1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * AMD Encrypted Register State Support
4  *
5  * Author: Joerg Roedel <jroedel@suse.de>
6  *
7  * This file is not compiled stand-alone. It contains code shared
8  * between the pre-decompression boot code and the running Linux kernel
9  * and is included directly into both code-bases.
10  */
11 
12 #ifndef __BOOT_COMPRESSED
13 #define error(v)	pr_err(v)
14 #define has_cpuflag(f)	boot_cpu_has(f)
15 #else
16 #undef WARN
17 #define WARN(condition, format...) (!!(condition))
18 #endif
19 
20 /* I/O parameters for CPUID-related helpers */
21 struct cpuid_leaf {
22 	u32 fn;
23 	u32 subfn;
24 	u32 eax;
25 	u32 ebx;
26 	u32 ecx;
27 	u32 edx;
28 };
29 
30 /*
31  * Individual entries of the SNP CPUID table, as defined by the SNP
32  * Firmware ABI, Revision 0.9, Section 7.1, Table 14.
33  */
34 struct snp_cpuid_fn {
35 	u32 eax_in;
36 	u32 ecx_in;
37 	u64 xcr0_in;
38 	u64 xss_in;
39 	u32 eax;
40 	u32 ebx;
41 	u32 ecx;
42 	u32 edx;
43 	u64 __reserved;
44 } __packed;
45 
46 /*
47  * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9,
48  * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit
49  * of 64 entries per CPUID table.
50  */
51 #define SNP_CPUID_COUNT_MAX 64
52 
53 struct snp_cpuid_table {
54 	u32 count;
55 	u32 __reserved1;
56 	u64 __reserved2;
57 	struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX];
58 } __packed;
59 
60 /*
61  * Since feature negotiation related variables are set early in the boot
62  * process they must reside in the .data section so as not to be zeroed
63  * out when the .bss section is later cleared.
64  *
65  * GHCB protocol version negotiated with the hypervisor.
66  */
67 static u16 ghcb_version __ro_after_init;
68 
69 /* Copy of the SNP firmware's CPUID page. */
70 static struct snp_cpuid_table cpuid_table_copy __ro_after_init;
71 
72 /*
73  * These will be initialized based on CPUID table so that non-present
74  * all-zero leaves (for sparse tables) can be differentiated from
75  * invalid/out-of-range leaves. This is needed since all-zero leaves
76  * still need to be post-processed.
77  */
78 static u32 cpuid_std_range_max __ro_after_init;
79 static u32 cpuid_hyp_range_max __ro_after_init;
80 static u32 cpuid_ext_range_max __ro_after_init;
81 
sev_es_check_cpu_features(void)82 static bool __init sev_es_check_cpu_features(void)
83 {
84 	if (!has_cpuflag(X86_FEATURE_RDRAND)) {
85 		error("RDRAND instruction not supported - no trusted source of randomness available\n");
86 		return false;
87 	}
88 
89 	return true;
90 }
91 
sev_es_terminate(unsigned int set,unsigned int reason)92 static void __noreturn sev_es_terminate(unsigned int set, unsigned int reason)
93 {
94 	u64 val = GHCB_MSR_TERM_REQ;
95 
96 	/* Tell the hypervisor what went wrong. */
97 	val |= GHCB_SEV_TERM_REASON(set, reason);
98 
99 	/* Request Guest Termination from Hypvervisor */
100 	sev_es_wr_ghcb_msr(val);
101 	VMGEXIT();
102 
103 	while (true)
104 		asm volatile("hlt\n" : : : "memory");
105 }
106 
107 /*
108  * The hypervisor features are available from GHCB version 2 onward.
109  */
get_hv_features(void)110 static u64 get_hv_features(void)
111 {
112 	u64 val;
113 
114 	if (ghcb_version < 2)
115 		return 0;
116 
117 	sev_es_wr_ghcb_msr(GHCB_MSR_HV_FT_REQ);
118 	VMGEXIT();
119 
120 	val = sev_es_rd_ghcb_msr();
121 	if (GHCB_RESP_CODE(val) != GHCB_MSR_HV_FT_RESP)
122 		return 0;
123 
124 	return GHCB_MSR_HV_FT_RESP_VAL(val);
125 }
126 
snp_register_ghcb_early(unsigned long paddr)127 static void snp_register_ghcb_early(unsigned long paddr)
128 {
129 	unsigned long pfn = paddr >> PAGE_SHIFT;
130 	u64 val;
131 
132 	sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn));
133 	VMGEXIT();
134 
135 	val = sev_es_rd_ghcb_msr();
136 
137 	/* If the response GPA is not ours then abort the guest */
138 	if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) ||
139 	    (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn))
140 		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER);
141 }
142 
sev_es_negotiate_protocol(void)143 static bool sev_es_negotiate_protocol(void)
144 {
145 	u64 val;
146 
147 	/* Do the GHCB protocol version negotiation */
148 	sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);
149 	VMGEXIT();
150 	val = sev_es_rd_ghcb_msr();
151 
152 	if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
153 		return false;
154 
155 	if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN ||
156 	    GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX)
157 		return false;
158 
159 	ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX);
160 
161 	return true;
162 }
163 
vc_ghcb_invalidate(struct ghcb * ghcb)164 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
165 {
166 	ghcb->save.sw_exit_code = 0;
167 	__builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
168 }
169 
vc_decoding_needed(unsigned long exit_code)170 static bool vc_decoding_needed(unsigned long exit_code)
171 {
172 	/* Exceptions don't require to decode the instruction */
173 	return !(exit_code >= SVM_EXIT_EXCP_BASE &&
174 		 exit_code <= SVM_EXIT_LAST_EXCP);
175 }
176 
vc_init_em_ctxt(struct es_em_ctxt * ctxt,struct pt_regs * regs,unsigned long exit_code)177 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
178 				      struct pt_regs *regs,
179 				      unsigned long exit_code)
180 {
181 	enum es_result ret = ES_OK;
182 
183 	memset(ctxt, 0, sizeof(*ctxt));
184 	ctxt->regs = regs;
185 
186 	if (vc_decoding_needed(exit_code))
187 		ret = vc_decode_insn(ctxt);
188 
189 	return ret;
190 }
191 
vc_finish_insn(struct es_em_ctxt * ctxt)192 static void vc_finish_insn(struct es_em_ctxt *ctxt)
193 {
194 	ctxt->regs->ip += ctxt->insn.length;
195 }
196 
verify_exception_info(struct ghcb * ghcb,struct es_em_ctxt * ctxt)197 static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
198 {
199 	u32 ret;
200 
201 	ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0);
202 	if (!ret)
203 		return ES_OK;
204 
205 	if (ret == 1) {
206 		u64 info = ghcb->save.sw_exit_info_2;
207 		unsigned long v = info & SVM_EVTINJ_VEC_MASK;
208 
209 		/* Check if exception information from hypervisor is sane. */
210 		if ((info & SVM_EVTINJ_VALID) &&
211 		    ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
212 		    ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
213 			ctxt->fi.vector = v;
214 
215 			if (info & SVM_EVTINJ_VALID_ERR)
216 				ctxt->fi.error_code = info >> 32;
217 
218 			return ES_EXCEPTION;
219 		}
220 	}
221 
222 	return ES_VMM_ERROR;
223 }
224 
sev_es_ghcb_hv_call(struct ghcb * ghcb,struct es_em_ctxt * ctxt,u64 exit_code,u64 exit_info_1,u64 exit_info_2)225 static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
226 					  struct es_em_ctxt *ctxt,
227 					  u64 exit_code, u64 exit_info_1,
228 					  u64 exit_info_2)
229 {
230 	/* Fill in protocol and format specifiers */
231 	ghcb->protocol_version = ghcb_version;
232 	ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
233 
234 	ghcb_set_sw_exit_code(ghcb, exit_code);
235 	ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
236 	ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
237 
238 	sev_es_wr_ghcb_msr(__pa(ghcb));
239 	VMGEXIT();
240 
241 	return verify_exception_info(ghcb, ctxt);
242 }
243 
__sev_cpuid_hv(u32 fn,int reg_idx,u32 * reg)244 static int __sev_cpuid_hv(u32 fn, int reg_idx, u32 *reg)
245 {
246 	u64 val;
247 
248 	sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, reg_idx));
249 	VMGEXIT();
250 	val = sev_es_rd_ghcb_msr();
251 	if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
252 		return -EIO;
253 
254 	*reg = (val >> 32);
255 
256 	return 0;
257 }
258 
__sev_cpuid_hv_msr(struct cpuid_leaf * leaf)259 static int __sev_cpuid_hv_msr(struct cpuid_leaf *leaf)
260 {
261 	int ret;
262 
263 	/*
264 	 * MSR protocol does not support fetching non-zero subfunctions, but is
265 	 * sufficient to handle current early-boot cases. Should that change,
266 	 * make sure to report an error rather than ignoring the index and
267 	 * grabbing random values. If this issue arises in the future, handling
268 	 * can be added here to use GHCB-page protocol for cases that occur late
269 	 * enough in boot that GHCB page is available.
270 	 */
271 	if (cpuid_function_is_indexed(leaf->fn) && leaf->subfn)
272 		return -EINVAL;
273 
274 	ret =         __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EAX, &leaf->eax);
275 	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EBX, &leaf->ebx);
276 	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_ECX, &leaf->ecx);
277 	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EDX, &leaf->edx);
278 
279 	return ret;
280 }
281 
__sev_cpuid_hv_ghcb(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)282 static int __sev_cpuid_hv_ghcb(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
283 {
284 	u32 cr4 = native_read_cr4();
285 	int ret;
286 
287 	ghcb_set_rax(ghcb, leaf->fn);
288 	ghcb_set_rcx(ghcb, leaf->subfn);
289 
290 	if (cr4 & X86_CR4_OSXSAVE)
291 		/* Safe to read xcr0 */
292 		ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
293 	else
294 		/* xgetbv will cause #UD - use reset value for xcr0 */
295 		ghcb_set_xcr0(ghcb, 1);
296 
297 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
298 	if (ret != ES_OK)
299 		return ret;
300 
301 	if (!(ghcb_rax_is_valid(ghcb) &&
302 	      ghcb_rbx_is_valid(ghcb) &&
303 	      ghcb_rcx_is_valid(ghcb) &&
304 	      ghcb_rdx_is_valid(ghcb)))
305 		return ES_VMM_ERROR;
306 
307 	leaf->eax = ghcb->save.rax;
308 	leaf->ebx = ghcb->save.rbx;
309 	leaf->ecx = ghcb->save.rcx;
310 	leaf->edx = ghcb->save.rdx;
311 
312 	return ES_OK;
313 }
314 
sev_cpuid_hv(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)315 static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
316 {
317 	return ghcb ? __sev_cpuid_hv_ghcb(ghcb, ctxt, leaf)
318 		    : __sev_cpuid_hv_msr(leaf);
319 }
320 
321 /*
322  * This may be called early while still running on the initial identity
323  * mapping. Use RIP-relative addressing to obtain the correct address
324  * while running with the initial identity mapping as well as the
325  * switch-over to kernel virtual addresses later.
326  */
snp_cpuid_get_table(void)327 static const struct snp_cpuid_table *snp_cpuid_get_table(void)
328 {
329 	void *ptr;
330 
331 	asm ("lea cpuid_table_copy(%%rip), %0"
332 	     : "=r" (ptr)
333 	     : "p" (&cpuid_table_copy));
334 
335 	return ptr;
336 }
337 
338 /*
339  * The SNP Firmware ABI, Revision 0.9, Section 7.1, details the use of
340  * XCR0_IN and XSS_IN to encode multiple versions of 0xD subfunctions 0
341  * and 1 based on the corresponding features enabled by a particular
342  * combination of XCR0 and XSS registers so that a guest can look up the
343  * version corresponding to the features currently enabled in its XCR0/XSS
344  * registers. The only values that differ between these versions/table
345  * entries is the enabled XSAVE area size advertised via EBX.
346  *
347  * While hypervisors may choose to make use of this support, it is more
348  * robust/secure for a guest to simply find the entry corresponding to the
349  * base/legacy XSAVE area size (XCR0=1 or XCR0=3), and then calculate the
350  * XSAVE area size using subfunctions 2 through 64, as documented in APM
351  * Volume 3, Rev 3.31, Appendix E.3.8, which is what is done here.
352  *
353  * Since base/legacy XSAVE area size is documented as 0x240, use that value
354  * directly rather than relying on the base size in the CPUID table.
355  *
356  * Return: XSAVE area size on success, 0 otherwise.
357  */
snp_cpuid_calc_xsave_size(u64 xfeatures_en,bool compacted)358 static u32 snp_cpuid_calc_xsave_size(u64 xfeatures_en, bool compacted)
359 {
360 	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
361 	u64 xfeatures_found = 0;
362 	u32 xsave_size = 0x240;
363 	int i;
364 
365 	for (i = 0; i < cpuid_table->count; i++) {
366 		const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
367 
368 		if (!(e->eax_in == 0xD && e->ecx_in > 1 && e->ecx_in < 64))
369 			continue;
370 		if (!(xfeatures_en & (BIT_ULL(e->ecx_in))))
371 			continue;
372 		if (xfeatures_found & (BIT_ULL(e->ecx_in)))
373 			continue;
374 
375 		xfeatures_found |= (BIT_ULL(e->ecx_in));
376 
377 		if (compacted)
378 			xsave_size += e->eax;
379 		else
380 			xsave_size = max(xsave_size, e->eax + e->ebx);
381 	}
382 
383 	/*
384 	 * Either the guest set unsupported XCR0/XSS bits, or the corresponding
385 	 * entries in the CPUID table were not present. This is not a valid
386 	 * state to be in.
387 	 */
388 	if (xfeatures_found != (xfeatures_en & GENMASK_ULL(63, 2)))
389 		return 0;
390 
391 	return xsave_size;
392 }
393 
394 static bool
snp_cpuid_get_validated_func(struct cpuid_leaf * leaf)395 snp_cpuid_get_validated_func(struct cpuid_leaf *leaf)
396 {
397 	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
398 	int i;
399 
400 	for (i = 0; i < cpuid_table->count; i++) {
401 		const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
402 
403 		if (e->eax_in != leaf->fn)
404 			continue;
405 
406 		if (cpuid_function_is_indexed(leaf->fn) && e->ecx_in != leaf->subfn)
407 			continue;
408 
409 		/*
410 		 * For 0xD subfunctions 0 and 1, only use the entry corresponding
411 		 * to the base/legacy XSAVE area size (XCR0=1 or XCR0=3, XSS=0).
412 		 * See the comments above snp_cpuid_calc_xsave_size() for more
413 		 * details.
414 		 */
415 		if (e->eax_in == 0xD && (e->ecx_in == 0 || e->ecx_in == 1))
416 			if (!(e->xcr0_in == 1 || e->xcr0_in == 3) || e->xss_in)
417 				continue;
418 
419 		leaf->eax = e->eax;
420 		leaf->ebx = e->ebx;
421 		leaf->ecx = e->ecx;
422 		leaf->edx = e->edx;
423 
424 		return true;
425 	}
426 
427 	return false;
428 }
429 
snp_cpuid_hv(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)430 static void snp_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
431 {
432 	if (sev_cpuid_hv(ghcb, ctxt, leaf))
433 		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV);
434 }
435 
snp_cpuid_postprocess(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)436 static int snp_cpuid_postprocess(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
437 				 struct cpuid_leaf *leaf)
438 {
439 	struct cpuid_leaf leaf_hv = *leaf;
440 
441 	switch (leaf->fn) {
442 	case 0x1:
443 		snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
444 
445 		/* initial APIC ID */
446 		leaf->ebx = (leaf_hv.ebx & GENMASK(31, 24)) | (leaf->ebx & GENMASK(23, 0));
447 		/* APIC enabled bit */
448 		leaf->edx = (leaf_hv.edx & BIT(9)) | (leaf->edx & ~BIT(9));
449 
450 		/* OSXSAVE enabled bit */
451 		if (native_read_cr4() & X86_CR4_OSXSAVE)
452 			leaf->ecx |= BIT(27);
453 		break;
454 	case 0x7:
455 		/* OSPKE enabled bit */
456 		leaf->ecx &= ~BIT(4);
457 		if (native_read_cr4() & X86_CR4_PKE)
458 			leaf->ecx |= BIT(4);
459 		break;
460 	case 0xB:
461 		leaf_hv.subfn = 0;
462 		snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
463 
464 		/* extended APIC ID */
465 		leaf->edx = leaf_hv.edx;
466 		break;
467 	case 0xD: {
468 		bool compacted = false;
469 		u64 xcr0 = 1, xss = 0;
470 		u32 xsave_size;
471 
472 		if (leaf->subfn != 0 && leaf->subfn != 1)
473 			return 0;
474 
475 		if (native_read_cr4() & X86_CR4_OSXSAVE)
476 			xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
477 		if (leaf->subfn == 1) {
478 			/* Get XSS value if XSAVES is enabled. */
479 			if (leaf->eax & BIT(3)) {
480 				unsigned long lo, hi;
481 
482 				asm volatile("rdmsr" : "=a" (lo), "=d" (hi)
483 						     : "c" (MSR_IA32_XSS));
484 				xss = (hi << 32) | lo;
485 			}
486 
487 			/*
488 			 * The PPR and APM aren't clear on what size should be
489 			 * encoded in 0xD:0x1:EBX when compaction is not enabled
490 			 * by either XSAVEC (feature bit 1) or XSAVES (feature
491 			 * bit 3) since SNP-capable hardware has these feature
492 			 * bits fixed as 1. KVM sets it to 0 in this case, but
493 			 * to avoid this becoming an issue it's safer to simply
494 			 * treat this as unsupported for SNP guests.
495 			 */
496 			if (!(leaf->eax & (BIT(1) | BIT(3))))
497 				return -EINVAL;
498 
499 			compacted = true;
500 		}
501 
502 		xsave_size = snp_cpuid_calc_xsave_size(xcr0 | xss, compacted);
503 		if (!xsave_size)
504 			return -EINVAL;
505 
506 		leaf->ebx = xsave_size;
507 		}
508 		break;
509 	case 0x8000001E:
510 		snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
511 
512 		/* extended APIC ID */
513 		leaf->eax = leaf_hv.eax;
514 		/* compute ID */
515 		leaf->ebx = (leaf->ebx & GENMASK(31, 8)) | (leaf_hv.ebx & GENMASK(7, 0));
516 		/* node ID */
517 		leaf->ecx = (leaf->ecx & GENMASK(31, 8)) | (leaf_hv.ecx & GENMASK(7, 0));
518 		break;
519 	default:
520 		/* No fix-ups needed, use values as-is. */
521 		break;
522 	}
523 
524 	return 0;
525 }
526 
527 /*
528  * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value
529  * should be treated as fatal by caller.
530  */
snp_cpuid(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)531 static int snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
532 {
533 	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
534 
535 	if (!cpuid_table->count)
536 		return -EOPNOTSUPP;
537 
538 	if (!snp_cpuid_get_validated_func(leaf)) {
539 		/*
540 		 * Some hypervisors will avoid keeping track of CPUID entries
541 		 * where all values are zero, since they can be handled the
542 		 * same as out-of-range values (all-zero). This is useful here
543 		 * as well as it allows virtually all guest configurations to
544 		 * work using a single SNP CPUID table.
545 		 *
546 		 * To allow for this, there is a need to distinguish between
547 		 * out-of-range entries and in-range zero entries, since the
548 		 * CPUID table entries are only a template that may need to be
549 		 * augmented with additional values for things like
550 		 * CPU-specific information during post-processing. So if it's
551 		 * not in the table, set the values to zero. Then, if they are
552 		 * within a valid CPUID range, proceed with post-processing
553 		 * using zeros as the initial values. Otherwise, skip
554 		 * post-processing and just return zeros immediately.
555 		 */
556 		leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0;
557 
558 		/* Skip post-processing for out-of-range zero leafs. */
559 		if (!(leaf->fn <= cpuid_std_range_max ||
560 		      (leaf->fn >= 0x40000000 && leaf->fn <= cpuid_hyp_range_max) ||
561 		      (leaf->fn >= 0x80000000 && leaf->fn <= cpuid_ext_range_max)))
562 			return 0;
563 	}
564 
565 	return snp_cpuid_postprocess(ghcb, ctxt, leaf);
566 }
567 
568 /*
569  * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
570  * page yet, so it only supports the MSR based communication with the
571  * hypervisor and only the CPUID exit-code.
572  */
do_vc_no_ghcb(struct pt_regs * regs,unsigned long exit_code)573 void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
574 {
575 	unsigned int subfn = lower_bits(regs->cx, 32);
576 	unsigned int fn = lower_bits(regs->ax, 32);
577 	struct cpuid_leaf leaf;
578 	int ret;
579 
580 	/* Only CPUID is supported via MSR protocol */
581 	if (exit_code != SVM_EXIT_CPUID)
582 		goto fail;
583 
584 	leaf.fn = fn;
585 	leaf.subfn = subfn;
586 
587 	ret = snp_cpuid(NULL, NULL, &leaf);
588 	if (!ret)
589 		goto cpuid_done;
590 
591 	if (ret != -EOPNOTSUPP)
592 		goto fail;
593 
594 	if (__sev_cpuid_hv_msr(&leaf))
595 		goto fail;
596 
597 cpuid_done:
598 	regs->ax = leaf.eax;
599 	regs->bx = leaf.ebx;
600 	regs->cx = leaf.ecx;
601 	regs->dx = leaf.edx;
602 
603 	/*
604 	 * This is a VC handler and the #VC is only raised when SEV-ES is
605 	 * active, which means SEV must be active too. Do sanity checks on the
606 	 * CPUID results to make sure the hypervisor does not trick the kernel
607 	 * into the no-sev path. This could map sensitive data unencrypted and
608 	 * make it accessible to the hypervisor.
609 	 *
610 	 * In particular, check for:
611 	 *	- Availability of CPUID leaf 0x8000001f
612 	 *	- SEV CPUID bit.
613 	 *
614 	 * The hypervisor might still report the wrong C-bit position, but this
615 	 * can't be checked here.
616 	 */
617 
618 	if (fn == 0x80000000 && (regs->ax < 0x8000001f))
619 		/* SEV leaf check */
620 		goto fail;
621 	else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
622 		/* SEV bit */
623 		goto fail;
624 
625 	/* Skip over the CPUID two-byte opcode */
626 	regs->ip += 2;
627 
628 	return;
629 
630 fail:
631 	/* Terminate the guest */
632 	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
633 }
634 
vc_insn_string_check(struct es_em_ctxt * ctxt,unsigned long address,bool write)635 static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt,
636 					   unsigned long address,
637 					   bool write)
638 {
639 	if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) {
640 		ctxt->fi.vector     = X86_TRAP_PF;
641 		ctxt->fi.error_code = X86_PF_USER;
642 		ctxt->fi.cr2        = address;
643 		if (write)
644 			ctxt->fi.error_code |= X86_PF_WRITE;
645 
646 		return ES_EXCEPTION;
647 	}
648 
649 	return ES_OK;
650 }
651 
vc_insn_string_read(struct es_em_ctxt * ctxt,void * src,char * buf,unsigned int data_size,unsigned int count,bool backwards)652 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
653 					  void *src, char *buf,
654 					  unsigned int data_size,
655 					  unsigned int count,
656 					  bool backwards)
657 {
658 	int i, b = backwards ? -1 : 1;
659 	unsigned long address = (unsigned long)src;
660 	enum es_result ret;
661 
662 	ret = vc_insn_string_check(ctxt, address, false);
663 	if (ret != ES_OK)
664 		return ret;
665 
666 	for (i = 0; i < count; i++) {
667 		void *s = src + (i * data_size * b);
668 		char *d = buf + (i * data_size);
669 
670 		ret = vc_read_mem(ctxt, s, d, data_size);
671 		if (ret != ES_OK)
672 			break;
673 	}
674 
675 	return ret;
676 }
677 
vc_insn_string_write(struct es_em_ctxt * ctxt,void * dst,char * buf,unsigned int data_size,unsigned int count,bool backwards)678 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
679 					   void *dst, char *buf,
680 					   unsigned int data_size,
681 					   unsigned int count,
682 					   bool backwards)
683 {
684 	int i, s = backwards ? -1 : 1;
685 	unsigned long address = (unsigned long)dst;
686 	enum es_result ret;
687 
688 	ret = vc_insn_string_check(ctxt, address, true);
689 	if (ret != ES_OK)
690 		return ret;
691 
692 	for (i = 0; i < count; i++) {
693 		void *d = dst + (i * data_size * s);
694 		char *b = buf + (i * data_size);
695 
696 		ret = vc_write_mem(ctxt, d, b, data_size);
697 		if (ret != ES_OK)
698 			break;
699 	}
700 
701 	return ret;
702 }
703 
704 #define IOIO_TYPE_STR  BIT(2)
705 #define IOIO_TYPE_IN   1
706 #define IOIO_TYPE_INS  (IOIO_TYPE_IN | IOIO_TYPE_STR)
707 #define IOIO_TYPE_OUT  0
708 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
709 
710 #define IOIO_REP       BIT(3)
711 
712 #define IOIO_ADDR_64   BIT(9)
713 #define IOIO_ADDR_32   BIT(8)
714 #define IOIO_ADDR_16   BIT(7)
715 
716 #define IOIO_DATA_32   BIT(6)
717 #define IOIO_DATA_16   BIT(5)
718 #define IOIO_DATA_8    BIT(4)
719 
720 #define IOIO_SEG_ES    (0 << 10)
721 #define IOIO_SEG_DS    (3 << 10)
722 
vc_ioio_exitinfo(struct es_em_ctxt * ctxt,u64 * exitinfo)723 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
724 {
725 	struct insn *insn = &ctxt->insn;
726 	size_t size;
727 	u64 port;
728 
729 	*exitinfo = 0;
730 
731 	switch (insn->opcode.bytes[0]) {
732 	/* INS opcodes */
733 	case 0x6c:
734 	case 0x6d:
735 		*exitinfo |= IOIO_TYPE_INS;
736 		*exitinfo |= IOIO_SEG_ES;
737 		port	   = ctxt->regs->dx & 0xffff;
738 		break;
739 
740 	/* OUTS opcodes */
741 	case 0x6e:
742 	case 0x6f:
743 		*exitinfo |= IOIO_TYPE_OUTS;
744 		*exitinfo |= IOIO_SEG_DS;
745 		port	   = ctxt->regs->dx & 0xffff;
746 		break;
747 
748 	/* IN immediate opcodes */
749 	case 0xe4:
750 	case 0xe5:
751 		*exitinfo |= IOIO_TYPE_IN;
752 		port	   = (u8)insn->immediate.value & 0xffff;
753 		break;
754 
755 	/* OUT immediate opcodes */
756 	case 0xe6:
757 	case 0xe7:
758 		*exitinfo |= IOIO_TYPE_OUT;
759 		port	   = (u8)insn->immediate.value & 0xffff;
760 		break;
761 
762 	/* IN register opcodes */
763 	case 0xec:
764 	case 0xed:
765 		*exitinfo |= IOIO_TYPE_IN;
766 		port	   = ctxt->regs->dx & 0xffff;
767 		break;
768 
769 	/* OUT register opcodes */
770 	case 0xee:
771 	case 0xef:
772 		*exitinfo |= IOIO_TYPE_OUT;
773 		port	   = ctxt->regs->dx & 0xffff;
774 		break;
775 
776 	default:
777 		return ES_DECODE_FAILED;
778 	}
779 
780 	*exitinfo |= port << 16;
781 
782 	switch (insn->opcode.bytes[0]) {
783 	case 0x6c:
784 	case 0x6e:
785 	case 0xe4:
786 	case 0xe6:
787 	case 0xec:
788 	case 0xee:
789 		/* Single byte opcodes */
790 		*exitinfo |= IOIO_DATA_8;
791 		size       = 1;
792 		break;
793 	default:
794 		/* Length determined by instruction parsing */
795 		*exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
796 						     : IOIO_DATA_32;
797 		size       = (insn->opnd_bytes == 2) ? 2 : 4;
798 	}
799 
800 	switch (insn->addr_bytes) {
801 	case 2:
802 		*exitinfo |= IOIO_ADDR_16;
803 		break;
804 	case 4:
805 		*exitinfo |= IOIO_ADDR_32;
806 		break;
807 	case 8:
808 		*exitinfo |= IOIO_ADDR_64;
809 		break;
810 	}
811 
812 	if (insn_has_rep_prefix(insn))
813 		*exitinfo |= IOIO_REP;
814 
815 	return vc_ioio_check(ctxt, (u16)port, size);
816 }
817 
vc_handle_ioio(struct ghcb * ghcb,struct es_em_ctxt * ctxt)818 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
819 {
820 	struct pt_regs *regs = ctxt->regs;
821 	u64 exit_info_1, exit_info_2;
822 	enum es_result ret;
823 
824 	ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
825 	if (ret != ES_OK)
826 		return ret;
827 
828 	if (exit_info_1 & IOIO_TYPE_STR) {
829 
830 		/* (REP) INS/OUTS */
831 
832 		bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
833 		unsigned int io_bytes, exit_bytes;
834 		unsigned int ghcb_count, op_count;
835 		unsigned long es_base;
836 		u64 sw_scratch;
837 
838 		/*
839 		 * For the string variants with rep prefix the amount of in/out
840 		 * operations per #VC exception is limited so that the kernel
841 		 * has a chance to take interrupts and re-schedule while the
842 		 * instruction is emulated.
843 		 */
844 		io_bytes   = (exit_info_1 >> 4) & 0x7;
845 		ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
846 
847 		op_count    = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
848 		exit_info_2 = min(op_count, ghcb_count);
849 		exit_bytes  = exit_info_2 * io_bytes;
850 
851 		es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
852 
853 		/* Read bytes of OUTS into the shared buffer */
854 		if (!(exit_info_1 & IOIO_TYPE_IN)) {
855 			ret = vc_insn_string_read(ctxt,
856 					       (void *)(es_base + regs->si),
857 					       ghcb->shared_buffer, io_bytes,
858 					       exit_info_2, df);
859 			if (ret)
860 				return ret;
861 		}
862 
863 		/*
864 		 * Issue an VMGEXIT to the HV to consume the bytes from the
865 		 * shared buffer or to have it write them into the shared buffer
866 		 * depending on the instruction: OUTS or INS.
867 		 */
868 		sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
869 		ghcb_set_sw_scratch(ghcb, sw_scratch);
870 		ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
871 					  exit_info_1, exit_info_2);
872 		if (ret != ES_OK)
873 			return ret;
874 
875 		/* Read bytes from shared buffer into the guest's destination. */
876 		if (exit_info_1 & IOIO_TYPE_IN) {
877 			ret = vc_insn_string_write(ctxt,
878 						   (void *)(es_base + regs->di),
879 						   ghcb->shared_buffer, io_bytes,
880 						   exit_info_2, df);
881 			if (ret)
882 				return ret;
883 
884 			if (df)
885 				regs->di -= exit_bytes;
886 			else
887 				regs->di += exit_bytes;
888 		} else {
889 			if (df)
890 				regs->si -= exit_bytes;
891 			else
892 				regs->si += exit_bytes;
893 		}
894 
895 		if (exit_info_1 & IOIO_REP)
896 			regs->cx -= exit_info_2;
897 
898 		ret = regs->cx ? ES_RETRY : ES_OK;
899 
900 	} else {
901 
902 		/* IN/OUT into/from rAX */
903 
904 		int bits = (exit_info_1 & 0x70) >> 1;
905 		u64 rax = 0;
906 
907 		if (!(exit_info_1 & IOIO_TYPE_IN))
908 			rax = lower_bits(regs->ax, bits);
909 
910 		ghcb_set_rax(ghcb, rax);
911 
912 		ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
913 		if (ret != ES_OK)
914 			return ret;
915 
916 		if (exit_info_1 & IOIO_TYPE_IN) {
917 			if (!ghcb_rax_is_valid(ghcb))
918 				return ES_VMM_ERROR;
919 			regs->ax = lower_bits(ghcb->save.rax, bits);
920 		}
921 	}
922 
923 	return ret;
924 }
925 
vc_handle_cpuid_snp(struct ghcb * ghcb,struct es_em_ctxt * ctxt)926 static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
927 {
928 	struct pt_regs *regs = ctxt->regs;
929 	struct cpuid_leaf leaf;
930 	int ret;
931 
932 	leaf.fn = regs->ax;
933 	leaf.subfn = regs->cx;
934 	ret = snp_cpuid(ghcb, ctxt, &leaf);
935 	if (!ret) {
936 		regs->ax = leaf.eax;
937 		regs->bx = leaf.ebx;
938 		regs->cx = leaf.ecx;
939 		regs->dx = leaf.edx;
940 	}
941 
942 	return ret;
943 }
944 
vc_handle_cpuid(struct ghcb * ghcb,struct es_em_ctxt * ctxt)945 static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
946 				      struct es_em_ctxt *ctxt)
947 {
948 	struct pt_regs *regs = ctxt->regs;
949 	u32 cr4 = native_read_cr4();
950 	enum es_result ret;
951 	int snp_cpuid_ret;
952 
953 	snp_cpuid_ret = vc_handle_cpuid_snp(ghcb, ctxt);
954 	if (!snp_cpuid_ret)
955 		return ES_OK;
956 	if (snp_cpuid_ret != -EOPNOTSUPP)
957 		return ES_VMM_ERROR;
958 
959 	ghcb_set_rax(ghcb, regs->ax);
960 	ghcb_set_rcx(ghcb, regs->cx);
961 
962 	if (cr4 & X86_CR4_OSXSAVE)
963 		/* Safe to read xcr0 */
964 		ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
965 	else
966 		/* xgetbv will cause #GP - use reset value for xcr0 */
967 		ghcb_set_xcr0(ghcb, 1);
968 
969 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
970 	if (ret != ES_OK)
971 		return ret;
972 
973 	if (!(ghcb_rax_is_valid(ghcb) &&
974 	      ghcb_rbx_is_valid(ghcb) &&
975 	      ghcb_rcx_is_valid(ghcb) &&
976 	      ghcb_rdx_is_valid(ghcb)))
977 		return ES_VMM_ERROR;
978 
979 	regs->ax = ghcb->save.rax;
980 	regs->bx = ghcb->save.rbx;
981 	regs->cx = ghcb->save.rcx;
982 	regs->dx = ghcb->save.rdx;
983 
984 	return ES_OK;
985 }
986 
vc_handle_rdtsc(struct ghcb * ghcb,struct es_em_ctxt * ctxt,unsigned long exit_code)987 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
988 				      struct es_em_ctxt *ctxt,
989 				      unsigned long exit_code)
990 {
991 	bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
992 	enum es_result ret;
993 
994 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
995 	if (ret != ES_OK)
996 		return ret;
997 
998 	if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
999 	     (!rdtscp || ghcb_rcx_is_valid(ghcb))))
1000 		return ES_VMM_ERROR;
1001 
1002 	ctxt->regs->ax = ghcb->save.rax;
1003 	ctxt->regs->dx = ghcb->save.rdx;
1004 	if (rdtscp)
1005 		ctxt->regs->cx = ghcb->save.rcx;
1006 
1007 	return ES_OK;
1008 }
1009 
1010 struct cc_setup_data {
1011 	struct setup_data header;
1012 	u32 cc_blob_address;
1013 };
1014 
1015 /*
1016  * Search for a Confidential Computing blob passed in as a setup_data entry
1017  * via the Linux Boot Protocol.
1018  */
find_cc_blob_setup_data(struct boot_params * bp)1019 static struct cc_blob_sev_info *find_cc_blob_setup_data(struct boot_params *bp)
1020 {
1021 	struct cc_setup_data *sd = NULL;
1022 	struct setup_data *hdr;
1023 
1024 	hdr = (struct setup_data *)bp->hdr.setup_data;
1025 
1026 	while (hdr) {
1027 		if (hdr->type == SETUP_CC_BLOB) {
1028 			sd = (struct cc_setup_data *)hdr;
1029 			return (struct cc_blob_sev_info *)(unsigned long)sd->cc_blob_address;
1030 		}
1031 		hdr = (struct setup_data *)hdr->next;
1032 	}
1033 
1034 	return NULL;
1035 }
1036 
1037 /*
1038  * Initialize the kernel's copy of the SNP CPUID table, and set up the
1039  * pointer that will be used to access it.
1040  *
1041  * Maintaining a direct mapping of the SNP CPUID table used by firmware would
1042  * be possible as an alternative, but the approach is brittle since the
1043  * mapping needs to be updated in sync with all the changes to virtual memory
1044  * layout and related mapping facilities throughout the boot process.
1045  */
setup_cpuid_table(const struct cc_blob_sev_info * cc_info)1046 static void __init setup_cpuid_table(const struct cc_blob_sev_info *cc_info)
1047 {
1048 	const struct snp_cpuid_table *cpuid_table_fw, *cpuid_table;
1049 	int i;
1050 
1051 	if (!cc_info || !cc_info->cpuid_phys || cc_info->cpuid_len < PAGE_SIZE)
1052 		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
1053 
1054 	cpuid_table_fw = (const struct snp_cpuid_table *)cc_info->cpuid_phys;
1055 	if (!cpuid_table_fw->count || cpuid_table_fw->count > SNP_CPUID_COUNT_MAX)
1056 		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
1057 
1058 	cpuid_table = snp_cpuid_get_table();
1059 	memcpy((void *)cpuid_table, cpuid_table_fw, sizeof(*cpuid_table));
1060 
1061 	/* Initialize CPUID ranges for range-checking. */
1062 	for (i = 0; i < cpuid_table->count; i++) {
1063 		const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
1064 
1065 		if (fn->eax_in == 0x0)
1066 			cpuid_std_range_max = fn->eax;
1067 		else if (fn->eax_in == 0x40000000)
1068 			cpuid_hyp_range_max = fn->eax;
1069 		else if (fn->eax_in == 0x80000000)
1070 			cpuid_ext_range_max = fn->eax;
1071 	}
1072 }
1073 
pvalidate_pages(struct snp_psc_desc * desc)1074 static void pvalidate_pages(struct snp_psc_desc *desc)
1075 {
1076 	struct psc_entry *e;
1077 	unsigned long vaddr;
1078 	unsigned int size;
1079 	unsigned int i;
1080 	bool validate;
1081 	int rc;
1082 
1083 	for (i = 0; i <= desc->hdr.end_entry; i++) {
1084 		e = &desc->entries[i];
1085 
1086 		vaddr = (unsigned long)pfn_to_kaddr(e->gfn);
1087 		size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
1088 		validate = e->operation == SNP_PAGE_STATE_PRIVATE;
1089 
1090 		rc = pvalidate(vaddr, size, validate);
1091 		if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) {
1092 			unsigned long vaddr_end = vaddr + PMD_SIZE;
1093 
1094 			for (; vaddr < vaddr_end; vaddr += PAGE_SIZE) {
1095 				rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
1096 				if (rc)
1097 					break;
1098 			}
1099 		}
1100 
1101 		if (rc) {
1102 			WARN(1, "Failed to validate address 0x%lx ret %d", vaddr, rc);
1103 			sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
1104 		}
1105 	}
1106 }
1107 
vmgexit_psc(struct ghcb * ghcb,struct snp_psc_desc * desc)1108 static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc)
1109 {
1110 	int cur_entry, end_entry, ret = 0;
1111 	struct snp_psc_desc *data;
1112 	struct es_em_ctxt ctxt;
1113 
1114 	vc_ghcb_invalidate(ghcb);
1115 
1116 	/* Copy the input desc into GHCB shared buffer */
1117 	data = (struct snp_psc_desc *)ghcb->shared_buffer;
1118 	memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc)));
1119 
1120 	/*
1121 	 * As per the GHCB specification, the hypervisor can resume the guest
1122 	 * before processing all the entries. Check whether all the entries
1123 	 * are processed. If not, then keep retrying. Note, the hypervisor
1124 	 * will update the data memory directly to indicate the status, so
1125 	 * reference the data->hdr everywhere.
1126 	 *
1127 	 * The strategy here is to wait for the hypervisor to change the page
1128 	 * state in the RMP table before guest accesses the memory pages. If the
1129 	 * page state change was not successful, then later memory access will
1130 	 * result in a crash.
1131 	 */
1132 	cur_entry = data->hdr.cur_entry;
1133 	end_entry = data->hdr.end_entry;
1134 
1135 	while (data->hdr.cur_entry <= data->hdr.end_entry) {
1136 		ghcb_set_sw_scratch(ghcb, (u64)__pa(data));
1137 
1138 		/* This will advance the shared buffer data points to. */
1139 		ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0);
1140 
1141 		/*
1142 		 * Page State Change VMGEXIT can pass error code through
1143 		 * exit_info_2.
1144 		 */
1145 		if (WARN(ret || ghcb->save.sw_exit_info_2,
1146 			 "SNP: PSC failed ret=%d exit_info_2=%llx\n",
1147 			 ret, ghcb->save.sw_exit_info_2)) {
1148 			ret = 1;
1149 			goto out;
1150 		}
1151 
1152 		/* Verify that reserved bit is not set */
1153 		if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) {
1154 			ret = 1;
1155 			goto out;
1156 		}
1157 
1158 		/*
1159 		 * Sanity check that entry processing is not going backwards.
1160 		 * This will happen only if hypervisor is tricking us.
1161 		 */
1162 		if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry,
1163 "SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n",
1164 			 end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) {
1165 			ret = 1;
1166 			goto out;
1167 		}
1168 	}
1169 
1170 out:
1171 	return ret;
1172 }
1173