{	vcpu_clear(to_vmx(vcpu));}static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu){	return vmcs_readl(GUEST_RFLAGS);}static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags){	if (vcpu->rmode.active)		rflags |= IOPL_MASK | X86_EFLAGS_VM;	vmcs_writel(GUEST_RFLAGS, rflags);}static void skip_emulated_instruction(struct kvm_vcpu *vcpu){	unsigned long rip;	u32 interruptibility;	rip = vmcs_readl(GUEST_RIP);	rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);	vmcs_writel(GUEST_RIP, rip);	/*	 * We emulated an instruction, so temporary interrupt blocking	 * should be removed, if set.	 */	interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);	if (interruptibility & 3)		vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,			     interruptibility & ~3);	vcpu->interrupt_window_open = 1;}static void vmx_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code){	printk(KERN_DEBUG "inject_general_protection: rip 0x%lx\n",	       vmcs_readl(GUEST_RIP));	vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,		     GP_VECTOR |		     INTR_TYPE_EXCEPTION |		     INTR_INFO_DELIEVER_CODE_MASK |		     INTR_INFO_VALID_MASK);}/* * Swap MSR entry in host/guest MSR entry array. */#ifdef CONFIG_X86_64static void move_msr_up(struct vcpu_vmx *vmx, int from, int to){	struct kvm_msr_entry tmp;	tmp = vmx->guest_msrs[to];	vmx->guest_msrs[to] = vmx->guest_msrs[from];	vmx->guest_msrs[from] = tmp;	tmp = vmx->host_msrs[to];	vmx->host_msrs[to] = vmx->host_msrs[from];	vmx->host_msrs[from] = tmp;}#endif/* * Set up the vmcs to automatically save and restore system * msrs.  Don't touch the 64-bit msrs if the guest is in legacy * mode, as fiddling with msrs is very expensive. */static void setup_msrs(struct vcpu_vmx *vmx){	int save_nmsrs;	save_nmsrs = 0;#ifdef CONFIG_X86_64	if (is_long_mode(&vmx->vcpu)) {		int index;		index = __find_msr_index(vmx, MSR_SYSCALL_MASK);		if (index >= 0)			move_msr_up(vmx, index, save_nmsrs++);		index = __find_msr_index(vmx, MSR_LSTAR);		if (index >= 0)			move_msr_up(vmx, index, save_nmsrs++);		index = __find_msr_index(vmx, MSR_CSTAR);		if (index >= 0)			move_msr_up(vmx, index, save_nmsrs++);		index = __find_msr_index(vmx, MSR_KERNEL_GS_BASE);		if (index >= 0)			move_msr_up(vmx, index, save_nmsrs++);		/*		 * MSR_K6_STAR is only needed on long mode guests, and only		 * if efer.sce is enabled.		 */		index = __find_msr_index(vmx, MSR_K6_STAR);		if ((index >= 0) && (vmx->vcpu.shadow_efer & EFER_SCE))			move_msr_up(vmx, index, save_nmsrs++);	}#endif	vmx->save_nmsrs = save_nmsrs;#ifdef CONFIG_X86_64	vmx->msr_offset_kernel_gs_base =		__find_msr_index(vmx, MSR_KERNEL_GS_BASE);#endif	vmx->msr_offset_efer = __find_msr_index(vmx, MSR_EFER);}/* * reads and returns guest's timestamp counter "register" * guest_tsc = host_tsc + tsc_offset    -- 21.3 */static u64 guest_read_tsc(void){	u64 host_tsc, tsc_offset;	rdtscll(host_tsc);	tsc_offset = vmcs_read64(TSC_OFFSET);	return host_tsc + tsc_offset;}/* * writes 'guest_tsc' into guest's timestamp counter "register" * guest_tsc = host_tsc + tsc_offset ==> tsc_offset = guest_tsc - host_tsc */static void guest_write_tsc(u64 guest_tsc){	u64 host_tsc;	rdtscll(host_tsc);	vmcs_write64(TSC_OFFSET, guest_tsc - host_tsc);}/* * Reads an msr value (of 'msr_index') into 'pdata'. * Returns 0 on success, non-0 otherwise. * Assumes vcpu_load() was already called. */static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata){	u64 data;	struct kvm_msr_entry *msr;	if (!pdata) {		printk(KERN_ERR "BUG: get_msr called with NULL pdata\n");		return -EINVAL;	}	switch (msr_index) {#ifdef CONFIG_X86_64	case MSR_FS_BASE:		data = vmcs_readl(GUEST_FS_BASE);		break;	case MSR_GS_BASE:		data = vmcs_readl(GUEST_GS_BASE);		break;	case MSR_EFER:		return kvm_get_msr_common(vcpu, msr_index, pdata);#endif	case MSR_IA32_TIME_STAMP_COUNTER:		data = guest_read_tsc();		break;	case MSR_IA32_SYSENTER_CS:		data = vmcs_read32(GUEST_SYSENTER_CS);		break;	case MSR_IA32_SYSENTER_EIP:		data = vmcs_readl(GUEST_SYSENTER_EIP);		break;	case MSR_IA32_SYSENTER_ESP:		data = vmcs_readl(GUEST_SYSENTER_ESP);		break;	default:		msr = find_msr_entry(to_vmx(vcpu), msr_index);		if (msr) {			data = msr->data;			break;		}		return kvm_get_msr_common(vcpu, msr_index, pdata);	}	*pdata = data;	return 0;}/* * Writes msr value into into the appropriate "register". * Returns 0 on success, non-0 otherwise. * Assumes vcpu_load() was already called. */static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data){	struct vcpu_vmx *vmx = to_vmx(vcpu);	struct kvm_msr_entry *msr;	int ret = 0;	switch (msr_index) {#ifdef CONFIG_X86_64	case MSR_EFER:		ret = kvm_set_msr_common(vcpu, msr_index, data);		if (vmx->host_state.loaded)			load_transition_efer(vmx);		break;	case MSR_FS_BASE:		vmcs_writel(GUEST_FS_BASE, data);		break;	case MSR_GS_BASE:		vmcs_writel(GUEST_GS_BASE, data);		break;#endif	case MSR_IA32_SYSENTER_CS:		vmcs_write32(GUEST_SYSENTER_CS, data);		break;	case MSR_IA32_SYSENTER_EIP:		vmcs_writel(GUEST_SYSENTER_EIP, data);		break;	case MSR_IA32_SYSENTER_ESP:		vmcs_writel(GUEST_SYSENTER_ESP, data);		break;	case MSR_IA32_TIME_STAMP_COUNTER:		guest_write_tsc(data);		break;	default:		msr = find_msr_entry(vmx, msr_index);		if (msr) {			msr->data = data;			if (vmx->host_state.loaded)				load_msrs(vmx->guest_msrs, vmx->save_nmsrs);			break;		}		ret = kvm_set_msr_common(vcpu, msr_index, data);	}	return ret;}/* * Sync the rsp and rip registers into the vcpu structure.  This allows * registers to be accessed by indexing vcpu->regs. */static void vcpu_load_rsp_rip(struct kvm_vcpu *vcpu){	vcpu->regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);	vcpu->rip = vmcs_readl(GUEST_RIP);}/* * Syncs rsp and rip back into the vmcs.  Should be called after possible * modification. */static void vcpu_put_rsp_rip(struct kvm_vcpu *vcpu){	vmcs_writel(GUEST_RSP, vcpu->regs[VCPU_REGS_RSP]);	vmcs_writel(GUEST_RIP, vcpu->rip);}static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg){	unsigned long dr7 = 0x400;	int old_singlestep;	old_singlestep = vcpu->guest_debug.singlestep;	vcpu->guest_debug.enabled = dbg->enabled;	if (vcpu->guest_debug.enabled) {		int i;		dr7 |= 0x200;  /* exact */		for (i = 0; i < 4; ++i) {			if (!dbg->breakpoints[i].enabled)				continue;			vcpu->guest_debug.bp[i] = dbg->breakpoints[i].address;			dr7 |= 2 << (i*2);    /* global enable */			dr7 |= 0 << (i*4+16); /* execution breakpoint */		}		vcpu->guest_debug.singlestep = dbg->singlestep;	} else		vcpu->guest_debug.singlestep = 0;	if (old_singlestep && !vcpu->guest_debug.singlestep) {		unsigned long flags;		flags = vmcs_readl(GUEST_RFLAGS);		flags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);		vmcs_writel(GUEST_RFLAGS, flags);	}	update_exception_bitmap(vcpu);	vmcs_writel(GUEST_DR7, dr7);	return 0;}static int vmx_get_irq(struct kvm_vcpu *vcpu){	u32 idtv_info_field;	idtv_info_field = vmcs_read32(IDT_VECTORING_INFO_FIELD);	if (idtv_info_field & INTR_INFO_VALID_MASK) {		if (is_external_interrupt(idtv_info_field))			return idtv_info_field & VECTORING_INFO_VECTOR_MASK;		else			printk("pending exception: not handled yet\n");	}	return -1;}static __init int cpu_has_kvm_support(void){	unsigned long ecx = cpuid_ecx(1);	return test_bit(5, &ecx); /* CPUID.1:ECX.VMX[bit 5] -> VT */}static __init int vmx_disabled_by_bios(void){	u64 msr;	rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);	return (msr & (MSR_IA32_FEATURE_CONTROL_LOCKED |		       MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED))	    == MSR_IA32_FEATURE_CONTROL_LOCKED;	/* locked but not enabled */}static void hardware_enable(void *garbage){	int cpu = raw_smp_processor_id();	u64 phys_addr = __pa(per_cpu(vmxarea, cpu));	u64 old;	rdmsrl(MSR_IA32_FEATURE_CONTROL, old);	if ((old & (MSR_IA32_FEATURE_CONTROL_LOCKED |		    MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED))	    != (MSR_IA32_FEATURE_CONTROL_LOCKED |		MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED))		/* enable and lock */		wrmsrl(MSR_IA32_FEATURE_CONTROL, old |		       MSR_IA32_FEATURE_CONTROL_LOCKED |		       MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED);	write_cr4(read_cr4() | X86_CR4_VMXE); /* FIXME: not cpu hotplug safe */	asm volatile (ASM_VMX_VMXON_RAX : : "a"(&phys_addr), "m"(phys_addr)		      : "memory", "cc");}static void hardware_disable(void *garbage){	asm volatile (ASM_VMX_VMXOFF : : : "cc");}static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,				      u32 msr, u32* result){	u32 vmx_msr_low, vmx_msr_high;	u32 ctl = ctl_min | ctl_opt;	rdmsr(msr, vmx_msr_low, vmx_msr_high);	ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */	ctl |= vmx_msr_low;  /* bit == 1 in low word  ==> must be one  */	/* Ensure minimum (required) set of control bits are supported. */	if (ctl_min & ~ctl)		return -EIO;	*result = ctl;	return 0;}static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf){	u32 vmx_msr_low, vmx_msr_high;	u32 min, opt;	u32 _pin_based_exec_control = 0;	u32 _cpu_based_exec_control = 0;	u32 _vmexit_control = 0;	u32 _vmentry_control = 0;	min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;	opt = 0;	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,				&_pin_based_exec_control) < 0)		return -EIO;	min = CPU_BASED_HLT_EXITING |#ifdef CONFIG_X86_64	      CPU_BASED_CR8_LOAD_EXITING |	      CPU_BASED_CR8_STORE_EXITING |#endif	      CPU_BASED_USE_IO_BITMAPS |	      CPU_BASED_MOV_DR_EXITING |	      CPU_BASED_USE_TSC_OFFSETING;#ifdef CONFIG_X86_64	opt = CPU_BASED_TPR_SHADOW;#else	opt = 0;#endif	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,				&_cpu_based_exec_control) < 0)		return -EIO;#ifdef CONFIG_X86_64	if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))		_cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &					   ~CPU_BASED_CR8_STORE_EXITING;#endif	min = 0;#ifdef CONFIG_X86_64	min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;#endif	opt = 0;	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,				&_vmexit_control) < 0)		return -EIO;	min = opt = 0;	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,				&_vmentry_control) < 0)		return -EIO;	rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);	/* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */	if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)		return -EIO;#ifdef CONFIG_X86_64	/* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */	if (vmx_msr_high & (1u<<16))		return -EIO;#endif	/* Require Write-Back (WB) memory type for VMCS accesses. */	if (((vmx_msr_high >> 18) & 15) != 6)		return -EIO;	vmcs_conf->size = vmx_msr_high & 0x1fff;	vmcs_conf->order = get_order(vmcs_config.size);	vmcs_conf->revision_id = vmx_msr_low;	vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;	vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;	vmcs_conf->vmexit_ctrl         = _vmexit_control;	vmcs_conf->vmentry_ctrl        = _vmentry_control;	return 0;}static struct vmcs *alloc_vmcs_cpu(int cpu){	int node = cpu_to_node(cpu);	struct page *pages;	struct vmcs *vmcs;	pages = alloc_pages_node(node, GFP_KERNEL, vmcs_config.order);	if (!pages)		return NULL;	vmcs = page_address(pages);	memset(vmcs, 0, vmcs_config.size);	vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */	return vmcs;}static struct vmcs *alloc_vmcs(void){	return alloc_vmcs_cpu(raw_smp_processor_id());}static void free_vmcs(struct vmcs *vmcs){	free_pages((unsigned long)vmcs, vmcs_config.order);}static void free_kvm_area(void){	int cpu;	for_each_online_cpu(cpu)		free_vmcs(per_cpu(vmxarea, cpu));}static __init int alloc_kvm_area(void){	int cpu;	for_each_online_cpu(cpu) {		struct vmcs *vmcs;		vmcs = alloc_vmcs_cpu(cpu);		if (!vmcs) {			free_kvm_area();			return -ENOMEM;		}		per_cpu(vmxarea, cpu) = vmcs;	}	return 0;}static __init int hardware_setup(void){	if (setup_vmcs_config(&vmcs_config) < 0)		return -EIO;	return alloc_kvm_area();}static __exit void hardware_unsetup(void){	free_kvm_area();}static void fix_pmode_dataseg(int seg, struct kvm_save_segment *save){	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];