249 lines
7.4 KiB
C
249 lines
7.4 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef ASM_KVM_CACHE_REGS_H
|
|
#define ASM_KVM_CACHE_REGS_H
|
|
|
|
#include <linux/kvm_host.h>
|
|
|
|
#define KVM_POSSIBLE_CR0_GUEST_BITS (X86_CR0_TS | X86_CR0_WP)
|
|
#define KVM_POSSIBLE_CR4_GUEST_BITS \
|
|
(X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR \
|
|
| X86_CR4_OSXMMEXCPT | X86_CR4_PGE | X86_CR4_TSD | X86_CR4_FSGSBASE)
|
|
|
|
#define X86_CR0_PDPTR_BITS (X86_CR0_CD | X86_CR0_NW | X86_CR0_PG)
|
|
#define X86_CR4_TLBFLUSH_BITS (X86_CR4_PGE | X86_CR4_PCIDE | X86_CR4_PAE | X86_CR4_SMEP)
|
|
#define X86_CR4_PDPTR_BITS (X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_SMEP)
|
|
|
|
static_assert(!(KVM_POSSIBLE_CR0_GUEST_BITS & X86_CR0_PDPTR_BITS));
|
|
|
|
#define BUILD_KVM_GPR_ACCESSORS(lname, uname) \
|
|
static __always_inline unsigned long kvm_##lname##_read(struct kvm_vcpu *vcpu)\
|
|
{ \
|
|
return vcpu->arch.regs[VCPU_REGS_##uname]; \
|
|
} \
|
|
static __always_inline void kvm_##lname##_write(struct kvm_vcpu *vcpu, \
|
|
unsigned long val) \
|
|
{ \
|
|
vcpu->arch.regs[VCPU_REGS_##uname] = val; \
|
|
}
|
|
BUILD_KVM_GPR_ACCESSORS(rax, RAX)
|
|
BUILD_KVM_GPR_ACCESSORS(rbx, RBX)
|
|
BUILD_KVM_GPR_ACCESSORS(rcx, RCX)
|
|
BUILD_KVM_GPR_ACCESSORS(rdx, RDX)
|
|
BUILD_KVM_GPR_ACCESSORS(rbp, RBP)
|
|
BUILD_KVM_GPR_ACCESSORS(rsi, RSI)
|
|
BUILD_KVM_GPR_ACCESSORS(rdi, RDI)
|
|
#ifdef CONFIG_X86_64
|
|
BUILD_KVM_GPR_ACCESSORS(r8, R8)
|
|
BUILD_KVM_GPR_ACCESSORS(r9, R9)
|
|
BUILD_KVM_GPR_ACCESSORS(r10, R10)
|
|
BUILD_KVM_GPR_ACCESSORS(r11, R11)
|
|
BUILD_KVM_GPR_ACCESSORS(r12, R12)
|
|
BUILD_KVM_GPR_ACCESSORS(r13, R13)
|
|
BUILD_KVM_GPR_ACCESSORS(r14, R14)
|
|
BUILD_KVM_GPR_ACCESSORS(r15, R15)
|
|
#endif
|
|
|
|
/*
|
|
* Using the register cache from interrupt context is generally not allowed, as
|
|
* caching a register and marking it available/dirty can't be done atomically,
|
|
* i.e. accesses from interrupt context may clobber state or read stale data if
|
|
* the vCPU task is in the process of updating the cache. The exception is if
|
|
* KVM is handling a PMI IRQ/NMI VM-Exit, as that bound code sequence doesn't
|
|
* touch the cache, it runs after the cache is reset (post VM-Exit), and PMIs
|
|
* need to access several registers that are cacheable.
|
|
*/
|
|
#define kvm_assert_register_caching_allowed(vcpu) \
|
|
lockdep_assert_once(in_task() || kvm_arch_pmi_in_guest(vcpu))
|
|
|
|
/*
|
|
* avail dirty
|
|
* 0 0 register in VMCS/VMCB
|
|
* 0 1 *INVALID*
|
|
* 1 0 register in vcpu->arch
|
|
* 1 1 register in vcpu->arch, needs to be stored back
|
|
*/
|
|
static inline bool kvm_register_is_available(struct kvm_vcpu *vcpu,
|
|
enum kvm_reg reg)
|
|
{
|
|
kvm_assert_register_caching_allowed(vcpu);
|
|
return test_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
|
|
}
|
|
|
|
static inline bool kvm_register_is_dirty(struct kvm_vcpu *vcpu,
|
|
enum kvm_reg reg)
|
|
{
|
|
kvm_assert_register_caching_allowed(vcpu);
|
|
return test_bit(reg, (unsigned long *)&vcpu->arch.regs_dirty);
|
|
}
|
|
|
|
static inline void kvm_register_mark_available(struct kvm_vcpu *vcpu,
|
|
enum kvm_reg reg)
|
|
{
|
|
kvm_assert_register_caching_allowed(vcpu);
|
|
__set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
|
|
}
|
|
|
|
static inline void kvm_register_mark_dirty(struct kvm_vcpu *vcpu,
|
|
enum kvm_reg reg)
|
|
{
|
|
kvm_assert_register_caching_allowed(vcpu);
|
|
__set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
|
|
__set_bit(reg, (unsigned long *)&vcpu->arch.regs_dirty);
|
|
}
|
|
|
|
/*
|
|
* kvm_register_test_and_mark_available() is a special snowflake that uses an
|
|
* arch bitop directly to avoid the explicit instrumentation that comes with
|
|
* the generic bitops. This allows code that cannot be instrumented (noinstr
|
|
* functions), e.g. the low level VM-Enter/VM-Exit paths, to cache registers.
|
|
*/
|
|
static __always_inline bool kvm_register_test_and_mark_available(struct kvm_vcpu *vcpu,
|
|
enum kvm_reg reg)
|
|
{
|
|
kvm_assert_register_caching_allowed(vcpu);
|
|
return arch___test_and_set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
|
|
}
|
|
|
|
/*
|
|
* The "raw" register helpers are only for cases where the full 64 bits of a
|
|
* register are read/written irrespective of current vCPU mode. In other words,
|
|
* odds are good you shouldn't be using the raw variants.
|
|
*/
|
|
static inline unsigned long kvm_register_read_raw(struct kvm_vcpu *vcpu, int reg)
|
|
{
|
|
if (WARN_ON_ONCE((unsigned int)reg >= NR_VCPU_REGS))
|
|
return 0;
|
|
|
|
if (!kvm_register_is_available(vcpu, reg))
|
|
kvm_x86_call(cache_reg)(vcpu, reg);
|
|
|
|
return vcpu->arch.regs[reg];
|
|
}
|
|
|
|
static inline void kvm_register_write_raw(struct kvm_vcpu *vcpu, int reg,
|
|
unsigned long val)
|
|
{
|
|
if (WARN_ON_ONCE((unsigned int)reg >= NR_VCPU_REGS))
|
|
return;
|
|
|
|
vcpu->arch.regs[reg] = val;
|
|
kvm_register_mark_dirty(vcpu, reg);
|
|
}
|
|
|
|
static inline unsigned long kvm_rip_read(struct kvm_vcpu *vcpu)
|
|
{
|
|
return kvm_register_read_raw(vcpu, VCPU_REGS_RIP);
|
|
}
|
|
|
|
static inline void kvm_rip_write(struct kvm_vcpu *vcpu, unsigned long val)
|
|
{
|
|
kvm_register_write_raw(vcpu, VCPU_REGS_RIP, val);
|
|
}
|
|
|
|
static inline unsigned long kvm_rsp_read(struct kvm_vcpu *vcpu)
|
|
{
|
|
return kvm_register_read_raw(vcpu, VCPU_REGS_RSP);
|
|
}
|
|
|
|
static inline void kvm_rsp_write(struct kvm_vcpu *vcpu, unsigned long val)
|
|
{
|
|
kvm_register_write_raw(vcpu, VCPU_REGS_RSP, val);
|
|
}
|
|
|
|
static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index)
|
|
{
|
|
might_sleep(); /* on svm */
|
|
|
|
if (!kvm_register_is_available(vcpu, VCPU_EXREG_PDPTR))
|
|
kvm_x86_call(cache_reg)(vcpu, VCPU_EXREG_PDPTR);
|
|
|
|
return vcpu->arch.walk_mmu->pdptrs[index];
|
|
}
|
|
|
|
static inline void kvm_pdptr_write(struct kvm_vcpu *vcpu, int index, u64 value)
|
|
{
|
|
vcpu->arch.walk_mmu->pdptrs[index] = value;
|
|
}
|
|
|
|
static inline ulong kvm_read_cr0_bits(struct kvm_vcpu *vcpu, ulong mask)
|
|
{
|
|
ulong tmask = mask & KVM_POSSIBLE_CR0_GUEST_BITS;
|
|
if ((tmask & vcpu->arch.cr0_guest_owned_bits) &&
|
|
!kvm_register_is_available(vcpu, VCPU_EXREG_CR0))
|
|
kvm_x86_call(cache_reg)(vcpu, VCPU_EXREG_CR0);
|
|
return vcpu->arch.cr0 & mask;
|
|
}
|
|
|
|
static __always_inline bool kvm_is_cr0_bit_set(struct kvm_vcpu *vcpu,
|
|
unsigned long cr0_bit)
|
|
{
|
|
BUILD_BUG_ON(!is_power_of_2(cr0_bit));
|
|
|
|
return !!kvm_read_cr0_bits(vcpu, cr0_bit);
|
|
}
|
|
|
|
static inline ulong kvm_read_cr0(struct kvm_vcpu *vcpu)
|
|
{
|
|
return kvm_read_cr0_bits(vcpu, ~0UL);
|
|
}
|
|
|
|
static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask)
|
|
{
|
|
ulong tmask = mask & KVM_POSSIBLE_CR4_GUEST_BITS;
|
|
if ((tmask & vcpu->arch.cr4_guest_owned_bits) &&
|
|
!kvm_register_is_available(vcpu, VCPU_EXREG_CR4))
|
|
kvm_x86_call(cache_reg)(vcpu, VCPU_EXREG_CR4);
|
|
return vcpu->arch.cr4 & mask;
|
|
}
|
|
|
|
static __always_inline bool kvm_is_cr4_bit_set(struct kvm_vcpu *vcpu,
|
|
unsigned long cr4_bit)
|
|
{
|
|
BUILD_BUG_ON(!is_power_of_2(cr4_bit));
|
|
|
|
return !!kvm_read_cr4_bits(vcpu, cr4_bit);
|
|
}
|
|
|
|
static inline ulong kvm_read_cr3(struct kvm_vcpu *vcpu)
|
|
{
|
|
if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3))
|
|
kvm_x86_call(cache_reg)(vcpu, VCPU_EXREG_CR3);
|
|
return vcpu->arch.cr3;
|
|
}
|
|
|
|
static inline ulong kvm_read_cr4(struct kvm_vcpu *vcpu)
|
|
{
|
|
return kvm_read_cr4_bits(vcpu, ~0UL);
|
|
}
|
|
|
|
static inline u64 kvm_read_edx_eax(struct kvm_vcpu *vcpu)
|
|
{
|
|
return (kvm_rax_read(vcpu) & -1u)
|
|
| ((u64)(kvm_rdx_read(vcpu) & -1u) << 32);
|
|
}
|
|
|
|
static inline void enter_guest_mode(struct kvm_vcpu *vcpu)
|
|
{
|
|
vcpu->arch.hflags |= HF_GUEST_MASK;
|
|
vcpu->stat.guest_mode = 1;
|
|
}
|
|
|
|
static inline void leave_guest_mode(struct kvm_vcpu *vcpu)
|
|
{
|
|
vcpu->arch.hflags &= ~HF_GUEST_MASK;
|
|
|
|
if (vcpu->arch.load_eoi_exitmap_pending) {
|
|
vcpu->arch.load_eoi_exitmap_pending = false;
|
|
kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);
|
|
}
|
|
|
|
vcpu->stat.guest_mode = 0;
|
|
}
|
|
|
|
static inline bool is_guest_mode(struct kvm_vcpu *vcpu)
|
|
{
|
|
return vcpu->arch.hflags & HF_GUEST_MASK;
|
|
}
|
|
|
|
#endif
|