From a7c992b4c4c1336129fc63f8587330d13b8d5c73 Mon Sep 17 00:00:00 2001 From: Vinay Chandra Dommeti Date: Tue, 7 Jul 2020 11:41:03 -0700 Subject: [PATCH 1/4] initial pcid support --- src/asm/asm.s | 6 ++++ src/asm/mod.rs | 6 ++++ src/instructions/tlb.rs | 75 ++++++++++++++++++++++++++++++++++++++++ src/registers/control.rs | 42 +++++++++++++++++++++- 4 files changed, 128 insertions(+), 1 deletion(-) diff --git a/src/asm/asm.s b/src/asm/asm.s index 39d577976..26e49052f 100644 --- a/src/asm/asm.s +++ b/src/asm/asm.s @@ -94,6 +94,12 @@ _x86_64_asm_invlpg: invlpg (%rdi) retq +.global _x86_64_asm_invpcid +.p2align 4 +_x86_64_asm_invpcid: + invpcid (%rsi), %rdi + retq + .global _x86_64_asm_ltr .p2align 4 _x86_64_asm_ltr: diff --git a/src/asm/mod.rs b/src/asm/mod.rs index 72140dc26..85e8da191 100644 --- a/src/asm/mod.rs +++ b/src/asm/mod.rs @@ -138,6 +138,12 @@ extern "C" { )] pub(crate) fn x86_64_asm_invlpg(addr: u64); + #[cfg_attr( + any(target_env = "gnu", target_env = "musl"), + link_name = "_x86_64_asm_invpcid" + )] + pub(crate) fn x86_64_asm_invpcid(kind: u64, desc: u64); + #[cfg_attr( any(target_env = "gnu", target_env = "musl"), link_name = "_x86_64_asm_read_cr0" diff --git a/src/instructions/tlb.rs b/src/instructions/tlb.rs index 553494b2a..39715d965 100644 --- a/src/instructions/tlb.rs +++ b/src/instructions/tlb.rs @@ -23,3 +23,78 @@ pub fn flush_all() { let (frame, flags) = Cr3::read(); unsafe { Cr3::write(frame, flags) } } + +/// The Invalidate PCID Command to execute. +#[derive(Debug)] +pub enum InvPicdCommand { + /// The logical processor invalidates mappings—except global translations—for the linear address and PCID specified. + IndividualAddressInvalidation(VirtAddr, Pcid), + + /// The logical processor invalidates all mappings—except global translations—associated with the PCID. + SingleContextInvalidation(Pcid), + + /// The logical processor invalidates all mappings—including global translations—associated with any PCID. + AllContextInvalidationIncludeGlobal, + + /// The logical processor invalidates all mappings—except global translations—associated with any PCID. + AllContextInvalidationExcludeGlobal, +} + +/// The INVPCID descriptor comprises 128 bits and consists of a PCID and a linear address. +/// For INVPCID type 0, the processor uses the full 64 bits of the linear address even outside 64-bit mode; the linear address is not used for other INVPCID types. +#[repr(u128)] +#[derive(Debug)] +struct InvpcidDescriptor { + address: u64, + pcid: u64, +} + +/// Structure of a PCID. A PCID has to be <= 4096 for x86_64. +#[repr(transparent)] +#[derive(Debug)] +pub struct Pcid(u64); + +impl Pcid { + /// Create a new PCID. Will result in a failure if the value of + /// PCID is out of expected bounds. + pub fn new(pcid: u16) -> Result { + if pcid >= 4096 { + Err("PCID should be < 4096.") + } else { + Ok(Pcid(pcid as u64)) + } + } + + /// Get the value of the current PCID. + pub fn value(&self) -> u16 { + self.0 as u16 + } +} + +/// Invalidate the given address in the TLB using the `invpcid` instruction. +#[inline] +pub fn flush_pcid(command: InvPicdCommand) { + + let mut desc = InvpcidDescriptor { + address: 0, + pcid: 0, + }; + + let kind; + match command { + InvPicdCommand::IndividualAddressInvalidation(addr, pcid) => { kind = 0; desc.pcid = pcid.value() as u64; desc.address = addr.as_u64() } + InvPicdCommand::SingleContextInvalidation(pcid) => { kind = 1; desc.pcid = pcid.0}, + InvPicdCommand::AllContextInvalidationIncludeGlobal => kind = 2, + InvPicdCommand::AllContextInvalidationExcludeGlobal => kind = 3, + } + + #[cfg(feature = "inline_asm")] + unsafe { + llvm_asm!("invpcid ($0), $1" :: "r" (&desc as *const InvpcidDescriptor as u64), "r" (kind) : "memory") + }; + + #[cfg(not(feature = "inline_asm"))] + unsafe { + crate::asm::x86_64_asm_invpcid(kind, &desc as *const InvpcidDescriptor as u64) + }; +} diff --git a/src/registers/control.rs b/src/registers/control.rs index 3a559c31c..775eacd3f 100644 --- a/src/registers/control.rs +++ b/src/registers/control.rs @@ -129,7 +129,7 @@ bitflags! { mod x86_64 { use super::*; use crate::structures::paging::PhysFrame; - use crate::{PhysAddr, VirtAddr}; + use crate::{PhysAddr, VirtAddr, instructions::tlb::Pcid}; impl Cr0 { /// Read the current set of CR0 flags. @@ -251,6 +251,28 @@ mod x86_64 { (frame, flags) } + /// Read the current P4 table address from the CR3 register along with PCID. + /// The correct functioning of this requires CR4.PCIDE = 1. + #[inline] + pub fn read_as_pcid() -> (PhysFrame, u16) { + let value: u64; + + #[cfg(feature = "inline_asm")] + unsafe { + llvm_asm!("mov %cr3, $0" : "=r" (value)); + } + + #[cfg(not(feature = "inline_asm"))] + unsafe { + value = crate::asm::x86_64_asm_read_cr3(); + } + + let addr = PhysAddr::new(value & 0x_000f_ffff_ffff_f000); + let frame = PhysFrame::containing_address(addr); + let pcid = value & 0xFFF; + (frame, pcid as u16) + } + /// Write a new P4 table address into the CR3 register. /// /// ## Safety @@ -267,6 +289,24 @@ mod x86_64 { #[cfg(not(feature = "inline_asm"))] crate::asm::x86_64_asm_write_cr3(value) } + + /// Write a new P4 table address into the CR3 register. + /// + /// ## Safety + /// Changing the level 4 page table is unsafe, because it's possible to violate memory safety by + /// changing the page mapping. + /// Also, using this requires CR4.PCIDE flag to be set. + #[inline] + pub unsafe fn write_pcid(frame: PhysFrame, pcid: Pcid) { + let addr = frame.start_address(); + let value = addr.as_u64() | pcid.value() as u64; + + #[cfg(feature = "inline_asm")] + llvm_asm!("mov $0, %cr3" :: "r" (value) : "memory"); + + #[cfg(not(feature = "inline_asm"))] + crate::asm::x86_64_asm_write_cr3(value) + } } impl Cr4 { From 59eaf668566a2802f4675e6e543746addbb41091 Mon Sep 17 00:00:00 2001 From: Vinay Chandra Dommeti Date: Tue, 7 Jul 2020 11:44:31 -0700 Subject: [PATCH 2/4] Formatting --- src/instructions/tlb.rs | 16 +++++++++++----- src/registers/control.rs | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/instructions/tlb.rs b/src/instructions/tlb.rs index 39715d965..a7165609d 100644 --- a/src/instructions/tlb.rs +++ b/src/instructions/tlb.rs @@ -55,7 +55,7 @@ struct InvpcidDescriptor { pub struct Pcid(u64); impl Pcid { - /// Create a new PCID. Will result in a failure if the value of + /// Create a new PCID. Will result in a failure if the value of /// PCID is out of expected bounds. pub fn new(pcid: u16) -> Result { if pcid >= 4096 { @@ -74,16 +74,22 @@ impl Pcid { /// Invalidate the given address in the TLB using the `invpcid` instruction. #[inline] pub fn flush_pcid(command: InvPicdCommand) { - let mut desc = InvpcidDescriptor { address: 0, pcid: 0, }; - + let kind; match command { - InvPicdCommand::IndividualAddressInvalidation(addr, pcid) => { kind = 0; desc.pcid = pcid.value() as u64; desc.address = addr.as_u64() } - InvPicdCommand::SingleContextInvalidation(pcid) => { kind = 1; desc.pcid = pcid.0}, + InvPicdCommand::IndividualAddressInvalidation(addr, pcid) => { + kind = 0; + desc.pcid = pcid.value() as u64; + desc.address = addr.as_u64() + } + InvPicdCommand::SingleContextInvalidation(pcid) => { + kind = 1; + desc.pcid = pcid.0 + } InvPicdCommand::AllContextInvalidationIncludeGlobal => kind = 2, InvPicdCommand::AllContextInvalidationExcludeGlobal => kind = 3, } diff --git a/src/registers/control.rs b/src/registers/control.rs index 775eacd3f..a7cc276a5 100644 --- a/src/registers/control.rs +++ b/src/registers/control.rs @@ -129,7 +129,7 @@ bitflags! { mod x86_64 { use super::*; use crate::structures::paging::PhysFrame; - use crate::{PhysAddr, VirtAddr, instructions::tlb::Pcid}; + use crate::{instructions::tlb::Pcid, PhysAddr, VirtAddr}; impl Cr0 { /// Read the current set of CR0 flags. From caf483bc0d15f9db9a7939b8278d4f080df5206d Mon Sep 17 00:00:00 2001 From: Vinay Chandra Dommeti Date: Mon, 28 Dec 2020 16:27:07 +0530 Subject: [PATCH 3/4] Comments --- src/instructions/tlb.rs | 46 ++++++++++++++++-------------- src/registers/control.rs | 60 +++++++++++++++++++--------------------- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/src/instructions/tlb.rs b/src/instructions/tlb.rs index 7deb2ace6..df918fa37 100644 --- a/src/instructions/tlb.rs +++ b/src/instructions/tlb.rs @@ -28,21 +28,21 @@ pub fn flush_all() { #[derive(Debug)] pub enum InvPicdCommand { /// The logical processor invalidates mappings—except global translations—for the linear address and PCID specified. - IndividualAddressInvalidation(VirtAddr, Pcid), + Address(VirtAddr, Pcid), /// The logical processor invalidates all mappings—except global translations—associated with the PCID. - SingleContextInvalidation(Pcid), + Single(Pcid), /// The logical processor invalidates all mappings—including global translations—associated with any PCID. - AllContextInvalidationIncludeGlobal, + All, /// The logical processor invalidates all mappings—except global translations—associated with any PCID. - AllContextInvalidationExcludeGlobal, + AllExceptGlobal, } /// The INVPCID descriptor comprises 128 bits and consists of a PCID and a linear address. /// For INVPCID type 0, the processor uses the full 64 bits of the linear address even outside 64-bit mode; the linear address is not used for other INVPCID types. -#[repr(u128)] +#[repr(C)] #[derive(Debug)] struct InvpcidDescriptor { address: u64, @@ -52,55 +52,59 @@ struct InvpcidDescriptor { /// Structure of a PCID. A PCID has to be <= 4096 for x86_64. #[repr(transparent)] #[derive(Debug)] -pub struct Pcid(u64); +pub struct Pcid(u16); impl Pcid { /// Create a new PCID. Will result in a failure if the value of /// PCID is out of expected bounds. - pub fn new(pcid: u16) -> Result { + pub const fn new(pcid: u16) -> Result { if pcid >= 4096 { Err("PCID should be < 4096.") } else { - Ok(Pcid(pcid as u64)) + Ok(Pcid(pcid)) } } /// Get the value of the current PCID. - pub fn value(&self) -> u16 { - self.0 as u16 + pub const fn value(&self) -> u16 { + self.0 } } /// Invalidate the given address in the TLB using the `invpcid` instruction. +/// +/// ## Safety +/// This function is unsafe as it requires CPUID.(EAX=07H, ECX=0H):EBX.INVPCID to be 1. #[inline] -pub fn flush_pcid(command: InvPicdCommand) { +pub unsafe fn flush_pcid(command: InvPicdCommand) { let mut desc = InvpcidDescriptor { address: 0, pcid: 0, }; - let kind; + let kind: u64; match command { - InvPicdCommand::IndividualAddressInvalidation(addr, pcid) => { + InvPicdCommand::Address(addr, pcid) => { kind = 0; - desc.pcid = pcid.value() as u64; + desc.pcid = pcid.value().into(); desc.address = addr.as_u64() } - InvPicdCommand::SingleContextInvalidation(pcid) => { + InvPicdCommand::Single(pcid) => { kind = 1; - desc.pcid = pcid.0 + desc.pcid = pcid.0.into() } - InvPicdCommand::AllContextInvalidationIncludeGlobal => kind = 2, - InvPicdCommand::AllContextInvalidationExcludeGlobal => kind = 3, + InvPicdCommand::All => kind = 2, + InvPicdCommand::AllExceptGlobal => kind = 3, } #[cfg(feature = "inline_asm")] - unsafe { - llvm_asm!("invpcid ($0), $1" :: "r" (&desc as *const InvpcidDescriptor as u64), "r" (kind) : "memory") + { + let desc_value = &desc as *const InvpcidDescriptor as u64; + asm!("invpcid ({0}), {1}", in(reg) desc_value, in(reg) kind); }; #[cfg(not(feature = "inline_asm"))] - unsafe { + { crate::asm::x86_64_asm_invpcid(kind, &desc as *const InvpcidDescriptor as u64) }; } diff --git a/src/registers/control.rs b/src/registers/control.rs index c114bdb00..97abb027e 100644 --- a/src/registers/control.rs +++ b/src/registers/control.rs @@ -128,8 +128,7 @@ bitflags! { #[cfg(feature = "instructions")] mod x86_64 { use super::*; - use crate::structures::paging::PhysFrame; - use crate::{instructions::tlb::Pcid, PhysAddr, VirtAddr}; + use crate::{instructions::tlb::Pcid, structures::paging::PhysFrame, PhysAddr, VirtAddr}; impl Cr0 { /// Read the current set of CR0 flags. @@ -233,6 +232,14 @@ mod x86_64 { /// Read the current P4 table address from the CR3 register. #[inline] pub fn read() -> (PhysFrame, Cr3Flags) { + let (frame, value) = Cr3::read_raw(); + let flags = Cr3Flags::from_bits_truncate(value.into()); + (frame, flags) + } + + /// Read the current P4 table address from the CR3 register + #[inline] + pub fn read_raw() -> (PhysFrame, u16) { let value: u64; #[cfg(feature = "inline_asm")] @@ -245,32 +252,18 @@ mod x86_64 { value = crate::asm::x86_64_asm_read_cr3(); } - let flags = Cr3Flags::from_bits_truncate(value); let addr = PhysAddr::new(value & 0x_000f_ffff_ffff_f000); let frame = PhysFrame::containing_address(addr); - (frame, flags) + (frame, (value & 0xFFF) as u16) } /// Read the current P4 table address from the CR3 register along with PCID. /// The correct functioning of this requires CR4.PCIDE = 1. + /// See [`Cr4Flags::PCID`] #[inline] - pub fn read_as_pcid() -> (PhysFrame, u16) { - let value: u64; - - #[cfg(feature = "inline_asm")] - unsafe { - llvm_asm!("mov %cr3, $0" : "=r" (value)); - } - - #[cfg(not(feature = "inline_asm"))] - unsafe { - value = crate::asm::x86_64_asm_read_cr3(); - } - - let addr = PhysAddr::new(value & 0x_000f_ffff_ffff_f000); - let frame = PhysFrame::containing_address(addr); - let pcid = value & 0xFFF; - (frame, pcid as u16) + pub fn read_pcid() -> (PhysFrame, Pcid) { + let (frame, value) = Cr3::read_raw(); + (frame, Pcid::new(value as u16).unwrap()) } /// Write a new P4 table address into the CR3 register. @@ -280,14 +273,7 @@ mod x86_64 { /// changing the page mapping. #[inline] pub unsafe fn write(frame: PhysFrame, flags: Cr3Flags) { - let addr = frame.start_address(); - let value = addr.as_u64() | flags.bits(); - - #[cfg(feature = "inline_asm")] - asm!("mov cr3, {}", in(reg) value, options(nostack)); - - #[cfg(not(feature = "inline_asm"))] - crate::asm::x86_64_asm_write_cr3(value) + Cr3::write_raw(frame, flags.bits() as u16); } /// Write a new P4 table address into the CR3 register. @@ -295,14 +281,24 @@ mod x86_64 { /// ## Safety /// Changing the level 4 page table is unsafe, because it's possible to violate memory safety by /// changing the page mapping. - /// Also, using this requires CR4.PCIDE flag to be set. + /// [`Cr4Flags::PCID`] must be set before calling this method. #[inline] pub unsafe fn write_pcid(frame: PhysFrame, pcid: Pcid) { + Cr3::write_raw(frame, pcid.value()); + } + + /// Write a new P4 table address into the CR3 register. + /// + /// ## Safety + /// Changing the level 4 page table is unsafe, because it's possible to violate memory safety by + /// changing the page mapping. + #[inline] + unsafe fn write_raw(frame: PhysFrame, val: u16) { let addr = frame.start_address(); - let value = addr.as_u64() | pcid.value() as u64; + let value = addr.as_u64() | val as u64; #[cfg(feature = "inline_asm")] - llvm_asm!("mov $0, %cr3" :: "r" (value) : "memory"); + asm!("mov cr3, {}", in(reg) value, options(nostack)); #[cfg(not(feature = "inline_asm"))] crate::asm::x86_64_asm_write_cr3(value) From b41372ce096d19b55568b9ca23d3088743fd5078 Mon Sep 17 00:00:00 2001 From: Vinay Chandra Dommeti Date: Tue, 29 Dec 2020 09:55:10 +0530 Subject: [PATCH 4/4] Fix asm syntax --- src/instructions/tlb.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/instructions/tlb.rs b/src/instructions/tlb.rs index df918fa37..0a97e7540 100644 --- a/src/instructions/tlb.rs +++ b/src/instructions/tlb.rs @@ -100,7 +100,7 @@ pub unsafe fn flush_pcid(command: InvPicdCommand) { #[cfg(feature = "inline_asm")] { let desc_value = &desc as *const InvpcidDescriptor as u64; - asm!("invpcid ({0}), {1}", in(reg) desc_value, in(reg) kind); + asm!("invpcid {1}, [{0}]", in(reg) desc_value, in(reg) kind); }; #[cfg(not(feature = "inline_asm"))]