From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from ams.source.kernel.org (ams.source.kernel.org [145.40.68.75]) by mx.groups.io with SMTP id smtpd.web10.240.1662476814137499351 for ; Tue, 06 Sep 2022 08:06:54 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@kernel.org header.s=k20201202 header.b=u01j2T/Q; spf=pass (domain: kernel.org, ip: 145.40.68.75, mailfrom: ardb@kernel.org) Received: from smtp.kernel.org (relay.kernel.org [52.25.139.140]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ams.source.kernel.org (Postfix) with ESMTPS id 1AAF9B81916; Tue, 6 Sep 2022 15:06:51 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id D47AEC433B5; Tue, 6 Sep 2022 15:06:48 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1662476809; bh=eKgwoqJnTBqpSKQsOZJh6RtsZxijIaSTYFGpNiEDRjY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=u01j2T/QPKyOhdRZUzQtXrlt4uHp4iK9QS36TttvqLrzkd174G2soItUUGAlyKU3q m2Y2fps1qrNhyG3GRZbVtdXO+BKTsQatYD8/V9D/amw062A8WkLuWUXK/kETBwE9HW XSLJw1Y9Ndvx1BDSvKHe7DQjrVILAMRoDHIO5yERFepIrq0KMu37iGyazDL0/hx5TX 1zTLgDQCM+hfHBwCzF3EW+lWIwlH24fDXdZx8NvHDSjVaP/gCv+1zNnqIW4l+UZUwp +3I9Qztn5IOkZdx/oICq02ZlpIN36aoCmfR03tt5KMsgnKzYlermVzLPuIwlZZmE7a 2B13Qkh4wnI8g== From: "Ard Biesheuvel" To: devel@edk2.groups.io Cc: quic_llindhol@quicinc.com, sami.mujawar@arm.com, Ard Biesheuvel Subject: [PATCH v2 2/7] ArmPkg/ArmMmuLib: use shadow page tables for break-before-make at EL1 Date: Tue, 6 Sep 2022 17:06:34 +0200 Message-Id: <20220906150639.157227-3-ardb@kernel.org> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20220906150639.157227-1-ardb@kernel.org> References: <20220906150639.157227-1-ardb@kernel.org> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable When executing at EL1, disabling and re-enabling the MMU every time we need to replace a live translation entry is slightly problematic, given that memory accesses performed with the MMU off have non-cacheable attributes and are therefore non-coherent. On bare metal, we can deal with this by adding some barriers and cache invalidation instructions, but when running under virtualization, elaborate trapping and cache maintenance logic is necessary on the part of the hypervisor, and this is better avoided. So let's switch to a different approach when running at EL1, and use two sets of page tables with different ASIDs, and non-global attributes for all mappings. This allows us to switch between those sets without having to care about break-before-make, which means we can manipulate the primary translation while running from the secondary. To avoid splitting block mappings unnecessarily in the shadow page tables, add a special case to the recursive mapping routines to retain a block mapping that already covers a region that we are trying to map, and has the right attributes. Signed-off-by: Ard Biesheuvel --- ArmPkg/Drivers/CpuDxe/AArch64/Mmu.c | 4 + ArmPkg/Include/Chipset/AArch64Mmu.h | 1 + ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c | 154 +++++++++++= ++++----- ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibReplaceEntry.S | 15 +- 4 files changed, 138 insertions(+), 36 deletions(-) diff --git a/ArmPkg/Drivers/CpuDxe/AArch64/Mmu.c b/ArmPkg/Drivers/CpuDxe/AA= rch64/Mmu.c index 8bb33046e707..d15eb158ad60 100644 --- a/ArmPkg/Drivers/CpuDxe/AArch64/Mmu.c +++ b/ArmPkg/Drivers/CpuDxe/AArch64/Mmu.c @@ -313,6 +313,10 @@ EfiAttributeToArmAttribute ( ArmAttributes |=3D TT_PXN_MASK;=0D }=0D =0D + if (ArmReadCurrentEL () =3D=3D AARCH64_EL1) {=0D + ArmAttributes |=3D TT_NG;=0D + }=0D +=0D return ArmAttributes;=0D }=0D =0D diff --git a/ArmPkg/Include/Chipset/AArch64Mmu.h b/ArmPkg/Include/Chipset/A= Arch64Mmu.h index 2ea2cc0a874d..763dc53908e2 100644 --- a/ArmPkg/Include/Chipset/AArch64Mmu.h +++ b/ArmPkg/Include/Chipset/AArch64Mmu.h @@ -67,6 +67,7 @@ =0D #define TT_NS BIT5=0D #define TT_AF BIT10=0D +#define TT_NG BIT11=0D =0D #define TT_SH_NON_SHAREABLE (0x0 << 8)=0D #define TT_SH_OUTER_SHAREABLE (0x2 << 8)=0D diff --git a/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c b/ArmPkg/Libr= ary/ArmMmuLib/AArch64/ArmMmuLibCore.c index 34f1031c4de3..747ebc533511 100644 --- a/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c +++ b/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c @@ -25,23 +25,31 @@ ArmMemoryAttributeToPageAttribute ( IN ARM_MEMORY_REGION_ATTRIBUTES Attributes=0D )=0D {=0D + UINT64 NonGlobal;=0D +=0D + if (ArmReadCurrentEL () =3D=3D AARCH64_EL1) {=0D + NonGlobal =3D TT_NG;=0D + } else {=0D + NonGlobal =3D 0;=0D + }=0D +=0D switch (Attributes) {=0D case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK_NONSHAREABLE:=0D case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK_NONSHAREABLE:=0D - return TT_ATTR_INDX_MEMORY_WRITE_BACK;=0D + return TT_ATTR_INDX_MEMORY_WRITE_BACK | NonGlobal;=0D =0D case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK:=0D case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK:=0D - return TT_ATTR_INDX_MEMORY_WRITE_BACK | TT_SH_INNER_SHAREABLE;=0D + return TT_ATTR_INDX_MEMORY_WRITE_BACK | TT_SH_INNER_SHAREABLE | NonG= lobal;=0D =0D case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_THROUGH:=0D case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_THROUGH:=0D - return TT_ATTR_INDX_MEMORY_WRITE_THROUGH | TT_SH_INNER_SHAREABLE;=0D + return TT_ATTR_INDX_MEMORY_WRITE_THROUGH | TT_SH_INNER_SHAREABLE | N= onGlobal;=0D =0D // Uncached and device mappings are treated as outer shareable by defa= ult,=0D case ARM_MEMORY_REGION_ATTRIBUTE_UNCACHED_UNBUFFERED:=0D case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_UNCACHED_UNBUFFERED:=0D - return TT_ATTR_INDX_MEMORY_NON_CACHEABLE;=0D + return TT_ATTR_INDX_MEMORY_NON_CACHEABLE | NonGlobal;=0D =0D default:=0D ASSERT (0);=0D @@ -50,7 +58,7 @@ ArmMemoryAttributeToPageAttribute ( if (ArmReadCurrentEL () =3D=3D AARCH64_EL2) {=0D return TT_ATTR_INDX_DEVICE_MEMORY | TT_XN_MASK;=0D } else {=0D - return TT_ATTR_INDX_DEVICE_MEMORY | TT_UXN_MASK | TT_PXN_MASK;=0D + return TT_ATTR_INDX_DEVICE_MEMORY | TT_UXN_MASK | TT_PXN_MASK | TT= _NG;=0D }=0D }=0D }=0D @@ -203,7 +211,14 @@ UpdateRegionMappingRecursive ( {=0D ASSERT (Level < 3);=0D =0D - if (!IsTableEntry (*Entry, Level)) {=0D + if (IsBlockEntry (*Entry, Level) && (AttributeClearMask =3D=3D 0) &&= =0D + ((*Entry & TT_ATTRIBUTES_MASK) =3D=3D AttributeSetMask))=0D + {=0D + // The existing block entry already covers the region we are=0D + // trying to map with the correct attributes so no need to do=0D + // anything here=0D + continue;=0D + } else if (!IsTableEntry (*Entry, Level)) {=0D //=0D // No table entry exists yet, so we need to allocate a page table= =0D // for the next level.=0D @@ -304,7 +319,8 @@ UpdateRegionMapping ( IN UINT64 RegionStart,=0D IN UINT64 RegionLength,=0D IN UINT64 AttributeSetMask,=0D - IN UINT64 AttributeClearMask=0D + IN UINT64 AttributeClearMask,=0D + IN UINT64 *RootTable=0D )=0D {=0D UINTN T0SZ;=0D @@ -320,7 +336,7 @@ UpdateRegionMapping ( RegionStart + RegionLength,=0D AttributeSetMask,=0D AttributeClearMask,=0D - ArmGetTTBR0BaseAddress (),=0D + RootTable,=0D GetRootTableLevel (T0SZ)=0D );=0D }=0D @@ -329,14 +345,26 @@ STATIC EFI_STATUS=0D FillTranslationTable (=0D IN UINT64 *RootTable,=0D - IN ARM_MEMORY_REGION_DESCRIPTOR *MemoryRegion=0D + IN ARM_MEMORY_REGION_DESCRIPTOR *MemoryRegion,=0D + IN BOOLEAN IsSecondary=0D )=0D {=0D + //=0D + // Omit non-memory mappings from the shadow page tables, which only need= to=0D + // cover system RAM.=0D + //=0D + if (IsSecondary &&=0D + (MemoryRegion->Attributes !=3D ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BAC= K))=0D + {=0D + return EFI_SUCCESS;=0D + }=0D +=0D return UpdateRegionMapping (=0D MemoryRegion->VirtualBase,=0D MemoryRegion->Length,=0D ArmMemoryAttributeToPageAttribute (MemoryRegion->Attributes) | = TT_AF,=0D - 0=0D + 0,=0D + RootTable=0D );=0D }=0D =0D @@ -380,6 +408,10 @@ GcdAttributeToPageAttribute ( PageAttributes |=3D TT_AP_NO_RO;=0D }=0D =0D + if (ArmReadCurrentEL () =3D=3D AARCH64_EL1) {=0D + PageAttributes |=3D TT_NG;=0D + }=0D +=0D return PageAttributes | TT_AF;=0D }=0D =0D @@ -390,8 +422,9 @@ ArmSetMemoryAttributes ( IN UINT64 Attributes=0D )=0D {=0D - UINT64 PageAttributes;=0D - UINT64 PageAttributeMask;=0D + UINT64 PageAttributes;=0D + UINT64 PageAttributeMask;=0D + EFI_STATUS Status;=0D =0D PageAttributes =3D GcdAttributeToPageAttribute (Attributes);=0D PageAttributeMask =3D 0;=0D @@ -404,13 +437,36 @@ ArmSetMemoryAttributes ( PageAttributes &=3D TT_AP_MASK | TT_UXN_MASK | TT_PXN_MASK;=0D PageAttributeMask =3D ~(TT_ADDRESS_MASK_BLOCK_ENTRY | TT_AP_MASK |=0D TT_PXN_MASK | TT_XN_MASK);=0D + } else if ((ArmReadCurrentEL () =3D=3D AARCH64_EL1) &&=0D + ((PageAttributes & TT_ATTR_INDX_MASK) =3D=3D TT_ATTR_INDX_MEM= ORY_WRITE_BACK))=0D + {=0D + //=0D + // Update the shadow page tables if we are running at EL1 and are mapp= ing=0D + // memory. This is needed because we may be adding memory that may be = used=0D + // later on for allocating page tables, and these need to be shadowed = as=0D + // well so we can update them safely. Strip the attributes so we don't= =0D + // fragment the shadow page tables unnecessarily. (Note that adding d= evice=0D + // memory here and stripping the XN attributes would be bad, as it cou= ld=0D + // result in speculative instruction fetches from MMIO regions.)=0D + //=0D + Status =3D UpdateRegionMapping (=0D + BaseAddress,=0D + Length,=0D + (PageAttributes & ~(TT_AP_MASK | TT_PXN_MASK | TT_UXN_MASK)= ) | TT_AP_NO_RW,=0D + 0,=0D + ArmGetTTBR0BaseAddress () + EFI_PAGE_SIZE=0D + );=0D + if (EFI_ERROR (Status)) {=0D + return Status;=0D + }=0D }=0D =0D return UpdateRegionMapping (=0D BaseAddress,=0D Length,=0D PageAttributes,=0D - PageAttributeMask=0D + PageAttributeMask,=0D + ArmGetTTBR0BaseAddress ()=0D );=0D }=0D =0D @@ -423,7 +479,13 @@ SetMemoryRegionAttribute ( IN UINT64 BlockEntryMask=0D )=0D {=0D - return UpdateRegionMapping (BaseAddress, Length, Attributes, BlockEntryM= ask);=0D + return UpdateRegionMapping (=0D + BaseAddress,=0D + Length,=0D + Attributes,=0D + BlockEntryMask,=0D + ArmGetTTBR0BaseAddress ()=0D + );=0D }=0D =0D EFI_STATUS=0D @@ -503,13 +565,15 @@ ArmConfigureMmu ( OUT UINTN *TranslationTableSize OPTIONAL=0D )=0D {=0D - VOID *TranslationTable;=0D - UINTN MaxAddressBits;=0D - UINT64 MaxAddress;=0D - UINTN T0SZ;=0D - UINTN RootTableEntryCount;=0D - UINT64 TCR;=0D - EFI_STATUS Status;=0D + UINT64 *TranslationTable;=0D + UINTN MaxAddressBits;=0D + UINT64 MaxAddress;=0D + UINTN T0SZ;=0D + UINTN RootTableEntryCount;=0D + UINT64 TCR;=0D + EFI_STATUS Status;=0D + ARM_MEMORY_REGION_DESCRIPTOR *MemTab;=0D + UINTN NumRootPages;=0D =0D if (MemoryTable =3D=3D NULL) {=0D ASSERT (MemoryTable !=3D NULL);=0D @@ -538,6 +602,8 @@ ArmConfigureMmu ( // Note: Bits 23 and 31 are reserved(RES1) bits in TCR_EL2=0D TCR =3D T0SZ | (1UL << 31) | (1UL << 23) | TCR_TG0_4KB;=0D =0D + NumRootPages =3D 1;=0D +=0D // Set the Physical Address Size using MaxAddress=0D if (MaxAddress < SIZE_4GB) {=0D TCR |=3D TCR_PS_4GB;=0D @@ -564,6 +630,8 @@ ArmConfigureMmu ( // Due to Cortex-A57 erratum #822227 we must set TG1[1] =3D=3D 1, rega= rdless of EPD1.=0D TCR =3D T0SZ | TCR_TG0_4KB | TCR_TG1_4KB | TCR_EPD1;=0D =0D + NumRootPages =3D 2;=0D +=0D // Set the Physical Address Size using MaxAddress=0D if (MaxAddress < SIZE_4GB) {=0D TCR |=3D TCR_IPS_4GB;=0D @@ -608,19 +676,11 @@ ArmConfigureMmu ( ArmSetTCR (TCR);=0D =0D // Allocate pages for translation table=0D - TranslationTable =3D AllocatePages (1);=0D + TranslationTable =3D AllocatePages (NumRootPages);=0D if (TranslationTable =3D=3D NULL) {=0D return EFI_OUT_OF_RESOURCES;=0D }=0D =0D - //=0D - // We set TTBR0 just after allocating the table to retrieve its location= from=0D - // the subsequent functions without needing to pass this value across th= e=0D - // functions. The MMU is only enabled after the translation tables are=0D - // populated.=0D - //=0D - ArmSetTTBR0 (TranslationTable);=0D -=0D if (TranslationTableBase !=3D NULL) {=0D *TranslationTableBase =3D TranslationTable;=0D }=0D @@ -637,15 +697,14 @@ ArmConfigureMmu ( TranslationTable,=0D RootTableEntryCount * sizeof (UINT64)=0D );=0D +=0D ZeroMem (TranslationTable, RootTableEntryCount * sizeof (UINT64));=0D =0D - while (MemoryTable->Length !=3D 0) {=0D - Status =3D FillTranslationTable (TranslationTable, MemoryTable);=0D + for (MemTab =3D MemoryTable; MemTab->Length !=3D 0; MemTab++) {=0D + Status =3D FillTranslationTable (TranslationTable, MemTab, FALSE);=0D if (EFI_ERROR (Status)) {=0D goto FreeTranslationTable;=0D }=0D -=0D - MemoryTable++;=0D }=0D =0D //=0D @@ -661,16 +720,41 @@ ArmConfigureMmu ( MAIR_ATTR (TT_ATTR_INDX_MEMORY_WRITE_BACK, MAIR_ATTR_NORMAL_MEMORY_WRI= TE_BACK)=0D );=0D =0D + ArmSetTTBR0 (TranslationTable);=0D +=0D ArmDisableAlignmentCheck ();=0D ArmEnableStackAlignmentCheck ();=0D ArmEnableInstructionCache ();=0D ArmEnableDataCache ();=0D =0D ArmEnableMmu ();=0D +=0D + if (NumRootPages > 1) {=0D + //=0D + // Clone all memory ranges into the shadow page tables that we will us= e=0D + // to temporarily switch to when manipulating live entries=0D + //=0D + ZeroMem (=0D + TranslationTable + TT_ENTRY_COUNT,=0D + RootTableEntryCount * sizeof (UINT64)=0D + );=0D +=0D + for (MemTab =3D MemoryTable; MemTab->Length !=3D 0; MemTab++) {=0D + Status =3D FillTranslationTable (=0D + TranslationTable + TT_ENTRY_COUNT,=0D + MemTab,=0D + TRUE=0D + );=0D + if (EFI_ERROR (Status)) {=0D + goto FreeTranslationTable;=0D + }=0D + }=0D + }=0D +=0D return EFI_SUCCESS;=0D =0D FreeTranslationTable:=0D - FreePages (TranslationTable, 1);=0D + FreePages (TranslationTable, NumRootPages);=0D return Status;=0D }=0D =0D diff --git a/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibReplaceEntry.S b/Arm= Pkg/Library/ArmMmuLib/AArch64/ArmMmuLibReplaceEntry.S index 66ebca571e63..6929e081ed8d 100644 --- a/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibReplaceEntry.S +++ b/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibReplaceEntry.S @@ -59,7 +59,20 @@ ASM_FUNC(ArmReplaceLiveTranslationEntry) dsb nsh=0D =0D EL1_OR_EL2_OR_EL3(x3)=0D -1:__replace_entry 1=0D +1:mrs x8, ttbr0_el1=0D + add x9, x8, #0x1000 // advance to shadow page table=0D + orr x9, x9, #1 << 48 // use different ASID for shadow translatio= ns=0D + msr ttbr0_el1, x9=0D + isb=0D + str x1, [x0] // install the entry and make it observeabl= e=0D + dsb ishst // to the page table walker=0D + isb=0D + lsr x2, x2, #12=0D + tlbi vae1is, x2 // invalidate the updated entry=0D + dsb ish=0D + isb=0D + msr ttbr0_el1, x8 // switch back to original translation=0D + isb=0D b 4f=0D 2:__replace_entry 2=0D b 4f=0D --=20 2.35.1