From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from dfw.source.kernel.org (dfw.source.kernel.org [139.178.84.217]) by mx.groups.io with SMTP id smtpd.web11.74763.1656957195536778091 for ; Mon, 04 Jul 2022 10:53:15 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@kernel.org header.s=k20201202 header.b=E20v0GFW; spf=pass (domain: kernel.org, ip: 139.178.84.217, mailfrom: ardb@kernel.org) Received: from smtp.kernel.org (relay.kernel.org [52.25.139.140]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dfw.source.kernel.org (Postfix) with ESMTPS id D4928615F6; Mon, 4 Jul 2022 17:53:14 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id D9A19C341CB; Mon, 4 Jul 2022 17:53:12 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1656957194; bh=aTDmPKjlKnw984etZeVvIBimne0Q63ChvKwlnZHsW/w=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=E20v0GFWzk2m9Qs8RcRIyzzWXNeJU2EvZSbvrCp/xY+ExnBq7Le6fdVpd53s6L17r 2nwnXRgDE/1ggRi8I10NpEbKm+Ie7w6NpCXJRbGYDnSRKz8EF0tzS8nr/h73l3sa6a zrIq6MZqPV997uK7AIs332w8YMyQ+OYLyflBp+nYeefkOvQk3nAZoi01RNNrOby8en Mcc9aFdwfthIVujhDJcERFmOSlAnwUjE/3xsGyoM14zKOFCjY9XpsGj5T1Art4YN8m je5Q0I6ZyL3aaO2htNZIZ3zpnMwCaJ4P8jlX5b+2viye90tH5vqwF3fRDsVXcEllZr CSKYk2toc5MEA== From: "Ard Biesheuvel" To: devel@edk2.groups.io Cc: quic_llindhol@quicinc.com, sami.mujawar@arm.com, Ard Biesheuvel , Marc Zyngier , Alexander Graf Subject: [PATCH 2/7] ArmPkg/ArmMmuLib: use shadow page tables for break-before-make at EL1 Date: Mon, 4 Jul 2022 19:52:47 +0200 Message-Id: <20220704175252.4015120-3-ardb@kernel.org> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20220704175252.4015120-1-ardb@kernel.org> References: <20220704175252.4015120-1-ardb@kernel.org> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable When executing at EL1, disabling and re-enabling the MMU every time we need to replace a live translation entry is slightly problematic, given that memory accesses performed with the MMU off have non-cacheable attributes and are therefore non-coherent. On bare metal, we can deal with this by adding some barriers and cache invalidation instructions, but when running under virtualization, elaborate trapping and cache maintenance logic is necessary on the part of the hypervisor, and this is better avoided. So let's switch to a different approach when running at EL1, and use two sets of page tables with different ASIDs, and non-global attributes for all mappings. This allows us to switch between those sets without having to care about break-before-make, which means we can manipulate the primary translation while running from the secondary. To avoid splitting block mappings unnecessarily in the shadow page tables, add a special case to the recursive mapping routines to retain a block mapping that already covers a region that we are trying to map, and has the right attributes. Signed-off-by: Ard Biesheuvel --- ArmPkg/Drivers/CpuDxe/AArch64/Mmu.c | 3 + ArmPkg/Include/Chipset/AArch64Mmu.h | 1 + ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c | 112 +++++++++++= +++------ ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibReplaceEntry.S | 15 ++- 4 files changed, 96 insertions(+), 35 deletions(-) diff --git a/ArmPkg/Drivers/CpuDxe/AArch64/Mmu.c b/ArmPkg/Drivers/CpuDxe/AA= rch64/Mmu.c index 8bb33046e707..1ccd9f7fece8 100644 --- a/ArmPkg/Drivers/CpuDxe/AArch64/Mmu.c +++ b/ArmPkg/Drivers/CpuDxe/AArch64/Mmu.c @@ -313,6 +313,9 @@ EfiAttributeToArmAttribute ( ArmAttributes |=3D TT_PXN_MASK;=0D }=0D =0D + if (ArmReadCurrentEL () =3D=3D AARCH64_EL1) {=0D + ArmAttributes |=3D TT_NG;=0D + }=0D return ArmAttributes;=0D }=0D =0D diff --git a/ArmPkg/Include/Chipset/AArch64Mmu.h b/ArmPkg/Include/Chipset/A= Arch64Mmu.h index 2ea2cc0a874d..763dc53908e2 100644 --- a/ArmPkg/Include/Chipset/AArch64Mmu.h +++ b/ArmPkg/Include/Chipset/AArch64Mmu.h @@ -67,6 +67,7 @@ =0D #define TT_NS BIT5=0D #define TT_AF BIT10=0D +#define TT_NG BIT11=0D =0D #define TT_SH_NON_SHAREABLE (0x0 << 8)=0D #define TT_SH_OUTER_SHAREABLE (0x2 << 8)=0D diff --git a/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c b/ArmPkg/Libr= ary/ArmMmuLib/AArch64/ArmMmuLibCore.c index b85ae5a30768..d5c19600dce8 100644 --- a/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c +++ b/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c @@ -25,23 +25,31 @@ ArmMemoryAttributeToPageAttribute ( IN ARM_MEMORY_REGION_ATTRIBUTES Attributes=0D )=0D {=0D + UINT64 NonGlobal;=0D +=0D + if (ArmReadCurrentEL () =3D=3D AARCH64_EL1) {=0D + NonGlobal =3D TT_NG;=0D + } else {=0D + NonGlobal =3D 0;=0D + }=0D +=0D switch (Attributes) {=0D case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK_NONSHAREABLE:=0D case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK_NONSHAREABLE:=0D - return TT_ATTR_INDX_MEMORY_WRITE_BACK;=0D + return TT_ATTR_INDX_MEMORY_WRITE_BACK | NonGlobal;=0D =0D case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK:=0D case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK:=0D - return TT_ATTR_INDX_MEMORY_WRITE_BACK | TT_SH_INNER_SHAREABLE;=0D + return TT_ATTR_INDX_MEMORY_WRITE_BACK | TT_SH_INNER_SHAREABLE | NonG= lobal;=0D =0D case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_THROUGH:=0D case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_THROUGH:=0D - return TT_ATTR_INDX_MEMORY_WRITE_THROUGH | TT_SH_INNER_SHAREABLE;=0D + return TT_ATTR_INDX_MEMORY_WRITE_THROUGH | TT_SH_INNER_SHAREABLE | N= onGlobal;=0D =0D // Uncached and device mappings are treated as outer shareable by defa= ult,=0D case ARM_MEMORY_REGION_ATTRIBUTE_UNCACHED_UNBUFFERED:=0D case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_UNCACHED_UNBUFFERED:=0D - return TT_ATTR_INDX_MEMORY_NON_CACHEABLE;=0D + return TT_ATTR_INDX_MEMORY_NON_CACHEABLE | NonGlobal;=0D =0D default:=0D ASSERT (0);=0D @@ -50,7 +58,7 @@ ArmMemoryAttributeToPageAttribute ( if (ArmReadCurrentEL () =3D=3D AARCH64_EL2) {=0D return TT_ATTR_INDX_DEVICE_MEMORY | TT_XN_MASK;=0D } else {=0D - return TT_ATTR_INDX_DEVICE_MEMORY | TT_UXN_MASK | TT_PXN_MASK;=0D + return TT_ATTR_INDX_DEVICE_MEMORY | TT_UXN_MASK | TT_PXN_MASK | TT= _NG;=0D }=0D }=0D }=0D @@ -203,7 +211,13 @@ UpdateRegionMappingRecursive ( {=0D ASSERT (Level < 3);=0D =0D - if (!IsTableEntry (*Entry, Level)) {=0D + if (IsBlockEntry (*Entry, Level) && AttributeClearMask =3D=3D 0 &&=0D + ((*Entry & TT_ATTRIBUTES_MASK) ^ AttributeSetMask) =3D=3D 0) {=0D + // The existing block entry already covers the region we are=0D + // trying to map with the correct attributes so no need to do=0D + // anything here=0D + continue;=0D + } else if (!IsTableEntry (*Entry, Level)) {=0D //=0D // No table entry exists yet, so we need to allocate a page table= =0D // for the next level.=0D @@ -304,7 +318,8 @@ UpdateRegionMapping ( IN UINT64 RegionStart,=0D IN UINT64 RegionLength,=0D IN UINT64 AttributeSetMask,=0D - IN UINT64 AttributeClearMask=0D + IN UINT64 AttributeClearMask,=0D + IN UINT64 *RootTable=0D )=0D {=0D UINTN T0SZ;=0D @@ -320,7 +335,7 @@ UpdateRegionMapping ( RegionStart + RegionLength,=0D AttributeSetMask,=0D AttributeClearMask,=0D - ArmGetTTBR0BaseAddress (),=0D + RootTable,=0D GetRootTableLevel (T0SZ)=0D );=0D }=0D @@ -336,7 +351,7 @@ FillTranslationTable ( MemoryRegion->VirtualBase,=0D MemoryRegion->Length,=0D ArmMemoryAttributeToPageAttribute (MemoryRegion->Attributes) | = TT_AF,=0D - 0=0D + 0, RootTable=0D );=0D }=0D =0D @@ -380,6 +395,10 @@ GcdAttributeToPageAttribute ( PageAttributes |=3D TT_AP_NO_RO;=0D }=0D =0D + if (ArmReadCurrentEL () =3D=3D AARCH64_EL1) {=0D + PageAttributes |=3D TT_NG;=0D + }=0D +=0D return PageAttributes | TT_AF;=0D }=0D =0D @@ -390,8 +409,9 @@ ArmSetMemoryAttributes ( IN UINT64 Attributes=0D )=0D {=0D - UINT64 PageAttributes;=0D - UINT64 PageAttributeMask;=0D + UINT64 PageAttributes;=0D + UINT64 PageAttributeMask;=0D + EFI_STATUS Status;=0D =0D PageAttributes =3D GcdAttributeToPageAttribute (Attributes);=0D PageAttributeMask =3D 0;=0D @@ -404,13 +424,22 @@ ArmSetMemoryAttributes ( PageAttributes &=3D TT_AP_MASK | TT_UXN_MASK | TT_PXN_MASK;=0D PageAttributeMask =3D ~(TT_ADDRESS_MASK_BLOCK_ENTRY | TT_AP_MASK |=0D TT_PXN_MASK | TT_XN_MASK);=0D + } else if (ArmReadCurrentEL () =3D=3D AARCH64_EL1) {=0D + // Update the shadow page tables=0D + Status =3D UpdateRegionMapping (BaseAddress, Length,=0D + (PageAttributes & ~(TT_AP_MASK | TT_PXN_MASK | TT_UXN_MASK)= ) | TT_AP_NO_RW,=0D + 0, ArmGetTTBR0BaseAddress () + EFI_PAGE_SIZE);=0D + if (EFI_ERROR (Status)) {=0D + return Status;=0D + }=0D }=0D =0D return UpdateRegionMapping (=0D BaseAddress,=0D Length,=0D PageAttributes,=0D - PageAttributeMask=0D + PageAttributeMask,=0D + ArmGetTTBR0BaseAddress ()=0D );=0D }=0D =0D @@ -423,7 +452,8 @@ SetMemoryRegionAttribute ( IN UINT64 BlockEntryMask=0D )=0D {=0D - return UpdateRegionMapping (BaseAddress, Length, Attributes, BlockEntryM= ask);=0D + return UpdateRegionMapping (BaseAddress, Length, Attributes,=0D + BlockEntryMask, ArmGetTTBR0BaseAddress ());=0D }=0D =0D EFI_STATUS=0D @@ -503,13 +533,15 @@ ArmConfigureMmu ( OUT UINTN *TranslationTableSize OPTIONAL=0D )=0D {=0D - VOID *TranslationTable;=0D - UINTN MaxAddressBits;=0D - UINT64 MaxAddress;=0D - UINTN T0SZ;=0D - UINTN RootTableEntryCount;=0D - UINT64 TCR;=0D - EFI_STATUS Status;=0D + UINT64 *TranslationTable;=0D + UINTN MaxAddressBits;=0D + UINT64 MaxAddress;=0D + UINTN T0SZ;=0D + UINTN RootTableEntryCount;=0D + UINT64 TCR;=0D + EFI_STATUS Status;=0D + ARM_MEMORY_REGION_DESCRIPTOR *MemTab;=0D + UINTN NumRootPages;=0D =0D if (MemoryTable =3D=3D NULL) {=0D ASSERT (MemoryTable !=3D NULL);=0D @@ -538,6 +570,8 @@ ArmConfigureMmu ( // Note: Bits 23 and 31 are reserved(RES1) bits in TCR_EL2=0D TCR =3D T0SZ | (1UL << 31) | (1UL << 23) | TCR_TG0_4KB;=0D =0D + NumRootPages =3D 1;=0D +=0D // Set the Physical Address Size using MaxAddress=0D if (MaxAddress < SIZE_4GB) {=0D TCR |=3D TCR_PS_4GB;=0D @@ -564,6 +598,8 @@ ArmConfigureMmu ( // Due to Cortex-A57 erratum #822227 we must set TG1[1] =3D=3D 1, rega= rdless of EPD1.=0D TCR =3D T0SZ | TCR_TG0_4KB | TCR_TG1_4KB | TCR_EPD1;=0D =0D + NumRootPages =3D 2;=0D +=0D // Set the Physical Address Size using MaxAddress=0D if (MaxAddress < SIZE_4GB) {=0D TCR |=3D TCR_IPS_4GB;=0D @@ -608,19 +644,11 @@ ArmConfigureMmu ( ArmSetTCR (TCR);=0D =0D // Allocate pages for translation table=0D - TranslationTable =3D AllocatePages (1);=0D + TranslationTable =3D AllocatePages (NumRootPages);=0D if (TranslationTable =3D=3D NULL) {=0D return EFI_OUT_OF_RESOURCES;=0D }=0D =0D - //=0D - // We set TTBR0 just after allocating the table to retrieve its location= from=0D - // the subsequent functions without needing to pass this value across th= e=0D - // functions. The MMU is only enabled after the translation tables are=0D - // populated.=0D - //=0D - ArmSetTTBR0 (TranslationTable);=0D -=0D if (TranslationTableBase !=3D NULL) {=0D *TranslationTableBase =3D TranslationTable;=0D }=0D @@ -639,13 +667,11 @@ ArmConfigureMmu ( );=0D ZeroMem (TranslationTable, RootTableEntryCount * sizeof (UINT64));=0D =0D - while (MemoryTable->Length !=3D 0) {=0D - Status =3D FillTranslationTable (TranslationTable, MemoryTable);=0D + for (MemTab =3D MemoryTable; MemTab->Length !=3D 0; MemTab++) {=0D + Status =3D FillTranslationTable (TranslationTable, MemTab);=0D if (EFI_ERROR (Status)) {=0D goto FreeTranslationTable;=0D }=0D -=0D - MemoryTable++;=0D }=0D =0D //=0D @@ -661,16 +687,34 @@ ArmConfigureMmu ( MAIR_ATTR (TT_ATTR_INDX_MEMORY_WRITE_BACK, MAIR_ATTR_NORMAL_MEMORY_WRI= TE_BACK)=0D );=0D =0D + ArmSetTTBR0 (TranslationTable);=0D +=0D ArmDisableAlignmentCheck ();=0D ArmEnableStackAlignmentCheck ();=0D ArmEnableInstructionCache ();=0D ArmEnableDataCache ();=0D =0D ArmEnableMmu ();=0D +=0D + if (NumRootPages > 1) {=0D + //=0D + // Clone all memory ranges into the shadow page tables that we will us= e=0D + // to temporarily switch to when manipulating live entries=0D + //=0D + ZeroMem (TranslationTable + TT_ENTRY_COUNT,=0D + RootTableEntryCount * sizeof (UINT64));=0D +=0D + for (MemTab =3D MemoryTable; MemTab->Length !=3D 0; MemTab++) {=0D + Status =3D FillTranslationTable (TranslationTable + TT_ENTRY_COUNT, = MemTab);=0D + if (EFI_ERROR (Status)) {=0D + goto FreeTranslationTable;=0D + }=0D + }=0D + }=0D return EFI_SUCCESS;=0D =0D FreeTranslationTable:=0D - FreePages (TranslationTable, 1);=0D + FreePages (TranslationTable, NumRootPages);=0D return Status;=0D }=0D =0D diff --git a/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibReplaceEntry.S b/Arm= Pkg/Library/ArmMmuLib/AArch64/ArmMmuLibReplaceEntry.S index 66ebca571e63..6929e081ed8d 100644 --- a/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibReplaceEntry.S +++ b/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibReplaceEntry.S @@ -59,7 +59,20 @@ ASM_FUNC(ArmReplaceLiveTranslationEntry) dsb nsh=0D =0D EL1_OR_EL2_OR_EL3(x3)=0D -1:__replace_entry 1=0D +1:mrs x8, ttbr0_el1=0D + add x9, x8, #0x1000 // advance to shadow page table=0D + orr x9, x9, #1 << 48 // use different ASID for shadow translatio= ns=0D + msr ttbr0_el1, x9=0D + isb=0D + str x1, [x0] // install the entry and make it observeabl= e=0D + dsb ishst // to the page table walker=0D + isb=0D + lsr x2, x2, #12=0D + tlbi vae1is, x2 // invalidate the updated entry=0D + dsb ish=0D + isb=0D + msr ttbr0_el1, x8 // switch back to original translation=0D + isb=0D b 4f=0D 2:__replace_entry 2=0D b 4f=0D --=20 2.35.1