[RFC] [PATCH V3 2/3] MdeModulePkg/PciHostBridge: Add IOMMU support.

[Jiewen Yao <jiewen.yao@intel.com>]

The responsibility of PciHostBridge is to allocate IOMMU
page aligned memory for Map and AllocateBuffer,
because PciHostBridge driver already handles Map() request
to allocate another buffer for DMA read/write.

If the max address requirement can not be satisfied,
PciHostBridge may also allocate any IOMMU page aligned memory
and use IOMMU Remapping feature to map to lower address to
satisfy device requirement.

PciHostBridge does not set IOMMU attribute because it does
not know which device request the DMA. This work is done
by PciBus driver.

Cc: Ruiyu Ni <ruiyu.ni@intel.com>
Cc: Leo Duran <leo.duran@amd.com>
Cc: Brijesh Singh <brijesh.singh@amd.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: Jiewen Yao <jiewen.yao@intel.com>
---
 MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciHostBridge.c      |   3 +
 MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciHostBridgeDxe.inf |   1 +
 MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciRootBridge.h      |   8 +
 MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciRootBridgeIo.c    | 350 ++++++++++++++++++--
 4 files changed, 326 insertions(+), 36 deletions(-)

diff --git a/MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciHostBridge.c b/MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciHostBridge.c
index 9005dee..35233a7 100644
--- a/MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciHostBridge.c
+++ b/MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciHostBridge.c
@@ -28,6 +28,9 @@ GLOBAL_REMOVE_IF_UNREFERENCED CHAR16 *mPciResourceTypeStr[] = {
   L"I/O", L"Mem", L"PMem", L"Mem64", L"PMem64", L"Bus"
 };
 
+EDKII_IOMMU_PROTOCOL        *gIoMmuProtocol;
+UINTN                       mIoMmuPageSize = 1;
+
 /**
   Ensure the compatibility of an IO space descriptor with the IO aperture.
 
diff --git a/MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciHostBridgeDxe.inf b/MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciHostBridgeDxe.inf
index d8b0439..2d3c8c9 100644
--- a/MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciHostBridgeDxe.inf
+++ b/MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciHostBridgeDxe.inf
@@ -49,6 +49,7 @@
   gEfiDevicePathProtocolGuid                      ## BY_START
   gEfiPciRootBridgeIoProtocolGuid                 ## BY_START
   gEfiPciHostBridgeResourceAllocationProtocolGuid ## BY_START
+  gEdkiiIoMmuProtocolGuid                         ## CONSUMES
 
 [Depex]
   gEfiCpuIo2ProtocolGuid AND
diff --git a/MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciRootBridge.h b/MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciRootBridge.h
index 13185b4..77c3490 100644
--- a/MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciRootBridge.h
+++ b/MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciRootBridge.h
@@ -27,6 +27,7 @@ WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
 #include <Protocol/CpuIo2.h>
 #include <Protocol/DevicePath.h>
 #include <Protocol/PciRootBridgeIo.h>
+#include <Protocol/IoMmu.h>
 #include <Library/DebugLib.h>
 #include <Library/DevicePathLib.h>
 #include <Library/BaseMemoryLib.h>
@@ -50,8 +51,11 @@ typedef struct {
   EFI_PCI_ROOT_BRIDGE_IO_PROTOCOL_OPERATION Operation;
   UINTN                                     NumberOfBytes;
   UINTN                                     NumberOfPages;
+  UINTN                                     MappedNumberOfBytes;
+  UINTN                                     MappedNumberOfPages;
   EFI_PHYSICAL_ADDRESS                      HostAddress;
   EFI_PHYSICAL_ADDRESS                      MappedHostAddress;
+  BOOLEAN                                   RemapNonExisting;
 } MAP_INFO;
 #define MAP_INFO_FROM_LINK(a) CR (a, MAP_INFO, Link, MAP_INFO_SIGNATURE)
 
@@ -575,4 +579,8 @@ RootBridgeIoConfiguration (
 
 extern EFI_METRONOME_ARCH_PROTOCOL *mMetronome;
 extern EFI_CPU_IO2_PROTOCOL         *mCpuIo;
+
+extern EDKII_IOMMU_PROTOCOL        *gIoMmuProtocol;
+extern UINTN                       mIoMmuPageSize;
+
 #endif
diff --git a/MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciRootBridgeIo.c b/MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciRootBridgeIo.c
index 8af131b..2a17eb1 100644
--- a/MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciRootBridgeIo.c
+++ b/MdeModulePkg/Bus/Pci/PciHostBridgeDxe/PciRootBridgeIo.c
@@ -1022,6 +1022,121 @@ RootBridgeIoPciWrite (
 }
 
 /**
+  Allocates one or more 4KB pages of a certain memory type at a specified alignment.
+
+  Allocates the number of 4KB pages specified by Pages of a certain memory type with an alignment
+  specified by Alignment.  The allocated buffer is returned.  If Pages is 0, then NULL is returned.
+  If there is not enough memory at the specified alignment remaining to satisfy the request, then
+  NULL is returned.
+  If Alignment is not a power of two and Alignment is not zero, then ASSERT().
+  If Pages plus EFI_SIZE_TO_PAGES (Alignment) overflows, then ASSERT().
+
+  @param  Type                  The type of allocation to perform.
+  @param  MemoryType            The type of memory to allocate.
+  @param  Pages                 The number of 4 KB pages to allocate.
+  @param  Alignment             The requested alignment of the allocation.  Must be a power of two.
+                                If Alignment is zero, then byte alignment is used.
+  @param  Address               Pointer to a physical address.
+
+  @return Memory Allocation Status.
+
+**/
+EFI_STATUS
+InternalAllocateAlignedPagesWithAllocateType (
+  IN EFI_ALLOCATE_TYPE         Type,
+  IN EFI_MEMORY_TYPE           MemoryType,
+  IN UINTN                     Pages,
+  IN UINTN                     Alignment,
+  IN OUT EFI_PHYSICAL_ADDRESS  *Address
+  )
+{
+  EFI_STATUS            Status;
+  EFI_PHYSICAL_ADDRESS  Memory;
+  UINTN                 AlignedMemory;
+  UINTN                 AlignmentMask;
+  UINTN                 UnalignedPages;
+  UINTN                 RealPages;
+
+  //
+  // Alignment must be a power of two or zero.
+  //
+  ASSERT ((Alignment & (Alignment - 1)) == 0);
+
+  if (Pages == 0) {
+    return EFI_INVALID_PARAMETER;
+  }
+  if (Alignment > EFI_PAGE_SIZE) {
+    //
+    // Calculate the total number of pages since alignment is larger than page size.
+    //
+    AlignmentMask  = Alignment - 1;
+    RealPages      = Pages + EFI_SIZE_TO_PAGES (Alignment);
+    //
+    // Make sure that Pages plus EFI_SIZE_TO_PAGES (Alignment) does not overflow.
+    //
+    ASSERT (RealPages > Pages);
+
+    Memory         = *Address;
+    Status         = gBS->AllocatePages (Type, MemoryType, RealPages, &Memory);
+    if (EFI_ERROR (Status)) {
+      return Status;
+    }
+    AlignedMemory  = ((UINTN) Memory + AlignmentMask) & ~AlignmentMask;
+    UnalignedPages = EFI_SIZE_TO_PAGES (AlignedMemory - (UINTN) Memory);
+    if (UnalignedPages > 0) {
+      //
+      // Free first unaligned page(s).
+      //
+      Status = gBS->FreePages (Memory, UnalignedPages);
+      ASSERT_EFI_ERROR (Status);
+    }
+    Memory         = (EFI_PHYSICAL_ADDRESS) (AlignedMemory + EFI_PAGES_TO_SIZE (Pages));
+    UnalignedPages = RealPages - Pages - UnalignedPages;
+    if (UnalignedPages > 0) {
+      //
+      // Free last unaligned page(s).
+      //
+      Status = gBS->FreePages (Memory, UnalignedPages);
+      ASSERT_EFI_ERROR (Status);
+    }
+  } else {
+    //
+    // Do not over-allocate pages in this case.
+    //
+    Memory = *Address;
+    Status = gBS->AllocatePages (Type, MemoryType, Pages, &Memory);
+    if (EFI_ERROR (Status)) {
+      return Status;
+    }
+    AlignedMemory  = (UINTN) Memory;
+  }
+  *Address = AlignedMemory;
+  return EFI_SUCCESS;
+}
+
+/**
+  Return if a value is aligned.
+
+  @param Value the value to be checked
+  @param Alignment the alignment to be checked with.
+
+  @retval TRUE  The value is aligned
+  @retval FALSE The value is not aligned.
+**/
+BOOLEAN
+InternalIsAlgined (
+  IN UINTN  Value,
+  IN UINTN  Alignment
+  )
+{
+  if (Value == ALIGN_VALUE(Value, Alignment)) {
+    return TRUE;
+  } else {
+    return FALSE;
+  }
+}
+
+/**
   Provides the PCI controller-specific address needed to access
   system memory for DMA.
 
@@ -1057,6 +1172,9 @@ RootBridgeIoMap (
   PCI_ROOT_BRIDGE_INSTANCE                          *RootBridge;
   EFI_PHYSICAL_ADDRESS                              PhysicalAddress;
   MAP_INFO                                          *MapInfo;
+  EFI_PHYSICAL_ADDRESS                              MaxAddress;
+  BOOLEAN                                           NeedMap;
+  BOOLEAN                                           NeedAllocateNonExisting;
 
   if (HostAddress == NULL || NumberOfBytes == NULL || DeviceAddress == NULL ||
       Mapping == NULL) {
@@ -1072,13 +1190,41 @@ RootBridgeIoMap (
 
   RootBridge = ROOT_BRIDGE_FROM_THIS (This);
 
+  if (gIoMmuProtocol == NULL) {
+    gBS->LocateProtocol (
+          &gEdkiiIoMmuProtocolGuid,
+          NULL,
+          (VOID **) &gIoMmuProtocol
+          );
+    if (gIoMmuProtocol != NULL) {
+      gIoMmuProtocol->GetPageSize (gIoMmuProtocol, &mIoMmuPageSize);
+      ASSERT ((mIoMmuPageSize & (mIoMmuPageSize - 1)) == 0);
+    }
+  }
+
   PhysicalAddress = (EFI_PHYSICAL_ADDRESS) (UINTN) HostAddress;
+
+  MaxAddress = (EFI_PHYSICAL_ADDRESS)-1;
+  NeedMap = FALSE;
+  NeedAllocateNonExisting = FALSE;
+
   if ((!RootBridge->DmaAbove4G ||
        (Operation != EfiPciOperationBusMasterRead64 &&
         Operation != EfiPciOperationBusMasterWrite64 &&
         Operation != EfiPciOperationBusMasterCommonBuffer64)) &&
       ((PhysicalAddress + *NumberOfBytes) > SIZE_4GB)) {
+    NeedMap = TRUE;
+    MaxAddress = SIZE_4GB - 1;
+  }
 
+  if (gIoMmuProtocol != NULL) {
+    if ((!InternalIsAlgined (*NumberOfBytes, mIoMmuPageSize)) ||
+        (!InternalIsAlgined ((UINTN)HostAddress, mIoMmuPageSize))) {
+      NeedMap = TRUE;
+    }
+  }
+
+  if (NeedMap) {
     //
     // If the root bridge or the device cannot handle performing DMA above
     // 4GB but any part of the DMA transfer being mapped is above 4GB, then
@@ -1090,9 +1236,17 @@ RootBridgeIoMap (
       //
       // Common Buffer operations can not be remapped.  If the common buffer
       // if above 4GB, then it is not possible to generate a mapping, so return
-      // an error.
+      // an error if there is no IOMMU.
       //
-      return EFI_UNSUPPORTED;
+      if (gIoMmuProtocol == NULL) {
+        return EFI_UNSUPPORTED;
+      } else {
+        //
+        // We can try to allocate non-existing memory for below 4GiB address
+        // and use IOMMU to remap it.
+        //
+        NeedAllocateNonExisting = TRUE;
+      }
     }
 
     //
@@ -1113,21 +1267,81 @@ RootBridgeIoMap (
     MapInfo->NumberOfBytes     = *NumberOfBytes;
     MapInfo->NumberOfPages     = EFI_SIZE_TO_PAGES (MapInfo->NumberOfBytes);
     MapInfo->HostAddress       = PhysicalAddress;
-    MapInfo->MappedHostAddress = SIZE_4GB - 1;
+    MapInfo->MappedHostAddress = MaxAddress;
+    MapInfo->MappedNumberOfBytes = ALIGN_VALUE (MapInfo->NumberOfBytes, mIoMmuPageSize);
+    MapInfo->MappedNumberOfPages = EFI_SIZE_TO_PAGES (MapInfo->MappedNumberOfBytes);
+    MapInfo->RemapNonExisting    = FALSE;
 
-    //
-    // Allocate a buffer below 4GB to map the transfer to.
-    //
-    Status = gBS->AllocatePages (
-                    AllocateMaxAddress,
-                    EfiBootServicesData,
-                    MapInfo->NumberOfPages,
-                    &MapInfo->MappedHostAddress
-                    );
-    if (EFI_ERROR (Status)) {
-      FreePool (MapInfo);
-      *NumberOfBytes = 0;
-      return Status;
+    if (!((Operation == EfiPciOperationBusMasterCommonBuffer) ||
+          (Operation == EfiPciOperationBusMasterCommonBuffer64))) {
+      //
+      // Allocate a buffer below 4GB to map the transfer to.
+      //
+      Status = InternalAllocateAlignedPagesWithAllocateType (
+                      AllocateMaxAddress,
+                      EfiBootServicesData,
+                      MapInfo->MappedNumberOfPages,
+                      mIoMmuPageSize,
+                      &MapInfo->MappedHostAddress
+                      );
+      if (EFI_ERROR (Status)) {
+        if (gIoMmuProtocol == NULL) {
+          FreePool (MapInfo);
+          *NumberOfBytes = 0;
+          return Status;
+        }
+        //
+        // We can try to allocate non-existing memory for below 4GiB address
+        // and use IOMMU to remap it.
+        //
+        NeedAllocateNonExisting = TRUE;
+      }
+    }
+
+    if (NeedAllocateNonExisting) {
+      if ((!InternalIsAlgined (*NumberOfBytes, mIoMmuPageSize)) ||
+          (!InternalIsAlgined ((UINTN)HostAddress, mIoMmuPageSize))) {
+        //
+        // If original memory is not IOMMU aligned, we cannot remap it.
+        //
+        FreePool (MapInfo);
+        *NumberOfBytes = 0;
+        return EFI_UNSUPPORTED;
+      }
+      //
+      // If the code runs to here, it means:
+      // 1) We need remap a common buffer to below 4G non-existing memory.
+      // 2) We need rempa a read/write buffer to below 4G non-existing memory.
+      //
+      MapInfo->MappedHostAddress = MaxAddress;
+      Status = gDS->AllocateMemorySpace (
+                      EfiGcdAllocateMaxAddressSearchTopDown,
+                      EfiGcdMemoryTypeNonExistent,
+                      mIoMmuPageSize,
+                      MapInfo->MappedNumberOfBytes,
+                      &MapInfo->MappedHostAddress,
+                      gImageHandle,
+                      RootBridge->Handle
+                      );
+      if (EFI_ERROR(Status)) {
+        FreePool (MapInfo);
+        *NumberOfBytes = 0;
+        return Status;
+      }
+      Status = gIoMmuProtocol->SetRemapAddress (
+                                 gIoMmuProtocol,
+                                 NULL,
+                                 MapInfo->MappedHostAddress,
+                                 (VOID *)(UINTN)MapInfo->HostAddress,
+                                 MapInfo->MappedNumberOfBytes
+                                 );
+      if (EFI_ERROR(Status)) {
+        gDS->FreeMemorySpace (MapInfo->MappedHostAddress, MapInfo->MappedNumberOfBytes);
+        FreePool (MapInfo);
+        *NumberOfBytes = 0;
+        return Status;
+      }
+      MapInfo->RemapNonExisting = TRUE;
     }
 
     //
@@ -1135,19 +1349,26 @@ RootBridgeIoMap (
     // then copy the contents of the real buffer into the mapped buffer
     // so the Bus Master can read the contents of the real buffer.
     //
-    if (Operation == EfiPciOperationBusMasterRead ||
-        Operation == EfiPciOperationBusMasterRead64) {
-      CopyMem (
-        (VOID *) (UINTN) MapInfo->MappedHostAddress,
-        (VOID *) (UINTN) MapInfo->HostAddress,
-        MapInfo->NumberOfBytes
-        );
+    // This action can be skipped if there is IOMMU, because PciBus does
+    // the action after setting IOMMU attributes.
+    //
+    if (gIoMmuProtocol == NULL) {
+      if (Operation == EfiPciOperationBusMasterRead ||
+          Operation == EfiPciOperationBusMasterRead64) {
+        CopyMem (
+          (VOID *) (UINTN) MapInfo->MappedHostAddress,
+          (VOID *) (UINTN) MapInfo->HostAddress,
+          MapInfo->NumberOfBytes
+          );
+      }
     }
 
     InsertTailList (&RootBridge->Maps, &MapInfo->Link);
 
     //
     // The DeviceAddress is the address of the maped buffer below 4GB
+    // NOTE: It can be a valid system memory address.
+    //       Or a non-existing memory but mapped by IOMMU.
     //
     *DeviceAddress = MapInfo->MappedHostAddress;
     //
@@ -1228,19 +1449,42 @@ RootBridgeIoUnmap (
   // then copy the contents of the mapped buffer into the real buffer
   // so the processor can read the contents of the real buffer.
   //
-  if (MapInfo->Operation == EfiPciOperationBusMasterWrite ||
-      MapInfo->Operation == EfiPciOperationBusMasterWrite64) {
-    CopyMem (
-      (VOID *) (UINTN) MapInfo->HostAddress,
-      (VOID *) (UINTN) MapInfo->MappedHostAddress,
-      MapInfo->NumberOfBytes
-      );
+  // This action can be skipped if there is IOMMU, because PciBus does
+  // the action.
+  //
+  if (gIoMmuProtocol == NULL) {
+    if (MapInfo->Operation == EfiPciOperationBusMasterWrite ||
+        MapInfo->Operation == EfiPciOperationBusMasterWrite64) {
+      CopyMem (
+        (VOID *) (UINTN) MapInfo->HostAddress,
+        (VOID *) (UINTN) MapInfo->MappedHostAddress,
+        MapInfo->NumberOfBytes
+        );
+    }
+  }
+
+  if (gIoMmuProtocol != NULL) {
+    //
+    // Free GCD non existing memory.
+    //
+    if (MapInfo->RemapNonExisting) {
+      gIoMmuProtocol->SetRemapAddress (
+                        gIoMmuProtocol,
+                        NULL,
+                        MapInfo->MappedHostAddress,
+                        (VOID *)(UINTN)MapInfo->MappedHostAddress,
+                        MapInfo->MappedNumberOfBytes
+                        );
+      gDS->FreeMemorySpace (MapInfo->MappedHostAddress, MapInfo->MappedNumberOfBytes);
+      FreePool (Mapping);
+      return EFI_SUCCESS;
+    }
   }
 
   //
   // Free the mapped buffer and the MAP_INFO structure.
   //
-  gBS->FreePages (MapInfo->MappedHostAddress, MapInfo->NumberOfPages);
+  gBS->FreePages (MapInfo->MappedHostAddress, MapInfo->MappedNumberOfPages);
   FreePool (Mapping);
   return EFI_SUCCESS;
 }
@@ -1285,7 +1529,7 @@ RootBridgeIoAllocateBuffer (
   EFI_STATUS                Status;
   EFI_PHYSICAL_ADDRESS      PhysicalAddress;
   PCI_ROOT_BRIDGE_INSTANCE  *RootBridge;
-  EFI_ALLOCATE_TYPE         AllocateType;
+  UINTN                     Size;
 
   //
   // Validate Attributes
@@ -1312,25 +1556,52 @@ RootBridgeIoAllocateBuffer (
 
   RootBridge = ROOT_BRIDGE_FROM_THIS (This);
 
-  AllocateType = AllocateAnyPages;
+  PhysicalAddress = (EFI_PHYSICAL_ADDRESS) (-1);
   if (!RootBridge->DmaAbove4G ||
       (Attributes & EFI_PCI_ATTRIBUTE_DUAL_ADDRESS_CYCLE) == 0) {
     //
     // Limit allocations to memory below 4GB
     //
-    AllocateType    = AllocateMaxAddress;
     PhysicalAddress = (EFI_PHYSICAL_ADDRESS) (SIZE_4GB - 1);
   }
-  Status = gBS->AllocatePages (
-                  AllocateType,
+  if (gIoMmuProtocol != NULL) {
+    Size = EFI_PAGES_TO_SIZE(Pages);
+    Size = ALIGN_VALUE(EFI_PAGES_TO_SIZE(Pages), mIoMmuPageSize);
+    Pages = EFI_SIZE_TO_PAGES (Size);
+  }
+  Status = InternalAllocateAlignedPagesWithAllocateType (
+                  AllocateMaxAddress,
                   MemoryType,
                   Pages,
+                  mIoMmuPageSize,
                   &PhysicalAddress
                   );
   if (!EFI_ERROR (Status)) {
     *HostAddress = (VOID *) (UINTN) PhysicalAddress;
+    return EFI_SUCCESS;
   }
 
+  if (gIoMmuProtocol != NULL) {
+    //
+    // Try to allocate AnyAddress here.
+    //
+    // We can try to allocate non-existing memory for below 4GiB address
+    // and use IOMMU to remap it later.
+    //
+    PhysicalAddress = (EFI_PHYSICAL_ADDRESS) (-1);
+    Status = InternalAllocateAlignedPagesWithAllocateType (
+               AllocateMaxAddress,
+               MemoryType,
+               Pages,
+               mIoMmuPageSize,
+               &PhysicalAddress
+               );
+    if (EFI_ERROR(Status)) {
+      return Status;
+    }
+    *HostAddress = (VOID *) (UINTN) PhysicalAddress;
+    return EFI_SUCCESS;
+  }
   return Status;
 }
 
@@ -1356,6 +1627,13 @@ RootBridgeIoFreeBuffer (
   OUT VOID                             *HostAddress
   )
 {
+  UINTN                     Size;
+
+  if (gIoMmuProtocol != NULL) {
+    Size = EFI_PAGES_TO_SIZE(Pages);
+    Size = ALIGN_VALUE(EFI_PAGES_TO_SIZE(Pages), mIoMmuPageSize);
+    Pages = EFI_SIZE_TO_PAGES (Size);
+  }
   return gBS->FreePages ((EFI_PHYSICAL_ADDRESS) (UINTN) HostAddress, Pages);
 }
 
-- 
2.7.4.windows.1