[PATCH v2 4/5] MdeModulePkg/NonDiscoverablePciDeviceDxe: add support for non-coherent DMA

[Ard Biesheuvel <ard.biesheuvel@linaro.org>]

Add support for non-coherent DMA, either by performing explicit cache
maintenance when DMA mappings are aligned to the CPU's DMA buffer alignment,
or by bounce buffering via uncached mappings otherwise.

Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
 MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.c   |  17 +-
 MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.inf |   2 +
 MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.c    | 254 ++++++++++++++++++++
 MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.h    |   5 +
 4 files changed, 271 insertions(+), 7 deletions(-)

diff --git a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.c b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.c
index c7f1c42208d7..16cc3fb7e242 100644
--- a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.c
+++ b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.c
@@ -16,6 +16,8 @@
 
 #include <Protocol/DriverBinding.h>
 
+EFI_CPU_ARCH_PROTOCOL      *mCpu;
+
 //
 // Probe, start and stop functions of this driver, called by the DXE core for
 // specific devices.
@@ -62,13 +64,9 @@ NonDiscoverablePciDeviceSupported (
   case NonDiscoverableDeviceTypeSdhci:
   case NonDiscoverableDeviceTypeUfs:
   case NonDiscoverableDeviceTypeNvme:
-    //
-    // Restricted to DMA coherent for now
-    //
-    if (Device->DmaType == NonDiscoverableDeviceDmaTypeCoherent) {
-      Status = EFI_SUCCESS;
-      break;
-    }
+    Status = EFI_SUCCESS;
+    break;
+
   default:
     Status = EFI_UNSUPPORTED;
   }
@@ -194,6 +192,11 @@ NonDiscoverablePciDeviceDxeEntryPoint (
   IN EFI_SYSTEM_TABLE *SystemTable
   )
 {
+  EFI_STATUS      Status;
+
+  Status = gBS->LocateProtocol (&gEfiCpuArchProtocolGuid, NULL, (VOID **)&mCpu);
+  ASSERT_EFI_ERROR(Status);
+
   return EfiLibInstallDriverBindingComponentName2 (
            ImageHandle,
            SystemTable,
diff --git a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.inf b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.inf
index da1e986b6e9e..e28562533ae8 100644
--- a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.inf
+++ b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.inf
@@ -32,11 +32,13 @@ [Packages]
 [LibraryClasses]
   BaseMemoryLib
   DebugLib
+  DxeServicesTableLib
   MemoryAllocationLib
   UefiBootServicesTableLib
   UefiDriverEntryPoint
   UefiLib
 
 [Protocols]
+  gEfiCpuArchProtocolGuid                 ## CONSUMES
   gEfiPciIoProtocolGuid                   ## BY_START
   gNonDiscoverableDeviceProtocolGuid      ## TO_START
diff --git a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.c b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.c
index 1269194a9eca..d7d687a853bb 100644
--- a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.c
+++ b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.c
@@ -15,6 +15,8 @@
 
 #include "NonDiscoverablePciDeviceIo.h"
 
+#include <Library/DxeServicesTableLib.h>
+
 #include <IndustryStandard/Acpi.h>
 
 #include <Protocol/PciRootBridgeIo.h>
@@ -475,6 +477,251 @@ CoherentPciIoFreeBuffer (
   return EFI_SUCCESS;
 }
 
+STATIC
+EFI_STATUS
+NonCoherentPciIoFreeBuffer (
+  IN  EFI_PCI_IO_PROTOCOL         *This,
+  IN  UINTN                       Pages,
+  IN  VOID                        *HostAddress
+  )
+{
+  EFI_STATUS        Status;
+
+  Status = gDS->SetMemorySpaceAttributes (
+                  (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress,
+                  EFI_PAGES_TO_SIZE (Pages),
+                  EFI_MEMORY_WB);
+  if (EFI_ERROR (Status)) {
+    return Status;
+  }
+
+  FreePages (HostAddress, Pages);
+  return EFI_SUCCESS;
+}
+
+STATIC
+EFI_STATUS
+NonCoherentPciIoAllocateBuffer (
+  IN  EFI_PCI_IO_PROTOCOL         *This,
+  IN  EFI_ALLOCATE_TYPE           Type,
+  IN  EFI_MEMORY_TYPE             MemoryType,
+  IN  UINTN                       Pages,
+  OUT VOID                        **HostAddress,
+  IN  UINT64                      Attributes
+  )
+{
+  EFI_GCD_MEMORY_SPACE_DESCRIPTOR       GcdDescriptor;
+  EFI_STATUS                            Status;
+  UINT64                                MemType;
+
+  Status = CoherentPciIoAllocateBuffer (This, Type, MemoryType, Pages,
+             HostAddress, Attributes);
+  if (EFI_ERROR (Status)) {
+    return Status;
+  }
+
+  Status = gDS->GetMemorySpaceDescriptor (
+                  (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress,
+                  &GcdDescriptor);
+  if (EFI_ERROR (Status)) {
+    goto FreeBuffer;
+  }
+
+  if ((GcdDescriptor.Capabilities & (EFI_MEMORY_WC | EFI_MEMORY_UC)) == 0) {
+    return EFI_UNSUPPORTED;
+  }
+
+  //
+  // Set the preferred memory attributes
+  //
+  if ((Attributes & EFI_PCI_ATTRIBUTE_MEMORY_WRITE_COMBINE) != 0 ||
+      (GcdDescriptor.Capabilities & EFI_MEMORY_UC) == 0) {
+    //
+    // Use write combining if it was requested, or if it is the only
+    // type supported by the region.
+    //
+    MemType = EFI_MEMORY_WC;
+  } else {
+    MemType = EFI_MEMORY_UC;
+  }
+
+  Status = gDS->SetMemorySpaceAttributes (
+                  (EFI_PHYSICAL_ADDRESS)(UINTN)*HostAddress,
+                  EFI_PAGES_TO_SIZE (Pages),
+                  MemType);
+  if (EFI_ERROR (Status)) {
+    goto FreeBuffer;
+  }
+
+  Status = mCpu->FlushDataCache (
+                   mCpu,
+                   (EFI_PHYSICAL_ADDRESS)(UINTN)*HostAddress,
+                   EFI_PAGES_TO_SIZE (Pages),
+                   EfiCpuFlushTypeInvalidate);
+  if (EFI_ERROR (Status)) {
+    goto FreeBuffer;
+  }
+  return EFI_SUCCESS;
+
+FreeBuffer:
+  NonCoherentPciIoFreeBuffer (This, Pages, *HostAddress);
+  return Status;
+}
+
+STATIC
+EFI_STATUS
+NonCoherentPciIoMap (
+  IN     EFI_PCI_IO_PROTOCOL            *This,
+  IN     EFI_PCI_IO_PROTOCOL_OPERATION  Operation,
+  IN     VOID                           *HostAddress,
+  IN OUT UINTN                          *NumberOfBytes,
+  OUT    EFI_PHYSICAL_ADDRESS           *DeviceAddress,
+  OUT    VOID                           **Mapping
+  )
+{
+  NON_DISCOVERABLE_PCI_DEVICE           *Dev;
+  EFI_STATUS                            Status;
+  NON_DISCOVERABLE_PCI_DEVICE_MAP_INFO  *MapInfo;
+  UINTN                                 AlignMask;
+  VOID                                  *AllocAddress;
+  EFI_GCD_MEMORY_SPACE_DESCRIPTOR       GcdDescriptor;
+  BOOLEAN                               Bounce;
+
+  MapInfo = AllocatePool (sizeof *MapInfo);
+  if (MapInfo == NULL) {
+    return EFI_OUT_OF_RESOURCES;
+  }
+
+  MapInfo->HostAddress = HostAddress;
+  MapInfo->Operation = Operation;
+  MapInfo->NumberOfBytes = *NumberOfBytes;
+
+  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);
+
+  //
+  // If this device does not support 64-bit DMA addressing, we need to allocate
+  // a bounce buffer and copy over the data in case HostAddress >= 4 GB.
+  //
+  Bounce = ((Dev->Attributes & EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE) == 0 &&
+            (UINTN) HostAddress >= SIZE_4GB);
+
+  if (!Bounce) {
+    switch (Operation) {
+    case EfiPciIoOperationBusMasterRead:
+    case EfiPciIoOperationBusMasterWrite:
+      //
+      // For streaming DMA, it is sufficient if the buffer is aligned to
+      // the CPUs DMA buffer alignment.
+      //
+      AlignMask = mCpu->DmaBufferAlignment - 1;
+      if ((((UINTN) HostAddress | *NumberOfBytes) & AlignMask) == 0) {
+        break;
+      }
+      // fall through
+
+    case EfiPciIoOperationBusMasterCommonBuffer:
+      //
+      // Check whether the host address refers to an uncached mapping.
+      //
+      Status = gDS->GetMemorySpaceDescriptor (
+                      (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress,
+                      &GcdDescriptor);
+      if (EFI_ERROR (Status) ||
+          (GcdDescriptor.Attributes & (EFI_MEMORY_WB|EFI_MEMORY_WT)) != 0) {
+        Bounce = TRUE;
+      }
+      break;
+
+    default:
+      ASSERT (FALSE);
+    }
+  }
+
+  if (Bounce) {
+    if (Operation == EfiPciIoOperationBusMasterCommonBuffer) {
+      Status = EFI_DEVICE_ERROR;
+      goto FreeMapInfo;
+    }
+
+    Status = NonCoherentPciIoAllocateBuffer (This, AllocateAnyPages,
+               EfiBootServicesData, EFI_SIZE_TO_PAGES (MapInfo->NumberOfBytes),
+               &AllocAddress, EFI_PCI_ATTRIBUTE_MEMORY_WRITE_COMBINE);
+    if (EFI_ERROR (Status)) {
+      goto FreeMapInfo;
+    }
+    MapInfo->AllocAddress = (EFI_PHYSICAL_ADDRESS)(UINTN)AllocAddress;
+    if (Operation == EfiPciIoOperationBusMasterRead) {
+      gBS->CopyMem (AllocAddress, HostAddress, *NumberOfBytes);
+    }
+    *DeviceAddress = MapInfo->AllocAddress;
+  } else {
+    MapInfo->AllocAddress = 0;
+    *DeviceAddress = (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress;
+
+    //
+    // We are not using a bounce buffer: the mapping is sufficiently
+    // aligned to allow us to simply flush the caches. Note that cleaning
+    // the caches is necessary for both data directions:
+    // - for bus master read, we want the latest data to be present
+    //   in main memory
+    // - for bus master write, we don't want any stale dirty cachelines that
+    //   may be written back unexpectedly, and clobber the data written to
+    //   main memory by the device.
+    //
+    mCpu->FlushDataCache (mCpu, (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress,
+            *NumberOfBytes, EfiCpuFlushTypeWriteBack);
+  }
+
+  *Mapping = MapInfo;
+  return EFI_SUCCESS;
+
+FreeMapInfo:
+  FreePool (MapInfo);
+
+  return Status;
+}
+
+STATIC
+EFI_STATUS
+NonCoherentPciIoUnmap (
+  IN  EFI_PCI_IO_PROTOCOL          *This,
+  IN  VOID                         *Mapping
+  )
+{
+  NON_DISCOVERABLE_PCI_DEVICE_MAP_INFO  *MapInfo;
+
+  if (Mapping == NULL) {
+    return EFI_DEVICE_ERROR;
+  }
+
+  MapInfo = Mapping;
+  if (MapInfo->AllocAddress != 0) {
+    //
+    // We are using a bounce buffer: copy back the data if necessary,
+    // and free the buffer.
+    //
+    if (MapInfo->Operation == EfiPciIoOperationBusMasterWrite) {
+      gBS->CopyMem (MapInfo->HostAddress, (VOID *)(UINTN)MapInfo->AllocAddress,
+             MapInfo->NumberOfBytes);
+    }
+    NonCoherentPciIoFreeBuffer (This,
+      EFI_SIZE_TO_PAGES (MapInfo->NumberOfBytes),
+      (VOID *)(UINTN)MapInfo->AllocAddress);
+  } else {
+    //
+    // We are *not* using a bounce buffer: if this is a bus master write,
+    // we have to invalidate the caches so the CPU will see the uncached
+    // data written by the device.
+    //
+    if (MapInfo->Operation == EfiPciIoOperationBusMasterWrite) {
+      mCpu->FlushDataCache (mCpu,
+              (EFI_PHYSICAL_ADDRESS)(UINTN)MapInfo->HostAddress,
+              MapInfo->NumberOfBytes, EfiCpuFlushTypeInvalidate);
+    }
+  }
+  FreePool (MapInfo);
+  return EFI_SUCCESS;
+}
 
 STATIC
 EFI_STATUS
@@ -737,4 +984,11 @@ InitializePciIoProtocol (
 
   // Copy protocol structure
   CopyMem(&Dev->PciIo, &PciIoTemplate, sizeof PciIoTemplate);
+
+  if (Dev->Device->DmaType == NonDiscoverableDeviceDmaTypeNonCoherent) {
+    Dev->PciIo.AllocateBuffer = NonCoherentPciIoAllocateBuffer;
+    Dev->PciIo.FreeBuffer = NonCoherentPciIoFreeBuffer;
+    Dev->PciIo.Map = NonCoherentPciIoMap;
+    Dev->PciIo.Unmap = NonCoherentPciIoUnmap;
+  }
 }
diff --git a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.h b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.h
index 5c6086fe6c6b..e7ab60338141 100644
--- a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.h
+++ b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.h
@@ -15,6 +15,8 @@
 #ifndef __NON_DISCOVERABLE_PCI_DEVICE_IO_H__
 #define __NON_DISCOVERABLE_PCI_DEVICE_IO_H__
 
+#include <PiDxe.h>
+
 #include <Library/BaseMemoryLib.h>
 #include <Library/DebugLib.h>
 #include <Library/MemoryAllocationLib.h>
@@ -25,6 +27,7 @@
 
 #include <Protocol/ComponentName.h>
 #include <Protocol/NonDiscoverableDevice.h>
+#include <Protocol/Cpu.h>
 #include <Protocol/PciIo.h>
 
 #define NON_DISCOVERABLE_PCI_DEVICE_SIG SIGNATURE_32 ('P', 'P', 'I', 'D')
@@ -33,6 +36,8 @@
         CR (PciIoPointer, NON_DISCOVERABLE_PCI_DEVICE, PciIo, \
             NON_DISCOVERABLE_PCI_DEVICE_SIG)
 
+extern EFI_CPU_ARCH_PROTOCOL      *mCpu;
+
 typedef struct {
   UINT32                    Signature;
   //
-- 
2.7.4