[PATCH v3 4/5] MdeModulePkg/NonDiscoverablePciDeviceDxe: add support for non-coherent DMA

[Ard Biesheuvel <ard.biesheuvel@linaro.org>]

Add support for non-coherent DMA, either by performing explicit cache
maintenance when DMA mappings are aligned to the CPU's DMA buffer alignment,
or by bounce buffering via uncached mappings otherwise.

Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
 MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.c   |  17 +-
 MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.inf |   2 +
 MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.c    | 319 ++++++++++++++++++++
 MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.h    |  29 ++
 4 files changed, 360 insertions(+), 7 deletions(-)

diff --git a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.c b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.c
index 32da576485aa..8159c98338fb 100644
--- a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.c
+++ b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.c
@@ -16,6 +16,8 @@
 
 #include <Protocol/DriverBinding.h>
 
+EFI_CPU_ARCH_PROTOCOL      *mCpu;
+
 //
 // Probe, start and stop functions of this driver, called by the DXE core for
 // specific devices.
@@ -64,13 +66,9 @@ NonDiscoverablePciDeviceSupported (
   case NonDiscoverableDeviceTypeSdhci:
   case NonDiscoverableDeviceTypeUfs:
   case NonDiscoverableDeviceTypeNvme:
-    //
-    // Restricted to DMA coherent for now
-    //
-    if (Device->DmaType == NonDiscoverableDeviceDmaTypeCoherent) {
-      Status = EFI_SUCCESS;
-      break;
-    }
+    Status = EFI_SUCCESS;
+    break;
+
   default:
     Status = EFI_UNSUPPORTED;
     goto CloseProtocol;
@@ -212,6 +210,11 @@ NonDiscoverablePciDeviceDxeEntryPoint (
   IN EFI_SYSTEM_TABLE *SystemTable
   )
 {
+  EFI_STATUS      Status;
+
+  Status = gBS->LocateProtocol (&gEfiCpuArchProtocolGuid, NULL, (VOID **)&mCpu);
+  ASSERT_EFI_ERROR(Status);
+
   return EfiLibInstallDriverBindingComponentName2 (
            ImageHandle,
            SystemTable,
diff --git a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.inf b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.inf
index 543e622694a0..3ffc634840c7 100644
--- a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.inf
+++ b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.inf
@@ -32,6 +32,7 @@ [Packages]
 [LibraryClasses]
   BaseMemoryLib
   DebugLib
+  DxeServicesTableLib
   MemoryAllocationLib
   UefiBootServicesTableLib
   UefiDriverEntryPoint
@@ -40,3 +41,4 @@ [LibraryClasses]
 [Protocols]
   gEfiPciIoProtocolGuid                         ## BY_START
   gEdkiiNonDiscoverableDeviceProtocolGuid       ## TO_START
+  gEfiCpuArchProtocolGuid                       ## CONSUMES
diff --git a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.c b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.c
index 23466318bfc1..db91d065a330 100644
--- a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.c
+++ b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.c
@@ -15,6 +15,8 @@
 
 #include "NonDiscoverablePciDeviceIo.h"
 
+#include <Library/DxeServicesTableLib.h>
+
 #include <IndustryStandard/Acpi.h>
 
 #include <Protocol/PciRootBridgeIo.h>
@@ -579,6 +581,316 @@ CoherentPciIoFreeBuffer (
   return EFI_SUCCESS;
 }
 
+STATIC
+EFI_STATUS
+NonCoherentPciIoFreeBuffer (
+  IN  EFI_PCI_IO_PROTOCOL         *This,
+  IN  UINTN                       Pages,
+  IN  VOID                        *HostAddress
+  )
+{
+  NON_DISCOVERABLE_PCI_DEVICE                   *Dev;
+  LIST_ENTRY                                    *Entry;
+  EFI_STATUS                                    Status;
+  NON_DISCOVERABLE_DEVICE_UNCACHED_ALLOCATION   *Alloc;
+  BOOLEAN                                       Found;
+
+  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);
+
+  Found = FALSE;
+
+  //
+  // Find the uncached allocation list entry associated
+  // with this allocation
+  //
+  for (Entry = Dev->UncachedAllocationList.ForwardLink;
+       Entry != &Dev->UncachedAllocationList;
+       Entry = Entry->ForwardLink) {
+
+    Alloc = BASE_CR (Entry, NON_DISCOVERABLE_DEVICE_UNCACHED_ALLOCATION, List);
+    if (Alloc->HostAddress == HostAddress && Alloc->NumPages == Pages) {
+      //
+      // We are freeing the exact allocation we were given
+      // before by AllocateBuffer()
+      //
+      Found = TRUE;
+      break;
+    }
+  }
+
+  if (!Found) {
+    ASSERT_EFI_ERROR (EFI_NOT_FOUND);
+    return EFI_NOT_FOUND;
+  }
+
+  RemoveEntryList (&Alloc->List);
+
+  Status = gDS->SetMemorySpaceAttributes (
+                  (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress,
+                  EFI_PAGES_TO_SIZE (Pages),
+                  Alloc->Attributes);
+  if (EFI_ERROR (Status)) {
+    goto FreeAlloc;
+  }
+
+  //
+  // If we fail to restore the original attributes, it is better to leak the
+  // memory than to return it to the heap
+  //
+  FreePages (HostAddress, Pages);
+
+FreeAlloc:
+  FreePool (Alloc);
+  return Status;
+}
+
+STATIC
+EFI_STATUS
+NonCoherentPciIoAllocateBuffer (
+  IN  EFI_PCI_IO_PROTOCOL         *This,
+  IN  EFI_ALLOCATE_TYPE           Type,
+  IN  EFI_MEMORY_TYPE             MemoryType,
+  IN  UINTN                       Pages,
+  OUT VOID                        **HostAddress,
+  IN  UINT64                      Attributes
+  )
+{
+  NON_DISCOVERABLE_PCI_DEVICE                 *Dev;
+  EFI_GCD_MEMORY_SPACE_DESCRIPTOR             GcdDescriptor;
+  EFI_STATUS                                  Status;
+  UINT64                                      MemType;
+  NON_DISCOVERABLE_DEVICE_UNCACHED_ALLOCATION *Alloc;
+
+  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);
+
+  Status = CoherentPciIoAllocateBuffer (This, Type, MemoryType, Pages,
+             HostAddress, Attributes);
+  if (EFI_ERROR (Status)) {
+    return Status;
+  }
+
+  Status = gDS->GetMemorySpaceDescriptor (
+                  (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress,
+                  &GcdDescriptor);
+  if (EFI_ERROR (Status)) {
+    goto FreeBuffer;
+  }
+
+  if ((GcdDescriptor.Capabilities & (EFI_MEMORY_WC | EFI_MEMORY_UC)) == 0) {
+    return EFI_UNSUPPORTED;
+  }
+
+  //
+  // Set the preferred memory attributes
+  //
+  if ((Attributes & EFI_PCI_ATTRIBUTE_MEMORY_WRITE_COMBINE) != 0 ||
+      (GcdDescriptor.Capabilities & EFI_MEMORY_UC) == 0) {
+    //
+    // Use write combining if it was requested, or if it is the only
+    // type supported by the region.
+    //
+    MemType = EFI_MEMORY_WC;
+  } else {
+    MemType = EFI_MEMORY_UC;
+  }
+
+  Alloc = AllocatePool (sizeof *Alloc);
+  if (Alloc == NULL) {
+    goto FreeBuffer;
+  }
+
+  Alloc->HostAddress = *HostAddress;
+  Alloc->NumPages = Pages;
+  Alloc->Attributes = GcdDescriptor.Attributes;
+
+  //
+  // Record this allocation in the linked list, so we
+  // can restore the memory space attributes later
+  //
+  InsertHeadList (&Dev->UncachedAllocationList, &Alloc->List);
+
+  Status = gDS->SetMemorySpaceAttributes (
+                  (EFI_PHYSICAL_ADDRESS)(UINTN)*HostAddress,
+                  EFI_PAGES_TO_SIZE (Pages),
+                  MemType);
+  if (EFI_ERROR (Status)) {
+    goto RemoveList;
+  }
+
+  Status = mCpu->FlushDataCache (
+                   mCpu,
+                   (EFI_PHYSICAL_ADDRESS)(UINTN)*HostAddress,
+                   EFI_PAGES_TO_SIZE (Pages),
+                   EfiCpuFlushTypeInvalidate);
+  if (EFI_ERROR (Status)) {
+    goto RemoveList;
+  }
+
+  return EFI_SUCCESS;
+
+RemoveList:
+  RemoveEntryList (&Alloc->List);
+  FreePool (Alloc);
+
+FreeBuffer:
+  NonCoherentPciIoFreeBuffer (This, Pages, *HostAddress);
+  return Status;
+}
+
+STATIC
+EFI_STATUS
+NonCoherentPciIoMap (
+  IN     EFI_PCI_IO_PROTOCOL            *This,
+  IN     EFI_PCI_IO_PROTOCOL_OPERATION  Operation,
+  IN     VOID                           *HostAddress,
+  IN OUT UINTN                          *NumberOfBytes,
+  OUT    EFI_PHYSICAL_ADDRESS           *DeviceAddress,
+  OUT    VOID                           **Mapping
+  )
+{
+  NON_DISCOVERABLE_PCI_DEVICE           *Dev;
+  EFI_STATUS                            Status;
+  NON_DISCOVERABLE_PCI_DEVICE_MAP_INFO  *MapInfo;
+  UINTN                                 AlignMask;
+  VOID                                  *AllocAddress;
+  EFI_GCD_MEMORY_SPACE_DESCRIPTOR       GcdDescriptor;
+  BOOLEAN                               Bounce;
+
+  MapInfo = AllocatePool (sizeof *MapInfo);
+  if (MapInfo == NULL) {
+    return EFI_OUT_OF_RESOURCES;
+  }
+
+  MapInfo->HostAddress = HostAddress;
+  MapInfo->Operation = Operation;
+  MapInfo->NumberOfBytes = *NumberOfBytes;
+
+  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);
+
+  //
+  // If this device does not support 64-bit DMA addressing, we need to allocate
+  // a bounce buffer and copy over the data in case HostAddress >= 4 GB.
+  //
+  Bounce = ((Dev->Attributes & EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE) == 0 &&
+            (UINTN)HostAddress + *NumberOfBytes > SIZE_4GB);
+
+  if (!Bounce) {
+    switch (Operation) {
+    case EfiPciIoOperationBusMasterRead:
+    case EfiPciIoOperationBusMasterWrite:
+      //
+      // For streaming DMA, it is sufficient if the buffer is aligned to
+      // the CPUs DMA buffer alignment.
+      //
+      AlignMask = mCpu->DmaBufferAlignment - 1;
+      if ((((UINTN) HostAddress | *NumberOfBytes) & AlignMask) == 0) {
+        break;
+      }
+      // fall through
+
+    case EfiPciIoOperationBusMasterCommonBuffer:
+      //
+      // Check whether the host address refers to an uncached mapping.
+      //
+      Status = gDS->GetMemorySpaceDescriptor (
+                      (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress,
+                      &GcdDescriptor);
+      if (EFI_ERROR (Status) ||
+          (GcdDescriptor.Attributes & (EFI_MEMORY_WB|EFI_MEMORY_WT)) != 0) {
+        Bounce = TRUE;
+      }
+      break;
+
+    default:
+      ASSERT (FALSE);
+    }
+  }
+
+  if (Bounce) {
+    if (Operation == EfiPciIoOperationBusMasterCommonBuffer) {
+      Status = EFI_DEVICE_ERROR;
+      goto FreeMapInfo;
+    }
+
+    Status = NonCoherentPciIoAllocateBuffer (This, AllocateAnyPages,
+               EfiBootServicesData, EFI_SIZE_TO_PAGES (MapInfo->NumberOfBytes),
+               &AllocAddress, EFI_PCI_ATTRIBUTE_MEMORY_WRITE_COMBINE);
+    if (EFI_ERROR (Status)) {
+      goto FreeMapInfo;
+    }
+    MapInfo->AllocAddress = (EFI_PHYSICAL_ADDRESS)(UINTN)AllocAddress;
+    if (Operation == EfiPciIoOperationBusMasterRead) {
+      gBS->CopyMem (AllocAddress, HostAddress, *NumberOfBytes);
+    }
+    *DeviceAddress = MapInfo->AllocAddress;
+  } else {
+    MapInfo->AllocAddress = 0;
+    *DeviceAddress = (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress;
+
+    //
+    // We are not using a bounce buffer: the mapping is sufficiently
+    // aligned to allow us to simply flush the caches. Note that cleaning
+    // the caches is necessary for both data directions:
+    // - for bus master read, we want the latest data to be present
+    //   in main memory
+    // - for bus master write, we don't want any stale dirty cachelines that
+    //   may be written back unexpectedly, and clobber the data written to
+    //   main memory by the device.
+    //
+    mCpu->FlushDataCache (mCpu, (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress,
+            *NumberOfBytes, EfiCpuFlushTypeWriteBack);
+  }
+
+  *Mapping = MapInfo;
+  return EFI_SUCCESS;
+
+FreeMapInfo:
+  FreePool (MapInfo);
+
+  return Status;
+}
+
+STATIC
+EFI_STATUS
+NonCoherentPciIoUnmap (
+  IN  EFI_PCI_IO_PROTOCOL          *This,
+  IN  VOID                         *Mapping
+  )
+{
+  NON_DISCOVERABLE_PCI_DEVICE_MAP_INFO  *MapInfo;
+
+  if (Mapping == NULL) {
+    return EFI_DEVICE_ERROR;
+  }
+
+  MapInfo = Mapping;
+  if (MapInfo->AllocAddress != 0) {
+    //
+    // We are using a bounce buffer: copy back the data if necessary,
+    // and free the buffer.
+    //
+    if (MapInfo->Operation == EfiPciIoOperationBusMasterWrite) {
+      gBS->CopyMem (MapInfo->HostAddress, (VOID *)(UINTN)MapInfo->AllocAddress,
+             MapInfo->NumberOfBytes);
+    }
+    NonCoherentPciIoFreeBuffer (This,
+      EFI_SIZE_TO_PAGES (MapInfo->NumberOfBytes),
+      (VOID *)(UINTN)MapInfo->AllocAddress);
+  } else {
+    //
+    // We are *not* using a bounce buffer: if this is a bus master write,
+    // we have to invalidate the caches so the CPU will see the uncached
+    // data written by the device.
+    //
+    if (MapInfo->Operation == EfiPciIoOperationBusMasterWrite) {
+      mCpu->FlushDataCache (mCpu,
+              (EFI_PHYSICAL_ADDRESS)(UINTN)MapInfo->HostAddress,
+              MapInfo->NumberOfBytes, EfiCpuFlushTypeInvalidate);
+    }
+  }
+  FreePool (MapInfo);
+  return EFI_SUCCESS;
+}
 
 STATIC
 EFI_STATUS
@@ -771,6 +1083,13 @@ InitializePciIoProtocol (
   // Copy protocol structure
   CopyMem(&Dev->PciIo, &PciIoTemplate, sizeof PciIoTemplate);
 
+  if (Dev->Device->DmaType == NonDiscoverableDeviceDmaTypeNonCoherent) {
+    Dev->PciIo.AllocateBuffer   = NonCoherentPciIoAllocateBuffer;
+    Dev->PciIo.FreeBuffer       = NonCoherentPciIoFreeBuffer;
+    Dev->PciIo.Map              = NonCoherentPciIoMap;
+    Dev->PciIo.Unmap            = NonCoherentPciIoUnmap;
+  }
+
   switch (Dev->Device->Type) {
   case NonDiscoverableDeviceTypeOhci:
     Dev->ConfigSpace.Hdr.ClassCode[0] = PCI_IF_OHCI;
diff --git a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.h b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.h
index bc0a3d3258f9..449614862911 100644
--- a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.h
+++ b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.h
@@ -15,6 +15,8 @@
 #ifndef __NON_DISCOVERABLE_PCI_DEVICE_IO_H__
 #define __NON_DISCOVERABLE_PCI_DEVICE_IO_H__
 
+#include <PiDxe.h>
+
 #include <Library/BaseMemoryLib.h>
 #include <Library/DebugLib.h>
 #include <Library/MemoryAllocationLib.h>
@@ -25,6 +27,7 @@
 
 #include <Protocol/ComponentName.h>
 #include <Protocol/NonDiscoverableDevice.h>
+#include <Protocol/Cpu.h>
 #include <Protocol/PciIo.h>
 
 #define NON_DISCOVERABLE_PCI_DEVICE_SIG SIGNATURE_32 ('P', 'P', 'I', 'D')
@@ -38,6 +41,27 @@
 
 #define PCI_MAX_BARS                  6
 
+extern EFI_CPU_ARCH_PROTOCOL      *mCpu;
+
+typedef struct {
+  //
+  // The linked-list next pointer
+  //
+  LIST_ENTRY          List;
+  //
+  // The address of the uncached allocation
+  //
+  VOID                *HostAddress;
+  //
+  // The number of pages in the allocation
+  //
+  UINTN               NumPages;
+  //
+  // The attributes of the allocation
+  //
+  UINT64              Attributes;
+} NON_DISCOVERABLE_DEVICE_UNCACHED_ALLOCATION;
+
 typedef struct {
   UINT32                    Signature;
   //
@@ -71,6 +95,11 @@ typedef struct {
   // Whether this device has been enabled
   //
   BOOLEAN                   Enabled;
+  //
+  // Linked list to keep track of uncached allocations performed
+  // on behalf of this device
+  //
+  LIST_ENTRY                UncachedAllocationList;
 } NON_DISCOVERABLE_PCI_DEVICE;
 
 VOID
-- 
2.7.4