[PATCH v3 8/8] MdeModulePkg/PciBusDxe: Handling of pre-populated PCI BARs

["Ojeda Leon, Nicolas" <ncoleon@amazon.com>]

Extend the PCI BAR placement logic in order to consider pre-populated
resources first, if indicated by the Pcd token for such purpose.

The PCI_BAR type is augmented by one field for mapping the absolute
address of prepopulated BARs into a root bridge relative offset.
As part of the CreateResourceMap stage of the PCI resource allocation
process, all the resources belonging to a root bridge are analyzed and
if a BaseAddress (absolute address read from the PCI device during
enumeration) is present, it is translated into a root-bridge relative
offset and the space occupied by it is marked as used in the root
bridge's aperture. This process is performed before the regular
placement logic is executed.

The remaining unassigned BARs are then placed according to default
logic starting after the last pre-populated resource.

The motivation behind respecting pre-populated BARs is that the
hypervisor can decide on placement of some resources and the guest must
use the same values to benefit from performance optimizations or
specific requirements.

Cc: Alexander Graf <graf@amazon.de>
Cc: Gerd Hoffmann <kraxel@redhat.com>

Signed-off-by: Nicolas Ojeda Leon <ncoleon@amazon.com>
---
 MdeModulePkg/Bus/Pci/PciBusDxe/PciBus.h       |   1 +
 .../Bus/Pci/PciBusDxe/PciEnumeratorSupport.c  |   5 +-
 MdeModulePkg/Bus/Pci/PciBusDxe/PciLib.c       |  11 +
 .../Bus/Pci/PciBusDxe/PciResourceSupport.c    | 277 +++++++++++++++++-
 .../Bus/Pci/PciBusDxe/PciResourceSupport.h    |  20 ++
 5 files changed, 310 insertions(+), 4 deletions(-)

diff --git a/MdeModulePkg/Bus/Pci/PciBusDxe/PciBus.h b/MdeModulePkg/Bus/Pci/PciBusDxe/PciBus.h
index 4b58c3ea9b..9218566bf6 100644
--- a/MdeModulePkg/Bus/Pci/PciBusDxe/PciBus.h
+++ b/MdeModulePkg/Bus/Pci/PciBusDxe/PciBus.h
@@ -94,6 +94,7 @@ typedef enum {
 //
 struct _PCI_BAR {
   UINT64          BaseAddress;
+  UINT64          BaseAddressOffset;
   UINT64          Length;
   UINT64          Alignment;
   PCI_BAR_TYPE    BarType;
diff --git a/MdeModulePkg/Bus/Pci/PciBusDxe/PciEnumeratorSupport.c b/MdeModulePkg/Bus/Pci/PciBusDxe/PciEnumeratorSupport.c
index 9251388bc2..91caf162e0 100644
--- a/MdeModulePkg/Bus/Pci/PciBusDxe/PciEnumeratorSupport.c
+++ b/MdeModulePkg/Bus/Pci/PciBusDxe/PciEnumeratorSupport.c
@@ -1776,8 +1776,9 @@ PciParseBar (
     return Offset + 4;
   }
 
-  PciIoDevice->PciBar[BarIndex].BarTypeFixed = FALSE;
-  PciIoDevice->PciBar[BarIndex].Offset       = (UINT8)Offset;
+  PciIoDevice->PciBar[BarIndex].BarTypeFixed      = FALSE;
+  PciIoDevice->PciBar[BarIndex].Offset            = (UINT8)Offset;
+  PciIoDevice->PciBar[BarIndex].BaseAddressOffset = 0;
   if ((Value & 0x01) != 0) {
     //
     // Device I/Os
diff --git a/MdeModulePkg/Bus/Pci/PciBusDxe/PciLib.c b/MdeModulePkg/Bus/Pci/PciBusDxe/PciLib.c
index 2ffbc08256..452ca10540 100644
--- a/MdeModulePkg/Bus/Pci/PciBusDxe/PciLib.c
+++ b/MdeModulePkg/Bus/Pci/PciBusDxe/PciLib.c
@@ -605,6 +605,17 @@ PciHostBridgeResourceAllocator (
         GetResourceFromDevice (RootBridgeDev, IoBridge, Mem32Bridge, PMem32Bridge, Mem64Bridge, PMem64Bridge);
       }
 
+      //
+      // Handle pre-populated resources
+      //
+      PopulateResourceAperture (
+        Mem32Base,
+        PMem32Base,
+        Mem64Base,
+        PMem64Base,
+        RootBridgeDev
+        );
+
       //
       // Create resourcemap by going through all the devices subject to this root bridge
       //
diff --git a/MdeModulePkg/Bus/Pci/PciBusDxe/PciResourceSupport.c b/MdeModulePkg/Bus/Pci/PciBusDxe/PciResourceSupport.c
index 8ffd05f327..e2a550bf8e 100644
--- a/MdeModulePkg/Bus/Pci/PciBusDxe/PciResourceSupport.c
+++ b/MdeModulePkg/Bus/Pci/PciBusDxe/PciResourceSupport.c
@@ -324,6 +324,266 @@ CalculateApertureIo16 (
   Bridge->Length = MAX (Bridge->Length, PaddingAperture);
 }
 
+/**
+  Analyze a single BAR and if it is pre-populated (its baseAddress is already
+  set), calculate the offset inside the root bridge.
+
+  @param Bar    PCI BAR to be placed if pre-populated
+  @param Base   Base address of the root bridge
+
+**/
+STATIC
+VOID
+PrePopulateBAR (
+  IN OUT PCI_BAR  *Bar,
+  IN     UINT64   Base
+  )
+{
+  if (Base == gAllOne) {
+    return;
+  }
+
+  //
+  // Pre-populated IO space BARs are not supported
+  //
+  if ((Bar->BarType == PciBarTypeIo16) || (Bar->BarType == PciBarTypeIo32)) {
+    return;
+  }
+
+  if (Bar->BaseAddress < Base) {
+    return;
+  }
+
+  if (Bar->BaseAddress != 0) {
+    Bar->BaseAddressOffset = Bar->BaseAddress - Base;
+  }
+}
+
+/**
+  Calculate and set the address offset inside the root bridge of device
+  BARs that are pre-populated.
+
+  @param Dev      PCI device whose BARs are to be analyzed
+  @param Base     Base address of the root bridge under which the device
+                  is located.
+
+ */
+STATIC
+VOID
+PopulateDeviceBars (
+  IN UINT64         Mem32Base,
+  IN UINT64         PMem32Base,
+  IN UINT64         Mem64Base,
+  IN UINT64         PMem64Base,
+  IN PCI_IO_DEVICE  *Dev
+  )
+{
+  UINTN  Index;
+
+  for (Index = 0; Index < PCI_MAX_BAR; Index++) {
+    switch (Dev->PciBar[Index].BarType) {
+      case PciBarTypeMem32:
+        PrePopulateBAR (&Dev->PciBar[Index], Mem32Base);
+        break;
+
+      case PciBarTypePMem32:
+        PrePopulateBAR (&Dev->PciBar[Index], PMem32Base);
+        break;
+
+      case PciBarTypeMem64:
+        PrePopulateBAR (&Dev->PciBar[Index], Mem64Base);
+        break;
+
+      case PciBarTypePMem64:
+        PrePopulateBAR (&Dev->PciBar[Index], PMem64Base);
+        break;
+
+      default:
+        break;
+    }
+  }
+}
+
+/**
+  Considering the Bridge attributes specifying the supported resources,
+  merge the base addresses to fake the non-suported types belonging to
+  the alternative type.
+
+  @param Mem32Base      32-bit base address
+  @param PMem32Base     32-bit prefetchable base address
+  @param Mem64Base      64-bit base address
+  @param PMem64Base     64-bit prefetchable base address
+  @param Bridge         PCI resource node for given bridge device.
+
+ */
+STATIC
+VOID
+MergeBridgeResourceBases (
+  IN OUT UINT64         *Mem32Base,
+  IN OUT UINT64         *PMem32Base,
+  IN OUT UINT64         *Mem64Base,
+  IN OUT UINT64         *PMem64Base,
+  IN     PCI_IO_DEVICE  *Bridge
+  )
+{
+  //
+  // if root bridge supports combined Pmem Mem decoding
+  // merge these two type of resource
+  //
+  if (BridgeSupportResourceDecode (Bridge, EFI_BRIDGE_PMEM_MEM_COMBINE_SUPPORTED)) {
+    *PMem32Base = *Mem32Base;
+    *PMem64Base = *Mem64Base;
+  }
+
+  //
+  // If bridge doesn't support Pmem32
+  // degrade it to mem32
+  //
+  if (!BridgeSupportResourceDecode (Bridge, EFI_BRIDGE_PMEM32_DECODE_SUPPORTED)) {
+    *PMem32Base = *Mem32Base;
+  }
+
+  //
+  // If firmware is in 32-bit mode,
+  // then degrade PMEM64/MEM64 requests
+  //
+  if (sizeof (UINTN) <= 4) {
+    *Mem64Base  = *Mem32Base;
+    *PMem64Base = *PMem32Base;
+  } else {
+    //
+    // if the bridge does not support MEM64, degrade MEM64 to MEM32
+    //
+    if (!BridgeSupportResourceDecode (Bridge, EFI_BRIDGE_MEM64_DECODE_SUPPORTED)) {
+      *Mem64Base = *Mem32Base;
+    }
+
+    //
+    // if the bridge does not support PMEM64, degrade PMEM64 to PMEM32
+    //
+    if (!BridgeSupportResourceDecode (Bridge, EFI_BRIDGE_PMEM64_DECODE_SUPPORTED)) {
+      *PMem64Base = *PMem32Base;
+    }
+  }
+}
+
+/**
+  Populate a root bridge's resource aperture according to
+  initial conditions ruled by individual pre-populated
+  resources
+
+   @param Bridge    PCI resource node for given bridge device.
+   @param Base      Resource aperture base address
+
+**/
+VOID
+PopulateResourceAperture (
+  IN     UINT64         Mem32Base,
+  IN     UINT64         PMem32Base,
+  IN     UINT64         Mem64Base,
+  IN     UINT64         PMem64Base,
+  IN OUT PCI_IO_DEVICE  *Bridge
+  )
+{
+  PCI_IO_DEVICE  *Temp;
+  LIST_ENTRY     *CurrentLink;
+
+  //
+  // Verify if pre-populated BARs need to be respected, otherwise
+  // there is no need to pre-populate any resource
+  //
+  if (!PcdGetBool (PcdPciPreservePopulatedMappings)) {
+    return;
+  }
+
+  CurrentLink = Bridge->ChildList.ForwardLink;
+
+  //
+  // Merge base addresses of the different types depending on resource
+  // decoding supported by the bridge
+  //
+  MergeBridgeResourceBases (
+    &Mem32Base,
+    &PMem32Base,
+    &Mem64Base,
+    &PMem64Base,
+    Bridge
+    );
+
+  while (CurrentLink != NULL && CurrentLink != &Bridge->ChildList) {
+    Temp = PCI_IO_DEVICE_FROM_LINK (CurrentLink);
+
+    PopulateDeviceBars (
+      Mem32Base,
+      PMem32Base,
+      Mem64Base,
+      PMem64Base,
+      Temp
+      );
+
+    //
+    // Recursive call to analyze the hierarchical tree under a bridge
+    //
+    if (IS_PCI_BRIDGE (&Temp->Pci)) {
+      PopulateResourceAperture (
+        Mem32Base,
+        PMem32Base,
+        Mem64Base,
+        PMem64Base,
+        Temp
+        );
+    }
+
+    CurrentLink = CurrentLink->ForwardLink;
+  }
+}
+
+/**
+  Calculate the current used resource of the bridge provided that some
+  of the resources under it might been pre-populated.
+
+  @param Bridge     PCI resource node for given bridge device.
+
+**/
+STATIC
+UINT64
+GetBridgePopulatedAperture (
+  IN PCI_RESOURCE_NODE  *Bridge
+  )
+{
+  UINT64             Aperture;
+  UINT64             EndAddress;
+  LIST_ENTRY         *CurrentLink;
+  PCI_RESOURCE_NODE  *Node;
+  PCI_BAR            *Bar;
+
+  Aperture = 0;
+
+  //
+  // Analyze all resource nodes looking for the prepopulated with the
+  // highest address used.
+  //
+  for ( CurrentLink = GetFirstNode (&Bridge->ChildList)
+        ; !IsNull (&Bridge->ChildList, CurrentLink)
+        ; CurrentLink = GetNextNode (&Bridge->ChildList, CurrentLink)
+        )
+  {
+    Node = RESOURCE_NODE_FROM_LINK (CurrentLink);
+
+    Bar = &Node->PciDev->PciBar[Node->Bar];
+
+    if (Bar->BaseAddress != 0) {
+      EndAddress = Bar->BaseAddressOffset + Bar->Length;
+
+      if (EndAddress > Aperture) {
+        Aperture = EndAddress;
+      }
+    }
+  }
+
+  return Aperture;
+}
+
 /**
   This function is used to calculate the resource aperture
   for a given bridge device.
@@ -337,6 +597,7 @@ CalculateResourceAperture (
   )
 {
   UINT64             Aperture[2];
+  UINT64             InitialAperture;
   LIST_ENTRY         *CurrentLink;
   PCI_RESOURCE_NODE  *Node;
 
@@ -349,8 +610,14 @@ CalculateResourceAperture (
     return;
   }
 
-  Aperture[PciResUsageTypical] = 0;
-  Aperture[PciResUsagePadding] = 0;
+  //
+  // Initialize apertures at bridge's current resource usage
+  // which might be occupied by pre-populated resources.
+  //
+  InitialAperture              = GetBridgePopulatedAperture (Bridge);
+  Aperture[PciResUsageTypical] = InitialAperture;
+  Aperture[PciResUsagePadding] = InitialAperture;
+
   //
   // Assume the bridge is aligned
   //
@@ -361,6 +628,12 @@ CalculateResourceAperture (
   {
     Node = RESOURCE_NODE_FROM_LINK (CurrentLink);
 
+    PCI_BAR  *Bar = &Node->PciDev->PciBar[Node->Bar];
+    if (PcdGetBool (PcdPciPreservePopulatedMappings) && Bar->BaseAddress) {
+      Node->Offset = Bar->BaseAddressOffset;
+      continue;
+    }
+
     //
     // It's possible for a bridge to contain multiple padding resource
     // nodes due to DegradeResource().
diff --git a/MdeModulePkg/Bus/Pci/PciBusDxe/PciResourceSupport.h b/MdeModulePkg/Bus/Pci/PciBusDxe/PciResourceSupport.h
index 1527d4eafa..c5ab5f3fd9 100644
--- a/MdeModulePkg/Bus/Pci/PciBusDxe/PciResourceSupport.h
+++ b/MdeModulePkg/Bus/Pci/PciBusDxe/PciResourceSupport.h
@@ -111,6 +111,26 @@ CalculateApertureIo16 (
   IN PCI_RESOURCE_NODE  *Bridge
   );
 
+/**
+  Populate a root bridge's resource aperture according to
+  initial conditions ruled by individual pre-populated
+  resources
+
+   @note Pre-populated PIO BARs are not supported.
+
+   @param Bridge    PCI resource node for given bridge device.
+   @param Base      Resource aperture base address
+
+**/
+VOID
+PopulateResourceAperture (
+  IN     UINT64         Mem32Base,
+  IN     UINT64         PMem32Base,
+  IN     UINT64         Mem64Base,
+  IN     UINT64         PMem64Base,
+  IN OUT PCI_IO_DEVICE  *Bridge
+  );
+
 /**
   This function is used to calculate the resource aperture
   for a given bridge device.
-- 
2.17.1




Amazon Development Center Germany GmbH
Krausenstr. 38
10117 Berlin
Geschaeftsfuehrung: Christian Schlaeger, Jonathan Weiss
Eingetragen am Amtsgericht Charlottenburg unter HRB 149173 B
Sitz: Berlin
Ust-ID: DE 289 237 879