[edk2-staging/UEFI_PCI_ENHANCE-2 PATCH 05/12] PciBusDxe: Setup sub-phases for PCI feature enumeration

["Javeed, Ashraf" <ashraf.javeed@intel.com>]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2194

The code changes are made to setup the following internal sub-phases for
enumerating the PCI features in the late phase of the PCI Bus driver.
(1) PciFeatureRootBridgeScan - initial phase in configuring the other PCI
    features to record the primary root ports
(2) PciFeatureGetDevicePolicy - get the PCI device-specific platform pol-
    icies and align with device capabilities
(3) PciFeatureSetupPhase - align all PCI nodes in the PCI heirarchical
    tree (if required for that PCI feature)
(4) PciFeatureConfigurationPhase - finally override to complete configu-
    ration of the PCI feature

The code changes are made to support the configuration of other PCIe
features, like MPS, which require a common value to be assigned among
all the child PCI devices and its parent root port device.

Signed-off-by: Ashraf Javeed <ashraf.javeed@intel.com>
Cc: Jian J Wang <jian.j.wang@intel.com>
Cc: Hao A Wu <hao.a.wu@intel.com>
Cc: Ray Ni <ray.ni@intel.com>
---
 MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.c | 859 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.h | 146 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 1005 insertions(+)

diff --git a/MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.c b/MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.c
index 8be227a..ab0e096 100644
--- a/MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.c
+++ b/MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.c
@@ -9,6 +9,23 @@ SPDX-License-Identifier: BSD-2-Clause-Patent
 #include "PciBus.h"
 #include "PciFeatureSupport.h"
 
+/**
+  A gobal pointer to PRIMARY_ROOT_PORT_NODE buffer to track all the primary physical
+  PCI Root Ports (PCI Controllers) for a given PCI Root Bridge instance while
+  enumerating to configure the PCI features
+**/
+PRIMARY_ROOT_PORT_NODE                      *mPrimaryRootPortList;
+
+/**
+  A global pointer to PCI_FEATURE_CONFIGURATION_COMPLETION_LIST, which stores all
+  the PCI Root Bridge instances that are enumerated for the other PCI features,
+  like MaxPayloadSize & MaxReadReqSize; during the the Start() interface of the
+  driver binding protocol. The records pointed by this pointer would be destroyed
+  when the DXE core invokes the Stop() interface.
+**/
+PCI_FEATURE_CONFIGURATION_COMPLETION_LIST   *mPciFeaturesConfigurationCompletionList = NULL;
+
+
 /**
   Main routine to indicate whether the platform has selected the Max_Payload_Size
   PCI feature to be configured by this driver
@@ -175,3 +192,845 @@ SetupPtm (
 {
   return (PcdGet32 (PcdOtherPciFeatures) & PCI_FEATURE_SUPPORT_FLAG_PTM) ? TRUE : FALSE;
 }
+
+/**
+ Helper routine to determine the existence of previously enumerated PCI device
+
+ @retval  TRUE  PCI device exist
+          FALSE does not exist
+**/
+BOOLEAN
+DeviceExist (
+  PCI_IO_DEVICE                   *PciDevice
+  )
+{
+  EFI_PCI_IO_PROTOCOL   *PciIoProtocol = &PciDevice->PciIo;
+  UINT16                VendorId = 0xFFFF;
+
+  PciIoProtocol->Pci.Read (
+                      PciIoProtocol,
+                      EfiPciIoWidthUint16,
+                      PCI_VENDOR_ID_OFFSET,
+                      1,
+                      &VendorId
+                      );
+  if (VendorId == 0 || VendorId == 0xFFFF) {
+    return FALSE;
+  } else {
+    return TRUE;
+  }
+}
+
+/**
+  Helper routine which determines whether the given PCI Root Bridge instance
+  record already exist. This routine shall help avoid duplicate record creation
+  in case of re-enumeration of PCI configuation features.
+
+  @param  RootBridge              A pointer to the PCI_IO_DEVICE for the Root Bridge
+  @param  PciFeatureConfigRecord  A pointer to a pointer for type
+                                  PCI_FEATURE_CONFIGURATION_COMPLETION_LIST
+                                  record, Use to return the specific record.
+
+  @retval TRUE                    Record already exist
+          FALSE                   Record does not exist for the given PCI Root Bridge
+**/
+BOOLEAN
+CheckPciFeatureConfigurationRecordExist (
+  IN  PCI_IO_DEVICE                             *RootBridge,
+  OUT PCI_FEATURE_CONFIGURATION_COMPLETION_LIST **PciFeatureConfigRecord
+  )
+{
+  LIST_ENTRY                                  *Link;
+  PCI_FEATURE_CONFIGURATION_COMPLETION_LIST   *Temp;
+
+  if (mPciFeaturesConfigurationCompletionList) {
+    Link = &mPciFeaturesConfigurationCompletionList->RootBridgeLink;
+
+    do {
+      Temp = PCI_FEATURE_CONFIGURATION_COMPLETION_LIST_FROM_LINK (Link);
+      if (Temp->RootBridgeHandle == RootBridge->Handle) {
+        *PciFeatureConfigRecord = Temp;
+        return TRUE;
+      }
+      Link = Link->ForwardLink;
+    } while (Link != &mPciFeaturesConfigurationCompletionList->RootBridgeLink);
+  }
+  //
+  // not found on the PCI feature configuration completion list
+  //
+  *PciFeatureConfigRecord = NULL;
+  return FALSE;
+}
+
+/**
+  This routine is primarily to avoid multiple configuration of PCI features
+  to the same PCI Root Bridge due to EDK2 core's ConnectController calls on
+  all the EFI handles. This routine also provide re-enumeration of the PCI
+  features on the same PCI Root Bridge based on the policy of ReEnumeratePciFeatureConfiguration
+  of the PCI_FEATURE_CONFIGURATION_COMPLETION_LIST.
+
+  @param  RootBridge              A pointer to the PCI_IO_DEVICE for the Root Bridge
+
+  @retval TRUE                    PCI Feature configuration required for the PCI
+                                  Root Bridge
+          FALSE                   PCI Feature configuration is not required to be
+                                  re-enumerated for the PCI Root Bridge
+**/
+BOOLEAN
+CheckPciFeaturesConfigurationRequired (
+  IN PCI_IO_DEVICE          *RootBridge
+  )
+{
+  LIST_ENTRY                                  *Link;
+  PCI_FEATURE_CONFIGURATION_COMPLETION_LIST   *Temp;
+
+  if (mPciFeaturesConfigurationCompletionList) {
+    Link = &mPciFeaturesConfigurationCompletionList->RootBridgeLink;
+
+    do {
+      Temp = PCI_FEATURE_CONFIGURATION_COMPLETION_LIST_FROM_LINK (Link);
+      if (Temp->RootBridgeHandle == RootBridge->Handle) {
+        return Temp->ReEnumeratePciFeatureConfiguration;
+      }
+      Link = Link->ForwardLink;
+    } while (Link != &mPciFeaturesConfigurationCompletionList->RootBridgeLink);
+  }
+  //
+  // not found on the PCI feature configuration completion list, return as required
+  //
+  return TRUE;
+}
+
+/**
+  This routine finds the duplicate record if exist and assigns the re-enumeration
+  requirement flag, as passed as input. It creates new record for the PCI Root
+  Bridge and appends the list after updating its re-enumeration flag.
+
+  @param  RootBridge            A pointer to PCI_IO_DEVICE of the Root Bridge
+  @param  ReEnumerationRequired A BOOLEAN for recording the re-enumeration requirement
+
+  @retval EFI_SUCCESS           new record inserted into the list or updated the
+                                existing record
+          EFI_INVALID_PARAMETER Unexpected error as CheckPciFeatureConfigurationRecordExist
+                                reports as record exist but does not return its pointer
+          EFI_OUT_OF_RESOURCES  Not able to create PCI features configuratin complete
+                                record for the RootBridge
+**/
+EFI_STATUS
+AddRootBridgeInPciFeaturesConfigCompletionList (
+  IN PCI_IO_DEVICE          *RootBridge,
+  IN BOOLEAN                ReEnumerationRequired
+  )
+{
+  PCI_FEATURE_CONFIGURATION_COMPLETION_LIST   *Temp;
+
+  if (CheckPciFeatureConfigurationRecordExist (RootBridge, &Temp)) {
+    //
+    // this PCI Root Bridge record already exist; it may have been re-enumerated
+    // hence just update its enumeration required flag again to exit
+    //
+    if (Temp) {
+      Temp->ReEnumeratePciFeatureConfiguration  = ReEnumerationRequired;
+      return EFI_SUCCESS;
+    } else {
+      //
+      // PCI feature configuration complete record reported as exist and no
+      // record pointer returned
+      //
+      return EFI_INVALID_PARAMETER;
+    }
+
+  } else {
+
+    Temp = AllocateZeroPool (sizeof (PCI_FEATURE_CONFIGURATION_COMPLETION_LIST));
+    if (Temp == NULL) {
+      return EFI_OUT_OF_RESOURCES;
+    }
+    Temp->Signature                           = PCI_FEATURE_CONFIGURATION_SIGNATURE;
+    Temp->RootBridgeHandle                    = RootBridge->Handle;
+    Temp->ReEnumeratePciFeatureConfiguration  = ReEnumerationRequired;
+    if (mPciFeaturesConfigurationCompletionList) {
+      InsertTailList (
+          &mPciFeaturesConfigurationCompletionList->RootBridgeLink,
+          &Temp->RootBridgeLink
+          );
+    } else {
+      //
+      // init the very first node of the Root Bridge
+      //
+      mPciFeaturesConfigurationCompletionList = Temp;
+      InitializeListHead (&mPciFeaturesConfigurationCompletionList->RootBridgeLink);
+    }
+  }
+  return EFI_SUCCESS;
+}
+
+/**
+  Free up memory alloted for the primary physical PCI Root ports of the PCI Root
+  Bridge instance. Free up all the nodes of type PRIMARY_ROOT_PORT_NODE.
+**/
+VOID
+DestroyPrimaryRootPortNodes ()
+{
+  LIST_ENTRY                *Link;
+  PRIMARY_ROOT_PORT_NODE    *Temp;
+
+  if (mPrimaryRootPortList) {
+    Link = &mPrimaryRootPortList->NeighborRootPort;
+
+    if (IsListEmpty (Link)) {
+      FreePool (mPrimaryRootPortList->OtherPciFeaturesConfigurationTable);
+      FreePool (mPrimaryRootPortList);
+    } else {
+      do {
+        if (Link->ForwardLink != &mPrimaryRootPortList->NeighborRootPort) {
+          Link = Link->ForwardLink;
+        }
+        Temp = PRIMARY_ROOT_PORT_NODE_FROM_LINK (Link);
+        Link = RemoveEntryList (Link);
+        FreePool (Temp->OtherPciFeaturesConfigurationTable);
+        FreePool (Temp);
+      } while (!IsListEmpty (Link));
+      FreePool (mPrimaryRootPortList->OtherPciFeaturesConfigurationTable);
+      FreePool (mPrimaryRootPortList);
+    }
+    mPrimaryRootPortList = NULL;
+  }
+}
+
+/**
+  Routine meant for initializing any global variables used. It primarily cleans
+  up the internal data structure memory allocated for the previous PCI Root Bridge
+  instance. This should be the first routine to call for any virtual PCI Root
+  Bridge instance.
+**/
+VOID
+SetupPciFeaturesConfigurationDefaults ()
+{
+  //
+  // delete the primary root port list
+  //
+  if (mPrimaryRootPortList) {
+    DestroyPrimaryRootPortNodes ();
+  }
+}
+
+/**
+  Main routine to determine the child PCI devices of a PCI bridge device
+  and group them under a common internal PCI features Configuration table.
+
+  @param  PciDevice                       A pointer to the PCI_IO_DEVICE.
+  @param  PciFeaturesConfigTable          A pointer to a pointer to the
+                                          OTHER_PCI_FEATURES_CONFIGURATION_TABLE.
+                                          Returns NULL in case of RCiEP or the PCI
+                                          device does match with any of the physical
+                                          Root ports, or it does not belong to any
+                                          Root port's PCI bus range (not a child)
+
+  @retval EFI_SUCCESS                     able to determine the PCI feature
+                                          configuration table. For RCiEP since
+                                          since it is not prepared.
+          EFI_DEVICE_ERROR                the PCI device has invalid EFI device
+                                          path
+**/
+EFI_STATUS
+GetPciFeaturesConfigurationTable (
+  IN  PCI_IO_DEVICE                           *PciDevice,
+  OUT OTHER_PCI_FEATURES_CONFIGURATION_TABLE  **PciFeaturesConfigTable
+  )
+{
+  LIST_ENTRY                *Link;
+  PRIMARY_ROOT_PORT_NODE    *Temp;
+  BOOLEAN                   NodeMatch;
+  EFI_DEVICE_PATH_PROTOCOL  *RootPortPath;
+  EFI_DEVICE_PATH_PROTOCOL  *PciDevicePath;
+
+  if (mPrimaryRootPortList == NULL) {
+    //
+    // no populated PCI primary root ports to parse and match the PCI features
+    // configuration table
+    //
+    *PciFeaturesConfigTable = NULL;
+    return EFI_SUCCESS;
+  }
+
+
+  if (IsDevicePathEnd (PciDevice->DevicePath)){
+    //
+    // the given PCI device does not have a valid device path
+    //
+    *PciFeaturesConfigTable = NULL;
+    return EFI_DEVICE_ERROR;
+  }
+
+
+  Link = &mPrimaryRootPortList->NeighborRootPort;
+  do {
+    Temp = PRIMARY_ROOT_PORT_NODE_FROM_LINK (Link);
+    RootPortPath = Temp->RootPortDevicePath;
+    PciDevicePath = PciDevice->DevicePath;
+    NodeMatch = FALSE;
+    //
+    // match the device path from the list of primary Root Ports with the given
+    // device; the initial nodes matching in sequence indicate that the given PCI
+    // device belongs to that PCI tree from the root port
+    //
+    if (IsDevicePathEnd (RootPortPath)) {
+      //
+      // critical error as no device path available in root
+      //
+      *PciFeaturesConfigTable = NULL;
+      return EFI_DEVICE_ERROR;
+    }
+
+    if (EfiCompareDevicePath (RootPortPath, PciDevicePath)) {
+      //
+      // the given PCI device is the primary root port itself
+      //
+      *PciFeaturesConfigTable = Temp->OtherPciFeaturesConfigurationTable;
+      return EFI_SUCCESS;
+    }
+    //
+    // check this PCI device belongs to the primary root port of the root bridge
+    // any child PCI device will have the same initial device path nodes  as
+    // its parent root port
+    //
+    while (!IsDevicePathEnd (RootPortPath)){
+
+      if (DevicePathNodeLength (RootPortPath) != DevicePathNodeLength (PciDevicePath)) {
+        //
+        // break to check the next primary root port nodes as does not match
+        //
+        NodeMatch = FALSE;
+        break;
+      }
+      if (CompareMem (RootPortPath, PciDevicePath, DevicePathNodeLength (RootPortPath)) != 0) {
+        //
+        // node does not match, break to check next node
+        //
+        NodeMatch = FALSE;
+        break;
+      }
+      NodeMatch = TRUE;
+      //
+      // advance to next node
+      //
+      RootPortPath = NextDevicePathNode (RootPortPath);
+      PciDevicePath = NextDevicePathNode (PciDevicePath);
+    }
+
+    if (NodeMatch == TRUE) {
+      //
+      // device belongs to primary root port, return its PCI feature configuration
+      // table
+      //
+      *PciFeaturesConfigTable = Temp->OtherPciFeaturesConfigurationTable;
+      return EFI_SUCCESS;
+    }
+
+    //
+    // advance to next Root port node
+    //
+    Link = Link->ForwardLink;
+  } while (Link != &mPrimaryRootPortList->NeighborRootPort);
+  //
+  // the PCI device must be RCiEP, does not belong to any primary root port
+  //
+  *PciFeaturesConfigTable = NULL;
+  return EFI_SUCCESS;
+}
+
+/**
+  This routine determines the existance of the child PCI device for the given
+  PCI Root / Bridge Port device. Always assumes the input PCI device is the bridge
+  or PCI-PCI Bridge device. This routine should not be used with PCI endpoint device.
+
+  @param  PciDevice                       A pointer to the PCI_IO_DEVICE.
+
+  @retval TRUE                            child device exist
+          FALSE                           no child device
+**/
+BOOLEAN
+IsPciRootPortEmpty (
+  IN  PCI_IO_DEVICE                           *PciDevice
+  )
+{
+  if (IsListEmpty (&PciDevice->ChildList)){
+    return TRUE;
+  }
+  return FALSE;
+}
+
+
+/**
+   Process each PCI device as per the pltaform and device-specific policy.
+
+  @param RootBridge             A pointer to the PCI_IO_DEVICE.
+
+  @retval EFI_SUCCESS           processing each PCI feature as per policy defined
+                                was successful.
+ **/
+EFI_STATUS
+SetupDevicePciFeatures (
+  IN  PCI_IO_DEVICE                   *PciDevice,
+  IN  PCI_FEATURE_CONFIGURATION_PHASE PciConfigPhase
+  )
+{
+  EFI_STATUS                              Status;
+  OTHER_PCI_FEATURES_CONFIGURATION_TABLE  *OtherPciFeaturesConfigTable;
+
+  OtherPciFeaturesConfigTable = NULL;
+  Status = GetPciFeaturesConfigurationTable (PciDevice, &OtherPciFeaturesConfigTable);
+  if (EFI_ERROR( Status)) {
+    DEBUG ((
+       DEBUG_WARN, "[Cfg group: 0 {error in dev path}]"
+    ));
+  } else if (OtherPciFeaturesConfigTable == NULL) {
+    DEBUG ((
+        DEBUG_INFO, "[Cfg group: 0]"
+    ));
+  } else {
+    DEBUG ((
+        DEBUG_INFO, "[Cfg group: %d]",
+        OtherPciFeaturesConfigTable->ID
+    ));
+  }
+
+  if (PciConfigPhase == PciFeatureGetDevicePolicy) {
+    Status = GetPciDevicePlatformPolicy (PciDevice);
+    if (EFI_ERROR(Status)) {
+      DEBUG ((
+          DEBUG_ERROR, "Error in obtaining PCI device policy!!!\n"
+      ));
+    }
+  }
+
+  return Status;
+}
+
+/**
+  Traverse all the nodes from the root bridge or PCI-PCI bridge instance, to
+  configure the PCI features as per the device-specific platform policy, and
+  as per the device capability, as applicable.
+
+  @param RootBridge             A pointer to the PCI_IO_DEVICE.
+
+  @retval EFI_SUCCESS           Traversing all the nodes of the root bridge
+                                instances were successfull.
+**/
+EFI_STATUS
+SetupPciFeatures (
+  IN  PCI_IO_DEVICE                   *RootBridge,
+  IN  PCI_FEATURE_CONFIGURATION_PHASE PciConfigPhase
+  )
+{
+  EFI_STATUS            Status;
+  LIST_ENTRY            *Link;
+  PCI_IO_DEVICE         *Device;
+
+  for ( Link = RootBridge->ChildList.ForwardLink
+      ; Link != &RootBridge->ChildList
+      ; Link = Link->ForwardLink
+  ) {
+    Device = PCI_IO_DEVICE_FROM_LINK (Link);
+    if (!DeviceExist (Device)) {
+      DEBUG ((
+          DEBUG_ERROR, "::Device [%02x|%02x|%02x] - does not exist!!!\n",
+          Device->BusNumber, Device->DeviceNumber, Device->FunctionNumber
+      ));
+      continue;
+    }
+    if (IS_PCI_BRIDGE (&Device->Pci)) {
+      DEBUG ((
+          DEBUG_INFO, "::Bridge [%02x|%02x|%02x] -",
+          Device->BusNumber, Device->DeviceNumber, Device->FunctionNumber
+      ));
+      if (Device->IsPciExp) {
+        Status = SetupDevicePciFeatures (Device, PciConfigPhase);
+      } else {
+        DEBUG (( DEBUG_INFO, "Not a PCIe capable device!\n"));
+        //
+        // PCI Bridge which does not have PCI Express Capability structure
+        // cannot process this kind of PCI Bridge device
+        //
+
+      }
+
+      SetupPciFeatures (Device, PciConfigPhase);
+    } else {
+      DEBUG ((
+          DEBUG_INFO, "::Device [%02x|%02x|%02x] -",
+          Device->BusNumber, Device->DeviceNumber, Device->FunctionNumber
+      ));
+      if (Device->IsPciExp) {
+
+        Status = SetupDevicePciFeatures (Device, PciConfigPhase);
+      } else {
+        DEBUG (( DEBUG_INFO, "Not a PCIe capable device!\n"));
+        //
+        // PCI Device which does not have PCI Express Capability structure
+        // cannot process this kind of PCI device
+        //
+      }
+    }
+  }
+
+  return EFI_SUCCESS;
+}
+
+/**
+  Program the PCI device, to override the PCI features as per the policy,
+  resolved from previous traverse.
+
+  @param RootBridge             A pointer to the PCI_IO_DEVICE.
+
+  @retval EFI_SUCCESS           The other PCI features configuration during enumeration
+                                of all the nodes of the PCI root bridge instance were
+                                programmed in PCI-compliance pattern along with the
+                                device-specific policy, as applicable.
+  @retval EFI_UNSUPPORTED       One of the override operation maong the nodes of
+                                the PCI hierarchy resulted in a incompatible address
+                                range.
+  @retval EFI_INVALID_PARAMETER The override operation is performed with invalid input
+                                parameters.
+**/
+EFI_STATUS
+ProgramDevicePciFeatures (
+  IN PCI_IO_DEVICE          *PciDevice
+  )
+{
+  EFI_STATUS           Status;
+
+  return Status;
+}
+
+/**
+  Program all the nodes of the specified root bridge or PCI-PCI Bridge, to
+  override the PCI features.
+
+  @param RootBridge             A pointer to the PCI_IO_DEVICE.
+
+  @retval EFI_SUCCESS           The other PCI features configuration during enumeration
+                                of all the nodes of the PCI root bridge instance were
+                                programmed in PCI-compliance pattern along with the
+                                device-specific policy, as applicable.
+  @retval EFI_UNSUPPORTED       One of the override operation maong the nodes of
+                                the PCI hierarchy resulted in a incompatible address
+                                range.
+  @retval EFI_INVALID_PARAMETER The override operation is performed with invalid input
+                                parameters.
+**/
+EFI_STATUS
+ProgramPciFeatures (
+  IN PCI_IO_DEVICE          *RootBridge
+  )
+{
+  EFI_STATUS           Status;
+  LIST_ENTRY           *Link;
+  PCI_IO_DEVICE        *Device;
+
+  for ( Link = RootBridge->ChildList.ForwardLink
+      ; Link != &RootBridge->ChildList
+      ; Link = Link->ForwardLink
+  ) {
+    Device = PCI_IO_DEVICE_FROM_LINK (Link);
+    if (!DeviceExist (Device)) {
+      DEBUG ((
+          DEBUG_ERROR, "::Device [%02x|%02x|%02x] - does not exist!!!\n",
+          Device->BusNumber, Device->DeviceNumber, Device->FunctionNumber
+      ));
+      continue;
+    }
+    if (IS_PCI_BRIDGE (&Device->Pci)) {
+      DEBUG ((
+          DEBUG_INFO, "::Bridge [%02x|%02x|%02x] -",
+          Device->BusNumber, Device->DeviceNumber, Device->FunctionNumber
+      ));
+      if (Device->IsPciExp) {
+        DEBUG (( DEBUG_INFO, "ready to override!\n"));
+
+        Status = ProgramDevicePciFeatures (Device);
+      } else {
+        DEBUG (( DEBUG_INFO, "skipped!\n"));
+        //
+        // PCI Bridge which does not have PCI Express Capability structure
+        // cannot process this kind of PCI Bridge device
+        //
+      }
+
+      Status = ProgramPciFeatures (Device);
+    } else {
+      DEBUG ((
+          DEBUG_INFO, "::Device [%02x|%02x|%02x] -",
+          Device->BusNumber, Device->DeviceNumber, Device->FunctionNumber
+      ));
+      if (Device->IsPciExp) {
+        DEBUG (( DEBUG_INFO, "ready to override!\n"));
+
+        Status = ProgramDevicePciFeatures (Device);
+      } else {
+        DEBUG (( DEBUG_INFO, "skipped!\n"));
+        //
+        // PCI Device which does not have PCI Express Capability structure
+        // cannot process this kind of PCI device
+        //
+      }
+    }
+  }
+
+  return Status;
+}
+
+/**
+  Create and add a node of type PRIMARY_ROOT_PORT_NODE in the list for the primary
+  Root Port so that all its child PCI devices can be identified against the PCI
+  features configuration table group ID, of type OTHER_PCI_FEATURES_CONFIGURATION_TABLE.
+
+  @param BridgePort    A pointer to the PCI_IO_DEVICE
+  @param PortNumber    A UINTN value to identify the PCI feature configuration
+                       table group
+
+  @retval EFI_SUCCESS           success in adding a node of PRIMARY_ROOT_PORT_NODE
+                                to the list
+          EFI_OUT_OF_RESOURCES  unable to get memory for creating the node
+**/
+EFI_STATUS
+AddPrimaryRootPortNode (
+  IN  PCI_IO_DEVICE           *BridgePort,
+  IN  UINTN                   PortNumber
+  )
+{
+  PRIMARY_ROOT_PORT_NODE                  *RootPortNode = NULL;
+  OTHER_PCI_FEATURES_CONFIGURATION_TABLE  *PciConfigTable = NULL;
+
+  RootPortNode = AllocateZeroPool (sizeof (PRIMARY_ROOT_PORT_NODE));
+  if (RootPortNode == NULL) {
+    return EFI_OUT_OF_RESOURCES;
+  }
+  RootPortNode->Signature                           = PCI_ROOT_PORT_SIGNATURE;
+  RootPortNode->RootPortDevicePath                  = BridgePort->DevicePath;
+  PciConfigTable = AllocateZeroPool (
+                     sizeof (OTHER_PCI_FEATURES_CONFIGURATION_TABLE)
+                    );
+  if (PciConfigTable) {
+    PciConfigTable->ID                          = PortNumber;
+  }
+  RootPortNode->OtherPciFeaturesConfigurationTable  = PciConfigTable;
+
+  if (mPrimaryRootPortList != NULL) {
+    InsertTailList (&mPrimaryRootPortList->NeighborRootPort, &RootPortNode->NeighborRootPort);
+  } else {
+    InitializeListHead (&RootPortNode->NeighborRootPort);
+    mPrimaryRootPortList = RootPortNode;
+  }
+
+  if (PciConfigTable == NULL) {
+    return EFI_OUT_OF_RESOURCES;
+  }
+  return EFI_SUCCESS;
+}
+
+/**
+  Scan all the nodes of the RootBridge to identify and create a separate list
+  of all primary physical PCI root ports and link each with its own instance of
+  the PCI Feature Configuration Table.
+
+  @param  RootBridge    A pointer to the PCI_IO_DEVICE of the PCI Root Bridge
+
+  @retval EFI_OUT_OF_RESOURCES  unable to allocate buffer to store PCI feature
+                                configuration table for all the physical PCI root
+                                ports given
+          EFI_NOT_FOUND         No PCI Bridge device found
+          EFI_SUCCESS           PCI Feature COnfiguration table created for all
+                                the PCI Rooot ports found
+          EFI_INVALID_PARAMETER invalid parameter passed to the routine which
+                                creates the PCI controller node for the primary
+                                Root post list
+**/
+EFI_STATUS
+RecordPciRootPortBridges (
+  IN  PCI_IO_DEVICE           *RootBridge
+  )
+{
+  EFI_STATUS              Status = EFI_NOT_FOUND;
+  LIST_ENTRY              *Link;
+  PCI_IO_DEVICE           *Device;
+  UINTN                   NumberOfRootPorts;
+
+  DEBUG ((
+      DEBUG_INFO, "<<********** RecordPciRootPortBridges -start *************>>\n"
+  ));
+  NumberOfRootPorts = 0;
+  for ( Link = RootBridge->ChildList.ForwardLink
+      ; Link != &RootBridge->ChildList
+      ; Link = Link->ForwardLink
+  ) {
+    Device = PCI_IO_DEVICE_FROM_LINK (Link);
+    if (!DeviceExist (Device)) {
+      continue;
+    }
+    if (IS_PCI_BRIDGE (&Device->Pci)) {
+      NumberOfRootPorts++;
+      DEBUG ((
+          DEBUG_INFO, "#%d ::Bridge [%02x|%02x|%02x]",
+          NumberOfRootPorts, Device->BusNumber, Device->DeviceNumber, Device->FunctionNumber
+      ));
+      //
+      // create a primary root port list if that port is connected
+      //
+      if (!IsListEmpty (&Device->ChildList)) {
+        DEBUG ((
+            DEBUG_INFO, "- has downstream device!\n"
+        ));
+        Status = AddPrimaryRootPortNode (Device, NumberOfRootPorts);
+        if (EFI_ERROR (Status)) {
+          DEBUG ((
+              DEBUG_ERROR, "PCI configuration table allocation failure for #%d ::Bridge [%02x|%02x|%02x]\n",
+              NumberOfRootPorts, Device->BusNumber, Device->DeviceNumber, Device->FunctionNumber
+          ));
+        }
+      } else {
+        DEBUG ((
+            DEBUG_INFO, "- no downstream device!\n"
+        ));
+      }
+    }
+  }
+  DEBUG ((
+      DEBUG_INFO, "<<********** RecordPciRootPortBridges - end **********>>\n"
+  ));
+  return Status;
+}
+
+/**
+  Enumerate all the nodes of the specified root bridge or PCI-PCI Bridge, to
+  configure the other PCI features.
+
+  @param RootBridge          A pointer to the PCI_IO_DEVICE.
+
+  @retval EFI_SUCCESS           The other PCI features configuration during enumeration
+                                of all the nodes of the PCI root bridge instance were
+                                programmed in PCI-compliance pattern along with the
+                                device-specific policy, as applicable.
+  @retval EFI_UNSUPPORTED       One of the override operation maong the nodes of
+                                the PCI hierarchy resulted in a incompatible address
+                                range.
+  @retval EFI_INVALID_PARAMETER The override operation is performed with invalid input
+                                parameters.
+**/
+EFI_STATUS
+EnumerateOtherPciFeatures (
+  IN PCI_IO_DEVICE          *RootBridge
+  )
+{
+  EFI_STATUS            Status;
+  CHAR16                *Str;
+  UINTN                 OtherPciFeatureConfigPhase;
+
+  //
+  // check on PCI features configuration is complete and re-enumeration is required
+  //
+  if (!CheckPciFeaturesConfigurationRequired (RootBridge)) {
+    return EFI_ALREADY_STARTED;
+  }
+
+  Str = ConvertDevicePathToText (
+          DevicePathFromHandle (RootBridge->Handle),
+          FALSE,
+          FALSE
+        );
+  DEBUG ((DEBUG_INFO, "Enumerating PCI features for Root Bridge %s\n", Str != NULL ? Str : L""));
+
+  for ( OtherPciFeatureConfigPhase = PciFeatureRootBridgeScan
+      ; OtherPciFeatureConfigPhase <= PciFeatureConfigurationComplete
+      ; OtherPciFeatureConfigPhase++
+      ) {
+    switch (OtherPciFeatureConfigPhase){
+      case  PciFeatureRootBridgeScan:
+        SetupPciFeaturesConfigurationDefaults ();
+        //
+        //first scan the entire root bridge heirarchy for the primary PCI root ports
+        //
+        RecordPciRootPortBridges (RootBridge);
+        break;
+
+      case  PciFeatureGetDevicePolicy:
+      case  PciFeatureSetupPhase:
+        DEBUG ((
+            DEBUG_INFO, "<<********** SetupPciFeatures - start **********>>\n"
+        ));
+        //
+        // enumerate the other PCI features
+        //
+        Status = SetupPciFeatures (RootBridge, OtherPciFeatureConfigPhase);
+
+        DEBUG ((
+            DEBUG_INFO, "<<********** SetupPciFeatures - end **********>>\n"
+        ));
+        break;
+
+      case  PciFeatureConfigurationPhase:
+        //
+        // override the PCI features as per enumeration phase
+        //
+        DEBUG ((DEBUG_INFO, "PCI features override for Root Bridge %s\n", Str != NULL ? Str : L""));
+        DEBUG ((
+            DEBUG_INFO, "<<********** ProgramPciFeatures - start **********>>\n"
+        ));
+        Status = ProgramPciFeatures (RootBridge);
+        DEBUG ((
+            DEBUG_INFO, "<<********** ProgramPciFeatures - end **********>>\n"
+        ));
+        break;
+
+      case  PciFeatureConfigurationComplete:
+        //
+        // clean up the temporary resource nodes created for this root bridge
+        //
+        DestroyPrimaryRootPortNodes ();
+    }
+  }
+
+  if (Str != NULL) {
+    FreePool (Str);
+  }
+  //
+  // mark this root bridge as PCI features configuration complete, and no new
+  // enumeration is required
+  //
+  AddRootBridgeInPciFeaturesConfigCompletionList (RootBridge, FALSE);
+  return Status;
+}
+
+/**
+  This routine is invoked from the Stop () interface for the EFI handle of the
+  RootBridge. Free up its node of type PCI_FEATURE_CONFIGURATION_COMPLETION_LIST.
+
+  @param  RootBridge      A pointer to the PCI_IO_DEVICE
+**/
+VOID
+DestroyRootBridgePciFeaturesConfigCompletionList (
+  IN PCI_IO_DEVICE          *RootBridge
+  )
+{
+  LIST_ENTRY                                  *Link;
+  PCI_FEATURE_CONFIGURATION_COMPLETION_LIST   *Temp;
+
+  if (mPciFeaturesConfigurationCompletionList) {
+    Link = &mPciFeaturesConfigurationCompletionList->RootBridgeLink;
+
+    do {
+      Temp = PCI_FEATURE_CONFIGURATION_COMPLETION_LIST_FROM_LINK (Link);
+      if (Temp->RootBridgeHandle == RootBridge->Handle) {
+        RemoveEntryList (Link);
+        FreePool (Temp);
+        return;
+      }
+      Link = Link->ForwardLink;
+    } while (Link != &mPciFeaturesConfigurationCompletionList->RootBridgeLink);
+  }
+  //
+  // not found on the PCI feature configuration completion list, return
+  //
+  return;
+}
diff --git a/MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.h b/MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.h
index d06a5e8..b06c140 100644
--- a/MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.h
+++ b/MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.h
@@ -23,4 +23,150 @@ SPDX-License-Identifier: BSD-2-Clause-Patent
 #define PCI_FEATURE_SUPPORT_FLAG_CCC  BIT13
 #define PCI_FEATURE_SUPPORT_FLAG_ESYN BIT14
 #define PCI_FEATURE_SUPPORT_FLAG_PTM  BIT20
+
+//
+// defines the data structure to hold the details of the PCI Root port devices
+//
+typedef struct _PRIMARY_ROOT_PORT_NODE  PRIMARY_ROOT_PORT_NODE;
+
+//
+// defines the data structure to hold the configuration data for the other PCI
+// features
+//
+typedef struct _OTHER_PCI_FEATURES_CONFIGURATION_TABLE  OTHER_PCI_FEATURES_CONFIGURATION_TABLE;
+
+//
+// Defines for the PCI features configuration completion and re-enumeration list
+//
+typedef struct _PCI_FEATURE_CONFIGURATION_COMPLETION_LIST  PCI_FEATURE_CONFIGURATION_COMPLETION_LIST;
+
+//
+// Signature value for the PCI Root Port node
+//
+#define PCI_ROOT_PORT_SIGNATURE               SIGNATURE_32 ('p', 'c', 'i', 'p')
+
+//
+// Definitions of the PCI Root Port data structure members
+//
+struct _PRIMARY_ROOT_PORT_NODE {
+  //
+  // Signature header
+  //
+  UINT32                                    Signature;
+  //
+  // linked list pointers to next node
+  //
+  LIST_ENTRY                                NeighborRootPort;
+  //
+  // pointer to PCI_IO_DEVICE of the primary PCI Controller device
+  //
+  EFI_DEVICE_PATH_PROTOCOL                  *RootPortDevicePath;
+  //
+  // pointer to the corresponding PCI feature configuration Table node
+  // all the child PCI devices of the controller are aligned based on this table
+  //
+  OTHER_PCI_FEATURES_CONFIGURATION_TABLE    *OtherPciFeaturesConfigurationTable;
+};
+
+#define PRIMARY_ROOT_PORT_NODE_FROM_LINK(a) \
+  CR (a, PRIMARY_ROOT_PORT_NODE, NeighborRootPort, PCI_ROOT_PORT_SIGNATURE)
+
+//
+// Definition of the PCI Feature configuration Table members
+//
+struct _OTHER_PCI_FEATURES_CONFIGURATION_TABLE {
+  //
+  // Configuration Table ID
+  //
+  UINTN                                     ID;
+};
+
+
+//
+// PCI feature configuration node signature value
+//
+#define PCI_FEATURE_CONFIGURATION_SIGNATURE               SIGNATURE_32 ('p', 'c', 'i', 'f')
+
+struct _PCI_FEATURE_CONFIGURATION_COMPLETION_LIST {
+  //
+  // Signature header
+  //
+  UINT32                                    Signature;
+  //
+  // link to next Root Bridge whose PCI Feature configuration is complete
+  //
+  LIST_ENTRY                                RootBridgeLink;
+  //
+  // EFI handle of the Root Bridge whose PCI feature configuration is complete
+  //
+  EFI_HANDLE                                RootBridgeHandle;
+  //
+  // indication for complete re-enumeration of the PCI feature configuration
+  //
+  BOOLEAN                                   ReEnumeratePciFeatureConfiguration;
+};
+
+#define PCI_FEATURE_CONFIGURATION_COMPLETION_LIST_FROM_LINK(a) \
+  CR (a, PCI_FEATURE_CONFIGURATION_COMPLETION_LIST, RootBridgeLink, PCI_FEATURE_CONFIGURATION_SIGNATURE)
+
+//
+// Declaration of the internal sub-phases within the PCI Feature enumeration
+//
+typedef enum {
+  //
+  // initial phase in configuring the other PCI features to record the primary
+  // root ports
+  //
+  PciFeatureRootBridgeScan,
+  //
+  // get the PCI device-specific platform policies and align with device capabilities
+  //
+  PciFeatureGetDevicePolicy,
+  //
+  // align all PCI nodes in the PCI heirarchical tree
+  //
+  PciFeatureSetupPhase,
+  //
+  // finally override to complete configuration of the PCI feature
+  //
+  PciFeatureConfigurationPhase,
+  //
+  // PCI feature configuration complete
+  //
+  PciFeatureConfigurationComplete
+
+}PCI_FEATURE_CONFIGURATION_PHASE;
+
+
+/**
+  Enumerate all the nodes of the specified root bridge or PCI-PCI Bridge, to
+  configure the other PCI features.
+
+  @param RootBridge          A pointer to the PCI_IO_DEVICE.
+
+  @retval EFI_SUCCESS           The other PCI features configuration during enumeration
+                                of all the nodes of the PCI root bridge instance were
+                                programmed in PCI-compliance pattern along with the
+                                device-specific policy, as applicable.
+  @retval EFI_UNSUPPORTED       One of the override operation maong the nodes of
+                                the PCI hierarchy resulted in a incompatible address
+                                range.
+  @retval EFI_INVALID_PARAMETER The override operation is performed with invalid input
+                                parameters.
+**/
+EFI_STATUS
+EnumerateOtherPciFeatures (
+  IN PCI_IO_DEVICE           *RootBridge
+  );
+
+/**
+  This routine is invoked from the Stop () interface for the EFI handle of the
+  RootBridge. Free up its node of type PCI_FEATURE_CONFIGURATION_COMPLETION_LIST.
+
+  @param  RootBridge      A pointer to the PCI_IO_DEVICE
+**/
+VOID
+DestroyRootBridgePciFeaturesConfigCompletionList (
+  IN PCI_IO_DEVICE          *RootBridge
+  );
 #endif
-- 
2.21.0.windows.1