Re: [edk2-platforms][PATCH v4 25/31] Ampere: Utilize the PCIe User setting

["Leif Lindholm" <leif@nuviainc.com>]

On Fri, Oct 22, 2021 at 13:18:03 +0700, Nhi Pham wrote:
> From: Vu Nguyen <vunguyen@os.amperecomputing.com>
> 
> This change allows to configure the PCIe bifurcation mode and update
> the ACPI IORT tables based on the PCIe User setting.
> 
> Cc: Thang Nguyen <thang@os.amperecomputing.com>
> Cc: Chuong Tran <chuong@os.amperecomputing.com>
> Cc: Phong Vo <phong@os.amperecomputing.com>
> Cc: Leif Lindholm <leif@nuviainc.com>
> Cc: Michael D Kinney <michael.d.kinney@intel.com>
> Cc: Ard Biesheuvel <ardb+tianocore@kernel.org>
> Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
> 
> Signed-off-by: Nhi Pham <nhi@os.amperecomputing.com>

Reviewed-by: Leif Lindholm <leif@nuviainc.com>

> ---
>  Platform/Ampere/JadePkg/Drivers/AcpiPlatformDxe/AcpiPlatformDxe.inf |  2 +
>  Silicon/Ampere/AmpereAltraPkg/Drivers/PcieInitPei/PcieInitPei.inf   |  5 +++
>  Platform/Ampere/JadePkg/Drivers/AcpiPlatformDxe/AcpiIort.c          | 32 ++++++++++++----
>  Silicon/Ampere/AmpereAltraPkg/Drivers/PcieInitPei/PcieInitPei.c     | 40 ++++++++++++++++++--
>  4 files changed, 69 insertions(+), 10 deletions(-)
> 
> diff --git a/Platform/Ampere/JadePkg/Drivers/AcpiPlatformDxe/AcpiPlatformDxe.inf b/Platform/Ampere/JadePkg/Drivers/AcpiPlatformDxe/AcpiPlatformDxe.inf
> index 415f795d2a54..804e761a1524 100644
> --- a/Platform/Ampere/JadePkg/Drivers/AcpiPlatformDxe/AcpiPlatformDxe.inf
> +++ b/Platform/Ampere/JadePkg/Drivers/AcpiPlatformDxe/AcpiPlatformDxe.inf
> @@ -52,6 +52,7 @@ [LibraryClasses]
>    UefiBootServicesTableLib
>    UefiDriverEntryPoint
>    UefiLib
> +  UefiRuntimeServicesTableLib
>  
>  [Pcd]
>    gArmPlatformTokenSpaceGuid.PcdCoreCount
> @@ -70,6 +71,7 @@ [Guids]
>    gEfiEventReadyToBootGuid
>    gPlatformInfoHobGuid
>    gRootComplexInfoHobGuid
> +  gRootComplexConfigFormSetGuid
>  
>  [Protocols]
>    gEfiAcpiTableProtocolGuid                     ## ALWAYS_CONSUMED
> diff --git a/Silicon/Ampere/AmpereAltraPkg/Drivers/PcieInitPei/PcieInitPei.inf b/Silicon/Ampere/AmpereAltraPkg/Drivers/PcieInitPei/PcieInitPei.inf
> index 17ac1672dac8..32d60bec1440 100644
> --- a/Silicon/Ampere/AmpereAltraPkg/Drivers/PcieInitPei/PcieInitPei.inf
> +++ b/Silicon/Ampere/AmpereAltraPkg/Drivers/PcieInitPei/PcieInitPei.inf
> @@ -31,9 +31,14 @@ [LibraryClasses]
>    DebugLib
>    HobLib
>    PeimEntryPoint
> +  PeiServicesLib
> +
> +[Ppis]
> +  gEfiPeiReadOnlyVariable2PpiGuid
>  
>  [Guids]
>    gRootComplexInfoHobGuid
> +  gRootComplexConfigFormSetGuid
>    gPlatformInfoHobGuid
>  
>  [Depex]
> diff --git a/Platform/Ampere/JadePkg/Drivers/AcpiPlatformDxe/AcpiIort.c b/Platform/Ampere/JadePkg/Drivers/AcpiPlatformDxe/AcpiIort.c
> index b8f8cfa356af..97be85c51f25 100644
> --- a/Platform/Ampere/JadePkg/Drivers/AcpiPlatformDxe/AcpiIort.c
> +++ b/Platform/Ampere/JadePkg/Drivers/AcpiPlatformDxe/AcpiIort.c
> @@ -7,6 +7,7 @@
>  **/
>  
>  #include <AcpiHeader.h>
> +#include <Guid/RootComplexConfigHii.h>
>  #include <Guid/RootComplexInfoHob.h>
>  #include <IndustryStandard/Acpi30.h>
>  #include <IndustryStandard/IoRemappingTable.h>
> @@ -17,6 +18,7 @@
>  #include <Library/HobLib.h>
>  #include <Library/MemoryAllocationLib.h>
>  #include <Library/UefiBootServicesTableLib.h>
> +#include <Library/UefiRuntimeServicesTableLib.h>
>  #include <Platform/Ac01.h>
>  #include <Protocol/AcpiTable.h>
>  
> @@ -282,8 +284,10 @@ AcpiInstallIort (
>    EFI_ACPI_TABLE_PROTOCOL           *AcpiTableProtocol;
>    EFI_STATUS                        Status;
>    INT32                             EnabledRCs[AC01_PCIE_MAX_ROOT_COMPLEX];
> +  ROOT_COMPLEX_CONFIG_VARSTORE_DATA VarStoreConfig;
>    UINT32                            RcCount, SmmuPmuAgentCount, TotalCount;
>    UINT8                             Idx;
> +  UINTN                             BufferSize;
>    UINTN                             TableKey;
>    VOID                              *Hob;
>    VOID                              *IortBuffer;
> @@ -313,14 +317,28 @@ AcpiInstallIort (
>    }
>  
>    SmmuPmuAgentCount = 0;
> -  for (Idx = 0; Idx < RcCount; Idx++) {
> -    if (mRootComplexList[EnabledRCs[Idx]].Type == RootComplexTypeA) {
> -      SmmuPmuAgentCount += AC01_RCA_MAX_TBU_PMU;
> -    } else {
> -      SmmuPmuAgentCount += AC01_RCB_MAX_TBU_PMU;
> +
> +  //
> +  // Check SMMU setting
> +  //
> +  BufferSize = sizeof (VarStoreConfig);
> +  Status = gRT->GetVariable (
> +                  ROOT_COMPLEX_CONFIG_VARSTORE_NAME,
> +                  &gRootComplexConfigFormSetGuid,
> +                  NULL,
> +                  &BufferSize,
> +                  &VarStoreConfig
> +                  );
> +  if (!EFI_ERROR (Status) && VarStoreConfig.SmmuPmu) {
> +    for (Idx = 0; Idx < RcCount; Idx++) {
> +      if (mRootComplexList[EnabledRCs[Idx]].Type == RootComplexTypeA) {
> +        SmmuPmuAgentCount += AC01_RCA_MAX_TBU_PMU;
> +      } else {
> +        SmmuPmuAgentCount += AC01_RCB_MAX_TBU_PMU;
> +      }
> +      // Plus 1 TCU
> +      SmmuPmuAgentCount += 1;
>      }
> -    // Plus 1 TCU
> -    SmmuPmuAgentCount += 1;
>    }
>  
>    TotalCount = sizeof (EFI_ACPI_6_0_IO_REMAPPING_TABLE) +
> diff --git a/Silicon/Ampere/AmpereAltraPkg/Drivers/PcieInitPei/PcieInitPei.c b/Silicon/Ampere/AmpereAltraPkg/Drivers/PcieInitPei/PcieInitPei.c
> index 76cbb76f38f0..3e873a669332 100644
> --- a/Silicon/Ampere/AmpereAltraPkg/Drivers/PcieInitPei/PcieInitPei.c
> +++ b/Silicon/Ampere/AmpereAltraPkg/Drivers/PcieInitPei/PcieInitPei.c
> @@ -8,6 +8,7 @@
>  
>  #include <PiPei.h>
>  
> +#include <Guid/RootComplexConfigHii.h>
>  #include <Guid/RootComplexInfoHob.h>
>  #include <Library/AmpereCpuLib.h>
>  #include <Library/BaseMemoryLib.h>
> @@ -15,7 +16,9 @@
>  #include <Library/DebugLib.h>
>  #include <Library/HobLib.h>
>  #include <Library/Ac01PcieLib.h>
> +#include <Library/PeiServicesLib.h>
>  #include <Platform/Ac01.h>
> +#include <Ppi/ReadOnlyVariable2.h>
>  
>  #include "RootComplexNVParam.h"
>  
> @@ -43,8 +46,39 @@ BuildRootComplexData (
>    )
>  {
>    AC01_ROOT_COMPLEX                    *RootComplex;
> +  BOOLEAN                              ConfigFound;
> +  EFI_PEI_READ_ONLY_VARIABLE2_PPI      *VariablePpi;
> +  EFI_STATUS                           Status;
> +  ROOT_COMPLEX_CONFIG_VARSTORE_DATA    RootComplexConfig;
>    UINT8                                RCIndex;
>    UINT8                                PcieIndex;
> +  UINTN                                DataSize;
> +
> +  ConfigFound = FALSE;
> +
> +  //
> +  // Get the Root Complex config from NVRAM
> +  //
> +  Status = PeiServicesLocatePpi (
> +             &gEfiPeiReadOnlyVariable2PpiGuid,
> +             0,
> +             NULL,
> +             (VOID **)&VariablePpi
> +             );
> +  if (!EFI_ERROR (Status)) {
> +    DataSize = sizeof (RootComplexConfig);
> +    Status = VariablePpi->GetVariable (
> +                            VariablePpi,
> +                            ROOT_COMPLEX_CONFIG_VARSTORE_NAME,
> +                            &gRootComplexConfigFormSetGuid,
> +                            NULL,
> +                            &DataSize,
> +                            &RootComplexConfig
> +                            );
> +    if (!EFI_ERROR (Status)) {
> +      ConfigFound = TRUE;
> +    }
> +  }
>  
>    ZeroMem (&mRootComplexList, sizeof (AC01_ROOT_COMPLEX) * AC01_PCIE_MAX_ROOT_COMPLEX);
>  
> @@ -58,9 +92,9 @@ BuildRootComplexData (
>  
>    for (RCIndex = 0; RCIndex < AC01_PCIE_MAX_ROOT_COMPLEX; RCIndex++) {
>      RootComplex = &mRootComplexList[RCIndex];
> -    RootComplex->Active = TRUE;
> -    RootComplex->DevMapLow = 0;
> -    RootComplex->DevMapHigh = 0;
> +    RootComplex->Active = ConfigFound ? RootComplexConfig.RCStatus[RCIndex] : TRUE;
> +    RootComplex->DevMapLow = ConfigFound ? RootComplexConfig.RCBifurcationLow[RCIndex] : 0;
> +    RootComplex->DevMapHigh = ConfigFound ? RootComplexConfig.RCBifurcationLow[RCIndex] : 0;
>      RootComplex->Socket = RCIndex / AC01_PCIE_MAX_RCS_PER_SOCKET;
>      RootComplex->ID = RCIndex % AC01_PCIE_MAX_RCS_PER_SOCKET;
>      RootComplex->CsrBase = mCsrBase[RCIndex];
> -- 
> 2.17.1
>