Re: [edk2-devel] [edk2-staging/UEFI_PCI_ENHANCE-2 PATCH 09/12] PciBusDxe: New PCI Express feature LTR

["Javeed, Ashraf" <ashraf.javeed@intel.com>]

This patch can also be viewed in the following repo:-
https://github.com/ashrafj/edk2-staging/commit/fd03852add274895880971e38f07e30d5fd79863

Thanks
Ashraf

> -----Original Message-----
> From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Javeed,
> Ashraf
> Sent: Saturday, February 8, 2020 1:35 AM
> To: devel@edk2.groups.io
> Cc: Wang, Jian J <jian.j.wang@intel.com>; Wu, Hao A <hao.a.wu@intel.com>;
> Ni, Ray <ray.ni@intel.com>
> Subject: [edk2-devel] [edk2-staging/UEFI_PCI_ENHANCE-2 PATCH 09/12]
> PciBusDxe: New PCI Express feature LTR
> 
> BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2499
> 
> This code change enables the PCI Express feature LTR, in compliance with the
> PCI Express Base Specification 5, as well as in accordance its device policy, as
> follows:
> (1) all the devices capability register needs to indicate that LTR mecha-
>     nism is supported. ANy device not capable shall lead to all devices
>     state in LTR disable
> (2) if device policy of any device requests LTR mechanism enabled, than
>     based on above condition (1) is TRUE, all of the devices from root
>     bridge are enabled.
> (3) Even in case of RCiEP device without any root bridge device; if device
>     policy requests LTR enabled and device capability does not support LTR
>     shall enforce the default LTR disabled state for that device
> 
> This programming of LTR, gets the device-specific platform policy using the new
> PCI Express Platform Protocol interface (ECR version 0.8), defined in the below
> feature request:-
>   https://bugzilla.tianocore.org/show_bug.cgi?id=1954
> 
> Signed-off-by: Ashraf Javeed <ashraf.javeed@intel.com>
> Cc: Jian J Wang <jian.j.wang@intel.com>
> Cc: Hao A Wu <hao.a.wu@intel.com>
> Cc: Ray Ni <ray.ni@intel.com>
> ---
>  MdeModulePkg/Bus/Pci/PciBusDxe/PciBus.h             |   1 +
>  MdeModulePkg/Bus/Pci/PciBusDxe/PciExpressFeatures.c | 171
> +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
> +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
> +++++++++++++++++++++++++++++++++++++++++
>  MdeModulePkg/Bus/Pci/PciBusDxe/PciExpressFeatures.h |  40
> ++++++++++++++++++++++++++++++++++++++++
>  MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.c  |  19
> ++++++++++++++++++-
>  MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.h  |   9 +++++++++
>  MdeModulePkg/Bus/Pci/PciBusDxe/PciPlatformSupport.c |  53
> +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  6 files changed, 292 insertions(+), 1 deletion(-)
> 
> diff --git a/MdeModulePkg/Bus/Pci/PciBusDxe/PciBus.h
> b/MdeModulePkg/Bus/Pci/PciBusDxe/PciBus.h
> index 57ef1b2..7e43a26 100644
> --- a/MdeModulePkg/Bus/Pci/PciBusDxe/PciBus.h
> +++ b/MdeModulePkg/Bus/Pci/PciBusDxe/PciBus.h
> @@ -296,6 +296,7 @@ struct _PCI_IO_DEVICE {
>    PCI_FEATURE_POLICY                        SetupNS;
>    PCI_FEATURE_POLICY                        SetupCTO;
>    EFI_PCI_EXPRESS_ATOMIC_OP                 SetupAtomicOp;
> +  BOOLEAN                                   SetupLtr;
>  };
> 
>  #define PCI_IO_DEVICE_FROM_PCI_IO_THIS(a) \ diff --git
> a/MdeModulePkg/Bus/Pci/PciBusDxe/PciExpressFeatures.c
> b/MdeModulePkg/Bus/Pci/PciBusDxe/PciExpressFeatures.c
> index 5c76ba4..63a243b 100644
> --- a/MdeModulePkg/Bus/Pci/PciBusDxe/PciExpressFeatures.c
> +++ b/MdeModulePkg/Bus/Pci/PciBusDxe/PciExpressFeatures.c
> @@ -1069,3 +1069,174 @@ ProgramAtomicOp (
>    return Status;
>  }
> 
> +/**
> +  The main routine which process the PCI feature LTR enable/disable as
> +per the
> +  device-specific platform policy, as well as in complaince with the
> +PCI Express
> +  Base specification Revision 5.
> +
> +  @param PciDevice                      A pointer to the PCI_IO_DEVICE.
> +  @param PciExpressConfigurationTable  pointer to
> + PCI_EXPRESS_FEATURES_CONFIGURATION_TABLE
> +
> +  @retval EFI_SUCCESS                   setup of PCI feature LTR is successful.
> +**/
> +EFI_STATUS
> +SetupLtr (
> +  IN  PCI_IO_DEVICE                             *PciDevice,
> +  IN  PCI_EXPRESS_FEATURES_CONFIGURATION_TABLE
> +*PciExpressConfigurationTable
> +  )
> +{
> +  PCI_REG_PCIE_DEVICE_CAPABILITY2 DeviceCap2;
> +  //
> +  // as per the PCI-Express Base Specification, in order to enable LTR
> +mechanism
> +  // in the upstream ports, all the upstream ports and its downstream
> +ports has
> +  // to support the LTR mechanism reported in its Device Capability 2
> +register
> +  //
> +  DeviceCap2.Uint32 =
> +PciDevice->PciExpressCapabilityStructure.DeviceCapability2.Uint32;
> +
> +  if (PciExpressConfigurationTable) {
> +    //
> +    // in this phase establish 2 requirements:
> +    // (1) all the PCI devices in the hierarchy supports the LTR mechanism
> +    // (2) check and record any device-specific platform policy that wants to
> +    //     enable the LTR mechanism
> +    //
> +    if
> + (!PciDevice->PciExpressCapabilityStructure.DeviceCapability2.Bits.LtrM
> + echanism) {
> +
> +      //
> +      // it starts with the assumption that all the PCI devices support LTR
> mechanism
> +      // and negates the flag if any PCI device Device Capability 2 register
> advertizes
> +      // as not supported
> +      //
> +      PciExpressConfigurationTable->LtrSupported = FALSE;
> +    }
> +
> +    if (PciDevice->SetupLtr == TRUE) {
> +      //
> +      // it starts with the assumption that device-specific platform policy would
> +      // be set to LTR disable, and negates the flag if any PCI device platform
> +      // policy wants to override to enable the LTR mechanism
> +      //
> +      PciExpressConfigurationTable->LtrEnable = TRUE;
> +    }
> +  } else {
> +    //
> +    // in case of RCiEP device or the bridge device with out any child device,
> +    // overrule the device policy if the device in not capable
> +    //
> +    if (!PciDevice-
> >PciExpressCapabilityStructure.DeviceCapability2.Bits.LtrMechanism
> +        && PciDevice->SetupLtr == TRUE) {
> +      PciDevice->SetupLtr = FALSE;
> +    }
> +    //
> +    // for any bridge device which is Hot-Plug capable, it is expected that
> platform
> +    // will not enforce the enabling of LTR mechanism only for the bridge device
> +    //
> +  }
> +
> +  DEBUG (( DEBUG_INFO, "LTR En: %d (LTR Cap: %d),",
> +    PciDevice->SetupLtr ? 1 : 0,
> +    PciDevice-
> >PciExpressCapabilityStructure.DeviceCapability2.Bits.LtrMechanism
> +    ));
> +  return EFI_SUCCESS;
> +}
> +
> +EFI_STATUS
> +ReSetupLtr (
> +  IN  PCI_IO_DEVICE                             *PciDevice,
> +  IN  PCI_EXPRESS_FEATURES_CONFIGURATION_TABLE
> +*PciExpressConfigurationTable
> +  )
> +{
> +  //
> +  // not applicable to RCiEP device...
> +  // for the bridge device without any child device, the policy is
> +already overruled
> +  // based on capability in the above routine
> +  //
> +  if (PciExpressConfigurationTable) {
> +    //
> +    // in this phase align the device policy to enable LTR policy of any PCI device
> +    // in the tree if all the devices are capable to support the LTR mechanism
> +    //
> +    if (PciExpressConfigurationTable->LtrSupported == TRUE
> +        && PciExpressConfigurationTable->LtrEnable == TRUE
> +    ) {
> +      PciDevice->SetupLtr = TRUE;
> +    } else {
> +      PciDevice->SetupLtr = FALSE;
> +    }
> +  }
> +
> +  DEBUG (( DEBUG_INFO, "LTR En: %d (LTR Cap: %d),",
> +    PciDevice->SetupLtr ? 1 : 0,
> +    PciDevice-
> >PciExpressCapabilityStructure.DeviceCapability2.Bits.LtrMechanism
> +    ));
> +  return EFI_SUCCESS;
> +}
> +
> +/**
> +  Program the PCI Device Control 2 register LTR mechanism field; if
> +  the hardware value is different than the intended value.
> +
> +  @param  PciDevice             A pointer to the PCI_IO_DEVICE instance.
> +
> +  @retval EFI_SUCCESS           The data was read from or written to the PCI
> device.
> +  @retval EFI_UNSUPPORTED       The address range specified by Offset, Width,
> and Count is not
> +                                valid for the PCI configuration header of the PCI controller.
> +  @retval EFI_INVALID_PARAMETER Buffer is NULL or Width is invalid.
> +
> +**/
> +EFI_STATUS
> +ProgramLtr (
> +  IN PCI_IO_DEVICE          *PciDevice,
> +  IN VOID                   *PciExFeatureConfiguration
> +  )
> +{
> +  PCI_REG_PCIE_DEVICE_CONTROL2  PcieDev;
> +  UINT32                        Offset;
> +  EFI_STATUS                    Status;
> +  EFI_TPL                       OldTpl;
> +
> +  PcieDev.Uint16 = 0;
> +  Offset = PciDevice->PciExpressCapabilityOffset +
> +               OFFSET_OF (PCI_CAPABILITY_PCIEXP, DeviceControl2);
> + Status = PciDevice->PciIo.Pci.Read (
> +                                  &PciDevice->PciIo,
> +                                  EfiPciIoWidthUint16,
> +                                  Offset,
> +                                  1,
> +                                  &PcieDev.Uint16
> +                                  );
> +  ASSERT (Status == EFI_SUCCESS);
> +
> +  if (PciDevice->SetupLtr != (BOOLEAN) PcieDev.Bits.LtrMechanism) {
> +    PcieDev.Bits.LtrMechanism = PciDevice->SetupLtr ? 1 : 0;
> +    DEBUG (( DEBUG_INFO, "LTR=%d,", PcieDev.Bits.LtrMechanism));
> +
> +    //
> +    // Raise TPL to high level to disable timer interrupt while the write operation
> completes
> +    //
> +    OldTpl = gBS->RaiseTPL (TPL_HIGH_LEVEL);
> +
> +    Status = PciDevice->PciIo.Pci.Write (
> +                                    &PciDevice->PciIo,
> +                                    EfiPciIoWidthUint16,
> +                                    Offset,
> +                                    1,
> +                                    &PcieDev.Uint16
> +                                    );
> +    //
> +    // Restore TPL to its original level
> +    //
> +    gBS->RestoreTPL (OldTpl);
> +
> +    if (!EFI_ERROR(Status)) {
> +      PciDevice->PciExpressCapabilityStructure.DeviceControl2.Uint16 =
> PcieDev.Uint16;
> +    } else {
> +      ReportPciWriteError (PciDevice->BusNumber, PciDevice->DeviceNumber,
> PciDevice->FunctionNumber, Offset);
> +    }
> +  } else {
> +    DEBUG (( DEBUG_INFO, "no LTR,"));
> +  }
> +
> +  return Status;
> +}
> +
> diff --git a/MdeModulePkg/Bus/Pci/PciBusDxe/PciExpressFeatures.h
> b/MdeModulePkg/Bus/Pci/PciBusDxe/PciExpressFeatures.h
> index 1e287fc..374fe49 100644
> --- a/MdeModulePkg/Bus/Pci/PciBusDxe/PciExpressFeatures.h
> +++ b/MdeModulePkg/Bus/Pci/PciBusDxe/PciExpressFeatures.h
> @@ -203,4 +203,44 @@ ProgramAtomicOp (
>    IN VOID                   *PciExFeatureConfiguration
>    );
> 
> +/**
> +  The main routine which process the PCI feature LTR enable/disable as
> +per the
> +  device-specific platform policy, as well as in complaince with the
> +PCI Express
> +  Base specification Revision 5.
> +
> +  @param PciDevice                      A pointer to the PCI_IO_DEVICE.
> +  @param PciFeaturesConfigurationTable  pointer to
> + PCI_EXPRESS_FEATURES_CONFIGURATION_TABLE
> +
> +  @retval EFI_SUCCESS                   setup of PCI feature LTR is successful.
> +**/
> +EFI_STATUS
> +SetupLtr (
> +  IN  PCI_IO_DEVICE                             *PciDevice,
> +  IN  PCI_EXPRESS_FEATURES_CONFIGURATION_TABLE
> +*PciFeaturesConfigurationTable
> +  );
> +
> +EFI_STATUS
> +ReSetupLtr (
> +  IN  PCI_IO_DEVICE                             *PciDevice,
> +  IN  PCI_EXPRESS_FEATURES_CONFIGURATION_TABLE
> +*PciFeaturesConfigurationTable
> +  );
> +
> +/**
> +  Program the PCI Device Control 2 register LTR mechanism field; if
> +  the hardware value is different than the intended value.
> +
> +  @param  PciDevice             A pointer to the PCI_IO_DEVICE instance.
> +
> +  @retval EFI_SUCCESS           The data was read from or written to the PCI
> device.
> +  @retval EFI_UNSUPPORTED       The address range specified by Offset, Width,
> and Count is not
> +                                valid for the PCI configuration header of the PCI controller.
> +  @retval EFI_INVALID_PARAMETER Buffer is NULL or Width is invalid.
> +
> +**/
> +EFI_STATUS
> +ProgramLtr (
> +  IN PCI_IO_DEVICE          *PciDevice,
> +  IN VOID                   *PciExFeatureConfiguration
> +  );
> +
>  #endif
> diff --git a/MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.c
> b/MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.c
> index 9d624a0..bdeb0d2 100644
> --- a/MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.c
> +++ b/MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.c
> @@ -66,7 +66,7 @@ EFI_PCI_EXPRESS_PLATFORM_POLICY
> mPciExpressPlatformPolicy = {
>      //
>      // support for PCI Express feature - LTR
>      //
> -    FALSE,
> +    TRUE,
>      //
>      // support for PCI Express feature - PTM
>      //
> @@ -131,6 +131,15 @@ PCI_EXPRESS_FEATURE_INITIALIZATION_POINT
> mPciExpressFeatureInitializationList[]
>    },
>    {
>      PciExpressFeatureProgramPhase,        PciExpressAtomicOp,
> ProgramAtomicOp
> +  },
> +  {
> +    PciExpressFeatureSetupPhase,          PciExpressLtr,        SetupLtr
> +  },
> +  {
> +    PciExpressFeatureEntendedSetupPhase,  PciExpressLtr,        ReSetupLtr
> +  },
> +  {
> +    PciExpressFeatureProgramPhase,        PciExpressLtr,        ProgramLtr
>    }
>  };
> 
> @@ -654,6 +663,14 @@ CreatePciRootBridgeDeviceNode (
>      //
>      PciConfigTable->Lock_Max_Read_Request_Size  = FALSE;
>      //
> +    // start by assuming the LTR mechanism is supported in a PCI tree
> +    //
> +    PciConfigTable->LtrSupported                = TRUE;
> +    //
> +    // the default LTR mechanism is disabled as per the PCI Base specification
> +    //
> +    PciConfigTable->LtrEnable                   = FALSE;
> +    //
>      // start by assuming the AtomicOp Routing capability is supported in the PCI
>      // tree
>      //
> diff --git a/MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.h
> b/MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.h
> index 2bd565e..5dded7c 100644
> --- a/MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.h
> +++ b/MdeModulePkg/Bus/Pci/PciBusDxe/PciFeatureSupport.h
> @@ -81,6 +81,15 @@ struct
> _PCI_EXPRESS_FEATURES_CONFIGURATION_TABLE {
>    //
>    BOOLEAN                                   Lock_Max_Read_Request_Size;
>    //
> +  // to record the adversity in LTR mechanism support capability among
> + the PCI  // device of an heirarchy  //
> +  BOOLEAN                                   LtrSupported;
> +  //
> +  // to enable the LTR mechansim for the entire PCI tree from a root
> + port  //
> +  BOOLEAN                                   LtrEnable;
> +  //
>    // to record the AtomicOp Routing capability of the PCI Heirarchy to enable
>    // the AtomicOp of the EP device
>    //
> diff --git a/MdeModulePkg/Bus/Pci/PciBusDxe/PciPlatformSupport.c
> b/MdeModulePkg/Bus/Pci/PciBusDxe/PciPlatformSupport.c
> index 2707976..83b3aa7 100644
> --- a/MdeModulePkg/Bus/Pci/PciBusDxe/PciPlatformSupport.c
> +++ b/MdeModulePkg/Bus/Pci/PciBusDxe/PciPlatformSupport.c
> @@ -307,6 +307,36 @@ SetDevicePolicyPciExpressCto (
>    }
>  }
> 
> +/**
> +  Routine to set the device-specific policy for the PCI feature LTR
> +enable/disable
> +
> +  @param  AtomicOp      value corresponding to data type
> EFI_PCI_EXPRESS_ATOMIC_OP
> +  @param  PciDevice     A pointer to PCI_IO_DEVICE
> +
> +**/
> +VOID
> +SetDevicePolicyPciExpressLtr (
> +  IN  EFI_PCI_EXPRESS_LTR            Ltr,
> +  OUT PCI_IO_DEVICE               *PciDevice
> +  )
> +{
> +  switch (Ltr){
> +    case EFI_PCI_EXPRESS_LTR_AUTO:
> +    case EFI_PCI_EXPRESS_LTR_DISABLE:
> +      //
> +      // leave the LTR mechanism disable or restore to its default state
> +      //
> +      PciDevice->SetupLtr = FALSE;
> +      break;
> +    case EFI_PCI_EXPRESS_LTR_ENABLE:
> +      //
> +      // LTR mechanism enable
> +      //
> +      PciDevice->SetupLtr = TRUE;
> +      break;
> +  }
> +}
> +
>  /**
>    Generic routine to setup the PCI features as per its predetermined defaults.
>  **/
> @@ -336,6 +366,8 @@ SetupDefaultPciExpressDevicePolicy (
> 
>    PciDevice->SetupAtomicOp.Override = 0;
> 
> +  PciDevice->SetupLtr = FALSE;
> +
>  }
> 
>  /**
> @@ -461,6 +493,16 @@ GetPciExpressDevicePolicy (
>        PciDevice->SetupAtomicOp.Override = 0;
>      }
> 
> +    //
> +    // set the device-specific policy for LTR mechanism in the function
> +    //
> +    if (mPciExpressPlatformPolicy.Ltr) {
> +      SetDevicePolicyPciExpressLtr (PciExpressDevicePolicy.DeviceCtl2LTR,
> PciDevice);
> +    } else {
> +      PciDevice->SetupLtr = FALSE;
> +    }
> +
> +
>      DEBUG ((
>        DEBUG_INFO,
>        "[device policy: platform]"
> @@ -715,6 +757,17 @@ PciExpressPlatformNotifyDeviceState (
>      PciExDeviceConfiguration.DeviceCtl2AtomicOp.Enable_AtomicOpRequester =
> 0;
> 
> PciExDeviceConfiguration.DeviceCtl2AtomicOp.Enable_AtomicOpEgressBlockin
> g = 0;
>    }
> +  //
> +  // get the device-specific state for LTR mechanism in the function
> + //  if (mPciExpressPlatformPolicy.Ltr) {
> +    PciExDeviceConfiguration.DeviceCtl2LTR = PciDevice-
> >PciExpressCapabilityStructure.DeviceControl2.Bits.LtrMechanism
> +                                                ? EFI_PCI_EXPRESS_LTR_ENABLE
> +                                                :
> + EFI_PCI_EXPRESS_LTR_DISABLE;  } else {
> +    PciExDeviceConfiguration.DeviceCtl2LTR =
> + EFI_PCI_EXPRESS_NOT_APPLICABLE;  }
> +
> 
>    if (mPciExPlatformProtocol != NULL) {
>      return mPciExPlatformProtocol->NotifyDeviceState (
> --
> 2.21.0.windows.1
> 
> 
>