Re: [PATCH V10 14/47] UefiCpuPkg: Enable Tdx support in MpInitLib

["Ni, Ray" <ray.ni@intel.com>]

All look good! 3 minor comments:
1. DxeMpLib.c may not need to include MpIntelTdx.h
2. You may use "ASSERT (!ExcludeBsp)" in below code. Otherwise, it may confuse reader that when ExcludeBsp is TRUE, SUCCESS is returned when ProcessorCount is 1 which is not right.

  if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
    if (!ExcludeBsp) {
      //
      // Start BSP.
      //
      Procedure (ProcedureArgument);
    }

    return EFI_SUCCESS;
  }

3. TdxMpInitLibGetProcessorInfo may not need to call GetProcessorLocationByApicId. You can directly set Package/Core/Thread to 0.

-----Original Message-----
From: Xu, Min M <min.m.xu@intel.com> 
Sent: Thursday, March 24, 2022 8:10 AM
To: devel@edk2.groups.io
Cc: Xu, Min M <min.m.xu@intel.com>; Brijesh Singh <brijesh.singh@amd.com>; Aktas, Erdem <erdemaktas@google.com>; James Bottomley <jejb@linux.ibm.com>; Yao, Jiewen <jiewen.yao@intel.com>; Tom Lendacky <thomas.lendacky@amd.com>; Dong, Eric <eric.dong@intel.com>; Ni, Ray <ray.ni@intel.com>; Kumar, Rahul1 <rahul1.kumar@intel.com>; Gerd Hoffmann <kraxel@redhat.com>
Subject: [PATCH V10 14/47] UefiCpuPkg: Enable Tdx support in MpInitLib

RFC: https://bugzilla.tianocore.org/show_bug.cgi?id=3429

In TDVF BSP and APs are simplified. BSP is the vCPU-0, while the others are treated as APs.

So MP intialization is rather simple. The processor info is retrieved by TDCALL, ApWorker is not supported, BSP is always the working processor, while the APs are just in a wait-for-precedure state.

Cc: Brijesh Singh <brijesh.singh@amd.com>
Cc: Erdem Aktas <erdemaktas@google.com>
Cc: James Bottomley <jejb@linux.ibm.com>
Cc: Jiewen Yao <jiewen.yao@intel.com>
Cc: Tom Lendacky <thomas.lendacky@amd.com>
Cc: Eric Dong <eric.dong@intel.com>
Cc: Ray Ni <ray.ni@intel.com>
Cc: Rahul Kumar <rahul1.kumar@intel.com>
Cc: Gerd Hoffmann <kraxel@redhat.com>
Acked-by: Gerd Hoffmann <kraxel@redhat.com>
Signed-off-by: Min Xu <min.m.xu@intel.com>
---
 UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf |   3 +
 UefiCpuPkg/Library/MpInitLib/DxeMpLib.c       |   2 +-
 UefiCpuPkg/Library/MpInitLib/MpIntelTdx.h     |  71 ++++++++++
 UefiCpuPkg/Library/MpInitLib/MpLib.c          |  60 ++++++++-
 UefiCpuPkg/Library/MpInitLib/MpLibTdx.c       | 125 ++++++++++++++++++
 UefiCpuPkg/Library/MpInitLib/MpLibTdxNull.c   |  73 ++++++++++
 UefiCpuPkg/Library/MpInitLib/PeiMpInitLib.inf |   3 +
 7 files changed, 331 insertions(+), 6 deletions(-)  create mode 100644 UefiCpuPkg/Library/MpInitLib/MpIntelTdx.h
 create mode 100644 UefiCpuPkg/Library/MpInitLib/MpLibTdx.c
 create mode 100644 UefiCpuPkg/Library/MpInitLib/MpLibTdxNull.c

diff --git a/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf b/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf
index e1cd0b350008..159b4d16ed0e 100644
--- a/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf
+++ b/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf
@@ -24,10 +24,12 @@
 [Sources.IA32]
   Ia32/AmdSev.c
   Ia32/MpFuncs.nasm
+  MpLibTdxNull.c
 
 [Sources.X64]
   X64/AmdSev.c
   X64/MpFuncs.nasm
+  MpLibTdx.c
 
 [Sources.common]
   AmdSev.c
@@ -36,6 +38,7 @@
   MpLib.c
   MpLib.h
   Microcode.c
+  MpIntelTdx.h
 
 [Packages]
   MdePkg/MdePkg.dec
diff --git a/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c b/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c
index 60d14a5a0e10..1b2b937e1c8e 100644
--- a/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c
+++ b/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c
@@ -7,6 +7,7 @@
 **/
 
 #include "MpLib.h"
+#include "MpIntelTdx.h"
 
 #include <Library/UefiLib.h>
 #include <Library/UefiBootServicesTableLib.h>
@@ -15,7 +16,6 @@
 #include <Library/VmgExitLib.h>
 #include <Register/Amd/Fam17Msr.h>
 #include <Register/Amd/Ghcb.h>
-
 #include <Protocol/Timer.h>
 
 #define  AP_SAFE_STACK_SIZE  128
diff --git a/UefiCpuPkg/Library/MpInitLib/MpIntelTdx.h b/UefiCpuPkg/Library/MpInitLib/MpIntelTdx.h
new file mode 100644
index 000000000000..b2136f466ce6
--- /dev/null
+++ b/UefiCpuPkg/Library/MpInitLib/MpIntelTdx.h
@@ -0,0 +1,71 @@
+/** @file
+  Intel Tdx header file.
+
+  Copyright (c) 2021, Intel Corporation. All rights reserved.<BR>
+
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+
+**/
+
+#ifndef MP_INTEL_TDX_H_
+#define MP_INTEL_TDX_H_
+
+#include <PiPei.h>
+#include <Library/BaseLib.h>
+#include <Library/DebugLib.h>
+#include <Uefi/UefiBaseType.h>
+#include <Protocol/MpService.h>
+
+/**
+  Gets detailed MP-related information on the requested processor at 
+the
+  instant this call is made. This service may only be called from the BSP.
+
+  @param[in]  ProcessorNumber       The handle number of processor.
+  @param[out] ProcessorInfoBuffer   A pointer to the buffer where information for
+                                    the requested processor is deposited.
+  @param[out]  HealthData            Return processor health data.
+
+  @retval EFI_SUCCESS             Processor information was returned.
+  @retval EFI_DEVICE_ERROR        The calling processor is an AP.
+  @retval EFI_INVALID_PARAMETER   ProcessorInfoBuffer is NULL.
+  @retval EFI_NOT_FOUND           The processor with the handle specified by
+                                  ProcessorNumber does not exist in the platform.
+  @retval EFI_NOT_READY           MP Initialize Library is not initialized.
+
+**/
+EFI_STATUS
+EFIAPI
+TdxMpInitLibGetProcessorInfo (
+  IN  UINTN                      ProcessorNumber,
+  OUT EFI_PROCESSOR_INFORMATION  *ProcessorInfoBuffer,
+  OUT EFI_HEALTH_FLAGS           *HealthData  OPTIONAL
+  );
+
+/**
+  Retrieves the number of logical processor in the platform and the 
+number of
+  those logical processors that are enabled on this boot. This service 
+may only
+  be called from the BSP.
+
+  @param[out] NumberOfProcessors          Pointer to the total number of logical
+                                          processors in the system, including the BSP
+                                          and disabled APs.
+  @param[out] NumberOfEnabledProcessors   Pointer to the number of enabled logical
+                                          processors that exist in system, including
+                                          the BSP.
+
+  @retval EFI_SUCCESS             The number of logical processors and enabled
+                                  logical processors was retrieved.
+  @retval EFI_DEVICE_ERROR        The calling processor is an AP.
+  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL and NumberOfEnabledProcessors
+                                  is NULL.
+  @retval EFI_NOT_READY           MP Initialize Library is not initialized.
+
+**/
+EFI_STATUS
+EFIAPI
+TdxMpInitLibGetNumberOfProcessors (
+  OUT UINTN *NumberOfProcessors, OPTIONAL
+  OUT UINTN                     *NumberOfEnabledProcessors OPTIONAL
+  );
+
+#endif
diff --git a/UefiCpuPkg/Library/MpInitLib/MpLib.c b/UefiCpuPkg/Library/MpInitLib/MpLib.c
index 4a73787ee43a..469dda256b8b 100644
--- a/UefiCpuPkg/Library/MpInitLib/MpLib.c
+++ b/UefiCpuPkg/Library/MpInitLib/MpLib.c
@@ -9,9 +9,11 @@
 **/
 
 #include "MpLib.h"
+#include "MpIntelTdx.h"
 #include <Library/VmgExitLib.h>
 #include <Register/Amd/Fam17Msr.h>
 #include <Register/Amd/Ghcb.h>
+#include <ConfidentialComputingGuestAttr.h>
 
 EFI_GUID  mCpuInitMpLibHobGuid = CPU_INIT_MP_LIB_HOB_GUID;
 
@@ -1803,6 +1805,10 @@ MpInitLibInitialize (
   UINTN                    BackupBufferAddr;
   UINTN                    ApIdtBase;
 
+  if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
+    return EFI_SUCCESS;
+  }
+
   OldCpuMpData = GetCpuMpDataFromGuidedHob ();
   if (OldCpuMpData == NULL) {
     MaxLogicalProcessorNumber = PcdGet32 (PcdCpuMaxLogicalProcessorNumber);
@@ -2073,6 +2079,10 @@ MpInitLibGetProcessorInfo (
   CPU_INFO_IN_HOB  *CpuInfoInHob;
   UINTN            OriginalProcessorNumber;
 
+  if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
+    return TdxMpInitLibGetProcessorInfo (ProcessorNumber, 
+ ProcessorInfoBuffer, HealthData);  }
+
   CpuMpData    = GetCpuMpData ();
   CpuInfoInHob = (CPU_INFO_IN_HOB *)(UINTN)CpuMpData->CpuInfoInHob;
 
@@ -2167,6 +2177,10 @@ SwitchBSPWorker (
   BOOLEAN                      OldInterruptState;
   BOOLEAN                      OldTimerInterruptState;
 
+  if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
+    return EFI_UNSUPPORTED;
+  }
+
   //
   // Save and Disable Local APIC timer interrupt
   //
@@ -2307,6 +2321,10 @@ EnableDisableApWorker (
   CPU_MP_DATA  *CpuMpData;
   UINTN        CallerNumber;
 
+  if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
+    return EFI_UNSUPPORTED;
+  }
+
   CpuMpData = GetCpuMpData ();
 
   //
@@ -2367,6 +2385,11 @@ MpInitLibWhoAmI (
     return EFI_INVALID_PARAMETER;
   }
 
+  if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
+    *ProcessorNumber = 0;
+    return EFI_SUCCESS;
+  }
+
   CpuMpData = GetCpuMpData ();
 
   return GetProcessorNumber (CpuMpData, ProcessorNumber); @@ -2405,12 +2428,16 @@ MpInitLibGetNumberOfProcessors (
   UINTN        EnabledProcessorNumber;
   UINTN        Index;
 
-  CpuMpData = GetCpuMpData ();
-
   if ((NumberOfProcessors == NULL) && (NumberOfEnabledProcessors == NULL)) {
     return EFI_INVALID_PARAMETER;
   }
 
+  if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
+    return TdxMpInitLibGetNumberOfProcessors (NumberOfProcessors, 
+ NumberOfEnabledProcessors);  }
+
+  CpuMpData = GetCpuMpData ();
+
   //
   // Check whether caller processor is BSP
   //
@@ -2490,13 +2517,16 @@ StartupAllCPUsWorker (
   BOOLEAN      HasEnabledAp;
   CPU_STATE    ApState;
 
-  CpuMpData = GetCpuMpData ();
-
   if (FailedCpuList != NULL) {
     *FailedCpuList = NULL;
   }
 
-  if ((CpuMpData->CpuCount == 1) && ExcludeBsp) {
+  Status = MpInitLibGetNumberOfProcessors (&ProcessorCount, NULL);  if 
+ (EFI_ERROR (Status)) {
+    return Status;
+  }
+
+  if ((ProcessorCount == 1) && ExcludeBsp) {
     return EFI_NOT_STARTED;
   }
 
@@ -2504,6 +2534,19 @@ StartupAllCPUsWorker (
     return EFI_INVALID_PARAMETER;
   }
 
+  if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
+    if (!ExcludeBsp) {
+      //
+      // Start BSP.
+      //
+      Procedure (ProcedureArgument);
+    }
+
+    return EFI_SUCCESS;
+  }
+
+  CpuMpData = GetCpuMpData ();
+
   //
   // Check whether caller processor is BSP
   //
@@ -2643,6 +2686,13 @@ StartupThisAPWorker (
   CPU_AP_DATA  *CpuData;
   UINTN        CallerNumber;
 
+  //
+  // In Td guest, startup of AP is not supported in current stage.
+  //
+  if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
+    return EFI_UNSUPPORTED;
+  }
+
   CpuMpData = GetCpuMpData ();
 
   if (Finished != NULL) {
diff --git a/UefiCpuPkg/Library/MpInitLib/MpLibTdx.c b/UefiCpuPkg/Library/MpInitLib/MpLibTdx.c
new file mode 100644
index 000000000000..1a8d7d6dab64
--- /dev/null
+++ b/UefiCpuPkg/Library/MpInitLib/MpLibTdx.c
@@ -0,0 +1,125 @@
+/** @file
+  CPU MP Initialize Library common functions.
+
+  Copyright (c) 2016 - 2020, Intel Corporation. All rights 
+ reserved.<BR>  Copyright (c) 2020, AMD Inc. All rights reserved.<BR>
+
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+
+**/
+
+#include "MpLib.h"
+#include "MpIntelTdx.h"
+#include <Library/BaseLib.h>
+#include <Library/PcdLib.h>
+#include <Library/DebugLib.h>
+#include <IndustryStandard/Tdx.h>
+
+/**
+  Gets detailed MP-related information on the requested processor at 
+the
+  instant this call is made. This service may only be called from the BSP.
+
+  In current stage only the BSP is workable. So ProcessorNumber should be 0.
+
+  @param[in]  ProcessorNumber       The handle number of processor.
+  @param[out] ProcessorInfoBuffer   A pointer to the buffer where information for
+                                    the requested processor is deposited.
+  @param[out]  HealthData            Return processor health data.
+
+  @retval EFI_SUCCESS             Processor information was returned.
+  @retval EFI_DEVICE_ERROR        The calling processor is an AP.
+  @retval EFI_INVALID_PARAMETER   ProcessorInfoBuffer is NULL or ProcessorNumber is not 0.
+  @retval EFI_NOT_FOUND           The processor with the handle specified by
+                                  ProcessorNumber does not exist in the platform.
+  @retval EFI_NOT_READY           MP Initialize Library is not initialized.
+
+**/
+EFI_STATUS
+EFIAPI
+TdxMpInitLibGetProcessorInfo (
+  IN  UINTN                      ProcessorNumber,
+  OUT EFI_PROCESSOR_INFORMATION  *ProcessorInfoBuffer,
+  OUT EFI_HEALTH_FLAGS           *HealthData  OPTIONAL
+  )
+{
+  EFI_STATUS      Status;
+  TD_RETURN_DATA  TdReturnData;
+
+  if ((ProcessorInfoBuffer == NULL) || (ProcessorNumber != 0)) {
+    return EFI_INVALID_PARAMETER;
+  }
+
+  Status = TdCall (TDCALL_TDINFO, 0, 0, 0, &TdReturnData);  ASSERT 
+ (Status == EFI_SUCCESS);
+
+  if (ProcessorNumber >= TdReturnData.TdInfo.NumVcpus) {
+    return EFI_NOT_FOUND;
+  }
+
+  ProcessorInfoBuffer->ProcessorId = ProcessorNumber;  
+ ProcessorInfoBuffer->StatusFlag  = 0;  if (ProcessorNumber == 0) {
+    ProcessorInfoBuffer->StatusFlag |= PROCESSOR_AS_BSP_BIT;  }
+
+  ProcessorInfoBuffer->StatusFlag |= PROCESSOR_ENABLED_BIT;
+
+  //
+  // Get processor location information  //  
+ GetProcessorLocationByApicId (
+    (UINT32)ProcessorNumber,
+    &ProcessorInfoBuffer->Location.Package,
+    &ProcessorInfoBuffer->Location.Core,
+    &ProcessorInfoBuffer->Location.Thread
+    );
+
+  if (HealthData != NULL) {
+    HealthData->Uint32 = 0;
+  }
+
+  return Status;
+}
+
+/**
+  Retrieves the number of logical processor in the platform and the 
+number of
+  those logical processors that are enabled on this boot. This service 
+may only
+  be called from the BSP.
+
+  @param[out] NumberOfProcessors          Pointer to the total number of logical
+                                          processors in the system, including the BSP
+                                          and disabled APs.
+  @param[out] NumberOfEnabledProcessors   Pointer to the number of enabled logical
+                                          processors that exist in system, including
+                                          the BSP.
+
+  @retval EFI_SUCCESS             The number of logical processors and enabled
+                                  logical processors was retrieved.
+  @retval EFI_DEVICE_ERROR        The calling processor is an AP.
+  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL and NumberOfEnabledProcessors
+                                  is NULL.
+  @retval EFI_NOT_READY           MP Initialize Library is not initialized.
+
+**/
+EFI_STATUS
+EFIAPI
+TdxMpInitLibGetNumberOfProcessors (
+  OUT UINTN *NumberOfProcessors, OPTIONAL
+  OUT UINTN *NumberOfEnabledProcessors OPTIONAL
+  )
+{
+  ASSERT (NumberOfProcessors != NULL || NumberOfEnabledProcessors != 
+NULL);
+  //
+  // In current stage only the BSP is workable. So NumberOfProcessors
+  // & NumberOfEnableddProcessors are both 1.
+  //
+  if (NumberOfProcessors != NULL) {
+    *NumberOfProcessors = 1;
+  }
+
+  if (NumberOfEnabledProcessors != NULL) {
+    *NumberOfEnabledProcessors = 1;
+  }
+
+  return EFI_SUCCESS;
+}
diff --git a/UefiCpuPkg/Library/MpInitLib/MpLibTdxNull.c b/UefiCpuPkg/Library/MpInitLib/MpLibTdxNull.c
new file mode 100644
index 000000000000..2849e13f2aaa
--- /dev/null
+++ b/UefiCpuPkg/Library/MpInitLib/MpLibTdxNull.c
@@ -0,0 +1,73 @@
+/** @file
+  CPU MP Initialize Library common functions.
+
+  Copyright (c) 2016 - 2020, Intel Corporation. All rights 
+ reserved.<BR>  Copyright (c) 2020, AMD Inc. All rights reserved.<BR>
+
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+
+**/
+
+#include "MpLib.h"
+#include "MpIntelTdx.h"
+#include <Library/DebugLib.h>
+
+/**
+  Gets detailed MP-related information on the requested processor at 
+the
+  instant this call is made. This service may only be called from the BSP.
+
+  @param[in]  ProcessorNumber       The handle number of processor.
+  @param[out] ProcessorInfoBuffer   A pointer to the buffer where information for
+                                    the requested processor is deposited.
+  @param[out]  HealthData            Return processor health data.
+
+  @retval EFI_SUCCESS             Processor information was returned.
+  @retval EFI_DEVICE_ERROR        The calling processor is an AP.
+  @retval EFI_INVALID_PARAMETER   ProcessorInfoBuffer is NULL.
+  @retval EFI_NOT_FOUND           The processor with the handle specified by
+                                  ProcessorNumber does not exist in the platform.
+  @retval EFI_NOT_READY           MP Initialize Library is not initialized.
+
+**/
+EFI_STATUS
+EFIAPI
+TdxMpInitLibGetProcessorInfo (
+  IN  UINTN                      ProcessorNumber,
+  OUT EFI_PROCESSOR_INFORMATION  *ProcessorInfoBuffer,
+  OUT EFI_HEALTH_FLAGS           *HealthData  OPTIONAL
+  )
+{
+  ASSERT (FALSE);
+  return EFI_UNSUPPORTED;
+}
+
+/**
+  Retrieves the number of logical processor in the platform and the 
+number of
+  those logical processors that are enabled on this boot. This service 
+may only
+  be called from the BSP.
+
+  @param[out] NumberOfProcessors          Pointer to the total number of logical
+                                          processors in the system, including the BSP
+                                          and disabled APs.
+  @param[out] NumberOfEnabledProcessors   Pointer to the number of enabled logical
+                                          processors that exist in system, including
+                                          the BSP.
+
+  @retval EFI_SUCCESS             The number of logical processors and enabled
+                                  logical processors was retrieved.
+  @retval EFI_DEVICE_ERROR        The calling processor is an AP.
+  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL and NumberOfEnabledProcessors
+                                  is NULL.
+  @retval EFI_NOT_READY           MP Initialize Library is not initialized.
+
+**/
+EFI_STATUS
+EFIAPI
+TdxMpInitLibGetNumberOfProcessors (
+  OUT UINTN *NumberOfProcessors, OPTIONAL
+  OUT UINTN                     *NumberOfEnabledProcessors OPTIONAL
+  )
+{
+  ASSERT (FALSE);
+  return EFI_UNSUPPORTED;
+}
diff --git a/UefiCpuPkg/Library/MpInitLib/PeiMpInitLib.inf b/UefiCpuPkg/Library/MpInitLib/PeiMpInitLib.inf
index 5facf4db9499..894be0f8daab 100644
--- a/UefiCpuPkg/Library/MpInitLib/PeiMpInitLib.inf
+++ b/UefiCpuPkg/Library/MpInitLib/PeiMpInitLib.inf
@@ -24,10 +24,12 @@
 [Sources.IA32]
   Ia32/AmdSev.c
   Ia32/MpFuncs.nasm
+  MpLibTdxNull.c
 
 [Sources.X64]
   X64/AmdSev.c
   X64/MpFuncs.nasm
+  MpLibTdx.c
 
 [Sources.common]
   AmdSev.c
@@ -36,6 +38,7 @@
   MpLib.c
   MpLib.h
   Microcode.c
+  MpIntelTdx.h
 
 [Packages]
   MdePkg/MdePkg.dec
--
2.29.2.windows.2