[PATCH 1/5] UefiCpuPkg: Create MpHandOff to transfer info from PEI to DXE.

["Yuanhao Xie" <yuanhao.xie@intel.com>]

The purpose of the Hob characterized by mCpuInitMpLibHobGuid was
twofold: To transfer information from PEI to DXE and acted as a global
variable for PEI. However, during the  DXE phase,only a few fields
from the CPU_MP_DATA which collected in PEI phase were needed.

To enhance this process, a new Hob was specifically created to transfer
information to the DXE phase. This new Hob contained only the essential
fields required for reuse in DXE. Instead of directly including the
BspNumber in MpHandOff, the DXE phase introduced the use of
GetBspNumber() to collect the BspNumber from ApicID and CpuCount.

The addition of MP_HAND_OFF_SIGNAL serves the sole purpose of awakening
 the APs and transitioning their context from PEI to DXE. The
 WaitLoopExecutionMode field indicates whether the bit mode of PEI
 matches that of DXE. Both of these fields are filled only when the
 ApLoopMode is not ApInHltLoop. In the case of ApLoopMode, it remains
 necessary to wake up the APs using the init-sipi-sipi sequence.

The function GetMpHandOffHob() was added to the collected MpHandOff in
the DXE phase. The CpuMpData in the DXE phase was updated by gathering
information from MpHandOff.

Cc: Gerd Hoffmann <kraxel@redhat.com>
Cc: Eric Dong <eric.dong@intel.com>
Cc: Ray Ni <ray.ni@intel.com>
Cc: Rahul Kumar <rahul1.kumar@intel.com>
Cc: Tom Lendacky <thomas.lendacky@amd.com>
Signed-off-by: Yuanhao Xie <yuanhao.xie@intel.com>
---
 UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf |   1 +
 UefiCpuPkg/Library/MpInitLib/MpHandOff.h      |  64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 UefiCpuPkg/Library/MpInitLib/MpLib.c          | 100 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------
 UefiCpuPkg/Library/MpInitLib/MpLib.h          |   2 ++
 UefiCpuPkg/Library/MpInitLib/PeiMpInitLib.inf |   1 +
 UefiCpuPkg/Library/MpInitLib/PeiMpLib.c       |  32 +++++++++++++++++++++++++++++++-
 6 files changed, 186 insertions(+), 14 deletions(-)

diff --git a/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf b/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf
index 67ebf7254c..7d45d3ad4d 100644
--- a/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf
+++ b/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf
@@ -38,6 +38,7 @@
   MpLib.c
   MpLib.h
   Microcode.c
+  MpHandOff.h
 
 [Packages]
   MdePkg/MdePkg.dec
diff --git a/UefiCpuPkg/Library/MpInitLib/MpHandOff.h b/UefiCpuPkg/Library/MpInitLib/MpHandOff.h
new file mode 100644
index 0000000000..65c97d865c
--- /dev/null
+++ b/UefiCpuPkg/Library/MpInitLib/MpHandOff.h
@@ -0,0 +1,64 @@
+/** @file
+  Defines the HOB GUID used to describe the MSEG memory region allocated in PEI.
+  Copyright (c) 2015 - 2023, Intel Corporation. All rights reserved.<BR>
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+**/
+
+#ifndef MP_HANDOFF_H_
+#define MP_HANDOFF_H_
+//
+// To trigger the start-up signal, BSP writes the specified
+// StartupSignalValue to the StartupSignalAddress of each processor.
+// This address is monitored by the APs, and as soon as they receive
+// the value that matches the MP_HAND_OFF_SIGNAL, they will wake up
+// and switch the context from PEI to DXE phase.
+//
+#define MP_HAND_OFF_SIGNAL  SIGNATURE_32 ('M', 'P', 'H', 'O')
+
+#define MP_HANDOFF_GUID \
+  { \
+    0x11e2bd88, 0xed38, 0x4abd, {0xa3, 0x99, 0x21, 0xf2, 0x5f, 0xd0, 0x7a, 0x60 } \
+  }
+extern EFI_GUID  mMpHandOffGuid;
+//
+// The information required to transfer from the PEI phase to the
+// DXE phase is contained within the MP_HAND_OFF and PROCESSOR_HAND_OFF.
+// If the SizeOfPointer (WaitLoopExecutionMode) of both phases are equal,
+// and the APs is not in halt mode,
+// then the APs can be awakened by triggering the start-up
+// signal, rather than using INIT-SIPI-SIPI.
+// To trigger the start-up signal, BSP writes the specified
+// StartupSignalValue to the StartupSignalAddress of each processor.
+// This address is monitored by the APs.
+//
+typedef struct {
+  UINT32    ApicId;
+  UINT32    Health;
+  UINT64    StartupSignalAddress;
+  UINT64    StartupProcedureAddress;
+} PROCESSOR_HAND_OFF;
+
+typedef struct {
+  //
+  // The ProcessorIndex indicates the range of processors. If it is set to 0, it signifies
+  // processors from 0 to CpuCount - 1. Multiple instances in the HOB list describe
+  // processors from ProcessorIndex to ProcessorIndex + CpuCount - 1.
+  //
+  UINT32                ProcessorIndex;
+  UINT32                CpuCount;
+  UINT32                WaitLoopExecutionMode;
+  UINT32                StartupSignalValue;
+  PROCESSOR_HAND_OFF    Info[];
+} MP_HAND_OFF;
+
+/**
+  Get pointer to MP_HAND_OFF GUIDed HOB.
+
+  @return  The pointer to MP_HAND_OFF structure.
+**/
+MP_HAND_OFF *
+GetMpHandOffHob (
+  VOID
+  );
+
+#endif
diff --git a/UefiCpuPkg/Library/MpInitLib/MpLib.c b/UefiCpuPkg/Library/MpInitLib/MpLib.c
index f1f2840714..c55150e7b6 100644
--- a/UefiCpuPkg/Library/MpInitLib/MpLib.c
+++ b/UefiCpuPkg/Library/MpInitLib/MpLib.c
@@ -14,6 +14,7 @@
 #include <Register/Amd/Ghcb.h>
 
 EFI_GUID  mCpuInitMpLibHobGuid = CPU_INIT_MP_LIB_HOB_GUID;
+EFI_GUID  mMpHandOffGuid       = MP_HANDOFF_GUID;
 
 /**
   Save the volatile registers required to be restored following INIT IPI.
@@ -1784,6 +1785,54 @@ CheckAllAPs (
   return EFI_NOT_READY;
 }
 
+/**
+  This function Get BspNumber.
+
+  @param[in] MpHandOff        Pointer to MpHandOff
+  @return                     BspNumber
+**/
+UINT32
+GetBspNumber (
+  IN CONST MP_HAND_OFF  *MpHandOff
+  )
+{
+  UINT32  ApicId;
+  UINT32  BspNumber;
+  UINT32  Index;
+
+  //
+  // Get the processor number for the BSP
+  //
+  BspNumber = MAX_UINT32;
+  ApicId    = GetInitialApicId ();
+  for (Index = 0; Index < MpHandOff->CpuCount; Index++) {
+    if (MpHandOff->Info[Index].ApicId == ApicId) {
+      BspNumber = Index;
+    }
+  }
+
+  ASSERT (BspNumber != MAX_UINT32);
+
+  return BspNumber;
+}
+
+/**
+  Get pointer to CPU MP Data structure from GUIDed HOB.
+
+  @param[in] CpuMpData  The pointer to CPU MP Data structure.
+**/
+VOID
+AmdSevUpdateCpuMpData (
+  IN CPU_MP_DATA  *CpuMpData
+  )
+{
+  CPU_MP_DATA  *OldCpuMpData;
+
+  OldCpuMpData = GetCpuMpDataFromGuidedHob ();
+
+  OldCpuMpData->NewCpuMpData = CpuMpData;
+}
+
 /**
   MP Initialize Library initialization.
 
@@ -1803,7 +1852,7 @@ MpInitLibInitialize (
   VOID
   )
 {
-  CPU_MP_DATA              *OldCpuMpData;
+  MP_HAND_OFF              *MpHandOff;
   CPU_INFO_IN_HOB          *CpuInfoInHob;
   UINT32                   MaxLogicalProcessorNumber;
   UINT32                   ApStackSize;
@@ -1822,11 +1871,11 @@ MpInitLibInitialize (
   UINTN                    BackupBufferAddr;
   UINTN                    ApIdtBase;
 
-  OldCpuMpData = GetCpuMpDataFromGuidedHob ();
-  if (OldCpuMpData == NULL) {
+  MpHandOff = GetMpHandOffHob ();
+  if (MpHandOff == NULL) {
     MaxLogicalProcessorNumber = PcdGet32 (PcdCpuMaxLogicalProcessorNumber);
   } else {
-    MaxLogicalProcessorNumber = OldCpuMpData->CpuCount;
+    MaxLogicalProcessorNumber = MpHandOff->CpuCount;
   }
 
   ASSERT (MaxLogicalProcessorNumber != 0);
@@ -1970,7 +2019,7 @@ MpInitLibInitialize (
   //
   ProgramVirtualWireMode ();
 
-  if (OldCpuMpData == NULL) {
+  if (MpHandOff == NULL) {
     if (MaxLogicalProcessorNumber > 1) {
       //
       // Wakeup all APs and calculate the processor count in system
@@ -1982,15 +2031,18 @@ MpInitLibInitialize (
     // APs have been wakeup before, just get the CPU Information
     // from HOB
     //
-    OldCpuMpData->NewCpuMpData = CpuMpData;
-    CpuMpData->CpuCount        = OldCpuMpData->CpuCount;
-    CpuMpData->BspNumber       = OldCpuMpData->BspNumber;
-    CpuMpData->CpuInfoInHob    = OldCpuMpData->CpuInfoInHob;
-    CpuInfoInHob               = (CPU_INFO_IN_HOB *)(UINTN)CpuMpData->CpuInfoInHob;
+    AmdSevUpdateCpuMpData (CpuMpData);
+    CpuMpData->CpuCount  = MpHandOff->CpuCount;
+    CpuMpData->BspNumber = GetBspNumber (MpHandOff);
+    CpuInfoInHob         = (CPU_INFO_IN_HOB *)(UINTN)CpuMpData->CpuInfoInHob;
     for (Index = 0; Index < CpuMpData->CpuCount; Index++) {
       InitializeSpinLock (&CpuMpData->CpuData[Index].ApLock);
-      CpuMpData->CpuData[Index].CpuHealthy = (CpuInfoInHob[Index].Health == 0) ? TRUE : FALSE;
+      CpuMpData->CpuData[Index].CpuHealthy = (MpHandOff->Info[Index].Health == 0) ? TRUE : FALSE;
       CpuMpData->CpuData[Index].ApFunction = 0;
+      CpuInfoInHob[Index].InitialApicId    = MpHandOff->Info[Index].ApicId;
+      CpuInfoInHob[Index].ApTopOfStack     = CpuMpData->Buffer + (Index + 1) * CpuMpData->CpuApStackSize;
+      CpuInfoInHob[Index].ApicId           = MpHandOff->Info[Index].ApicId;
+      CpuInfoInHob[Index].Health           = MpHandOff->Info[Index].Health;
     }
   }
 
@@ -2019,7 +2071,7 @@ MpInitLibInitialize (
   // Wakeup APs to do some AP initialize sync (Microcode & MTRR)
   //
   if (CpuMpData->CpuCount > 1) {
-    if (OldCpuMpData != NULL) {
+    if (MpHandOff != NULL) {
       //
       // Only needs to use this flag for DXE phase to update the wake up
       // buffer. Wakeup buffer allocated in PEI phase is no longer valid
@@ -2036,7 +2088,7 @@ MpInitLibInitialize (
       CpuPause ();
     }
 
-    if (OldCpuMpData != NULL) {
+    if (MpHandOff != NULL) {
       CpuMpData->InitFlag = ApInitDone;
     }
 
@@ -2763,6 +2815,28 @@ StartupThisAPWorker (
   return Status;
 }
 
+/**
+  Get pointer to MP_HAND_OFF GUIDed HOB.
+
+  @return  The pointer to MP_HAND_OFF structure.
+**/
+MP_HAND_OFF *
+GetMpHandOffHob (
+  VOID
+  )
+{
+  EFI_HOB_GUID_TYPE  *GuidHob;
+  MP_HAND_OFF        *MpHandOff;
+
+  MpHandOff = NULL;
+  GuidHob   = GetFirstGuidHob (&mMpHandOffGuid);
+  if (GuidHob != NULL) {
+    MpHandOff = (MP_HAND_OFF *)GET_GUID_HOB_DATA (GuidHob);
+  }
+
+  return MpHandOff;
+}
+
 /**
   Get pointer to CPU MP Data structure from GUIDed HOB.
 
diff --git a/UefiCpuPkg/Library/MpInitLib/MpLib.h b/UefiCpuPkg/Library/MpInitLib/MpLib.h
index b694c7b40f..9bb70415f3 100644
--- a/UefiCpuPkg/Library/MpInitLib/MpLib.h
+++ b/UefiCpuPkg/Library/MpInitLib/MpLib.h
@@ -40,6 +40,8 @@
 
 #include <Guid/MicrocodePatchHob.h>
 
+#include "MpHandOff.h"
+
 #define WAKEUP_AP_SIGNAL  SIGNATURE_32 ('S', 'T', 'A', 'P')
 
 #define CPU_INIT_MP_LIB_HOB_GUID \
diff --git a/UefiCpuPkg/Library/MpInitLib/PeiMpInitLib.inf b/UefiCpuPkg/Library/MpInitLib/PeiMpInitLib.inf
index 78eded96bf..1aacb9b4cf 100644
--- a/UefiCpuPkg/Library/MpInitLib/PeiMpInitLib.inf
+++ b/UefiCpuPkg/Library/MpInitLib/PeiMpInitLib.inf
@@ -36,6 +36,7 @@
   MpLib.c
   MpLib.h
   Microcode.c
+  MpHandOff.h
 
 [Packages]
   MdePkg/MdePkg.dec
diff --git a/UefiCpuPkg/Library/MpInitLib/PeiMpLib.c b/UefiCpuPkg/Library/MpInitLib/PeiMpLib.c
index 013f89b197..f80e00edcf 100644
--- a/UefiCpuPkg/Library/MpInitLib/PeiMpLib.c
+++ b/UefiCpuPkg/Library/MpInitLib/PeiMpLib.c
@@ -126,7 +126,37 @@ SaveCpuMpData (
   IN CPU_MP_DATA  *CpuMpData
   )
 {
-  UINT64  Data64;
+  UINT64           Data64;
+  UINTN            Index;
+  CPU_INFO_IN_HOB  *CpuInfoInHob;
+  MP_HAND_OFF      *MpHandOff;
+  UINTN            MpHandOffSize;
+
+  //
+  // When APs are in a state that can be waken up by a store operation to a memory address,
+  // report the MP_HAND_OFF data for DXE to use.
+  //
+  CpuInfoInHob  = (CPU_INFO_IN_HOB *)(UINTN)CpuMpData->CpuInfoInHob;
+  MpHandOffSize = sizeof (MP_HAND_OFF) + sizeof (PROCESSOR_HAND_OFF) * CpuMpData->CpuCount;
+  MpHandOff     = (MP_HAND_OFF *)BuildGuidHob (&mMpHandOffGuid, MpHandOffSize);
+  ASSERT (MpHandOff != NULL);
+  ZeroMem (MpHandOff, MpHandOffSize);
+  MpHandOff->ProcessorIndex = 0;
+
+  MpHandOff->CpuCount = CpuMpData->CpuCount;
+  if (CpuMpData->ApLoopMode != ApInHltLoop) {
+    MpHandOff->StartupSignalValue    = MP_HAND_OFF_SIGNAL;
+    MpHandOff->WaitLoopExecutionMode = sizeof (VOID *);
+  }
+
+  for (Index = 0; Index < MpHandOff->CpuCount; Index++) {
+    MpHandOff->Info[Index].ApicId = CpuInfoInHob[Index].ApicId;
+    MpHandOff->Info[Index].Health = CpuInfoInHob[Index].Health;
+    if (CpuMpData->ApLoopMode != ApInHltLoop) {
+      MpHandOff->Info[Index].StartupSignalAddress    = (UINT64)(UINTN)CpuMpData->CpuData[Index].StartupApSignal;
+      MpHandOff->Info[Index].StartupProcedureAddress = (UINT64)(UINTN)&CpuMpData->CpuData[Index].ApFunction;
+    }
+  }
 
   //
   // Build location of CPU MP DATA buffer in HOB
-- 
2.36.1.windows.1