[Patch 0/2] Enable new MM MP protocol.

[Patch 0/2] Enable new MM MP protocol.

[Dong, Eric]

PI spec added a new protocol named MM MP protocol. This protocol allows
for better remote queuing of execution of procedures on an AP.
This extends the existing procedures to allow:
1. A function to be called in blocking and non-blocking manner explicitly
2. Allow broadcasts.
3. Allow execution of a procedure when a processor powers up.

This patch serial enable this new protocol.

Cc: Ray Ni <ray.ni@intel.com>
Cc: Laszlo Ersek <lersek@redhat.com>

Eric Dong (2):
  MdePkg: Add new MM MP Protocol definition.
  UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP Protocol.

 MdePkg/Include/Pi/PiMultiPhase.h             |  16 +
 MdePkg/Include/Protocol/MmMp.h               | 334 +++++++++++++
 MdePkg/Include/Protocol/SmmMp.h              |  44 ++
 MdePkg/MdePkg.dec                            |   6 +
 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c       | 375 +++++++++++++++
 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h       | 283 +++++++++++
 UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c        | 468 ++++++++++++++++++-
 UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c   |  11 +
 UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h   | 172 ++++++-
 UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf |   3 +
 10 files changed, 1696 insertions(+), 16 deletions(-)
 create mode 100644 MdePkg/Include/Protocol/MmMp.h
 create mode 100644 MdePkg/Include/Protocol/SmmMp.h
 create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
 create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h

-- 
2.21.0.windows.1

[Patch 1/2] MdePkg: Add new MM MP Protocol definition.

[Dong, Eric]

EFI MM MP Protocol is defined in the PI 1.7 specification.

The MM MP protocol provides a set of functions to allow execution of
procedures on processors that have entered MM. This protocol has the
following properties:
1. The caller can invoke execution of a procedure on a processor, other
than the caller, that has also entered MM. Supports blocking and
non-blocking modes of operation.
2. The caller can invoke a procedure on multiple processors. Supports
blocking and non-blocking modes of operation.

Cc: Ray Ni <ray.ni@intel.com>
Cc: Laszlo Ersek <lersek@redhat.com>
Signed-off-by: Eric Dong <eric.dong@intel.com>
---
 MdePkg/Include/Pi/PiMultiPhase.h |  16 ++
 MdePkg/Include/Protocol/MmMp.h   | 334 +++++++++++++++++++++++++++++++
 MdePkg/Include/Protocol/SmmMp.h  |  44 ++++
 MdePkg/MdePkg.dec                |   6 +
 4 files changed, 400 insertions(+)
 create mode 100644 MdePkg/Include/Protocol/MmMp.h
 create mode 100644 MdePkg/Include/Protocol/SmmMp.h

diff --git a/MdePkg/Include/Pi/PiMultiPhase.h b/MdePkg/Include/Pi/PiMultiPhase.h
index eb12527767..a5056799e1 100644
--- a/MdePkg/Include/Pi/PiMultiPhase.h
+++ b/MdePkg/Include/Pi/PiMultiPhase.h
@@ -176,4 +176,20 @@ VOID
   IN OUT VOID  *Buffer
   );
 
+/**
+  The function prototype for invoking a function on an Application Processor.
+
+  This definition is used by the UEFI MM MP Serices Protocol.
+
+  @param[in] ProcedureArgument    The pointer to private data buffer.
+
+  @retval EFI_SUCCESS             Excutive the procedure successfully
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_AP_PROCEDURE2)(
+  IN VOID  *ProcedureArgument
+);
+
 #endif
diff --git a/MdePkg/Include/Protocol/MmMp.h b/MdePkg/Include/Protocol/MmMp.h
new file mode 100644
index 0000000000..0467072bec
--- /dev/null
+++ b/MdePkg/Include/Protocol/MmMp.h
@@ -0,0 +1,334 @@
+/** @file
+  EFI MM MP Protocol is defined in the PI 1.5 specification.
+
+  The MM MP protocol provides a set of functions to allow execution of procedures on processors that
+  have entered MM. This protocol has the following properties:
+  1. The caller can only invoke execution of a procedure on a processor, other than the caller, that
+     has also entered MM.
+  2. It is possible to invoke a procedure on multiple processors. Supports blocking and non-blocking
+     modes of operation.
+
+  Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+
+**/
+
+#ifndef _MM_MP_H_
+#define _MM_MP_H_
+
+#include <Pi/PiMmCis.h>
+
+#define EFI_MM_MP_PROTOCOL_GUID \
+  { \
+    0x5d5450d7, 0x990c, 0x4180, {0xa8, 0x3, 0x8e, 0x63, 0xf0, 0x60, 0x83, 0x7  }  \
+  }
+
+//
+// Revision definition.
+//
+#define EFI_MM_MP_PROTOCOL_REVISION    0x00
+
+//
+// Attribute flags
+//
+#define EFI_MM_MP_TIMEOUT_SUPPORTED    0x01
+
+//
+// Completion token
+//
+typedef VOID* MM_COMPLETION;
+
+typedef struct {
+  MM_COMPLETION  Completion;
+  EFI_STATUS     Status;
+} MM_DISPATCH_COMPLETION_TOKEN;
+
+typedef struct _EFI_MM_MP_PROTOCOL  EFI_MM_MP_PROTOCOL;
+
+/**
+  Service to retrieves the number of logical processor in the platform.
+
+  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
+  @param[out] NumberOfProcessors  Pointer to the total number of logical processors in the system,
+                                  including the BSP and all APs.
+
+  @retval EFI_SUCCESS             The number of processors was retrieved successfully
+  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MM_GET_NUMBER_OF_PROCESSORS) (
+  IN CONST EFI_MM_MP_PROTOCOL  *This,
+  OUT      UINTN               *NumberOfProcessors
+);
+
+
+/**
+  This service allows the caller to invoke a procedure one of the application processors (AP). This
+  function uses an optional token parameter to support blocking and non-blocking modes. If the token
+  is passed into the call, the function will operate in a non-blocking fashion and the caller can
+  check for completion with CheckOnProcedure or WaitForProcedure.
+
+  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
+  @param[in]     Procedure              A pointer to the procedure to be run on the designated target
+                                        AP of the system. Type EFI_AP_PROCEDURE2 is defined below in
+                                        related definitions.
+  @param[in]     CpuNumber              The zero-based index of the processor number of the target
+                                        AP, on which the code stream is supposed to run. If the number
+                                        points to the calling processor then it will not run the
+                                        supplied code.
+  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for this AP to
+                                        finish execution of Procedure, either for blocking or
+                                        non-blocking mode. Zero means infinity. If the timeout
+                                        expires before this AP returns from Procedure, then Procedure
+                                        on the AP is terminated. If the timeout expires in blocking
+                                        mode, the call returns EFI_TIMEOUT. If the timeout expires
+                                        in non-blocking mode, the timeout determined can be through
+                                        CheckOnProcedure or WaitForProcedure.
+                                        Note that timeout support is optional. Whether an
+                                        implementation supports this feature, can be determined via
+                                        the Attributes data member.
+  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
+                                        that is run by the AP. It is an optional common mailbox
+                                        between APs and the caller to share information.
+  @param[in,out] Token                  This is parameter is broken into two components:
+                                        1.Token->Completion is an optional parameter that allows the
+                                        caller to execute the procedure in a blocking or non-blocking
+                                        fashion. If it is NULL the call is blocking, and the call will
+                                        not return until the AP has completed the procedure. If the
+                                        token is not NULL, the call will return immediately. The caller
+                                        can check whether the procedure has completed with
+                                        CheckOnProcedure or WaitForProcedure.
+                                        2.Token->Status The implementation updates the address pointed
+                                        at by this variable with the status code returned by Procedure
+                                        when it completes execution on the target AP, or with EFI_TIMEOUT
+                                        if the Procedure fails to complete within the optional timeout.
+                                        The implementation will update this variable with EFI_NOT_READY
+                                        prior to starting Procedure on the target AP.
+  @param[in,out] CPUStatus              This optional pointer may be used to get the status code returned
+                                        by Procedure when it completes execution on the target AP, or with
+                                        EFI_TIMEOUT if the Procedure fails to complete within the optional
+                                        timeout. The implementation will update this variable with
+                                        EFI_NOT_READY prior to starting Procedure on the target AP.
+
+  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
+                                        execution on the target AP.
+                                        In the non-blocking case this indicates that the procedure has
+                                        been successfully scheduled for execution on the target AP.
+  @retval EFI_INVALID_PARAMETER         The input arguments are out of range. Either the target AP is the
+                                        caller of the function, or the Procedure or Token is NULL
+  @retval EFI_NOT_READY                 If the target AP is busy executing another procedure
+  @retval EFI_ALREADY_STARTED           Token is already in use for another procedure
+  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before the specified AP
+                                        has finished
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MM_DISPATCH_PROCEDURE) (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       EFI_AP_PROCEDURE2             Procedure,
+  IN       UINTN                         CpuNumber,
+  IN       UINTN                         TimeoutInMicroseconds,
+  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
+  IN OUT   MM_COMPLETION                 *Token,
+  IN OUT   EFI_STATUS                    *CPUStatus
+);
+
+/**
+  This service allows the caller to invoke a procedure on all running application processors (AP)
+  except the caller. This function uses an optional token parameter to support blocking and
+  nonblocking modes. If the token is passed into the call, the function will operate in a non-blocking
+  fashion and the caller can check for completion with CheckOnProcedure or WaitForProcedure.
+
+  It is not necessary for the implementation to run the procedure on every processor on the platform.
+  Processors that are powered down in such a way that they cannot respond to interrupts, may be
+  excluded from the broadcast.
+
+
+  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
+  @param[in]     Procedure              A pointer to the code stream to be run on the APs that have
+                                        entered MM. Type EFI_AP_PROCEDURE is defined below in related
+                                        definitions.
+  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for the APs to finish
+                                        execution of Procedure, either for blocking or non-blocking mode.
+                                        Zero means infinity. If the timeout expires before all APs return
+                                        from Procedure, then Procedure on the failed APs is terminated. If
+                                        the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
+                                        If the timeout expires in non-blocking mode, the timeout determined
+                                        can be through CheckOnProcedure or WaitForProcedure.
+                                        Note that timeout support is optional. Whether an implementation
+                                        supports this feature can be determined via the Attributes data
+                                        member.
+  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
+                                        that is run by the AP. It is an optional common mailbox
+                                        between APs and the caller to share information.
+  @param[in,out] Token                  This is parameter is broken into two components:
+                                        1.Token->Completion is an optional parameter that allows the
+                                        caller to execute the procedure in a blocking or non-blocking
+                                        fashion. If it is NULL the call is blocking, and the call will
+                                        not return until the AP has completed the procedure. If the
+                                        token is not NULL, the call will return immediately. The caller
+                                        can check whether the procedure has completed with
+                                        CheckOnProcedure or WaitForProcedure.
+                                        2.Token->Status The implementation updates the address pointed
+                                        at by this variable with the status code returned by Procedure
+                                        when it completes execution on the target AP, or with EFI_TIMEOUT
+                                        if the Procedure fails to complete within the optional timeout.
+                                        The implementation will update this variable with EFI_NOT_READY
+                                        prior to starting Procedure on the target AP
+  @param[in,out] CPUStatus              This optional pointer may be used to get the individual status
+                                        returned by every AP that participated in the broadcast. This
+                                        parameter if used provides the base address of an array to hold
+                                        the EFI_STATUS value of each AP in the system. The size of the
+                                        array can be ascertained by the GetNumberOfProcessors function.
+                                        As mentioned above, the broadcast may not include every processor
+                                        in the system. Some implementations may exclude processors that
+                                        have been powered down in such a way that they are not responsive
+                                        to interrupts. Additionally the broadcast excludes the processor
+                                        which is making the BroadcastProcedure call. For every excluded
+                                        processor, the array entry must contain a value of EFI_NOT_STARTED
+
+  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
+                                        execution on the APs. In the non-blocking case this indicates that
+                                        the procedure has been successfully scheduled for execution on the
+                                        APs.
+  @retval EFI_INVALID_PARAMETER         Procedure or Token is NULL.
+  @retval EFI_NOT_READY                 If a target AP is busy executing another procedure.
+  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before all enabled APs have
+                                        finished.
+  @retval EFI_ALREADY_STARTED           Before the AP procedure associated with the Token is finished, the
+                                        same Token cannot be used to dispatch or broadcast another procedure.
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MM_BROADCAST_PROCEDURE) (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       EFI_AP_PROCEDURE2             Procedure,
+  IN       UINTN                         TimeoutInMicroseconds,
+  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
+  IN OUT   MM_COMPLETION                 *Token,
+  IN OUT   EFI_STATUS                    *CPUStatus
+);
+
+
+/**
+  This service allows the caller to set a startup procedure that will be executed when an AP powers
+  up from a state where core configuration and context is lost. The procedure is execution has the
+  following properties:
+  1. The procedure executes before the processor is handed over to the operating system.
+  2. All processors execute the same startup procedure.
+  3. The procedure may run in parallel with other procedures invoked through the functions in this
+  protocol, or with processors that are executing an MM handler or running in the operating system.
+
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Procedure            A pointer to the code stream to be run on the designated target AP
+                                       of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
+                                       with the related definitions of
+                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
+                                       If caller may pass a value of NULL to deregister any existing
+                                       startup procedure.
+  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of parameters to the code that is
+                                       run by the AP. It is an optional common mailbox between APs and
+                                       the caller to share information
+
+  @retval EFI_SUCCESS                  The Procedure has been set successfully.
+  @retval EFI_INVALID_PARAMETER        The Procedure is NULL.
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MM_SET_STARTUP_PROCEDURE) (
+  IN CONST EFI_MM_MP_PROTOCOL *This,
+  IN       EFI_AP_PROCEDURE   Procedure,
+  IN OUT   VOID               *ProcedureArguments OPTIONAL
+);
+
+/**
+  When non-blocking execution of a procedure on an AP is invoked with DispatchProcedure,
+  via the use of a token, this function can be used to check for completion of the procedure on the AP.
+  The function takes the token that was passed into the DispatchProcedure call. If the procedure
+  is complete, and therefore it is now possible to run another procedure on the same AP, this function
+  returns EFI_SUCESS. In this case the status returned by the procedure that executed on the AP is
+  returned in the token's Status field. If the procedure has not yet completed, then this function
+  returns EFI_NOT_READY.
+
+  When a non-blocking execution of a procedure is invoked with BroadcastProcedure, via the
+  use of a token, this function can be used to check for completion of the procedure on all the
+  broadcast APs. The function takes the token that was passed into the BroadcastProcedure
+  call. If the procedure is complete on all broadcast APs this function returns EFI_SUCESS. In this
+  case the Status field in the token passed into the function reflects the overall result of the
+  invocation, which may be EFI_SUCCESS, if all executions succeeded, or the first observed failure.
+  If the procedure has not yet completed on the broadcast APs, the function returns
+  EFI_NOT_READY.
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Token                This parameter describes the token that was passed into
+                                       DispatchProcedure or BroadcastProcedure.
+
+  @retval EFI_SUCCESS                  Procedure has completed.
+  @retval EFI_NOT_READY                The Procedure has not completed.
+  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is NULL
+  @retval EFI_NOT_FOUND                Token is not currently in use for a non-blocking call
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_CHECK_FOR_PROCEDURE) (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       MM_COMPLETION                 Token
+);
+
+/**
+  When a non-blocking execution of a procedure on an AP is invoked via DispatchProcedure,
+  this function will block the caller until the remote procedure has completed on the designated AP.
+  The non-blocking procedure invocation is identified by the Token parameter, which must match the
+  token that used when DispatchProcedure was called. Upon completion the status returned by
+  the procedure that executed on the AP is used to update the token's Status field.
+
+  When a non-blocking execution of a procedure on an AP is invoked via BroadcastProcedure
+  this function will block the caller until the remote procedure has completed on all of the APs that
+  entered MM. The non-blocking procedure invocation is identified by the Token parameter, which
+  must match the token that used when BroadcastProcedure was called. Upon completion the
+  overall status returned by the procedures that executed on the broadcast AP is used to update the
+  token's Status field. The overall status may be EFI_SUCCESS, if all executions succeeded, or the
+  first observed failure.
+
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Token                This parameter describes the token that was passed into
+                                       DispatchProcedure or BroadcastProcedure.
+
+  @retval EFI_SUCCESS                  Procedure has completed.
+  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is NULL
+  @retval EFI_NOT_FOUND                Token is not currently in use for a non-blocking call
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_WAIT_FOR_PROCEDURE) (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       MM_COMPLETION                 Token
+);
+
+
+
+///
+/// The MM MP protocol provides a set of functions to allow execution of procedures on processors that
+/// have entered MM.
+///
+struct _EFI_MM_MP_PROTOCOL {
+  UINT32                            Revision;
+  UINT32                            Attributes;
+  EFI_MM_GET_NUMBER_OF_PROCESSORS   GetNumberOfProcessors;
+  EFI_MM_DISPATCH_PROCEDURE         DispatchProcedure;
+  EFI_MM_BROADCAST_PROCEDURE        BroadcastProcedure;
+  EFI_MM_SET_STARTUP_PROCEDURE      SetStartupProcedure;
+  EFI_CHECK_FOR_PROCEDURE           CheckForProcedure;
+  EFI_WAIT_FOR_PROCEDURE            WaitForProcedure;
+};
+
+extern EFI_GUID gEfiMmMpProtocolGuid;
+
+#endif
+
diff --git a/MdePkg/Include/Protocol/SmmMp.h b/MdePkg/Include/Protocol/SmmMp.h
new file mode 100644
index 0000000000..5f391a3860
--- /dev/null
+++ b/MdePkg/Include/Protocol/SmmMp.h
@@ -0,0 +1,44 @@
+/** @file
+  EFI SMM MP Protocol is defined in the PI 1.5 specification.
+
+  The SMM MP protocol provides a set of functions to allow execution of procedures on processors that
+  have entered SMM. This protocol has the following properties:
+  1. The caller can only invoke execution of a procedure on a processor, other than the caller, that
+     has also entered SMM.
+  2. It is possible to invoke a procedure on multiple processors. Supports blocking and non-blocking
+     modes of operation.
+
+  Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+
+**/
+
+#ifndef _SMM_MP_H_
+#define _SMM_MP_H_
+
+#include <Protocol/MmMp.h>
+
+#define EFI_SMM_MP_PROTOCOL_GUID EFI_MM_MP_PROTOCOL_GUID
+
+#define EFI_SMM_MP_TIMEOUT_SUPPORTED EFI_MM_MP_TIMEOUT_SUPPORTED
+
+#define EFI_SMM_MP_PROTOCOL_REVISION    EFI_MM_MP_PROTOCOL_REVISION
+
+typedef MM_COMPLETION SMM_COMPLETION;
+
+typedef MM_DISPATCH_COMPLETION_TOKEN SMM_DISPATCH_COMPLETION_TOKEN;
+
+typedef EFI_MM_MP_PROTOCOL  EFI_SMM_MP_PROTOCOL;
+
+typedef EFI_MM_GET_NUMBER_OF_PROCESSORS EFI_SMM_GET_NUMBER_OF_PROCESSORS;
+
+typedef EFI_MM_DISPATCH_PROCEDURE EFI_SMM_DISPATCH_PROCEDURE;
+
+typedef EFI_MM_BROADCAST_PROCEDURE EFI_SMM_BROADCAST_PROCEDURE;
+
+typedef EFI_MM_SET_STARTUP_PROCEDURE EFI_SMM_SET_STARTUP_PROCEDURE;
+
+extern EFI_GUID gEfiSmmMpProtocolGuid;
+
+#endif
+
diff --git a/MdePkg/MdePkg.dec b/MdePkg/MdePkg.dec
index 6c563375ee..0b8da9f2cb 100644
--- a/MdePkg/MdePkg.dec
+++ b/MdePkg/MdePkg.dec
@@ -1167,6 +1167,12 @@
   # Protocols defined in PI 1.5.
   #
 
+  ## Include/Protocol/SmmMp.h
+  gEfiSmmMpProtocolGuid = { 0x5d5450d7, 0x990c, 0x4180, { 0xa8, 0x3, 0x8e, 0x63, 0xf0, 0x60, 0x83, 0x7 }}
+
+  ## Include/Protocol/MmMp.h
+  gEfiMmMpProtocolGuid = { 0x5d5450d7, 0x990c, 0x4180, { 0xa8, 0x3, 0x8e, 0x63, 0xf0, 0x60, 0x83, 0x7 }}
+
   ## Include/Protocol/MmEndOfDxe.h
   gEfiMmEndOfDxeProtocolGuid = { 0x24e70042, 0xd5c5, 0x4260, { 0x8c, 0x39, 0xa, 0xd3, 0xaa, 0x32, 0xe9, 0x3d }}
 
-- 
2.21.0.windows.1

Re: [edk2-devel] [Patch 1/2] MdePkg: Add new MM MP Protocol definition.

[Laszlo Ersek]

Hi Eric,

On 06/19/19 07:51, Dong, Eric wrote:
> EFI MM MP Protocol is defined in the PI 1.7 specification.

I can only offer regression-testing for this series. I'll report back
with such results hopefully soon.

Regarding the above sentence. I think we should clarify it. The MM MP
protocol is present in PI 1.6 already (possibly in even earlier PI
releases). But then it was amended in PI 1.7.

In fact, the file-level comments in your code below reference "PI 1.5".
So please update the above statement, in the commit message.

Thanks
Laszlo

> The MM MP protocol provides a set of functions to allow execution of
> procedures on processors that have entered MM. This protocol has the
> following properties:
> 1. The caller can invoke execution of a procedure on a processor, other
> than the caller, that has also entered MM. Supports blocking and
> non-blocking modes of operation.
> 2. The caller can invoke a procedure on multiple processors. Supports
> blocking and non-blocking modes of operation.
> 
> Cc: Ray Ni <ray.ni@intel.com>
> Cc: Laszlo Ersek <lersek@redhat.com>
> Signed-off-by: Eric Dong <eric.dong@intel.com>
> ---
>  MdePkg/Include/Pi/PiMultiPhase.h |  16 ++
>  MdePkg/Include/Protocol/MmMp.h   | 334 +++++++++++++++++++++++++++++++
>  MdePkg/Include/Protocol/SmmMp.h  |  44 ++++
>  MdePkg/MdePkg.dec                |   6 +
>  4 files changed, 400 insertions(+)
>  create mode 100644 MdePkg/Include/Protocol/MmMp.h
>  create mode 100644 MdePkg/Include/Protocol/SmmMp.h
> 
> diff --git a/MdePkg/Include/Pi/PiMultiPhase.h b/MdePkg/Include/Pi/PiMultiPhase.h
> index eb12527767..a5056799e1 100644
> --- a/MdePkg/Include/Pi/PiMultiPhase.h
> +++ b/MdePkg/Include/Pi/PiMultiPhase.h
> @@ -176,4 +176,20 @@ VOID
>    IN OUT VOID  *Buffer
>    );
>  
> +/**
> +  The function prototype for invoking a function on an Application Processor.
> +
> +  This definition is used by the UEFI MM MP Serices Protocol.
> +
> +  @param[in] ProcedureArgument    The pointer to private data buffer.
> +
> +  @retval EFI_SUCCESS             Excutive the procedure successfully
> +
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_AP_PROCEDURE2)(
> +  IN VOID  *ProcedureArgument
> +);
> +
>  #endif
> diff --git a/MdePkg/Include/Protocol/MmMp.h b/MdePkg/Include/Protocol/MmMp.h
> new file mode 100644
> index 0000000000..0467072bec
> --- /dev/null
> +++ b/MdePkg/Include/Protocol/MmMp.h
> @@ -0,0 +1,334 @@
> +/** @file
> +  EFI MM MP Protocol is defined in the PI 1.5 specification.
> +
> +  The MM MP protocol provides a set of functions to allow execution of procedures on processors that
> +  have entered MM. This protocol has the following properties:
> +  1. The caller can only invoke execution of a procedure on a processor, other than the caller, that
> +     has also entered MM.
> +  2. It is possible to invoke a procedure on multiple processors. Supports blocking and non-blocking
> +     modes of operation.
> +
> +  Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
> +  SPDX-License-Identifier: BSD-2-Clause-Patent
> +
> +**/
> +
> +#ifndef _MM_MP_H_
> +#define _MM_MP_H_
> +
> +#include <Pi/PiMmCis.h>
> +
> +#define EFI_MM_MP_PROTOCOL_GUID \
> +  { \
> +    0x5d5450d7, 0x990c, 0x4180, {0xa8, 0x3, 0x8e, 0x63, 0xf0, 0x60, 0x83, 0x7  }  \
> +  }
> +
> +//
> +// Revision definition.
> +//
> +#define EFI_MM_MP_PROTOCOL_REVISION    0x00
> +
> +//
> +// Attribute flags
> +//
> +#define EFI_MM_MP_TIMEOUT_SUPPORTED    0x01
> +
> +//
> +// Completion token
> +//
> +typedef VOID* MM_COMPLETION;
> +
> +typedef struct {
> +  MM_COMPLETION  Completion;
> +  EFI_STATUS     Status;
> +} MM_DISPATCH_COMPLETION_TOKEN;
> +
> +typedef struct _EFI_MM_MP_PROTOCOL  EFI_MM_MP_PROTOCOL;
> +
> +/**
> +  Service to retrieves the number of logical processor in the platform.
> +
> +  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
> +  @param[out] NumberOfProcessors  Pointer to the total number of logical processors in the system,
> +                                  including the BSP and all APs.
> +
> +  @retval EFI_SUCCESS             The number of processors was retrieved successfully
> +  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_MM_GET_NUMBER_OF_PROCESSORS) (
> +  IN CONST EFI_MM_MP_PROTOCOL  *This,
> +  OUT      UINTN               *NumberOfProcessors
> +);
> +
> +
> +/**
> +  This service allows the caller to invoke a procedure one of the application processors (AP). This
> +  function uses an optional token parameter to support blocking and non-blocking modes. If the token
> +  is passed into the call, the function will operate in a non-blocking fashion and the caller can
> +  check for completion with CheckOnProcedure or WaitForProcedure.
> +
> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]     Procedure              A pointer to the procedure to be run on the designated target
> +                                        AP of the system. Type EFI_AP_PROCEDURE2 is defined below in
> +                                        related definitions.
> +  @param[in]     CpuNumber              The zero-based index of the processor number of the target
> +                                        AP, on which the code stream is supposed to run. If the number
> +                                        points to the calling processor then it will not run the
> +                                        supplied code.
> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for this AP to
> +                                        finish execution of Procedure, either for blocking or
> +                                        non-blocking mode. Zero means infinity. If the timeout
> +                                        expires before this AP returns from Procedure, then Procedure
> +                                        on the AP is terminated. If the timeout expires in blocking
> +                                        mode, the call returns EFI_TIMEOUT. If the timeout expires
> +                                        in non-blocking mode, the timeout determined can be through
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        Note that timeout support is optional. Whether an
> +                                        implementation supports this feature, can be determined via
> +                                        the Attributes data member.
> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
> +                                        that is run by the AP. It is an optional common mailbox
> +                                        between APs and the caller to share information.
> +  @param[in,out] Token                  This is parameter is broken into two components:
> +                                        1.Token->Completion is an optional parameter that allows the
> +                                        caller to execute the procedure in a blocking or non-blocking
> +                                        fashion. If it is NULL the call is blocking, and the call will
> +                                        not return until the AP has completed the procedure. If the
> +                                        token is not NULL, the call will return immediately. The caller
> +                                        can check whether the procedure has completed with
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        2.Token->Status The implementation updates the address pointed
> +                                        at by this variable with the status code returned by Procedure
> +                                        when it completes execution on the target AP, or with EFI_TIMEOUT
> +                                        if the Procedure fails to complete within the optional timeout.
> +                                        The implementation will update this variable with EFI_NOT_READY
> +                                        prior to starting Procedure on the target AP.
> +  @param[in,out] CPUStatus              This optional pointer may be used to get the status code returned
> +                                        by Procedure when it completes execution on the target AP, or with
> +                                        EFI_TIMEOUT if the Procedure fails to complete within the optional
> +                                        timeout. The implementation will update this variable with
> +                                        EFI_NOT_READY prior to starting Procedure on the target AP.
> +
> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
> +                                        execution on the target AP.
> +                                        In the non-blocking case this indicates that the procedure has
> +                                        been successfully scheduled for execution on the target AP.
> +  @retval EFI_INVALID_PARAMETER         The input arguments are out of range. Either the target AP is the
> +                                        caller of the function, or the Procedure or Token is NULL
> +  @retval EFI_NOT_READY                 If the target AP is busy executing another procedure
> +  @retval EFI_ALREADY_STARTED           Token is already in use for another procedure
> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before the specified AP
> +                                        has finished
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_MM_DISPATCH_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         CpuNumber,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   MM_COMPLETION                 *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +);
> +
> +/**
> +  This service allows the caller to invoke a procedure on all running application processors (AP)
> +  except the caller. This function uses an optional token parameter to support blocking and
> +  nonblocking modes. If the token is passed into the call, the function will operate in a non-blocking
> +  fashion and the caller can check for completion with CheckOnProcedure or WaitForProcedure.
> +
> +  It is not necessary for the implementation to run the procedure on every processor on the platform.
> +  Processors that are powered down in such a way that they cannot respond to interrupts, may be
> +  excluded from the broadcast.
> +
> +
> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]     Procedure              A pointer to the code stream to be run on the APs that have
> +                                        entered MM. Type EFI_AP_PROCEDURE is defined below in related
> +                                        definitions.
> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for the APs to finish
> +                                        execution of Procedure, either for blocking or non-blocking mode.
> +                                        Zero means infinity. If the timeout expires before all APs return
> +                                        from Procedure, then Procedure on the failed APs is terminated. If
> +                                        the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
> +                                        If the timeout expires in non-blocking mode, the timeout determined
> +                                        can be through CheckOnProcedure or WaitForProcedure.
> +                                        Note that timeout support is optional. Whether an implementation
> +                                        supports this feature can be determined via the Attributes data
> +                                        member.
> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
> +                                        that is run by the AP. It is an optional common mailbox
> +                                        between APs and the caller to share information.
> +  @param[in,out] Token                  This is parameter is broken into two components:
> +                                        1.Token->Completion is an optional parameter that allows the
> +                                        caller to execute the procedure in a blocking or non-blocking
> +                                        fashion. If it is NULL the call is blocking, and the call will
> +                                        not return until the AP has completed the procedure. If the
> +                                        token is not NULL, the call will return immediately. The caller
> +                                        can check whether the procedure has completed with
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        2.Token->Status The implementation updates the address pointed
> +                                        at by this variable with the status code returned by Procedure
> +                                        when it completes execution on the target AP, or with EFI_TIMEOUT
> +                                        if the Procedure fails to complete within the optional timeout.
> +                                        The implementation will update this variable with EFI_NOT_READY
> +                                        prior to starting Procedure on the target AP
> +  @param[in,out] CPUStatus              This optional pointer may be used to get the individual status
> +                                        returned by every AP that participated in the broadcast. This
> +                                        parameter if used provides the base address of an array to hold
> +                                        the EFI_STATUS value of each AP in the system. The size of the
> +                                        array can be ascertained by the GetNumberOfProcessors function.
> +                                        As mentioned above, the broadcast may not include every processor
> +                                        in the system. Some implementations may exclude processors that
> +                                        have been powered down in such a way that they are not responsive
> +                                        to interrupts. Additionally the broadcast excludes the processor
> +                                        which is making the BroadcastProcedure call. For every excluded
> +                                        processor, the array entry must contain a value of EFI_NOT_STARTED
> +
> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
> +                                        execution on the APs. In the non-blocking case this indicates that
> +                                        the procedure has been successfully scheduled for execution on the
> +                                        APs.
> +  @retval EFI_INVALID_PARAMETER         Procedure or Token is NULL.
> +  @retval EFI_NOT_READY                 If a target AP is busy executing another procedure.
> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before all enabled APs have
> +                                        finished.
> +  @retval EFI_ALREADY_STARTED           Before the AP procedure associated with the Token is finished, the
> +                                        same Token cannot be used to dispatch or broadcast another procedure.
> +
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_MM_BROADCAST_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   MM_COMPLETION                 *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +);
> +
> +
> +/**
> +  This service allows the caller to set a startup procedure that will be executed when an AP powers
> +  up from a state where core configuration and context is lost. The procedure is execution has the
> +  following properties:
> +  1. The procedure executes before the processor is handed over to the operating system.
> +  2. All processors execute the same startup procedure.
> +  3. The procedure may run in parallel with other procedures invoked through the functions in this
> +  protocol, or with processors that are executing an MM handler or running in the operating system.
> +
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Procedure            A pointer to the code stream to be run on the designated target AP
> +                                       of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
> +                                       with the related definitions of
> +                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
> +                                       If caller may pass a value of NULL to deregister any existing
> +                                       startup procedure.
> +  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of parameters to the code that is
> +                                       run by the AP. It is an optional common mailbox between APs and
> +                                       the caller to share information
> +
> +  @retval EFI_SUCCESS                  The Procedure has been set successfully.
> +  @retval EFI_INVALID_PARAMETER        The Procedure is NULL.
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_MM_SET_STARTUP_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL *This,
> +  IN       EFI_AP_PROCEDURE   Procedure,
> +  IN OUT   VOID               *ProcedureArguments OPTIONAL
> +);
> +
> +/**
> +  When non-blocking execution of a procedure on an AP is invoked with DispatchProcedure,
> +  via the use of a token, this function can be used to check for completion of the procedure on the AP.
> +  The function takes the token that was passed into the DispatchProcedure call. If the procedure
> +  is complete, and therefore it is now possible to run another procedure on the same AP, this function
> +  returns EFI_SUCESS. In this case the status returned by the procedure that executed on the AP is
> +  returned in the token's Status field. If the procedure has not yet completed, then this function
> +  returns EFI_NOT_READY.
> +
> +  When a non-blocking execution of a procedure is invoked with BroadcastProcedure, via the
> +  use of a token, this function can be used to check for completion of the procedure on all the
> +  broadcast APs. The function takes the token that was passed into the BroadcastProcedure
> +  call. If the procedure is complete on all broadcast APs this function returns EFI_SUCESS. In this
> +  case the Status field in the token passed into the function reflects the overall result of the
> +  invocation, which may be EFI_SUCCESS, if all executions succeeded, or the first observed failure.
> +  If the procedure has not yet completed on the broadcast APs, the function returns
> +  EFI_NOT_READY.
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Token                This parameter describes the token that was passed into
> +                                       DispatchProcedure or BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS                  Procedure has completed.
> +  @retval EFI_NOT_READY                The Procedure has not completed.
> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is NULL
> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-blocking call
> +
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_CHECK_FOR_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       MM_COMPLETION                 Token
> +);
> +
> +/**
> +  When a non-blocking execution of a procedure on an AP is invoked via DispatchProcedure,
> +  this function will block the caller until the remote procedure has completed on the designated AP.
> +  The non-blocking procedure invocation is identified by the Token parameter, which must match the
> +  token that used when DispatchProcedure was called. Upon completion the status returned by
> +  the procedure that executed on the AP is used to update the token's Status field.
> +
> +  When a non-blocking execution of a procedure on an AP is invoked via BroadcastProcedure
> +  this function will block the caller until the remote procedure has completed on all of the APs that
> +  entered MM. The non-blocking procedure invocation is identified by the Token parameter, which
> +  must match the token that used when BroadcastProcedure was called. Upon completion the
> +  overall status returned by the procedures that executed on the broadcast AP is used to update the
> +  token's Status field. The overall status may be EFI_SUCCESS, if all executions succeeded, or the
> +  first observed failure.
> +
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Token                This parameter describes the token that was passed into
> +                                       DispatchProcedure or BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS                  Procedure has completed.
> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is NULL
> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-blocking call
> +
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_WAIT_FOR_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       MM_COMPLETION                 Token
> +);
> +
> +
> +
> +///
> +/// The MM MP protocol provides a set of functions to allow execution of procedures on processors that
> +/// have entered MM.
> +///
> +struct _EFI_MM_MP_PROTOCOL {
> +  UINT32                            Revision;
> +  UINT32                            Attributes;
> +  EFI_MM_GET_NUMBER_OF_PROCESSORS   GetNumberOfProcessors;
> +  EFI_MM_DISPATCH_PROCEDURE         DispatchProcedure;
> +  EFI_MM_BROADCAST_PROCEDURE        BroadcastProcedure;
> +  EFI_MM_SET_STARTUP_PROCEDURE      SetStartupProcedure;
> +  EFI_CHECK_FOR_PROCEDURE           CheckForProcedure;
> +  EFI_WAIT_FOR_PROCEDURE            WaitForProcedure;
> +};
> +
> +extern EFI_GUID gEfiMmMpProtocolGuid;
> +
> +#endif
> +
> diff --git a/MdePkg/Include/Protocol/SmmMp.h b/MdePkg/Include/Protocol/SmmMp.h
> new file mode 100644
> index 0000000000..5f391a3860
> --- /dev/null
> +++ b/MdePkg/Include/Protocol/SmmMp.h
> @@ -0,0 +1,44 @@
> +/** @file
> +  EFI SMM MP Protocol is defined in the PI 1.5 specification.
> +
> +  The SMM MP protocol provides a set of functions to allow execution of procedures on processors that
> +  have entered SMM. This protocol has the following properties:
> +  1. The caller can only invoke execution of a procedure on a processor, other than the caller, that
> +     has also entered SMM.
> +  2. It is possible to invoke a procedure on multiple processors. Supports blocking and non-blocking
> +     modes of operation.
> +
> +  Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
> +  SPDX-License-Identifier: BSD-2-Clause-Patent
> +
> +**/
> +
> +#ifndef _SMM_MP_H_
> +#define _SMM_MP_H_
> +
> +#include <Protocol/MmMp.h>
> +
> +#define EFI_SMM_MP_PROTOCOL_GUID EFI_MM_MP_PROTOCOL_GUID
> +
> +#define EFI_SMM_MP_TIMEOUT_SUPPORTED EFI_MM_MP_TIMEOUT_SUPPORTED
> +
> +#define EFI_SMM_MP_PROTOCOL_REVISION    EFI_MM_MP_PROTOCOL_REVISION
> +
> +typedef MM_COMPLETION SMM_COMPLETION;
> +
> +typedef MM_DISPATCH_COMPLETION_TOKEN SMM_DISPATCH_COMPLETION_TOKEN;
> +
> +typedef EFI_MM_MP_PROTOCOL  EFI_SMM_MP_PROTOCOL;
> +
> +typedef EFI_MM_GET_NUMBER_OF_PROCESSORS EFI_SMM_GET_NUMBER_OF_PROCESSORS;
> +
> +typedef EFI_MM_DISPATCH_PROCEDURE EFI_SMM_DISPATCH_PROCEDURE;
> +
> +typedef EFI_MM_BROADCAST_PROCEDURE EFI_SMM_BROADCAST_PROCEDURE;
> +
> +typedef EFI_MM_SET_STARTUP_PROCEDURE EFI_SMM_SET_STARTUP_PROCEDURE;
> +
> +extern EFI_GUID gEfiSmmMpProtocolGuid;
> +
> +#endif
> +
> diff --git a/MdePkg/MdePkg.dec b/MdePkg/MdePkg.dec
> index 6c563375ee..0b8da9f2cb 100644
> --- a/MdePkg/MdePkg.dec
> +++ b/MdePkg/MdePkg.dec
> @@ -1167,6 +1167,12 @@
>    # Protocols defined in PI 1.5.
>    #
>  
> +  ## Include/Protocol/SmmMp.h
> +  gEfiSmmMpProtocolGuid = { 0x5d5450d7, 0x990c, 0x4180, { 0xa8, 0x3, 0x8e, 0x63, 0xf0, 0x60, 0x83, 0x7 }}
> +
> +  ## Include/Protocol/MmMp.h
> +  gEfiMmMpProtocolGuid = { 0x5d5450d7, 0x990c, 0x4180, { 0xa8, 0x3, 0x8e, 0x63, 0xf0, 0x60, 0x83, 0x7 }}
> +
>    ## Include/Protocol/MmEndOfDxe.h
>    gEfiMmEndOfDxeProtocolGuid = { 0x24e70042, 0xd5c5, 0x4260, { 0x8c, 0x39, 0xa, 0xd3, 0xaa, 0x32, 0xe9, 0x3d }}
>  
> 

Re: [Patch 1/2] MdePkg: Add new MM MP Protocol definition.

[Ni, Ray]

Eric,
Is SmmMp.h necessary? If PI spec doesn't define the EFI_SMM_MP_PROTOCOL, SmmMp.h is not needed.

Consumer is expected to use the new MM_MP protocol. There is no backward compatibility issue here.

Thanks,
Ray

> -----Original Message-----
> From: Dong, Eric <eric.dong@intel.com>
> Sent: Wednesday, June 19, 2019 1:51 PM
> To: devel@edk2.groups.io
> Cc: Ni, Ray <ray.ni@intel.com>; Laszlo Ersek <lersek@redhat.com>
> Subject: [Patch 1/2] MdePkg: Add new MM MP Protocol definition.
> 
> EFI MM MP Protocol is defined in the PI 1.7 specification.
> 
> The MM MP protocol provides a set of functions to allow execution of
> procedures on processors that have entered MM. This protocol has the
> following properties:
> 1. The caller can invoke execution of a procedure on a processor, other than
> the caller, that has also entered MM. Supports blocking and non-blocking
> modes of operation.
> 2. The caller can invoke a procedure on multiple processors. Supports
> blocking and non-blocking modes of operation.
> 
> Cc: Ray Ni <ray.ni@intel.com>
> Cc: Laszlo Ersek <lersek@redhat.com>
> Signed-off-by: Eric Dong <eric.dong@intel.com>
> ---
>  MdePkg/Include/Pi/PiMultiPhase.h |  16 ++
>  MdePkg/Include/Protocol/MmMp.h   | 334
> +++++++++++++++++++++++++++++++
>  MdePkg/Include/Protocol/SmmMp.h  |  44 ++++
>  MdePkg/MdePkg.dec                |   6 +
>  4 files changed, 400 insertions(+)
>  create mode 100644 MdePkg/Include/Protocol/MmMp.h  create mode
> 100644 MdePkg/Include/Protocol/SmmMp.h
> 
> diff --git a/MdePkg/Include/Pi/PiMultiPhase.h
> b/MdePkg/Include/Pi/PiMultiPhase.h
> index eb12527767..a5056799e1 100644
> --- a/MdePkg/Include/Pi/PiMultiPhase.h
> +++ b/MdePkg/Include/Pi/PiMultiPhase.h
> @@ -176,4 +176,20 @@ VOID
>    IN OUT VOID  *Buffer
>    );
> 
> +/**
> +  The function prototype for invoking a function on an Application Processor.
> +
> +  This definition is used by the UEFI MM MP Serices Protocol.
> +
> +  @param[in] ProcedureArgument    The pointer to private data buffer.
> +
> +  @retval EFI_SUCCESS             Excutive the procedure successfully
> +
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_AP_PROCEDURE2)(
> +  IN VOID  *ProcedureArgument
> +);
> +
>  #endif
> diff --git a/MdePkg/Include/Protocol/MmMp.h
> b/MdePkg/Include/Protocol/MmMp.h new file mode 100644 index
> 0000000000..0467072bec
> --- /dev/null
> +++ b/MdePkg/Include/Protocol/MmMp.h
> @@ -0,0 +1,334 @@
> +/** @file
> +  EFI MM MP Protocol is defined in the PI 1.5 specification.
> +
> +  The MM MP protocol provides a set of functions to allow execution of
> + procedures on processors that  have entered MM. This protocol has the
> following properties:
> +  1. The caller can only invoke execution of a procedure on a processor,
> other than the caller, that
> +     has also entered MM.
> +  2. It is possible to invoke a procedure on multiple processors. Supports
> blocking and non-blocking
> +     modes of operation.
> +
> +  Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
> +  SPDX-License-Identifier: BSD-2-Clause-Patent
> +
> +**/
> +
> +#ifndef _MM_MP_H_
> +#define _MM_MP_H_
> +
> +#include <Pi/PiMmCis.h>
> +
> +#define EFI_MM_MP_PROTOCOL_GUID \
> +  { \
> +    0x5d5450d7, 0x990c, 0x4180, {0xa8, 0x3, 0x8e, 0x63, 0xf0, 0x60,
> +0x83, 0x7  }  \
> +  }
> +
> +//
> +// Revision definition.
> +//
> +#define EFI_MM_MP_PROTOCOL_REVISION    0x00
> +
> +//
> +// Attribute flags
> +//
> +#define EFI_MM_MP_TIMEOUT_SUPPORTED    0x01
> +
> +//
> +// Completion token
> +//
> +typedef VOID* MM_COMPLETION;
> +
> +typedef struct {
> +  MM_COMPLETION  Completion;
> +  EFI_STATUS     Status;
> +} MM_DISPATCH_COMPLETION_TOKEN;
> +
> +typedef struct _EFI_MM_MP_PROTOCOL  EFI_MM_MP_PROTOCOL;
> +
> +/**
> +  Service to retrieves the number of logical processor in the platform.
> +
> +  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
> +  @param[out] NumberOfProcessors  Pointer to the total number of logical
> processors in the system,
> +                                  including the BSP and all APs.
> +
> +  @retval EFI_SUCCESS             The number of processors was retrieved
> successfully
> +  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_MM_GET_NUMBER_OF_PROCESSORS) (
> +  IN CONST EFI_MM_MP_PROTOCOL  *This,
> +  OUT      UINTN               *NumberOfProcessors
> +);
> +
> +
> +/**
> +  This service allows the caller to invoke a procedure one of the
> +application processors (AP). This
> +  function uses an optional token parameter to support blocking and
> +non-blocking modes. If the token
> +  is passed into the call, the function will operate in a non-blocking
> +fashion and the caller can
> +  check for completion with CheckOnProcedure or WaitForProcedure.
> +
> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]     Procedure              A pointer to the procedure to be run on
> the designated target
> +                                        AP of the system. Type EFI_AP_PROCEDURE2 is defined
> below in
> +                                        related definitions.
> +  @param[in]     CpuNumber              The zero-based index of the processor
> number of the target
> +                                        AP, on which the code stream is supposed to run. If the
> number
> +                                        points to the calling processor then it will not run the
> +                                        supplied code.
> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
> microseconds for this AP to
> +                                        finish execution of Procedure, either for blocking or
> +                                        non-blocking mode. Zero means infinity. If the timeout
> +                                        expires before this AP returns from Procedure, then
> Procedure
> +                                        on the AP is terminated. If the timeout expires in
> blocking
> +                                        mode, the call returns EFI_TIMEOUT. If the timeout
> expires
> +                                        in non-blocking mode, the timeout determined can be
> through
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        Note that timeout support is optional. Whether an
> +                                        implementation supports this feature, can be
> determined via
> +                                        the Attributes data member.
> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
> parameters to the code
> +                                        that is run by the AP. It is an optional common mailbox
> +                                        between APs and the caller to share information.
> +  @param[in,out] Token                  This is parameter is broken into two
> components:
> +                                        1.Token->Completion is an optional parameter that
> allows the
> +                                        caller to execute the procedure in a blocking or non-
> blocking
> +                                        fashion. If it is NULL the call is blocking, and the call will
> +                                        not return until the AP has completed the procedure. If
> the
> +                                        token is not NULL, the call will return immediately. The
> caller
> +                                        can check whether the procedure has completed with
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        2.Token->Status The implementation updates the
> address pointed
> +                                        at by this variable with the status code returned by
> Procedure
> +                                        when it completes execution on the target AP, or with
> EFI_TIMEOUT
> +                                        if the Procedure fails to complete within the optional
> timeout.
> +                                        The implementation will update this variable with
> EFI_NOT_READY
> +                                        prior to starting Procedure on the target AP.
> +  @param[in,out] CPUStatus              This optional pointer may be used to get
> the status code returned
> +                                        by Procedure when it completes execution on the
> target AP, or with
> +                                        EFI_TIMEOUT if the Procedure fails to complete within
> the optional
> +                                        timeout. The implementation will update this variable
> with
> +                                        EFI_NOT_READY prior to starting Procedure on the
> target AP.
> +
> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
> Procedure has completed
> +                                        execution on the target AP.
> +                                        In the non-blocking case this indicates that the
> procedure has
> +                                        been successfully scheduled for execution on the target
> AP.
> +  @retval EFI_INVALID_PARAMETER         The input arguments are out of
> range. Either the target AP is the
> +                                        caller of the function, or the Procedure or Token is NULL
> +  @retval EFI_NOT_READY                 If the target AP is busy executing another
> procedure
> +  @retval EFI_ALREADY_STARTED           Token is already in use for another
> procedure
> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
> before the specified AP
> +                                        has finished **/ typedef
> +EFI_STATUS (EFIAPI *EFI_MM_DISPATCH_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         CpuNumber,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   MM_COMPLETION                 *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +);
> +
> +/**
> +  This service allows the caller to invoke a procedure on all running
> +application processors (AP)
> +  except the caller. This function uses an optional token parameter to
> +support blocking and
> +  nonblocking modes. If the token is passed into the call, the function
> +will operate in a non-blocking
> +  fashion and the caller can check for completion with CheckOnProcedure or
> WaitForProcedure.
> +
> +  It is not necessary for the implementation to run the procedure on every
> processor on the platform.
> +  Processors that are powered down in such a way that they cannot
> + respond to interrupts, may be  excluded from the broadcast.
> +
> +
> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]     Procedure              A pointer to the code stream to be run on
> the APs that have
> +                                        entered MM. Type EFI_AP_PROCEDURE is defined
> below in related
> +                                        definitions.
> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
> microseconds for the APs to finish
> +                                        execution of Procedure, either for blocking or non-
> blocking mode.
> +                                        Zero means infinity. If the timeout expires before all
> APs return
> +                                        from Procedure, then Procedure on the failed APs is
> terminated. If
> +                                        the timeout expires in blocking mode, the call returns
> EFI_TIMEOUT.
> +                                        If the timeout expires in non-blocking mode, the
> timeout determined
> +                                        can be through CheckOnProcedure or
> WaitForProcedure.
> +                                        Note that timeout support is optional. Whether an
> implementation
> +                                        supports this feature can be determined via the
> Attributes data
> +                                        member.
> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
> parameters to the code
> +                                        that is run by the AP. It is an optional common mailbox
> +                                        between APs and the caller to share information.
> +  @param[in,out] Token                  This is parameter is broken into two
> components:
> +                                        1.Token->Completion is an optional parameter that
> allows the
> +                                        caller to execute the procedure in a blocking or non-
> blocking
> +                                        fashion. If it is NULL the call is blocking, and the call will
> +                                        not return until the AP has completed the procedure. If
> the
> +                                        token is not NULL, the call will return immediately. The
> caller
> +                                        can check whether the procedure has completed with
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        2.Token->Status The implementation updates the
> address pointed
> +                                        at by this variable with the status code returned by
> Procedure
> +                                        when it completes execution on the target AP, or with
> EFI_TIMEOUT
> +                                        if the Procedure fails to complete within the optional
> timeout.
> +                                        The implementation will update this variable with
> EFI_NOT_READY
> +                                        prior to starting Procedure on the target AP
> +  @param[in,out] CPUStatus              This optional pointer may be used to get
> the individual status
> +                                        returned by every AP that participated in the broadcast.
> This
> +                                        parameter if used provides the base address of an array
> to hold
> +                                        the EFI_STATUS value of each AP in the system. The size
> of the
> +                                        array can be ascertained by the
> GetNumberOfProcessors function.
> +                                        As mentioned above, the broadcast may not include
> every processor
> +                                        in the system. Some implementations may exclude
> processors that
> +                                        have been powered down in such a way that they are
> not responsive
> +                                        to interrupts. Additionally the broadcast excludes the
> processor
> +                                        which is making the BroadcastProcedure call. For every
> excluded
> +                                        processor, the array entry must
> + contain a value of EFI_NOT_STARTED
> +
> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
> Procedure has completed
> +                                        execution on the APs. In the non-blocking case this
> indicates that
> +                                        the procedure has been successfully scheduled for
> execution on the
> +                                        APs.
> +  @retval EFI_INVALID_PARAMETER         Procedure or Token is NULL.
> +  @retval EFI_NOT_READY                 If a target AP is busy executing another
> procedure.
> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
> before all enabled APs have
> +                                        finished.
> +  @retval EFI_ALREADY_STARTED           Before the AP procedure associated
> with the Token is finished, the
> +                                        same Token cannot be used to dispatch or broadcast
> another procedure.
> +
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_MM_BROADCAST_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   MM_COMPLETION                 *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +);
> +
> +
> +/**
> +  This service allows the caller to set a startup procedure that will
> +be executed when an AP powers
> +  up from a state where core configuration and context is lost. The
> +procedure is execution has the
> +  following properties:
> +  1. The procedure executes before the processor is handed over to the
> operating system.
> +  2. All processors execute the same startup procedure.
> +  3. The procedure may run in parallel with other procedures invoked
> +through the functions in this
> +  protocol, or with processors that are executing an MM handler or running
> in the operating system.
> +
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Procedure            A pointer to the code stream to be run on
> the designated target AP
> +                                       of the system. Type EFI_AP_PROCEDURE is defined
> below in Volume 2
> +                                       with the related definitions of
> +                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
> +                                       If caller may pass a value of NULL to deregister any
> existing
> +                                       startup procedure.
> +  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of
> parameters to the code that is
> +                                       run by the AP. It is an optional common mailbox
> between APs and
> +                                       the caller to share information
> +
> +  @retval EFI_SUCCESS                  The Procedure has been set successfully.
> +  @retval EFI_INVALID_PARAMETER        The Procedure is NULL.
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_MM_SET_STARTUP_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL *This,
> +  IN       EFI_AP_PROCEDURE   Procedure,
> +  IN OUT   VOID               *ProcedureArguments OPTIONAL
> +);
> +
> +/**
> +  When non-blocking execution of a procedure on an AP is invoked with
> +DispatchProcedure,
> +  via the use of a token, this function can be used to check for completion of
> the procedure on the AP.
> +  The function takes the token that was passed into the
> +DispatchProcedure call. If the procedure
> +  is complete, and therefore it is now possible to run another
> +procedure on the same AP, this function
> +  returns EFI_SUCESS. In this case the status returned by the procedure
> +that executed on the AP is
> +  returned in the token's Status field. If the procedure has not yet
> +completed, then this function
> +  returns EFI_NOT_READY.
> +
> +  When a non-blocking execution of a procedure is invoked with
> + BroadcastProcedure, via the  use of a token, this function can be used
> + to check for completion of the procedure on all the  broadcast APs.
> + The function takes the token that was passed into the
> + BroadcastProcedure  call. If the procedure is complete on all
> + broadcast APs this function returns EFI_SUCESS. In this  case the Status
> field in the token passed into the function reflects the overall result of the
> invocation, which may be EFI_SUCCESS, if all executions succeeded, or the
> first observed failure.
> +  If the procedure has not yet completed on the broadcast APs, the
> + function returns  EFI_NOT_READY.
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Token                This parameter describes the token that was
> passed into
> +                                       DispatchProcedure or BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS                  Procedure has completed.
> +  @retval EFI_NOT_READY                The Procedure has not completed.
> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is
> NULL
> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-
> blocking call
> +
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_CHECK_FOR_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       MM_COMPLETION                 Token
> +);
> +
> +/**
> +  When a non-blocking execution of a procedure on an AP is invoked via
> +DispatchProcedure,
> +  this function will block the caller until the remote procedure has completed
> on the designated AP.
> +  The non-blocking procedure invocation is identified by the Token
> +parameter, which must match the
> +  token that used when DispatchProcedure was called. Upon completion
> +the status returned by
> +  the procedure that executed on the AP is used to update the token's
> Status field.
> +
> +  When a non-blocking execution of a procedure on an AP is invoked via
> + BroadcastProcedure  this function will block the caller until the
> + remote procedure has completed on all of the APs that  entered MM. The
> + non-blocking procedure invocation is identified by the Token
> + parameter, which  must match the token that used when
> + BroadcastProcedure was called. Upon completion the  overall status
> + returned by the procedures that executed on the broadcast AP is used to
> update the  token's Status field. The overall status may be EFI_SUCCESS, if all
> executions succeeded, or the  first observed failure.
> +
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Token                This parameter describes the token that was
> passed into
> +                                       DispatchProcedure or BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS                  Procedure has completed.
> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is
> NULL
> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-
> blocking call
> +
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_WAIT_FOR_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       MM_COMPLETION                 Token
> +);
> +
> +
> +
> +///
> +/// The MM MP protocol provides a set of functions to allow execution
> +of procedures on processors that /// have entered MM.
> +///
> +struct _EFI_MM_MP_PROTOCOL {
> +  UINT32                            Revision;
> +  UINT32                            Attributes;
> +  EFI_MM_GET_NUMBER_OF_PROCESSORS   GetNumberOfProcessors;
> +  EFI_MM_DISPATCH_PROCEDURE         DispatchProcedure;
> +  EFI_MM_BROADCAST_PROCEDURE        BroadcastProcedure;
> +  EFI_MM_SET_STARTUP_PROCEDURE      SetStartupProcedure;
> +  EFI_CHECK_FOR_PROCEDURE           CheckForProcedure;
> +  EFI_WAIT_FOR_PROCEDURE            WaitForProcedure;
> +};
> +
> +extern EFI_GUID gEfiMmMpProtocolGuid;
> +
> +#endif
> +
> diff --git a/MdePkg/Include/Protocol/SmmMp.h
> b/MdePkg/Include/Protocol/SmmMp.h new file mode 100644 index
> 0000000000..5f391a3860
> --- /dev/null
> +++ b/MdePkg/Include/Protocol/SmmMp.h
> @@ -0,0 +1,44 @@
> +/** @file
> +  EFI SMM MP Protocol is defined in the PI 1.5 specification.
> +
> +  The SMM MP protocol provides a set of functions to allow execution of
> + procedures on processors that  have entered SMM. This protocol has the
> following properties:
> +  1. The caller can only invoke execution of a procedure on a processor,
> other than the caller, that
> +     has also entered SMM.
> +  2. It is possible to invoke a procedure on multiple processors. Supports
> blocking and non-blocking
> +     modes of operation.
> +
> +  Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
> +  SPDX-License-Identifier: BSD-2-Clause-Patent
> +
> +**/
> +
> +#ifndef _SMM_MP_H_
> +#define _SMM_MP_H_
> +
> +#include <Protocol/MmMp.h>
> +
> +#define EFI_SMM_MP_PROTOCOL_GUID EFI_MM_MP_PROTOCOL_GUID
> +
> +#define EFI_SMM_MP_TIMEOUT_SUPPORTED
> EFI_MM_MP_TIMEOUT_SUPPORTED
> +
> +#define EFI_SMM_MP_PROTOCOL_REVISION
> EFI_MM_MP_PROTOCOL_REVISION
> +
> +typedef MM_COMPLETION SMM_COMPLETION;
> +
> +typedef MM_DISPATCH_COMPLETION_TOKEN
> SMM_DISPATCH_COMPLETION_TOKEN;
> +
> +typedef EFI_MM_MP_PROTOCOL  EFI_SMM_MP_PROTOCOL;
> +
> +typedef EFI_MM_GET_NUMBER_OF_PROCESSORS
> +EFI_SMM_GET_NUMBER_OF_PROCESSORS;
> +
> +typedef EFI_MM_DISPATCH_PROCEDURE
> EFI_SMM_DISPATCH_PROCEDURE;
> +
> +typedef EFI_MM_BROADCAST_PROCEDURE
> EFI_SMM_BROADCAST_PROCEDURE;
> +
> +typedef EFI_MM_SET_STARTUP_PROCEDURE
> EFI_SMM_SET_STARTUP_PROCEDURE;
> +
> +extern EFI_GUID gEfiSmmMpProtocolGuid;
> +
> +#endif
> +
> diff --git a/MdePkg/MdePkg.dec b/MdePkg/MdePkg.dec index
> 6c563375ee..0b8da9f2cb 100644
> --- a/MdePkg/MdePkg.dec
> +++ b/MdePkg/MdePkg.dec
> @@ -1167,6 +1167,12 @@
>    # Protocols defined in PI 1.5.
>    #
> 
> +  ## Include/Protocol/SmmMp.h
> +  gEfiSmmMpProtocolGuid = { 0x5d5450d7, 0x990c, 0x4180, { 0xa8, 0x3,
> + 0x8e, 0x63, 0xf0, 0x60, 0x83, 0x7 }}
> +
> +  ## Include/Protocol/MmMp.h
> +  gEfiMmMpProtocolGuid = { 0x5d5450d7, 0x990c, 0x4180, { 0xa8, 0x3,
> + 0x8e, 0x63, 0xf0, 0x60, 0x83, 0x7 }}
> +
>    ## Include/Protocol/MmEndOfDxe.h
>    gEfiMmEndOfDxeProtocolGuid = { 0x24e70042, 0xd5c5, 0x4260, { 0x8c, 0x39,
> 0xa, 0xd3, 0xaa, 0x32, 0xe9, 0x3d }}
> 
> --
> 2.21.0.windows.1

[Patch 2/2] UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP Protocol.

[Dong, Eric]

Add MM Mp Protocol in PiSmmCpuDxeSmm driver.

Cc: Ray Ni <ray.ni@intel.com>
Cc: Laszlo Ersek <lersek@redhat.com>
Signed-off-by: Eric Dong <eric.dong@intel.com>
---
 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c       | 375 +++++++++++++++
 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h       | 283 +++++++++++
 UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c        | 468 ++++++++++++++++++-
 UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c   |  11 +
 UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h   | 172 ++++++-
 UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf |   3 +
 6 files changed, 1296 insertions(+), 16 deletions(-)
 create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
 create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h

diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c b/UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
new file mode 100644
index 0000000000..8cf69428c2
--- /dev/null
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
@@ -0,0 +1,375 @@
+/** @file
+SMM MP protocol implementation
+
+Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
+
+SPDX-License-Identifier: BSD-2-Clause-Patent
+
+**/
+
+#include "PiSmmCpuDxeSmm.h"
+#include "MpProtocol.h"
+
+///
+/// SMM MP Protocol instance
+///
+EFI_SMM_MP_PROTOCOL  mSmmMp  = {
+  EFI_SMM_MP_PROTOCOL_REVISION,
+  0,
+  SmmMpGetNumberOfProcessors,
+  SmmMpDispatchProcedure,
+  SmmMpBroadcastProcedure,
+  SmmMpSetStartupProcedure,
+  SmmMpCheckForProcedure,
+  SmmMpWaitForProcedure
+};
+
+/**
+  Service to retrieves the number of logical processor in the platform.
+
+  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
+  @param[out] NumberOfProcessors  Pointer to the total number of logical processors in the system,
+                                  including the BSP and all APs.
+
+  @retval EFI_SUCCESS             The number of processors was retrieved successfully
+  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
+**/
+
+EFI_STATUS
+EFIAPI
+SmmMpGetNumberOfProcessors (
+  IN CONST EFI_SMM_MP_PROTOCOL  *This,
+  OUT      UINTN                *NumberOfProcessors
+  )
+{
+  if (NumberOfProcessors == NULL) {
+    return EFI_INVALID_PARAMETER;
+  }
+
+  *NumberOfProcessors = gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus;
+
+  return EFI_SUCCESS;
+}
+
+
+/**
+  This service allows the caller to invoke a procedure one of the application processors (AP). This
+  function uses an optional token parameter to support blocking and non-blocking modes. If the token
+  is passed into the call, the function will operate in a non-blocking fashion and the caller can
+  check for completion with CheckOnProcedure or WaitForProcedure.
+
+  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
+  @param[in]     Procedure              A pointer to the procedure to be run on the designated target
+                                        AP of the system. Type EFI_AP_PROCEDURE2 is defined below in
+                                        related definitions.
+  @param[in]     CpuNumber              The zero-based index of the processor number of the target
+                                        AP, on which the code stream is supposed to run. If the number
+                                        points to the calling processor then it will not run the
+                                        supplied code.
+  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for this AP to
+                                        finish execution of Procedure, either for blocking or
+                                        non-blocking mode. Zero means infinity. If the timeout
+                                        expires before this AP returns from Procedure, then Procedure
+                                        on the AP is terminated. If the timeout expires in blocking
+                                        mode, the call returns EFI_TIMEOUT. If the timeout expires
+                                        in non-blocking mode, the timeout determined can be through
+                                        CheckOnProcedure or WaitForProcedure.
+                                        Note that timeout support is optional. Whether an
+                                        implementation supports this feature, can be determined via
+                                        the Attributes data member.
+  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
+                                        that is run by the AP. It is an optional common mailbox
+                                        between APs and the caller to share information.
+  @param[in,out] Token                  This is parameter is broken into two components:
+                                        1.Token->Completion is an optional parameter that allows the
+                                        caller to execute the procedure in a blocking or non-blocking
+                                        fashion. If it is NULL the call is blocking, and the call will
+                                        not return until the AP has completed the procedure. If the
+                                        token is not NULL, the call will return immediately. The caller
+                                        can check whether the procedure has completed with
+                                        CheckOnProcedure or WaitForProcedure.
+                                        2.Token->Status The implementation updates the address pointed
+                                        at by this variable with the status code returned by Procedure
+                                        when it completes execution on the target AP, or with EFI_TIMEOUT
+                                        if the Procedure fails to complete within the optional timeout.
+                                        The implementation will update this variable with EFI_NOT_READY
+                                        prior to starting Procedure on the target AP
+  @param[in,out] CPUStatus              This optional pointer may be used to get the status code returned
+                                        by Procedure when it completes execution on the target AP, or with
+                                        EFI_TIMEOUT if the Procedure fails to complete within the optional
+                                        timeout. The implementation will update this variable with
+                                        EFI_NOT_READY prior to starting Procedure on the target AP.
+
+  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
+                                        execution on the target AP.
+                                        In the non-blocking case this indicates that the procedure has
+                                        been successfully scheduled for execution on the target AP.
+  @retval EFI_INVALID_PARAMETER         The input arguments are out of range. Either the target AP is the
+                                        caller of the function, or the Procedure or Token is NULL
+  @retval EFI_NOT_READY                 If the target AP is busy executing another procedure
+  @retval EFI_ALREADY_STARTED           Token is already in use for another procedure
+  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before the specified AP
+                                        has finished
+**/
+EFI_STATUS
+EFIAPI
+SmmMpDispatchProcedure (
+  IN CONST EFI_SMM_MP_PROTOCOL           *This,
+  IN       EFI_AP_PROCEDURE2             Procedure,
+  IN       UINTN                         CpuNumber,
+  IN       UINTN                         TimeoutInMicroseconds,
+  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
+  IN OUT   SMM_COMPLETION                *Token,
+  IN OUT   EFI_STATUS                    *CPUStatus
+  )
+{
+  if (Token != NULL) {
+    *Token = AllocatePool (sizeof (SPIN_LOCK));
+    ASSERT (*Token != NULL);
+    InitializeSpinLock ((SPIN_LOCK *)(*Token));
+
+    return InternalSmmStartupThisAp(
+      Procedure,
+      CpuNumber,
+      ProcedureArguments,
+      SmmCpuCallInNewType,
+      (SPIN_LOCK *)(*Token),
+      TimeoutInMicroseconds,
+      CPUStatus
+      );
+  }
+
+  return InternalSmmStartupThisAp(
+    Procedure,
+    CpuNumber,
+    ProcedureArguments,
+    SmmCpuCallInNewType,
+    NULL,
+    TimeoutInMicroseconds,
+    CPUStatus
+    );
+}
+
+/**
+  This service allows the caller to invoke a procedure on all running application processors (AP)
+  except the caller. This function uses an optional token parameter to support blocking and
+  nonblocking modes. If the token is passed into the call, the function will operate in a non-blocking
+  fashion and the caller can check for completion with CheckOnProcedure or WaitForProcedure.
+
+  It is not necessary for the implementation to run the procedure on every processor on the platform.
+  Processors that are powered down in such a way that they cannot respond to interrupts, may be
+  excluded from the broadcast.
+
+
+  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
+  @param[in]     Procedure              A pointer to the code stream to be run on the APs that have
+                                        entered MM. Type EFI_AP_PROCEDURE is defined below in related
+                                        definitions.
+  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for the APs to finish
+                                        execution of Procedure, either for blocking or non-blocking mode.
+                                        Zero means infinity. If the timeout expires before all APs return
+                                        from Procedure, then Procedure on the failed APs is terminated. If
+                                        the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
+                                        If the timeout expires in non-blocking mode, the timeout determined
+                                        can be through CheckOnProcedure or WaitForProcedure.
+                                        Note that timeout support is optional. Whether an implementation
+                                        supports this feature can be determined via the Attributes data
+                                        member.
+  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
+                                        that is run by the AP. It is an optional common mailbox
+                                        between APs and the caller to share information.
+  @param[in,out] Token                  This is parameter is broken into two components:
+                                        1.Token->Completion is an optional parameter that allows the
+                                        caller to execute the procedure in a blocking or non-blocking
+                                        fashion. If it is NULL the call is blocking, and the call will
+                                        not return until the AP has completed the procedure. If the
+                                        token is not NULL, the call will return immediately. The caller
+                                        can check whether the procedure has completed with
+                                        CheckOnProcedure or WaitForProcedure.
+                                        2.Token->Status The implementation updates the address pointed
+                                        at by this variable with the status code returned by Procedure
+                                        when it completes execution on the target AP, or with EFI_TIMEOUT
+                                        if the Procedure fails to complete within the optional timeout.
+                                        The implementation will update this variable with EFI_NOT_READY
+                                        prior to starting Procedure on the target AP
+  @param[in,out] CPUStatus              This optional pointer may be used to get the individual status
+                                        returned by every AP that participated in the broadcast. This
+                                        parameter if used provides the base address of an array to hold
+                                        the EFI_STATUS value of each AP in the system. The size of the
+                                        array can be ascertained by the GetNumberOfProcessors function.
+                                        As mentioned above, the broadcast may not include every processor
+                                        in the system. Some implementations may exclude processors that
+                                        have been powered down in such a way that they are not responsive
+                                        to interrupts. Additionally the broadcast excludes the processor
+                                        which is making the BroadcastProcedure call. For every excluded
+                                        processor, the array entry must contain a value of EFI_NOT_STARTED
+
+  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
+                                        execution on the APs.
+                                        In the non-blocking case this indicates that the procedure has
+                                        been successfully scheduled for execution on the APs.
+  @retval EFI_INVALID_PARAMETER         The Procedure or Token is NULL
+  @retval EFI_NOT_READY                 If the target AP is busy executing another procedure
+  @retval EFI_ALREADY_STARTED           Token is already in use for another procedure
+  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before the specified AP
+                                        has finished.
+
+**/
+EFI_STATUS
+EFIAPI
+SmmMpBroadcastProcedure (
+  IN CONST EFI_SMM_MP_PROTOCOL           *This,
+  IN       EFI_AP_PROCEDURE2             Procedure,
+  IN       UINTN                         TimeoutInMicroseconds,
+  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
+  IN OUT   SMM_COMPLETION                *Token,
+  IN OUT   EFI_STATUS                    *CPUStatus
+  )
+{
+  if (Token != NULL) {
+    *Token = AllocatePool (sizeof (SPIN_LOCK));
+    ASSERT (*Token != NULL);
+    InitializeSpinLock ((SPIN_LOCK *) (*Token));
+
+    return InternalSmmStartupAllAPs(
+      Procedure,
+      TimeoutInMicroseconds,
+      ProcedureArguments,
+      (SPIN_LOCK *) (*Token),
+      CPUStatus
+      );
+  }
+
+  return InternalSmmStartupAllAPs(
+    Procedure,
+    TimeoutInMicroseconds,
+    ProcedureArguments,
+    NULL,
+    CPUStatus
+    );
+}
+
+
+/**
+  This service allows the caller to set a startup procedure that will be executed when an AP powers
+  up from a state where core configuration and context is lost. The procedure is execution has the
+  following properties:
+  1. The procedure executes before the processor is handed over to the operating system.
+  2. All processors execute the same startup procedure.
+  3. The procedure may run in parallel with other procedures invoked through the functions in this
+  protocol, or with processors that are executing an MM handler or running in the operating system.
+
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Procedure            A pointer to the code stream to be run on the designated target AP
+                                       of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
+                                       with the related definitions of
+                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
+                                       If caller may pass a value of NULL to deregister any existing
+                                       startup procedure.
+  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of parameters to the code that is
+                                       run by the AP. It is an optional common mailbox between APs and
+                                       the caller to share information
+
+  @retval EFI_SUCCESS                  The Procedure has been set successfully.
+  @retval EFI_INVALID_PARAMETER        The Procedure is NULL but ProcedureArguments not NULL.
+
+**/
+EFI_STATUS
+EFIAPI
+SmmMpSetStartupProcedure (
+  IN CONST EFI_SMM_MP_PROTOCOL *This,
+  IN       EFI_AP_PROCEDURE    Procedure,
+  IN OUT   VOID                *ProcedureArguments OPTIONAL
+  )
+{
+  return RegisterStartupProcedure (Procedure, ProcedureArguments);
+}
+
+/**
+  When non-blocking execution of a procedure on an AP is invoked with DispatchProcedure,
+  via the use of a token, this function can be used to check for completion of the procedure on the AP.
+  The function takes the token that was passed into the DispatchProcedure call. If the procedure
+  is complete, and therefore it is now possible to run another procedure on the same AP, this function
+  returns EFI_SUCESS. In this case the status returned by the procedure that executed on the AP is
+  returned in the token's Status field. If the procedure has not yet completed, then this function
+  returns EFI_NOT_READY.
+
+  When a non-blocking execution of a procedure is invoked with BroadcastProcedure, via the
+  use of a token, this function can be used to check for completion of the procedure on all the
+  broadcast APs. The function takes the token that was passed into the BroadcastProcedure
+  call. If the procedure is complete on all broadcast APs this function returns EFI_SUCESS. In this
+  case the Status field in the token passed into the function reflects the overall result of the
+  invocation, which may be EFI_SUCCESS, if all executions succeeded, or the first observed failure.
+  If the procedure has not yet completed on the broadcast APs, the function returns
+  EFI_NOT_READY.
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Token                This parameter describes the token that was passed into
+                                       DispatchProcedure or BroadcastProcedure.
+
+  @retval EFI_SUCCESS                  Procedure has completed.
+  @retval EFI_NOT_READY                The Procedure has not completed.
+  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is NULL
+  @retval EFI_NOT_FOUND                Token is not currently in use for a non-blocking call
+
+**/
+EFI_STATUS
+EFIAPI
+SmmMpCheckForProcedure (
+  IN CONST EFI_SMM_MP_PROTOCOL            *This,
+  IN       SMM_COMPLETION                 Token
+  )
+{
+  if (Token == NULL) {
+    return EFI_INVALID_PARAMETER;
+  }
+
+  if (!CurrentUsedToken ((SPIN_LOCK *)Token)) {
+    return EFI_NOT_FOUND;
+  }
+
+  return CheckStatusForThisAP ((SPIN_LOCK *)Token);
+}
+
+/**
+  When a non-blocking execution of a procedure on an AP is invoked via DispatchProcedure,
+  this function will block the caller until the remote procedure has completed on the designated AP.
+  The non-blocking procedure invocation is identified by the Token parameter, which must match the
+  token that used when DispatchProcedure was called. Upon completion the status returned by
+  the procedure that executed on the AP is used to update the token's Status field.
+
+  When a non-blocking execution of a procedure on an AP is invoked via BroadcastProcedure
+  this function will block the caller until the remote procedure has completed on all of the APs that
+  entered MM. The non-blocking procedure invocation is identified by the Token parameter, which
+  must match the token that used when BroadcastProcedure was called. Upon completion the
+  overall status returned by the procedures that executed on the broadcast AP is used to update the
+  token's Status field. The overall status may be EFI_SUCCESS, if all executions succeeded, or the
+  first observed failure.
+
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Token                This parameter describes the token that was passed into
+                                       DispatchProcedure or BroadcastProcedure.
+
+  @retval EFI_SUCCESS                  Procedure has completed.
+  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is NULL
+  @retval EFI_NOT_FOUND                Token is not currently in use for a non-blocking call
+
+**/
+EFI_STATUS
+EFIAPI
+SmmMpWaitForProcedure (
+  IN CONST EFI_SMM_MP_PROTOCOL            *This,
+  IN       SMM_COMPLETION                 Token
+  )
+{
+  EFI_STATUS    Status;
+
+  do {
+    Status = SmmMpCheckForProcedure (This, Token);
+  } while (Status == EFI_NOT_READY);
+
+  return Status;
+}
+
diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h b/UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h
new file mode 100644
index 0000000000..a1b4a0a671
--- /dev/null
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h
@@ -0,0 +1,283 @@
+/** @file
+Include file for SMM MP protocol implementation.
+
+Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
+
+SPDX-License-Identifier: BSD-2-Clause-Patent
+
+**/
+
+#ifndef _SMM_MP_PROTOCOL_H_
+#define _SMM_MP_PROTOCOL_H_
+
+//
+// SMM MP Protocol function prototypes.
+//
+
+/**
+  Service to retrieves the number of logical processor in the platform.
+
+  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
+  @param[out] NumberOfProcessors  Pointer to the total number of logical processors in the system,
+                                  including the BSP and all APs.
+
+  @retval EFI_SUCCESS             The number of processors was retrieved successfully
+  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
+**/
+
+EFI_STATUS
+EFIAPI
+SmmMpGetNumberOfProcessors (
+  IN CONST EFI_SMM_MP_PROTOCOL  *This,
+  OUT      UINTN                *NumberOfProcessors
+  );
+
+
+/**
+  This service allows the caller to invoke a procedure one of the application processors (AP). This
+  function uses an optional token parameter to support blocking and non-blocking modes. If the token
+  is passed into the call, the function will operate in a non-blocking fashion and the caller can
+  check for completion with CheckOnProcedure or WaitForProcedure.
+
+  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
+  @param[in]     Procedure              A pointer to the procedure to be run on the designated target
+                                        AP of the system. Type EFI_AP_PROCEDURE2 is defined below in
+                                        related definitions.
+  @param[in]     CpuNumber              The zero-based index of the processor number of the target
+                                        AP, on which the code stream is supposed to run. If the number
+                                        points to the calling processor then it will not run the
+                                        supplied code.
+  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for this AP to
+                                        finish execution of Procedure, either for blocking or
+                                        non-blocking mode. Zero means infinity. If the timeout
+                                        expires before this AP returns from Procedure, then Procedure
+                                        on the AP is terminated. If the timeout expires in blocking
+                                        mode, the call returns EFI_TIMEOUT. If the timeout expires
+                                        in non-blocking mode, the timeout determined can be through
+                                        CheckOnProcedure or WaitForProcedure.
+                                        Note that timeout support is optional. Whether an
+                                        implementation supports this feature, can be determined via
+                                        the Attributes data member.
+  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
+                                        that is run by the AP. It is an optional common mailbox
+                                        between APs and the caller to share information.
+  @param[in,out] Token                  This is parameter is broken into two components:
+                                        1.Token->Completion is an optional parameter that allows the
+                                        caller to execute the procedure in a blocking or non-blocking
+                                        fashion. If it is NULL the call is blocking, and the call will
+                                        not return until the AP has completed the procedure. If the
+                                        token is not NULL, the call will return immediately. The caller
+                                        can check whether the procedure has completed with
+                                        CheckOnProcedure or WaitForProcedure.
+                                        2.Token->Status The implementation updates the address pointed
+                                        at by this variable with the status code returned by Procedure
+                                        when it completes execution on the target AP, or with EFI_TIMEOUT
+                                        if the Procedure fails to complete within the optional timeout.
+                                        The implementation will update this variable with EFI_NOT_READY
+                                        prior to starting Procedure on the target AP
+  @param[in,out] CPUStatus              This optional pointer may be used to get the status code returned
+                                        by Procedure when it completes execution on the target AP, or with
+                                        EFI_TIMEOUT if the Procedure fails to complete within the optional
+                                        timeout. The implementation will update this variable with
+                                        EFI_NOT_READY prior to starting Procedure on the target AP.
+
+  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
+                                        execution on the target AP.
+                                        In the non-blocking case this indicates that the procedure has
+                                        been successfully scheduled for execution on the target AP.
+  @retval EFI_INVALID_PARAMETER         The input arguments are out of range. Either the target AP is the
+                                        caller of the function, or the Procedure or Token is NULL
+  @retval EFI_NOT_READY                 If the target AP is busy executing another procedure
+  @retval EFI_ALREADY_STARTED           Token is already in use for another procedure
+  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before the specified AP
+                                        has finished
+**/
+EFI_STATUS
+EFIAPI
+SmmMpDispatchProcedure (
+  IN CONST EFI_SMM_MP_PROTOCOL            *This,
+  IN       EFI_AP_PROCEDURE2              Procedure,
+  IN       UINTN                          CpuNumber,
+  IN       UINTN                          TimeoutInMicroseconds,
+  IN OUT   VOID                           *ProcedureArguments OPTIONAL,
+  IN OUT   SMM_COMPLETION                 *Token,
+  IN OUT   EFI_STATUS                     *CPUStatus
+  );
+
+/**
+  This service allows the caller to invoke a procedure on all running application processors (AP)
+  except the caller. This function uses an optional token parameter to support blocking and
+  nonblocking modes. If the token is passed into the call, the function will operate in a non-blocking
+  fashion and the caller can check for completion with CheckOnProcedure or WaitForProcedure.
+
+  It is not necessary for the implementation to run the procedure on every processor on the platform.
+  Processors that are powered down in such a way that they cannot respond to interrupts, may be
+  excluded from the broadcast.
+
+
+  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
+  @param[in]     Procedure              A pointer to the code stream to be run on the APs that have
+                                        entered MM. Type EFI_AP_PROCEDURE is defined below in related
+                                        definitions.
+  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for the APs to finish
+                                        execution of Procedure, either for blocking or non-blocking mode.
+                                        Zero means infinity. If the timeout expires before all APs return
+                                        from Procedure, then Procedure on the failed APs is terminated. If
+                                        the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
+                                        If the timeout expires in non-blocking mode, the timeout determined
+                                        can be through CheckOnProcedure or WaitForProcedure.
+                                        Note that timeout support is optional. Whether an implementation
+                                        supports this feature can be determined via the Attributes data
+                                        member.
+  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
+                                        that is run by the AP. It is an optional common mailbox
+                                        between APs and the caller to share information.
+  @param[in,out] Token                  This is parameter is broken into two components:
+                                        1.Token->Completion is an optional parameter that allows the
+                                        caller to execute the procedure in a blocking or non-blocking
+                                        fashion. If it is NULL the call is blocking, and the call will
+                                        not return until the AP has completed the procedure. If the
+                                        token is not NULL, the call will return immediately. The caller
+                                        can check whether the procedure has completed with
+                                        CheckOnProcedure or WaitForProcedure.
+                                        2.Token->Status The implementation updates the address pointed
+                                        at by this variable with the status code returned by Procedure
+                                        when it completes execution on the target AP, or with EFI_TIMEOUT
+                                        if the Procedure fails to complete within the optional timeout.
+                                        The implementation will update this variable with EFI_NOT_READY
+                                        prior to starting Procedure on the target AP
+  @param[in,out] CPUStatus              This optional pointer may be used to get the individual status
+                                        returned by every AP that participated in the broadcast. This
+                                        parameter if used provides the base address of an array to hold
+                                        the EFI_STATUS value of each AP in the system. The size of the
+                                        array can be ascertained by the GetNumberOfProcessors function.
+                                        As mentioned above, the broadcast may not include every processor
+                                        in the system. Some implementations may exclude processors that
+                                        have been powered down in such a way that they are not responsive
+                                        to interrupts. Additionally the broadcast excludes the processor
+                                        which is making the BroadcastProcedure call. For every excluded
+                                        processor, the array entry must contain a value of EFI_NOT_STARTED
+
+  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
+                                        execution on the APs.
+                                        In the non-blocking case this indicates that the procedure has
+                                        been successfully scheduled for execution on the APs.
+  @retval EFI_INVALID_PARAMETER         The Procedure or Token is NULL
+  @retval EFI_NOT_READY                 If the target AP is busy executing another procedure
+  @retval EFI_ALREADY_STARTED           Token is already in use for another procedure
+  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before the specified AP
+                                        has finished
+
+**/
+EFI_STATUS
+EFIAPI
+SmmMpBroadcastProcedure (
+  IN CONST EFI_SMM_MP_PROTOCOL            *This,
+  IN       EFI_AP_PROCEDURE2              Procedure,
+  IN       UINTN                          TimeoutInMicroseconds,
+  IN OUT   VOID                           *ProcedureArguments OPTIONAL,
+  IN OUT   SMM_COMPLETION                 *Token,
+  IN OUT   EFI_STATUS                     *CPUStatus
+  );
+
+
+/**
+  This service allows the caller to set a startup procedure that will be executed when an AP powers
+  up from a state where core configuration and context is lost. The procedure is execution has the
+  following properties:
+  1. The procedure executes before the processor is handed over to the operating system.
+  2. All processors execute the same startup procedure.
+  3. The procedure may run in parallel with other procedures invoked through the functions in this
+  protocol, or with processors that are executing an MM handler or running in the operating system.
+
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Procedure            A pointer to the code stream to be run on the designated target AP
+                                       of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
+                                       with the related definitions of
+                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
+                                       If caller may pass a value of NULL to deregister any existing
+                                       startup procedure.
+  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of parameters to the code that is
+                                       run by the AP. It is an optional common mailbox between APs and
+                                       the caller to share information
+
+  @retval EFI_SUCCESS                  The Procedure has been set successfully.
+  @retval EFI_INVALID_PARAMETER        The Procedure is NULL but ProcedureArguments not NULL.
+**/
+EFI_STATUS
+EFIAPI
+SmmMpSetStartupProcedure (
+  IN CONST EFI_SMM_MP_PROTOCOL *This,
+  IN       EFI_AP_PROCEDURE    Procedure,
+  IN OUT   VOID                *ProcedureArguments OPTIONAL
+  );
+
+/**
+  When non-blocking execution of a procedure on an AP is invoked with DispatchProcedure,
+  via the use of a token, this function can be used to check for completion of the procedure on the AP.
+  The function takes the token that was passed into the DispatchProcedure call. If the procedure
+  is complete, and therefore it is now possible to run another procedure on the same AP, this function
+  returns EFI_SUCESS. In this case the status returned by the procedure that executed on the AP is
+  returned in the token's Status field. If the procedure has not yet completed, then this function
+  returns EFI_NOT_READY.
+
+  When a non-blocking execution of a procedure is invoked with BroadcastProcedure, via the
+  use of a token, this function can be used to check for completion of the procedure on all the
+  broadcast APs. The function takes the token that was passed into the BroadcastProcedure
+  call. If the procedure is complete on all broadcast APs this function returns EFI_SUCESS. In this
+  case the Status field in the token passed into the function reflects the overall result of the
+  invocation, which may be EFI_SUCCESS, if all executions succeeded, or the first observed failure.
+  If the procedure has not yet completed on the broadcast APs, the function returns
+  EFI_NOT_READY.
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Token                This parameter describes the token that was passed into
+                                       DispatchProcedure or BroadcastProcedure.
+
+  @retval EFI_SUCCESS                  Procedure has completed.
+  @retval EFI_NOT_READY                The Procedure has not completed.
+  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is NULL
+  @retval EFI_NOT_FOUND                Token is not currently in use for a non-blocking call
+
+**/
+EFI_STATUS
+EFIAPI
+SmmMpCheckForProcedure (
+  IN CONST EFI_SMM_MP_PROTOCOL            *This,
+  IN       SMM_COMPLETION                 Token
+  );
+
+/**
+  When a non-blocking execution of a procedure on an AP is invoked via DispatchProcedure,
+  this function will block the caller until the remote procedure has completed on the designated AP.
+  The non-blocking procedure invocation is identified by the Token parameter, which must match the
+  token that used when DispatchProcedure was called. Upon completion the status returned by
+  the procedure that executed on the AP is used to update the token's Status field.
+
+  When a non-blocking execution of a procedure on an AP is invoked via BroadcastProcedure
+  this function will block the caller until the remote procedure has completed on all of the APs that
+  entered MM. The non-blocking procedure invocation is identified by the Token parameter, which
+  must match the token that used when BroadcastProcedure was called. Upon completion the
+  overall status returned by the procedures that executed on the broadcast AP is used to update the
+  token's Status field. The overall status may be EFI_SUCCESS, if all executions succeeded, or the
+  first observed failure.
+
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Token                This parameter describes the token that was passed into
+                                       DispatchProcedure or BroadcastProcedure.
+
+  @retval EFI_SUCCESS                  Procedure has completed.
+  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is NULL
+  @retval EFI_NOT_FOUND                Token is not currently in use for a non-blocking call
+
+**/
+EFI_STATUS
+EFIAPI
+SmmMpWaitForProcedure (
+  IN CONST EFI_SMM_MP_PROTOCOL            *This,
+  IN       SMM_COMPLETION                 Token
+  );
+
+#endif
diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
index 64fb4d6344..6b08dc0c4e 100644
--- a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
@@ -146,6 +146,56 @@ ReleaseAllAPs (
   }
 }
 
+/**
+  Wheck whether task has been finished by all APs.
+
+  @retval      TRUE        Task has been finished by all APs.
+  @retval      FALSE       Task not has been finished by all APs.
+
+**/
+BOOLEAN
+TaskFinishInAllAPs (
+  VOID
+  )
+{
+  UINTN                             Index;
+  UINTN                             BspIndex;
+
+  BspIndex = mSmmMpSyncData->BspIndex;
+
+  for (Index = mMaxNumberOfCpus; Index-- > 0;) {
+    if (Index != BspIndex && *(mSmmMpSyncData->CpuData[Index].Present) && !mSmmMpSyncData->FinishTask[Index]) {
+      return FALSE;
+    }
+  }
+
+  return TRUE;
+}
+
+/**
+  Checks whenther executive procedure in sync mode.
+
+  @param[in]       CallInType               Which call for this function.
+  @param[in]       Token                    Token input by caller.
+
+
+  @retval   TRUE    Executive procedure in sync mode.
+  @retval   FALSE   Executive procedure not in sync mode.
+
+**/
+BOOLEAN
+ExecutiveInSyncMode (
+  IN      SMM_CPU_CALL_IN_TYPE           CallInType,
+  IN      SPIN_LOCK                      *Token
+  )
+{
+  if (CallInType == SmmCpuCallInOldSyncType) {
+    return TRUE;
+  }
+
+  return Token == NULL;
+}
+
 /**
   Checks if all CPUs (with certain exceptions) have checked in for this SMI run
 
@@ -347,6 +397,30 @@ ReplaceOSMtrrs (
   MtrrSetAllMtrrs (&gSmiMtrrs);
 }
 
+/**
+  Clear the flags used for execute one procedure.
+
+  @param     BlockStyle        Is this procedure block style or non-block.
+  @param     SingleProcessor   Is this procedure only for single processor.
+
+**/
+VOID
+ClearProcedureFlags (
+  IN BOOLEAN             BlockStyle,
+  IN BOOLEAN             SingleProcessor
+  )
+{
+  mSmmMpSyncData->CpuCheckMode = SmmCpuCheckModeMax;
+
+  if (SingleProcessor) {
+    mSmmMpSyncData->CpuIndex = (UINTN) -1;
+  }
+
+  if (!BlockStyle) {
+    ReleaseSpinLock (mSmmMpSyncData->CurrentToken);
+  }
+}
+
 /**
   SMI handler for BSP.
 
@@ -604,6 +678,7 @@ APHandler (
   UINT64                            Timer;
   UINTN                             BspIndex;
   MTRR_SETTINGS                     Mtrrs;
+  EFI_STATUS                        ProcedureStatus;
 
   //
   // Timeout BSP
@@ -730,14 +805,43 @@ APHandler (
     //
     // Invoke the scheduled procedure
     //
-    (*mSmmMpSyncData->CpuData[CpuIndex].Procedure) (
-      (VOID*)mSmmMpSyncData->CpuData[CpuIndex].Parameter
-      );
+    ProcedureStatus = (*mSmmMpSyncData->CpuData[CpuIndex].Procedure) (
+                          (VOID*)mSmmMpSyncData->CpuData[CpuIndex].Parameter
+                          );
+
+    if (mSmmMpSyncData->CurrentToken != NULL) {
+      if (mSmmMpSyncData->CpuCheckMode == SmmCpuCheckModeSingle) {
+        if (mSmmMpSyncData->CpuStatus != NULL) {
+          *mSmmMpSyncData->CpuStatus = ProcedureStatus;
+        }
+      } else {
+        ASSERT (mSmmMpSyncData->CpuCheckMode == SmmCpuCheckModeAll);
+        if (mSmmMpSyncData->CpuStatus != NULL) {
+          mSmmMpSyncData->CpuStatus[CpuIndex] = ProcedureStatus;
+        }
+      }
+    }
 
     //
     // Release BUSY
     //
     ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
+
+    //
+    // In non-block mode, Update the sync flags (CpuCheckMode, CpuIndex, FinishTask, CurrentToken) here.
+    //
+    if (mSmmMpSyncData->CurrentToken != NULL) {
+      if (mSmmMpSyncData->CpuCheckMode == SmmCpuCheckModeSingle) {
+        if (mSmmMpSyncData->CpuIndex == CpuIndex) {
+          ClearProcedureFlags (FALSE, TRUE);
+        }
+      } else if (mSmmMpSyncData->CpuCheckMode == SmmCpuCheckModeAll) {
+        mSmmMpSyncData->FinishTask[CpuIndex] = TRUE;
+        if (TaskFinishInAllAPs()) {
+          ClearProcedureFlags (FALSE, FALSE);
+        }
+      }
+    }
   }
 
   if (SmmCpuFeaturesNeedConfigureMtrrs()) {
@@ -906,13 +1010,82 @@ Gen4GPageTable (
   return (UINT32)(UINTN)PageTable;
 }
 
+/**
+  Checks whether the input token is the current used token.
+
+
+  @param[in]  Token      This parameter describes the token that was passed into DispatchProcedure or
+                         BroadcastProcedure.
+
+  @retval TRUE           The input token is the current used token.
+  @retval FALSE          The input token is not the current used token.
+**/
+BOOLEAN
+CurrentUsedToken (
+  IN SPIN_LOCK           *Token
+  )
+{
+  return mSmmMpSyncData->CurrentToken == Token;
+}
+
+/**
+  Checks status of specified AP.
+
+  This function checks whether the specified AP has finished the task assigned
+  by StartupThisAP(), and whether timeout expires.
+
+  @param[in]  Token             This parameter describes the token that was passed into DispatchProcedure or
+                                BroadcastProcedure.
+
+  @retval EFI_SUCCESS           Specified AP has finished task assigned by StartupThisAPs().
+  @retval EFI_NOT_READY         Specified AP has not finished task and timeout has not expired.
+**/
+EFI_STATUS
+CheckStatusForThisAP (
+  IN SPIN_LOCK          *Token
+  )
+{
+  if (AcquireSpinLockOrFail (Token)) {
+    ReleaseSpinLock (Token);
+    return EFI_SUCCESS;
+  }
+
+  return EFI_NOT_READY;
+}
+
+
 /**
   Schedule a procedure to run on the specified CPU.
 
   @param[in]       Procedure                The address of the procedure to run
   @param[in]       CpuIndex                 Target CPU Index
   @param[in, out]  ProcArguments            The parameter to pass to the procedure
-  @param[in]       BlockingMode             Startup AP in blocking mode or not
+  @param[in]       CallInType               Which call for this function.
+  @param[in, out]  Token                    This is parameter is broken into two components:
+                                            1.Token->Completion is an optional parameter that allows the
+                                            caller to execute the procedure in a blocking or non-blocking
+                                            fashion. If it is NULL the call is blocking, and the call will
+                                            not return until the AP has completed the procedure. If the
+                                            token is not NULL, the call will return immediately. The caller
+                                            can check whether the procedure has completed with
+                                            CheckOnProcedure or WaitForProcedure.
+                                            2.Token->Status The implementation updates the address pointed
+                                            at by this variable with the status code returned by Procedure
+                                            when it completes execution on the target AP, or with EFI_TIMEOUT
+                                            if the Procedure fails to complete within the optional timeout.
+                                            The implementation will update this variable with EFI_NOT_READY
+                                            prior to starting Procedure on the target AP
+  @param[in]       TimeoutInMicroseconds    Indicates the time limit in microseconds for the APs to finish
+                                            execution of Procedure, either for blocking or non-blocking mode.
+                                            Zero means infinity. If the timeout expires before all APs return
+                                            from Procedure, then Procedure on the failed APs is terminated. If
+                                            the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
+                                            If the timeout expires in non-blocking mode, the timeout determined
+                                            can be through CheckOnProcedure or WaitForProcedure.
+                                            Note that timeout support is optional. Whether an implementation
+                                            supports this feature can be determined via the Attributes data
+                                            member.
+
 
   @retval EFI_INVALID_PARAMETER    CpuNumber not valid
   @retval EFI_INVALID_PARAMETER    CpuNumber specifying BSP
@@ -923,10 +1096,13 @@ Gen4GPageTable (
 **/
 EFI_STATUS
 InternalSmmStartupThisAp (
-  IN      EFI_AP_PROCEDURE          Procedure,
-  IN      UINTN                     CpuIndex,
-  IN OUT  VOID                      *ProcArguments OPTIONAL,
-  IN      BOOLEAN                   BlockingMode
+  IN      EFI_AP_PROCEDURE               Procedure,
+  IN      UINTN                          CpuIndex,
+  IN OUT  VOID                           *ProcArguments OPTIONAL,
+  IN      SMM_CPU_CALL_IN_TYPE           CallInType,
+  IN      SPIN_LOCK                      *Token,
+  IN      UINTN                          TimeoutInMicroseconds,
+  IN      EFI_STATUS                     *CpuStatus
   )
 {
   if (CpuIndex >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus) {
@@ -952,27 +1128,231 @@ InternalSmmStartupThisAp (
     }
     return EFI_INVALID_PARAMETER;
   }
+  if ((TimeoutInMicroseconds != 0) && ((mSmmMp.Attributes & EFI_SMM_MP_TIMEOUT_SUPPORTED) == 0)) {
+    DEBUG((DEBUG_ERROR, "Input TimeoutInMicroseconds != 0 but EFI_SMM_MP_TIMEOUT_SUPPORTED not supported!\n"));
+    return EFI_INVALID_PARAMETER;
+  }
+  if (Procedure == NULL) {
+    return EFI_INVALID_PARAMETER;
+  }
+  if (Token != NULL && Token == mSmmMpSyncData->CurrentToken) {
+    return EFI_ALREADY_STARTED;
+  }
+  if (!AcquireSpinLockOrFail (mSmmMpSyncData->CpuData[CpuIndex].Busy)) {
+    return EFI_NOT_READY;
+  } else {
+    ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
+  }
 
-  if (BlockingMode) {
+  if (ExecutiveInSyncMode (CallInType, Token)) {
     AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
   } else {
+    if (Token != NULL && !AcquireSpinLockOrFail (Token)) {
+      DEBUG((DEBUG_ERROR, "Token->Completion can't acquire\n"));
+      return EFI_INVALID_PARAMETER;
+    }
     if (!AcquireSpinLockOrFail (mSmmMpSyncData->CpuData[CpuIndex].Busy)) {
       DEBUG((DEBUG_ERROR, "mSmmMpSyncData->CpuData[%d].Busy\n", CpuIndex));
-      return EFI_INVALID_PARAMETER;
+      ReleaseSpinLock (Token);
+      return EFI_NOT_READY;
     }
   }
 
-  mSmmMpSyncData->CpuData[CpuIndex].Procedure = Procedure;
+  //
+  // PI spec requirement in EFI_MM_MP_PROTOCOL.DispatchProcedure():
+  // The implementation will update this variable with
+  // EFI_NOT_READY prior to starting Procedure on the target AP.
+  //
+  mSmmMpSyncData->CurrentToken = Token;
+  mSmmMpSyncData->CpuCheckMode = SmmCpuCheckModeSingle;
+  mSmmMpSyncData->CpuIndex = CpuIndex;
+  mSmmMpSyncData->CpuStatus = CpuStatus;
+  if (mSmmMpSyncData->CpuStatus != NULL) {
+    *mSmmMpSyncData->CpuStatus = EFI_NOT_READY;
+  }
+
+  mSmmMpSyncData->CpuData[CpuIndex].Procedure = (EFI_AP_PROCEDURE2) Procedure;
   mSmmMpSyncData->CpuData[CpuIndex].Parameter = ProcArguments;
+
   ReleaseSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);
 
-  if (BlockingMode) {
+  if (ExecutiveInSyncMode (CallInType, Token)) {
     AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
     ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
+
+    //
+    // In block mode, update the sync flags (CpuCheckMode, CpuIndex, FinishTask, CurrentToken) here.
+    //
+    if (Token == NULL) {
+      ClearProcedureFlags (TRUE, TRUE);
+    }
+  }
+
+  return EFI_SUCCESS;
+}
+
+/**
+  Worker function to execute a caller provided function on all enabled APs.
+
+  @param[in]  Procedure               A pointer to the function to be run on
+                                      enabled APs of the system.
+  @param[in]  SingleThread            If TRUE, then all the enabled APs execute
+                                      the function specified by Procedure one by
+                                      one, in ascending order of processor handle
+                                      number.  If FALSE, then all the enabled APs
+                                      execute the function specified by Procedure
+                                      simultaneously.
+  @param[in]  WaitEvent               The event created by the caller with CreateEvent()
+                                      service.
+  @param[in]  TimeoutInMicroseconds   Indicates the time limit in microseconds for
+                                      APs to return from Procedure, either for
+                                      blocking or non-blocking mode.
+  @param[in]  ProcedureArgument       The parameter passed into Procedure for
+                                      all APs.
+  @param[out] FailedCpuList           If all APs finish successfully, then its
+                                      content is set to NULL. If not all APs
+                                      finish before timeout expires, then its
+                                      content is set to address of the buffer
+                                      holding handle numbers of the failed APs.
+
+  @retval EFI_SUCCESS             In blocking mode, all APs have finished before
+                                  the timeout expired.
+  @retval EFI_SUCCESS             In non-blocking mode, function has been dispatched
+                                  to all enabled APs.
+  @retval others                  Failed to Startup all APs.
+
+**/
+EFI_STATUS
+InternalSmmStartupAllAPs (
+  IN       EFI_AP_PROCEDURE2             Procedure,
+  IN       UINTN                         TimeoutInMicroseconds,
+  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
+  IN OUT   SPIN_LOCK                     *Token,
+  IN OUT   EFI_STATUS                    *CPUStatus
+  )
+{
+  UINTN               Index;
+  UINTN               CpuCount;
+
+  if (Procedure == NULL) {
+    return EFI_INVALID_PARAMETER;
+  }
+  if ((TimeoutInMicroseconds != 0) && ((mSmmMp.Attributes & EFI_SMM_MP_TIMEOUT_SUPPORTED) == 0)) {
+    return EFI_INVALID_PARAMETER;
+  }
+  if (Token != NULL && Token == mSmmMpSyncData->CurrentToken) {
+    return EFI_ALREADY_STARTED;
+  }
+
+  CpuCount = 0;
+  for (Index = mMaxNumberOfCpus; Index-- > 0;) {
+    if (*mSmmMpSyncData->CpuData[Index].Present && (Index != gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu)) {
+      CpuCount ++;
+
+      if (gSmmCpuPrivate->Operation[Index] == SmmCpuRemove) {
+        return EFI_INVALID_PARAMETER;
+      }
+
+      if (!AcquireSpinLockOrFail(mSmmMpSyncData->CpuData[Index].Busy)) {
+        return EFI_NOT_READY;
+      }
+      ReleaseSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
+    }
+  }
+  if (CpuCount == 0) {
+    return EFI_NOT_STARTED;
+  }
+
+  if (Token == NULL) {
+    //
+    // Make sure all BUSY should be acquired.
+    //
+    for (Index = mMaxNumberOfCpus; Index-- > 0;) {
+      if (Index != gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu && *(mSmmMpSyncData->CpuData[Index].Present)) {
+        AcquireSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
+      }
+    }
+  } else {
+    if (!AcquireSpinLockOrFail (Token)) {
+      return EFI_INVALID_PARAMETER;
+    }
+
+    //
+    // Make sure all BUSY should be acquired.
+    //
+    for (Index = mMaxNumberOfCpus; Index-- > 0;) {
+      if (Index != gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu && *(mSmmMpSyncData->CpuData[Index].Present)) {
+        if (!AcquireSpinLockOrFail (mSmmMpSyncData->CpuData[Index].Busy)) {
+          DEBUG((DEBUG_ERROR, "mSmmMpSyncData->CpuData[%d].Busy\n", Index));
+
+          //
+          // Release BUSY accquired before.
+          //
+          for (CpuCount = mMaxNumberOfCpus; CpuCount -- > Index;) {
+            if (CpuCount != gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu && *(mSmmMpSyncData->CpuData[CpuCount].Present)) {
+              ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuCount].Busy);
+            }
+          }
+
+          ReleaseSpinLock (Token);
+          return EFI_INVALID_PARAMETER;
+        }
+      }
+    }
+  }
+
+  //
+  // PI spec requirement:
+  // The implementation will update this variable with
+  // EFI_NOT_READY prior to starting Procedure on the target AP.
+  //
+  mSmmMpSyncData->CurrentToken = Token;
+  mSmmMpSyncData->CpuCheckMode = SmmCpuCheckModeAll;
+  mSmmMpSyncData->CpuStatus = CPUStatus;
+
+  for (Index = mMaxNumberOfCpus; Index-- > 0;) {
+    if (Index != gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu && *(mSmmMpSyncData->CpuData[Index].Present)) {
+      mSmmMpSyncData->CpuData[Index].Procedure = (EFI_AP_PROCEDURE2) Procedure;
+      mSmmMpSyncData->CpuData[Index].Parameter = ProcedureArguments;
+    } else {
+      //
+      // PI spec requirement:
+      // For every excluded processor, the array entry must contain a value of EFI_NOT_STARTED.
+      //
+      if (mSmmMpSyncData->CpuStatus != NULL) {
+        mSmmMpSyncData->CpuStatus[Index] = EFI_NOT_STARTED;
+      }
+    }
+    //
+    // Clear all status no matter this processor is used or not.
+    // May not need to clear all but add it to make the status clear.
+    //
+    mSmmMpSyncData->FinishTask[Index] = FALSE;
   }
+
+  ReleaseAllAPs ();
+
+  if (Token == NULL) {
+    //
+    // Make sure all APs have completed their tasks.
+    //
+    for (Index = mMaxNumberOfCpus; Index-- > 0;) {
+      if (Index != gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu && *(mSmmMpSyncData->CpuData[Index].Present)) {
+        AcquireSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
+        ReleaseSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
+      }
+    }
+
+    //
+    // Update the sync flags (CpuCheckMode, CpuIndex, FinishTask, CurrentToken) in block mode here.
+    //
+    ClearProcedureFlags (TRUE, FALSE);
+  }
+
   return EFI_SUCCESS;
 }
 
+
 /**
   Schedule a procedure to run on the specified CPU in blocking mode.
 
@@ -995,7 +1375,7 @@ SmmBlockingStartupThisAp (
   IN OUT  VOID                      *ProcArguments OPTIONAL
   )
 {
-  return InternalSmmStartupThisAp(Procedure, CpuIndex, ProcArguments, TRUE);
+  return InternalSmmStartupThisAp(Procedure, CpuIndex, ProcArguments, SmmCpuCallInOldSyncType, NULL, 0, NULL);
 }
 
 /**
@@ -1020,7 +1400,15 @@ SmmStartupThisAp (
   IN OUT  VOID                      *ProcArguments OPTIONAL
   )
 {
-  return InternalSmmStartupThisAp(Procedure, CpuIndex, ProcArguments, FeaturePcdGet (PcdCpuSmmBlockStartupThisAp));
+  SMM_CPU_CALL_IN_TYPE    CallInType;
+
+  if (FeaturePcdGet (PcdCpuSmmBlockStartupThisAp)) {
+    CallInType = SmmCpuCallInOldSyncType;
+  } else {
+    CallInType = SmmCpuCallInOldAsyncType;
+  }
+
+  return InternalSmmStartupThisAp(Procedure, CpuIndex, ProcArguments, CallInType, NULL, 0, NULL);
 }
 
 /**
@@ -1112,6 +1500,13 @@ SmiRendezvous (
   Cr2 = 0;
   SaveCr2 (&Cr2);
 
+  //
+  // Call the user register Startup function first.
+  //
+  if (mSmmMpSyncData->StartupProcedure != NULL) {
+    mSmmMpSyncData->StartupProcedure (mSmmMpSyncData->StartupProcArgs);
+  }
+
   //
   // Perform CPU specific entry hooks
   //
@@ -1332,6 +1727,7 @@ InitializeMpSyncData (
     ZeroMem (mSmmMpSyncData, mSmmMpSyncDataSize);
     mSmmMpSyncData->CpuData = (SMM_CPU_DATA_BLOCK *)((UINT8 *)mSmmMpSyncData + sizeof (SMM_DISPATCHER_MP_SYNC_DATA));
     mSmmMpSyncData->CandidateBsp = (BOOLEAN *)(mSmmMpSyncData->CpuData + gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus);
+    mSmmMpSyncData->FinishTask = (BOOLEAN *)(mSmmMpSyncData->CandidateBsp + gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus);
     if (FeaturePcdGet (PcdCpuSmmEnableBspElection)) {
       //
       // Enable BSP election by setting BspIndex to -1
@@ -1360,6 +1756,10 @@ InitializeMpSyncData (
       *(mSmmMpSyncData->CpuData[CpuIndex].Run)     = 0;
       *(mSmmMpSyncData->CpuData[CpuIndex].Present) = FALSE;
     }
+
+    mSmmMpSyncData->CpuCheckMode = SmmCpuCheckModeMax;
+    mSmmMpSyncData->CpuIndex = CpuIndex;
+    mSmmMpSyncData->CurrentToken = NULL;
   }
 }
 
@@ -1399,7 +1799,7 @@ InitializeMpServiceData (
   // Initialize mSmmMpSyncData
   //
   mSmmMpSyncDataSize = sizeof (SMM_DISPATCHER_MP_SYNC_DATA) +
-                       (sizeof (SMM_CPU_DATA_BLOCK) + sizeof (BOOLEAN)) * gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus;
+                       (sizeof (SMM_CPU_DATA_BLOCK) + sizeof (BOOLEAN) + sizeof (BOOLEAN)) * gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus;
   mSmmMpSyncData = (SMM_DISPATCHER_MP_SYNC_DATA*) AllocatePages (EFI_SIZE_TO_PAGES (mSmmMpSyncDataSize));
   ASSERT (mSmmMpSyncData != NULL);
   mCpuSmmSyncMode = (SMM_CPU_SYNC_MODE)PcdGet8 (PcdCpuSmmSyncMode);
@@ -1469,3 +1869,41 @@ RegisterSmmEntry (
   gSmmCpuPrivate->SmmCoreEntry = SmmEntryPoint;
   return EFI_SUCCESS;
 }
+
+/**
+
+  Register the SMM Foundation entry point.
+
+  @param[in]      Procedure            A pointer to the code stream to be run on the designated target AP
+                                       of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
+                                       with the related definitions of
+                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
+                                       If caller may pass a value of NULL to deregister any existing
+                                       startup procedure.
+  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of parameters to the code that is
+                                       run by the AP. It is an optional common mailbox between APs and
+                                       the caller to share information
+
+  @retval EFI_SUCCESS                  The Procedure has been set successfully.
+  @retval EFI_INVALID_PARAMETER        The Procedure is NULL but ProcedureArguments not NULL.
+
+**/
+EFI_STATUS
+RegisterStartupProcedure (
+  IN EFI_AP_PROCEDURE    Procedure,
+  IN VOID                *ProcedureArguments OPTIONAL
+  )
+{
+  if (Procedure == NULL && ProcedureArguments != NULL) {
+    return EFI_INVALID_PARAMETER;
+  }
+  if (mSmmMpSyncData == NULL) {
+    return EFI_NOT_READY;
+  }
+
+  mSmmMpSyncData->StartupProcedure = Procedure;
+  mSmmMpSyncData->StartupProcArgs  = ProcedureArguments;
+
+  return EFI_SUCCESS;
+}
+
diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c
index 2f7d777ee7..e498f72801 100644
--- a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c
@@ -996,6 +996,17 @@ PiCpuSmmEntry (
                     );
   ASSERT_EFI_ERROR (Status);
 
+  //
+  // Install the SMM Mp Protocol into SMM protocol database
+  //
+  Status = gSmst->SmmInstallProtocolInterface (
+                    &mSmmCpuHandle,
+                    &gEfiSmmMpProtocolGuid,
+                    EFI_NATIVE_INTERFACE,
+                    &mSmmMp
+                    );
+  ASSERT_EFI_ERROR (Status);
+
   //
   // Expose address of CPU Hot Plug Data structure if CPU hot plug is supported.
   //
diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h
index 2bb35a424d..b321ff15ee 100644
--- a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h
@@ -20,6 +20,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent
 #include <Protocol/SmmReadyToLock.h>
 #include <Protocol/SmmCpuService.h>
 #include <Protocol/SmmMemoryAttribute.h>
+#include <Protocol/SmmMp.h>
 
 #include <Guid/AcpiS3Context.h>
 #include <Guid/MemoryAttributesTable.h>
@@ -226,6 +227,7 @@ extern CPU_HOT_PLUG_DATA      mCpuHotPlugData;
 extern UINTN                  mMaxNumberOfCpus;
 extern UINTN                  mNumberOfCpus;
 extern EFI_SMM_CPU_PROTOCOL   mSmmCpu;
+extern EFI_SMM_MP_PROTOCOL    mSmmMp;
 
 ///
 /// The mode of the CPU at the time an SMI occurs
@@ -363,7 +365,7 @@ SmmRelocationSemaphoreComplete (
 ///
 typedef struct {
   SPIN_LOCK                         *Busy;
-  volatile EFI_AP_PROCEDURE         Procedure;
+  volatile EFI_AP_PROCEDURE2        Procedure;
   volatile VOID                     *Parameter;
   volatile UINT32                   *Run;
   volatile BOOLEAN                  *Present;
@@ -375,6 +377,19 @@ typedef enum {
   SmmCpuSyncModeMax
 } SMM_CPU_SYNC_MODE;
 
+typedef enum {
+  SmmCpuCheckModeSingle,
+  SmmCpuCheckModeAll,
+  SmmCpuCheckModeMax
+} SMM_CPU_CHECK_MODE;
+
+typedef enum {
+  SmmCpuCallInOldSyncType,
+  SmmCpuCallInOldAsyncType,
+  SmmCpuCallInNewType,
+  SmmCpuCallInTypeMax
+} SMM_CPU_CALL_IN_TYPE;
+
 typedef struct {
   //
   // Pointer to an array. The array should be located immediately after this structure
@@ -388,6 +403,14 @@ typedef struct {
   volatile SMM_CPU_SYNC_MODE    EffectiveSyncMode;
   volatile BOOLEAN              SwitchBsp;
   volatile BOOLEAN              *CandidateBsp;
+  SPIN_LOCK                     *CurrentToken;
+  SMM_CPU_CHECK_MODE            CpuCheckMode;
+  UINTN                         CpuIndex;
+  EFI_STATUS                    *CpuStatus;
+  volatile BOOLEAN              *FinishTask;
+
+  EFI_AP_PROCEDURE              StartupProcedure;
+  VOID                          *StartupProcArgs;
 } SMM_DISPATCHER_MP_SYNC_DATA;
 
 #define SMM_PSD_OFFSET              0xfb00
@@ -1259,4 +1282,151 @@ RestoreCr2 (
   IN UINTN  Cr2
   );
 
+/**
+  Schedule a procedure to run on the specified CPU.
+
+  @param[in]       Procedure                The address of the procedure to run
+  @param[in]       CpuIndex                 Target CPU Index
+  @param[in, out]  ProcArguments            The parameter to pass to the procedure
+  @param[in]       CallInType               Which call for this function.
+  @param[in, out]  Token                    This is parameter is broken into two components:
+                                            1.Token->Completion is an optional parameter that allows the
+                                            caller to execute the procedure in a blocking or non-blocking
+                                            fashion. If it is NULL the call is blocking, and the call will
+                                            not return until the AP has completed the procedure. If the
+                                            token is not NULL, the call will return immediately. The caller
+                                            can check whether the procedure has completed with
+                                            CheckOnProcedure or WaitForProcedure.
+                                            2.Token->Status The implementation updates the address pointed
+                                            at by this variable with the status code returned by Procedure
+                                            when it completes execution on the target AP, or with EFI_TIMEOUT
+                                            if the Procedure fails to complete within the optional timeout.
+                                            The implementation will update this variable with EFI_NOT_READY
+                                            prior to starting Procedure on the target AP
+  @param[in]       TimeoutInMicroseconds    Indicates the time limit in microseconds for the APs to finish
+                                            execution of Procedure, either for blocking or non-blocking mode.
+                                            Zero means infinity. If the timeout expires before all APs return
+                                            from Procedure, then Procedure on the failed APs is terminated. If
+                                            the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
+                                            If the timeout expires in non-blocking mode, the timeout determined
+                                            can be through CheckOnProcedure or WaitForProcedure.
+                                            Note that timeout support is optional. Whether an implementation
+                                            supports this feature can be determined via the Attributes data
+                                            member.
+
+
+  @retval EFI_INVALID_PARAMETER    CpuNumber not valid
+  @retval EFI_INVALID_PARAMETER    CpuNumber specifying BSP
+  @retval EFI_INVALID_PARAMETER    The AP specified by CpuNumber did not enter SMM
+  @retval EFI_INVALID_PARAMETER    The AP specified by CpuNumber is busy
+  @retval EFI_SUCCESS              The procedure has been successfully scheduled
+
+**/
+EFI_STATUS
+InternalSmmStartupThisAp (
+  IN      EFI_AP_PROCEDURE               Procedure,
+  IN      UINTN                          CpuIndex,
+  IN OUT  VOID                           *ProcArguments OPTIONAL,
+  IN      SMM_CPU_CALL_IN_TYPE           CallInType,
+  IN      SPIN_LOCK                      *Token,
+  IN      UINTN                          TimeoutInMicroseconds,
+  IN      EFI_STATUS                     *CpuStatus
+  );
+
+/**
+  Checks whether the input token is the current used token.
+
+
+  @param[in]  Token      This parameter describes the token that was passed into DispatchProcedure or
+                         BroadcastProcedure.
+
+  @retval TRUE           The input token is the current used token.
+  @retval FALSE          The input token is not the current used token.
+**/
+BOOLEAN
+CurrentUsedToken (
+  IN SPIN_LOCK  *Token
+  );
+
+/**
+  Checks status of specified AP.
+
+  This function checks whether the specified AP has finished the task assigned
+  by StartupThisAP(), and whether timeout expires.
+
+  @param[in]  Token             This parameter describes the token that was passed into DispatchProcedure or
+                                BroadcastProcedure.
+
+  @retval EFI_SUCCESS           Specified AP has finished task assigned by StartupThisAPs().
+  @retval EFI_NOT_READY         Specified AP has not finished task and timeout has not expired.
+**/
+EFI_STATUS
+CheckStatusForThisAP (
+  IN SPIN_LOCK  *Token
+  );
+
+/**
+  Worker function to execute a caller provided function on all enabled APs.
+
+  @param[in]  Procedure               A pointer to the function to be run on
+                                      enabled APs of the system.
+  @param[in]  SingleThread            If TRUE, then all the enabled APs execute
+                                      the function specified by Procedure one by
+                                      one, in ascending order of processor handle
+                                      number.  If FALSE, then all the enabled APs
+                                      execute the function specified by Procedure
+                                      simultaneously.
+  @param[in]  WaitEvent               The event created by the caller with CreateEvent()
+                                      service.
+  @param[in]  TimeoutInMicroseconds   Indicates the time limit in microseconds for
+                                      APs to return from Procedure, either for
+                                      blocking or non-blocking mode.
+  @param[in]  ProcedureArgument       The parameter passed into Procedure for
+                                      all APs.
+  @param[out] FailedCpuList           If all APs finish successfully, then its
+                                      content is set to NULL. If not all APs
+                                      finish before timeout expires, then its
+                                      content is set to address of the buffer
+                                      holding handle numbers of the failed APs.
+
+  @retval EFI_SUCCESS             In blocking mode, all APs have finished before
+                                  the timeout expired.
+  @retval EFI_SUCCESS             In non-blocking mode, function has been dispatched
+                                  to all enabled APs.
+  @retval others                  Failed to Startup all APs.
+
+**/
+EFI_STATUS
+InternalSmmStartupAllAPs (
+  IN       EFI_AP_PROCEDURE2             Procedure,
+  IN       UINTN                         TimeoutInMicroseconds,
+  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
+  IN OUT   SPIN_LOCK                     *Token,
+  IN OUT   EFI_STATUS                    *CPUStatus
+  );
+
+/**
+
+  Register the SMM Foundation entry point.
+
+  @param[in]      Procedure            A pointer to the code stream to be run on the designated target AP
+                                       of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
+                                       with the related definitions of
+                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
+                                       If caller may pass a value of NULL to deregister any existing
+                                       startup procedure.
+  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of parameters to the code that is
+                                       run by the AP. It is an optional common mailbox between APs and
+                                       the caller to share information
+
+  @retval EFI_SUCCESS                  The Procedure has been set successfully.
+  @retval EFI_INVALID_PARAMETER        The Procedure is NULL but ProcedureArguments not NULL.
+
+**/
+EFI_STATUS
+RegisterStartupProcedure (
+  IN EFI_AP_PROCEDURE    Procedure,
+  IN VOID                *ProcedureArguments OPTIONAL
+  );
+
 #endif
diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf
index 466c568d49..f478ae62a7 100644
--- a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf
@@ -40,6 +40,8 @@
   SmmProfileInternal.h
   SmramSaveState.c
   SmmCpuMemoryManagement.c
+  MpProtocol.h
+  MpProtocol.c
 
 [Sources.Ia32]
   Ia32/Semaphore.c
@@ -105,6 +107,7 @@
   gEfiSmmReadyToLockProtocolGuid           ## NOTIFY
   gEfiSmmCpuServiceProtocolGuid            ## PRODUCES
   gEdkiiSmmMemoryAttributeProtocolGuid     ## PRODUCES
+  gEfiSmmMpProtocolGuid                    ## PRODUCES
 
 [Guids]
   gEfiAcpiVariableGuid                     ## SOMETIMES_CONSUMES ## HOB # it is used for S3 boot.
-- 
2.21.0.windows.1

Re: [edk2-devel] [Patch 2/2] UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP Protocol.

[Laszlo Ersek]

On 06/19/19 07:51, Dong, Eric wrote:
> Add MM Mp Protocol in PiSmmCpuDxeSmm driver.
> 
> Cc: Ray Ni <ray.ni@intel.com>
> Cc: Laszlo Ersek <lersek@redhat.com>
> Signed-off-by: Eric Dong <eric.dong@intel.com>
> ---
>  UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c       | 375 +++++++++++++++
>  UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h       | 283 +++++++++++
>  UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c        | 468 ++++++++++++++++++-
>  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c   |  11 +
>  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h   | 172 ++++++-
>  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf |   3 +
>  6 files changed, 1296 insertions(+), 16 deletions(-)
>  create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
>  create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h
> 
> diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c b/UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
> new file mode 100644
> index 0000000000..8cf69428c2
> --- /dev/null
> +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
> @@ -0,0 +1,375 @@
> +/** @file
> +SMM MP protocol implementation
> +
> +Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
> +
> +SPDX-License-Identifier: BSD-2-Clause-Patent
> +
> +**/
> +
> +#include "PiSmmCpuDxeSmm.h"
> +#include "MpProtocol.h"
> +
> +///
> +/// SMM MP Protocol instance
> +///
> +EFI_SMM_MP_PROTOCOL  mSmmMp  = {
> +  EFI_SMM_MP_PROTOCOL_REVISION,
> +  0,
> +  SmmMpGetNumberOfProcessors,
> +  SmmMpDispatchProcedure,
> +  SmmMpBroadcastProcedure,
> +  SmmMpSetStartupProcedure,
> +  SmmMpCheckForProcedure,
> +  SmmMpWaitForProcedure
> +};
> +
> +/**
> +  Service to retrieves the number of logical processor in the platform.
> +
> +  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
> +  @param[out] NumberOfProcessors  Pointer to the total number of logical processors in the system,
> +                                  including the BSP and all APs.
> +
> +  @retval EFI_SUCCESS             The number of processors was retrieved successfully
> +  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
> +**/
> +
> +EFI_STATUS
> +EFIAPI
> +SmmMpGetNumberOfProcessors (
> +  IN CONST EFI_SMM_MP_PROTOCOL  *This,
> +  OUT      UINTN                *NumberOfProcessors
> +  )
> +{
> +  if (NumberOfProcessors == NULL) {
> +    return EFI_INVALID_PARAMETER;
> +  }
> +
> +  *NumberOfProcessors = gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus;
> +
> +  return EFI_SUCCESS;
> +}
> +
> +
> +/**
> +  This service allows the caller to invoke a procedure one of the application processors (AP). This
> +  function uses an optional token parameter to support blocking and non-blocking modes. If the token
> +  is passed into the call, the function will operate in a non-blocking fashion and the caller can
> +  check for completion with CheckOnProcedure or WaitForProcedure.
> +
> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]     Procedure              A pointer to the procedure to be run on the designated target
> +                                        AP of the system. Type EFI_AP_PROCEDURE2 is defined below in
> +                                        related definitions.
> +  @param[in]     CpuNumber              The zero-based index of the processor number of the target
> +                                        AP, on which the code stream is supposed to run. If the number
> +                                        points to the calling processor then it will not run the
> +                                        supplied code.
> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for this AP to
> +                                        finish execution of Procedure, either for blocking or
> +                                        non-blocking mode. Zero means infinity. If the timeout
> +                                        expires before this AP returns from Procedure, then Procedure
> +                                        on the AP is terminated. If the timeout expires in blocking
> +                                        mode, the call returns EFI_TIMEOUT. If the timeout expires
> +                                        in non-blocking mode, the timeout determined can be through
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        Note that timeout support is optional. Whether an
> +                                        implementation supports this feature, can be determined via
> +                                        the Attributes data member.
> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
> +                                        that is run by the AP. It is an optional common mailbox
> +                                        between APs and the caller to share information.
> +  @param[in,out] Token                  This is parameter is broken into two components:
> +                                        1.Token->Completion is an optional parameter that allows the
> +                                        caller to execute the procedure in a blocking or non-blocking
> +                                        fashion. If it is NULL the call is blocking, and the call will
> +                                        not return until the AP has completed the procedure. If the
> +                                        token is not NULL, the call will return immediately. The caller
> +                                        can check whether the procedure has completed with
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        2.Token->Status The implementation updates the address pointed
> +                                        at by this variable with the status code returned by Procedure
> +                                        when it completes execution on the target AP, or with EFI_TIMEOUT
> +                                        if the Procedure fails to complete within the optional timeout.
> +                                        The implementation will update this variable with EFI_NOT_READY
> +                                        prior to starting Procedure on the target AP
> +  @param[in,out] CPUStatus              This optional pointer may be used to get the status code returned
> +                                        by Procedure when it completes execution on the target AP, or with
> +                                        EFI_TIMEOUT if the Procedure fails to complete within the optional
> +                                        timeout. The implementation will update this variable with
> +                                        EFI_NOT_READY prior to starting Procedure on the target AP.
> +
> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
> +                                        execution on the target AP.
> +                                        In the non-blocking case this indicates that the procedure has
> +                                        been successfully scheduled for execution on the target AP.
> +  @retval EFI_INVALID_PARAMETER         The input arguments are out of range. Either the target AP is the
> +                                        caller of the function, or the Procedure or Token is NULL
> +  @retval EFI_NOT_READY                 If the target AP is busy executing another procedure
> +  @retval EFI_ALREADY_STARTED           Token is already in use for another procedure
> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before the specified AP
> +                                        has finished
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpDispatchProcedure (
> +  IN CONST EFI_SMM_MP_PROTOCOL           *This,
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         CpuNumber,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   SMM_COMPLETION                *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +  )
> +{
> +  if (Token != NULL) {
> +    *Token = AllocatePool (sizeof (SPIN_LOCK));
> +    ASSERT (*Token != NULL);
> +    InitializeSpinLock ((SPIN_LOCK *)(*Token));
> +
> +    return InternalSmmStartupThisAp(
> +      Procedure,
> +      CpuNumber,
> +      ProcedureArguments,
> +      SmmCpuCallInNewType,
> +      (SPIN_LOCK *)(*Token),
> +      TimeoutInMicroseconds,
> +      CPUStatus
> +      );
> +  }
> +
> +  return InternalSmmStartupThisAp(
> +    Procedure,
> +    CpuNumber,
> +    ProcedureArguments,
> +    SmmCpuCallInNewType,
> +    NULL,
> +    TimeoutInMicroseconds,
> +    CPUStatus
> +    );
> +}

This function breaks the build for me:

  passing argument 1 of 'InternalSmmStartupThisAp' from incompatible
  pointer type [-Werror]

InternalSmmStartupThisAp() expects an EFI_AP_PROCEDURE as first parameter:

  typedef
  VOID
  (EFIAPI *EFI_AP_PROCEDURE)(
    IN OUT VOID  *Buffer
    );

but you pass EFI_AP_PROCEDURE2:

  typedef
  EFI_STATUS
  (EFIAPI *EFI_AP_PROCEDURE2)(
    IN VOID  *ProcedureArgument
  );

Assuming the return status propagation is otherwise correctly
implemented through "CpuStatus" -- I didn't check --, please use a
dedicated wrapper function for the callback, so that we can avoid type
punning of function pointers. (It's possible that you'll have to
introduce a separate wrapper structure too, for passing in both the
function pointer and the procedure argument pointer -- and then that
structure should be allocated dynamically.)


--*--


Independently, I think that catching "out of memory" situations in
PiSmmCpuDxeSmm with ASSERTs is bad practice. This is a privileged /
security-relevant driver, and the function in question is exposed as a
protocol member function (i.e. it's not *only* a part of driver
startup). Which I believe might make it possible for the OS, indirectly
(via EFI runtime calls for example), to trigger this function.

So, even though the PI spec doesn't list EFI_OUT_OF_RESOURCES as a valid
return code for this member function, we should return that, if
AllocatePool() fails. (This applies to SmmMpBroadcastProcedure() too.)

Thanks
Laszlo

Re: [edk2-devel] [Patch 2/2] UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP Protocol.

[Dong, Eric]

> -----Original Message-----
> From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of
> Laszlo Ersek
> Sent: Friday, June 21, 2019 12:45 AM
> To: devel@edk2.groups.io; Dong, Eric <eric.dong@intel.com>
> Cc: Ni, Ray <ray.ni@intel.com>
> Subject: Re: [edk2-devel] [Patch 2/2] UefiCpuPkg/PiSmmCpuDxeSmm:
> Enable MM MP Protocol.
> 
> On 06/19/19 07:51, Dong, Eric wrote:
> > Add MM Mp Protocol in PiSmmCpuDxeSmm driver.
> >
> > Cc: Ray Ni <ray.ni@intel.com>
> > Cc: Laszlo Ersek <lersek@redhat.com>
> > Signed-off-by: Eric Dong <eric.dong@intel.com>
> > ---
> >  UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c       | 375
> +++++++++++++++
> >  UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h       | 283 +++++++++++
> >  UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c        | 468
> ++++++++++++++++++-
> >  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c   |  11 +
> >  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h   | 172 ++++++-
> >  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf |   3 +
> >  6 files changed, 1296 insertions(+), 16 deletions(-)  create mode
> > 100644 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
> >  create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h
> >
> > diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
> > b/UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
> > new file mode 100644
> > index 0000000000..8cf69428c2
> > --- /dev/null
> > +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
> > @@ -0,0 +1,375 @@
> > +/** @file
> > +SMM MP protocol implementation
> > +
> > +Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
> > +
> > +SPDX-License-Identifier: BSD-2-Clause-Patent
> > +
> > +**/
> > +
> > +#include "PiSmmCpuDxeSmm.h"
> > +#include "MpProtocol.h"
> > +
> > +///
> > +/// SMM MP Protocol instance
> > +///
> > +EFI_SMM_MP_PROTOCOL  mSmmMp  = {
> > +  EFI_SMM_MP_PROTOCOL_REVISION,
> > +  0,
> > +  SmmMpGetNumberOfProcessors,
> > +  SmmMpDispatchProcedure,
> > +  SmmMpBroadcastProcedure,
> > +  SmmMpSetStartupProcedure,
> > +  SmmMpCheckForProcedure,
> > +  SmmMpWaitForProcedure
> > +};
> > +
> > +/**
> > +  Service to retrieves the number of logical processor in the platform.
> > +
> > +  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
> > +  @param[out] NumberOfProcessors  Pointer to the total number of
> logical processors in the system,
> > +                                  including the BSP and all APs.
> > +
> > +  @retval EFI_SUCCESS             The number of processors was retrieved
> successfully
> > +  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
> > +**/
> > +
> > +EFI_STATUS
> > +EFIAPI
> > +SmmMpGetNumberOfProcessors (
> > +  IN CONST EFI_SMM_MP_PROTOCOL  *This,
> > +  OUT      UINTN                *NumberOfProcessors
> > +  )
> > +{
> > +  if (NumberOfProcessors == NULL) {
> > +    return EFI_INVALID_PARAMETER;
> > +  }
> > +
> > +  *NumberOfProcessors =
> > + gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus;
> > +
> > +  return EFI_SUCCESS;
> > +}
> > +
> > +
> > +/**
> > +  This service allows the caller to invoke a procedure one of the
> > +application processors (AP). This
> > +  function uses an optional token parameter to support blocking and
> > +non-blocking modes. If the token
> > +  is passed into the call, the function will operate in a
> > +non-blocking fashion and the caller can
> > +  check for completion with CheckOnProcedure or WaitForProcedure.
> > +
> > +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
> > +  @param[in]     Procedure              A pointer to the procedure to be run on
> the designated target
> > +                                        AP of the system. Type EFI_AP_PROCEDURE2 is
> defined below in
> > +                                        related definitions.
> > +  @param[in]     CpuNumber              The zero-based index of the processor
> number of the target
> > +                                        AP, on which the code stream is supposed to run. If
> the number
> > +                                        points to the calling processor then it will not run the
> > +                                        supplied code.
> > +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
> microseconds for this AP to
> > +                                        finish execution of Procedure, either for blocking or
> > +                                        non-blocking mode. Zero means infinity. If the
> timeout
> > +                                        expires before this AP returns from Procedure, then
> Procedure
> > +                                        on the AP is terminated. If the timeout expires in
> blocking
> > +                                        mode, the call returns EFI_TIMEOUT. If the timeout
> expires
> > +                                        in non-blocking mode, the timeout determined can be
> through
> > +                                        CheckOnProcedure or WaitForProcedure.
> > +                                        Note that timeout support is optional. Whether an
> > +                                        implementation supports this feature, can be
> determined via
> > +                                        the Attributes data member.
> > +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
> parameters to the code
> > +                                        that is run by the AP. It is an optional common mailbox
> > +                                        between APs and the caller to share information.
> > +  @param[in,out] Token                  This is parameter is broken into two
> components:
> > +                                        1.Token->Completion is an optional parameter that
> allows the
> > +                                        caller to execute the procedure in a blocking or non-
> blocking
> > +                                        fashion. If it is NULL the call is blocking, and the call will
> > +                                        not return until the AP has completed the procedure.
> If the
> > +                                        token is not NULL, the call will return immediately. The
> caller
> > +                                        can check whether the procedure has completed with
> > +                                        CheckOnProcedure or WaitForProcedure.
> > +                                        2.Token->Status The implementation updates the
> address pointed
> > +                                        at by this variable with the status code returned by
> Procedure
> > +                                        when it completes execution on the target AP, or with
> EFI_TIMEOUT
> > +                                        if the Procedure fails to complete within the optional
> timeout.
> > +                                        The implementation will update this variable with
> EFI_NOT_READY
> > +                                        prior to starting Procedure on the target AP
> > +  @param[in,out] CPUStatus              This optional pointer may be used to
> get the status code returned
> > +                                        by Procedure when it completes execution on the
> target AP, or with
> > +                                        EFI_TIMEOUT if the Procedure fails to complete within
> the optional
> > +                                        timeout. The implementation will update this variable
> with
> > +                                        EFI_NOT_READY prior to starting Procedure on the
> target AP.
> > +
> > +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
> Procedure has completed
> > +                                        execution on the target AP.
> > +                                        In the non-blocking case this indicates that the
> procedure has
> > +                                        been successfully scheduled for execution on the
> target AP.
> > +  @retval EFI_INVALID_PARAMETER         The input arguments are out of
> range. Either the target AP is the
> > +                                        caller of the function, or the Procedure or Token is
> NULL
> > +  @retval EFI_NOT_READY                 If the target AP is busy executing
> another procedure
> > +  @retval EFI_ALREADY_STARTED           Token is already in use for another
> procedure
> > +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
> before the specified AP
> > +                                        has finished **/ EFI_STATUS
> > +EFIAPI SmmMpDispatchProcedure (
> > +  IN CONST EFI_SMM_MP_PROTOCOL           *This,
> > +  IN       EFI_AP_PROCEDURE2             Procedure,
> > +  IN       UINTN                         CpuNumber,
> > +  IN       UINTN                         TimeoutInMicroseconds,
> > +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> > +  IN OUT   SMM_COMPLETION                *Token,
> > +  IN OUT   EFI_STATUS                    *CPUStatus
> > +  )
> > +{
> > +  if (Token != NULL) {
> > +    *Token = AllocatePool (sizeof (SPIN_LOCK));
> > +    ASSERT (*Token != NULL);
> > +    InitializeSpinLock ((SPIN_LOCK *)(*Token));
> > +
> > +    return InternalSmmStartupThisAp(
> > +      Procedure,
> > +      CpuNumber,
> > +      ProcedureArguments,
> > +      SmmCpuCallInNewType,
> > +      (SPIN_LOCK *)(*Token),
> > +      TimeoutInMicroseconds,
> > +      CPUStatus
> > +      );
> > +  }
> > +
> > +  return InternalSmmStartupThisAp(
> > +    Procedure,
> > +    CpuNumber,
> > +    ProcedureArguments,
> > +    SmmCpuCallInNewType,
> > +    NULL,
> > +    TimeoutInMicroseconds,
> > +    CPUStatus
> > +    );
> > +}
> 
> This function breaks the build for me:
> 
>   passing argument 1 of 'InternalSmmStartupThisAp' from incompatible
>   pointer type [-Werror]
> 
> InternalSmmStartupThisAp() expects an EFI_AP_PROCEDURE as first
> parameter:
> 
>   typedef
>   VOID
>   (EFIAPI *EFI_AP_PROCEDURE)(
>     IN OUT VOID  *Buffer
>     );
> 
> but you pass EFI_AP_PROCEDURE2:
> 
>   typedef
>   EFI_STATUS
>   (EFIAPI *EFI_AP_PROCEDURE2)(
>     IN VOID  *ProcedureArgument
>   );
> 
> Assuming the return status propagation is otherwise correctly implemented
> through "CpuStatus" -- I didn't check --, please use a dedicated wrapper
> function for the callback, so that we can avoid type punning of function
> pointers. (It's possible that you'll have to introduce a separate wrapper
> structure too, for passing in both the function pointer and the procedure
> argument pointer -- and then that structure should be allocated dynamically.)

Hi Laszlo,

Can you help to explain more about how to use dedicated wrapper function for the callback? What I can image is using type cast here to fix this issue. Why you think type cast is not enough here? 

> 
> 
> --*--
> 
> 
> Independently, I think that catching "out of memory" situations in
> PiSmmCpuDxeSmm with ASSERTs is bad practice. This is a privileged /
> security-relevant driver, and the function in question is exposed as a protocol
> member function (i.e. it's not *only* a part of driver startup). Which I believe
> might make it possible for the OS, indirectly (via EFI runtime calls for
> example), to trigger this function.
> 
> So, even though the PI spec doesn't list EFI_OUT_OF_RESOURCES as a valid
> return code for this member function, we should return that, if
> AllocatePool() fails. (This applies to SmmMpBroadcastProcedure() too.)

Agree, will add error handling here and add the return status code to the function header comments.

Thanks,
Eric
> 
> Thanks
> Laszlo
> 
> 

Re: [edk2-devel] [Patch 2/2] UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP Protocol.

[Laszlo Ersek]

On 06/25/19 04:15, Dong, Eric wrote:
>> -----Original Message-----
>> From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of
>> Laszlo Ersek
>> Sent: Friday, June 21, 2019 12:45 AM
>> To: devel@edk2.groups.io; Dong, Eric <eric.dong@intel.com>
>> Cc: Ni, Ray <ray.ni@intel.com>
>> Subject: Re: [edk2-devel] [Patch 2/2] UefiCpuPkg/PiSmmCpuDxeSmm:
>> Enable MM MP Protocol.
>>
>> On 06/19/19 07:51, Dong, Eric wrote:

>>> EFI_STATUS
>>> +EFIAPI SmmMpDispatchProcedure (
>>> +  IN CONST EFI_SMM_MP_PROTOCOL           *This,
>>> +  IN       EFI_AP_PROCEDURE2             Procedure,
>>> +  IN       UINTN                         CpuNumber,
>>> +  IN       UINTN                         TimeoutInMicroseconds,
>>> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
>>> +  IN OUT   SMM_COMPLETION                *Token,
>>> +  IN OUT   EFI_STATUS                    *CPUStatus
>>> +  )
>>> +{
>>> +  if (Token != NULL) {
>>> +    *Token = AllocatePool (sizeof (SPIN_LOCK));
>>> +    ASSERT (*Token != NULL);
>>> +    InitializeSpinLock ((SPIN_LOCK *)(*Token));
>>> +
>>> +    return InternalSmmStartupThisAp(
>>> +      Procedure,
>>> +      CpuNumber,
>>> +      ProcedureArguments,
>>> +      SmmCpuCallInNewType,
>>> +      (SPIN_LOCK *)(*Token),
>>> +      TimeoutInMicroseconds,
>>> +      CPUStatus
>>> +      );
>>> +  }
>>> +
>>> +  return InternalSmmStartupThisAp(
>>> +    Procedure,
>>> +    CpuNumber,
>>> +    ProcedureArguments,
>>> +    SmmCpuCallInNewType,
>>> +    NULL,
>>> +    TimeoutInMicroseconds,
>>> +    CPUStatus
>>> +    );
>>> +}
>>
>> This function breaks the build for me:
>>
>>   passing argument 1 of 'InternalSmmStartupThisAp' from incompatible
>>   pointer type [-Werror]
>>
>> InternalSmmStartupThisAp() expects an EFI_AP_PROCEDURE as first
>> parameter:
>>
>>   typedef
>>   VOID
>>   (EFIAPI *EFI_AP_PROCEDURE)(
>>     IN OUT VOID  *Buffer
>>     );
>>
>> but you pass EFI_AP_PROCEDURE2:
>>
>>   typedef
>>   EFI_STATUS
>>   (EFIAPI *EFI_AP_PROCEDURE2)(
>>     IN VOID  *ProcedureArgument
>>   );
>>
>> Assuming the return status propagation is otherwise correctly implemented
>> through "CpuStatus" -- I didn't check --, please use a dedicated wrapper
>> function for the callback, so that we can avoid type punning of function
>> pointers. (It's possible that you'll have to introduce a separate wrapper
>> structure too, for passing in both the function pointer and the procedure
>> argument pointer -- and then that structure should be allocated dynamically.)
> 
> Hi Laszlo,
> 
> Can you help to explain more about how to use dedicated wrapper function for the callback? What I can image is using type cast here to fix this issue. Why you think type cast is not enough here? 

[1] ISO C99 6.5.2.2 "Function calls", paragraph 9:

    If the function is defined with a type that is not compatible with
    the type (of the expression) pointed to by the expression that
    denotes the called function, the behavior is undefined.

After you pass "Procedure" to InternalSmmStartupThisAp(), then something
inside InternalSmmStartupThisAp() will call "Procedure" through the
following prototype:

  VOID
  EFIAPI
  Procedure (
    IN OUT VOID  *Buffer
    );

However, "Procedure" has the following prototype in reality:

  EFI_STATUS
  EFIAPI
  Procedure (
    IN VOID  *ProcedureArgument
    );

Therefore -- the standard says --, if these two prototypes are not
compatible with each other, then the behavior is undefined, in the code
above.

In simpler terms, you must not call a function as another function type,
unless those function types are compatible.

So let's see if the two function types in question are compatible:

[2] ISO C99 6.7.5.3 "Function declarators (including prototypes)",
paragraph 15:

  For two function types to be compatible, both shall specify compatible
  return types. [...]

In our case, one return type is VOID, and the other return type is
EFI_STATUS.

Those types are not compatible. Therefore the function types are not
compatible. Therefore you must not call EFI_AP_PROCEDURE *as* an
EFI_AP_PROCEDURE2, or vice versa -- otherwise the behavior is undefined.


Regarding the wrapper function and structure, this is what I propose:


typedef struct {
  EFI_AP_PROCEDURE2 Procedure2;
  VOID              *ProcedureArgument;
} PROCEDURE2_WRAPPER;


STATIC
VOID
EFIAPI
Procedure2Wrapper (
  IN OUT VOID *Buffer
  )
{
  PROCEDURE2_WRAPPER *Wrapper;
  EFI_STATUS         Status;

  Wrapper = Buffer;
  Status = Wrapper->Procedure2 (Wrapper->ProcedureArgument);
  //
  // Ignore status deliberately.
  //
}


And then in SmmMpDispatchProcedure():

  PROCEDURE2_WRAPPER *Wrapper;

  Wrapper = AllocatePool (sizeof *Wrapper);
  if (Wrapper = NULL) {
    return EFI_OUT_OF_RESOURCES;
  }
  Wrapper->Procedure2 = Procedure;
  Wrapper->ProcedureArgument = ProcedureArguments;
  InternalSmmStartupThisAp (... Procedure2Wrapper ... Wrapper ...);


The point is that InternalSmmStartupThisAp() receives a pointer to
Procedure2Wrapper(), and that argument *really* has the correct type
(EFI_AP_PROCEDURE).

Of course, "Wrapper" should be released as well:

- For synchronous calls -- i.e., when Token is NULL --, the
SmmMpDispatchProcedure() function should call

  FreePool (Wrapper);

right after InternalSmmStartupThisAp() returns.

- For the async case, when Token is *not* NULL, the SMM_COMPLETION data
type has to be extended with a pointer to the PROCEDURE2_WRAPPER oject.
And then, when the completion is *collected*, Wrapper should be freed.

(Of course, AllocatePool() and FreePool() operate on SMRAM.)

This entire idea is about inserting another level of indirection so that
we can avoid undefined function calls in C code.

Thanks
Laszlo

Re: [edk2-devel] [Patch 2/2] UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP Protocol.

[Laszlo Ersek]

On 06/25/19 12:24, Laszlo Ersek wrote:

>   if (Wrapper = NULL) {
>     return EFI_OUT_OF_RESOURCES;
>   }

Clearly, I meant "Wrapper == NULL" above :)

Re: [Patch 2/2] UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP Protocol.

[Ni, Ray]

> -----Original Message-----
> From: Dong, Eric <eric.dong@intel.com>
> Sent: Wednesday, June 19, 2019 1:51 PM
> To: devel@edk2.groups.io
> Cc: Ni, Ray <ray.ni@intel.com>; Laszlo Ersek <lersek@redhat.com>
> Subject: [Patch 2/2] UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP
> Protocol.
> 
> Add MM Mp Protocol in PiSmmCpuDxeSmm driver.
> 
> Cc: Ray Ni <ray.ni@intel.com>
> Cc: Laszlo Ersek <lersek@redhat.com>
> Signed-off-by: Eric Dong <eric.dong@intel.com>
> ---
>  UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c       | 375 +++++++++++++++
>  UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h       | 283 +++++++++++
>  UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c        | 468
> ++++++++++++++++++-
>  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c   |  11 +
>  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h   | 172 ++++++-
>  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf |   3 +
>  6 files changed, 1296 insertions(+), 16 deletions(-)
>  create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
>  create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h
> 
> diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
> b/UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
> new file mode 100644
> index 0000000000..8cf69428c2
> --- /dev/null
> +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
> @@ -0,0 +1,375 @@
> +/** @file
> +SMM MP protocol implementation
> +
> +Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
> +
> +SPDX-License-Identifier: BSD-2-Clause-Patent
> +
> +**/
> +
> +#include "PiSmmCpuDxeSmm.h"
> +#include "MpProtocol.h"
> +
> +///
> +/// SMM MP Protocol instance
> +///
> +EFI_SMM_MP_PROTOCOL  mSmmMp  = {
> +  EFI_SMM_MP_PROTOCOL_REVISION,
> +  0,
> +  SmmMpGetNumberOfProcessors,
> +  SmmMpDispatchProcedure,
> +  SmmMpBroadcastProcedure,
> +  SmmMpSetStartupProcedure,
> +  SmmMpCheckForProcedure,
> +  SmmMpWaitForProcedure
> +};
> +
> +/**
> +  Service to retrieves the number of logical processor in the platform.
> +
> +  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
> +  @param[out] NumberOfProcessors  Pointer to the total number of logical
> processors in the system,
> +                                  including the BSP and all APs.
> +
> +  @retval EFI_SUCCESS             The number of processors was retrieved
> successfully
> +  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
> +**/
> +
> +EFI_STATUS
> +EFIAPI
> +SmmMpGetNumberOfProcessors (
> +  IN CONST EFI_SMM_MP_PROTOCOL  *This,
> +  OUT      UINTN                *NumberOfProcessors
> +  )
> +{
> +  if (NumberOfProcessors == NULL) {
> +    return EFI_INVALID_PARAMETER;
> +  }
> +
> +  *NumberOfProcessors = gSmmCpuPrivate-
> >SmmCoreEntryContext.NumberOfCpus;
> +
> +  return EFI_SUCCESS;
> +}
> +
> +
> +/**
> +  This service allows the caller to invoke a procedure one of the application
> processors (AP). This
> +  function uses an optional token parameter to support blocking and non-
> blocking modes. If the token
> +  is passed into the call, the function will operate in a non-blocking fashion
> and the caller can
> +  check for completion with CheckOnProcedure or WaitForProcedure.
> +
> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]     Procedure              A pointer to the procedure to be run on
> the designated target
> +                                        AP of the system. Type EFI_AP_PROCEDURE2 is defined
> below in
> +                                        related definitions.
> +  @param[in]     CpuNumber              The zero-based index of the processor
> number of the target
> +                                        AP, on which the code stream is supposed to run. If the
> number
> +                                        points to the calling processor then it will not run the
> +                                        supplied code.
> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
> microseconds for this AP to
> +                                        finish execution of Procedure, either for blocking or
> +                                        non-blocking mode. Zero means infinity. If the timeout
> +                                        expires before this AP returns from Procedure, then
> Procedure
> +                                        on the AP is terminated. If the timeout expires in
> blocking
> +                                        mode, the call returns EFI_TIMEOUT. If the timeout
> expires
> +                                        in non-blocking mode, the timeout determined can be
> through
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        Note that timeout support is optional. Whether an
> +                                        implementation supports this feature, can be
> determined via
> +                                        the Attributes data member.
> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
> parameters to the code
> +                                        that is run by the AP. It is an optional common mailbox
> +                                        between APs and the caller to share information.
> +  @param[in,out] Token                  This is parameter is broken into two
> components:
> +                                        1.Token->Completion is an optional parameter that
> allows the
> +                                        caller to execute the procedure in a blocking or non-
> blocking
> +                                        fashion. If it is NULL the call is blocking, and the call will
> +                                        not return until the AP has completed the procedure. If
> the
> +                                        token is not NULL, the call will return immediately. The
> caller
> +                                        can check whether the procedure has completed with
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        2.Token->Status The implementation updates the
> address pointed
> +                                        at by this variable with the status code returned by
> Procedure
> +                                        when it completes execution on the target AP, or with
> EFI_TIMEOUT
> +                                        if the Procedure fails to complete within the optional
> timeout.
> +                                        The implementation will update this variable with
> EFI_NOT_READY
> +                                        prior to starting Procedure on the target AP
> +  @param[in,out] CPUStatus              This optional pointer may be used to get
> the status code returned
> +                                        by Procedure when it completes execution on the
> target AP, or with
> +                                        EFI_TIMEOUT if the Procedure fails to complete within
> the optional
> +                                        timeout. The implementation will update this variable
> with
> +                                        EFI_NOT_READY prior to starting Procedure on the
> target AP.
> +
> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
> Procedure has completed
> +                                        execution on the target AP.
> +                                        In the non-blocking case this indicates that the
> procedure has
> +                                        been successfully scheduled for execution on the target
> AP.
> +  @retval EFI_INVALID_PARAMETER         The input arguments are out of
> range. Either the target AP is the
> +                                        caller of the function, or the Procedure or Token is NULL
> +  @retval EFI_NOT_READY                 If the target AP is busy executing another
> procedure
> +  @retval EFI_ALREADY_STARTED           Token is already in use for another
> procedure
> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
> before the specified AP
> +                                        has finished
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpDispatchProcedure (
> +  IN CONST EFI_SMM_MP_PROTOCOL           *This,
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         CpuNumber,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   SMM_COMPLETION                *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +  )
> +{

1. SPIN_LOCK      *Lock;

> +  if (Token != NULL) {
> +    *Token = AllocatePool (sizeof (SPIN_LOCK));

Lock = AllocatePool (...);

> +    ASSERT (*Token != NULL); 
If (Lock == NULL) {
  Return EFI_DEVICE_ERROR;
}
> +    InitializeSpinLock ((SPIN_LOCK *)(*Token));
InitializeSpinLock (Lock);
> +
*Token = Lock;
> +    return InternalSmmStartupThisAp(
> +      Procedure,
> +      CpuNumber,
> +      ProcedureArguments,
> +      SmmCpuCallInNewType,
> +      (SPIN_LOCK *)(*Token),
(Token != NULL) ? Lock : NULL

The two InternalSmmStartupThisAp() can be combined to one call.

> +      TimeoutInMicroseconds,
> +      CPUStatus
> +      );
> +  }
> +
> +  return InternalSmmStartupThisAp(
> +    Procedure,
> +    CpuNumber,
> +    ProcedureArguments,
> +    SmmCpuCallInNewType,
> +    NULL,
> +    TimeoutInMicroseconds,
> +    CPUStatus
> +    );
> +}
> +
> +/**
> +  This service allows the caller to invoke a procedure on all running
> application processors (AP)
> +  except the caller. This function uses an optional token parameter to
> support blocking and
> +  nonblocking modes. If the token is passed into the call, the function will
> operate in a non-blocking
> +  fashion and the caller can check for completion with CheckOnProcedure or
> WaitForProcedure.
> +
> +  It is not necessary for the implementation to run the procedure on every
> processor on the platform.
> +  Processors that are powered down in such a way that they cannot respond
> to interrupts, may be
> +  excluded from the broadcast.
> +
> +
> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]     Procedure              A pointer to the code stream to be run on
> the APs that have
> +                                        entered MM. Type EFI_AP_PROCEDURE is defined
> below in related
> +                                        definitions.
> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
> microseconds for the APs to finish
> +                                        execution of Procedure, either for blocking or non-
> blocking mode.
> +                                        Zero means infinity. If the timeout expires before all
> APs return
> +                                        from Procedure, then Procedure on the failed APs is
> terminated. If
> +                                        the timeout expires in blocking mode, the call returns
> EFI_TIMEOUT.
> +                                        If the timeout expires in non-blocking mode, the
> timeout determined
> +                                        can be through CheckOnProcedure or
> WaitForProcedure.
> +                                        Note that timeout support is optional. Whether an
> implementation
> +                                        supports this feature can be determined via the
> Attributes data
> +                                        member.
> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
> parameters to the code
> +                                        that is run by the AP. It is an optional common mailbox
> +                                        between APs and the caller to share information.
> +  @param[in,out] Token                  This is parameter is broken into two
> components:
> +                                        1.Token->Completion is an optional parameter that
> allows the
> +                                        caller to execute the procedure in a blocking or non-
> blocking
> +                                        fashion. If it is NULL the call is blocking, and the call will
> +                                        not return until the AP has completed the procedure. If
> the
> +                                        token is not NULL, the call will return immediately. The
> caller
> +                                        can check whether the procedure has completed with
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        2.Token->Status The implementation updates the
> address pointed
> +                                        at by this variable with the status code returned by
> Procedure
> +                                        when it completes execution on the target AP, or with
> EFI_TIMEOUT
> +                                        if the Procedure fails to complete within the optional
> timeout.
> +                                        The implementation will update this variable with
> EFI_NOT_READY
> +                                        prior to starting Procedure on the target AP
> +  @param[in,out] CPUStatus              This optional pointer may be used to get
> the individual status
> +                                        returned by every AP that participated in the broadcast.
> This
> +                                        parameter if used provides the base address of an array
> to hold
> +                                        the EFI_STATUS value of each AP in the system. The size
> of the
> +                                        array can be ascertained by the
> GetNumberOfProcessors function.
> +                                        As mentioned above, the broadcast may not include
> every processor
> +                                        in the system. Some implementations may exclude
> processors that
> +                                        have been powered down in such a way that they are
> not responsive
> +                                        to interrupts. Additionally the broadcast excludes the
> processor
> +                                        which is making the BroadcastProcedure call. For every
> excluded
> +                                        processor, the array entry must contain a value of
> EFI_NOT_STARTED
> +
> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
> Procedure has completed
> +                                        execution on the APs.
> +                                        In the non-blocking case this indicates that the
> procedure has
> +                                        been successfully scheduled for execution on the APs.
> +  @retval EFI_INVALID_PARAMETER         The Procedure or Token is NULL
> +  @retval EFI_NOT_READY                 If the target AP is busy executing another
> procedure
> +  @retval EFI_ALREADY_STARTED           Token is already in use for another
> procedure
> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
> before the specified AP
> +                                        has finished.
> +
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpBroadcastProcedure (
> +  IN CONST EFI_SMM_MP_PROTOCOL           *This,
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   SMM_COMPLETION                *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +  )
> +{
> +  if (Token != NULL) {
> +    *Token = AllocatePool (sizeof (SPIN_LOCK));
> +    ASSERT (*Token != NULL);
> +    InitializeSpinLock ((SPIN_LOCK *) (*Token));
> +
> +    return InternalSmmStartupAllAPs(
> +      Procedure,
> +      TimeoutInMicroseconds,
> +      ProcedureArguments,
> +      (SPIN_LOCK *) (*Token),
> +      CPUStatus
> +      );
> +  }

2. Similar comments.

> +
> +  return InternalSmmStartupAllAPs(
> +    Procedure,
> +    TimeoutInMicroseconds,
> +    ProcedureArguments,
> +    NULL,
> +    CPUStatus
> +    );
> +}
> +
> +
> +/**
> +  This service allows the caller to set a startup procedure that will be
> executed when an AP powers
> +  up from a state where core configuration and context is lost. The
> procedure is execution has the
> +  following properties:
> +  1. The procedure executes before the processor is handed over to the
> operating system.
> +  2. All processors execute the same startup procedure.
> +  3. The procedure may run in parallel with other procedures invoked
> through the functions in this
> +  protocol, or with processors that are executing an MM handler or running
> in the operating system.
> +
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Procedure            A pointer to the code stream to be run on
> the designated target AP
> +                                       of the system. Type EFI_AP_PROCEDURE is defined
> below in Volume 2
> +                                       with the related definitions of
> +                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
> +                                       If caller may pass a value of NULL to deregister any
> existing
> +                                       startup procedure.
> +  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of
> parameters to the code that is
> +                                       run by the AP. It is an optional common mailbox
> between APs and
> +                                       the caller to share information
> +
> +  @retval EFI_SUCCESS                  The Procedure has been set successfully.
> +  @retval EFI_INVALID_PARAMETER        The Procedure is NULL but
> ProcedureArguments not NULL.
> +
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpSetStartupProcedure (
> +  IN CONST EFI_SMM_MP_PROTOCOL *This,
> +  IN       EFI_AP_PROCEDURE    Procedure,
> +  IN OUT   VOID                *ProcedureArguments OPTIONAL
> +  )
> +{
> +  return RegisterStartupProcedure (Procedure, ProcedureArguments);
> +}
> +
> +/**
> +  When non-blocking execution of a procedure on an AP is invoked with
> DispatchProcedure,
> +  via the use of a token, this function can be used to check for completion of
> the procedure on the AP.
> +  The function takes the token that was passed into the DispatchProcedure
> call. If the procedure
> +  is complete, and therefore it is now possible to run another procedure on
> the same AP, this function
> +  returns EFI_SUCESS. In this case the status returned by the procedure that
> executed on the AP is
> +  returned in the token's Status field. If the procedure has not yet
> completed, then this function
> +  returns EFI_NOT_READY.
> +
> +  When a non-blocking execution of a procedure is invoked with
> BroadcastProcedure, via the
> +  use of a token, this function can be used to check for completion of the
> procedure on all the
> +  broadcast APs. The function takes the token that was passed into the
> BroadcastProcedure
> +  call. If the procedure is complete on all broadcast APs this function returns
> EFI_SUCESS. In this
> +  case the Status field in the token passed into the function reflects the
> overall result of the
> +  invocation, which may be EFI_SUCCESS, if all executions succeeded, or the
> first observed failure.
> +  If the procedure has not yet completed on the broadcast APs, the function
> returns
> +  EFI_NOT_READY.
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Token                This parameter describes the token that was
> passed into
> +                                       DispatchProcedure or BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS                  Procedure has completed.
> +  @retval EFI_NOT_READY                The Procedure has not completed.
> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is
> NULL
> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-
> blocking call
> +
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpCheckForProcedure (
> +  IN CONST EFI_SMM_MP_PROTOCOL            *This,
> +  IN       SMM_COMPLETION                 Token
> +  )
> +{
> +  if (Token == NULL) {
> +    return EFI_INVALID_PARAMETER;
> +  }
> +
> +  if (!CurrentUsedToken ((SPIN_LOCK *)Token)) {

3. CurrentUsedToken --> IsCurrentToken()?

> +    return EFI_NOT_FOUND;
> +  }
> +
> +  return CheckStatusForThisAP ((SPIN_LOCK *)Token);


4. return IsApReady(Token) ? EFI_SUCCESS : EFI_NOT_READY;

> +}
> +
> +/**
> +  When a non-blocking execution of a procedure on an AP is invoked via
> DispatchProcedure,
> +  this function will block the caller until the remote procedure has completed
> on the designated AP.
> +  The non-blocking procedure invocation is identified by the Token
> parameter, which must match the
> +  token that used when DispatchProcedure was called. Upon completion the
> status returned by
> +  the procedure that executed on the AP is used to update the token's
> Status field.
> +
> +  When a non-blocking execution of a procedure on an AP is invoked via
> BroadcastProcedure
> +  this function will block the caller until the remote procedure has completed
> on all of the APs that
> +  entered MM. The non-blocking procedure invocation is identified by the
> Token parameter, which
> +  must match the token that used when BroadcastProcedure was called.
> Upon completion the
> +  overall status returned by the procedures that executed on the broadcast
> AP is used to update the
> +  token's Status field. The overall status may be EFI_SUCCESS, if all
> executions succeeded, or the
> +  first observed failure.
> +
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Token                This parameter describes the token that was
> passed into
> +                                       DispatchProcedure or BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS                  Procedure has completed.
> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is
> NULL
> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-
> blocking call
> +
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpWaitForProcedure (
> +  IN CONST EFI_SMM_MP_PROTOCOL            *This,
> +  IN       SMM_COMPLETION                 Token
> +  )
> +{
> +  EFI_STATUS    Status;
> +
> +  do {
> +    Status = SmmMpCheckForProcedure (This, Token);
> +  } while (Status == EFI_NOT_READY);


5. Use IsApReady()?

> +
> +  return Status;
> +}
> +

6. The file name can be SmmMp.c/h?

> diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> index 64fb4d6344..6b08dc0c4e 100644
> --- a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> @@ -146,6 +146,56 @@ ReleaseAllAPs (
>    }
>  }
> 
> +/**
> +  Wheck whether task has been finished by all APs.
> +
> +  @retval      TRUE        Task has been finished by all APs.
> +  @retval      FALSE       Task not has been finished by all APs.
> +
> +**/
> +BOOLEAN
> +TaskFinishInAllAPs (
> +  VOID
> +  )
> +{
> +  UINTN                             Index;
> +  UINTN                             BspIndex;
> +
> +  BspIndex = mSmmMpSyncData->BspIndex;
> +
> +  for (Index = mMaxNumberOfCpus; Index-- > 0;) {
> +    if (Index != BspIndex && *(mSmmMpSyncData->CpuData[Index].Present)
> && !mSmmMpSyncData->FinishTask[Index]) {
> +      return FALSE;
> +    }
> +  }
> +
> +  return TRUE;
> +}
> +
> +/**
> +  Checks whenther executive procedure in sync mode.
> +
> +  @param[in]       CallInType               Which call for this function.
> +  @param[in]       Token                    Token input by caller.
> +
> +
> +  @retval   TRUE    Executive procedure in sync mode.
> +  @retval   FALSE   Executive procedure not in sync mode.
> +
> +**/
> +BOOLEAN
> +ExecutiveInSyncMode (
> +  IN      SMM_CPU_CALL_IN_TYPE           CallInType,
> +  IN      SPIN_LOCK                      *Token
> +  )
> +{
> +  if (CallInType == SmmCpuCallInOldSyncType) {
> +    return TRUE;
> +  }
> +
> +  return Token == NULL;
> +}
> +
>  /**
>    Checks if all CPUs (with certain exceptions) have checked in for this SMI run
> 
> @@ -347,6 +397,30 @@ ReplaceOSMtrrs (
>    MtrrSetAllMtrrs (&gSmiMtrrs);
>  }
> 
> +/**
> +  Clear the flags used for execute one procedure.
> +
> +  @param     BlockStyle        Is this procedure block style or non-block.
> +  @param     SingleProcessor   Is this procedure only for single processor.
> +
> +**/
> +VOID
> +ClearProcedureFlags (
> +  IN BOOLEAN             BlockStyle,
> +  IN BOOLEAN             SingleProcessor
> +  )
> +{
> +  mSmmMpSyncData->CpuCheckMode = SmmCpuCheckModeMax;
> +
> +  if (SingleProcessor) {
> +    mSmmMpSyncData->CpuIndex = (UINTN) -1;
> +  }
> +
> +  if (!BlockStyle) {
> +    ReleaseSpinLock (mSmmMpSyncData->CurrentToken);
> +  }
> +}
> +
>  /**
>    SMI handler for BSP.
> 
> @@ -604,6 +678,7 @@ APHandler (
>    UINT64                            Timer;
>    UINTN                             BspIndex;
>    MTRR_SETTINGS                     Mtrrs;
> +  EFI_STATUS                        ProcedureStatus;
> 
>    //
>    // Timeout BSP
> @@ -730,14 +805,43 @@ APHandler (
>      //
>      // Invoke the scheduled procedure
>      //
> -    (*mSmmMpSyncData->CpuData[CpuIndex].Procedure) (
> -      (VOID*)mSmmMpSyncData->CpuData[CpuIndex].Parameter
> -      );
> +    ProcedureStatus = (*mSmmMpSyncData-
> >CpuData[CpuIndex].Procedure) (
> +                          (VOID*)mSmmMpSyncData->CpuData[CpuIndex].Parameter
> +                          );
> +
> +    if (mSmmMpSyncData->CurrentToken != NULL) {
> +      if (mSmmMpSyncData->CpuCheckMode == SmmCpuCheckModeSingle)
> {
> +        if (mSmmMpSyncData->CpuStatus != NULL) {
> +          *mSmmMpSyncData->CpuStatus = ProcedureStatus;
> +        }
> +      } else {
> +        ASSERT (mSmmMpSyncData->CpuCheckMode ==
> SmmCpuCheckModeAll);
> +        if (mSmmMpSyncData->CpuStatus != NULL) {
> +          mSmmMpSyncData->CpuStatus[CpuIndex] = ProcedureStatus;
> +        }
> +      }
> +    }
> 
>      //
>      // Release BUSY
>      //
>      ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
> +
> +    //
> +    // In non-block mode, Update the sync flags (CpuCheckMode, CpuIndex,
> FinishTask, CurrentToken) here.
> +    //
> +    if (mSmmMpSyncData->CurrentToken != NULL) {
> +      if (mSmmMpSyncData->CpuCheckMode == SmmCpuCheckModeSingle)
> {
> +        if (mSmmMpSyncData->CpuIndex == CpuIndex) {
> +          ClearProcedureFlags (FALSE, TRUE);
> +        }
> +      } else if (mSmmMpSyncData->CpuCheckMode ==
> SmmCpuCheckModeAll) {
> +        mSmmMpSyncData->FinishTask[CpuIndex] = TRUE;

7. Can FinishTask be eliminated by just using mSmmMpSyncData->CpuData[CpuIndex].Busy?

> +        if (TaskFinishInAllAPs()) {
> +          ClearProcedureFlags (FALSE, FALSE);
> +        }
> +      }
> +    }
>    }
> 
>    if (SmmCpuFeaturesNeedConfigureMtrrs()) {
> @@ -906,13 +1010,82 @@ Gen4GPageTable (
>    return (UINT32)(UINTN)PageTable;
>  }
> 
> +/**
> +  Checks whether the input token is the current used token.
> +
> +
> +  @param[in]  Token      This parameter describes the token that was passed
> into DispatchProcedure or
> +                         BroadcastProcedure.
> +
> +  @retval TRUE           The input token is the current used token.
> +  @retval FALSE          The input token is not the current used token.
> +**/
> +BOOLEAN
> +CurrentUsedToken (
> +  IN SPIN_LOCK           *Token
> +  )
> +{
> +  return mSmmMpSyncData->CurrentToken == Token;
> +}
> +
> +/**
> +  Checks status of specified AP.
> +
> +  This function checks whether the specified AP has finished the task
> assigned
> +  by StartupThisAP(), and whether timeout expires.
> +
> +  @param[in]  Token             This parameter describes the token that was
> passed into DispatchProcedure or
> +                                BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS           Specified AP has finished task assigned by
> StartupThisAPs().
> +  @retval EFI_NOT_READY         Specified AP has not finished task and
> timeout has not expired.
> +**/
> +EFI_STATUS
> +CheckStatusForThisAP (
> +  IN SPIN_LOCK          *Token
> +  )
> +{
> +  if (AcquireSpinLockOrFail (Token)) {
> +    ReleaseSpinLock (Token);
> +    return EFI_SUCCESS;
> +  }
> +
> +  return EFI_NOT_READY;
> +}
> +
> +
>  /**
>    Schedule a procedure to run on the specified CPU.
> 
>    @param[in]       Procedure                The address of the procedure to run
>    @param[in]       CpuIndex                 Target CPU Index
>    @param[in, out]  ProcArguments            The parameter to pass to the
> procedure
> -  @param[in]       BlockingMode             Startup AP in blocking mode or not
> +  @param[in]       CallInType               Which call for this function.
> +  @param[in, out]  Token                    This is parameter is broken into two
> components:
> +                                            1.Token->Completion is an optional parameter that
> allows the
> +                                            caller to execute the procedure in a blocking or non-
> blocking
> +                                            fashion. If it is NULL the call is blocking, and the call will
> +                                            not return until the AP has completed the procedure.
> If the
> +                                            token is not NULL, the call will return immediately.
> The caller
> +                                            can check whether the procedure has completed with
> +                                            CheckOnProcedure or WaitForProcedure.
> +                                            2.Token->Status The implementation updates the
> address pointed
> +                                            at by this variable with the status code returned by
> Procedure
> +                                            when it completes execution on the target AP, or with
> EFI_TIMEOUT
> +                                            if the Procedure fails to complete within the optional
> timeout.
> +                                            The implementation will update this variable with
> EFI_NOT_READY
> +                                            prior to starting Procedure on the target AP
> +  @param[in]       TimeoutInMicroseconds    Indicates the time limit in
> microseconds for the APs to finish
> +                                            execution of Procedure, either for blocking or non-
> blocking mode.
> +                                            Zero means infinity. If the timeout expires before all
> APs return
> +                                            from Procedure, then Procedure on the failed APs is
> terminated. If
> +                                            the timeout expires in blocking mode, the call returns
> EFI_TIMEOUT.
> +                                            If the timeout expires in non-blocking mode, the
> timeout determined
> +                                            can be through CheckOnProcedure or
> WaitForProcedure.
> +                                            Note that timeout support is optional. Whether an
> implementation
> +                                            supports this feature can be determined via the
> Attributes data
> +                                            member.
> +
> 
>    @retval EFI_INVALID_PARAMETER    CpuNumber not valid
>    @retval EFI_INVALID_PARAMETER    CpuNumber specifying BSP
> @@ -923,10 +1096,13 @@ Gen4GPageTable (
>  **/
>  EFI_STATUS
>  InternalSmmStartupThisAp (
> -  IN      EFI_AP_PROCEDURE          Procedure,
> -  IN      UINTN                     CpuIndex,
> -  IN OUT  VOID                      *ProcArguments OPTIONAL,
> -  IN      BOOLEAN                   BlockingMode
> +  IN      EFI_AP_PROCEDURE               Procedure,
> +  IN      UINTN                          CpuIndex,
> +  IN OUT  VOID                           *ProcArguments OPTIONAL,
> +  IN      SMM_CPU_CALL_IN_TYPE           CallInType,
> +  IN      SPIN_LOCK                      *Token,
> +  IN      UINTN                          TimeoutInMicroseconds,
> +  IN      EFI_STATUS                     *CpuStatus
>    )
>  {
>    if (CpuIndex >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus)
> {
> @@ -952,27 +1128,231 @@ InternalSmmStartupThisAp (
>      }
>      return EFI_INVALID_PARAMETER;
>    }
> +  if ((TimeoutInMicroseconds != 0) && ((mSmmMp.Attributes &
> EFI_SMM_MP_TIMEOUT_SUPPORTED) == 0)) {
> +    DEBUG((DEBUG_ERROR, "Input TimeoutInMicroseconds != 0 but
> EFI_SMM_MP_TIMEOUT_SUPPORTED not supported!\n"));
> +    return EFI_INVALID_PARAMETER;

8. Is mSmmMp.Attributes changed at runtime? If it's a constant value, can
the code directly reject non-zero TimeoutInMicroseconds?

> +  }
> +  if (Procedure == NULL) {
> +    return EFI_INVALID_PARAMETER;
> +  }
> +  if (Token != NULL && Token == mSmmMpSyncData->CurrentToken) {

9. Can IsCurrentToken() be used?

> +    return EFI_ALREADY_STARTED;
> +  }
> +  if (!AcquireSpinLockOrFail (mSmmMpSyncData->CpuData[CpuIndex].Busy))

10. In old style blocking mode, consumer expects code is waiting here. It might cause backward compatibility issue here.

> {
> +    return EFI_NOT_READY;
> +  } else {
> +    ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
> +  }
> 
> -  if (BlockingMode) {
> +  if (ExecutiveInSyncMode (CallInType, Token)) {

11. Can we simplify the CallInType? SmmCpuCallInOldSyncType is the same as when
Token equals to NULL, right?
SmmCpuCallInOldAsyncType is assigned but I cannot find code that checks this type.
So please think about if this type can be eliminated or replaced with a BOOLEAN flag.


>      AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
>    } else {
> +    if (Token != NULL && !AcquireSpinLockOrFail (Token)) {
> +      DEBUG((DEBUG_ERROR, "Token->Completion can't acquire\n"));
> +      return EFI_INVALID_PARAMETER;
> +    }
>      if (!AcquireSpinLockOrFail (mSmmMpSyncData-
> >CpuData[CpuIndex].Busy)) {
>        DEBUG((DEBUG_ERROR, "mSmmMpSyncData->CpuData[%d].Busy\n",
> CpuIndex));
> -      return EFI_INVALID_PARAMETER;
> +      ReleaseSpinLock (Token);
> +      return EFI_NOT_READY;
>      }
>    }
> 
> -  mSmmMpSyncData->CpuData[CpuIndex].Procedure = Procedure;
> +  //
> +  // PI spec requirement in EFI_MM_MP_PROTOCOL.DispatchProcedure():
> +  // The implementation will update this variable with
> +  // EFI_NOT_READY prior to starting Procedure on the target AP.
> +  //
> +  mSmmMpSyncData->CurrentToken = Token;
> +  mSmmMpSyncData->CpuCheckMode = SmmCpuCheckModeSingle;
> +  mSmmMpSyncData->CpuIndex = CpuIndex;

12. Can CpuCheckMode be eliminated since mSmmMpSyncData->CpuIndex 
having value other (-1) indicates it's a StartupThisAp call.
Above code in ApHandler() can just check mSmmMpSyncData->CpuIndex
instead of checking mSmmMpSyncData->CpuCheckMode.

> +  mSmmMpSyncData->CpuStatus = CpuStatus;
> +  if (mSmmMpSyncData->CpuStatus != NULL) {
> +    *mSmmMpSyncData->CpuStatus = EFI_NOT_READY;
> +  }
> +
> +  mSmmMpSyncData->CpuData[CpuIndex].Procedure =
> (EFI_AP_PROCEDURE2) Procedure;
>    mSmmMpSyncData->CpuData[CpuIndex].Parameter = ProcArguments;
> +
>    ReleaseSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);
> 
> -  if (BlockingMode) {
> +  if (ExecutiveInSyncMode (CallInType, Token)) {
>      AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
>      ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
> +
> +    //
> +    // In block mode, update the sync flags (CpuCheckMode, CpuIndex,
> FinishTask, CurrentToken) here.
> +    //
> +    if (Token == NULL) {
> +      ClearProcedureFlags (TRUE, TRUE);
> +    }
> +  }
> +
> +  return EFI_SUCCESS;
> +}
> +
> +/**
> +  Worker function to execute a caller provided function on all enabled APs.
> +
> +  @param[in]  Procedure               A pointer to the function to be run on
> +                                      enabled APs of the system.
> +  @param[in]  SingleThread            If TRUE, then all the enabled APs execute
> +                                      the function specified by Procedure one by
> +                                      one, in ascending order of processor handle
> +                                      number.  If FALSE, then all the enabled APs
> +                                      execute the function specified by Procedure
> +                                      simultaneously.
> +  @param[in]  WaitEvent               The event created by the caller with
> CreateEvent()
> +                                      service.
> +  @param[in]  TimeoutInMicroseconds   Indicates the time limit in
> microseconds for
> +                                      APs to return from Procedure, either for
> +                                      blocking or non-blocking mode.
> +  @param[in]  ProcedureArgument       The parameter passed into
> Procedure for
> +                                      all APs.
> +  @param[out] FailedCpuList           If all APs finish successfully, then its
> +                                      content is set to NULL. If not all APs
> +                                      finish before timeout expires, then its
> +                                      content is set to address of the buffer
> +                                      holding handle numbers of the failed APs.
> +
> +  @retval EFI_SUCCESS             In blocking mode, all APs have finished before
> +                                  the timeout expired.
> +  @retval EFI_SUCCESS             In non-blocking mode, function has been
> dispatched
> +                                  to all enabled APs.
> +  @retval others                  Failed to Startup all APs.
> +
> +**/
> +EFI_STATUS
> +InternalSmmStartupAllAPs (
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   SPIN_LOCK                     *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +  )
> +{
> +  UINTN               Index;
> +  UINTN               CpuCount;
> +
> +  if (Procedure == NULL) {
> +    return EFI_INVALID_PARAMETER;
> +  }
> +  if ((TimeoutInMicroseconds != 0) && ((mSmmMp.Attributes &
> EFI_SMM_MP_TIMEOUT_SUPPORTED) == 0)) {
> +    return EFI_INVALID_PARAMETER;
> +  }
> +  if (Token != NULL && Token == mSmmMpSyncData->CurrentToken) {
> +    return EFI_ALREADY_STARTED;
> +  }
> +
> +  CpuCount = 0;
> +  for (Index = mMaxNumberOfCpus; Index-- > 0;) {
> +    if (*mSmmMpSyncData->CpuData[Index].Present && (Index !=
> gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu)) {
> +      CpuCount ++;
> +
> +      if (gSmmCpuPrivate->Operation[Index] == SmmCpuRemove) {
> +        return EFI_INVALID_PARAMETER;
> +      }
> +
> +      if (!AcquireSpinLockOrFail(mSmmMpSyncData->CpuData[Index].Busy)) {
> +        return EFI_NOT_READY;
> +      }
> +      ReleaseSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
> +    }
> +  }
> +  if (CpuCount == 0) {
> +    return EFI_NOT_STARTED;
> +  }
> +
> +  if (Token == NULL) {
> +    //
> +    // Make sure all BUSY should be acquired.
> +    //
> +    for (Index = mMaxNumberOfCpus; Index-- > 0;) {
> +      if (Index != gSmmCpuPrivate-
> >SmmCoreEntryContext.CurrentlyExecutingCpu && *(mSmmMpSyncData-
> >CpuData[Index].Present)) {
> +        AcquireSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
> +      }
> +    }
> +  } else {
> +    if (!AcquireSpinLockOrFail (Token)) {
> +      return EFI_INVALID_PARAMETER;
> +    }
> +
> +    //
> +    // Make sure all BUSY should be acquired.
> +    //
> +    for (Index = mMaxNumberOfCpus; Index-- > 0;) {
> +      if (Index != gSmmCpuPrivate-
> >SmmCoreEntryContext.CurrentlyExecutingCpu && *(mSmmMpSyncData-
> >CpuData[Index].Present)) {
> +        if (!AcquireSpinLockOrFail (mSmmMpSyncData->CpuData[Index].Busy))
> {
> +          DEBUG((DEBUG_ERROR, "mSmmMpSyncData->CpuData[%d].Busy\n",
> Index));
> +
> +          //
> +          // Release BUSY accquired before.
> +          //
> +          for (CpuCount = mMaxNumberOfCpus; CpuCount -- > Index;) {
> +            if (CpuCount != gSmmCpuPrivate-
> >SmmCoreEntryContext.CurrentlyExecutingCpu && *(mSmmMpSyncData-
> >CpuData[CpuCount].Present)) {
> +              ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuCount].Busy);
> +            }
> +          }
> +
> +          ReleaseSpinLock (Token);
> +          return EFI_INVALID_PARAMETER;
> +        }
> +      }
> +    }
> +  }
> +
> +  //
> +  // PI spec requirement:
> +  // The implementation will update this variable with
> +  // EFI_NOT_READY prior to starting Procedure on the target AP.
> +  //
> +  mSmmMpSyncData->CurrentToken = Token;
> +  mSmmMpSyncData->CpuCheckMode = SmmCpuCheckModeAll;
> +  mSmmMpSyncData->CpuStatus = CPUStatus;


12. Similar comments as above. mSmmMpSyncData->CpuIndex can be assigned to -1 indicating
it's a StartupALLAP call.

> +
> +  for (Index = mMaxNumberOfCpus; Index-- > 0;) {
> +    if (Index != gSmmCpuPrivate-
> >SmmCoreEntryContext.CurrentlyExecutingCpu && *(mSmmMpSyncData-
> >CpuData[Index].Present)) {
> +      mSmmMpSyncData->CpuData[Index].Procedure =
> (EFI_AP_PROCEDURE2) Procedure;
> +      mSmmMpSyncData->CpuData[Index].Parameter =
> ProcedureArguments;
> +    } else {
> +      //
> +      // PI spec requirement:
> +      // For every excluded processor, the array entry must contain a value of
> EFI_NOT_STARTED.
> +      //
> +      if (mSmmMpSyncData->CpuStatus != NULL) {
> +        mSmmMpSyncData->CpuStatus[Index] = EFI_NOT_STARTED;
> +      }
> +    }
> +    //
> +    // Clear all status no matter this processor is used or not.
> +    // May not need to clear all but add it to make the status clear.
> +    //
> +    mSmmMpSyncData->FinishTask[Index] = FALSE;
>    }
> +
> +  ReleaseAllAPs ();
> +
> +  if (Token == NULL) {
> +    //
> +    // Make sure all APs have completed their tasks.
> +    //
> +    for (Index = mMaxNumberOfCpus; Index-- > 0;) {
> +      if (Index != gSmmCpuPrivate-
> >SmmCoreEntryContext.CurrentlyExecutingCpu && *(mSmmMpSyncData-
> >CpuData[Index].Present)) {
> +        AcquireSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
> +        ReleaseSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
> +      }
> +    }
> +
> +    //
> +    // Update the sync flags (CpuCheckMode, CpuIndex, FinishTask,
> CurrentToken) in block mode here.
> +    //
> +    ClearProcedureFlags (TRUE, FALSE);
> +  }
> +
>    return EFI_SUCCESS;
>  }
> 
> +
>  /**
>    Schedule a procedure to run on the specified CPU in blocking mode.
> 
> @@ -995,7 +1375,7 @@ SmmBlockingStartupThisAp (
>    IN OUT  VOID                      *ProcArguments OPTIONAL
>    )
>  {
> -  return InternalSmmStartupThisAp(Procedure, CpuIndex, ProcArguments,
> TRUE);
> +  return InternalSmmStartupThisAp(Procedure, CpuIndex, ProcArguments,
> SmmCpuCallInOldSyncType, NULL, 0, NULL);
>  }
> 
>  /**
> @@ -1020,7 +1400,15 @@ SmmStartupThisAp (
>    IN OUT  VOID                      *ProcArguments OPTIONAL
>    )
>  {
> -  return InternalSmmStartupThisAp(Procedure, CpuIndex, ProcArguments,
> FeaturePcdGet (PcdCpuSmmBlockStartupThisAp));
> +  SMM_CPU_CALL_IN_TYPE    CallInType;
> +
> +  if (FeaturePcdGet (PcdCpuSmmBlockStartupThisAp)) {
> +    CallInType = SmmCpuCallInOldSyncType;
> +  } else {
> +    CallInType = SmmCpuCallInOldAsyncType;
> +  }
> +
> +  return InternalSmmStartupThisAp(Procedure, CpuIndex, ProcArguments,
> CallInType, NULL, 0, NULL);
>  }
> 
>  /**
> @@ -1112,6 +1500,13 @@ SmiRendezvous (
>    Cr2 = 0;
>    SaveCr2 (&Cr2);
> 
> +  //
> +  // Call the user register Startup function first.
> +  //
> +  if (mSmmMpSyncData->StartupProcedure != NULL) {
> +    mSmmMpSyncData->StartupProcedure (mSmmMpSyncData-
> >StartupProcArgs);
> +  }
> +
>    //
>    // Perform CPU specific entry hooks
>    //
> @@ -1332,6 +1727,7 @@ InitializeMpSyncData (
>      ZeroMem (mSmmMpSyncData, mSmmMpSyncDataSize);
>      mSmmMpSyncData->CpuData = (SMM_CPU_DATA_BLOCK *)((UINT8
> *)mSmmMpSyncData + sizeof (SMM_DISPATCHER_MP_SYNC_DATA));
>      mSmmMpSyncData->CandidateBsp = (BOOLEAN *)(mSmmMpSyncData-
> >CpuData + gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus);
> +    mSmmMpSyncData->FinishTask = (BOOLEAN *)(mSmmMpSyncData-
> >CandidateBsp + gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus);
>      if (FeaturePcdGet (PcdCpuSmmEnableBspElection)) {
>        //
>        // Enable BSP election by setting BspIndex to -1
> @@ -1360,6 +1756,10 @@ InitializeMpSyncData (
>        *(mSmmMpSyncData->CpuData[CpuIndex].Run)     = 0;
>        *(mSmmMpSyncData->CpuData[CpuIndex].Present) = FALSE;
>      }
> +
> +    mSmmMpSyncData->CpuCheckMode = SmmCpuCheckModeMax;
> +    mSmmMpSyncData->CpuIndex = CpuIndex;
> +    mSmmMpSyncData->CurrentToken = NULL;
>    }
>  }
> 
> @@ -1399,7 +1799,7 @@ InitializeMpServiceData (
>    // Initialize mSmmMpSyncData
>    //
>    mSmmMpSyncDataSize = sizeof (SMM_DISPATCHER_MP_SYNC_DATA) +
> -                       (sizeof (SMM_CPU_DATA_BLOCK) + sizeof (BOOLEAN)) *
> gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus;
> +                       (sizeof (SMM_CPU_DATA_BLOCK) + sizeof (BOOLEAN) + sizeof
> (BOOLEAN)) * gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus;
>    mSmmMpSyncData = (SMM_DISPATCHER_MP_SYNC_DATA*)
> AllocatePages (EFI_SIZE_TO_PAGES (mSmmMpSyncDataSize));
>    ASSERT (mSmmMpSyncData != NULL);
>    mCpuSmmSyncMode = (SMM_CPU_SYNC_MODE)PcdGet8
> (PcdCpuSmmSyncMode);
> @@ -1469,3 +1869,41 @@ RegisterSmmEntry (
>    gSmmCpuPrivate->SmmCoreEntry = SmmEntryPoint;
>    return EFI_SUCCESS;
>  }
> +
> +/**
> +
> +  Register the SMM Foundation entry point.
> +
> +  @param[in]      Procedure            A pointer to the code stream to be run on
> the designated target AP
> +                                       of the system. Type EFI_AP_PROCEDURE is defined
> below in Volume 2
> +                                       with the related definitions of
> +                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
> +                                       If caller may pass a value of NULL to deregister any
> existing
> +                                       startup procedure.
> +  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of
> parameters to the code that is
> +                                       run by the AP. It is an optional common mailbox
> between APs and
> +                                       the caller to share information
> +
> +  @retval EFI_SUCCESS                  The Procedure has been set successfully.
> +  @retval EFI_INVALID_PARAMETER        The Procedure is NULL but
> ProcedureArguments not NULL.
> +
> +**/
> +EFI_STATUS
> +RegisterStartupProcedure (
> +  IN EFI_AP_PROCEDURE    Procedure,
> +  IN VOID                *ProcedureArguments OPTIONAL
> +  )
> +{
> +  if (Procedure == NULL && ProcedureArguments != NULL) {
> +    return EFI_INVALID_PARAMETER;
> +  }
> +  if (mSmmMpSyncData == NULL) {
> +    return EFI_NOT_READY;
> +  }
> +
> +  mSmmMpSyncData->StartupProcedure = Procedure;
> +  mSmmMpSyncData->StartupProcArgs  = ProcedureArguments;
> +
> +  return EFI_SUCCESS;
> +}
> +
> diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c
> b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c
> index 2f7d777ee7..e498f72801 100644
> --- a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c
> +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c
> @@ -996,6 +996,17 @@ PiCpuSmmEntry (
>                      );
>    ASSERT_EFI_ERROR (Status);
> 
> +  //
> +  // Install the SMM Mp Protocol into SMM protocol database
> +  //
> +  Status = gSmst->SmmInstallProtocolInterface (
> +                    &mSmmCpuHandle,
> +                    &gEfiSmmMpProtocolGuid,
> +                    EFI_NATIVE_INTERFACE,
> +                    &mSmmMp
> +                    );
> +  ASSERT_EFI_ERROR (Status);
> +
>    //
>    // Expose address of CPU Hot Plug Data structure if CPU hot plug is
> supported.
>    //
> diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h
> b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h
> index 2bb35a424d..b321ff15ee 100644
> --- a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h
> +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h
> @@ -20,6 +20,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent
>  #include <Protocol/SmmReadyToLock.h>
>  #include <Protocol/SmmCpuService.h>
>  #include <Protocol/SmmMemoryAttribute.h>
> +#include <Protocol/SmmMp.h>
> 
>  #include <Guid/AcpiS3Context.h>
>  #include <Guid/MemoryAttributesTable.h>
> @@ -226,6 +227,7 @@ extern CPU_HOT_PLUG_DATA      mCpuHotPlugData;
>  extern UINTN                  mMaxNumberOfCpus;
>  extern UINTN                  mNumberOfCpus;
>  extern EFI_SMM_CPU_PROTOCOL   mSmmCpu;
> +extern EFI_SMM_MP_PROTOCOL    mSmmMp;
> 
>  ///
>  /// The mode of the CPU at the time an SMI occurs
> @@ -363,7 +365,7 @@ SmmRelocationSemaphoreComplete (
>  ///
>  typedef struct {
>    SPIN_LOCK                         *Busy;
> -  volatile EFI_AP_PROCEDURE         Procedure;
> +  volatile EFI_AP_PROCEDURE2        Procedure;
>    volatile VOID                     *Parameter;
>    volatile UINT32                   *Run;
>    volatile BOOLEAN                  *Present;
> @@ -375,6 +377,19 @@ typedef enum {
>    SmmCpuSyncModeMax
>  } SMM_CPU_SYNC_MODE;
> 
> +typedef enum {
> +  SmmCpuCheckModeSingle,
> +  SmmCpuCheckModeAll,
> +  SmmCpuCheckModeMax
> +} SMM_CPU_CHECK_MODE;
> +
> +typedef enum {
> +  SmmCpuCallInOldSyncType,
> +  SmmCpuCallInOldAsyncType,
> +  SmmCpuCallInNewType,
> +  SmmCpuCallInTypeMax
> +} SMM_CPU_CALL_IN_TYPE;
> +
>  typedef struct {
>    //
>    // Pointer to an array. The array should be located immediately after this
> structure
> @@ -388,6 +403,14 @@ typedef struct {
>    volatile SMM_CPU_SYNC_MODE    EffectiveSyncMode;
>    volatile BOOLEAN              SwitchBsp;
>    volatile BOOLEAN              *CandidateBsp;
> +  SPIN_LOCK                     *CurrentToken;
> +  SMM_CPU_CHECK_MODE            CpuCheckMode;
> +  UINTN                         CpuIndex;
> +  EFI_STATUS                    *CpuStatus;
> +  volatile BOOLEAN              *FinishTask;
> +
> +  EFI_AP_PROCEDURE              StartupProcedure;
> +  VOID                          *StartupProcArgs;
>  } SMM_DISPATCHER_MP_SYNC_DATA;
> 
>  #define SMM_PSD_OFFSET              0xfb00
> @@ -1259,4 +1282,151 @@ RestoreCr2 (
>    IN UINTN  Cr2
>    );
> 
> +/**
> +  Schedule a procedure to run on the specified CPU.
> +
> +  @param[in]       Procedure                The address of the procedure to run
> +  @param[in]       CpuIndex                 Target CPU Index
> +  @param[in, out]  ProcArguments            The parameter to pass to the
> procedure
> +  @param[in]       CallInType               Which call for this function.
> +  @param[in, out]  Token                    This is parameter is broken into two
> components:
> +                                            1.Token->Completion is an optional parameter that
> allows the
> +                                            caller to execute the procedure in a blocking or non-
> blocking
> +                                            fashion. If it is NULL the call is blocking, and the call will
> +                                            not return until the AP has completed the procedure.
> If the
> +                                            token is not NULL, the call will return immediately.
> The caller
> +                                            can check whether the procedure has completed with
> +                                            CheckOnProcedure or WaitForProcedure.
> +                                            2.Token->Status The implementation updates the
> address pointed
> +                                            at by this variable with the status code returned by
> Procedure
> +                                            when it completes execution on the target AP, or with
> EFI_TIMEOUT
> +                                            if the Procedure fails to complete within the optional
> timeout.
> +                                            The implementation will update this variable with
> EFI_NOT_READY
> +                                            prior to starting Procedure on the target AP
> +  @param[in]       TimeoutInMicroseconds    Indicates the time limit in
> microseconds for the APs to finish
> +                                            execution of Procedure, either for blocking or non-
> blocking mode.
> +                                            Zero means infinity. If the timeout expires before all
> APs return
> +                                            from Procedure, then Procedure on the failed APs is
> terminated. If
> +                                            the timeout expires in blocking mode, the call returns
> EFI_TIMEOUT.
> +                                            If the timeout expires in non-blocking mode, the
> timeout determined
> +                                            can be through CheckOnProcedure or
> WaitForProcedure.
> +                                            Note that timeout support is optional. Whether an
> implementation
> +                                            supports this feature can be determined via the
> Attributes data
> +                                            member.
> +
> +
> +  @retval EFI_INVALID_PARAMETER    CpuNumber not valid
> +  @retval EFI_INVALID_PARAMETER    CpuNumber specifying BSP
> +  @retval EFI_INVALID_PARAMETER    The AP specified by CpuNumber did
> not enter SMM
> +  @retval EFI_INVALID_PARAMETER    The AP specified by CpuNumber is
> busy
> +  @retval EFI_SUCCESS              The procedure has been successfully
> scheduled
> +
> +**/
> +EFI_STATUS
> +InternalSmmStartupThisAp (
> +  IN      EFI_AP_PROCEDURE               Procedure,
> +  IN      UINTN                          CpuIndex,
> +  IN OUT  VOID                           *ProcArguments OPTIONAL,
> +  IN      SMM_CPU_CALL_IN_TYPE           CallInType,
> +  IN      SPIN_LOCK                      *Token,
> +  IN      UINTN                          TimeoutInMicroseconds,
> +  IN      EFI_STATUS                     *CpuStatus
> +  );
> +
> +/**
> +  Checks whether the input token is the current used token.
> +
> +
> +  @param[in]  Token      This parameter describes the token that was passed
> into DispatchProcedure or
> +                         BroadcastProcedure.
> +
> +  @retval TRUE           The input token is the current used token.
> +  @retval FALSE          The input token is not the current used token.
> +**/
> +BOOLEAN
> +CurrentUsedToken (
> +  IN SPIN_LOCK  *Token
> +  );
> +
> +/**
> +  Checks status of specified AP.
> +
> +  This function checks whether the specified AP has finished the task
> assigned
> +  by StartupThisAP(), and whether timeout expires.
> +
> +  @param[in]  Token             This parameter describes the token that was
> passed into DispatchProcedure or
> +                                BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS           Specified AP has finished task assigned by
> StartupThisAPs().
> +  @retval EFI_NOT_READY         Specified AP has not finished task and
> timeout has not expired.
> +**/
> +EFI_STATUS
> +CheckStatusForThisAP (
> +  IN SPIN_LOCK  *Token
> +  );
> +
> +/**
> +  Worker function to execute a caller provided function on all enabled APs.
> +
> +  @param[in]  Procedure               A pointer to the function to be run on
> +                                      enabled APs of the system.
> +  @param[in]  SingleThread            If TRUE, then all the enabled APs execute
> +                                      the function specified by Procedure one by
> +                                      one, in ascending order of processor handle
> +                                      number.  If FALSE, then all the enabled APs
> +                                      execute the function specified by Procedure
> +                                      simultaneously.
> +  @param[in]  WaitEvent               The event created by the caller with
> CreateEvent()
> +                                      service.
> +  @param[in]  TimeoutInMicroseconds   Indicates the time limit in
> microseconds for
> +                                      APs to return from Procedure, either for
> +                                      blocking or non-blocking mode.
> +  @param[in]  ProcedureArgument       The parameter passed into
> Procedure for
> +                                      all APs.
> +  @param[out] FailedCpuList           If all APs finish successfully, then its
> +                                      content is set to NULL. If not all APs
> +                                      finish before timeout expires, then its
> +                                      content is set to address of the buffer
> +                                      holding handle numbers of the failed APs.
> +
> +  @retval EFI_SUCCESS             In blocking mode, all APs have finished before
> +                                  the timeout expired.
> +  @retval EFI_SUCCESS             In non-blocking mode, function has been
> dispatched
> +                                  to all enabled APs.
> +  @retval others                  Failed to Startup all APs.
> +
> +**/
> +EFI_STATUS
> +InternalSmmStartupAllAPs (
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   SPIN_LOCK                     *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +  );
> +
> +/**
> +
> +  Register the SMM Foundation entry point.
> +
> +  @param[in]      Procedure            A pointer to the code stream to be run on
> the designated target AP
> +                                       of the system. Type EFI_AP_PROCEDURE is defined
> below in Volume 2
> +                                       with the related definitions of
> +                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
> +                                       If caller may pass a value of NULL to deregister any
> existing
> +                                       startup procedure.
> +  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of
> parameters to the code that is
> +                                       run by the AP. It is an optional common mailbox
> between APs and
> +                                       the caller to share information
> +
> +  @retval EFI_SUCCESS                  The Procedure has been set successfully.
> +  @retval EFI_INVALID_PARAMETER        The Procedure is NULL but
> ProcedureArguments not NULL.
> +
> +**/
> +EFI_STATUS
> +RegisterStartupProcedure (
> +  IN EFI_AP_PROCEDURE    Procedure,
> +  IN VOID                *ProcedureArguments OPTIONAL
> +  );
> +
>  #endif
> diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf
> b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf
> index 466c568d49..f478ae62a7 100644
> --- a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf
> +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf
> @@ -40,6 +40,8 @@
>    SmmProfileInternal.h
>    SmramSaveState.c
>    SmmCpuMemoryManagement.c
> +  MpProtocol.h
> +  MpProtocol.c
> 
>  [Sources.Ia32]
>    Ia32/Semaphore.c
> @@ -105,6 +107,7 @@
>    gEfiSmmReadyToLockProtocolGuid           ## NOTIFY
>    gEfiSmmCpuServiceProtocolGuid            ## PRODUCES
>    gEdkiiSmmMemoryAttributeProtocolGuid     ## PRODUCES
> +  gEfiSmmMpProtocolGuid                    ## PRODUCES
> 
>  [Guids]
>    gEfiAcpiVariableGuid                     ## SOMETIMES_CONSUMES ## HOB # it is
> used for S3 boot.
> --
> 2.21.0.windows.1

Re: [edk2-devel] [Patch 0/2] Enable new MM MP protocol.

[Laszlo Ersek]

On 06/19/19 07:51, Dong, Eric wrote:
> PI spec added a new protocol named MM MP protocol. This protocol allows
> for better remote queuing of execution of procedures on an AP.
> This extends the existing procedures to allow:
> 1. A function to be called in blocking and non-blocking manner explicitly
> 2. Allow broadcasts.
> 3. Allow execution of a procedure when a processor powers up.
>
> This patch serial enable this new protocol.
>
> Cc: Ray Ni <ray.ni@intel.com>
> Cc: Laszlo Ersek <lersek@redhat.com>
>
> Eric Dong (2):
>   MdePkg: Add new MM MP Protocol definition.
>   UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP Protocol.
>
>  MdePkg/Include/Pi/PiMultiPhase.h             |  16 +
>  MdePkg/Include/Protocol/MmMp.h               | 334 +++++++++++++
>  MdePkg/Include/Protocol/SmmMp.h              |  44 ++
>  MdePkg/MdePkg.dec                            |   6 +
>  UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c       | 375 +++++++++++++++
>  UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h       | 283 +++++++++++
>  UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c        | 468 ++++++++++++++++++-
>  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c   |  11 +
>  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h   | 172 ++++++-
>  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf |   3 +
>  10 files changed, 1696 insertions(+), 16 deletions(-)
>  create mode 100644 MdePkg/Include/Protocol/MmMp.h
>  create mode 100644 MdePkg/Include/Protocol/SmmMp.h
>  create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
>  create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h
>

I applied this set (for regression testing) on top of commit
f17935321a5b ("MdeModulePkg: Add CapsuleOnDiskLoadPei PEIM.",
2019-06-20).


First of all, git-am complained like this:

> Applying: MdePkg: Add new MM MP Protocol definition.
> .git/rebase-apply/patch:374: new blank line at EOF.
> +
> .git/rebase-apply/patch:424: new blank line at EOF.
> +
> warning: 2 lines add whitespace errors.
> Applying: UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP Protocol.
> .git/rebase-apply/patch:392: new blank line at EOF.
> +
> .git/rebase-apply/patch:1276: new blank line at EOF.
> +
> warning: 2 lines add whitespace errors.

Please consider removing those empty lines.


Second, the series does not build for me. I'll describe the issue under
the affected patch.

Thanks,
Laszlo

Re: [edk2-devel] [Patch 0/2] Enable new MM MP protocol.

[Liming Gao]

Eric:
  Please submit one BZ for it. Seemly, this is a new feature. I will add it into Q3 stable tag feature plan. 

Thanks
Liming
>-----Original Message-----
>From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of
>Dong, Eric
>Sent: Wednesday, June 19, 2019 1:51 PM
>To: devel@edk2.groups.io
>Cc: Ni, Ray <ray.ni@intel.com>; Laszlo Ersek <lersek@redhat.com>
>Subject: [edk2-devel] [Patch 0/2] Enable new MM MP protocol.
>
>PI spec added a new protocol named MM MP protocol. This protocol allows
>for better remote queuing of execution of procedures on an AP.
>This extends the existing procedures to allow:
>1. A function to be called in blocking and non-blocking manner explicitly
>2. Allow broadcasts.
>3. Allow execution of a procedure when a processor powers up.
>
>This patch serial enable this new protocol.
>
>Cc: Ray Ni <ray.ni@intel.com>
>Cc: Laszlo Ersek <lersek@redhat.com>
>
>Eric Dong (2):
>  MdePkg: Add new MM MP Protocol definition.
>  UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP Protocol.
>
> MdePkg/Include/Pi/PiMultiPhase.h             |  16 +
> MdePkg/Include/Protocol/MmMp.h               | 334 +++++++++++++
> MdePkg/Include/Protocol/SmmMp.h              |  44 ++
> MdePkg/MdePkg.dec                            |   6 +
> UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c       | 375 +++++++++++++++
> UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h       | 283 +++++++++++
> UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c        | 468
>++++++++++++++++++-
> UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c   |  11 +
> UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h   | 172 ++++++-
> UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf |   3 +
> 10 files changed, 1696 insertions(+), 16 deletions(-)
> create mode 100644 MdePkg/Include/Protocol/MmMp.h
> create mode 100644 MdePkg/Include/Protocol/SmmMp.h
> create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.c
> create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/MpProtocol.h
>
>--
>2.21.0.windows.1
>
>
>

[Patch v4 0/2] Enable new MM MP protocol

[Dong, Eric]

V4 changes:
1. Use link list to save the used tokens.

v3 changes:
1. Fix Token clean up too early caused CheckProcedure return error.

https://bugzilla.tianocore.org/show_bug.cgi?id=1937

PI spec added a new protocol named MM MP protocol. This protocol allows for better remote queuing of execution of procedures on an AP.
This extends the existing procedures to allow:
1. A function to be called in blocking and non-blocking manner explicitly 2. Allow broadcasts.
3. Allow execution of a procedure when a processor powers up.

This patch serial enable this new protocol.

Cc: Ray Ni <ray.ni@intel.com>
Cc: Laszlo Ersek <lersek@redhat.com>


Eric Dong (2):
  MdePkg: Add new MM MP Protocol definition.
  UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP Protocol.

 MdePkg/Include/Pi/PiMultiPhase.h             |  16 +
 MdePkg/Include/Protocol/MmMp.h               | 333 +++++++++++
 MdePkg/MdePkg.dec                            |   3 +
 UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c        | 558 ++++++++++++++++++-
 UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c   |  16 +
 UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h   | 175 +++++-
 UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf |   3 +
 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c            | 376 +++++++++++++
 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h            | 286 ++++++++++
 9 files changed, 1744 insertions(+), 22 deletions(-)
 create mode 100644 MdePkg/Include/Protocol/MmMp.h
 create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c
 create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h

-- 
2.21.0.windows.1

[Patch 1/2] MdePkg: Add new MM MP Protocol definition.

[Dong, Eric]

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=1937

EFI MM MP Protocol is defined in the PI 1.5 specification.

The MM MP protocol provides a set of functions to allow execution of
procedures on processors that have entered MM. This protocol has the
following properties:
1. The caller can invoke execution of a procedure on a processor, other
than the caller, that has also entered MM. Supports blocking and
non-blocking modes of operation.
2. The caller can invoke a procedure on multiple processors. Supports
blocking and non-blocking modes of operation.

Cc: Ray Ni <ray.ni@intel.com>
Cc: Laszlo Ersek <lersek@redhat.com>
Signed-off-by: Eric Dong <eric.dong@intel.com>
Reviewed-by: Ray Ni <ray.ni@intel.com>
---
 MdePkg/Include/Pi/PiMultiPhase.h |  16 ++
 MdePkg/Include/Protocol/MmMp.h   | 333 +++++++++++++++++++++++++++++++
 MdePkg/MdePkg.dec                |   3 +
 3 files changed, 352 insertions(+)
 create mode 100644 MdePkg/Include/Protocol/MmMp.h

diff --git a/MdePkg/Include/Pi/PiMultiPhase.h b/MdePkg/Include/Pi/PiMultiPhase.h
index eb12527767..a5056799e1 100644
--- a/MdePkg/Include/Pi/PiMultiPhase.h
+++ b/MdePkg/Include/Pi/PiMultiPhase.h
@@ -176,4 +176,20 @@ VOID
   IN OUT VOID  *Buffer
   );
 
+/**
+  The function prototype for invoking a function on an Application Processor.
+
+  This definition is used by the UEFI MM MP Serices Protocol.
+
+  @param[in] ProcedureArgument    The pointer to private data buffer.
+
+  @retval EFI_SUCCESS             Excutive the procedure successfully
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_AP_PROCEDURE2)(
+  IN VOID  *ProcedureArgument
+);
+
 #endif
diff --git a/MdePkg/Include/Protocol/MmMp.h b/MdePkg/Include/Protocol/MmMp.h
new file mode 100644
index 0000000000..beace1386c
--- /dev/null
+++ b/MdePkg/Include/Protocol/MmMp.h
@@ -0,0 +1,333 @@
+/** @file
+  EFI MM MP Protocol is defined in the PI 1.5 specification.
+
+  The MM MP protocol provides a set of functions to allow execution of procedures on processors that
+  have entered MM. This protocol has the following properties:
+  1. The caller can only invoke execution of a procedure on a processor, other than the caller, that
+     has also entered MM.
+  2. It is possible to invoke a procedure on multiple processors. Supports blocking and non-blocking
+     modes of operation.
+
+  Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+
+**/
+
+#ifndef _MM_MP_H_
+#define _MM_MP_H_
+
+#include <Pi/PiMmCis.h>
+
+#define EFI_MM_MP_PROTOCOL_GUID \
+  { \
+    0x5d5450d7, 0x990c, 0x4180, {0xa8, 0x3, 0x8e, 0x63, 0xf0, 0x60, 0x83, 0x7  }  \
+  }
+
+//
+// Revision definition.
+//
+#define EFI_MM_MP_PROTOCOL_REVISION    0x00
+
+//
+// Attribute flags
+//
+#define EFI_MM_MP_TIMEOUT_SUPPORTED    0x01
+
+//
+// Completion token
+//
+typedef VOID* MM_COMPLETION;
+
+typedef struct {
+  MM_COMPLETION  Completion;
+  EFI_STATUS     Status;
+} MM_DISPATCH_COMPLETION_TOKEN;
+
+typedef struct _EFI_MM_MP_PROTOCOL  EFI_MM_MP_PROTOCOL;
+
+/**
+  Service to retrieves the number of logical processor in the platform.
+
+  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
+  @param[out] NumberOfProcessors  Pointer to the total number of logical processors in the system,
+                                  including the BSP and all APs.
+
+  @retval EFI_SUCCESS             The number of processors was retrieved successfully
+  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MM_GET_NUMBER_OF_PROCESSORS) (
+  IN CONST EFI_MM_MP_PROTOCOL  *This,
+  OUT      UINTN               *NumberOfProcessors
+);
+
+
+/**
+  This service allows the caller to invoke a procedure one of the application processors (AP). This
+  function uses an optional token parameter to support blocking and non-blocking modes. If the token
+  is passed into the call, the function will operate in a non-blocking fashion and the caller can
+  check for completion with CheckOnProcedure or WaitForProcedure.
+
+  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
+  @param[in]     Procedure              A pointer to the procedure to be run on the designated target
+                                        AP of the system. Type EFI_AP_PROCEDURE2 is defined below in
+                                        related definitions.
+  @param[in]     CpuNumber              The zero-based index of the processor number of the target
+                                        AP, on which the code stream is supposed to run. If the number
+                                        points to the calling processor then it will not run the
+                                        supplied code.
+  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for this AP to
+                                        finish execution of Procedure, either for blocking or
+                                        non-blocking mode. Zero means infinity. If the timeout
+                                        expires before this AP returns from Procedure, then Procedure
+                                        on the AP is terminated. If the timeout expires in blocking
+                                        mode, the call returns EFI_TIMEOUT. If the timeout expires
+                                        in non-blocking mode, the timeout determined can be through
+                                        CheckOnProcedure or WaitForProcedure.
+                                        Note that timeout support is optional. Whether an
+                                        implementation supports this feature, can be determined via
+                                        the Attributes data member.
+  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
+                                        that is run by the AP. It is an optional common mailbox
+                                        between APs and the caller to share information.
+  @param[in,out] Token                  This is parameter is broken into two components:
+                                        1.Token->Completion is an optional parameter that allows the
+                                        caller to execute the procedure in a blocking or non-blocking
+                                        fashion. If it is NULL the call is blocking, and the call will
+                                        not return until the AP has completed the procedure. If the
+                                        token is not NULL, the call will return immediately. The caller
+                                        can check whether the procedure has completed with
+                                        CheckOnProcedure or WaitForProcedure.
+                                        2.Token->Status The implementation updates the address pointed
+                                        at by this variable with the status code returned by Procedure
+                                        when it completes execution on the target AP, or with EFI_TIMEOUT
+                                        if the Procedure fails to complete within the optional timeout.
+                                        The implementation will update this variable with EFI_NOT_READY
+                                        prior to starting Procedure on the target AP.
+  @param[in,out] CPUStatus              This optional pointer may be used to get the status code returned
+                                        by Procedure when it completes execution on the target AP, or with
+                                        EFI_TIMEOUT if the Procedure fails to complete within the optional
+                                        timeout. The implementation will update this variable with
+                                        EFI_NOT_READY prior to starting Procedure on the target AP.
+
+  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
+                                        execution on the target AP.
+                                        In the non-blocking case this indicates that the procedure has
+                                        been successfully scheduled for execution on the target AP.
+  @retval EFI_INVALID_PARAMETER         The input arguments are out of range. Either the target AP is the
+                                        caller of the function, or the Procedure or Token is NULL
+  @retval EFI_NOT_READY                 If the target AP is busy executing another procedure
+  @retval EFI_ALREADY_STARTED           Token is already in use for another procedure
+  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before the specified AP
+                                        has finished
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MM_DISPATCH_PROCEDURE) (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       EFI_AP_PROCEDURE2             Procedure,
+  IN       UINTN                         CpuNumber,
+  IN       UINTN                         TimeoutInMicroseconds,
+  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
+  IN OUT   MM_COMPLETION                 *Token,
+  IN OUT   EFI_STATUS                    *CPUStatus
+);
+
+/**
+  This service allows the caller to invoke a procedure on all running application processors (AP)
+  except the caller. This function uses an optional token parameter to support blocking and
+  nonblocking modes. If the token is passed into the call, the function will operate in a non-blocking
+  fashion and the caller can check for completion with CheckOnProcedure or WaitForProcedure.
+
+  It is not necessary for the implementation to run the procedure on every processor on the platform.
+  Processors that are powered down in such a way that they cannot respond to interrupts, may be
+  excluded from the broadcast.
+
+
+  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
+  @param[in]     Procedure              A pointer to the code stream to be run on the APs that have
+                                        entered MM. Type EFI_AP_PROCEDURE is defined below in related
+                                        definitions.
+  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for the APs to finish
+                                        execution of Procedure, either for blocking or non-blocking mode.
+                                        Zero means infinity. If the timeout expires before all APs return
+                                        from Procedure, then Procedure on the failed APs is terminated. If
+                                        the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
+                                        If the timeout expires in non-blocking mode, the timeout determined
+                                        can be through CheckOnProcedure or WaitForProcedure.
+                                        Note that timeout support is optional. Whether an implementation
+                                        supports this feature can be determined via the Attributes data
+                                        member.
+  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
+                                        that is run by the AP. It is an optional common mailbox
+                                        between APs and the caller to share information.
+  @param[in,out] Token                  This is parameter is broken into two components:
+                                        1.Token->Completion is an optional parameter that allows the
+                                        caller to execute the procedure in a blocking or non-blocking
+                                        fashion. If it is NULL the call is blocking, and the call will
+                                        not return until the AP has completed the procedure. If the
+                                        token is not NULL, the call will return immediately. The caller
+                                        can check whether the procedure has completed with
+                                        CheckOnProcedure or WaitForProcedure.
+                                        2.Token->Status The implementation updates the address pointed
+                                        at by this variable with the status code returned by Procedure
+                                        when it completes execution on the target AP, or with EFI_TIMEOUT
+                                        if the Procedure fails to complete within the optional timeout.
+                                        The implementation will update this variable with EFI_NOT_READY
+                                        prior to starting Procedure on the target AP
+  @param[in,out] CPUStatus              This optional pointer may be used to get the individual status
+                                        returned by every AP that participated in the broadcast. This
+                                        parameter if used provides the base address of an array to hold
+                                        the EFI_STATUS value of each AP in the system. The size of the
+                                        array can be ascertained by the GetNumberOfProcessors function.
+                                        As mentioned above, the broadcast may not include every processor
+                                        in the system. Some implementations may exclude processors that
+                                        have been powered down in such a way that they are not responsive
+                                        to interrupts. Additionally the broadcast excludes the processor
+                                        which is making the BroadcastProcedure call. For every excluded
+                                        processor, the array entry must contain a value of EFI_NOT_STARTED
+
+  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
+                                        execution on the APs. In the non-blocking case this indicates that
+                                        the procedure has been successfully scheduled for execution on the
+                                        APs.
+  @retval EFI_INVALID_PARAMETER         Procedure or Token is NULL.
+  @retval EFI_NOT_READY                 If a target AP is busy executing another procedure.
+  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before all enabled APs have
+                                        finished.
+  @retval EFI_ALREADY_STARTED           Before the AP procedure associated with the Token is finished, the
+                                        same Token cannot be used to dispatch or broadcast another procedure.
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MM_BROADCAST_PROCEDURE) (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       EFI_AP_PROCEDURE2             Procedure,
+  IN       UINTN                         TimeoutInMicroseconds,
+  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
+  IN OUT   MM_COMPLETION                 *Token,
+  IN OUT   EFI_STATUS                    *CPUStatus
+);
+
+
+/**
+  This service allows the caller to set a startup procedure that will be executed when an AP powers
+  up from a state where core configuration and context is lost. The procedure is execution has the
+  following properties:
+  1. The procedure executes before the processor is handed over to the operating system.
+  2. All processors execute the same startup procedure.
+  3. The procedure may run in parallel with other procedures invoked through the functions in this
+  protocol, or with processors that are executing an MM handler or running in the operating system.
+
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Procedure            A pointer to the code stream to be run on the designated target AP
+                                       of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
+                                       with the related definitions of
+                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
+                                       If caller may pass a value of NULL to deregister any existing
+                                       startup procedure.
+  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of parameters to the code that is
+                                       run by the AP. It is an optional common mailbox between APs and
+                                       the caller to share information
+
+  @retval EFI_SUCCESS                  The Procedure has been set successfully.
+  @retval EFI_INVALID_PARAMETER        The Procedure is NULL.
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MM_SET_STARTUP_PROCEDURE) (
+  IN CONST EFI_MM_MP_PROTOCOL *This,
+  IN       EFI_AP_PROCEDURE   Procedure,
+  IN OUT   VOID               *ProcedureArguments OPTIONAL
+);
+
+/**
+  When non-blocking execution of a procedure on an AP is invoked with DispatchProcedure,
+  via the use of a token, this function can be used to check for completion of the procedure on the AP.
+  The function takes the token that was passed into the DispatchProcedure call. If the procedure
+  is complete, and therefore it is now possible to run another procedure on the same AP, this function
+  returns EFI_SUCESS. In this case the status returned by the procedure that executed on the AP is
+  returned in the token's Status field. If the procedure has not yet completed, then this function
+  returns EFI_NOT_READY.
+
+  When a non-blocking execution of a procedure is invoked with BroadcastProcedure, via the
+  use of a token, this function can be used to check for completion of the procedure on all the
+  broadcast APs. The function takes the token that was passed into the BroadcastProcedure
+  call. If the procedure is complete on all broadcast APs this function returns EFI_SUCESS. In this
+  case the Status field in the token passed into the function reflects the overall result of the
+  invocation, which may be EFI_SUCCESS, if all executions succeeded, or the first observed failure.
+  If the procedure has not yet completed on the broadcast APs, the function returns
+  EFI_NOT_READY.
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Token                This parameter describes the token that was passed into
+                                       DispatchProcedure or BroadcastProcedure.
+
+  @retval EFI_SUCCESS                  Procedure has completed.
+  @retval EFI_NOT_READY                The Procedure has not completed.
+  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is NULL
+  @retval EFI_NOT_FOUND                Token is not currently in use for a non-blocking call
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_CHECK_FOR_PROCEDURE) (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       MM_COMPLETION                 Token
+);
+
+/**
+  When a non-blocking execution of a procedure on an AP is invoked via DispatchProcedure,
+  this function will block the caller until the remote procedure has completed on the designated AP.
+  The non-blocking procedure invocation is identified by the Token parameter, which must match the
+  token that used when DispatchProcedure was called. Upon completion the status returned by
+  the procedure that executed on the AP is used to update the token's Status field.
+
+  When a non-blocking execution of a procedure on an AP is invoked via BroadcastProcedure
+  this function will block the caller until the remote procedure has completed on all of the APs that
+  entered MM. The non-blocking procedure invocation is identified by the Token parameter, which
+  must match the token that used when BroadcastProcedure was called. Upon completion the
+  overall status returned by the procedures that executed on the broadcast AP is used to update the
+  token's Status field. The overall status may be EFI_SUCCESS, if all executions succeeded, or the
+  first observed failure.
+
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Token                This parameter describes the token that was passed into
+                                       DispatchProcedure or BroadcastProcedure.
+
+  @retval EFI_SUCCESS                  Procedure has completed.
+  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is NULL
+  @retval EFI_NOT_FOUND                Token is not currently in use for a non-blocking call
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_WAIT_FOR_PROCEDURE) (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       MM_COMPLETION                 Token
+);
+
+
+
+///
+/// The MM MP protocol provides a set of functions to allow execution of procedures on processors that
+/// have entered MM.
+///
+struct _EFI_MM_MP_PROTOCOL {
+  UINT32                            Revision;
+  UINT32                            Attributes;
+  EFI_MM_GET_NUMBER_OF_PROCESSORS   GetNumberOfProcessors;
+  EFI_MM_DISPATCH_PROCEDURE         DispatchProcedure;
+  EFI_MM_BROADCAST_PROCEDURE        BroadcastProcedure;
+  EFI_MM_SET_STARTUP_PROCEDURE      SetStartupProcedure;
+  EFI_CHECK_FOR_PROCEDURE           CheckForProcedure;
+  EFI_WAIT_FOR_PROCEDURE            WaitForProcedure;
+};
+
+extern EFI_GUID gEfiMmMpProtocolGuid;
+
+#endif
diff --git a/MdePkg/MdePkg.dec b/MdePkg/MdePkg.dec
index 6c563375ee..b382efd578 100644
--- a/MdePkg/MdePkg.dec
+++ b/MdePkg/MdePkg.dec
@@ -1167,6 +1167,9 @@
   # Protocols defined in PI 1.5.
   #
 
+  ## Include/Protocol/MmMp.h
+  gEfiMmMpProtocolGuid = { 0x5d5450d7, 0x990c, 0x4180, { 0xa8, 0x3, 0x8e, 0x63, 0xf0, 0x60, 0x83, 0x7 }}
+
   ## Include/Protocol/MmEndOfDxe.h
   gEfiMmEndOfDxeProtocolGuid = { 0x24e70042, 0xd5c5, 0x4260, { 0x8c, 0x39, 0xa, 0xd3, 0xaa, 0x32, 0xe9, 0x3d }}
 
-- 
2.21.0.windows.1