[Patch v5 0/2] Enable new MM MP protocol

[Patch v5 0/2] Enable new MM MP protocol

[Dong, Eric]

V5:
1. Some small enhancement.

V4 changes:
1. Use link list to save the used tokens.

V3 changes:
1. Fix Token clean up too early caused CheckProcedure return error.

V1 changes:
RFC: https://bugzilla.tianocore.org/show_bug.cgi?id=1937

PI spec added a new protocol named MM MP protocol. This protocol allows for
better remote queuing of execution of procedures on an AP.
This extends the existing procedures to allow:
1. A function to be called in blocking and non-blocking manner explicitly 
2. Allow broadcasts.
3. Allow execution of a procedure when a processor powers up.

This patch serial enable this new protocol.

Cc: Ray Ni <ray.ni@intel.com>
Cc: Laszlo Ersek <lersek@redhat.com>


Eric Dong (2):
  MdePkg: Add new MM MP Protocol definition.
  V5 changes: 1. some small enhancement.

 MdePkg/Include/Pi/PiMultiPhase.h             |  16 +
 MdePkg/Include/Protocol/MmMp.h               | 333 +++++++++++
 MdePkg/MdePkg.dec                            |   3 +
 UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c        | 570 ++++++++++++++++++-
 UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c   |  18 +
 UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h   | 193 ++++++-
 UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf |   3 +
 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c            | 344 +++++++++++
 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h            | 286 ++++++++++
 9 files changed, 1743 insertions(+), 23 deletions(-)
 create mode 100644 MdePkg/Include/Protocol/MmMp.h
 create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c
 create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h

-- 
2.21.0.windows.1

[Patch v5 1/2] MdePkg: Add new MM MP Protocol definition.

[Dong, Eric]

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=1937

EFI MM MP Protocol is defined in the PI 1.5 specification.

The MM MP protocol provides a set of functions to allow execution of
procedures on processors that have entered MM. This protocol has the
following properties:
1. The caller can invoke execution of a procedure on a processor, other
than the caller, that has also entered MM. Supports blocking and
non-blocking modes of operation.
2. The caller can invoke a procedure on multiple processors. Supports
blocking and non-blocking modes of operation.

Cc: Ray Ni <ray.ni@intel.com>
Cc: Laszlo Ersek <lersek@redhat.com>
Signed-off-by: Eric Dong <eric.dong@intel.com>
Reviewed-by: Ray Ni <ray.ni@intel.com>
---
 MdePkg/Include/Pi/PiMultiPhase.h |  16 ++
 MdePkg/Include/Protocol/MmMp.h   | 333 +++++++++++++++++++++++++++++++
 MdePkg/MdePkg.dec                |   3 +
 3 files changed, 352 insertions(+)
 create mode 100644 MdePkg/Include/Protocol/MmMp.h

diff --git a/MdePkg/Include/Pi/PiMultiPhase.h b/MdePkg/Include/Pi/PiMultiPhase.h
index eb12527767..a5056799e1 100644
--- a/MdePkg/Include/Pi/PiMultiPhase.h
+++ b/MdePkg/Include/Pi/PiMultiPhase.h
@@ -176,4 +176,20 @@ VOID
   IN OUT VOID  *Buffer
   );
 
+/**
+  The function prototype for invoking a function on an Application Processor.
+
+  This definition is used by the UEFI MM MP Serices Protocol.
+
+  @param[in] ProcedureArgument    The pointer to private data buffer.
+
+  @retval EFI_SUCCESS             Excutive the procedure successfully
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_AP_PROCEDURE2)(
+  IN VOID  *ProcedureArgument
+);
+
 #endif
diff --git a/MdePkg/Include/Protocol/MmMp.h b/MdePkg/Include/Protocol/MmMp.h
new file mode 100644
index 0000000000..beace1386c
--- /dev/null
+++ b/MdePkg/Include/Protocol/MmMp.h
@@ -0,0 +1,333 @@
+/** @file
+  EFI MM MP Protocol is defined in the PI 1.5 specification.
+
+  The MM MP protocol provides a set of functions to allow execution of procedures on processors that
+  have entered MM. This protocol has the following properties:
+  1. The caller can only invoke execution of a procedure on a processor, other than the caller, that
+     has also entered MM.
+  2. It is possible to invoke a procedure on multiple processors. Supports blocking and non-blocking
+     modes of operation.
+
+  Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+
+**/
+
+#ifndef _MM_MP_H_
+#define _MM_MP_H_
+
+#include <Pi/PiMmCis.h>
+
+#define EFI_MM_MP_PROTOCOL_GUID \
+  { \
+    0x5d5450d7, 0x990c, 0x4180, {0xa8, 0x3, 0x8e, 0x63, 0xf0, 0x60, 0x83, 0x7  }  \
+  }
+
+//
+// Revision definition.
+//
+#define EFI_MM_MP_PROTOCOL_REVISION    0x00
+
+//
+// Attribute flags
+//
+#define EFI_MM_MP_TIMEOUT_SUPPORTED    0x01
+
+//
+// Completion token
+//
+typedef VOID* MM_COMPLETION;
+
+typedef struct {
+  MM_COMPLETION  Completion;
+  EFI_STATUS     Status;
+} MM_DISPATCH_COMPLETION_TOKEN;
+
+typedef struct _EFI_MM_MP_PROTOCOL  EFI_MM_MP_PROTOCOL;
+
+/**
+  Service to retrieves the number of logical processor in the platform.
+
+  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
+  @param[out] NumberOfProcessors  Pointer to the total number of logical processors in the system,
+                                  including the BSP and all APs.
+
+  @retval EFI_SUCCESS             The number of processors was retrieved successfully
+  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MM_GET_NUMBER_OF_PROCESSORS) (
+  IN CONST EFI_MM_MP_PROTOCOL  *This,
+  OUT      UINTN               *NumberOfProcessors
+);
+
+
+/**
+  This service allows the caller to invoke a procedure one of the application processors (AP). This
+  function uses an optional token parameter to support blocking and non-blocking modes. If the token
+  is passed into the call, the function will operate in a non-blocking fashion and the caller can
+  check for completion with CheckOnProcedure or WaitForProcedure.
+
+  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
+  @param[in]     Procedure              A pointer to the procedure to be run on the designated target
+                                        AP of the system. Type EFI_AP_PROCEDURE2 is defined below in
+                                        related definitions.
+  @param[in]     CpuNumber              The zero-based index of the processor number of the target
+                                        AP, on which the code stream is supposed to run. If the number
+                                        points to the calling processor then it will not run the
+                                        supplied code.
+  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for this AP to
+                                        finish execution of Procedure, either for blocking or
+                                        non-blocking mode. Zero means infinity. If the timeout
+                                        expires before this AP returns from Procedure, then Procedure
+                                        on the AP is terminated. If the timeout expires in blocking
+                                        mode, the call returns EFI_TIMEOUT. If the timeout expires
+                                        in non-blocking mode, the timeout determined can be through
+                                        CheckOnProcedure or WaitForProcedure.
+                                        Note that timeout support is optional. Whether an
+                                        implementation supports this feature, can be determined via
+                                        the Attributes data member.
+  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
+                                        that is run by the AP. It is an optional common mailbox
+                                        between APs and the caller to share information.
+  @param[in,out] Token                  This is parameter is broken into two components:
+                                        1.Token->Completion is an optional parameter that allows the
+                                        caller to execute the procedure in a blocking or non-blocking
+                                        fashion. If it is NULL the call is blocking, and the call will
+                                        not return until the AP has completed the procedure. If the
+                                        token is not NULL, the call will return immediately. The caller
+                                        can check whether the procedure has completed with
+                                        CheckOnProcedure or WaitForProcedure.
+                                        2.Token->Status The implementation updates the address pointed
+                                        at by this variable with the status code returned by Procedure
+                                        when it completes execution on the target AP, or with EFI_TIMEOUT
+                                        if the Procedure fails to complete within the optional timeout.
+                                        The implementation will update this variable with EFI_NOT_READY
+                                        prior to starting Procedure on the target AP.
+  @param[in,out] CPUStatus              This optional pointer may be used to get the status code returned
+                                        by Procedure when it completes execution on the target AP, or with
+                                        EFI_TIMEOUT if the Procedure fails to complete within the optional
+                                        timeout. The implementation will update this variable with
+                                        EFI_NOT_READY prior to starting Procedure on the target AP.
+
+  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
+                                        execution on the target AP.
+                                        In the non-blocking case this indicates that the procedure has
+                                        been successfully scheduled for execution on the target AP.
+  @retval EFI_INVALID_PARAMETER         The input arguments are out of range. Either the target AP is the
+                                        caller of the function, or the Procedure or Token is NULL
+  @retval EFI_NOT_READY                 If the target AP is busy executing another procedure
+  @retval EFI_ALREADY_STARTED           Token is already in use for another procedure
+  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before the specified AP
+                                        has finished
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MM_DISPATCH_PROCEDURE) (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       EFI_AP_PROCEDURE2             Procedure,
+  IN       UINTN                         CpuNumber,
+  IN       UINTN                         TimeoutInMicroseconds,
+  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
+  IN OUT   MM_COMPLETION                 *Token,
+  IN OUT   EFI_STATUS                    *CPUStatus
+);
+
+/**
+  This service allows the caller to invoke a procedure on all running application processors (AP)
+  except the caller. This function uses an optional token parameter to support blocking and
+  nonblocking modes. If the token is passed into the call, the function will operate in a non-blocking
+  fashion and the caller can check for completion with CheckOnProcedure or WaitForProcedure.
+
+  It is not necessary for the implementation to run the procedure on every processor on the platform.
+  Processors that are powered down in such a way that they cannot respond to interrupts, may be
+  excluded from the broadcast.
+
+
+  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
+  @param[in]     Procedure              A pointer to the code stream to be run on the APs that have
+                                        entered MM. Type EFI_AP_PROCEDURE is defined below in related
+                                        definitions.
+  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for the APs to finish
+                                        execution of Procedure, either for blocking or non-blocking mode.
+                                        Zero means infinity. If the timeout expires before all APs return
+                                        from Procedure, then Procedure on the failed APs is terminated. If
+                                        the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
+                                        If the timeout expires in non-blocking mode, the timeout determined
+                                        can be through CheckOnProcedure or WaitForProcedure.
+                                        Note that timeout support is optional. Whether an implementation
+                                        supports this feature can be determined via the Attributes data
+                                        member.
+  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
+                                        that is run by the AP. It is an optional common mailbox
+                                        between APs and the caller to share information.
+  @param[in,out] Token                  This is parameter is broken into two components:
+                                        1.Token->Completion is an optional parameter that allows the
+                                        caller to execute the procedure in a blocking or non-blocking
+                                        fashion. If it is NULL the call is blocking, and the call will
+                                        not return until the AP has completed the procedure. If the
+                                        token is not NULL, the call will return immediately. The caller
+                                        can check whether the procedure has completed with
+                                        CheckOnProcedure or WaitForProcedure.
+                                        2.Token->Status The implementation updates the address pointed
+                                        at by this variable with the status code returned by Procedure
+                                        when it completes execution on the target AP, or with EFI_TIMEOUT
+                                        if the Procedure fails to complete within the optional timeout.
+                                        The implementation will update this variable with EFI_NOT_READY
+                                        prior to starting Procedure on the target AP
+  @param[in,out] CPUStatus              This optional pointer may be used to get the individual status
+                                        returned by every AP that participated in the broadcast. This
+                                        parameter if used provides the base address of an array to hold
+                                        the EFI_STATUS value of each AP in the system. The size of the
+                                        array can be ascertained by the GetNumberOfProcessors function.
+                                        As mentioned above, the broadcast may not include every processor
+                                        in the system. Some implementations may exclude processors that
+                                        have been powered down in such a way that they are not responsive
+                                        to interrupts. Additionally the broadcast excludes the processor
+                                        which is making the BroadcastProcedure call. For every excluded
+                                        processor, the array entry must contain a value of EFI_NOT_STARTED
+
+  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
+                                        execution on the APs. In the non-blocking case this indicates that
+                                        the procedure has been successfully scheduled for execution on the
+                                        APs.
+  @retval EFI_INVALID_PARAMETER         Procedure or Token is NULL.
+  @retval EFI_NOT_READY                 If a target AP is busy executing another procedure.
+  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before all enabled APs have
+                                        finished.
+  @retval EFI_ALREADY_STARTED           Before the AP procedure associated with the Token is finished, the
+                                        same Token cannot be used to dispatch or broadcast another procedure.
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MM_BROADCAST_PROCEDURE) (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       EFI_AP_PROCEDURE2             Procedure,
+  IN       UINTN                         TimeoutInMicroseconds,
+  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
+  IN OUT   MM_COMPLETION                 *Token,
+  IN OUT   EFI_STATUS                    *CPUStatus
+);
+
+
+/**
+  This service allows the caller to set a startup procedure that will be executed when an AP powers
+  up from a state where core configuration and context is lost. The procedure is execution has the
+  following properties:
+  1. The procedure executes before the processor is handed over to the operating system.
+  2. All processors execute the same startup procedure.
+  3. The procedure may run in parallel with other procedures invoked through the functions in this
+  protocol, or with processors that are executing an MM handler or running in the operating system.
+
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Procedure            A pointer to the code stream to be run on the designated target AP
+                                       of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
+                                       with the related definitions of
+                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
+                                       If caller may pass a value of NULL to deregister any existing
+                                       startup procedure.
+  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of parameters to the code that is
+                                       run by the AP. It is an optional common mailbox between APs and
+                                       the caller to share information
+
+  @retval EFI_SUCCESS                  The Procedure has been set successfully.
+  @retval EFI_INVALID_PARAMETER        The Procedure is NULL.
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MM_SET_STARTUP_PROCEDURE) (
+  IN CONST EFI_MM_MP_PROTOCOL *This,
+  IN       EFI_AP_PROCEDURE   Procedure,
+  IN OUT   VOID               *ProcedureArguments OPTIONAL
+);
+
+/**
+  When non-blocking execution of a procedure on an AP is invoked with DispatchProcedure,
+  via the use of a token, this function can be used to check for completion of the procedure on the AP.
+  The function takes the token that was passed into the DispatchProcedure call. If the procedure
+  is complete, and therefore it is now possible to run another procedure on the same AP, this function
+  returns EFI_SUCESS. In this case the status returned by the procedure that executed on the AP is
+  returned in the token's Status field. If the procedure has not yet completed, then this function
+  returns EFI_NOT_READY.
+
+  When a non-blocking execution of a procedure is invoked with BroadcastProcedure, via the
+  use of a token, this function can be used to check for completion of the procedure on all the
+  broadcast APs. The function takes the token that was passed into the BroadcastProcedure
+  call. If the procedure is complete on all broadcast APs this function returns EFI_SUCESS. In this
+  case the Status field in the token passed into the function reflects the overall result of the
+  invocation, which may be EFI_SUCCESS, if all executions succeeded, or the first observed failure.
+  If the procedure has not yet completed on the broadcast APs, the function returns
+  EFI_NOT_READY.
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Token                This parameter describes the token that was passed into
+                                       DispatchProcedure or BroadcastProcedure.
+
+  @retval EFI_SUCCESS                  Procedure has completed.
+  @retval EFI_NOT_READY                The Procedure has not completed.
+  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is NULL
+  @retval EFI_NOT_FOUND                Token is not currently in use for a non-blocking call
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_CHECK_FOR_PROCEDURE) (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       MM_COMPLETION                 Token
+);
+
+/**
+  When a non-blocking execution of a procedure on an AP is invoked via DispatchProcedure,
+  this function will block the caller until the remote procedure has completed on the designated AP.
+  The non-blocking procedure invocation is identified by the Token parameter, which must match the
+  token that used when DispatchProcedure was called. Upon completion the status returned by
+  the procedure that executed on the AP is used to update the token's Status field.
+
+  When a non-blocking execution of a procedure on an AP is invoked via BroadcastProcedure
+  this function will block the caller until the remote procedure has completed on all of the APs that
+  entered MM. The non-blocking procedure invocation is identified by the Token parameter, which
+  must match the token that used when BroadcastProcedure was called. Upon completion the
+  overall status returned by the procedures that executed on the broadcast AP is used to update the
+  token's Status field. The overall status may be EFI_SUCCESS, if all executions succeeded, or the
+  first observed failure.
+
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Token                This parameter describes the token that was passed into
+                                       DispatchProcedure or BroadcastProcedure.
+
+  @retval EFI_SUCCESS                  Procedure has completed.
+  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is NULL
+  @retval EFI_NOT_FOUND                Token is not currently in use for a non-blocking call
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_WAIT_FOR_PROCEDURE) (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       MM_COMPLETION                 Token
+);
+
+
+
+///
+/// The MM MP protocol provides a set of functions to allow execution of procedures on processors that
+/// have entered MM.
+///
+struct _EFI_MM_MP_PROTOCOL {
+  UINT32                            Revision;
+  UINT32                            Attributes;
+  EFI_MM_GET_NUMBER_OF_PROCESSORS   GetNumberOfProcessors;
+  EFI_MM_DISPATCH_PROCEDURE         DispatchProcedure;
+  EFI_MM_BROADCAST_PROCEDURE        BroadcastProcedure;
+  EFI_MM_SET_STARTUP_PROCEDURE      SetStartupProcedure;
+  EFI_CHECK_FOR_PROCEDURE           CheckForProcedure;
+  EFI_WAIT_FOR_PROCEDURE            WaitForProcedure;
+};
+
+extern EFI_GUID gEfiMmMpProtocolGuid;
+
+#endif
diff --git a/MdePkg/MdePkg.dec b/MdePkg/MdePkg.dec
index 6c563375ee..b382efd578 100644
--- a/MdePkg/MdePkg.dec
+++ b/MdePkg/MdePkg.dec
@@ -1167,6 +1167,9 @@
   # Protocols defined in PI 1.5.
   #
 
+  ## Include/Protocol/MmMp.h
+  gEfiMmMpProtocolGuid = { 0x5d5450d7, 0x990c, 0x4180, { 0xa8, 0x3, 0x8e, 0x63, 0xf0, 0x60, 0x83, 0x7 }}
+
   ## Include/Protocol/MmEndOfDxe.h
   gEfiMmEndOfDxeProtocolGuid = { 0x24e70042, 0xd5c5, 0x4260, { 0x8c, 0x39, 0xa, 0xd3, 0xaa, 0x32, 0xe9, 0x3d }}
 
-- 
2.21.0.windows.1

Re: [Patch v5 1/2] MdePkg: Add new MM MP Protocol definition.

[Ni, Ray]

+ MdePkg maintainers

We need to follow the open source community process to include MdePkg
maintainers for this change review.

Laszlo,
You are in the Cc list and had given Regression-Tested-By for V3 patch series.
It would be great if you could give patch owner a hint to avoid breaking the process
though I understand it's not responsibility of the people in the CC list. 😊

Thanks,
Ray


> -----Original Message-----
> From: Dong, Eric
> Sent: Wednesday, July 10, 2019 3:56 PM
> To: devel@edk2.groups.io
> Cc: Ni, Ray <ray.ni@intel.com>; Laszlo Ersek <lersek@redhat.com>
> Subject: [Patch v5 1/2] MdePkg: Add new MM MP Protocol definition.
> 
> REF: https://bugzilla.tianocore.org/show_bug.cgi?id=1937
> 
> EFI MM MP Protocol is defined in the PI 1.5 specification.
> 
> The MM MP protocol provides a set of functions to allow execution of
> procedures on processors that have entered MM. This protocol has the
> following properties:
> 1. The caller can invoke execution of a procedure on a processor, other than
> the caller, that has also entered MM. Supports blocking and non-blocking
> modes of operation.
> 2. The caller can invoke a procedure on multiple processors. Supports
> blocking and non-blocking modes of operation.
> 
> Cc: Ray Ni <ray.ni@intel.com>
> Cc: Laszlo Ersek <lersek@redhat.com>
> Signed-off-by: Eric Dong <eric.dong@intel.com>
> Reviewed-by: Ray Ni <ray.ni@intel.com>
> ---
>  MdePkg/Include/Pi/PiMultiPhase.h |  16 ++
>  MdePkg/Include/Protocol/MmMp.h   | 333
> +++++++++++++++++++++++++++++++
>  MdePkg/MdePkg.dec                |   3 +
>  3 files changed, 352 insertions(+)
>  create mode 100644 MdePkg/Include/Protocol/MmMp.h
> 
> diff --git a/MdePkg/Include/Pi/PiMultiPhase.h
> b/MdePkg/Include/Pi/PiMultiPhase.h
> index eb12527767..a5056799e1 100644
> --- a/MdePkg/Include/Pi/PiMultiPhase.h
> +++ b/MdePkg/Include/Pi/PiMultiPhase.h
> @@ -176,4 +176,20 @@ VOID
>    IN OUT VOID  *Buffer
>    );
> 
> +/**
> +  The function prototype for invoking a function on an Application Processor.
> +
> +  This definition is used by the UEFI MM MP Serices Protocol.
> +
> +  @param[in] ProcedureArgument    The pointer to private data buffer.
> +
> +  @retval EFI_SUCCESS             Excutive the procedure successfully
> +
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_AP_PROCEDURE2)(
> +  IN VOID  *ProcedureArgument
> +);
> +
>  #endif
> diff --git a/MdePkg/Include/Protocol/MmMp.h
> b/MdePkg/Include/Protocol/MmMp.h new file mode 100644 index
> 0000000000..beace1386c
> --- /dev/null
> +++ b/MdePkg/Include/Protocol/MmMp.h
> @@ -0,0 +1,333 @@
> +/** @file
> +  EFI MM MP Protocol is defined in the PI 1.5 specification.
> +
> +  The MM MP protocol provides a set of functions to allow execution of
> + procedures on processors that  have entered MM. This protocol has the
> following properties:
> +  1. The caller can only invoke execution of a procedure on a processor,
> other than the caller, that
> +     has also entered MM.
> +  2. It is possible to invoke a procedure on multiple processors. Supports
> blocking and non-blocking
> +     modes of operation.
> +
> +  Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
> +  SPDX-License-Identifier: BSD-2-Clause-Patent
> +
> +**/
> +
> +#ifndef _MM_MP_H_
> +#define _MM_MP_H_
> +
> +#include <Pi/PiMmCis.h>
> +
> +#define EFI_MM_MP_PROTOCOL_GUID \
> +  { \
> +    0x5d5450d7, 0x990c, 0x4180, {0xa8, 0x3, 0x8e, 0x63, 0xf0, 0x60,
> +0x83, 0x7  }  \
> +  }
> +
> +//
> +// Revision definition.
> +//
> +#define EFI_MM_MP_PROTOCOL_REVISION    0x00
> +
> +//
> +// Attribute flags
> +//
> +#define EFI_MM_MP_TIMEOUT_SUPPORTED    0x01
> +
> +//
> +// Completion token
> +//
> +typedef VOID* MM_COMPLETION;
> +
> +typedef struct {
> +  MM_COMPLETION  Completion;
> +  EFI_STATUS     Status;
> +} MM_DISPATCH_COMPLETION_TOKEN;
> +
> +typedef struct _EFI_MM_MP_PROTOCOL  EFI_MM_MP_PROTOCOL;
> +
> +/**
> +  Service to retrieves the number of logical processor in the platform.
> +
> +  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
> +  @param[out] NumberOfProcessors  Pointer to the total number of logical
> processors in the system,
> +                                  including the BSP and all APs.
> +
> +  @retval EFI_SUCCESS             The number of processors was retrieved
> successfully
> +  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_MM_GET_NUMBER_OF_PROCESSORS) (
> +  IN CONST EFI_MM_MP_PROTOCOL  *This,
> +  OUT      UINTN               *NumberOfProcessors
> +);
> +
> +
> +/**
> +  This service allows the caller to invoke a procedure one of the
> +application processors (AP). This
> +  function uses an optional token parameter to support blocking and
> +non-blocking modes. If the token
> +  is passed into the call, the function will operate in a non-blocking
> +fashion and the caller can
> +  check for completion with CheckOnProcedure or WaitForProcedure.
> +
> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]     Procedure              A pointer to the procedure to be run on
> the designated target
> +                                        AP of the system. Type EFI_AP_PROCEDURE2 is defined
> below in
> +                                        related definitions.
> +  @param[in]     CpuNumber              The zero-based index of the processor
> number of the target
> +                                        AP, on which the code stream is supposed to run. If the
> number
> +                                        points to the calling processor then it will not run the
> +                                        supplied code.
> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
> microseconds for this AP to
> +                                        finish execution of Procedure, either for blocking or
> +                                        non-blocking mode. Zero means infinity. If the timeout
> +                                        expires before this AP returns from Procedure, then
> Procedure
> +                                        on the AP is terminated. If the timeout expires in
> blocking
> +                                        mode, the call returns EFI_TIMEOUT. If the timeout
> expires
> +                                        in non-blocking mode, the timeout determined can be
> through
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        Note that timeout support is optional. Whether an
> +                                        implementation supports this feature, can be
> determined via
> +                                        the Attributes data member.
> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
> parameters to the code
> +                                        that is run by the AP. It is an optional common mailbox
> +                                        between APs and the caller to share information.
> +  @param[in,out] Token                  This is parameter is broken into two
> components:
> +                                        1.Token->Completion is an optional parameter that
> allows the
> +                                        caller to execute the procedure in a blocking or non-
> blocking
> +                                        fashion. If it is NULL the call is blocking, and the call will
> +                                        not return until the AP has completed the procedure. If
> the
> +                                        token is not NULL, the call will return immediately. The
> caller
> +                                        can check whether the procedure has completed with
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        2.Token->Status The implementation updates the
> address pointed
> +                                        at by this variable with the status code returned by
> Procedure
> +                                        when it completes execution on the target AP, or with
> EFI_TIMEOUT
> +                                        if the Procedure fails to complete within the optional
> timeout.
> +                                        The implementation will update this variable with
> EFI_NOT_READY
> +                                        prior to starting Procedure on the target AP.
> +  @param[in,out] CPUStatus              This optional pointer may be used to get
> the status code returned
> +                                        by Procedure when it completes execution on the
> target AP, or with
> +                                        EFI_TIMEOUT if the Procedure fails to complete within
> the optional
> +                                        timeout. The implementation will update this variable
> with
> +                                        EFI_NOT_READY prior to starting Procedure on the
> target AP.
> +
> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
> Procedure has completed
> +                                        execution on the target AP.
> +                                        In the non-blocking case this indicates that the
> procedure has
> +                                        been successfully scheduled for execution on the target
> AP.
> +  @retval EFI_INVALID_PARAMETER         The input arguments are out of
> range. Either the target AP is the
> +                                        caller of the function, or the Procedure or Token is NULL
> +  @retval EFI_NOT_READY                 If the target AP is busy executing another
> procedure
> +  @retval EFI_ALREADY_STARTED           Token is already in use for another
> procedure
> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
> before the specified AP
> +                                        has finished **/ typedef
> +EFI_STATUS (EFIAPI *EFI_MM_DISPATCH_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         CpuNumber,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   MM_COMPLETION                 *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +);
> +
> +/**
> +  This service allows the caller to invoke a procedure on all running
> +application processors (AP)
> +  except the caller. This function uses an optional token parameter to
> +support blocking and
> +  nonblocking modes. If the token is passed into the call, the function
> +will operate in a non-blocking
> +  fashion and the caller can check for completion with CheckOnProcedure or
> WaitForProcedure.
> +
> +  It is not necessary for the implementation to run the procedure on every
> processor on the platform.
> +  Processors that are powered down in such a way that they cannot
> + respond to interrupts, may be  excluded from the broadcast.
> +
> +
> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]     Procedure              A pointer to the code stream to be run on
> the APs that have
> +                                        entered MM. Type EFI_AP_PROCEDURE is defined
> below in related
> +                                        definitions.
> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
> microseconds for the APs to finish
> +                                        execution of Procedure, either for blocking or non-
> blocking mode.
> +                                        Zero means infinity. If the timeout expires before all
> APs return
> +                                        from Procedure, then Procedure on the failed APs is
> terminated. If
> +                                        the timeout expires in blocking mode, the call returns
> EFI_TIMEOUT.
> +                                        If the timeout expires in non-blocking mode, the
> timeout determined
> +                                        can be through CheckOnProcedure or
> WaitForProcedure.
> +                                        Note that timeout support is optional. Whether an
> implementation
> +                                        supports this feature can be determined via the
> Attributes data
> +                                        member.
> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
> parameters to the code
> +                                        that is run by the AP. It is an optional common mailbox
> +                                        between APs and the caller to share information.
> +  @param[in,out] Token                  This is parameter is broken into two
> components:
> +                                        1.Token->Completion is an optional parameter that
> allows the
> +                                        caller to execute the procedure in a blocking or non-
> blocking
> +                                        fashion. If it is NULL the call is blocking, and the call will
> +                                        not return until the AP has completed the procedure. If
> the
> +                                        token is not NULL, the call will return immediately. The
> caller
> +                                        can check whether the procedure has completed with
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        2.Token->Status The implementation updates the
> address pointed
> +                                        at by this variable with the status code returned by
> Procedure
> +                                        when it completes execution on the target AP, or with
> EFI_TIMEOUT
> +                                        if the Procedure fails to complete within the optional
> timeout.
> +                                        The implementation will update this variable with
> EFI_NOT_READY
> +                                        prior to starting Procedure on the target AP
> +  @param[in,out] CPUStatus              This optional pointer may be used to get
> the individual status
> +                                        returned by every AP that participated in the broadcast.
> This
> +                                        parameter if used provides the base address of an array
> to hold
> +                                        the EFI_STATUS value of each AP in the system. The size
> of the
> +                                        array can be ascertained by the
> GetNumberOfProcessors function.
> +                                        As mentioned above, the broadcast may not include
> every processor
> +                                        in the system. Some implementations may exclude
> processors that
> +                                        have been powered down in such a way that they are
> not responsive
> +                                        to interrupts. Additionally the broadcast excludes the
> processor
> +                                        which is making the BroadcastProcedure call. For every
> excluded
> +                                        processor, the array entry must
> + contain a value of EFI_NOT_STARTED
> +
> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
> Procedure has completed
> +                                        execution on the APs. In the non-blocking case this
> indicates that
> +                                        the procedure has been successfully scheduled for
> execution on the
> +                                        APs.
> +  @retval EFI_INVALID_PARAMETER         Procedure or Token is NULL.
> +  @retval EFI_NOT_READY                 If a target AP is busy executing another
> procedure.
> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
> before all enabled APs have
> +                                        finished.
> +  @retval EFI_ALREADY_STARTED           Before the AP procedure associated
> with the Token is finished, the
> +                                        same Token cannot be used to dispatch or broadcast
> another procedure.
> +
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_MM_BROADCAST_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   MM_COMPLETION                 *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +);
> +
> +
> +/**
> +  This service allows the caller to set a startup procedure that will
> +be executed when an AP powers
> +  up from a state where core configuration and context is lost. The
> +procedure is execution has the
> +  following properties:
> +  1. The procedure executes before the processor is handed over to the
> operating system.
> +  2. All processors execute the same startup procedure.
> +  3. The procedure may run in parallel with other procedures invoked
> +through the functions in this
> +  protocol, or with processors that are executing an MM handler or running
> in the operating system.
> +
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Procedure            A pointer to the code stream to be run on
> the designated target AP
> +                                       of the system. Type EFI_AP_PROCEDURE is defined
> below in Volume 2
> +                                       with the related definitions of
> +                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
> +                                       If caller may pass a value of NULL to deregister any
> existing
> +                                       startup procedure.
> +  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of
> parameters to the code that is
> +                                       run by the AP. It is an optional common mailbox
> between APs and
> +                                       the caller to share information
> +
> +  @retval EFI_SUCCESS                  The Procedure has been set successfully.
> +  @retval EFI_INVALID_PARAMETER        The Procedure is NULL.
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_MM_SET_STARTUP_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL *This,
> +  IN       EFI_AP_PROCEDURE   Procedure,
> +  IN OUT   VOID               *ProcedureArguments OPTIONAL
> +);
> +
> +/**
> +  When non-blocking execution of a procedure on an AP is invoked with
> +DispatchProcedure,
> +  via the use of a token, this function can be used to check for completion of
> the procedure on the AP.
> +  The function takes the token that was passed into the
> +DispatchProcedure call. If the procedure
> +  is complete, and therefore it is now possible to run another
> +procedure on the same AP, this function
> +  returns EFI_SUCESS. In this case the status returned by the procedure
> +that executed on the AP is
> +  returned in the token's Status field. If the procedure has not yet
> +completed, then this function
> +  returns EFI_NOT_READY.
> +
> +  When a non-blocking execution of a procedure is invoked with
> + BroadcastProcedure, via the  use of a token, this function can be used
> + to check for completion of the procedure on all the  broadcast APs.
> + The function takes the token that was passed into the
> + BroadcastProcedure  call. If the procedure is complete on all
> + broadcast APs this function returns EFI_SUCESS. In this  case the Status
> field in the token passed into the function reflects the overall result of the
> invocation, which may be EFI_SUCCESS, if all executions succeeded, or the
> first observed failure.
> +  If the procedure has not yet completed on the broadcast APs, the
> + function returns  EFI_NOT_READY.
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Token                This parameter describes the token that was
> passed into
> +                                       DispatchProcedure or BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS                  Procedure has completed.
> +  @retval EFI_NOT_READY                The Procedure has not completed.
> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is
> NULL
> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-
> blocking call
> +
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_CHECK_FOR_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       MM_COMPLETION                 Token
> +);
> +
> +/**
> +  When a non-blocking execution of a procedure on an AP is invoked via
> +DispatchProcedure,
> +  this function will block the caller until the remote procedure has completed
> on the designated AP.
> +  The non-blocking procedure invocation is identified by the Token
> +parameter, which must match the
> +  token that used when DispatchProcedure was called. Upon completion
> +the status returned by
> +  the procedure that executed on the AP is used to update the token's
> Status field.
> +
> +  When a non-blocking execution of a procedure on an AP is invoked via
> + BroadcastProcedure  this function will block the caller until the
> + remote procedure has completed on all of the APs that  entered MM. The
> + non-blocking procedure invocation is identified by the Token
> + parameter, which  must match the token that used when
> + BroadcastProcedure was called. Upon completion the  overall status
> + returned by the procedures that executed on the broadcast AP is used to
> update the  token's Status field. The overall status may be EFI_SUCCESS, if all
> executions succeeded, or the  first observed failure.
> +
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Token                This parameter describes the token that was
> passed into
> +                                       DispatchProcedure or BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS                  Procedure has completed.
> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is
> NULL
> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-
> blocking call
> +
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_WAIT_FOR_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       MM_COMPLETION                 Token
> +);
> +
> +
> +
> +///
> +/// The MM MP protocol provides a set of functions to allow execution
> +of procedures on processors that /// have entered MM.
> +///
> +struct _EFI_MM_MP_PROTOCOL {
> +  UINT32                            Revision;
> +  UINT32                            Attributes;
> +  EFI_MM_GET_NUMBER_OF_PROCESSORS   GetNumberOfProcessors;
> +  EFI_MM_DISPATCH_PROCEDURE         DispatchProcedure;
> +  EFI_MM_BROADCAST_PROCEDURE        BroadcastProcedure;
> +  EFI_MM_SET_STARTUP_PROCEDURE      SetStartupProcedure;
> +  EFI_CHECK_FOR_PROCEDURE           CheckForProcedure;
> +  EFI_WAIT_FOR_PROCEDURE            WaitForProcedure;
> +};
> +
> +extern EFI_GUID gEfiMmMpProtocolGuid;
> +
> +#endif
> diff --git a/MdePkg/MdePkg.dec b/MdePkg/MdePkg.dec index
> 6c563375ee..b382efd578 100644
> --- a/MdePkg/MdePkg.dec
> +++ b/MdePkg/MdePkg.dec
> @@ -1167,6 +1167,9 @@
>    # Protocols defined in PI 1.5.
>    #
> 
> +  ## Include/Protocol/MmMp.h
> +  gEfiMmMpProtocolGuid = { 0x5d5450d7, 0x990c, 0x4180, { 0xa8, 0x3,
> + 0x8e, 0x63, 0xf0, 0x60, 0x83, 0x7 }}
> +
>    ## Include/Protocol/MmEndOfDxe.h
>    gEfiMmEndOfDxeProtocolGuid = { 0x24e70042, 0xd5c5, 0x4260, { 0x8c, 0x39,
> 0xa, 0xd3, 0xaa, 0x32, 0xe9, 0x3d }}
> 
> --
> 2.21.0.windows.1

Re: [edk2-devel] [Patch v5 1/2] MdePkg: Add new MM MP Protocol definition.

[Laszlo Ersek]

Hi Ray,

On 07/11/19 08:42, Ni, Ray wrote:
> + MdePkg maintainers
> 
> We need to follow the open source community process to include MdePkg
> maintainers for this change review.
> 
> Laszlo,
> You are in the Cc list and had given Regression-Tested-By for V3 patch series.
> It would be great if you could give patch owner a hint to avoid breaking the process
> though I understand it's not responsibility of the people in the CC list. 😊

I'm unaware of any changes to the process.

In the commit message of the patch, Cc: tags should be included for all
"M" folks under XxxPkg, and also for all "R" folks who are listed with
interest in the given area (possibly the entirety of XxxPkg). The
contributor's git config should not include a
"sendemail.suppresscc=bodycc" setting; otherwise those Cc: tags will not
actually copy the intended reviewers.

For a given XxxPkg patch, at least one Acked-by or Required-by is needed
from an "M" person listed under XxxPkg in "Maintainers.txt", on the
mailing list, before the patch can be pushed.

Reviews and comments from others are welcome for the patch. The patch
should not be pushed as long as reasonable questions are still open for
it (regardless of origin). A patch should generally spend at least 24
hours on the list before it is pushed, even if an "M" approval arrives
earlier than that (so that others have a chance at noticing the patch
and at commenting). Once no questions seem to remain open, the patch has
spent the minimum time on the list, and there is public "M" approval,
the patch can be pushed.

An "M" person can defer to another "M" or "R" person, if he/she chooses
so. This is best done formally, i.e., wait until the deferred-to person
approves the patch, and then the first "M" person can give their own
Acked-by. (Standing for "I haven't reviewed in detail, but given the
circumstances / other reviews, I'm OK with this patch".)

When the patch is pushed, all feedback tags, given on the list for the
patch (regardless of origin), should be picked up, by whoever is pushing
the patch.

Let's consider the current state of this series, as an example.

- The first patch (for MdePkg) has an "M" review, from Liming.
- The second patch (for UefiCpuPkg) has an "M" review, from you.
- The series has an open question (regarding regression testing), from
myself.
- The series was posted more than 24 hours ago (as far as I can see).

So once I (hopefully) report back with an R-t-b for the series, the
series becomes pushable (assuming noone opens another question
meanwhile). In that case, any "M" person is welcome to push the patch
series -- in the current case, it will likely be Eric. When Eric
prepares for pushing the series, he's expected to pick up
- Liming's R-b for the first patch,
- my (to be posted) R-t-b for the first patch,
- your R-b for the second patch,
- my (to be posted) R-t-b for the second patch.

(I will likely send my R-t-b in response to the blurb (the "v5 0/2"
email), which means that the feedback tag applies to every patch in the
series.)

This "tag pickup" procedure is difficult to get right when using a MUA
that does not support a "threaded" view, and it is easy to get right
with a MUA that does. When I'm about to push a series, I tend to perform
one full scan over the thread:

- Whenever I reach patch-level feedback, I run "git rebase -i" for the
full series, just to pick up that one tag. (A single "reword" action
among the "pick"s.)

- Whenever I reach series-level feedback (grouped under the blurb), I
run "git rebase -i" for the full series again, and I apply the tag to
every patch in the series (all actions are set to "reword").

Thanks
Laszlo

>> -----Original Message-----
>> From: Dong, Eric
>> Sent: Wednesday, July 10, 2019 3:56 PM
>> To: devel@edk2.groups.io
>> Cc: Ni, Ray <ray.ni@intel.com>; Laszlo Ersek <lersek@redhat.com>
>> Subject: [Patch v5 1/2] MdePkg: Add new MM MP Protocol definition.
>>
>> REF: https://bugzilla.tianocore.org/show_bug.cgi?id=1937
>>
>> EFI MM MP Protocol is defined in the PI 1.5 specification.
>>
>> The MM MP protocol provides a set of functions to allow execution of
>> procedures on processors that have entered MM. This protocol has the
>> following properties:
>> 1. The caller can invoke execution of a procedure on a processor, other than
>> the caller, that has also entered MM. Supports blocking and non-blocking
>> modes of operation.
>> 2. The caller can invoke a procedure on multiple processors. Supports
>> blocking and non-blocking modes of operation.
>>
>> Cc: Ray Ni <ray.ni@intel.com>
>> Cc: Laszlo Ersek <lersek@redhat.com>
>> Signed-off-by: Eric Dong <eric.dong@intel.com>
>> Reviewed-by: Ray Ni <ray.ni@intel.com>
>> ---
>>  MdePkg/Include/Pi/PiMultiPhase.h |  16 ++
>>  MdePkg/Include/Protocol/MmMp.h   | 333
>> +++++++++++++++++++++++++++++++
>>  MdePkg/MdePkg.dec                |   3 +
>>  3 files changed, 352 insertions(+)
>>  create mode 100644 MdePkg/Include/Protocol/MmMp.h
>>
>> diff --git a/MdePkg/Include/Pi/PiMultiPhase.h
>> b/MdePkg/Include/Pi/PiMultiPhase.h
>> index eb12527767..a5056799e1 100644
>> --- a/MdePkg/Include/Pi/PiMultiPhase.h
>> +++ b/MdePkg/Include/Pi/PiMultiPhase.h
>> @@ -176,4 +176,20 @@ VOID
>>    IN OUT VOID  *Buffer
>>    );
>>
>> +/**
>> +  The function prototype for invoking a function on an Application Processor.
>> +
>> +  This definition is used by the UEFI MM MP Serices Protocol.
>> +
>> +  @param[in] ProcedureArgument    The pointer to private data buffer.
>> +
>> +  @retval EFI_SUCCESS             Excutive the procedure successfully
>> +
>> +**/
>> +typedef
>> +EFI_STATUS
>> +(EFIAPI *EFI_AP_PROCEDURE2)(
>> +  IN VOID  *ProcedureArgument
>> +);
>> +
>>  #endif
>> diff --git a/MdePkg/Include/Protocol/MmMp.h
>> b/MdePkg/Include/Protocol/MmMp.h new file mode 100644 index
>> 0000000000..beace1386c
>> --- /dev/null
>> +++ b/MdePkg/Include/Protocol/MmMp.h
>> @@ -0,0 +1,333 @@
>> +/** @file
>> +  EFI MM MP Protocol is defined in the PI 1.5 specification.
>> +
>> +  The MM MP protocol provides a set of functions to allow execution of
>> + procedures on processors that  have entered MM. This protocol has the
>> following properties:
>> +  1. The caller can only invoke execution of a procedure on a processor,
>> other than the caller, that
>> +     has also entered MM.
>> +  2. It is possible to invoke a procedure on multiple processors. Supports
>> blocking and non-blocking
>> +     modes of operation.
>> +
>> +  Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
>> +  SPDX-License-Identifier: BSD-2-Clause-Patent
>> +
>> +**/
>> +
>> +#ifndef _MM_MP_H_
>> +#define _MM_MP_H_
>> +
>> +#include <Pi/PiMmCis.h>
>> +
>> +#define EFI_MM_MP_PROTOCOL_GUID \
>> +  { \
>> +    0x5d5450d7, 0x990c, 0x4180, {0xa8, 0x3, 0x8e, 0x63, 0xf0, 0x60,
>> +0x83, 0x7  }  \
>> +  }
>> +
>> +//
>> +// Revision definition.
>> +//
>> +#define EFI_MM_MP_PROTOCOL_REVISION    0x00
>> +
>> +//
>> +// Attribute flags
>> +//
>> +#define EFI_MM_MP_TIMEOUT_SUPPORTED    0x01
>> +
>> +//
>> +// Completion token
>> +//
>> +typedef VOID* MM_COMPLETION;
>> +
>> +typedef struct {
>> +  MM_COMPLETION  Completion;
>> +  EFI_STATUS     Status;
>> +} MM_DISPATCH_COMPLETION_TOKEN;
>> +
>> +typedef struct _EFI_MM_MP_PROTOCOL  EFI_MM_MP_PROTOCOL;
>> +
>> +/**
>> +  Service to retrieves the number of logical processor in the platform.
>> +
>> +  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
>> +  @param[out] NumberOfProcessors  Pointer to the total number of logical
>> processors in the system,
>> +                                  including the BSP and all APs.
>> +
>> +  @retval EFI_SUCCESS             The number of processors was retrieved
>> successfully
>> +  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
>> +**/
>> +typedef
>> +EFI_STATUS
>> +(EFIAPI *EFI_MM_GET_NUMBER_OF_PROCESSORS) (
>> +  IN CONST EFI_MM_MP_PROTOCOL  *This,
>> +  OUT      UINTN               *NumberOfProcessors
>> +);
>> +
>> +
>> +/**
>> +  This service allows the caller to invoke a procedure one of the
>> +application processors (AP). This
>> +  function uses an optional token parameter to support blocking and
>> +non-blocking modes. If the token
>> +  is passed into the call, the function will operate in a non-blocking
>> +fashion and the caller can
>> +  check for completion with CheckOnProcedure or WaitForProcedure.
>> +
>> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
>> +  @param[in]     Procedure              A pointer to the procedure to be run on
>> the designated target
>> +                                        AP of the system. Type EFI_AP_PROCEDURE2 is defined
>> below in
>> +                                        related definitions.
>> +  @param[in]     CpuNumber              The zero-based index of the processor
>> number of the target
>> +                                        AP, on which the code stream is supposed to run. If the
>> number
>> +                                        points to the calling processor then it will not run the
>> +                                        supplied code.
>> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
>> microseconds for this AP to
>> +                                        finish execution of Procedure, either for blocking or
>> +                                        non-blocking mode. Zero means infinity. If the timeout
>> +                                        expires before this AP returns from Procedure, then
>> Procedure
>> +                                        on the AP is terminated. If the timeout expires in
>> blocking
>> +                                        mode, the call returns EFI_TIMEOUT. If the timeout
>> expires
>> +                                        in non-blocking mode, the timeout determined can be
>> through
>> +                                        CheckOnProcedure or WaitForProcedure.
>> +                                        Note that timeout support is optional. Whether an
>> +                                        implementation supports this feature, can be
>> determined via
>> +                                        the Attributes data member.
>> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
>> parameters to the code
>> +                                        that is run by the AP. It is an optional common mailbox
>> +                                        between APs and the caller to share information.
>> +  @param[in,out] Token                  This is parameter is broken into two
>> components:
>> +                                        1.Token->Completion is an optional parameter that
>> allows the
>> +                                        caller to execute the procedure in a blocking or non-
>> blocking
>> +                                        fashion. If it is NULL the call is blocking, and the call will
>> +                                        not return until the AP has completed the procedure. If
>> the
>> +                                        token is not NULL, the call will return immediately. The
>> caller
>> +                                        can check whether the procedure has completed with
>> +                                        CheckOnProcedure or WaitForProcedure.
>> +                                        2.Token->Status The implementation updates the
>> address pointed
>> +                                        at by this variable with the status code returned by
>> Procedure
>> +                                        when it completes execution on the target AP, or with
>> EFI_TIMEOUT
>> +                                        if the Procedure fails to complete within the optional
>> timeout.
>> +                                        The implementation will update this variable with
>> EFI_NOT_READY
>> +                                        prior to starting Procedure on the target AP.
>> +  @param[in,out] CPUStatus              This optional pointer may be used to get
>> the status code returned
>> +                                        by Procedure when it completes execution on the
>> target AP, or with
>> +                                        EFI_TIMEOUT if the Procedure fails to complete within
>> the optional
>> +                                        timeout. The implementation will update this variable
>> with
>> +                                        EFI_NOT_READY prior to starting Procedure on the
>> target AP.
>> +
>> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
>> Procedure has completed
>> +                                        execution on the target AP.
>> +                                        In the non-blocking case this indicates that the
>> procedure has
>> +                                        been successfully scheduled for execution on the target
>> AP.
>> +  @retval EFI_INVALID_PARAMETER         The input arguments are out of
>> range. Either the target AP is the
>> +                                        caller of the function, or the Procedure or Token is NULL
>> +  @retval EFI_NOT_READY                 If the target AP is busy executing another
>> procedure
>> +  @retval EFI_ALREADY_STARTED           Token is already in use for another
>> procedure
>> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
>> before the specified AP
>> +                                        has finished **/ typedef
>> +EFI_STATUS (EFIAPI *EFI_MM_DISPATCH_PROCEDURE) (
>> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
>> +  IN       EFI_AP_PROCEDURE2             Procedure,
>> +  IN       UINTN                         CpuNumber,
>> +  IN       UINTN                         TimeoutInMicroseconds,
>> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
>> +  IN OUT   MM_COMPLETION                 *Token,
>> +  IN OUT   EFI_STATUS                    *CPUStatus
>> +);
>> +
>> +/**
>> +  This service allows the caller to invoke a procedure on all running
>> +application processors (AP)
>> +  except the caller. This function uses an optional token parameter to
>> +support blocking and
>> +  nonblocking modes. If the token is passed into the call, the function
>> +will operate in a non-blocking
>> +  fashion and the caller can check for completion with CheckOnProcedure or
>> WaitForProcedure.
>> +
>> +  It is not necessary for the implementation to run the procedure on every
>> processor on the platform.
>> +  Processors that are powered down in such a way that they cannot
>> + respond to interrupts, may be  excluded from the broadcast.
>> +
>> +
>> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
>> +  @param[in]     Procedure              A pointer to the code stream to be run on
>> the APs that have
>> +                                        entered MM. Type EFI_AP_PROCEDURE is defined
>> below in related
>> +                                        definitions.
>> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
>> microseconds for the APs to finish
>> +                                        execution of Procedure, either for blocking or non-
>> blocking mode.
>> +                                        Zero means infinity. If the timeout expires before all
>> APs return
>> +                                        from Procedure, then Procedure on the failed APs is
>> terminated. If
>> +                                        the timeout expires in blocking mode, the call returns
>> EFI_TIMEOUT.
>> +                                        If the timeout expires in non-blocking mode, the
>> timeout determined
>> +                                        can be through CheckOnProcedure or
>> WaitForProcedure.
>> +                                        Note that timeout support is optional. Whether an
>> implementation
>> +                                        supports this feature can be determined via the
>> Attributes data
>> +                                        member.
>> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
>> parameters to the code
>> +                                        that is run by the AP. It is an optional common mailbox
>> +                                        between APs and the caller to share information.
>> +  @param[in,out] Token                  This is parameter is broken into two
>> components:
>> +                                        1.Token->Completion is an optional parameter that
>> allows the
>> +                                        caller to execute the procedure in a blocking or non-
>> blocking
>> +                                        fashion. If it is NULL the call is blocking, and the call will
>> +                                        not return until the AP has completed the procedure. If
>> the
>> +                                        token is not NULL, the call will return immediately. The
>> caller
>> +                                        can check whether the procedure has completed with
>> +                                        CheckOnProcedure or WaitForProcedure.
>> +                                        2.Token->Status The implementation updates the
>> address pointed
>> +                                        at by this variable with the status code returned by
>> Procedure
>> +                                        when it completes execution on the target AP, or with
>> EFI_TIMEOUT
>> +                                        if the Procedure fails to complete within the optional
>> timeout.
>> +                                        The implementation will update this variable with
>> EFI_NOT_READY
>> +                                        prior to starting Procedure on the target AP
>> +  @param[in,out] CPUStatus              This optional pointer may be used to get
>> the individual status
>> +                                        returned by every AP that participated in the broadcast.
>> This
>> +                                        parameter if used provides the base address of an array
>> to hold
>> +                                        the EFI_STATUS value of each AP in the system. The size
>> of the
>> +                                        array can be ascertained by the
>> GetNumberOfProcessors function.
>> +                                        As mentioned above, the broadcast may not include
>> every processor
>> +                                        in the system. Some implementations may exclude
>> processors that
>> +                                        have been powered down in such a way that they are
>> not responsive
>> +                                        to interrupts. Additionally the broadcast excludes the
>> processor
>> +                                        which is making the BroadcastProcedure call. For every
>> excluded
>> +                                        processor, the array entry must
>> + contain a value of EFI_NOT_STARTED
>> +
>> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
>> Procedure has completed
>> +                                        execution on the APs. In the non-blocking case this
>> indicates that
>> +                                        the procedure has been successfully scheduled for
>> execution on the
>> +                                        APs.
>> +  @retval EFI_INVALID_PARAMETER         Procedure or Token is NULL.
>> +  @retval EFI_NOT_READY                 If a target AP is busy executing another
>> procedure.
>> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
>> before all enabled APs have
>> +                                        finished.
>> +  @retval EFI_ALREADY_STARTED           Before the AP procedure associated
>> with the Token is finished, the
>> +                                        same Token cannot be used to dispatch or broadcast
>> another procedure.
>> +
>> +**/
>> +typedef
>> +EFI_STATUS
>> +(EFIAPI *EFI_MM_BROADCAST_PROCEDURE) (
>> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
>> +  IN       EFI_AP_PROCEDURE2             Procedure,
>> +  IN       UINTN                         TimeoutInMicroseconds,
>> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
>> +  IN OUT   MM_COMPLETION                 *Token,
>> +  IN OUT   EFI_STATUS                    *CPUStatus
>> +);
>> +
>> +
>> +/**
>> +  This service allows the caller to set a startup procedure that will
>> +be executed when an AP powers
>> +  up from a state where core configuration and context is lost. The
>> +procedure is execution has the
>> +  following properties:
>> +  1. The procedure executes before the processor is handed over to the
>> operating system.
>> +  2. All processors execute the same startup procedure.
>> +  3. The procedure may run in parallel with other procedures invoked
>> +through the functions in this
>> +  protocol, or with processors that are executing an MM handler or running
>> in the operating system.
>> +
>> +
>> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
>> +  @param[in]      Procedure            A pointer to the code stream to be run on
>> the designated target AP
>> +                                       of the system. Type EFI_AP_PROCEDURE is defined
>> below in Volume 2
>> +                                       with the related definitions of
>> +                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
>> +                                       If caller may pass a value of NULL to deregister any
>> existing
>> +                                       startup procedure.
>> +  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of
>> parameters to the code that is
>> +                                       run by the AP. It is an optional common mailbox
>> between APs and
>> +                                       the caller to share information
>> +
>> +  @retval EFI_SUCCESS                  The Procedure has been set successfully.
>> +  @retval EFI_INVALID_PARAMETER        The Procedure is NULL.
>> +**/
>> +typedef
>> +EFI_STATUS
>> +(EFIAPI *EFI_MM_SET_STARTUP_PROCEDURE) (
>> +  IN CONST EFI_MM_MP_PROTOCOL *This,
>> +  IN       EFI_AP_PROCEDURE   Procedure,
>> +  IN OUT   VOID               *ProcedureArguments OPTIONAL
>> +);
>> +
>> +/**
>> +  When non-blocking execution of a procedure on an AP is invoked with
>> +DispatchProcedure,
>> +  via the use of a token, this function can be used to check for completion of
>> the procedure on the AP.
>> +  The function takes the token that was passed into the
>> +DispatchProcedure call. If the procedure
>> +  is complete, and therefore it is now possible to run another
>> +procedure on the same AP, this function
>> +  returns EFI_SUCESS. In this case the status returned by the procedure
>> +that executed on the AP is
>> +  returned in the token's Status field. If the procedure has not yet
>> +completed, then this function
>> +  returns EFI_NOT_READY.
>> +
>> +  When a non-blocking execution of a procedure is invoked with
>> + BroadcastProcedure, via the  use of a token, this function can be used
>> + to check for completion of the procedure on all the  broadcast APs.
>> + The function takes the token that was passed into the
>> + BroadcastProcedure  call. If the procedure is complete on all
>> + broadcast APs this function returns EFI_SUCESS. In this  case the Status
>> field in the token passed into the function reflects the overall result of the
>> invocation, which may be EFI_SUCCESS, if all executions succeeded, or the
>> first observed failure.
>> +  If the procedure has not yet completed on the broadcast APs, the
>> + function returns  EFI_NOT_READY.
>> +
>> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
>> +  @param[in]      Token                This parameter describes the token that was
>> passed into
>> +                                       DispatchProcedure or BroadcastProcedure.
>> +
>> +  @retval EFI_SUCCESS                  Procedure has completed.
>> +  @retval EFI_NOT_READY                The Procedure has not completed.
>> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is
>> NULL
>> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-
>> blocking call
>> +
>> +**/
>> +typedef
>> +EFI_STATUS
>> +(EFIAPI *EFI_CHECK_FOR_PROCEDURE) (
>> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
>> +  IN       MM_COMPLETION                 Token
>> +);
>> +
>> +/**
>> +  When a non-blocking execution of a procedure on an AP is invoked via
>> +DispatchProcedure,
>> +  this function will block the caller until the remote procedure has completed
>> on the designated AP.
>> +  The non-blocking procedure invocation is identified by the Token
>> +parameter, which must match the
>> +  token that used when DispatchProcedure was called. Upon completion
>> +the status returned by
>> +  the procedure that executed on the AP is used to update the token's
>> Status field.
>> +
>> +  When a non-blocking execution of a procedure on an AP is invoked via
>> + BroadcastProcedure  this function will block the caller until the
>> + remote procedure has completed on all of the APs that  entered MM. The
>> + non-blocking procedure invocation is identified by the Token
>> + parameter, which  must match the token that used when
>> + BroadcastProcedure was called. Upon completion the  overall status
>> + returned by the procedures that executed on the broadcast AP is used to
>> update the  token's Status field. The overall status may be EFI_SUCCESS, if all
>> executions succeeded, or the  first observed failure.
>> +
>> +
>> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
>> +  @param[in]      Token                This parameter describes the token that was
>> passed into
>> +                                       DispatchProcedure or BroadcastProcedure.
>> +
>> +  @retval EFI_SUCCESS                  Procedure has completed.
>> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is
>> NULL
>> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-
>> blocking call
>> +
>> +**/
>> +typedef
>> +EFI_STATUS
>> +(EFIAPI *EFI_WAIT_FOR_PROCEDURE) (
>> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
>> +  IN       MM_COMPLETION                 Token
>> +);
>> +
>> +
>> +
>> +///
>> +/// The MM MP protocol provides a set of functions to allow execution
>> +of procedures on processors that /// have entered MM.
>> +///
>> +struct _EFI_MM_MP_PROTOCOL {
>> +  UINT32                            Revision;
>> +  UINT32                            Attributes;
>> +  EFI_MM_GET_NUMBER_OF_PROCESSORS   GetNumberOfProcessors;
>> +  EFI_MM_DISPATCH_PROCEDURE         DispatchProcedure;
>> +  EFI_MM_BROADCAST_PROCEDURE        BroadcastProcedure;
>> +  EFI_MM_SET_STARTUP_PROCEDURE      SetStartupProcedure;
>> +  EFI_CHECK_FOR_PROCEDURE           CheckForProcedure;
>> +  EFI_WAIT_FOR_PROCEDURE            WaitForProcedure;
>> +};
>> +
>> +extern EFI_GUID gEfiMmMpProtocolGuid;
>> +
>> +#endif
>> diff --git a/MdePkg/MdePkg.dec b/MdePkg/MdePkg.dec index
>> 6c563375ee..b382efd578 100644
>> --- a/MdePkg/MdePkg.dec
>> +++ b/MdePkg/MdePkg.dec
>> @@ -1167,6 +1167,9 @@
>>    # Protocols defined in PI 1.5.
>>    #
>>
>> +  ## Include/Protocol/MmMp.h
>> +  gEfiMmMpProtocolGuid = { 0x5d5450d7, 0x990c, 0x4180, { 0xa8, 0x3,
>> + 0x8e, 0x63, 0xf0, 0x60, 0x83, 0x7 }}
>> +
>>    ## Include/Protocol/MmEndOfDxe.h
>>    gEfiMmEndOfDxeProtocolGuid = { 0x24e70042, 0xd5c5, 0x4260, { 0x8c, 0x39,
>> 0xa, 0xd3, 0xaa, 0x32, 0xe9, 0x3d }}
>>
>> --
>> 2.21.0.windows.1
> 
> 
> 
> 

[Patch v5 2/2] UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP Protocol

[Dong, Eric]

V5 changes:
1. some small enhancement.

v4 changes:
1. Use link list to save the token info.

v3 changes:
1. Fix Token clean up too early caused CheckProcedure return error.

v2 changes:
1. Remove some duplicated global variables.
2. Enhance token design to support multiple task trig for different APs at the same time.

V1 changes:
REF: https://bugzilla.tianocore.org/show_bug.cgi?id=1937

Add MM Mp Protocol in PiSmmCpuDxeSmm driver.

Cc: Ray Ni <ray.ni@intel.com>
Cc: Laszlo Ersek <lersek@redhat.com>
Signed-off-by: Eric Dong <eric.dong@intel.com>
---
 UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c        | 570 ++++++++++++++++++-
 UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c   |  18 +
 UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h   | 193 ++++++-
 UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf |   3 +
 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c            | 344 +++++++++++
 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h            | 286 ++++++++++
 6 files changed, 1391 insertions(+), 23 deletions(-)
 create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c
 create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h

diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
index 64fb4d6344..f09e2738c3 100644
--- a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
@@ -140,7 +140,7 @@ ReleaseAllAPs (
 
   BspIndex = mSmmMpSyncData->BspIndex;
   for (Index = mMaxNumberOfCpus; Index-- > 0;) {
-    if (Index != BspIndex && *(mSmmMpSyncData->CpuData[Index].Present)) {
+    if (IsPresentAp (Index)) {
       ReleaseSemaphore (mSmmMpSyncData->CpuData[Index].Run);
     }
   }
@@ -347,6 +347,165 @@ ReplaceOSMtrrs (
   MtrrSetAllMtrrs (&gSmiMtrrs);
 }
 
+/**
+  Wheck whether task has been finished by all APs.
+
+  @param       BlockMode   Whether did it in block mode or non-block mode.
+
+  @retval      TRUE        Task has been finished by all APs.
+  @retval      FALSE       Task not has been finished by all APs.
+
+**/
+BOOLEAN
+WaitForAllAPsNotBusy (
+  IN BOOLEAN                        BlockMode
+  )
+{
+  UINTN                             Index;
+
+  for (Index = mMaxNumberOfCpus; Index-- > 0;) {
+    //
+    // Ignore BSP and APs which not call in SMM.
+    //
+    if (!IsPresentAp(Index)) {
+      continue;
+    }
+
+    if (BlockMode) {
+      AcquireSpinLock(mSmmMpSyncData->CpuData[Index].Busy);
+      ReleaseSpinLock(mSmmMpSyncData->CpuData[Index].Busy);
+    } else {
+      if (AcquireSpinLockOrFail (mSmmMpSyncData->CpuData[Index].Busy)) {
+        ReleaseSpinLock(mSmmMpSyncData->CpuData[Index].Busy);
+      } else {
+        return FALSE;
+      }
+    }
+  }
+
+  return TRUE;
+}
+
+/**
+  Check whether it is an present AP.
+
+  @param   CpuIndex      The AP index which calls this function.
+
+  @retval  TRUE           It's a present AP.
+  @retval  TRUE           This is not an AP or it is not present.
+
+**/
+BOOLEAN
+IsPresentAp (
+  IN UINTN        CpuIndex
+  )
+{
+  return ((CpuIndex != gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu) &&
+    *(mSmmMpSyncData->CpuData[CpuIndex].Present));
+}
+
+/**
+  Check whether execute in single AP or all APs.
+
+  Compare two Tokens used by different APs to know whether in StartAllAps call.
+
+  Whether is an valid AP base on AP's Present flag.
+
+  @retval  TRUE      IN StartAllAps call.
+  @retval  FALSE     Not in StartAllAps call.
+
+**/
+BOOLEAN
+InStartAllApsCall (
+  VOID
+  )
+{
+  UINTN      ApIndex;
+  UINTN      ApIndex2;
+
+  for (ApIndex = mMaxNumberOfCpus; ApIndex-- > 0;) {
+    if (IsPresentAp (ApIndex) && (mSmmMpSyncData->CpuData[ApIndex].Token != NULL)) {
+      for (ApIndex2 = ApIndex; ApIndex2-- > 0;) {
+        if (IsPresentAp (ApIndex2) && (mSmmMpSyncData->CpuData[ApIndex2].Token != NULL)) {
+          return mSmmMpSyncData->CpuData[ApIndex2].Token == mSmmMpSyncData->CpuData[ApIndex].Token;
+        }
+      }
+    }
+  }
+
+  return FALSE;
+}
+
+/**
+  Clean up the status flags used during executing the procedure.
+
+  @param   CpuIndex      The AP index which calls this function.
+
+**/
+VOID
+ReleaseToken (
+  IN UINTN                  CpuIndex
+  )
+{
+  UINTN                             Index;
+  BOOLEAN                           Released;
+
+  if (InStartAllApsCall ()) {
+    //
+    // In Start All APs mode, make sure all APs have finished task.
+    //
+    if (WaitForAllAPsNotBusy (FALSE)) {
+      //
+      // Clean the flags update in the function call.
+      //
+      Released = FALSE;
+      for (Index = mMaxNumberOfCpus; Index-- > 0;) {
+        //
+        // Only In SMM APs need to be clean up.
+        //
+        if (mSmmMpSyncData->CpuData[Index].Present && mSmmMpSyncData->CpuData[Index].Token != NULL) {
+          if (!Released) {
+            ReleaseSpinLock (mSmmMpSyncData->CpuData[Index].Token);
+            Released = TRUE;
+          }
+          mSmmMpSyncData->CpuData[Index].Token = NULL;
+        }
+      }
+    }
+  } else {
+    //
+    // In single AP mode.
+    //
+    if (mSmmMpSyncData->CpuData[CpuIndex].Token != NULL) {
+      ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Token);
+      mSmmMpSyncData->CpuData[CpuIndex].Token = NULL;
+    }
+  }
+}
+
+/**
+  Free the tokens in the maintained list.
+
+**/
+VOID
+FreeTokens (
+  VOID
+  )
+{
+  LIST_ENTRY            *Link;
+  PROCEDURE_TOKEN       *ProcToken;
+
+  while (!IsListEmpty (&gSmmCpuPrivate->TokenList)) {
+    Link = GetFirstNode (&gSmmCpuPrivate->TokenList);
+    ProcToken = PROCEDURE_TOKEN_FROM_LINK (Link);
+
+    RemoveEntryList (&ProcToken->Link);
+
+    FreePool ((VOID *)ProcToken->ProcedureToken);
+    FreePool (ProcToken);
+  }
+}
+
 /**
   SMI handler for BSP.
 
@@ -476,12 +635,7 @@ BSPHandler (
   //
   // Make sure all APs have completed their pending none-block tasks
   //
-  for (Index = mMaxNumberOfCpus; Index-- > 0;) {
-    if (Index != CpuIndex && *(mSmmMpSyncData->CpuData[Index].Present)) {
-      AcquireSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
-      ReleaseSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
-    }
-  }
+  WaitForAllAPsNotBusy (TRUE);
 
   //
   // Perform the remaining tasks
@@ -572,6 +726,11 @@ BSPHandler (
   //
   WaitForAllAPs (ApCount);
 
+  //
+  // Clean the tokens buffer.
+  //
+  FreeTokens ();
+
   //
   // Reset BspIndex to -1, meaning BSP has not been elected.
   //
@@ -604,6 +763,7 @@ APHandler (
   UINT64                            Timer;
   UINTN                             BspIndex;
   MTRR_SETTINGS                     Mtrrs;
+  EFI_STATUS                        ProcedureStatus;
 
   //
   // Timeout BSP
@@ -730,14 +890,19 @@ APHandler (
     //
     // Invoke the scheduled procedure
     //
-    (*mSmmMpSyncData->CpuData[CpuIndex].Procedure) (
-      (VOID*)mSmmMpSyncData->CpuData[CpuIndex].Parameter
-      );
+    ProcedureStatus = (*mSmmMpSyncData->CpuData[CpuIndex].Procedure) (
+                          (VOID*)mSmmMpSyncData->CpuData[CpuIndex].Parameter
+                          );
+    if (mSmmMpSyncData->CpuData[CpuIndex].Status != NULL) {
+      *mSmmMpSyncData->CpuData[CpuIndex].Status = ProcedureStatus;
+    }
 
     //
     // Release BUSY
     //
     ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
+
+    ReleaseToken (CpuIndex);
   }
 
   if (SmmCpuFeaturesNeedConfigureMtrrs()) {
@@ -906,13 +1071,124 @@ Gen4GPageTable (
   return (UINT32)(UINTN)PageTable;
 }
 
+/**
+  Checks whether the input token is the current used token.
+
+  @param[in]  Token      This parameter describes the token that was passed into DispatchProcedure or
+                         BroadcastProcedure.
+
+  @retval TRUE           The input token is the current used token.
+  @retval FALSE          The input token is not the current used token.
+**/
+BOOLEAN
+IsTokenInUse (
+  IN SPIN_LOCK           *Token
+  )
+{
+  LIST_ENTRY        *Link;
+  PROCEDURE_TOKEN   *ProcToken;
+
+  if (Token == NULL) {
+    return FALSE;
+  }
+
+  Link = GetFirstNode (&gSmmCpuPrivate->TokenList);
+  while (!IsNull (&gSmmCpuPrivate->TokenList, Link)) {
+    ProcToken = PROCEDURE_TOKEN_FROM_LINK (Link);
+
+    if (ProcToken->ProcedureToken == Token) {
+      return TRUE;
+    }
+
+    Link = GetNextNode (&gSmmCpuPrivate->TokenList, Link);
+  }
+
+  return FALSE;
+}
+
+/**
+  create token and save it to the maintain list.
+
+  @retval    return the spin lock used as token.
+
+**/
+SPIN_LOCK *
+CreateToken (
+  VOID
+  )
+{
+  PROCEDURE_TOKEN    *ProcToken;
+  SPIN_LOCK           *CpuToken;
+  UINTN               SpinLockSize;
+
+  SpinLockSize = GetSpinLockProperties ();
+  CpuToken = AllocatePool (SpinLockSize);
+  ASSERT (CpuToken != NULL);
+  InitializeSpinLock (CpuToken);
+  AcquireSpinLock (CpuToken);
+
+  ProcToken = AllocatePool (sizeof (PROCEDURE_TOKEN));
+  ASSERT (ProcToken != NULL);
+  ProcToken->Signature = PROCEDURE_TOKEN_SIGNATURE;
+  ProcToken->ProcedureToken = CpuToken;
+
+  InsertTailList (&gSmmCpuPrivate->TokenList, &ProcToken->Link);
+
+  return CpuToken;
+}
+
+/**
+  Checks status of specified AP.
+
+  This function checks whether the specified AP has finished the task assigned
+  by StartupThisAP(), and whether timeout expires.
+
+  @param[in]  Token             This parameter describes the token that was passed into DispatchProcedure or
+                                BroadcastProcedure.
+
+  @retval EFI_SUCCESS           Specified AP has finished task assigned by StartupThisAPs().
+  @retval EFI_NOT_READY         Specified AP has not finished task and timeout has not expired.
+**/
+EFI_STATUS
+IsApReady (
+  IN SPIN_LOCK          *Token
+  )
+{
+  if (AcquireSpinLockOrFail (Token)) {
+    ReleaseSpinLock (Token);
+    return EFI_SUCCESS;
+  }
+
+  return EFI_NOT_READY;
+}
+
 /**
   Schedule a procedure to run on the specified CPU.
 
   @param[in]       Procedure                The address of the procedure to run
   @param[in]       CpuIndex                 Target CPU Index
-  @param[in, out]  ProcArguments            The parameter to pass to the procedure
-  @param[in]       BlockingMode             Startup AP in blocking mode or not
+  @param[in,out]   ProcArguments            The parameter to pass to the procedure
+  @param[in]       Token                    This is an optional parameter that allows the caller to execute the
+                                            procedure in a blocking or non-blocking fashion. If it is NULL the
+                                            call is blocking, and the call will not return until the AP has
+                                            completed the procedure. If the token is not NULL, the call will
+                                            return immediately. The caller can check whether the procedure has
+                                            completed with CheckOnProcedure or WaitForProcedure.
+  @param[in]       TimeoutInMicroseconds    Indicates the time limit in microseconds for the APs to finish
+                                            execution of Procedure, either for blocking or non-blocking mode.
+                                            Zero means infinity. If the timeout expires before all APs return
+                                            from Procedure, then Procedure on the failed APs is terminated. If
+                                            the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
+                                            If the timeout expires in non-blocking mode, the timeout determined
+                                            can be through CheckOnProcedure or WaitForProcedure.
+                                            Note that timeout support is optional. Whether an implementation
+                                            supports this feature can be determined via the Attributes data
+                                            member.
+  @param[in,out]   CpuStatus                This optional pointer may be used to get the status code returned
+                                            by Procedure when it completes execution on the target AP, or with
+                                            EFI_TIMEOUT if the Procedure fails to complete within the optional
+                                            timeout. The implementation will update this variable with
+                                            EFI_NOT_READY prior to starting Procedure on the target AP.
 
   @retval EFI_INVALID_PARAMETER    CpuNumber not valid
   @retval EFI_INVALID_PARAMETER    CpuNumber specifying BSP
@@ -923,10 +1199,12 @@ Gen4GPageTable (
 **/
 EFI_STATUS
 InternalSmmStartupThisAp (
-  IN      EFI_AP_PROCEDURE          Procedure,
-  IN      UINTN                     CpuIndex,
-  IN OUT  VOID                      *ProcArguments OPTIONAL,
-  IN      BOOLEAN                   BlockingMode
+  IN      EFI_AP_PROCEDURE2              Procedure,
+  IN      UINTN                          CpuIndex,
+  IN OUT  VOID                           *ProcArguments OPTIONAL,
+  IN      MM_COMPLETION                  *Token,
+  IN      UINTN                          TimeoutInMicroseconds,
+  IN OUT  EFI_STATUS                     *CpuStatus
   )
 {
   if (CpuIndex >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus) {
@@ -952,24 +1230,190 @@ InternalSmmStartupThisAp (
     }
     return EFI_INVALID_PARAMETER;
   }
+  if ((TimeoutInMicroseconds != 0) && ((mSmmMp.Attributes & EFI_MM_MP_TIMEOUT_SUPPORTED) == 0)) {
+    return EFI_INVALID_PARAMETER;
+  }
+  if (Procedure == NULL) {
+    return EFI_INVALID_PARAMETER;
+  }
 
-  if (BlockingMode) {
+  if (Token == NULL) {
     AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
   } else {
     if (!AcquireSpinLockOrFail (mSmmMpSyncData->CpuData[CpuIndex].Busy)) {
-      DEBUG((DEBUG_ERROR, "mSmmMpSyncData->CpuData[%d].Busy\n", CpuIndex));
-      return EFI_INVALID_PARAMETER;
+      DEBUG((DEBUG_ERROR, "Can't acquire mSmmMpSyncData->CpuData[%d].Busy\n", CpuIndex));
+      return EFI_NOT_READY;
     }
+
+    *Token = (MM_COMPLETION) CreateToken ();
   }
 
   mSmmMpSyncData->CpuData[CpuIndex].Procedure = Procedure;
   mSmmMpSyncData->CpuData[CpuIndex].Parameter = ProcArguments;
+  if (Token != NULL) {
+    mSmmMpSyncData->CpuData[CpuIndex].Token   = (SPIN_LOCK *)(*Token);
+  }
+  mSmmMpSyncData->CpuData[CpuIndex].Status    = CpuStatus;
+  if (mSmmMpSyncData->CpuData[CpuIndex].Status != NULL) {
+    *mSmmMpSyncData->CpuData[CpuIndex].Status = EFI_NOT_READY;
+  }
+
   ReleaseSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);
 
-  if (BlockingMode) {
+  if (Token == NULL) {
     AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
     ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
   }
+
+  return EFI_SUCCESS;
+}
+
+/**
+  Worker function to execute a caller provided function on all enabled APs.
+
+  @param[in]     Procedure               A pointer to the function to be run on
+                                         enabled APs of the system.
+  @param[in]     TimeoutInMicroseconds   Indicates the time limit in microseconds for
+                                         APs to return from Procedure, either for
+                                         blocking or non-blocking mode.
+  @param[in,out] ProcedureArguments      The parameter passed into Procedure for
+                                         all APs.
+  @param[in,out] Token                   This is an optional parameter that allows the caller to execute the
+                                         procedure in a blocking or non-blocking fashion. If it is NULL the
+                                         call is blocking, and the call will not return until the AP has
+                                         completed the procedure. If the token is not NULL, the call will
+                                         return immediately. The caller can check whether the procedure has
+                                         completed with CheckOnProcedure or WaitForProcedure.
+  @param[in,out] CPUStatus               This optional pointer may be used to get the status code returned
+                                         by Procedure when it completes execution on the target AP, or with
+                                         EFI_TIMEOUT if the Procedure fails to complete within the optional
+                                         timeout. The implementation will update this variable with
+                                         EFI_NOT_READY prior to starting Procedure on the target AP.
+
+
+  @retval EFI_SUCCESS             In blocking mode, all APs have finished before
+                                  the timeout expired.
+  @retval EFI_SUCCESS             In non-blocking mode, function has been dispatched
+                                  to all enabled APs.
+  @retval others                  Failed to Startup all APs.
+
+**/
+EFI_STATUS
+InternalSmmStartupAllAPs (
+  IN       EFI_AP_PROCEDURE2             Procedure,
+  IN       UINTN                         TimeoutInMicroseconds,
+  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
+  IN OUT   MM_COMPLETION                 *Token,
+  IN OUT   EFI_STATUS                    *CPUStatus
+  )
+{
+  UINTN               Index;
+  UINTN               CpuCount;
+
+  if ((TimeoutInMicroseconds != 0) && ((mSmmMp.Attributes & EFI_MM_MP_TIMEOUT_SUPPORTED) == 0)) {
+    return EFI_INVALID_PARAMETER;
+  }
+  if (Procedure == NULL) {
+    return EFI_INVALID_PARAMETER;
+  }
+
+  CpuCount = 0;
+  for (Index = mMaxNumberOfCpus; Index-- > 0;) {
+    if (IsPresentAp (Index)) {
+      CpuCount ++;
+
+      if (gSmmCpuPrivate->Operation[Index] == SmmCpuRemove) {
+        return EFI_INVALID_PARAMETER;
+      }
+
+      if (!AcquireSpinLockOrFail(mSmmMpSyncData->CpuData[Index].Busy)) {
+        return EFI_NOT_READY;
+      }
+      ReleaseSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
+    }
+  }
+  if (CpuCount == 0) {
+    return EFI_NOT_STARTED;
+  }
+
+  if (Token != NULL) {
+    *Token = (MM_COMPLETION) CreateToken ();
+  }
+
+  //
+  // Make sure all BUSY should be acquired.
+  //
+  // Because former code already check mSmmMpSyncData->CpuData[***].Busy for each AP.
+  // Here code always use AcquireSpinLock instead of AcquireSpinLockOrFail for not
+  // block mode.
+  //
+  for (Index = mMaxNumberOfCpus; Index-- > 0;) {
+    if (IsPresentAp (Index)) {
+      AcquireSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
+    }
+  }
+
+  for (Index = mMaxNumberOfCpus; Index-- > 0;) {
+    if (IsPresentAp (Index)) {
+      mSmmMpSyncData->CpuData[Index].Procedure = (EFI_AP_PROCEDURE2) Procedure;
+      mSmmMpSyncData->CpuData[Index].Parameter = ProcedureArguments;
+      if (Token != NULL) {
+        mSmmMpSyncData->CpuData[Index].Token   = (SPIN_LOCK *)(*Token);
+      }
+      if (CPUStatus != NULL) {
+        mSmmMpSyncData->CpuData[Index].Status    = &CPUStatus[Index];
+        if (mSmmMpSyncData->CpuData[Index].Status != NULL) {
+          *mSmmMpSyncData->CpuData[Index].Status = EFI_NOT_READY;
+        }
+      }
+    } else {
+      //
+      // PI spec requirement:
+      // For every excluded processor, the array entry must contain a value of EFI_NOT_STARTED.
+      //
+      if (CPUStatus != NULL) {
+        CPUStatus[Index] = EFI_NOT_STARTED;
+      }
+    }
+  }
+
+  ReleaseAllAPs ();
+
+  if (Token == NULL) {
+    //
+    // Make sure all APs have completed their tasks.
+    //
+    WaitForAllAPsNotBusy (TRUE);
+  }
+
+  return EFI_SUCCESS;
+}
+
+/**
+  ISO C99 6.5.2.2 "Function calls", paragraph 9:
+  If the function is defined with a type that is not compatible with
+  the type (of the expression) pointed to by the expression that
+  denotes the called function, the behavior is undefined.
+
+  So add below wrapper function to convert between EFI_AP_PROCEDURE
+  and EFI_AP_PROCEDURE2.
+
+  Wrapper for Procedures.
+
+  @param[in]  Buffer              Pointer to PROCEDURE_WRAPPER buffer.
+
+**/
+EFI_STATUS
+EFIAPI
+ProcedureWrapper (
+  IN OUT VOID *Buffer
+  )
+{
+  PROCEDURE_WRAPPER *Wrapper;
+
+  Wrapper = Buffer;
+  Wrapper->Procedure (Wrapper->ProcedureArgument);
+
   return EFI_SUCCESS;
 }
 
@@ -995,7 +1439,15 @@ SmmBlockingStartupThisAp (
   IN OUT  VOID                      *ProcArguments OPTIONAL
   )
 {
-  return InternalSmmStartupThisAp(Procedure, CpuIndex, ProcArguments, TRUE);
+  PROCEDURE_WRAPPER  Wrapper;
+
+  Wrapper.Procedure = Procedure;
+  Wrapper.ProcedureArgument = ProcArguments;
+
+  //
+  // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
+  //
+  return InternalSmmStartupThisAp (ProcedureWrapper, CpuIndex, &Wrapper, NULL, 0, NULL);
 }
 
 /**
@@ -1020,7 +1472,22 @@ SmmStartupThisAp (
   IN OUT  VOID                      *ProcArguments OPTIONAL
   )
 {
-  return InternalSmmStartupThisAp(Procedure, CpuIndex, ProcArguments, FeaturePcdGet (PcdCpuSmmBlockStartupThisAp));
+  MM_COMPLETION               Token;
+
+  gSmmCpuPrivate->ApWrapperFunc[CpuIndex].Procedure = Procedure;
+  gSmmCpuPrivate->ApWrapperFunc[CpuIndex].ProcedureArgument = ProcArguments;
+
+  //
+  // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
+  //
+  return InternalSmmStartupThisAp (
+    ProcedureWrapper,
+    CpuIndex,
+    &gSmmCpuPrivate->ApWrapperFunc[CpuIndex],
+    FeaturePcdGet (PcdCpuSmmBlockStartupThisAp) ? NULL : &Token,
+    0,
+    NULL
+    );
 }
 
 /**
@@ -1112,6 +1579,13 @@ SmiRendezvous (
   Cr2 = 0;
   SaveCr2 (&Cr2);
 
+  //
+  // Call the user register Startup function first.
+  //
+  if (mSmmMpSyncData->StartupProcedure != NULL) {
+    mSmmMpSyncData->StartupProcedure (mSmmMpSyncData->StartupProcArgs);
+  }
+
   //
   // Perform CPU specific entry hooks
   //
@@ -1256,6 +1730,21 @@ Exit:
   RestoreCr2 (Cr2);
 }
 
+/**
+  Allocate buffer for SpinLock and Wrapper function buffer.
+
+**/
+VOID
+InitializeDataForMmMp (
+  VOID
+  )
+{
+  gSmmCpuPrivate->ApWrapperFunc = AllocatePool (sizeof (PROCEDURE_WRAPPER) * gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus);
+  ASSERT (gSmmCpuPrivate->ApWrapperFunc != NULL);
+
+  InitializeListHead (&gSmmCpuPrivate->TokenList);
+}
+
 /**
   Allocate buffer for all semaphores and spin locks.
 
@@ -1469,3 +1958,40 @@ RegisterSmmEntry (
   gSmmCpuPrivate->SmmCoreEntry = SmmEntryPoint;
   return EFI_SUCCESS;
 }
+
+/**
+
+  Register the SMM Foundation entry point.
+
+  @param[in]      Procedure            A pointer to the code stream to be run on the designated target AP
+                                       of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
+                                       with the related definitions of
+                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
+                                       If caller may pass a value of NULL to deregister any existing
+                                       startup procedure.
+  @param[in]      ProcedureArguments   Allows the caller to pass a list of parameters to the code that is
+                                       run by the AP. It is an optional common mailbox between APs and
+                                       the caller to share information
+
+  @retval EFI_SUCCESS                  The Procedure has been set successfully.
+  @retval EFI_INVALID_PARAMETER        The Procedure is NULL but ProcedureArguments not NULL.
+
+**/
+EFI_STATUS
+RegisterStartupProcedure (
+  IN EFI_AP_PROCEDURE    Procedure,
+  IN VOID                *ProcedureArguments OPTIONAL
+  )
+{
+  if (Procedure == NULL && ProcedureArguments != NULL) {
+    return EFI_INVALID_PARAMETER;
+  }
+  if (mSmmMpSyncData == NULL) {
+    return EFI_NOT_READY;
+  }
+
+  mSmmMpSyncData->StartupProcedure = Procedure;
+  mSmmMpSyncData->StartupProcArgs  = ProcedureArguments;
+
+  return EFI_SUCCESS;
+}
diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c
index 2f7d777ee7..69a04dfb23 100644
--- a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c
@@ -34,6 +34,8 @@ SMM_CPU_PRIVATE_DATA  mSmmCpuPrivateData = {
     mSmmCpuPrivateData.SmmReservedSmramRegion,  // SmmConfiguration.SmramReservedRegions
     RegisterSmmEntry                            // SmmConfiguration.RegisterSmmEntry
   },
+  NULL,                                         // pointer to Ap Wrapper Func array
+  {NULL, NULL},                                 // List_Entry for Tokens.
 };
 
 CPU_HOT_PLUG_DATA mCpuHotPlugData = {
@@ -996,6 +998,22 @@ PiCpuSmmEntry (
                     );
   ASSERT_EFI_ERROR (Status);
 
+  //
+  // Initialize global buffer for MM MP.
+  //
+  InitializeDataForMmMp ();
+
+  //
+  // Install the SMM Mp Protocol into SMM protocol database
+  //
+  Status = gSmst->SmmInstallProtocolInterface (
+                    &mSmmCpuHandle,
+                    &gEfiMmMpProtocolGuid,
+                    EFI_NATIVE_INTERFACE,
+                    &mSmmMp
+                    );
+  ASSERT_EFI_ERROR (Status);
+
   //
   // Expose address of CPU Hot Plug Data structure if CPU hot plug is supported.
   //
diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h
index 2bb35a424d..186809f431 100644
--- a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h
@@ -20,6 +20,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent
 #include <Protocol/SmmReadyToLock.h>
 #include <Protocol/SmmCpuService.h>
 #include <Protocol/SmmMemoryAttribute.h>
+#include <Protocol/MmMp.h>
 
 #include <Guid/AcpiS3Context.h>
 #include <Guid/MemoryAttributesTable.h>
@@ -197,6 +198,25 @@ typedef UINT32                              SMM_CPU_ARRIVAL_EXCEPTIONS;
 #define ARRIVAL_EXCEPTION_DELAYED           0x2
 #define ARRIVAL_EXCEPTION_SMI_DISABLED      0x4
 
+//
+// Wrapper used to convert EFI_AP_PROCEDURE2 and EFI_AP_PROCEDURE.
+//
+typedef struct {
+  EFI_AP_PROCEDURE  Procedure;
+  VOID              *ProcedureArgument;
+} PROCEDURE_WRAPPER;
+
+#define PROCEDURE_TOKEN_SIGNATURE  SIGNATURE_32 ('P', 'R', 'T', 'S')
+
+typedef struct {
+  UINTN                   Signature;
+  LIST_ENTRY              Link;
+
+  SPIN_LOCK               *ProcedureToken;
+} PROCEDURE_TOKEN;
+
+#define PROCEDURE_TOKEN_FROM_LINK(a)  CR (a, PROCEDURE_TOKEN, Link, PROCEDURE_TOKEN_SIGNATURE)
+
 //
 // Private structure for the SMM CPU module that is stored in DXE Runtime memory
 // Contains the SMM Configuration Protocols that is produced.
@@ -219,6 +239,10 @@ typedef struct {
   EFI_SMM_ENTRY_POINT             SmmCoreEntry;
 
   EFI_SMM_CONFIGURATION_PROTOCOL  SmmConfiguration;
+
+  PROCEDURE_WRAPPER               *ApWrapperFunc;
+  LIST_ENTRY                      TokenList;
+
 } SMM_CPU_PRIVATE_DATA;
 
 extern SMM_CPU_PRIVATE_DATA  *gSmmCpuPrivate;
@@ -226,6 +250,7 @@ extern CPU_HOT_PLUG_DATA      mCpuHotPlugData;
 extern UINTN                  mMaxNumberOfCpus;
 extern UINTN                  mNumberOfCpus;
 extern EFI_SMM_CPU_PROTOCOL   mSmmCpu;
+extern EFI_MM_MP_PROTOCOL     mSmmMp;
 
 ///
 /// The mode of the CPU at the time an SMI occurs
@@ -363,10 +388,12 @@ SmmRelocationSemaphoreComplete (
 ///
 typedef struct {
   SPIN_LOCK                         *Busy;
-  volatile EFI_AP_PROCEDURE         Procedure;
+  volatile EFI_AP_PROCEDURE2        Procedure;
   volatile VOID                     *Parameter;
   volatile UINT32                   *Run;
   volatile BOOLEAN                  *Present;
+  SPIN_LOCK                         *Token;
+  EFI_STATUS                        *Status;
 } SMM_CPU_DATA_BLOCK;
 
 typedef enum {
@@ -388,6 +415,8 @@ typedef struct {
   volatile SMM_CPU_SYNC_MODE    EffectiveSyncMode;
   volatile BOOLEAN              SwitchBsp;
   volatile BOOLEAN              *CandidateBsp;
+  EFI_AP_PROCEDURE              StartupProcedure;
+  VOID                          *StartupProcArgs;
 } SMM_DISPATCHER_MP_SYNC_DATA;
 
 #define SMM_PSD_OFFSET              0xfb00
@@ -410,6 +439,7 @@ typedef struct {
   SPIN_LOCK                         *Busy;
   volatile UINT32                   *Run;
   volatile BOOLEAN                  *Present;
+  SPIN_LOCK                         *Token;
 } SMM_CPU_SEMAPHORE_CPU;
 
 ///
@@ -1259,4 +1289,165 @@ RestoreCr2 (
   IN UINTN  Cr2
   );
 
+/**
+  Schedule a procedure to run on the specified CPU.
+
+  @param[in]       Procedure                The address of the procedure to run
+  @param[in]       CpuIndex                 Target CPU Index
+  @param[in,out]   ProcArguments            The parameter to pass to the procedure
+  @param[in,out]   Token                    This is an optional parameter that allows the caller to execute the
+                                            procedure in a blocking or non-blocking fashion. If it is NULL the
+                                            call is blocking, and the call will not return until the AP has
+                                            completed the procedure. If the token is not NULL, the call will
+                                            return immediately. The caller can check whether the procedure has
+                                            completed with CheckOnProcedure or WaitForProcedure.
+  @param[in]       TimeoutInMicroseconds    Indicates the time limit in microseconds for the APs to finish
+                                            execution of Procedure, either for blocking or non-blocking mode.
+                                            Zero means infinity. If the timeout expires before all APs return
+                                            from Procedure, then Procedure on the failed APs is terminated. If
+                                            the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
+                                            If the timeout expires in non-blocking mode, the timeout determined
+                                            can be through CheckOnProcedure or WaitForProcedure.
+                                            Note that timeout support is optional. Whether an implementation
+                                            supports this feature can be determined via the Attributes data
+                                            member.
+  @param[in,out]   CPUStatus                This optional pointer may be used to get the status code returned
+                                            by Procedure when it completes execution on the target AP, or with
+                                            EFI_TIMEOUT if the Procedure fails to complete within the optional
+                                            timeout. The implementation will update this variable with
+                                            EFI_NOT_READY prior to starting Procedure on the target AP.
+
+  @retval EFI_INVALID_PARAMETER    CpuNumber not valid
+  @retval EFI_INVALID_PARAMETER    CpuNumber specifying BSP
+  @retval EFI_INVALID_PARAMETER    The AP specified by CpuNumber did not enter SMM
+  @retval EFI_INVALID_PARAMETER    The AP specified by CpuNumber is busy
+  @retval EFI_SUCCESS              The procedure has been successfully scheduled
+
+**/
+EFI_STATUS
+InternalSmmStartupThisAp (
+  IN      EFI_AP_PROCEDURE2              Procedure,
+  IN      UINTN                          CpuIndex,
+  IN OUT  VOID                           *ProcArguments OPTIONAL,
+  IN      MM_COMPLETION                  *Token,
+  IN      UINTN                          TimeoutInMicroseconds,
+  IN OUT  EFI_STATUS                     *CpuStatus
+  );
+
+/**
+  Checks whether the input token is the current used token.
+
+  @param[in]  Token      This parameter describes the token that was passed into DispatchProcedure or
+                         BroadcastProcedure.
+
+  @retval TRUE           The input token is the current used token.
+  @retval FALSE          The input token is not the current used token.
+**/
+BOOLEAN
+IsTokenInUse (
+  IN SPIN_LOCK           *Token
+  );
+
+/**
+  Checks status of specified AP.
+
+  This function checks whether the specified AP has finished the task assigned
+  by StartupThisAP(), and whether timeout expires.
+
+  @param[in]  Token             This parameter describes the token that was passed into DispatchProcedure or
+                                BroadcastProcedure.
+
+  @retval EFI_SUCCESS           Specified AP has finished task assigned by StartupThisAPs().
+  @retval EFI_NOT_READY         Specified AP has not finished task and timeout has not expired.
+**/
+EFI_STATUS
+IsApReady (
+  IN SPIN_LOCK  *Token
+  );
+
+/**
+  Check whether it is an present AP.
+
+  @param   CpuIndex      The AP index which calls this function.
+
+  @retval  TRUE           It's a present AP.
+  @retval  TRUE           This is not an AP or it is not present.
+
+**/
+BOOLEAN
+IsPresentAp (
+  IN UINTN        CpuIndex
+  );
+
+/**
+  Worker function to execute a caller provided function on all enabled APs.
+
+  @param[in]     Procedure               A pointer to the function to be run on
+                                         enabled APs of the system.
+  @param[in]     TimeoutInMicroseconds   Indicates the time limit in microseconds for
+                                         APs to return from Procedure, either for
+                                         blocking or non-blocking mode.
+  @param[in,out] ProcedureArgument       The parameter passed into Procedure for
+                                         all APs.
+  @param[in,out] Token                   This is an optional parameter that allows the caller to execute the
+                                         procedure in a blocking or non-blocking fashion. If it is NULL the
+                                         call is blocking, and the call will not return until the AP has
+                                         completed the procedure. If the token is not NULL, the call will
+                                         return immediately. The caller can check whether the procedure has
+                                         completed with CheckOnProcedure or WaitForProcedure.
+  @param[in,out] CPUStatus               This optional pointer may be used to get the status code returned
+                                         by Procedure when it completes execution on the target AP, or with
+                                         EFI_TIMEOUT if the Procedure fails to complete within the optional
+                                         timeout. The implementation will update this variable with
+                                         EFI_NOT_READY prior to starting Procedure on the target AP.
+
+  @retval EFI_SUCCESS             In blocking mode, all APs have finished before
+                                  the timeout expired.
+  @retval EFI_SUCCESS             In non-blocking mode, function has been dispatched
+                                  to all enabled APs.
+  @retval others                  Failed to Startup all APs.
+
+**/
+EFI_STATUS
+InternalSmmStartupAllAPs (
+  IN       EFI_AP_PROCEDURE2             Procedure,
+  IN       UINTN                         TimeoutInMicroseconds,
+  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
+  IN OUT   MM_COMPLETION                 *Token,
+  IN OUT   EFI_STATUS                    *CPUStatus
+  );
+
+/**
+
+  Register the SMM Foundation entry point.
+
+  @param[in]      Procedure            A pointer to the code stream to be run on the designated target AP
+                                       of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
+                                       with the related definitions of
+                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
+                                       If caller may pass a value of NULL to deregister any existing
+                                       startup procedure.
+  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of parameters to the code that is
+                                       run by the AP. It is an optional common mailbox between APs and
+                                       the caller to share information
+
+  @retval EFI_SUCCESS                  The Procedure has been set successfully.
+  @retval EFI_INVALID_PARAMETER        The Procedure is NULL but ProcedureArguments not NULL.
+
+**/
+EFI_STATUS
+RegisterStartupProcedure (
+  IN EFI_AP_PROCEDURE    Procedure,
+  IN VOID                *ProcedureArguments OPTIONAL
+  );
+
+/**
+  Allocate buffer for SpinLock and Wrapper function buffer.
+
+**/
+VOID
+InitializeDataForMmMp (
+  VOID
+  );
+
 #endif
diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf
index 466c568d49..da0308c47f 100644
--- a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf
@@ -40,6 +40,8 @@
   SmmProfileInternal.h
   SmramSaveState.c
   SmmCpuMemoryManagement.c
+  SmmMp.h
+  SmmMp.c
 
 [Sources.Ia32]
   Ia32/Semaphore.c
@@ -105,6 +107,7 @@
   gEfiSmmReadyToLockProtocolGuid           ## NOTIFY
   gEfiSmmCpuServiceProtocolGuid            ## PRODUCES
   gEdkiiSmmMemoryAttributeProtocolGuid     ## PRODUCES
+  gEfiMmMpProtocolGuid                    ## PRODUCES
 
 [Guids]
   gEfiAcpiVariableGuid                     ## SOMETIMES_CONSUMES ## HOB # it is used for S3 boot.
diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c b/UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c
new file mode 100644
index 0000000000..9b2b191e03
--- /dev/null
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c
@@ -0,0 +1,344 @@
+/** @file
+SMM MP protocol implementation
+
+Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
+
+SPDX-License-Identifier: BSD-2-Clause-Patent
+
+**/
+
+#include "PiSmmCpuDxeSmm.h"
+#include "SmmMp.h"
+
+///
+/// SMM MP Protocol instance
+///
+EFI_MM_MP_PROTOCOL  mSmmMp  = {
+  EFI_MM_MP_PROTOCOL_REVISION,
+  0,
+  SmmMpGetNumberOfProcessors,
+  SmmMpDispatchProcedure,
+  SmmMpBroadcastProcedure,
+  SmmMpSetStartupProcedure,
+  SmmMpCheckForProcedure,
+  SmmMpWaitForProcedure
+};
+
+/**
+  Service to retrieves the number of logical processor in the platform.
+
+  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
+  @param[out] NumberOfProcessors  Pointer to the total number of logical processors in the system,
+                                  including the BSP and all APs.
+
+  @retval EFI_SUCCESS             The number of processors was retrieved successfully
+  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
+**/
+EFI_STATUS
+EFIAPI
+SmmMpGetNumberOfProcessors (
+  IN CONST EFI_MM_MP_PROTOCOL   *This,
+  OUT      UINTN                *NumberOfProcessors
+  )
+{
+  if (NumberOfProcessors == NULL) {
+    return EFI_INVALID_PARAMETER;
+  }
+
+  *NumberOfProcessors = gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus;
+
+  return EFI_SUCCESS;
+}
+
+/**
+  This service allows the caller to invoke a procedure one of the application processors (AP). This
+  function uses an optional token parameter to support blocking and non-blocking modes. If the token
+  is passed into the call, the function will operate in a non-blocking fashion and the caller can
+  check for completion with CheckOnProcedure or WaitForProcedure.
+
+  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
+  @param[in]     Procedure              A pointer to the procedure to be run on the designated target
+                                        AP of the system. Type EFI_AP_PROCEDURE2 is defined below in
+                                        related definitions.
+  @param[in]     CpuNumber              The zero-based index of the processor number of the target
+                                        AP, on which the code stream is supposed to run. If the number
+                                        points to the calling processor then it will not run the
+                                        supplied code.
+  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for this AP to
+                                        finish execution of Procedure, either for blocking or
+                                        non-blocking mode. Zero means infinity. If the timeout
+                                        expires before this AP returns from Procedure, then Procedure
+                                        on the AP is terminated. If the timeout expires in blocking
+                                        mode, the call returns EFI_TIMEOUT. If the timeout expires
+                                        in non-blocking mode, the timeout determined can be through
+                                        CheckOnProcedure or WaitForProcedure.
+                                        Note that timeout support is optional. Whether an
+                                        implementation supports this feature, can be determined via
+                                        the Attributes data member.
+  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
+                                        that is run by the AP. It is an optional common mailbox
+                                        between APs and the caller to share information.
+  @param[in,out] Token                  This is parameter is broken into two components:
+                                        1.Token->Completion is an optional parameter that allows the
+                                        caller to execute the procedure in a blocking or non-blocking
+                                        fashion. If it is NULL the call is blocking, and the call will
+                                        not return until the AP has completed the procedure. If the
+                                        token is not NULL, the call will return immediately. The caller
+                                        can check whether the procedure has completed with
+                                        CheckOnProcedure or WaitForProcedure.
+                                        2.Token->Status The implementation updates the address pointed
+                                        at by this variable with the status code returned by Procedure
+                                        when it completes execution on the target AP, or with EFI_TIMEOUT
+                                        if the Procedure fails to complete within the optional timeout.
+                                        The implementation will update this variable with EFI_NOT_READY
+                                        prior to starting Procedure on the target AP
+  @param[in,out] CPUStatus              This optional pointer may be used to get the status code returned
+                                        by Procedure when it completes execution on the target AP, or with
+                                        EFI_TIMEOUT if the Procedure fails to complete within the optional
+                                        timeout. The implementation will update this variable with
+                                        EFI_NOT_READY prior to starting Procedure on the target AP.
+
+  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
+                                        execution on the target AP.
+                                        In the non-blocking case this indicates that the procedure has
+                                        been successfully scheduled for execution on the target AP.
+  @retval EFI_INVALID_PARAMETER         The input arguments are out of range. Either the target AP is the
+                                        caller of the function, or the Procedure or Token is NULL
+  @retval EFI_NOT_READY                 If the target AP is busy executing another procedure
+  @retval EFI_ALREADY_STARTED           Token is already in use for another procedure
+  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before the specified AP
+                                        has finished
+  @retval EFI_OUT_OF_RESOURCES          Could not allocate a required resource.
+
+**/
+EFI_STATUS
+EFIAPI
+SmmMpDispatchProcedure (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       EFI_AP_PROCEDURE2             Procedure,
+  IN       UINTN                         CpuNumber,
+  IN       UINTN                         TimeoutInMicroseconds,
+  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
+  IN OUT   MM_COMPLETION                 *Token,
+  IN OUT   EFI_STATUS                    *CPUStatus
+  )
+{
+  return InternalSmmStartupThisAp (
+    Procedure,
+    CpuNumber,
+    ProcedureArguments,
+    Token,
+    TimeoutInMicroseconds,
+    CPUStatus
+    );
+}
+
+/**
+  This service allows the caller to invoke a procedure on all running application processors (AP)
+  except the caller. This function uses an optional token parameter to support blocking and
+  nonblocking modes. If the token is passed into the call, the function will operate in a non-blocking
+  fashion and the caller can check for completion with CheckOnProcedure or WaitForProcedure.
+
+  It is not necessary for the implementation to run the procedure on every processor on the platform.
+  Processors that are powered down in such a way that they cannot respond to interrupts, may be
+  excluded from the broadcast.
+
+
+  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
+  @param[in]     Procedure              A pointer to the code stream to be run on the APs that have
+                                        entered MM. Type EFI_AP_PROCEDURE is defined below in related
+                                        definitions.
+  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for the APs to finish
+                                        execution of Procedure, either for blocking or non-blocking mode.
+                                        Zero means infinity. If the timeout expires before all APs return
+                                        from Procedure, then Procedure on the failed APs is terminated. If
+                                        the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
+                                        If the timeout expires in non-blocking mode, the timeout determined
+                                        can be through CheckOnProcedure or WaitForProcedure.
+                                        Note that timeout support is optional. Whether an implementation
+                                        supports this feature can be determined via the Attributes data
+                                        member.
+  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
+                                        that is run by the AP. It is an optional common mailbox
+                                        between APs and the caller to share information.
+  @param[in,out] Token                  This is parameter is broken into two components:
+                                        1.Token->Completion is an optional parameter that allows the
+                                        caller to execute the procedure in a blocking or non-blocking
+                                        fashion. If it is NULL the call is blocking, and the call will
+                                        not return until the AP has completed the procedure. If the
+                                        token is not NULL, the call will return immediately. The caller
+                                        can check whether the procedure has completed with
+                                        CheckOnProcedure or WaitForProcedure.
+                                        2.Token->Status The implementation updates the address pointed
+                                        at by this variable with the status code returned by Procedure
+                                        when it completes execution on the target AP, or with EFI_TIMEOUT
+                                        if the Procedure fails to complete within the optional timeout.
+                                        The implementation will update this variable with EFI_NOT_READY
+                                        prior to starting Procedure on the target AP
+  @param[in,out] CPUStatus              This optional pointer may be used to get the individual status
+                                        returned by every AP that participated in the broadcast. This
+                                        parameter if used provides the base address of an array to hold
+                                        the EFI_STATUS value of each AP in the system. The size of the
+                                        array can be ascertained by the GetNumberOfProcessors function.
+                                        As mentioned above, the broadcast may not include every processor
+                                        in the system. Some implementations may exclude processors that
+                                        have been powered down in such a way that they are not responsive
+                                        to interrupts. Additionally the broadcast excludes the processor
+                                        which is making the BroadcastProcedure call. For every excluded
+                                        processor, the array entry must contain a value of EFI_NOT_STARTED
+
+  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
+                                        execution on the APs.
+                                        In the non-blocking case this indicates that the procedure has
+                                        been successfully scheduled for execution on the APs.
+  @retval EFI_INVALID_PARAMETER         The Procedure or Token is NULL
+  @retval EFI_NOT_READY                 If the target AP is busy executing another procedure
+  @retval EFI_ALREADY_STARTED           Token is already in use for another procedure
+  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before the specified AP
+                                        has finished.
+  @retval EFI_OUT_OF_RESOURCES          Could not allocate a required resource.
+
+**/
+EFI_STATUS
+EFIAPI
+SmmMpBroadcastProcedure (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       EFI_AP_PROCEDURE2             Procedure,
+  IN       UINTN                         TimeoutInMicroseconds,
+  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
+  IN OUT   MM_COMPLETION                 *Token,
+  IN OUT   EFI_STATUS                    *CPUStatus
+  )
+{
+  return InternalSmmStartupAllAPs(
+    Procedure,
+    TimeoutInMicroseconds,
+    ProcedureArguments,
+    Token,
+    CPUStatus
+    );
+}
+
+/**
+  This service allows the caller to set a startup procedure that will be executed when an AP powers
+  up from a state where core configuration and context is lost. The procedure is execution has the
+  following properties:
+  1. The procedure executes before the processor is handed over to the operating system.
+  2. All processors execute the same startup procedure.
+  3. The procedure may run in parallel with other procedures invoked through the functions in this
+  protocol, or with processors that are executing an MM handler or running in the operating system.
+
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Procedure            A pointer to the code stream to be run on the designated target AP
+                                       of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
+                                       with the related definitions of
+                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
+                                       If caller may pass a value of NULL to deregister any existing
+                                       startup procedure.
+  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of parameters to the code that is
+                                       run by the AP. It is an optional common mailbox between APs and
+                                       the caller to share information
+
+  @retval EFI_SUCCESS                  The Procedure has been set successfully.
+  @retval EFI_INVALID_PARAMETER        The Procedure is NULL but ProcedureArguments not NULL.
+
+**/
+EFI_STATUS
+EFIAPI
+SmmMpSetStartupProcedure (
+  IN CONST EFI_MM_MP_PROTOCOL  *This,
+  IN       EFI_AP_PROCEDURE    Procedure,
+  IN OUT   VOID                *ProcedureArguments OPTIONAL
+  )
+{
+  return RegisterStartupProcedure (Procedure, ProcedureArguments);
+}
+
+/**
+  When non-blocking execution of a procedure on an AP is invoked with DispatchProcedure,
+  via the use of a token, this function can be used to check for completion of the procedure on the AP.
+  The function takes the token that was passed into the DispatchProcedure call. If the procedure
+  is complete, and therefore it is now possible to run another procedure on the same AP, this function
+  returns EFI_SUCESS. In this case the status returned by the procedure that executed on the AP is
+  returned in the token's Status field. If the procedure has not yet completed, then this function
+  returns EFI_NOT_READY.
+
+  When a non-blocking execution of a procedure is invoked with BroadcastProcedure, via the
+  use of a token, this function can be used to check for completion of the procedure on all the
+  broadcast APs. The function takes the token that was passed into the BroadcastProcedure
+  call. If the procedure is complete on all broadcast APs this function returns EFI_SUCESS. In this
+  case the Status field in the token passed into the function reflects the overall result of the
+  invocation, which may be EFI_SUCCESS, if all executions succeeded, or the first observed failure.
+  If the procedure has not yet completed on the broadcast APs, the function returns
+  EFI_NOT_READY.
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Token                This parameter describes the token that was passed into
+                                       DispatchProcedure or BroadcastProcedure.
+
+  @retval EFI_SUCCESS                  Procedure has completed.
+  @retval EFI_NOT_READY                The Procedure has not completed.
+  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is NULL
+  @retval EFI_NOT_FOUND                Token is not currently in use for a non-blocking call
+
+**/
+EFI_STATUS
+EFIAPI
+SmmMpCheckForProcedure (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       MM_COMPLETION                 Token
+  )
+{
+  if (Token == NULL) {
+    return EFI_INVALID_PARAMETER;
+  }
+
+  if (!IsTokenInUse ((SPIN_LOCK *)Token)) {
+    return EFI_NOT_FOUND;
+  }
+
+  return IsApReady ((SPIN_LOCK *)Token);
+}
+
+/**
+  When a non-blocking execution of a procedure on an AP is invoked via DispatchProcedure,
+  this function will block the caller until the remote procedure has completed on the designated AP.
+  The non-blocking procedure invocation is identified by the Token parameter, which must match the
+  token that used when DispatchProcedure was called. Upon completion the status returned by
+  the procedure that executed on the AP is used to update the token's Status field.
+
+  When a non-blocking execution of a procedure on an AP is invoked via BroadcastProcedure
+  this function will block the caller until the remote procedure has completed on all of the APs that
+  entered MM. The non-blocking procedure invocation is identified by the Token parameter, which
+  must match the token that used when BroadcastProcedure was called. Upon completion the
+  overall status returned by the procedures that executed on the broadcast AP is used to update the
+  token's Status field. The overall status may be EFI_SUCCESS, if all executions succeeded, or the
+  first observed failure.
+
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Token                This parameter describes the token that was passed into
+                                       DispatchProcedure or BroadcastProcedure.
+
+  @retval EFI_SUCCESS                  Procedure has completed.
+  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is NULL
+  @retval EFI_NOT_FOUND                Token is not currently in use for a non-blocking call
+
+**/
+EFI_STATUS
+EFIAPI
+SmmMpWaitForProcedure (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       MM_COMPLETION                 Token
+  )
+{
+  EFI_STATUS    Status;
+
+  do {
+    Status = SmmMpCheckForProcedure (This, Token);
+  } while (Status == EFI_NOT_READY);
+
+  return Status;
+}
+
diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h b/UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h
new file mode 100644
index 0000000000..e0d823a4b1
--- /dev/null
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h
@@ -0,0 +1,286 @@
+/** @file
+Include file for SMM MP protocol implementation.
+
+Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
+
+SPDX-License-Identifier: BSD-2-Clause-Patent
+
+**/
+
+#ifndef _SMM_MP_PROTOCOL_H_
+#define _SMM_MP_PROTOCOL_H_
+
+//
+// SMM MP Protocol function prototypes.
+//
+
+/**
+  Service to retrieves the number of logical processor in the platform.
+
+  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
+  @param[out] NumberOfProcessors  Pointer to the total number of logical processors in the system,
+                                  including the BSP and all APs.
+
+  @retval EFI_SUCCESS             The number of processors was retrieved successfully
+  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
+**/
+
+EFI_STATUS
+EFIAPI
+SmmMpGetNumberOfProcessors (
+  IN CONST EFI_MM_MP_PROTOCOL   *This,
+  OUT      UINTN                *NumberOfProcessors
+  );
+
+
+/**
+  This service allows the caller to invoke a procedure one of the application processors (AP). This
+  function uses an optional token parameter to support blocking and non-blocking modes. If the token
+  is passed into the call, the function will operate in a non-blocking fashion and the caller can
+  check for completion with CheckOnProcedure or WaitForProcedure.
+
+  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
+  @param[in]     Procedure              A pointer to the procedure to be run on the designated target
+                                        AP of the system. Type EFI_AP_PROCEDURE2 is defined below in
+                                        related definitions.
+  @param[in]     CpuNumber              The zero-based index of the processor number of the target
+                                        AP, on which the code stream is supposed to run. If the number
+                                        points to the calling processor then it will not run the
+                                        supplied code.
+  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for this AP to
+                                        finish execution of Procedure, either for blocking or
+                                        non-blocking mode. Zero means infinity. If the timeout
+                                        expires before this AP returns from Procedure, then Procedure
+                                        on the AP is terminated. If the timeout expires in blocking
+                                        mode, the call returns EFI_TIMEOUT. If the timeout expires
+                                        in non-blocking mode, the timeout determined can be through
+                                        CheckOnProcedure or WaitForProcedure.
+                                        Note that timeout support is optional. Whether an
+                                        implementation supports this feature, can be determined via
+                                        the Attributes data member.
+  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
+                                        that is run by the AP. It is an optional common mailbox
+                                        between APs and the caller to share information.
+  @param[in,out] Token                  This is parameter is broken into two components:
+                                        1.Token->Completion is an optional parameter that allows the
+                                        caller to execute the procedure in a blocking or non-blocking
+                                        fashion. If it is NULL the call is blocking, and the call will
+                                        not return until the AP has completed the procedure. If the
+                                        token is not NULL, the call will return immediately. The caller
+                                        can check whether the procedure has completed with
+                                        CheckOnProcedure or WaitForProcedure.
+                                        2.Token->Status The implementation updates the address pointed
+                                        at by this variable with the status code returned by Procedure
+                                        when it completes execution on the target AP, or with EFI_TIMEOUT
+                                        if the Procedure fails to complete within the optional timeout.
+                                        The implementation will update this variable with EFI_NOT_READY
+                                        prior to starting Procedure on the target AP
+  @param[in,out] CPUStatus              This optional pointer may be used to get the status code returned
+                                        by Procedure when it completes execution on the target AP, or with
+                                        EFI_TIMEOUT if the Procedure fails to complete within the optional
+                                        timeout. The implementation will update this variable with
+                                        EFI_NOT_READY prior to starting Procedure on the target AP.
+
+  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
+                                        execution on the target AP.
+                                        In the non-blocking case this indicates that the procedure has
+                                        been successfully scheduled for execution on the target AP.
+  @retval EFI_INVALID_PARAMETER         The input arguments are out of range. Either the target AP is the
+                                        caller of the function, or the Procedure or Token is NULL
+  @retval EFI_NOT_READY                 If the target AP is busy executing another procedure
+  @retval EFI_ALREADY_STARTED           Token is already in use for another procedure
+  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before the specified AP
+                                        has finished
+  @retval EFI_OUT_OF_RESOURCES          Could not allocate a required resource.
+
+**/
+EFI_STATUS
+EFIAPI
+SmmMpDispatchProcedure (
+  IN CONST EFI_MM_MP_PROTOCOL             *This,
+  IN       EFI_AP_PROCEDURE2              Procedure,
+  IN       UINTN                          CpuNumber,
+  IN       UINTN                          TimeoutInMicroseconds,
+  IN OUT   VOID                           *ProcedureArguments OPTIONAL,
+  IN OUT   MM_COMPLETION                  *Token,
+  IN OUT   EFI_STATUS                     *CPUStatus
+  );
+
+/**
+  This service allows the caller to invoke a procedure on all running application processors (AP)
+  except the caller. This function uses an optional token parameter to support blocking and
+  nonblocking modes. If the token is passed into the call, the function will operate in a non-blocking
+  fashion and the caller can check for completion with CheckOnProcedure or WaitForProcedure.
+
+  It is not necessary for the implementation to run the procedure on every processor on the platform.
+  Processors that are powered down in such a way that they cannot respond to interrupts, may be
+  excluded from the broadcast.
+
+
+  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
+  @param[in]     Procedure              A pointer to the code stream to be run on the APs that have
+                                        entered MM. Type EFI_AP_PROCEDURE is defined below in related
+                                        definitions.
+  @param[in]     TimeoutInMicroseconds  Indicates the time limit in microseconds for the APs to finish
+                                        execution of Procedure, either for blocking or non-blocking mode.
+                                        Zero means infinity. If the timeout expires before all APs return
+                                        from Procedure, then Procedure on the failed APs is terminated. If
+                                        the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
+                                        If the timeout expires in non-blocking mode, the timeout determined
+                                        can be through CheckOnProcedure or WaitForProcedure.
+                                        Note that timeout support is optional. Whether an implementation
+                                        supports this feature can be determined via the Attributes data
+                                        member.
+  @param[in,out] ProcedureArguments     Allows the caller to pass a list of parameters to the code
+                                        that is run by the AP. It is an optional common mailbox
+                                        between APs and the caller to share information.
+  @param[in,out] Token                  This is parameter is broken into two components:
+                                        1.Token->Completion is an optional parameter that allows the
+                                        caller to execute the procedure in a blocking or non-blocking
+                                        fashion. If it is NULL the call is blocking, and the call will
+                                        not return until the AP has completed the procedure. If the
+                                        token is not NULL, the call will return immediately. The caller
+                                        can check whether the procedure has completed with
+                                        CheckOnProcedure or WaitForProcedure.
+                                        2.Token->Status The implementation updates the address pointed
+                                        at by this variable with the status code returned by Procedure
+                                        when it completes execution on the target AP, or with EFI_TIMEOUT
+                                        if the Procedure fails to complete within the optional timeout.
+                                        The implementation will update this variable with EFI_NOT_READY
+                                        prior to starting Procedure on the target AP
+  @param[in,out] CPUStatus              This optional pointer may be used to get the individual status
+                                        returned by every AP that participated in the broadcast. This
+                                        parameter if used provides the base address of an array to hold
+                                        the EFI_STATUS value of each AP in the system. The size of the
+                                        array can be ascertained by the GetNumberOfProcessors function.
+                                        As mentioned above, the broadcast may not include every processor
+                                        in the system. Some implementations may exclude processors that
+                                        have been powered down in such a way that they are not responsive
+                                        to interrupts. Additionally the broadcast excludes the processor
+                                        which is making the BroadcastProcedure call. For every excluded
+                                        processor, the array entry must contain a value of EFI_NOT_STARTED
+
+  @retval EFI_SUCCESS                   In the blocking case, this indicates that Procedure has completed
+                                        execution on the APs.
+                                        In the non-blocking case this indicates that the procedure has
+                                        been successfully scheduled for execution on the APs.
+  @retval EFI_INVALID_PARAMETER         The Procedure or Token is NULL
+  @retval EFI_NOT_READY                 If the target AP is busy executing another procedure
+  @retval EFI_ALREADY_STARTED           Token is already in use for another procedure
+  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired before the specified AP
+                                        has finished
+  @retval EFI_OUT_OF_RESOURCES          Could not allocate a required resource.
+
+**/
+EFI_STATUS
+EFIAPI
+SmmMpBroadcastProcedure (
+  IN CONST EFI_MM_MP_PROTOCOL             *This,
+  IN       EFI_AP_PROCEDURE2              Procedure,
+  IN       UINTN                          TimeoutInMicroseconds,
+  IN OUT   VOID                           *ProcedureArguments OPTIONAL,
+  IN OUT   MM_COMPLETION                  *Token,
+  IN OUT   EFI_STATUS                     *CPUStatus
+  );
+
+
+/**
+  This service allows the caller to set a startup procedure that will be executed when an AP powers
+  up from a state where core configuration and context is lost. The procedure is execution has the
+  following properties:
+  1. The procedure executes before the processor is handed over to the operating system.
+  2. All processors execute the same startup procedure.
+  3. The procedure may run in parallel with other procedures invoked through the functions in this
+  protocol, or with processors that are executing an MM handler or running in the operating system.
+
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Procedure            A pointer to the code stream to be run on the designated target AP
+                                       of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
+                                       with the related definitions of
+                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
+                                       If caller may pass a value of NULL to deregister any existing
+                                       startup procedure.
+  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of parameters to the code that is
+                                       run by the AP. It is an optional common mailbox between APs and
+                                       the caller to share information
+
+  @retval EFI_SUCCESS                  The Procedure has been set successfully.
+  @retval EFI_INVALID_PARAMETER        The Procedure is NULL but ProcedureArguments not NULL.
+**/
+EFI_STATUS
+EFIAPI
+SmmMpSetStartupProcedure (
+  IN CONST EFI_MM_MP_PROTOCOL  *This,
+  IN       EFI_AP_PROCEDURE    Procedure,
+  IN OUT   VOID                *ProcedureArguments OPTIONAL
+  );
+
+/**
+  When non-blocking execution of a procedure on an AP is invoked with DispatchProcedure,
+  via the use of a token, this function can be used to check for completion of the procedure on the AP.
+  The function takes the token that was passed into the DispatchProcedure call. If the procedure
+  is complete, and therefore it is now possible to run another procedure on the same AP, this function
+  returns EFI_SUCESS. In this case the status returned by the procedure that executed on the AP is
+  returned in the token's Status field. If the procedure has not yet completed, then this function
+  returns EFI_NOT_READY.
+
+  When a non-blocking execution of a procedure is invoked with BroadcastProcedure, via the
+  use of a token, this function can be used to check for completion of the procedure on all the
+  broadcast APs. The function takes the token that was passed into the BroadcastProcedure
+  call. If the procedure is complete on all broadcast APs this function returns EFI_SUCESS. In this
+  case the Status field in the token passed into the function reflects the overall result of the
+  invocation, which may be EFI_SUCCESS, if all executions succeeded, or the first observed failure.
+  If the procedure has not yet completed on the broadcast APs, the function returns
+  EFI_NOT_READY.
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Token                This parameter describes the token that was passed into
+                                       DispatchProcedure or BroadcastProcedure.
+
+  @retval EFI_SUCCESS                  Procedure has completed.
+  @retval EFI_NOT_READY                The Procedure has not completed.
+  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is NULL
+  @retval EFI_NOT_FOUND                Token is not currently in use for a non-blocking call
+
+**/
+EFI_STATUS
+EFIAPI
+SmmMpCheckForProcedure (
+  IN CONST EFI_MM_MP_PROTOCOL             *This,
+  IN       MM_COMPLETION                  Token
+  );
+
+/**
+  When a non-blocking execution of a procedure on an AP is invoked via DispatchProcedure,
+  this function will block the caller until the remote procedure has completed on the designated AP.
+  The non-blocking procedure invocation is identified by the Token parameter, which must match the
+  token that used when DispatchProcedure was called. Upon completion the status returned by
+  the procedure that executed on the AP is used to update the token's Status field.
+
+  When a non-blocking execution of a procedure on an AP is invoked via BroadcastProcedure
+  this function will block the caller until the remote procedure has completed on all of the APs that
+  entered MM. The non-blocking procedure invocation is identified by the Token parameter, which
+  must match the token that used when BroadcastProcedure was called. Upon completion the
+  overall status returned by the procedures that executed on the broadcast AP is used to update the
+  token's Status field. The overall status may be EFI_SUCCESS, if all executions succeeded, or the
+  first observed failure.
+
+
+  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
+  @param[in]      Token                This parameter describes the token that was passed into
+                                       DispatchProcedure or BroadcastProcedure.
+
+  @retval EFI_SUCCESS                  Procedure has completed.
+  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is NULL
+  @retval EFI_NOT_FOUND                Token is not currently in use for a non-blocking call
+
+**/
+EFI_STATUS
+EFIAPI
+SmmMpWaitForProcedure (
+  IN CONST EFI_MM_MP_PROTOCOL            *This,
+  IN       MM_COMPLETION                 Token
+  );
+
+#endif
-- 
2.21.0.windows.1

Re: [Patch v5 2/2] UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP Protocol

[Ni, Ray]

Reviewed-by: Ray Ni <ray.ni@intel.com>

> -----Original Message-----
> From: Dong, Eric
> Sent: Wednesday, July 10, 2019 3:56 PM
> To: devel@edk2.groups.io
> Cc: Ni, Ray <ray.ni@intel.com>; Laszlo Ersek <lersek@redhat.com>
> Subject: [Patch v5 2/2] UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP
> Protocol
> 
> V5 changes:
> 1. some small enhancement.
> 
> v4 changes:
> 1. Use link list to save the token info.
> 
> v3 changes:
> 1. Fix Token clean up too early caused CheckProcedure return error.
> 
> v2 changes:
> 1. Remove some duplicated global variables.
> 2. Enhance token design to support multiple task trig for different APs at the
> same time.
> 
> V1 changes:
> REF: https://bugzilla.tianocore.org/show_bug.cgi?id=1937
> 
> Add MM Mp Protocol in PiSmmCpuDxeSmm driver.
> 
> Cc: Ray Ni <ray.ni@intel.com>
> Cc: Laszlo Ersek <lersek@redhat.com>
> Signed-off-by: Eric Dong <eric.dong@intel.com>
> ---
>  UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c        | 570
> ++++++++++++++++++-
>  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c   |  18 +
>  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h   | 193 ++++++-
>  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf |   3 +
>  UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c            | 344 +++++++++++
>  UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h            | 286 ++++++++++
>  6 files changed, 1391 insertions(+), 23 deletions(-)
>  create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c
>  create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h
> 
> diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> index 64fb4d6344..f09e2738c3 100644
> --- a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> @@ -140,7 +140,7 @@ ReleaseAllAPs (
> 
>    BspIndex = mSmmMpSyncData->BspIndex;
>    for (Index = mMaxNumberOfCpus; Index-- > 0;) {
> -    if (Index != BspIndex && *(mSmmMpSyncData->CpuData[Index].Present))
> {
> +    if (IsPresentAp (Index)) {
>        ReleaseSemaphore (mSmmMpSyncData->CpuData[Index].Run);
>      }
>    }
> @@ -347,6 +347,165 @@ ReplaceOSMtrrs (
>    MtrrSetAllMtrrs (&gSmiMtrrs);
>  }
> 
> +/**
> +  Wheck whether task has been finished by all APs.
> +
> +  @param       BlockMode   Whether did it in block mode or non-block mode.
> +
> +  @retval      TRUE        Task has been finished by all APs.
> +  @retval      FALSE       Task not has been finished by all APs.
> +
> +**/
> +BOOLEAN
> +WaitForAllAPsNotBusy (
> +  IN BOOLEAN                        BlockMode
> +  )
> +{
> +  UINTN                             Index;
> +
> +  for (Index = mMaxNumberOfCpus; Index-- > 0;) {
> +    //
> +    // Ignore BSP and APs which not call in SMM.
> +    //
> +    if (!IsPresentAp(Index)) {
> +      continue;
> +    }
> +
> +    if (BlockMode) {
> +      AcquireSpinLock(mSmmMpSyncData->CpuData[Index].Busy);
> +      ReleaseSpinLock(mSmmMpSyncData->CpuData[Index].Busy);
> +    } else {
> +      if (AcquireSpinLockOrFail (mSmmMpSyncData->CpuData[Index].Busy)) {
> +        ReleaseSpinLock(mSmmMpSyncData->CpuData[Index].Busy);
> +      } else {
> +        return FALSE;
> +      }
> +    }
> +  }
> +
> +  return TRUE;
> +}
> +
> +/**
> +  Check whether it is an present AP.
> +
> +  @param   CpuIndex      The AP index which calls this function.
> +
> +  @retval  TRUE           It's a present AP.
> +  @retval  TRUE           This is not an AP or it is not present.
> +
> +**/
> +BOOLEAN
> +IsPresentAp (
> +  IN UINTN        CpuIndex
> +  )
> +{
> +  return ((CpuIndex != gSmmCpuPrivate-
> >SmmCoreEntryContext.CurrentlyExecutingCpu) &&
> +    *(mSmmMpSyncData->CpuData[CpuIndex].Present));
> +}
> +
> +/**
> +  Check whether execute in single AP or all APs.
> +
> +  Compare two Tokens used by different APs to know whether in
> StartAllAps call.
> +
> +  Whether is an valid AP base on AP's Present flag.
> +
> +  @retval  TRUE      IN StartAllAps call.
> +  @retval  FALSE     Not in StartAllAps call.
> +
> +**/
> +BOOLEAN
> +InStartAllApsCall (
> +  VOID
> +  )
> +{
> +  UINTN      ApIndex;
> +  UINTN      ApIndex2;
> +
> +  for (ApIndex = mMaxNumberOfCpus; ApIndex-- > 0;) {
> +    if (IsPresentAp (ApIndex) && (mSmmMpSyncData-
> >CpuData[ApIndex].Token != NULL)) {
> +      for (ApIndex2 = ApIndex; ApIndex2-- > 0;) {
> +        if (IsPresentAp (ApIndex2) && (mSmmMpSyncData-
> >CpuData[ApIndex2].Token != NULL)) {
> +          return mSmmMpSyncData->CpuData[ApIndex2].Token ==
> mSmmMpSyncData->CpuData[ApIndex].Token;
> +        }
> +      }
> +    }
> +  }
> +
> +  return FALSE;
> +}
> +
> +/**
> +  Clean up the status flags used during executing the procedure.
> +
> +  @param   CpuIndex      The AP index which calls this function.
> +
> +**/
> +VOID
> +ReleaseToken (
> +  IN UINTN                  CpuIndex
> +  )
> +{
> +  UINTN                             Index;
> +  BOOLEAN                           Released;
> +
> +  if (InStartAllApsCall ()) {
> +    //
> +    // In Start All APs mode, make sure all APs have finished task.
> +    //
> +    if (WaitForAllAPsNotBusy (FALSE)) {
> +      //
> +      // Clean the flags update in the function call.
> +      //
> +      Released = FALSE;
> +      for (Index = mMaxNumberOfCpus; Index-- > 0;) {
> +        //
> +        // Only In SMM APs need to be clean up.
> +        //
> +        if (mSmmMpSyncData->CpuData[Index].Present &&
> mSmmMpSyncData->CpuData[Index].Token != NULL) {
> +          if (!Released) {
> +            ReleaseSpinLock (mSmmMpSyncData->CpuData[Index].Token);
> +            Released = TRUE;
> +          }
> +          mSmmMpSyncData->CpuData[Index].Token = NULL;
> +        }
> +      }
> +    }
> +  } else {
> +    //
> +    // In single AP mode.
> +    //
> +    if (mSmmMpSyncData->CpuData[CpuIndex].Token != NULL) {
> +      ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Token);
> +      mSmmMpSyncData->CpuData[CpuIndex].Token = NULL;
> +    }
> +  }
> +}
> +
> +/**
> +  Free the tokens in the maintained list.
> +
> +**/
> +VOID
> +FreeTokens (
> +  VOID
> +  )
> +{
> +  LIST_ENTRY            *Link;
> +  PROCEDURE_TOKEN       *ProcToken;
> +
> +  while (!IsListEmpty (&gSmmCpuPrivate->TokenList)) {
> +    Link = GetFirstNode (&gSmmCpuPrivate->TokenList);
> +    ProcToken = PROCEDURE_TOKEN_FROM_LINK (Link);
> +
> +    RemoveEntryList (&ProcToken->Link);
> +
> +    FreePool ((VOID *)ProcToken->ProcedureToken);
> +    FreePool (ProcToken);
> +  }
> +}
> +
>  /**
>    SMI handler for BSP.
> 
> @@ -476,12 +635,7 @@ BSPHandler (
>    //
>    // Make sure all APs have completed their pending none-block tasks
>    //
> -  for (Index = mMaxNumberOfCpus; Index-- > 0;) {
> -    if (Index != CpuIndex && *(mSmmMpSyncData-
> >CpuData[Index].Present)) {
> -      AcquireSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
> -      ReleaseSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
> -    }
> -  }
> +  WaitForAllAPsNotBusy (TRUE);
> 
>    //
>    // Perform the remaining tasks
> @@ -572,6 +726,11 @@ BSPHandler (
>    //
>    WaitForAllAPs (ApCount);
> 
> +  //
> +  // Clean the tokens buffer.
> +  //
> +  FreeTokens ();
> +
>    //
>    // Reset BspIndex to -1, meaning BSP has not been elected.
>    //
> @@ -604,6 +763,7 @@ APHandler (
>    UINT64                            Timer;
>    UINTN                             BspIndex;
>    MTRR_SETTINGS                     Mtrrs;
> +  EFI_STATUS                        ProcedureStatus;
> 
>    //
>    // Timeout BSP
> @@ -730,14 +890,19 @@ APHandler (
>      //
>      // Invoke the scheduled procedure
>      //
> -    (*mSmmMpSyncData->CpuData[CpuIndex].Procedure) (
> -      (VOID*)mSmmMpSyncData->CpuData[CpuIndex].Parameter
> -      );
> +    ProcedureStatus = (*mSmmMpSyncData-
> >CpuData[CpuIndex].Procedure) (
> +                          (VOID*)mSmmMpSyncData->CpuData[CpuIndex].Parameter
> +                          );
> +    if (mSmmMpSyncData->CpuData[CpuIndex].Status != NULL) {
> +      *mSmmMpSyncData->CpuData[CpuIndex].Status = ProcedureStatus;
> +    }
> 
>      //
>      // Release BUSY
>      //
>      ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
> +
> +    ReleaseToken (CpuIndex);
>    }
> 
>    if (SmmCpuFeaturesNeedConfigureMtrrs()) {
> @@ -906,13 +1071,124 @@ Gen4GPageTable (
>    return (UINT32)(UINTN)PageTable;
>  }
> 
> +/**
> +  Checks whether the input token is the current used token.
> +
> +  @param[in]  Token      This parameter describes the token that was passed
> into DispatchProcedure or
> +                         BroadcastProcedure.
> +
> +  @retval TRUE           The input token is the current used token.
> +  @retval FALSE          The input token is not the current used token.
> +**/
> +BOOLEAN
> +IsTokenInUse (
> +  IN SPIN_LOCK           *Token
> +  )
> +{
> +  LIST_ENTRY        *Link;
> +  PROCEDURE_TOKEN   *ProcToken;
> +
> +  if (Token == NULL) {
> +    return FALSE;
> +  }
> +
> +  Link = GetFirstNode (&gSmmCpuPrivate->TokenList);
> +  while (!IsNull (&gSmmCpuPrivate->TokenList, Link)) {
> +    ProcToken = PROCEDURE_TOKEN_FROM_LINK (Link);
> +
> +    if (ProcToken->ProcedureToken == Token) {
> +      return TRUE;
> +    }
> +
> +    Link = GetNextNode (&gSmmCpuPrivate->TokenList, Link);
> +  }
> +
> +  return FALSE;
> +}
> +
> +/**
> +  create token and save it to the maintain list.
> +
> +  @retval    return the spin lock used as token.
> +
> +**/
> +SPIN_LOCK *
> +CreateToken (
> +  VOID
> +  )
> +{
> +  PROCEDURE_TOKEN    *ProcToken;
> +  SPIN_LOCK           *CpuToken;
> +  UINTN               SpinLockSize;
> +
> +  SpinLockSize = GetSpinLockProperties ();
> +  CpuToken = AllocatePool (SpinLockSize);
> +  ASSERT (CpuToken != NULL);
> +  InitializeSpinLock (CpuToken);
> +  AcquireSpinLock (CpuToken);
> +
> +  ProcToken = AllocatePool (sizeof (PROCEDURE_TOKEN));
> +  ASSERT (ProcToken != NULL);
> +  ProcToken->Signature = PROCEDURE_TOKEN_SIGNATURE;
> +  ProcToken->ProcedureToken = CpuToken;
> +
> +  InsertTailList (&gSmmCpuPrivate->TokenList, &ProcToken->Link);
> +
> +  return CpuToken;
> +}
> +
> +/**
> +  Checks status of specified AP.
> +
> +  This function checks whether the specified AP has finished the task
> assigned
> +  by StartupThisAP(), and whether timeout expires.
> +
> +  @param[in]  Token             This parameter describes the token that was
> passed into DispatchProcedure or
> +                                BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS           Specified AP has finished task assigned by
> StartupThisAPs().
> +  @retval EFI_NOT_READY         Specified AP has not finished task and
> timeout has not expired.
> +**/
> +EFI_STATUS
> +IsApReady (
> +  IN SPIN_LOCK          *Token
> +  )
> +{
> +  if (AcquireSpinLockOrFail (Token)) {
> +    ReleaseSpinLock (Token);
> +    return EFI_SUCCESS;
> +  }
> +
> +  return EFI_NOT_READY;
> +}
> +
>  /**
>    Schedule a procedure to run on the specified CPU.
> 
>    @param[in]       Procedure                The address of the procedure to run
>    @param[in]       CpuIndex                 Target CPU Index
> -  @param[in, out]  ProcArguments            The parameter to pass to the
> procedure
> -  @param[in]       BlockingMode             Startup AP in blocking mode or not
> +  @param[in,out]   ProcArguments            The parameter to pass to the
> procedure
> +  @param[in]       Token                    This is an optional parameter that allows
> the caller to execute the
> +                                            procedure in a blocking or non-blocking fashion. If it is
> NULL the
> +                                            call is blocking, and the call will not return until the AP
> has
> +                                            completed the procedure. If the token is not NULL,
> the call will
> +                                            return immediately. The caller can check whether the
> procedure has
> +                                            completed with CheckOnProcedure or
> WaitForProcedure.
> +  @param[in]       TimeoutInMicroseconds    Indicates the time limit in
> microseconds for the APs to finish
> +                                            execution of Procedure, either for blocking or non-
> blocking mode.
> +                                            Zero means infinity. If the timeout expires before all
> APs return
> +                                            from Procedure, then Procedure on the failed APs is
> terminated. If
> +                                            the timeout expires in blocking mode, the call returns
> EFI_TIMEOUT.
> +                                            If the timeout expires in non-blocking mode, the
> timeout determined
> +                                            can be through CheckOnProcedure or
> WaitForProcedure.
> +                                            Note that timeout support is optional. Whether an
> implementation
> +                                            supports this feature can be determined via the
> Attributes data
> +                                            member.
> +  @param[in,out]   CpuStatus                This optional pointer may be used to
> get the status code returned
> +                                            by Procedure when it completes execution on the
> target AP, or with
> +                                            EFI_TIMEOUT if the Procedure fails to complete
> within the optional
> +                                            timeout. The implementation will update this variable
> with
> +                                            EFI_NOT_READY prior to starting Procedure on the
> target AP.
> 
>    @retval EFI_INVALID_PARAMETER    CpuNumber not valid
>    @retval EFI_INVALID_PARAMETER    CpuNumber specifying BSP
> @@ -923,10 +1199,12 @@ Gen4GPageTable (
>  **/
>  EFI_STATUS
>  InternalSmmStartupThisAp (
> -  IN      EFI_AP_PROCEDURE          Procedure,
> -  IN      UINTN                     CpuIndex,
> -  IN OUT  VOID                      *ProcArguments OPTIONAL,
> -  IN      BOOLEAN                   BlockingMode
> +  IN      EFI_AP_PROCEDURE2              Procedure,
> +  IN      UINTN                          CpuIndex,
> +  IN OUT  VOID                           *ProcArguments OPTIONAL,
> +  IN      MM_COMPLETION                  *Token,
> +  IN      UINTN                          TimeoutInMicroseconds,
> +  IN OUT  EFI_STATUS                     *CpuStatus
>    )
>  {
>    if (CpuIndex >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus)
> {
> @@ -952,24 +1230,190 @@ InternalSmmStartupThisAp (
>      }
>      return EFI_INVALID_PARAMETER;
>    }
> +  if ((TimeoutInMicroseconds != 0) && ((mSmmMp.Attributes &
> EFI_MM_MP_TIMEOUT_SUPPORTED) == 0)) {
> +    return EFI_INVALID_PARAMETER;
> +  }
> +  if (Procedure == NULL) {
> +    return EFI_INVALID_PARAMETER;
> +  }
> 
> -  if (BlockingMode) {
> +  if (Token == NULL) {
>      AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
>    } else {
>      if (!AcquireSpinLockOrFail (mSmmMpSyncData-
> >CpuData[CpuIndex].Busy)) {
> -      DEBUG((DEBUG_ERROR, "mSmmMpSyncData->CpuData[%d].Busy\n",
> CpuIndex));
> -      return EFI_INVALID_PARAMETER;
> +      DEBUG((DEBUG_ERROR, "Can't acquire mSmmMpSyncData-
> >CpuData[%d].Busy\n", CpuIndex));
> +      return EFI_NOT_READY;
>      }
> +
> +    *Token = (MM_COMPLETION) CreateToken ();
>    }
> 
>    mSmmMpSyncData->CpuData[CpuIndex].Procedure = Procedure;
>    mSmmMpSyncData->CpuData[CpuIndex].Parameter = ProcArguments;
> +  if (Token != NULL) {
> +    mSmmMpSyncData->CpuData[CpuIndex].Token   = (SPIN_LOCK
> *)(*Token);
> +  }
> +  mSmmMpSyncData->CpuData[CpuIndex].Status    = CpuStatus;
> +  if (mSmmMpSyncData->CpuData[CpuIndex].Status != NULL) {
> +    *mSmmMpSyncData->CpuData[CpuIndex].Status = EFI_NOT_READY;
> +  }
> +
>    ReleaseSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);
> 
> -  if (BlockingMode) {
> +  if (Token == NULL) {
>      AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
>      ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
>    }
> +
> +  return EFI_SUCCESS;
> +}
> +
> +/**
> +  Worker function to execute a caller provided function on all enabled APs.
> +
> +  @param[in]     Procedure               A pointer to the function to be run on
> +                                         enabled APs of the system.
> +  @param[in]     TimeoutInMicroseconds   Indicates the time limit in
> microseconds for
> +                                         APs to return from Procedure, either for
> +                                         blocking or non-blocking mode.
> +  @param[in,out] ProcedureArguments      The parameter passed into
> Procedure for
> +                                         all APs.
> +  @param[in,out] Token                   This is an optional parameter that allows
> the caller to execute the
> +                                         procedure in a blocking or non-blocking fashion. If it is
> NULL the
> +                                         call is blocking, and the call will not return until the AP
> has
> +                                         completed the procedure. If the token is not NULL, the
> call will
> +                                         return immediately. The caller can check whether the
> procedure has
> +                                         completed with CheckOnProcedure or
> WaitForProcedure.
> +  @param[in,out] CPUStatus               This optional pointer may be used to get
> the status code returned
> +                                         by Procedure when it completes execution on the
> target AP, or with
> +                                         EFI_TIMEOUT if the Procedure fails to complete within
> the optional
> +                                         timeout. The implementation will update this variable
> with
> +                                         EFI_NOT_READY prior to starting Procedure on the
> target AP.
> +
> +
> +  @retval EFI_SUCCESS             In blocking mode, all APs have finished before
> +                                  the timeout expired.
> +  @retval EFI_SUCCESS             In non-blocking mode, function has been
> dispatched
> +                                  to all enabled APs.
> +  @retval others                  Failed to Startup all APs.
> +
> +**/
> +EFI_STATUS
> +InternalSmmStartupAllAPs (
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   MM_COMPLETION                 *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +  )
> +{
> +  UINTN               Index;
> +  UINTN               CpuCount;
> +
> +  if ((TimeoutInMicroseconds != 0) && ((mSmmMp.Attributes &
> EFI_MM_MP_TIMEOUT_SUPPORTED) == 0)) {
> +    return EFI_INVALID_PARAMETER;
> +  }
> +  if (Procedure == NULL) {
> +    return EFI_INVALID_PARAMETER;
> +  }
> +
> +  CpuCount = 0;
> +  for (Index = mMaxNumberOfCpus; Index-- > 0;) {
> +    if (IsPresentAp (Index)) {
> +      CpuCount ++;
> +
> +      if (gSmmCpuPrivate->Operation[Index] == SmmCpuRemove) {
> +        return EFI_INVALID_PARAMETER;
> +      }
> +
> +      if (!AcquireSpinLockOrFail(mSmmMpSyncData->CpuData[Index].Busy)) {
> +        return EFI_NOT_READY;
> +      }
> +      ReleaseSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
> +    }
> +  }
> +  if (CpuCount == 0) {
> +    return EFI_NOT_STARTED;
> +  }
> +
> +  if (Token != NULL) {
> +    *Token = (MM_COMPLETION) CreateToken ();
> +  }
> +
> +  //
> +  // Make sure all BUSY should be acquired.
> +  //
> +  // Because former code already check mSmmMpSyncData-
> >CpuData[***].Busy for each AP.
> +  // Here code always use AcquireSpinLock instead of AcquireSpinLockOrFail
> for not
> +  // block mode.
> +  //
> +  for (Index = mMaxNumberOfCpus; Index-- > 0;) {
> +    if (IsPresentAp (Index)) {
> +      AcquireSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
> +    }
> +  }
> +
> +  for (Index = mMaxNumberOfCpus; Index-- > 0;) {
> +    if (IsPresentAp (Index)) {
> +      mSmmMpSyncData->CpuData[Index].Procedure =
> (EFI_AP_PROCEDURE2) Procedure;
> +      mSmmMpSyncData->CpuData[Index].Parameter =
> ProcedureArguments;
> +      if (Token != NULL) {
> +        mSmmMpSyncData->CpuData[Index].Token   = (SPIN_LOCK *)(*Token);
> +      }
> +      if (CPUStatus != NULL) {
> +        mSmmMpSyncData->CpuData[Index].Status    = &CPUStatus[Index];
> +        if (mSmmMpSyncData->CpuData[Index].Status != NULL) {
> +          *mSmmMpSyncData->CpuData[Index].Status = EFI_NOT_READY;
> +        }
> +      }
> +    } else {
> +      //
> +      // PI spec requirement:
> +      // For every excluded processor, the array entry must contain a value of
> EFI_NOT_STARTED.
> +      //
> +      if (CPUStatus != NULL) {
> +        CPUStatus[Index] = EFI_NOT_STARTED;
> +      }
> +    }
> +  }
> +
> +  ReleaseAllAPs ();
> +
> +  if (Token == NULL) {
> +    //
> +    // Make sure all APs have completed their tasks.
> +    //
> +    WaitForAllAPsNotBusy (TRUE);
> +  }
> +
> +  return EFI_SUCCESS;
> +}
> +
> +/**
> +  ISO C99 6.5.2.2 "Function calls", paragraph 9:
> +  If the function is defined with a type that is not compatible with
> +  the type (of the expression) pointed to by the expression that
> +  denotes the called function, the behavior is undefined.
> +
> +  So add below wrapper function to convert between EFI_AP_PROCEDURE
> +  and EFI_AP_PROCEDURE2.
> +
> +  Wrapper for Procedures.
> +
> +  @param[in]  Buffer              Pointer to PROCEDURE_WRAPPER buffer.
> +
> +**/
> +EFI_STATUS
> +EFIAPI
> +ProcedureWrapper (
> +  IN OUT VOID *Buffer
> +  )
> +{
> +  PROCEDURE_WRAPPER *Wrapper;
> +
> +  Wrapper = Buffer;
> +  Wrapper->Procedure (Wrapper->ProcedureArgument);
> +
>    return EFI_SUCCESS;
>  }
> 
> @@ -995,7 +1439,15 @@ SmmBlockingStartupThisAp (
>    IN OUT  VOID                      *ProcArguments OPTIONAL
>    )
>  {
> -  return InternalSmmStartupThisAp(Procedure, CpuIndex, ProcArguments,
> TRUE);
> +  PROCEDURE_WRAPPER  Wrapper;
> +
> +  Wrapper.Procedure = Procedure;
> +  Wrapper.ProcedureArgument = ProcArguments;
> +
> +  //
> +  // Use wrapper function to convert EFI_AP_PROCEDURE to
> EFI_AP_PROCEDURE2.
> +  //
> +  return InternalSmmStartupThisAp (ProcedureWrapper, CpuIndex,
> &Wrapper, NULL, 0, NULL);
>  }
> 
>  /**
> @@ -1020,7 +1472,22 @@ SmmStartupThisAp (
>    IN OUT  VOID                      *ProcArguments OPTIONAL
>    )
>  {
> -  return InternalSmmStartupThisAp(Procedure, CpuIndex, ProcArguments,
> FeaturePcdGet (PcdCpuSmmBlockStartupThisAp));
> +  MM_COMPLETION               Token;
> +
> +  gSmmCpuPrivate->ApWrapperFunc[CpuIndex].Procedure = Procedure;
> +  gSmmCpuPrivate->ApWrapperFunc[CpuIndex].ProcedureArgument =
> ProcArguments;
> +
> +  //
> +  // Use wrapper function to convert EFI_AP_PROCEDURE to
> EFI_AP_PROCEDURE2.
> +  //
> +  return InternalSmmStartupThisAp (
> +    ProcedureWrapper,
> +    CpuIndex,
> +    &gSmmCpuPrivate->ApWrapperFunc[CpuIndex],
> +    FeaturePcdGet (PcdCpuSmmBlockStartupThisAp) ? NULL : &Token,
> +    0,
> +    NULL
> +    );
>  }
> 
>  /**
> @@ -1112,6 +1579,13 @@ SmiRendezvous (
>    Cr2 = 0;
>    SaveCr2 (&Cr2);
> 
> +  //
> +  // Call the user register Startup function first.
> +  //
> +  if (mSmmMpSyncData->StartupProcedure != NULL) {
> +    mSmmMpSyncData->StartupProcedure (mSmmMpSyncData-
> >StartupProcArgs);
> +  }
> +
>    //
>    // Perform CPU specific entry hooks
>    //
> @@ -1256,6 +1730,21 @@ Exit:
>    RestoreCr2 (Cr2);
>  }
> 
> +/**
> +  Allocate buffer for SpinLock and Wrapper function buffer.
> +
> +**/
> +VOID
> +InitializeDataForMmMp (
> +  VOID
> +  )
> +{
> +  gSmmCpuPrivate->ApWrapperFunc = AllocatePool (sizeof
> (PROCEDURE_WRAPPER) * gSmmCpuPrivate-
> >SmmCoreEntryContext.NumberOfCpus);
> +  ASSERT (gSmmCpuPrivate->ApWrapperFunc != NULL);
> +
> +  InitializeListHead (&gSmmCpuPrivate->TokenList);
> +}
> +
>  /**
>    Allocate buffer for all semaphores and spin locks.
> 
> @@ -1469,3 +1958,40 @@ RegisterSmmEntry (
>    gSmmCpuPrivate->SmmCoreEntry = SmmEntryPoint;
>    return EFI_SUCCESS;
>  }
> +
> +/**
> +
> +  Register the SMM Foundation entry point.
> +
> +  @param[in]      Procedure            A pointer to the code stream to be run on
> the designated target AP
> +                                       of the system. Type EFI_AP_PROCEDURE is defined
> below in Volume 2
> +                                       with the related definitions of
> +                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
> +                                       If caller may pass a value of NULL to deregister any
> existing
> +                                       startup procedure.
> +  @param[in]      ProcedureArguments   Allows the caller to pass a list of
> parameters to the code that is
> +                                       run by the AP. It is an optional common mailbox
> between APs and
> +                                       the caller to share information
> +
> +  @retval EFI_SUCCESS                  The Procedure has been set successfully.
> +  @retval EFI_INVALID_PARAMETER        The Procedure is NULL but
> ProcedureArguments not NULL.
> +
> +**/
> +EFI_STATUS
> +RegisterStartupProcedure (
> +  IN EFI_AP_PROCEDURE    Procedure,
> +  IN VOID                *ProcedureArguments OPTIONAL
> +  )
> +{
> +  if (Procedure == NULL && ProcedureArguments != NULL) {
> +    return EFI_INVALID_PARAMETER;
> +  }
> +  if (mSmmMpSyncData == NULL) {
> +    return EFI_NOT_READY;
> +  }
> +
> +  mSmmMpSyncData->StartupProcedure = Procedure;
> +  mSmmMpSyncData->StartupProcArgs  = ProcedureArguments;
> +
> +  return EFI_SUCCESS;
> +}
> diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c
> b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c
> index 2f7d777ee7..69a04dfb23 100644
> --- a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c
> +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c
> @@ -34,6 +34,8 @@ SMM_CPU_PRIVATE_DATA  mSmmCpuPrivateData = {
>      mSmmCpuPrivateData.SmmReservedSmramRegion,  //
> SmmConfiguration.SmramReservedRegions
>      RegisterSmmEntry                            // SmmConfiguration.RegisterSmmEntry
>    },
> +  NULL,                                         // pointer to Ap Wrapper Func array
> +  {NULL, NULL},                                 // List_Entry for Tokens.
>  };
> 
>  CPU_HOT_PLUG_DATA mCpuHotPlugData = {
> @@ -996,6 +998,22 @@ PiCpuSmmEntry (
>                      );
>    ASSERT_EFI_ERROR (Status);
> 
> +  //
> +  // Initialize global buffer for MM MP.
> +  //
> +  InitializeDataForMmMp ();
> +
> +  //
> +  // Install the SMM Mp Protocol into SMM protocol database
> +  //
> +  Status = gSmst->SmmInstallProtocolInterface (
> +                    &mSmmCpuHandle,
> +                    &gEfiMmMpProtocolGuid,
> +                    EFI_NATIVE_INTERFACE,
> +                    &mSmmMp
> +                    );
> +  ASSERT_EFI_ERROR (Status);
> +
>    //
>    // Expose address of CPU Hot Plug Data structure if CPU hot plug is
> supported.
>    //
> diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h
> b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h
> index 2bb35a424d..186809f431 100644
> --- a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h
> +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h
> @@ -20,6 +20,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent
>  #include <Protocol/SmmReadyToLock.h>
>  #include <Protocol/SmmCpuService.h>
>  #include <Protocol/SmmMemoryAttribute.h>
> +#include <Protocol/MmMp.h>
> 
>  #include <Guid/AcpiS3Context.h>
>  #include <Guid/MemoryAttributesTable.h>
> @@ -197,6 +198,25 @@ typedef UINT32
> SMM_CPU_ARRIVAL_EXCEPTIONS;
>  #define ARRIVAL_EXCEPTION_DELAYED           0x2
>  #define ARRIVAL_EXCEPTION_SMI_DISABLED      0x4
> 
> +//
> +// Wrapper used to convert EFI_AP_PROCEDURE2 and EFI_AP_PROCEDURE.
> +//
> +typedef struct {
> +  EFI_AP_PROCEDURE  Procedure;
> +  VOID              *ProcedureArgument;
> +} PROCEDURE_WRAPPER;
> +
> +#define PROCEDURE_TOKEN_SIGNATURE  SIGNATURE_32 ('P', 'R', 'T', 'S')
> +
> +typedef struct {
> +  UINTN                   Signature;
> +  LIST_ENTRY              Link;
> +
> +  SPIN_LOCK               *ProcedureToken;
> +} PROCEDURE_TOKEN;
> +
> +#define PROCEDURE_TOKEN_FROM_LINK(a)  CR (a, PROCEDURE_TOKEN,
> Link, PROCEDURE_TOKEN_SIGNATURE)
> +
>  //
>  // Private structure for the SMM CPU module that is stored in DXE Runtime
> memory
>  // Contains the SMM Configuration Protocols that is produced.
> @@ -219,6 +239,10 @@ typedef struct {
>    EFI_SMM_ENTRY_POINT             SmmCoreEntry;
> 
>    EFI_SMM_CONFIGURATION_PROTOCOL  SmmConfiguration;
> +
> +  PROCEDURE_WRAPPER               *ApWrapperFunc;
> +  LIST_ENTRY                      TokenList;
> +
>  } SMM_CPU_PRIVATE_DATA;
> 
>  extern SMM_CPU_PRIVATE_DATA  *gSmmCpuPrivate;
> @@ -226,6 +250,7 @@ extern CPU_HOT_PLUG_DATA      mCpuHotPlugData;
>  extern UINTN                  mMaxNumberOfCpus;
>  extern UINTN                  mNumberOfCpus;
>  extern EFI_SMM_CPU_PROTOCOL   mSmmCpu;
> +extern EFI_MM_MP_PROTOCOL     mSmmMp;
> 
>  ///
>  /// The mode of the CPU at the time an SMI occurs
> @@ -363,10 +388,12 @@ SmmRelocationSemaphoreComplete (
>  ///
>  typedef struct {
>    SPIN_LOCK                         *Busy;
> -  volatile EFI_AP_PROCEDURE         Procedure;
> +  volatile EFI_AP_PROCEDURE2        Procedure;
>    volatile VOID                     *Parameter;
>    volatile UINT32                   *Run;
>    volatile BOOLEAN                  *Present;
> +  SPIN_LOCK                         *Token;
> +  EFI_STATUS                        *Status;
>  } SMM_CPU_DATA_BLOCK;
> 
>  typedef enum {
> @@ -388,6 +415,8 @@ typedef struct {
>    volatile SMM_CPU_SYNC_MODE    EffectiveSyncMode;
>    volatile BOOLEAN              SwitchBsp;
>    volatile BOOLEAN              *CandidateBsp;
> +  EFI_AP_PROCEDURE              StartupProcedure;
> +  VOID                          *StartupProcArgs;
>  } SMM_DISPATCHER_MP_SYNC_DATA;
> 
>  #define SMM_PSD_OFFSET              0xfb00
> @@ -410,6 +439,7 @@ typedef struct {
>    SPIN_LOCK                         *Busy;
>    volatile UINT32                   *Run;
>    volatile BOOLEAN                  *Present;
> +  SPIN_LOCK                         *Token;
>  } SMM_CPU_SEMAPHORE_CPU;
> 
>  ///
> @@ -1259,4 +1289,165 @@ RestoreCr2 (
>    IN UINTN  Cr2
>    );
> 
> +/**
> +  Schedule a procedure to run on the specified CPU.
> +
> +  @param[in]       Procedure                The address of the procedure to run
> +  @param[in]       CpuIndex                 Target CPU Index
> +  @param[in,out]   ProcArguments            The parameter to pass to the
> procedure
> +  @param[in,out]   Token                    This is an optional parameter that allows
> the caller to execute the
> +                                            procedure in a blocking or non-blocking fashion. If it is
> NULL the
> +                                            call is blocking, and the call will not return until the AP
> has
> +                                            completed the procedure. If the token is not NULL,
> the call will
> +                                            return immediately. The caller can check whether the
> procedure has
> +                                            completed with CheckOnProcedure or
> WaitForProcedure.
> +  @param[in]       TimeoutInMicroseconds    Indicates the time limit in
> microseconds for the APs to finish
> +                                            execution of Procedure, either for blocking or non-
> blocking mode.
> +                                            Zero means infinity. If the timeout expires before all
> APs return
> +                                            from Procedure, then Procedure on the failed APs is
> terminated. If
> +                                            the timeout expires in blocking mode, the call returns
> EFI_TIMEOUT.
> +                                            If the timeout expires in non-blocking mode, the
> timeout determined
> +                                            can be through CheckOnProcedure or
> WaitForProcedure.
> +                                            Note that timeout support is optional. Whether an
> implementation
> +                                            supports this feature can be determined via the
> Attributes data
> +                                            member.
> +  @param[in,out]   CPUStatus                This optional pointer may be used to
> get the status code returned
> +                                            by Procedure when it completes execution on the
> target AP, or with
> +                                            EFI_TIMEOUT if the Procedure fails to complete
> within the optional
> +                                            timeout. The implementation will update this variable
> with
> +                                            EFI_NOT_READY prior to starting Procedure on the
> target AP.
> +
> +  @retval EFI_INVALID_PARAMETER    CpuNumber not valid
> +  @retval EFI_INVALID_PARAMETER    CpuNumber specifying BSP
> +  @retval EFI_INVALID_PARAMETER    The AP specified by CpuNumber did
> not enter SMM
> +  @retval EFI_INVALID_PARAMETER    The AP specified by CpuNumber is
> busy
> +  @retval EFI_SUCCESS              The procedure has been successfully
> scheduled
> +
> +**/
> +EFI_STATUS
> +InternalSmmStartupThisAp (
> +  IN      EFI_AP_PROCEDURE2              Procedure,
> +  IN      UINTN                          CpuIndex,
> +  IN OUT  VOID                           *ProcArguments OPTIONAL,
> +  IN      MM_COMPLETION                  *Token,
> +  IN      UINTN                          TimeoutInMicroseconds,
> +  IN OUT  EFI_STATUS                     *CpuStatus
> +  );
> +
> +/**
> +  Checks whether the input token is the current used token.
> +
> +  @param[in]  Token      This parameter describes the token that was passed
> into DispatchProcedure or
> +                         BroadcastProcedure.
> +
> +  @retval TRUE           The input token is the current used token.
> +  @retval FALSE          The input token is not the current used token.
> +**/
> +BOOLEAN
> +IsTokenInUse (
> +  IN SPIN_LOCK           *Token
> +  );
> +
> +/**
> +  Checks status of specified AP.
> +
> +  This function checks whether the specified AP has finished the task
> assigned
> +  by StartupThisAP(), and whether timeout expires.
> +
> +  @param[in]  Token             This parameter describes the token that was
> passed into DispatchProcedure or
> +                                BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS           Specified AP has finished task assigned by
> StartupThisAPs().
> +  @retval EFI_NOT_READY         Specified AP has not finished task and
> timeout has not expired.
> +**/
> +EFI_STATUS
> +IsApReady (
> +  IN SPIN_LOCK  *Token
> +  );
> +
> +/**
> +  Check whether it is an present AP.
> +
> +  @param   CpuIndex      The AP index which calls this function.
> +
> +  @retval  TRUE           It's a present AP.
> +  @retval  TRUE           This is not an AP or it is not present.
> +
> +**/
> +BOOLEAN
> +IsPresentAp (
> +  IN UINTN        CpuIndex
> +  );
> +
> +/**
> +  Worker function to execute a caller provided function on all enabled APs.
> +
> +  @param[in]     Procedure               A pointer to the function to be run on
> +                                         enabled APs of the system.
> +  @param[in]     TimeoutInMicroseconds   Indicates the time limit in
> microseconds for
> +                                         APs to return from Procedure, either for
> +                                         blocking or non-blocking mode.
> +  @param[in,out] ProcedureArgument       The parameter passed into
> Procedure for
> +                                         all APs.
> +  @param[in,out] Token                   This is an optional parameter that allows
> the caller to execute the
> +                                         procedure in a blocking or non-blocking fashion. If it is
> NULL the
> +                                         call is blocking, and the call will not return until the AP
> has
> +                                         completed the procedure. If the token is not NULL, the
> call will
> +                                         return immediately. The caller can check whether the
> procedure has
> +                                         completed with CheckOnProcedure or
> WaitForProcedure.
> +  @param[in,out] CPUStatus               This optional pointer may be used to get
> the status code returned
> +                                         by Procedure when it completes execution on the
> target AP, or with
> +                                         EFI_TIMEOUT if the Procedure fails to complete within
> the optional
> +                                         timeout. The implementation will update this variable
> with
> +                                         EFI_NOT_READY prior to starting Procedure on the
> target AP.
> +
> +  @retval EFI_SUCCESS             In blocking mode, all APs have finished before
> +                                  the timeout expired.
> +  @retval EFI_SUCCESS             In non-blocking mode, function has been
> dispatched
> +                                  to all enabled APs.
> +  @retval others                  Failed to Startup all APs.
> +
> +**/
> +EFI_STATUS
> +InternalSmmStartupAllAPs (
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   MM_COMPLETION                 *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +  );
> +
> +/**
> +
> +  Register the SMM Foundation entry point.
> +
> +  @param[in]      Procedure            A pointer to the code stream to be run on
> the designated target AP
> +                                       of the system. Type EFI_AP_PROCEDURE is defined
> below in Volume 2
> +                                       with the related definitions of
> +                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
> +                                       If caller may pass a value of NULL to deregister any
> existing
> +                                       startup procedure.
> +  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of
> parameters to the code that is
> +                                       run by the AP. It is an optional common mailbox
> between APs and
> +                                       the caller to share information
> +
> +  @retval EFI_SUCCESS                  The Procedure has been set successfully.
> +  @retval EFI_INVALID_PARAMETER        The Procedure is NULL but
> ProcedureArguments not NULL.
> +
> +**/
> +EFI_STATUS
> +RegisterStartupProcedure (
> +  IN EFI_AP_PROCEDURE    Procedure,
> +  IN VOID                *ProcedureArguments OPTIONAL
> +  );
> +
> +/**
> +  Allocate buffer for SpinLock and Wrapper function buffer.
> +
> +**/
> +VOID
> +InitializeDataForMmMp (
> +  VOID
> +  );
> +
>  #endif
> diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf
> b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf
> index 466c568d49..da0308c47f 100644
> --- a/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf
> +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf
> @@ -40,6 +40,8 @@
>    SmmProfileInternal.h
>    SmramSaveState.c
>    SmmCpuMemoryManagement.c
> +  SmmMp.h
> +  SmmMp.c
> 
>  [Sources.Ia32]
>    Ia32/Semaphore.c
> @@ -105,6 +107,7 @@
>    gEfiSmmReadyToLockProtocolGuid           ## NOTIFY
>    gEfiSmmCpuServiceProtocolGuid            ## PRODUCES
>    gEdkiiSmmMemoryAttributeProtocolGuid     ## PRODUCES
> +  gEfiMmMpProtocolGuid                    ## PRODUCES
> 
>  [Guids]
>    gEfiAcpiVariableGuid                     ## SOMETIMES_CONSUMES ## HOB # it is
> used for S3 boot.
> diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c
> b/UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c
> new file mode 100644
> index 0000000000..9b2b191e03
> --- /dev/null
> +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c
> @@ -0,0 +1,344 @@
> +/** @file
> +SMM MP protocol implementation
> +
> +Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
> +
> +SPDX-License-Identifier: BSD-2-Clause-Patent
> +
> +**/
> +
> +#include "PiSmmCpuDxeSmm.h"
> +#include "SmmMp.h"
> +
> +///
> +/// SMM MP Protocol instance
> +///
> +EFI_MM_MP_PROTOCOL  mSmmMp  = {
> +  EFI_MM_MP_PROTOCOL_REVISION,
> +  0,
> +  SmmMpGetNumberOfProcessors,
> +  SmmMpDispatchProcedure,
> +  SmmMpBroadcastProcedure,
> +  SmmMpSetStartupProcedure,
> +  SmmMpCheckForProcedure,
> +  SmmMpWaitForProcedure
> +};
> +
> +/**
> +  Service to retrieves the number of logical processor in the platform.
> +
> +  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
> +  @param[out] NumberOfProcessors  Pointer to the total number of logical
> processors in the system,
> +                                  including the BSP and all APs.
> +
> +  @retval EFI_SUCCESS             The number of processors was retrieved
> successfully
> +  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpGetNumberOfProcessors (
> +  IN CONST EFI_MM_MP_PROTOCOL   *This,
> +  OUT      UINTN                *NumberOfProcessors
> +  )
> +{
> +  if (NumberOfProcessors == NULL) {
> +    return EFI_INVALID_PARAMETER;
> +  }
> +
> +  *NumberOfProcessors = gSmmCpuPrivate-
> >SmmCoreEntryContext.NumberOfCpus;
> +
> +  return EFI_SUCCESS;
> +}
> +
> +/**
> +  This service allows the caller to invoke a procedure one of the application
> processors (AP). This
> +  function uses an optional token parameter to support blocking and non-
> blocking modes. If the token
> +  is passed into the call, the function will operate in a non-blocking fashion
> and the caller can
> +  check for completion with CheckOnProcedure or WaitForProcedure.
> +
> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]     Procedure              A pointer to the procedure to be run on
> the designated target
> +                                        AP of the system. Type EFI_AP_PROCEDURE2 is defined
> below in
> +                                        related definitions.
> +  @param[in]     CpuNumber              The zero-based index of the processor
> number of the target
> +                                        AP, on which the code stream is supposed to run. If the
> number
> +                                        points to the calling processor then it will not run the
> +                                        supplied code.
> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
> microseconds for this AP to
> +                                        finish execution of Procedure, either for blocking or
> +                                        non-blocking mode. Zero means infinity. If the timeout
> +                                        expires before this AP returns from Procedure, then
> Procedure
> +                                        on the AP is terminated. If the timeout expires in
> blocking
> +                                        mode, the call returns EFI_TIMEOUT. If the timeout
> expires
> +                                        in non-blocking mode, the timeout determined can be
> through
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        Note that timeout support is optional. Whether an
> +                                        implementation supports this feature, can be
> determined via
> +                                        the Attributes data member.
> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
> parameters to the code
> +                                        that is run by the AP. It is an optional common mailbox
> +                                        between APs and the caller to share information.
> +  @param[in,out] Token                  This is parameter is broken into two
> components:
> +                                        1.Token->Completion is an optional parameter that
> allows the
> +                                        caller to execute the procedure in a blocking or non-
> blocking
> +                                        fashion. If it is NULL the call is blocking, and the call will
> +                                        not return until the AP has completed the procedure. If
> the
> +                                        token is not NULL, the call will return immediately. The
> caller
> +                                        can check whether the procedure has completed with
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        2.Token->Status The implementation updates the
> address pointed
> +                                        at by this variable with the status code returned by
> Procedure
> +                                        when it completes execution on the target AP, or with
> EFI_TIMEOUT
> +                                        if the Procedure fails to complete within the optional
> timeout.
> +                                        The implementation will update this variable with
> EFI_NOT_READY
> +                                        prior to starting Procedure on the target AP
> +  @param[in,out] CPUStatus              This optional pointer may be used to get
> the status code returned
> +                                        by Procedure when it completes execution on the
> target AP, or with
> +                                        EFI_TIMEOUT if the Procedure fails to complete within
> the optional
> +                                        timeout. The implementation will update this variable
> with
> +                                        EFI_NOT_READY prior to starting Procedure on the
> target AP.
> +
> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
> Procedure has completed
> +                                        execution on the target AP.
> +                                        In the non-blocking case this indicates that the
> procedure has
> +                                        been successfully scheduled for execution on the target
> AP.
> +  @retval EFI_INVALID_PARAMETER         The input arguments are out of
> range. Either the target AP is the
> +                                        caller of the function, or the Procedure or Token is NULL
> +  @retval EFI_NOT_READY                 If the target AP is busy executing another
> procedure
> +  @retval EFI_ALREADY_STARTED           Token is already in use for another
> procedure
> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
> before the specified AP
> +                                        has finished
> +  @retval EFI_OUT_OF_RESOURCES          Could not allocate a required
> resource.
> +
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpDispatchProcedure (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         CpuNumber,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   MM_COMPLETION                 *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +  )
> +{
> +  return InternalSmmStartupThisAp (
> +    Procedure,
> +    CpuNumber,
> +    ProcedureArguments,
> +    Token,
> +    TimeoutInMicroseconds,
> +    CPUStatus
> +    );
> +}
> +
> +/**
> +  This service allows the caller to invoke a procedure on all running
> application processors (AP)
> +  except the caller. This function uses an optional token parameter to
> support blocking and
> +  nonblocking modes. If the token is passed into the call, the function will
> operate in a non-blocking
> +  fashion and the caller can check for completion with CheckOnProcedure or
> WaitForProcedure.
> +
> +  It is not necessary for the implementation to run the procedure on every
> processor on the platform.
> +  Processors that are powered down in such a way that they cannot respond
> to interrupts, may be
> +  excluded from the broadcast.
> +
> +
> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]     Procedure              A pointer to the code stream to be run on
> the APs that have
> +                                        entered MM. Type EFI_AP_PROCEDURE is defined
> below in related
> +                                        definitions.
> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
> microseconds for the APs to finish
> +                                        execution of Procedure, either for blocking or non-
> blocking mode.
> +                                        Zero means infinity. If the timeout expires before all
> APs return
> +                                        from Procedure, then Procedure on the failed APs is
> terminated. If
> +                                        the timeout expires in blocking mode, the call returns
> EFI_TIMEOUT.
> +                                        If the timeout expires in non-blocking mode, the
> timeout determined
> +                                        can be through CheckOnProcedure or
> WaitForProcedure.
> +                                        Note that timeout support is optional. Whether an
> implementation
> +                                        supports this feature can be determined via the
> Attributes data
> +                                        member.
> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
> parameters to the code
> +                                        that is run by the AP. It is an optional common mailbox
> +                                        between APs and the caller to share information.
> +  @param[in,out] Token                  This is parameter is broken into two
> components:
> +                                        1.Token->Completion is an optional parameter that
> allows the
> +                                        caller to execute the procedure in a blocking or non-
> blocking
> +                                        fashion. If it is NULL the call is blocking, and the call will
> +                                        not return until the AP has completed the procedure. If
> the
> +                                        token is not NULL, the call will return immediately. The
> caller
> +                                        can check whether the procedure has completed with
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        2.Token->Status The implementation updates the
> address pointed
> +                                        at by this variable with the status code returned by
> Procedure
> +                                        when it completes execution on the target AP, or with
> EFI_TIMEOUT
> +                                        if the Procedure fails to complete within the optional
> timeout.
> +                                        The implementation will update this variable with
> EFI_NOT_READY
> +                                        prior to starting Procedure on the target AP
> +  @param[in,out] CPUStatus              This optional pointer may be used to get
> the individual status
> +                                        returned by every AP that participated in the broadcast.
> This
> +                                        parameter if used provides the base address of an array
> to hold
> +                                        the EFI_STATUS value of each AP in the system. The size
> of the
> +                                        array can be ascertained by the
> GetNumberOfProcessors function.
> +                                        As mentioned above, the broadcast may not include
> every processor
> +                                        in the system. Some implementations may exclude
> processors that
> +                                        have been powered down in such a way that they are
> not responsive
> +                                        to interrupts. Additionally the broadcast excludes the
> processor
> +                                        which is making the BroadcastProcedure call. For every
> excluded
> +                                        processor, the array entry must contain a value of
> EFI_NOT_STARTED
> +
> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
> Procedure has completed
> +                                        execution on the APs.
> +                                        In the non-blocking case this indicates that the
> procedure has
> +                                        been successfully scheduled for execution on the APs.
> +  @retval EFI_INVALID_PARAMETER         The Procedure or Token is NULL
> +  @retval EFI_NOT_READY                 If the target AP is busy executing another
> procedure
> +  @retval EFI_ALREADY_STARTED           Token is already in use for another
> procedure
> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
> before the specified AP
> +                                        has finished.
> +  @retval EFI_OUT_OF_RESOURCES          Could not allocate a required
> resource.
> +
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpBroadcastProcedure (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   MM_COMPLETION                 *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +  )
> +{
> +  return InternalSmmStartupAllAPs(
> +    Procedure,
> +    TimeoutInMicroseconds,
> +    ProcedureArguments,
> +    Token,
> +    CPUStatus
> +    );
> +}
> +
> +/**
> +  This service allows the caller to set a startup procedure that will be
> executed when an AP powers
> +  up from a state where core configuration and context is lost. The
> procedure is execution has the
> +  following properties:
> +  1. The procedure executes before the processor is handed over to the
> operating system.
> +  2. All processors execute the same startup procedure.
> +  3. The procedure may run in parallel with other procedures invoked
> through the functions in this
> +  protocol, or with processors that are executing an MM handler or running
> in the operating system.
> +
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Procedure            A pointer to the code stream to be run on
> the designated target AP
> +                                       of the system. Type EFI_AP_PROCEDURE is defined
> below in Volume 2
> +                                       with the related definitions of
> +                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
> +                                       If caller may pass a value of NULL to deregister any
> existing
> +                                       startup procedure.
> +  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of
> parameters to the code that is
> +                                       run by the AP. It is an optional common mailbox
> between APs and
> +                                       the caller to share information
> +
> +  @retval EFI_SUCCESS                  The Procedure has been set successfully.
> +  @retval EFI_INVALID_PARAMETER        The Procedure is NULL but
> ProcedureArguments not NULL.
> +
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpSetStartupProcedure (
> +  IN CONST EFI_MM_MP_PROTOCOL  *This,
> +  IN       EFI_AP_PROCEDURE    Procedure,
> +  IN OUT   VOID                *ProcedureArguments OPTIONAL
> +  )
> +{
> +  return RegisterStartupProcedure (Procedure, ProcedureArguments);
> +}
> +
> +/**
> +  When non-blocking execution of a procedure on an AP is invoked with
> DispatchProcedure,
> +  via the use of a token, this function can be used to check for completion of
> the procedure on the AP.
> +  The function takes the token that was passed into the DispatchProcedure
> call. If the procedure
> +  is complete, and therefore it is now possible to run another procedure on
> the same AP, this function
> +  returns EFI_SUCESS. In this case the status returned by the procedure that
> executed on the AP is
> +  returned in the token's Status field. If the procedure has not yet
> completed, then this function
> +  returns EFI_NOT_READY.
> +
> +  When a non-blocking execution of a procedure is invoked with
> BroadcastProcedure, via the
> +  use of a token, this function can be used to check for completion of the
> procedure on all the
> +  broadcast APs. The function takes the token that was passed into the
> BroadcastProcedure
> +  call. If the procedure is complete on all broadcast APs this function returns
> EFI_SUCESS. In this
> +  case the Status field in the token passed into the function reflects the
> overall result of the
> +  invocation, which may be EFI_SUCCESS, if all executions succeeded, or the
> first observed failure.
> +  If the procedure has not yet completed on the broadcast APs, the function
> returns
> +  EFI_NOT_READY.
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Token                This parameter describes the token that was
> passed into
> +                                       DispatchProcedure or BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS                  Procedure has completed.
> +  @retval EFI_NOT_READY                The Procedure has not completed.
> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is
> NULL
> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-
> blocking call
> +
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpCheckForProcedure (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       MM_COMPLETION                 Token
> +  )
> +{
> +  if (Token == NULL) {
> +    return EFI_INVALID_PARAMETER;
> +  }
> +
> +  if (!IsTokenInUse ((SPIN_LOCK *)Token)) {
> +    return EFI_NOT_FOUND;
> +  }
> +
> +  return IsApReady ((SPIN_LOCK *)Token);
> +}
> +
> +/**
> +  When a non-blocking execution of a procedure on an AP is invoked via
> DispatchProcedure,
> +  this function will block the caller until the remote procedure has completed
> on the designated AP.
> +  The non-blocking procedure invocation is identified by the Token
> parameter, which must match the
> +  token that used when DispatchProcedure was called. Upon completion the
> status returned by
> +  the procedure that executed on the AP is used to update the token's
> Status field.
> +
> +  When a non-blocking execution of a procedure on an AP is invoked via
> BroadcastProcedure
> +  this function will block the caller until the remote procedure has completed
> on all of the APs that
> +  entered MM. The non-blocking procedure invocation is identified by the
> Token parameter, which
> +  must match the token that used when BroadcastProcedure was called.
> Upon completion the
> +  overall status returned by the procedures that executed on the broadcast
> AP is used to update the
> +  token's Status field. The overall status may be EFI_SUCCESS, if all
> executions succeeded, or the
> +  first observed failure.
> +
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Token                This parameter describes the token that was
> passed into
> +                                       DispatchProcedure or BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS                  Procedure has completed.
> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is
> NULL
> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-
> blocking call
> +
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpWaitForProcedure (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       MM_COMPLETION                 Token
> +  )
> +{
> +  EFI_STATUS    Status;
> +
> +  do {
> +    Status = SmmMpCheckForProcedure (This, Token);
> +  } while (Status == EFI_NOT_READY);
> +
> +  return Status;
> +}
> +
> diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h
> b/UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h
> new file mode 100644
> index 0000000000..e0d823a4b1
> --- /dev/null
> +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h
> @@ -0,0 +1,286 @@
> +/** @file
> +Include file for SMM MP protocol implementation.
> +
> +Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
> +
> +SPDX-License-Identifier: BSD-2-Clause-Patent
> +
> +**/
> +
> +#ifndef _SMM_MP_PROTOCOL_H_
> +#define _SMM_MP_PROTOCOL_H_
> +
> +//
> +// SMM MP Protocol function prototypes.
> +//
> +
> +/**
> +  Service to retrieves the number of logical processor in the platform.
> +
> +  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
> +  @param[out] NumberOfProcessors  Pointer to the total number of logical
> processors in the system,
> +                                  including the BSP and all APs.
> +
> +  @retval EFI_SUCCESS             The number of processors was retrieved
> successfully
> +  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
> +**/
> +
> +EFI_STATUS
> +EFIAPI
> +SmmMpGetNumberOfProcessors (
> +  IN CONST EFI_MM_MP_PROTOCOL   *This,
> +  OUT      UINTN                *NumberOfProcessors
> +  );
> +
> +
> +/**
> +  This service allows the caller to invoke a procedure one of the application
> processors (AP). This
> +  function uses an optional token parameter to support blocking and non-
> blocking modes. If the token
> +  is passed into the call, the function will operate in a non-blocking fashion
> and the caller can
> +  check for completion with CheckOnProcedure or WaitForProcedure.
> +
> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]     Procedure              A pointer to the procedure to be run on
> the designated target
> +                                        AP of the system. Type EFI_AP_PROCEDURE2 is defined
> below in
> +                                        related definitions.
> +  @param[in]     CpuNumber              The zero-based index of the processor
> number of the target
> +                                        AP, on which the code stream is supposed to run. If the
> number
> +                                        points to the calling processor then it will not run the
> +                                        supplied code.
> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
> microseconds for this AP to
> +                                        finish execution of Procedure, either for blocking or
> +                                        non-blocking mode. Zero means infinity. If the timeout
> +                                        expires before this AP returns from Procedure, then
> Procedure
> +                                        on the AP is terminated. If the timeout expires in
> blocking
> +                                        mode, the call returns EFI_TIMEOUT. If the timeout
> expires
> +                                        in non-blocking mode, the timeout determined can be
> through
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        Note that timeout support is optional. Whether an
> +                                        implementation supports this feature, can be
> determined via
> +                                        the Attributes data member.
> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
> parameters to the code
> +                                        that is run by the AP. It is an optional common mailbox
> +                                        between APs and the caller to share information.
> +  @param[in,out] Token                  This is parameter is broken into two
> components:
> +                                        1.Token->Completion is an optional parameter that
> allows the
> +                                        caller to execute the procedure in a blocking or non-
> blocking
> +                                        fashion. If it is NULL the call is blocking, and the call will
> +                                        not return until the AP has completed the procedure. If
> the
> +                                        token is not NULL, the call will return immediately. The
> caller
> +                                        can check whether the procedure has completed with
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        2.Token->Status The implementation updates the
> address pointed
> +                                        at by this variable with the status code returned by
> Procedure
> +                                        when it completes execution on the target AP, or with
> EFI_TIMEOUT
> +                                        if the Procedure fails to complete within the optional
> timeout.
> +                                        The implementation will update this variable with
> EFI_NOT_READY
> +                                        prior to starting Procedure on the target AP
> +  @param[in,out] CPUStatus              This optional pointer may be used to get
> the status code returned
> +                                        by Procedure when it completes execution on the
> target AP, or with
> +                                        EFI_TIMEOUT if the Procedure fails to complete within
> the optional
> +                                        timeout. The implementation will update this variable
> with
> +                                        EFI_NOT_READY prior to starting Procedure on the
> target AP.
> +
> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
> Procedure has completed
> +                                        execution on the target AP.
> +                                        In the non-blocking case this indicates that the
> procedure has
> +                                        been successfully scheduled for execution on the target
> AP.
> +  @retval EFI_INVALID_PARAMETER         The input arguments are out of
> range. Either the target AP is the
> +                                        caller of the function, or the Procedure or Token is NULL
> +  @retval EFI_NOT_READY                 If the target AP is busy executing another
> procedure
> +  @retval EFI_ALREADY_STARTED           Token is already in use for another
> procedure
> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
> before the specified AP
> +                                        has finished
> +  @retval EFI_OUT_OF_RESOURCES          Could not allocate a required
> resource.
> +
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpDispatchProcedure (
> +  IN CONST EFI_MM_MP_PROTOCOL             *This,
> +  IN       EFI_AP_PROCEDURE2              Procedure,
> +  IN       UINTN                          CpuNumber,
> +  IN       UINTN                          TimeoutInMicroseconds,
> +  IN OUT   VOID                           *ProcedureArguments OPTIONAL,
> +  IN OUT   MM_COMPLETION                  *Token,
> +  IN OUT   EFI_STATUS                     *CPUStatus
> +  );
> +
> +/**
> +  This service allows the caller to invoke a procedure on all running
> application processors (AP)
> +  except the caller. This function uses an optional token parameter to
> support blocking and
> +  nonblocking modes. If the token is passed into the call, the function will
> operate in a non-blocking
> +  fashion and the caller can check for completion with CheckOnProcedure or
> WaitForProcedure.
> +
> +  It is not necessary for the implementation to run the procedure on every
> processor on the platform.
> +  Processors that are powered down in such a way that they cannot respond
> to interrupts, may be
> +  excluded from the broadcast.
> +
> +
> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]     Procedure              A pointer to the code stream to be run on
> the APs that have
> +                                        entered MM. Type EFI_AP_PROCEDURE is defined
> below in related
> +                                        definitions.
> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
> microseconds for the APs to finish
> +                                        execution of Procedure, either for blocking or non-
> blocking mode.
> +                                        Zero means infinity. If the timeout expires before all
> APs return
> +                                        from Procedure, then Procedure on the failed APs is
> terminated. If
> +                                        the timeout expires in blocking mode, the call returns
> EFI_TIMEOUT.
> +                                        If the timeout expires in non-blocking mode, the
> timeout determined
> +                                        can be through CheckOnProcedure or
> WaitForProcedure.
> +                                        Note that timeout support is optional. Whether an
> implementation
> +                                        supports this feature can be determined via the
> Attributes data
> +                                        member.
> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
> parameters to the code
> +                                        that is run by the AP. It is an optional common mailbox
> +                                        between APs and the caller to share information.
> +  @param[in,out] Token                  This is parameter is broken into two
> components:
> +                                        1.Token->Completion is an optional parameter that
> allows the
> +                                        caller to execute the procedure in a blocking or non-
> blocking
> +                                        fashion. If it is NULL the call is blocking, and the call will
> +                                        not return until the AP has completed the procedure. If
> the
> +                                        token is not NULL, the call will return immediately. The
> caller
> +                                        can check whether the procedure has completed with
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        2.Token->Status The implementation updates the
> address pointed
> +                                        at by this variable with the status code returned by
> Procedure
> +                                        when it completes execution on the target AP, or with
> EFI_TIMEOUT
> +                                        if the Procedure fails to complete within the optional
> timeout.
> +                                        The implementation will update this variable with
> EFI_NOT_READY
> +                                        prior to starting Procedure on the target AP
> +  @param[in,out] CPUStatus              This optional pointer may be used to get
> the individual status
> +                                        returned by every AP that participated in the broadcast.
> This
> +                                        parameter if used provides the base address of an array
> to hold
> +                                        the EFI_STATUS value of each AP in the system. The size
> of the
> +                                        array can be ascertained by the
> GetNumberOfProcessors function.
> +                                        As mentioned above, the broadcast may not include
> every processor
> +                                        in the system. Some implementations may exclude
> processors that
> +                                        have been powered down in such a way that they are
> not responsive
> +                                        to interrupts. Additionally the broadcast excludes the
> processor
> +                                        which is making the BroadcastProcedure call. For every
> excluded
> +                                        processor, the array entry must contain a value of
> EFI_NOT_STARTED
> +
> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
> Procedure has completed
> +                                        execution on the APs.
> +                                        In the non-blocking case this indicates that the
> procedure has
> +                                        been successfully scheduled for execution on the APs.
> +  @retval EFI_INVALID_PARAMETER         The Procedure or Token is NULL
> +  @retval EFI_NOT_READY                 If the target AP is busy executing another
> procedure
> +  @retval EFI_ALREADY_STARTED           Token is already in use for another
> procedure
> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
> before the specified AP
> +                                        has finished
> +  @retval EFI_OUT_OF_RESOURCES          Could not allocate a required
> resource.
> +
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpBroadcastProcedure (
> +  IN CONST EFI_MM_MP_PROTOCOL             *This,
> +  IN       EFI_AP_PROCEDURE2              Procedure,
> +  IN       UINTN                          TimeoutInMicroseconds,
> +  IN OUT   VOID                           *ProcedureArguments OPTIONAL,
> +  IN OUT   MM_COMPLETION                  *Token,
> +  IN OUT   EFI_STATUS                     *CPUStatus
> +  );
> +
> +
> +/**
> +  This service allows the caller to set a startup procedure that will be
> executed when an AP powers
> +  up from a state where core configuration and context is lost. The
> procedure is execution has the
> +  following properties:
> +  1. The procedure executes before the processor is handed over to the
> operating system.
> +  2. All processors execute the same startup procedure.
> +  3. The procedure may run in parallel with other procedures invoked
> through the functions in this
> +  protocol, or with processors that are executing an MM handler or running
> in the operating system.
> +
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Procedure            A pointer to the code stream to be run on
> the designated target AP
> +                                       of the system. Type EFI_AP_PROCEDURE is defined
> below in Volume 2
> +                                       with the related definitions of
> +                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
> +                                       If caller may pass a value of NULL to deregister any
> existing
> +                                       startup procedure.
> +  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of
> parameters to the code that is
> +                                       run by the AP. It is an optional common mailbox
> between APs and
> +                                       the caller to share information
> +
> +  @retval EFI_SUCCESS                  The Procedure has been set successfully.
> +  @retval EFI_INVALID_PARAMETER        The Procedure is NULL but
> ProcedureArguments not NULL.
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpSetStartupProcedure (
> +  IN CONST EFI_MM_MP_PROTOCOL  *This,
> +  IN       EFI_AP_PROCEDURE    Procedure,
> +  IN OUT   VOID                *ProcedureArguments OPTIONAL
> +  );
> +
> +/**
> +  When non-blocking execution of a procedure on an AP is invoked with
> DispatchProcedure,
> +  via the use of a token, this function can be used to check for completion of
> the procedure on the AP.
> +  The function takes the token that was passed into the DispatchProcedure
> call. If the procedure
> +  is complete, and therefore it is now possible to run another procedure on
> the same AP, this function
> +  returns EFI_SUCESS. In this case the status returned by the procedure that
> executed on the AP is
> +  returned in the token's Status field. If the procedure has not yet
> completed, then this function
> +  returns EFI_NOT_READY.
> +
> +  When a non-blocking execution of a procedure is invoked with
> BroadcastProcedure, via the
> +  use of a token, this function can be used to check for completion of the
> procedure on all the
> +  broadcast APs. The function takes the token that was passed into the
> BroadcastProcedure
> +  call. If the procedure is complete on all broadcast APs this function returns
> EFI_SUCESS. In this
> +  case the Status field in the token passed into the function reflects the
> overall result of the
> +  invocation, which may be EFI_SUCCESS, if all executions succeeded, or the
> first observed failure.
> +  If the procedure has not yet completed on the broadcast APs, the function
> returns
> +  EFI_NOT_READY.
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Token                This parameter describes the token that was
> passed into
> +                                       DispatchProcedure or BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS                  Procedure has completed.
> +  @retval EFI_NOT_READY                The Procedure has not completed.
> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is
> NULL
> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-
> blocking call
> +
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpCheckForProcedure (
> +  IN CONST EFI_MM_MP_PROTOCOL             *This,
> +  IN       MM_COMPLETION                  Token
> +  );
> +
> +/**
> +  When a non-blocking execution of a procedure on an AP is invoked via
> DispatchProcedure,
> +  this function will block the caller until the remote procedure has completed
> on the designated AP.
> +  The non-blocking procedure invocation is identified by the Token
> parameter, which must match the
> +  token that used when DispatchProcedure was called. Upon completion the
> status returned by
> +  the procedure that executed on the AP is used to update the token's
> Status field.
> +
> +  When a non-blocking execution of a procedure on an AP is invoked via
> BroadcastProcedure
> +  this function will block the caller until the remote procedure has completed
> on all of the APs that
> +  entered MM. The non-blocking procedure invocation is identified by the
> Token parameter, which
> +  must match the token that used when BroadcastProcedure was called.
> Upon completion the
> +  overall status returned by the procedures that executed on the broadcast
> AP is used to update the
> +  token's Status field. The overall status may be EFI_SUCCESS, if all
> executions succeeded, or the
> +  first observed failure.
> +
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Token                This parameter describes the token that was
> passed into
> +                                       DispatchProcedure or BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS                  Procedure has completed.
> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is
> NULL
> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-
> blocking call
> +
> +**/
> +EFI_STATUS
> +EFIAPI
> +SmmMpWaitForProcedure (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       MM_COMPLETION                 Token
> +  );
> +
> +#endif
> --
> 2.21.0.windows.1

Re: [edk2-devel] [Patch v5 2/2] UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP Protocol

[Laszlo Ersek]

Hello Eric,

On 07/10/19 09:56, Dong, Eric wrote:
> V5 changes:
> 1. some small enhancement.
>
> v4 changes:
> 1. Use link list to save the token info.
>
> v3 changes:
> 1. Fix Token clean up too early caused CheckProcedure return error.
>
> v2 changes:
> 1. Remove some duplicated global variables.
> 2. Enhance token design to support multiple task trig for different APs at the same time.
>
> V1 changes:
> REF: https://bugzilla.tianocore.org/show_bug.cgi?id=1937
>
> Add MM Mp Protocol in PiSmmCpuDxeSmm driver.
>
> Cc: Ray Ni <ray.ni@intel.com>
> Cc: Laszlo Ersek <lersek@redhat.com>
> Signed-off-by: Eric Dong <eric.dong@intel.com>
> ---
>  UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c        | 570 ++++++++++++++++++-
>  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c   |  18 +
>  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h   | 193 ++++++-
>  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf |   3 +
>  UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c            | 344 +++++++++++
>  UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h            | 286 ++++++++++
>  6 files changed, 1391 insertions(+), 23 deletions(-)
>  create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c
>  create mode 100644 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h
>
> diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> index 64fb4d6344..f09e2738c3 100644
> --- a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> @@ -140,7 +140,7 @@ ReleaseAllAPs (
>
>    BspIndex = mSmmMpSyncData->BspIndex;
>    for (Index = mMaxNumberOfCpus; Index-- > 0;) {
> -    if (Index != BspIndex && *(mSmmMpSyncData->CpuData[Index].Present)) {
> +    if (IsPresentAp (Index)) {
>        ReleaseSemaphore (mSmmMpSyncData->CpuData[Index].Run);
>      }
>    }

version 5 again fails to build for me, with the following error message:

> UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c: In function 'ReleaseAllAPs':
> UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c:139:37: error: variable 'BspIndex' set but not used [-Werror=unused-but-set-variable]

With the following incremental patch:

> diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> index f09e2738c30d..ef16997547b8 100644
> --- a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> @@ -136,9 +136,7 @@ ReleaseAllAPs (
>    )
>  {
>    UINTN                             Index;
> -  UINTN                             BspIndex;
>
> -  BspIndex = mSmmMpSyncData->BspIndex;
>    for (Index = mMaxNumberOfCpus; Index-- > 0;) {
>      if (IsPresentAp (Index)) {
>        ReleaseSemaphore (mSmmMpSyncData->CpuData[Index].Run);

the build completes fine (using GCC48).

If you change nothing on the series other than squashing the above fix,
you can add my:

Regression-tested-by: Laszlo Ersek <lersek@redhat.com>

to both patches in the series.

-*-

Important: please do not push the series until the 5-level paging
commits are reverted, and reapplied (with Mike's R-b on the MdePkg
patch).

Thanks!
Laszlo

Re: [edk2-devel] [Patch v5 2/2] UefiCpuPkg/PiSmmCpuDxeSmm: Enable MM MP Protocol

[Dong, Eric]

Hi Laszlo,

Thanks for your patch and regression test. I will include your change when I push the patches.

Thanks,
Eric

> -----Original Message-----
> From: Laszlo Ersek [mailto:lersek@redhat.com]
> Sent: Friday, July 12, 2019 1:54 AM
> To: devel@edk2.groups.io; Dong, Eric <eric.dong@intel.com>
> Cc: Ni, Ray <ray.ni@intel.com>
> Subject: Re: [edk2-devel] [Patch v5 2/2] UefiCpuPkg/PiSmmCpuDxeSmm:
> Enable MM MP Protocol
> 
> Hello Eric,
> 
> On 07/10/19 09:56, Dong, Eric wrote:
> > V5 changes:
> > 1. some small enhancement.
> >
> > v4 changes:
> > 1. Use link list to save the token info.
> >
> > v3 changes:
> > 1. Fix Token clean up too early caused CheckProcedure return error.
> >
> > v2 changes:
> > 1. Remove some duplicated global variables.
> > 2. Enhance token design to support multiple task trig for different APs at
> the same time.
> >
> > V1 changes:
> > REF: https://bugzilla.tianocore.org/show_bug.cgi?id=1937
> >
> > Add MM Mp Protocol in PiSmmCpuDxeSmm driver.
> >
> > Cc: Ray Ni <ray.ni@intel.com>
> > Cc: Laszlo Ersek <lersek@redhat.com>
> > Signed-off-by: Eric Dong <eric.dong@intel.com>
> > ---
> >  UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c        | 570
> ++++++++++++++++++-
> >  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.c   |  18 +
> >  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.h   | 193 ++++++-
> >  UefiCpuPkg/PiSmmCpuDxeSmm/PiSmmCpuDxeSmm.inf |   3 +
> >  UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c            | 344 +++++++++++
> >  UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h            | 286 ++++++++++
> >  6 files changed, 1391 insertions(+), 23 deletions(-)  create mode
> > 100644 UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.c  create mode 100644
> > UefiCpuPkg/PiSmmCpuDxeSmm/SmmMp.h
> >
> > diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> > b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> > index 64fb4d6344..f09e2738c3 100644
> > --- a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> > +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> > @@ -140,7 +140,7 @@ ReleaseAllAPs (
> >
> >    BspIndex = mSmmMpSyncData->BspIndex;
> >    for (Index = mMaxNumberOfCpus; Index-- > 0;) {
> > -    if (Index != BspIndex && *(mSmmMpSyncData-
> >CpuData[Index].Present)) {
> > +    if (IsPresentAp (Index)) {
> >        ReleaseSemaphore (mSmmMpSyncData->CpuData[Index].Run);
> >      }
> >    }
> 
> version 5 again fails to build for me, with the following error message:
> 
> > UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c: In function 'ReleaseAllAPs':
> > UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c:139:37: error: variable
> > 'BspIndex' set but not used [-Werror=unused-but-set-variable]
> 
> With the following incremental patch:
> 
> > diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> > b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> > index f09e2738c30d..ef16997547b8 100644
> > --- a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> > +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
> > @@ -136,9 +136,7 @@ ReleaseAllAPs (
> >    )
> >  {
> >    UINTN                             Index;
> > -  UINTN                             BspIndex;
> >
> > -  BspIndex = mSmmMpSyncData->BspIndex;
> >    for (Index = mMaxNumberOfCpus; Index-- > 0;) {
> >      if (IsPresentAp (Index)) {
> >        ReleaseSemaphore (mSmmMpSyncData->CpuData[Index].Run);
> 
> the build completes fine (using GCC48).
> 
> If you change nothing on the series other than squashing the above fix, you
> can add my:
> 
> Regression-tested-by: Laszlo Ersek <lersek@redhat.com>
> 
> to both patches in the series.
> 
> -*-
> 
> Important: please do not push the series until the 5-level paging commits are
> reverted, and reapplied (with Mike's R-b on the MdePkg patch).
> 
> Thanks!
> Laszlo

Re: [edk2-devel] [Patch v5 0/2] Enable new MM MP protocol

[Laszlo Ersek]

On 07/10/19 09:56, Dong, Eric wrote:
> V5:
> 1. Some small enhancement.
> 
> V4 changes:
> 1. Use link list to save the used tokens.
> 
> V3 changes:
> 1. Fix Token clean up too early caused CheckProcedure return error.
> 
> V1 changes:
> RFC: https://bugzilla.tianocore.org/show_bug.cgi?id=1937
> 
> PI spec added a new protocol named MM MP protocol. This protocol allows for
> better remote queuing of execution of procedures on an AP.
> This extends the existing procedures to allow:
> 1. A function to be called in blocking and non-blocking manner explicitly 
> 2. Allow broadcasts.
> 3. Allow execution of a procedure when a processor powers up.
> 
> This patch serial enable this new protocol.
> 
> Cc: Ray Ni <ray.ni@intel.com>
> Cc: Laszlo Ersek <lersek@redhat.com>

Thanks -- given Ray's approval, I plan to regression test this version
later this week.

Cheers
Laszlo

Re: [edk2-devel] [Patch v5 1/2] MdePkg: Add new MM MP Protocol definition.

[Dong, Eric]

Hi Liming,  Mike,

Can you help to review this patch? 

Thanks,
Eric

> -----Original Message-----
> From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of
> Dong, Eric
> Sent: Wednesday, July 10, 2019 3:56 PM
> To: devel@edk2.groups.io
> Cc: Ni, Ray <ray.ni@intel.com>; Laszlo Ersek <lersek@redhat.com>
> Subject: [edk2-devel] [Patch v5 1/2] MdePkg: Add new MM MP Protocol
> definition.
> 
> REF: https://bugzilla.tianocore.org/show_bug.cgi?id=1937
> 
> EFI MM MP Protocol is defined in the PI 1.5 specification.
> 
> The MM MP protocol provides a set of functions to allow execution of
> procedures on processors that have entered MM. This protocol has the
> following properties:
> 1. The caller can invoke execution of a procedure on a processor, other than
> the caller, that has also entered MM. Supports blocking and non-blocking
> modes of operation.
> 2. The caller can invoke a procedure on multiple processors. Supports
> blocking and non-blocking modes of operation.
> 
> Cc: Ray Ni <ray.ni@intel.com>
> Cc: Laszlo Ersek <lersek@redhat.com>
> Signed-off-by: Eric Dong <eric.dong@intel.com>
> Reviewed-by: Ray Ni <ray.ni@intel.com>
> ---
>  MdePkg/Include/Pi/PiMultiPhase.h |  16 ++
>  MdePkg/Include/Protocol/MmMp.h   | 333
> +++++++++++++++++++++++++++++++
>  MdePkg/MdePkg.dec                |   3 +
>  3 files changed, 352 insertions(+)
>  create mode 100644 MdePkg/Include/Protocol/MmMp.h
> 
> diff --git a/MdePkg/Include/Pi/PiMultiPhase.h
> b/MdePkg/Include/Pi/PiMultiPhase.h
> index eb12527767..a5056799e1 100644
> --- a/MdePkg/Include/Pi/PiMultiPhase.h
> +++ b/MdePkg/Include/Pi/PiMultiPhase.h
> @@ -176,4 +176,20 @@ VOID
>    IN OUT VOID  *Buffer
>    );
> 
> +/**
> +  The function prototype for invoking a function on an Application Processor.
> +
> +  This definition is used by the UEFI MM MP Serices Protocol.
> +
> +  @param[in] ProcedureArgument    The pointer to private data buffer.
> +
> +  @retval EFI_SUCCESS             Excutive the procedure successfully
> +
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_AP_PROCEDURE2)(
> +  IN VOID  *ProcedureArgument
> +);
> +
>  #endif
> diff --git a/MdePkg/Include/Protocol/MmMp.h
> b/MdePkg/Include/Protocol/MmMp.h new file mode 100644 index
> 0000000000..beace1386c
> --- /dev/null
> +++ b/MdePkg/Include/Protocol/MmMp.h
> @@ -0,0 +1,333 @@
> +/** @file
> +  EFI MM MP Protocol is defined in the PI 1.5 specification.
> +
> +  The MM MP protocol provides a set of functions to allow execution of
> + procedures on processors that  have entered MM. This protocol has the
> following properties:
> +  1. The caller can only invoke execution of a procedure on a processor,
> other than the caller, that
> +     has also entered MM.
> +  2. It is possible to invoke a procedure on multiple processors. Supports
> blocking and non-blocking
> +     modes of operation.
> +
> +  Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
> +  SPDX-License-Identifier: BSD-2-Clause-Patent
> +
> +**/
> +
> +#ifndef _MM_MP_H_
> +#define _MM_MP_H_
> +
> +#include <Pi/PiMmCis.h>
> +
> +#define EFI_MM_MP_PROTOCOL_GUID \
> +  { \
> +    0x5d5450d7, 0x990c, 0x4180, {0xa8, 0x3, 0x8e, 0x63, 0xf0, 0x60,
> +0x83, 0x7  }  \
> +  }
> +
> +//
> +// Revision definition.
> +//
> +#define EFI_MM_MP_PROTOCOL_REVISION    0x00
> +
> +//
> +// Attribute flags
> +//
> +#define EFI_MM_MP_TIMEOUT_SUPPORTED    0x01
> +
> +//
> +// Completion token
> +//
> +typedef VOID* MM_COMPLETION;
> +
> +typedef struct {
> +  MM_COMPLETION  Completion;
> +  EFI_STATUS     Status;
> +} MM_DISPATCH_COMPLETION_TOKEN;
> +
> +typedef struct _EFI_MM_MP_PROTOCOL  EFI_MM_MP_PROTOCOL;
> +
> +/**
> +  Service to retrieves the number of logical processor in the platform.
> +
> +  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
> +  @param[out] NumberOfProcessors  Pointer to the total number of logical
> processors in the system,
> +                                  including the BSP and all APs.
> +
> +  @retval EFI_SUCCESS             The number of processors was retrieved
> successfully
> +  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_MM_GET_NUMBER_OF_PROCESSORS) (
> +  IN CONST EFI_MM_MP_PROTOCOL  *This,
> +  OUT      UINTN               *NumberOfProcessors
> +);
> +
> +
> +/**
> +  This service allows the caller to invoke a procedure one of the
> +application processors (AP). This
> +  function uses an optional token parameter to support blocking and
> +non-blocking modes. If the token
> +  is passed into the call, the function will operate in a non-blocking
> +fashion and the caller can
> +  check for completion with CheckOnProcedure or WaitForProcedure.
> +
> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]     Procedure              A pointer to the procedure to be run on
> the designated target
> +                                        AP of the system. Type EFI_AP_PROCEDURE2 is defined
> below in
> +                                        related definitions.
> +  @param[in]     CpuNumber              The zero-based index of the processor
> number of the target
> +                                        AP, on which the code stream is supposed to run. If the
> number
> +                                        points to the calling processor then it will not run the
> +                                        supplied code.
> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
> microseconds for this AP to
> +                                        finish execution of Procedure, either for blocking or
> +                                        non-blocking mode. Zero means infinity. If the timeout
> +                                        expires before this AP returns from Procedure, then
> Procedure
> +                                        on the AP is terminated. If the timeout expires in
> blocking
> +                                        mode, the call returns EFI_TIMEOUT. If the timeout
> expires
> +                                        in non-blocking mode, the timeout determined can be
> through
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        Note that timeout support is optional. Whether an
> +                                        implementation supports this feature, can be
> determined via
> +                                        the Attributes data member.
> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
> parameters to the code
> +                                        that is run by the AP. It is an optional common mailbox
> +                                        between APs and the caller to share information.
> +  @param[in,out] Token                  This is parameter is broken into two
> components:
> +                                        1.Token->Completion is an optional parameter that
> allows the
> +                                        caller to execute the procedure in a blocking or non-
> blocking
> +                                        fashion. If it is NULL the call is blocking, and the call will
> +                                        not return until the AP has completed the procedure. If
> the
> +                                        token is not NULL, the call will return immediately. The
> caller
> +                                        can check whether the procedure has completed with
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        2.Token->Status The implementation updates the
> address pointed
> +                                        at by this variable with the status code returned by
> Procedure
> +                                        when it completes execution on the target AP, or with
> EFI_TIMEOUT
> +                                        if the Procedure fails to complete within the optional
> timeout.
> +                                        The implementation will update this variable with
> EFI_NOT_READY
> +                                        prior to starting Procedure on the target AP.
> +  @param[in,out] CPUStatus              This optional pointer may be used to get
> the status code returned
> +                                        by Procedure when it completes execution on the
> target AP, or with
> +                                        EFI_TIMEOUT if the Procedure fails to complete within
> the optional
> +                                        timeout. The implementation will update this variable
> with
> +                                        EFI_NOT_READY prior to starting Procedure on the
> target AP.
> +
> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
> Procedure has completed
> +                                        execution on the target AP.
> +                                        In the non-blocking case this indicates that the
> procedure has
> +                                        been successfully scheduled for execution on the target
> AP.
> +  @retval EFI_INVALID_PARAMETER         The input arguments are out of
> range. Either the target AP is the
> +                                        caller of the function, or the Procedure or Token is NULL
> +  @retval EFI_NOT_READY                 If the target AP is busy executing another
> procedure
> +  @retval EFI_ALREADY_STARTED           Token is already in use for another
> procedure
> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
> before the specified AP
> +                                        has finished **/ typedef
> +EFI_STATUS (EFIAPI *EFI_MM_DISPATCH_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         CpuNumber,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   MM_COMPLETION                 *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +);
> +
> +/**
> +  This service allows the caller to invoke a procedure on all running
> +application processors (AP)
> +  except the caller. This function uses an optional token parameter to
> +support blocking and
> +  nonblocking modes. If the token is passed into the call, the function
> +will operate in a non-blocking
> +  fashion and the caller can check for completion with CheckOnProcedure or
> WaitForProcedure.
> +
> +  It is not necessary for the implementation to run the procedure on every
> processor on the platform.
> +  Processors that are powered down in such a way that they cannot
> + respond to interrupts, may be  excluded from the broadcast.
> +
> +
> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]     Procedure              A pointer to the code stream to be run on
> the APs that have
> +                                        entered MM. Type EFI_AP_PROCEDURE is defined
> below in related
> +                                        definitions.
> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
> microseconds for the APs to finish
> +                                        execution of Procedure, either for blocking or non-
> blocking mode.
> +                                        Zero means infinity. If the timeout expires before all
> APs return
> +                                        from Procedure, then Procedure on the failed APs is
> terminated. If
> +                                        the timeout expires in blocking mode, the call returns
> EFI_TIMEOUT.
> +                                        If the timeout expires in non-blocking mode, the
> timeout determined
> +                                        can be through CheckOnProcedure or
> WaitForProcedure.
> +                                        Note that timeout support is optional. Whether an
> implementation
> +                                        supports this feature can be determined via the
> Attributes data
> +                                        member.
> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
> parameters to the code
> +                                        that is run by the AP. It is an optional common mailbox
> +                                        between APs and the caller to share information.
> +  @param[in,out] Token                  This is parameter is broken into two
> components:
> +                                        1.Token->Completion is an optional parameter that
> allows the
> +                                        caller to execute the procedure in a blocking or non-
> blocking
> +                                        fashion. If it is NULL the call is blocking, and the call will
> +                                        not return until the AP has completed the procedure. If
> the
> +                                        token is not NULL, the call will return immediately. The
> caller
> +                                        can check whether the procedure has completed with
> +                                        CheckOnProcedure or WaitForProcedure.
> +                                        2.Token->Status The implementation updates the
> address pointed
> +                                        at by this variable with the status code returned by
> Procedure
> +                                        when it completes execution on the target AP, or with
> EFI_TIMEOUT
> +                                        if the Procedure fails to complete within the optional
> timeout.
> +                                        The implementation will update this variable with
> EFI_NOT_READY
> +                                        prior to starting Procedure on the target AP
> +  @param[in,out] CPUStatus              This optional pointer may be used to get
> the individual status
> +                                        returned by every AP that participated in the broadcast.
> This
> +                                        parameter if used provides the base address of an array
> to hold
> +                                        the EFI_STATUS value of each AP in the system. The size
> of the
> +                                        array can be ascertained by the
> GetNumberOfProcessors function.
> +                                        As mentioned above, the broadcast may not include
> every processor
> +                                        in the system. Some implementations may exclude
> processors that
> +                                        have been powered down in such a way that they are
> not responsive
> +                                        to interrupts. Additionally the broadcast excludes the
> processor
> +                                        which is making the BroadcastProcedure call. For every
> excluded
> +                                        processor, the array entry must
> + contain a value of EFI_NOT_STARTED
> +
> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
> Procedure has completed
> +                                        execution on the APs. In the non-blocking case this
> indicates that
> +                                        the procedure has been successfully scheduled for
> execution on the
> +                                        APs.
> +  @retval EFI_INVALID_PARAMETER         Procedure or Token is NULL.
> +  @retval EFI_NOT_READY                 If a target AP is busy executing another
> procedure.
> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
> before all enabled APs have
> +                                        finished.
> +  @retval EFI_ALREADY_STARTED           Before the AP procedure associated
> with the Token is finished, the
> +                                        same Token cannot be used to dispatch or broadcast
> another procedure.
> +
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_MM_BROADCAST_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       EFI_AP_PROCEDURE2             Procedure,
> +  IN       UINTN                         TimeoutInMicroseconds,
> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
> +  IN OUT   MM_COMPLETION                 *Token,
> +  IN OUT   EFI_STATUS                    *CPUStatus
> +);
> +
> +
> +/**
> +  This service allows the caller to set a startup procedure that will
> +be executed when an AP powers
> +  up from a state where core configuration and context is lost. The
> +procedure is execution has the
> +  following properties:
> +  1. The procedure executes before the processor is handed over to the
> operating system.
> +  2. All processors execute the same startup procedure.
> +  3. The procedure may run in parallel with other procedures invoked
> +through the functions in this
> +  protocol, or with processors that are executing an MM handler or running
> in the operating system.
> +
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Procedure            A pointer to the code stream to be run on
> the designated target AP
> +                                       of the system. Type EFI_AP_PROCEDURE is defined
> below in Volume 2
> +                                       with the related definitions of
> +                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
> +                                       If caller may pass a value of NULL to deregister any
> existing
> +                                       startup procedure.
> +  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of
> parameters to the code that is
> +                                       run by the AP. It is an optional common mailbox
> between APs and
> +                                       the caller to share information
> +
> +  @retval EFI_SUCCESS                  The Procedure has been set successfully.
> +  @retval EFI_INVALID_PARAMETER        The Procedure is NULL.
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_MM_SET_STARTUP_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL *This,
> +  IN       EFI_AP_PROCEDURE   Procedure,
> +  IN OUT   VOID               *ProcedureArguments OPTIONAL
> +);
> +
> +/**
> +  When non-blocking execution of a procedure on an AP is invoked with
> +DispatchProcedure,
> +  via the use of a token, this function can be used to check for completion of
> the procedure on the AP.
> +  The function takes the token that was passed into the
> +DispatchProcedure call. If the procedure
> +  is complete, and therefore it is now possible to run another
> +procedure on the same AP, this function
> +  returns EFI_SUCESS. In this case the status returned by the procedure
> +that executed on the AP is
> +  returned in the token's Status field. If the procedure has not yet
> +completed, then this function
> +  returns EFI_NOT_READY.
> +
> +  When a non-blocking execution of a procedure is invoked with
> + BroadcastProcedure, via the  use of a token, this function can be used
> + to check for completion of the procedure on all the  broadcast APs.
> + The function takes the token that was passed into the
> + BroadcastProcedure  call. If the procedure is complete on all
> + broadcast APs this function returns EFI_SUCESS. In this  case the Status
> field in the token passed into the function reflects the overall result of the
> invocation, which may be EFI_SUCCESS, if all executions succeeded, or the
> first observed failure.
> +  If the procedure has not yet completed on the broadcast APs, the
> + function returns  EFI_NOT_READY.
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Token                This parameter describes the token that was
> passed into
> +                                       DispatchProcedure or BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS                  Procedure has completed.
> +  @retval EFI_NOT_READY                The Procedure has not completed.
> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is
> NULL
> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-
> blocking call
> +
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_CHECK_FOR_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       MM_COMPLETION                 Token
> +);
> +
> +/**
> +  When a non-blocking execution of a procedure on an AP is invoked via
> +DispatchProcedure,
> +  this function will block the caller until the remote procedure has completed
> on the designated AP.
> +  The non-blocking procedure invocation is identified by the Token
> +parameter, which must match the
> +  token that used when DispatchProcedure was called. Upon completion
> +the status returned by
> +  the procedure that executed on the AP is used to update the token's
> Status field.
> +
> +  When a non-blocking execution of a procedure on an AP is invoked via
> + BroadcastProcedure  this function will block the caller until the
> + remote procedure has completed on all of the APs that  entered MM. The
> + non-blocking procedure invocation is identified by the Token
> + parameter, which  must match the token that used when
> + BroadcastProcedure was called. Upon completion the  overall status
> + returned by the procedures that executed on the broadcast AP is used to
> update the  token's Status field. The overall status may be EFI_SUCCESS, if all
> executions succeeded, or the  first observed failure.
> +
> +
> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
> +  @param[in]      Token                This parameter describes the token that was
> passed into
> +                                       DispatchProcedure or BroadcastProcedure.
> +
> +  @retval EFI_SUCCESS                  Procedure has completed.
> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is
> NULL
> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-
> blocking call
> +
> +**/
> +typedef
> +EFI_STATUS
> +(EFIAPI *EFI_WAIT_FOR_PROCEDURE) (
> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
> +  IN       MM_COMPLETION                 Token
> +);
> +
> +
> +
> +///
> +/// The MM MP protocol provides a set of functions to allow execution
> +of procedures on processors that /// have entered MM.
> +///
> +struct _EFI_MM_MP_PROTOCOL {
> +  UINT32                            Revision;
> +  UINT32                            Attributes;
> +  EFI_MM_GET_NUMBER_OF_PROCESSORS   GetNumberOfProcessors;
> +  EFI_MM_DISPATCH_PROCEDURE         DispatchProcedure;
> +  EFI_MM_BROADCAST_PROCEDURE        BroadcastProcedure;
> +  EFI_MM_SET_STARTUP_PROCEDURE      SetStartupProcedure;
> +  EFI_CHECK_FOR_PROCEDURE           CheckForProcedure;
> +  EFI_WAIT_FOR_PROCEDURE            WaitForProcedure;
> +};
> +
> +extern EFI_GUID gEfiMmMpProtocolGuid;
> +
> +#endif
> diff --git a/MdePkg/MdePkg.dec b/MdePkg/MdePkg.dec index
> 6c563375ee..b382efd578 100644
> --- a/MdePkg/MdePkg.dec
> +++ b/MdePkg/MdePkg.dec
> @@ -1167,6 +1167,9 @@
>    # Protocols defined in PI 1.5.
>    #
> 
> +  ## Include/Protocol/MmMp.h
> +  gEfiMmMpProtocolGuid = { 0x5d5450d7, 0x990c, 0x4180, { 0xa8, 0x3,
> + 0x8e, 0x63, 0xf0, 0x60, 0x83, 0x7 }}
> +
>    ## Include/Protocol/MmEndOfDxe.h
>    gEfiMmEndOfDxeProtocolGuid = { 0x24e70042, 0xd5c5, 0x4260, { 0x8c, 0x39,
> 0xa, 0xd3, 0xaa, 0x32, 0xe9, 0x3d }}
> 
> --
> 2.21.0.windows.1
> 
> 
> 

Re: [edk2-devel] [Patch v5 1/2] MdePkg: Add new MM MP Protocol definition.

[Liming Gao]

Thanks for your CC.

The change is OK. Reviewed-by: Liming Gao <liming.gao@intel.com>

>-----Original Message-----
>From: Dong, Eric
>Sent: Thursday, July 11, 2019 2:40 PM
>To: devel@edk2.groups.io; Dong, Eric <eric.dong@intel.com>; Gao, Liming
><liming.gao@intel.com>; Kinney, Michael D <michael.d.kinney@intel.com>
>Cc: Ni, Ray <ray.ni@intel.com>; Laszlo Ersek <lersek@redhat.com>
>Subject: RE: [edk2-devel] [Patch v5 1/2] MdePkg: Add new MM MP Protocol
>definition.
>
>Hi Liming,  Mike,
>
>Can you help to review this patch?
>
>Thanks,
>Eric
>
>> -----Original Message-----
>> From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of
>> Dong, Eric
>> Sent: Wednesday, July 10, 2019 3:56 PM
>> To: devel@edk2.groups.io
>> Cc: Ni, Ray <ray.ni@intel.com>; Laszlo Ersek <lersek@redhat.com>
>> Subject: [edk2-devel] [Patch v5 1/2] MdePkg: Add new MM MP Protocol
>> definition.
>>
>> REF: https://bugzilla.tianocore.org/show_bug.cgi?id=1937
>>
>> EFI MM MP Protocol is defined in the PI 1.5 specification.
>>
>> The MM MP protocol provides a set of functions to allow execution of
>> procedures on processors that have entered MM. This protocol has the
>> following properties:
>> 1. The caller can invoke execution of a procedure on a processor, other than
>> the caller, that has also entered MM. Supports blocking and non-blocking
>> modes of operation.
>> 2. The caller can invoke a procedure on multiple processors. Supports
>> blocking and non-blocking modes of operation.
>>
>> Cc: Ray Ni <ray.ni@intel.com>
>> Cc: Laszlo Ersek <lersek@redhat.com>
>> Signed-off-by: Eric Dong <eric.dong@intel.com>
>> Reviewed-by: Ray Ni <ray.ni@intel.com>
>> ---
>>  MdePkg/Include/Pi/PiMultiPhase.h |  16 ++
>>  MdePkg/Include/Protocol/MmMp.h   | 333
>> +++++++++++++++++++++++++++++++
>>  MdePkg/MdePkg.dec                |   3 +
>>  3 files changed, 352 insertions(+)
>>  create mode 100644 MdePkg/Include/Protocol/MmMp.h
>>
>> diff --git a/MdePkg/Include/Pi/PiMultiPhase.h
>> b/MdePkg/Include/Pi/PiMultiPhase.h
>> index eb12527767..a5056799e1 100644
>> --- a/MdePkg/Include/Pi/PiMultiPhase.h
>> +++ b/MdePkg/Include/Pi/PiMultiPhase.h
>> @@ -176,4 +176,20 @@ VOID
>>    IN OUT VOID  *Buffer
>>    );
>>
>> +/**
>> +  The function prototype for invoking a function on an Application
>Processor.
>> +
>> +  This definition is used by the UEFI MM MP Serices Protocol.
>> +
>> +  @param[in] ProcedureArgument    The pointer to private data buffer.
>> +
>> +  @retval EFI_SUCCESS             Excutive the procedure successfully
>> +
>> +**/
>> +typedef
>> +EFI_STATUS
>> +(EFIAPI *EFI_AP_PROCEDURE2)(
>> +  IN VOID  *ProcedureArgument
>> +);
>> +
>>  #endif
>> diff --git a/MdePkg/Include/Protocol/MmMp.h
>> b/MdePkg/Include/Protocol/MmMp.h new file mode 100644 index
>> 0000000000..beace1386c
>> --- /dev/null
>> +++ b/MdePkg/Include/Protocol/MmMp.h
>> @@ -0,0 +1,333 @@
>> +/** @file
>> +  EFI MM MP Protocol is defined in the PI 1.5 specification.
>> +
>> +  The MM MP protocol provides a set of functions to allow execution of
>> + procedures on processors that  have entered MM. This protocol has the
>> following properties:
>> +  1. The caller can only invoke execution of a procedure on a processor,
>> other than the caller, that
>> +     has also entered MM.
>> +  2. It is possible to invoke a procedure on multiple processors. Supports
>> blocking and non-blocking
>> +     modes of operation.
>> +
>> +  Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
>> +  SPDX-License-Identifier: BSD-2-Clause-Patent
>> +
>> +**/
>> +
>> +#ifndef _MM_MP_H_
>> +#define _MM_MP_H_
>> +
>> +#include <Pi/PiMmCis.h>
>> +
>> +#define EFI_MM_MP_PROTOCOL_GUID \
>> +  { \
>> +    0x5d5450d7, 0x990c, 0x4180, {0xa8, 0x3, 0x8e, 0x63, 0xf0, 0x60,
>> +0x83, 0x7  }  \
>> +  }
>> +
>> +//
>> +// Revision definition.
>> +//
>> +#define EFI_MM_MP_PROTOCOL_REVISION    0x00
>> +
>> +//
>> +// Attribute flags
>> +//
>> +#define EFI_MM_MP_TIMEOUT_SUPPORTED    0x01
>> +
>> +//
>> +// Completion token
>> +//
>> +typedef VOID* MM_COMPLETION;
>> +
>> +typedef struct {
>> +  MM_COMPLETION  Completion;
>> +  EFI_STATUS     Status;
>> +} MM_DISPATCH_COMPLETION_TOKEN;
>> +
>> +typedef struct _EFI_MM_MP_PROTOCOL  EFI_MM_MP_PROTOCOL;
>> +
>> +/**
>> +  Service to retrieves the number of logical processor in the platform.
>> +
>> +  @param[in]  This                The EFI_MM_MP_PROTOCOL instance.
>> +  @param[out] NumberOfProcessors  Pointer to the total number of logical
>> processors in the system,
>> +                                  including the BSP and all APs.
>> +
>> +  @retval EFI_SUCCESS             The number of processors was retrieved
>> successfully
>> +  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL
>> +**/
>> +typedef
>> +EFI_STATUS
>> +(EFIAPI *EFI_MM_GET_NUMBER_OF_PROCESSORS) (
>> +  IN CONST EFI_MM_MP_PROTOCOL  *This,
>> +  OUT      UINTN               *NumberOfProcessors
>> +);
>> +
>> +
>> +/**
>> +  This service allows the caller to invoke a procedure one of the
>> +application processors (AP). This
>> +  function uses an optional token parameter to support blocking and
>> +non-blocking modes. If the token
>> +  is passed into the call, the function will operate in a non-blocking
>> +fashion and the caller can
>> +  check for completion with CheckOnProcedure or WaitForProcedure.
>> +
>> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
>> +  @param[in]     Procedure              A pointer to the procedure to be run on
>> the designated target
>> +                                        AP of the system. Type EFI_AP_PROCEDURE2 is
>defined
>> below in
>> +                                        related definitions.
>> +  @param[in]     CpuNumber              The zero-based index of the processor
>> number of the target
>> +                                        AP, on which the code stream is supposed to run. If the
>> number
>> +                                        points to the calling processor then it will not run the
>> +                                        supplied code.
>> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
>> microseconds for this AP to
>> +                                        finish execution of Procedure, either for blocking or
>> +                                        non-blocking mode. Zero means infinity. If the timeout
>> +                                        expires before this AP returns from Procedure, then
>> Procedure
>> +                                        on the AP is terminated. If the timeout expires in
>> blocking
>> +                                        mode, the call returns EFI_TIMEOUT. If the timeout
>> expires
>> +                                        in non-blocking mode, the timeout determined can be
>> through
>> +                                        CheckOnProcedure or WaitForProcedure.
>> +                                        Note that timeout support is optional. Whether an
>> +                                        implementation supports this feature, can be
>> determined via
>> +                                        the Attributes data member.
>> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
>> parameters to the code
>> +                                        that is run by the AP. It is an optional common mailbox
>> +                                        between APs and the caller to share information.
>> +  @param[in,out] Token                  This is parameter is broken into two
>> components:
>> +                                        1.Token->Completion is an optional parameter that
>> allows the
>> +                                        caller to execute the procedure in a blocking or non-
>> blocking
>> +                                        fashion. If it is NULL the call is blocking, and the call will
>> +                                        not return until the AP has completed the procedure. If
>> the
>> +                                        token is not NULL, the call will return immediately. The
>> caller
>> +                                        can check whether the procedure has completed with
>> +                                        CheckOnProcedure or WaitForProcedure.
>> +                                        2.Token->Status The implementation updates the
>> address pointed
>> +                                        at by this variable with the status code returned by
>> Procedure
>> +                                        when it completes execution on the target AP, or with
>> EFI_TIMEOUT
>> +                                        if the Procedure fails to complete within the optional
>> timeout.
>> +                                        The implementation will update this variable with
>> EFI_NOT_READY
>> +                                        prior to starting Procedure on the target AP.
>> +  @param[in,out] CPUStatus              This optional pointer may be used to
>get
>> the status code returned
>> +                                        by Procedure when it completes execution on the
>> target AP, or with
>> +                                        EFI_TIMEOUT if the Procedure fails to complete within
>> the optional
>> +                                        timeout. The implementation will update this variable
>> with
>> +                                        EFI_NOT_READY prior to starting Procedure on the
>> target AP.
>> +
>> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
>> Procedure has completed
>> +                                        execution on the target AP.
>> +                                        In the non-blocking case this indicates that the
>> procedure has
>> +                                        been successfully scheduled for execution on the
>target
>> AP.
>> +  @retval EFI_INVALID_PARAMETER         The input arguments are out of
>> range. Either the target AP is the
>> +                                        caller of the function, or the Procedure or Token is
>NULL
>> +  @retval EFI_NOT_READY                 If the target AP is busy executing
>another
>> procedure
>> +  @retval EFI_ALREADY_STARTED           Token is already in use for another
>> procedure
>> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
>> before the specified AP
>> +                                        has finished **/ typedef
>> +EFI_STATUS (EFIAPI *EFI_MM_DISPATCH_PROCEDURE) (
>> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
>> +  IN       EFI_AP_PROCEDURE2             Procedure,
>> +  IN       UINTN                         CpuNumber,
>> +  IN       UINTN                         TimeoutInMicroseconds,
>> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
>> +  IN OUT   MM_COMPLETION                 *Token,
>> +  IN OUT   EFI_STATUS                    *CPUStatus
>> +);
>> +
>> +/**
>> +  This service allows the caller to invoke a procedure on all running
>> +application processors (AP)
>> +  except the caller. This function uses an optional token parameter to
>> +support blocking and
>> +  nonblocking modes. If the token is passed into the call, the function
>> +will operate in a non-blocking
>> +  fashion and the caller can check for completion with CheckOnProcedure
>or
>> WaitForProcedure.
>> +
>> +  It is not necessary for the implementation to run the procedure on every
>> processor on the platform.
>> +  Processors that are powered down in such a way that they cannot
>> + respond to interrupts, may be  excluded from the broadcast.
>> +
>> +
>> +  @param[in]     This                   The EFI_MM_MP_PROTOCOL instance.
>> +  @param[in]     Procedure              A pointer to the code stream to be run on
>> the APs that have
>> +                                        entered MM. Type EFI_AP_PROCEDURE is defined
>> below in related
>> +                                        definitions.
>> +  @param[in]     TimeoutInMicroseconds  Indicates the time limit in
>> microseconds for the APs to finish
>> +                                        execution of Procedure, either for blocking or non-
>> blocking mode.
>> +                                        Zero means infinity. If the timeout expires before all
>> APs return
>> +                                        from Procedure, then Procedure on the failed APs is
>> terminated. If
>> +                                        the timeout expires in blocking mode, the call returns
>> EFI_TIMEOUT.
>> +                                        If the timeout expires in non-blocking mode, the
>> timeout determined
>> +                                        can be through CheckOnProcedure or
>> WaitForProcedure.
>> +                                        Note that timeout support is optional. Whether an
>> implementation
>> +                                        supports this feature can be determined via the
>> Attributes data
>> +                                        member.
>> +  @param[in,out] ProcedureArguments     Allows the caller to pass a list of
>> parameters to the code
>> +                                        that is run by the AP. It is an optional common mailbox
>> +                                        between APs and the caller to share information.
>> +  @param[in,out] Token                  This is parameter is broken into two
>> components:
>> +                                        1.Token->Completion is an optional parameter that
>> allows the
>> +                                        caller to execute the procedure in a blocking or non-
>> blocking
>> +                                        fashion. If it is NULL the call is blocking, and the call will
>> +                                        not return until the AP has completed the procedure. If
>> the
>> +                                        token is not NULL, the call will return immediately. The
>> caller
>> +                                        can check whether the procedure has completed with
>> +                                        CheckOnProcedure or WaitForProcedure.
>> +                                        2.Token->Status The implementation updates the
>> address pointed
>> +                                        at by this variable with the status code returned by
>> Procedure
>> +                                        when it completes execution on the target AP, or with
>> EFI_TIMEOUT
>> +                                        if the Procedure fails to complete within the optional
>> timeout.
>> +                                        The implementation will update this variable with
>> EFI_NOT_READY
>> +                                        prior to starting Procedure on the target AP
>> +  @param[in,out] CPUStatus              This optional pointer may be used to
>get
>> the individual status
>> +                                        returned by every AP that participated in the broadcast.
>> This
>> +                                        parameter if used provides the base address of an
>array
>> to hold
>> +                                        the EFI_STATUS value of each AP in the system. The
>size
>> of the
>> +                                        array can be ascertained by the
>> GetNumberOfProcessors function.
>> +                                        As mentioned above, the broadcast may not include
>> every processor
>> +                                        in the system. Some implementations may exclude
>> processors that
>> +                                        have been powered down in such a way that they are
>> not responsive
>> +                                        to interrupts. Additionally the broadcast excludes the
>> processor
>> +                                        which is making the BroadcastProcedure call. For every
>> excluded
>> +                                        processor, the array entry must
>> + contain a value of EFI_NOT_STARTED
>> +
>> +  @retval EFI_SUCCESS                   In the blocking case, this indicates that
>> Procedure has completed
>> +                                        execution on the APs. In the non-blocking case this
>> indicates that
>> +                                        the procedure has been successfully scheduled for
>> execution on the
>> +                                        APs.
>> +  @retval EFI_INVALID_PARAMETER         Procedure or Token is NULL.
>> +  @retval EFI_NOT_READY                 If a target AP is busy executing another
>> procedure.
>> +  @retval EFI_TIMEOUT                   In blocking mode, the timeout expired
>> before all enabled APs have
>> +                                        finished.
>> +  @retval EFI_ALREADY_STARTED           Before the AP procedure associated
>> with the Token is finished, the
>> +                                        same Token cannot be used to dispatch or broadcast
>> another procedure.
>> +
>> +**/
>> +typedef
>> +EFI_STATUS
>> +(EFIAPI *EFI_MM_BROADCAST_PROCEDURE) (
>> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
>> +  IN       EFI_AP_PROCEDURE2             Procedure,
>> +  IN       UINTN                         TimeoutInMicroseconds,
>> +  IN OUT   VOID                          *ProcedureArguments OPTIONAL,
>> +  IN OUT   MM_COMPLETION                 *Token,
>> +  IN OUT   EFI_STATUS                    *CPUStatus
>> +);
>> +
>> +
>> +/**
>> +  This service allows the caller to set a startup procedure that will
>> +be executed when an AP powers
>> +  up from a state where core configuration and context is lost. The
>> +procedure is execution has the
>> +  following properties:
>> +  1. The procedure executes before the processor is handed over to the
>> operating system.
>> +  2. All processors execute the same startup procedure.
>> +  3. The procedure may run in parallel with other procedures invoked
>> +through the functions in this
>> +  protocol, or with processors that are executing an MM handler or running
>> in the operating system.
>> +
>> +
>> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
>> +  @param[in]      Procedure            A pointer to the code stream to be run on
>> the designated target AP
>> +                                       of the system. Type EFI_AP_PROCEDURE is defined
>> below in Volume 2
>> +                                       with the related definitions of
>> +                                       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
>> +                                       If caller may pass a value of NULL to deregister any
>> existing
>> +                                       startup procedure.
>> +  @param[in,out]  ProcedureArguments   Allows the caller to pass a list of
>> parameters to the code that is
>> +                                       run by the AP. It is an optional common mailbox
>> between APs and
>> +                                       the caller to share information
>> +
>> +  @retval EFI_SUCCESS                  The Procedure has been set successfully.
>> +  @retval EFI_INVALID_PARAMETER        The Procedure is NULL.
>> +**/
>> +typedef
>> +EFI_STATUS
>> +(EFIAPI *EFI_MM_SET_STARTUP_PROCEDURE) (
>> +  IN CONST EFI_MM_MP_PROTOCOL *This,
>> +  IN       EFI_AP_PROCEDURE   Procedure,
>> +  IN OUT   VOID               *ProcedureArguments OPTIONAL
>> +);
>> +
>> +/**
>> +  When non-blocking execution of a procedure on an AP is invoked with
>> +DispatchProcedure,
>> +  via the use of a token, this function can be used to check for completion
>of
>> the procedure on the AP.
>> +  The function takes the token that was passed into the
>> +DispatchProcedure call. If the procedure
>> +  is complete, and therefore it is now possible to run another
>> +procedure on the same AP, this function
>> +  returns EFI_SUCESS. In this case the status returned by the procedure
>> +that executed on the AP is
>> +  returned in the token's Status field. If the procedure has not yet
>> +completed, then this function
>> +  returns EFI_NOT_READY.
>> +
>> +  When a non-blocking execution of a procedure is invoked with
>> + BroadcastProcedure, via the  use of a token, this function can be used
>> + to check for completion of the procedure on all the  broadcast APs.
>> + The function takes the token that was passed into the
>> + BroadcastProcedure  call. If the procedure is complete on all
>> + broadcast APs this function returns EFI_SUCESS. In this  case the Status
>> field in the token passed into the function reflects the overall result of the
>> invocation, which may be EFI_SUCCESS, if all executions succeeded, or the
>> first observed failure.
>> +  If the procedure has not yet completed on the broadcast APs, the
>> + function returns  EFI_NOT_READY.
>> +
>> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
>> +  @param[in]      Token                This parameter describes the token that was
>> passed into
>> +                                       DispatchProcedure or BroadcastProcedure.
>> +
>> +  @retval EFI_SUCCESS                  Procedure has completed.
>> +  @retval EFI_NOT_READY                The Procedure has not completed.
>> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is
>> NULL
>> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-
>> blocking call
>> +
>> +**/
>> +typedef
>> +EFI_STATUS
>> +(EFIAPI *EFI_CHECK_FOR_PROCEDURE) (
>> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
>> +  IN       MM_COMPLETION                 Token
>> +);
>> +
>> +/**
>> +  When a non-blocking execution of a procedure on an AP is invoked via
>> +DispatchProcedure,
>> +  this function will block the caller until the remote procedure has
>completed
>> on the designated AP.
>> +  The non-blocking procedure invocation is identified by the Token
>> +parameter, which must match the
>> +  token that used when DispatchProcedure was called. Upon completion
>> +the status returned by
>> +  the procedure that executed on the AP is used to update the token's
>> Status field.
>> +
>> +  When a non-blocking execution of a procedure on an AP is invoked via
>> + BroadcastProcedure  this function will block the caller until the
>> + remote procedure has completed on all of the APs that  entered MM. The
>> + non-blocking procedure invocation is identified by the Token
>> + parameter, which  must match the token that used when
>> + BroadcastProcedure was called. Upon completion the  overall status
>> + returned by the procedures that executed on the broadcast AP is used to
>> update the  token's Status field. The overall status may be EFI_SUCCESS, if
>all
>> executions succeeded, or the  first observed failure.
>> +
>> +
>> +  @param[in]      This                 The EFI_MM_MP_PROTOCOL instance.
>> +  @param[in]      Token                This parameter describes the token that was
>> passed into
>> +                                       DispatchProcedure or BroadcastProcedure.
>> +
>> +  @retval EFI_SUCCESS                  Procedure has completed.
>> +  @retval EFI_INVALID_PARAMETER        Token or Token->Completion is
>> NULL
>> +  @retval EFI_NOT_FOUND                Token is not currently in use for a non-
>> blocking call
>> +
>> +**/
>> +typedef
>> +EFI_STATUS
>> +(EFIAPI *EFI_WAIT_FOR_PROCEDURE) (
>> +  IN CONST EFI_MM_MP_PROTOCOL            *This,
>> +  IN       MM_COMPLETION                 Token
>> +);
>> +
>> +
>> +
>> +///
>> +/// The MM MP protocol provides a set of functions to allow execution
>> +of procedures on processors that /// have entered MM.
>> +///
>> +struct _EFI_MM_MP_PROTOCOL {
>> +  UINT32                            Revision;
>> +  UINT32                            Attributes;
>> +  EFI_MM_GET_NUMBER_OF_PROCESSORS   GetNumberOfProcessors;
>> +  EFI_MM_DISPATCH_PROCEDURE         DispatchProcedure;
>> +  EFI_MM_BROADCAST_PROCEDURE        BroadcastProcedure;
>> +  EFI_MM_SET_STARTUP_PROCEDURE      SetStartupProcedure;
>> +  EFI_CHECK_FOR_PROCEDURE           CheckForProcedure;
>> +  EFI_WAIT_FOR_PROCEDURE            WaitForProcedure;
>> +};
>> +
>> +extern EFI_GUID gEfiMmMpProtocolGuid;
>> +
>> +#endif
>> diff --git a/MdePkg/MdePkg.dec b/MdePkg/MdePkg.dec index
>> 6c563375ee..b382efd578 100644
>> --- a/MdePkg/MdePkg.dec
>> +++ b/MdePkg/MdePkg.dec
>> @@ -1167,6 +1167,9 @@
>>    # Protocols defined in PI 1.5.
>>    #
>>
>> +  ## Include/Protocol/MmMp.h
>> +  gEfiMmMpProtocolGuid = { 0x5d5450d7, 0x990c, 0x4180, { 0xa8, 0x3,
>> + 0x8e, 0x63, 0xf0, 0x60, 0x83, 0x7 }}
>> +
>>    ## Include/Protocol/MmEndOfDxe.h
>>    gEfiMmEndOfDxeProtocolGuid = { 0x24e70042, 0xd5c5, 0x4260, { 0x8c,
>0x39,
>> 0xa, 0xd3, 0xaa, 0x32, 0xe9, 0x3d }}
>>
>> --
>> 2.21.0.windows.1
>>
>>
>>