Re: [PATCH v2 10/16] OvmfPkg/CpuHotplugSmm: collect CPUs with events

["Ard Biesheuvel" <ard.biesheuvel@linaro.org>]

On Wed, 26 Feb 2020 at 23:12, Laszlo Ersek <lersek@redhat.com> wrote:
>
> Call QemuCpuhpCollectApicIds() in the root MMI handler. The APIC IDs of
> the hotplugged CPUs will be used for several purposes in subsequent
> patches.
>
> For calling QemuCpuhpCollectApicIds(), pre-allocate both of its output
> arrays "PluggedApicIds" and "ToUnplugApicIds" in the driver's entry point
> function. The allocation size is dictated by the possible CPU count, which
> we fetch from "CPU_HOT_PLUG_DATA.ArrayLength".
>
> The CPU_HOT_PLUG_DATA structure in SMRAM is an out-of-band information
> channel between this driver and PiSmmCpuDxeSmm, underlying
> EFI_SMM_CPU_SERVICE_PROTOCOL.
>
> In order to consume "CPU_HOT_PLUG_DATA.ArrayLength", extend the driver's
> DEPEX to EFI_SMM_CPU_SERVICE_PROTOCOL. PiSmmCpuDxeSmm stores the address
> of CPU_HOT_PLUG_DATA to "PcdCpuHotPlugDataAddress", before it produces
> EFI_SMM_CPU_SERVICE_PROTOCOL.
>
> Stash the protocol at once, as it will be needed later.
>
> Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
> Cc: Igor Mammedov <imammedo@redhat.com>
> Cc: Jiewen Yao <jiewen.yao@intel.com>
> Cc: Jordan Justen <jordan.l.justen@intel.com>
> Cc: Michael Kinney <michael.d.kinney@intel.com>
> Cc: Philippe Mathieu-Daudé <philmd@redhat.com>
> Ref: https://bugzilla.tianocore.org/show_bug.cgi?id=1512
> Signed-off-by: Laszlo Ersek <lersek@redhat.com>
> Acked-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>

Reviewed-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>

> ---
>
> Notes:
>     v2:
>
>     - Pick up Ard's Acked-by, which is conditional on approval from Intel
>       reviewers on Cc. (I'd like to save Ard the churn of re-acking
>       unmodified patches.)
>
>  OvmfPkg/CpuHotplugSmm/CpuHotplugSmm.inf |   7 +-
>  OvmfPkg/CpuHotplugSmm/CpuHotplug.c      | 111 +++++++++++++++++++-
>  2 files changed, 115 insertions(+), 3 deletions(-)
>
> diff --git a/OvmfPkg/CpuHotplugSmm/CpuHotplugSmm.inf b/OvmfPkg/CpuHotplugSmm/CpuHotplugSmm.inf
> index ab690a9e5e20..31c1ee1c9f6d 100644
> --- a/OvmfPkg/CpuHotplugSmm/CpuHotplugSmm.inf
> +++ b/OvmfPkg/CpuHotplugSmm/CpuHotplugSmm.inf
> @@ -11,41 +11,46 @@ [Defines]
>    PI_SPECIFICATION_VERSION   = 0x00010046                            # PI-1.7.0
>    BASE_NAME                  = CpuHotplugSmm
>    FILE_GUID                  = 84EEA114-C6BE-4445-8F90-51D97863E363
>    MODULE_TYPE                = DXE_SMM_DRIVER
>    ENTRY_POINT                = CpuHotplugEntry
>
>  #
>  # The following information is for reference only and not required by the build
>  # tools.
>  #
>  # VALID_ARCHITECTURES        = IA32 X64
>  #
>
>  [Sources]
>    ApicId.h
>    CpuHotplug.c
>    QemuCpuhp.c
>    QemuCpuhp.h
>
>  [Packages]
>    MdePkg/MdePkg.dec
>    OvmfPkg/OvmfPkg.dec
> +  UefiCpuPkg/UefiCpuPkg.dec
>
>  [LibraryClasses]
>    BaseLib
>    DebugLib
>    MmServicesTableLib
>    PcdLib
> +  SafeIntLib
>    UefiDriverEntryPoint
>
>  [Protocols]
>    gEfiMmCpuIoProtocolGuid                                           ## CONSUMES
> +  gEfiSmmCpuServiceProtocolGuid                                     ## CONSUMES
>
>  [Pcd]
> +  gUefiCpuPkgTokenSpaceGuid.PcdCpuHotPlugDataAddress                ## CONSUMES
>    gUefiOvmfPkgTokenSpaceGuid.PcdQ35SmramAtDefaultSmbase             ## CONSUMES
>
>  [FeaturePcd]
>    gUefiOvmfPkgTokenSpaceGuid.PcdSmmSmramRequire                     ## CONSUMES
>
>  [Depex]
> -  gEfiMmCpuIoProtocolGuid
> +  gEfiMmCpuIoProtocolGuid AND
> +  gEfiSmmCpuServiceProtocolGuid
> diff --git a/OvmfPkg/CpuHotplugSmm/CpuHotplug.c b/OvmfPkg/CpuHotplugSmm/CpuHotplug.c
> index 5df8c689c63a..42e023cb85c0 100644
> --- a/OvmfPkg/CpuHotplugSmm/CpuHotplug.c
> +++ b/OvmfPkg/CpuHotplugSmm/CpuHotplug.c
> @@ -1,46 +1,76 @@
>  /** @file
>    Root SMI handler for VCPU hotplug SMIs.
>
>    Copyright (c) 2020, Red Hat, Inc.
>
>    SPDX-License-Identifier: BSD-2-Clause-Patent
>  **/
>
> +#include <CpuHotPlugData.h>                  // CPU_HOT_PLUG_DATA
>  #include <IndustryStandard/Q35MchIch9.h>     // ICH9_APM_CNT
>  #include <IndustryStandard/QemuCpuHotplug.h> // QEMU_CPUHP_CMD_GET_PENDING
>  #include <Library/BaseLib.h>                 // CpuDeadLoop()
>  #include <Library/DebugLib.h>                // ASSERT()
>  #include <Library/MmServicesTableLib.h>      // gMmst
>  #include <Library/PcdLib.h>                  // PcdGetBool()
> +#include <Library/SafeIntLib.h>              // SafeUintnSub()
>  #include <Protocol/MmCpuIo.h>                // EFI_MM_CPU_IO_PROTOCOL
> +#include <Protocol/SmmCpuService.h>          // EFI_SMM_CPU_SERVICE_PROTOCOL
>  #include <Uefi/UefiBaseType.h>               // EFI_STATUS
>
> +#include "ApicId.h"                          // APIC_ID
>  #include "QemuCpuhp.h"                       // QemuCpuhpWriteCpuSelector()
>
>  //
>  // We use this protocol for accessing IO Ports.
>  //
>  STATIC EFI_MM_CPU_IO_PROTOCOL *mMmCpuIo;
>  //
> +// The following protocol is used to report the addition or removal of a CPU to
> +// the SMM CPU driver (PiSmmCpuDxeSmm).
> +//
> +STATIC EFI_SMM_CPU_SERVICE_PROTOCOL *mMmCpuService;
> +//
> +// This structure is a communication side-channel between the
> +// EFI_SMM_CPU_SERVICE_PROTOCOL consumer (i.e., this driver) and provider
> +// (i.e., PiSmmCpuDxeSmm).
> +//
> +STATIC CPU_HOT_PLUG_DATA *mCpuHotPlugData;
> +//
> +// SMRAM arrays for fetching the APIC IDs of processors with pending events (of
> +// known event types), for the time of just one MMI.
> +//
> +// The lifetimes of these arrays match that of this driver only because we
> +// don't want to allocate SMRAM at OS runtime, and potentially fail (or
> +// fragment the SMRAM map).
> +//
> +// These arrays provide room for ("possible CPU count" minus one) APIC IDs
> +// each, as we don't expect every possible CPU to appear, or disappear, in a
> +// single MMI. The numbers of used (populated) elements in the arrays are
> +// determined on every MMI separately.
> +//
> +STATIC APIC_ID *mPluggedApicIds;
> +STATIC APIC_ID *mToUnplugApicIds;
> +//
>  // Represents the registration of the CPU Hotplug MMI handler.
>  //
>  STATIC EFI_HANDLE mDispatchHandle;
>
>
>  /**
>    CPU Hotplug MMI handler function.
>
>    This is a root MMI handler.
>
>    @param[in] DispatchHandle      The unique handle assigned to this handler by
>                                   EFI_MM_SYSTEM_TABLE.MmiHandlerRegister().
>
>    @param[in] Context             Context passed in by
>                                   EFI_MM_SYSTEM_TABLE.MmiManage(). Due to
>                                   CpuHotplugMmi() being a root MMI handler,
>                                   Context is ASSERT()ed to be NULL.
>
>    @param[in,out] CommBuffer      Ignored, due to CpuHotplugMmi() being a root
>                                   MMI handler.
>
>    @param[in,out] CommBufferSize  Ignored, due to CpuHotplugMmi() being a root
> @@ -65,162 +95,239 @@ STATIC EFI_HANDLE mDispatchHandle;
>                                                but other handlers should still
>                                                be called.
>
>    @retval EFI_WARN_INTERRUPT_SOURCE_PENDING   The MMI source is still pending,
>                                                and other handlers should still
>                                                be called.
>
>    @retval EFI_INTERRUPT_PENDING               The MMI source could not be
>                                                quiesced.
>  **/
>  STATIC
>  EFI_STATUS
>  EFIAPI
>  CpuHotplugMmi (
>    IN EFI_HANDLE DispatchHandle,
>    IN CONST VOID *Context        OPTIONAL,
>    IN OUT VOID   *CommBuffer     OPTIONAL,
>    IN OUT UINTN  *CommBufferSize OPTIONAL
>    )
>  {
>    EFI_STATUS Status;
>    UINT8      ApmControl;
> +  UINT32     PluggedCount;
> +  UINT32     ToUnplugCount;
>
>    //
>    // Assert that we are entering this function due to our root MMI handler
>    // registration.
>    //
>    ASSERT (DispatchHandle == mDispatchHandle);
>    //
>    // When MmiManage() is invoked to process root MMI handlers, the caller (the
>    // MM Core) is expected to pass in a NULL Context. MmiManage() then passes
>    // the same NULL Context to individual handlers.
>    //
>    ASSERT (Context == NULL);
>    //
>    // Read the MMI command value from the APM Control Port, to see if this is an
>    // MMI we should care about.
>    //
>    Status = mMmCpuIo->Io.Read (mMmCpuIo, MM_IO_UINT8, ICH9_APM_CNT, 1,
>                            &ApmControl);
>    if (EFI_ERROR (Status)) {
>      DEBUG ((DEBUG_ERROR, "%a: failed to read ICH9_APM_CNT: %r\n", __FUNCTION__,
>        Status));
>      //
>      // We couldn't even determine if the MMI was for us or not.
>      //
>      goto Fatal;
>    }
>
>    if (ApmControl != ICH9_APM_CNT_CPU_HOTPLUG) {
>      //
>      // The MMI is not for us.
>      //
>      return EFI_WARN_INTERRUPT_SOURCE_QUIESCED;
>    }
>
> +  //
> +  // Collect the CPUs with pending events.
> +  //
> +  Status = QemuCpuhpCollectApicIds (
> +             mMmCpuIo,
> +             mCpuHotPlugData->ArrayLength,     // PossibleCpuCount
> +             mCpuHotPlugData->ArrayLength - 1, // ApicIdCount
> +             mPluggedApicIds,
> +             &PluggedCount,
> +             mToUnplugApicIds,
> +             &ToUnplugCount
> +             );
> +  if (EFI_ERROR (Status)) {
> +    goto Fatal;
> +  }
> +  if (ToUnplugCount > 0) {
> +    DEBUG ((DEBUG_ERROR, "%a: hot-unplug is not supported yet\n",
> +      __FUNCTION__));
> +    goto Fatal;
> +  }
> +
>    //
>    // We've handled this MMI.
>    //
>    return EFI_SUCCESS;
>
>  Fatal:
>    ASSERT (FALSE);
>    CpuDeadLoop ();
>    //
>    // We couldn't handle this MMI.
>    //
>    return EFI_INTERRUPT_PENDING;
>  }
>
>
>  //
>  // Entry point function of this driver.
>  //
>  EFI_STATUS
>  EFIAPI
>  CpuHotplugEntry (
>    IN EFI_HANDLE       ImageHandle,
>    IN EFI_SYSTEM_TABLE *SystemTable
>    )
>  {
>    EFI_STATUS Status;
> +  UINTN      Size;
>
>    //
>    // This module should only be included when SMM support is required.
>    //
>    ASSERT (FeaturePcdGet (PcdSmmSmramRequire));
>    //
>    // This driver depends on the dynamically detected "SMRAM at default SMBASE"
>    // feature.
>    //
>    if (!PcdGetBool (PcdQ35SmramAtDefaultSmbase)) {
>      return EFI_UNSUPPORTED;
>    }
>
>    //
>    // Errors from here on are fatal; we cannot allow the boot to proceed if we
>    // can't set up this driver to handle CPU hotplug.
>    //
>    // First, collect the protocols needed later. All of these protocols are
>    // listed in our module DEPEX.
>    //
>    Status = gMmst->MmLocateProtocol (&gEfiMmCpuIoProtocolGuid,
>                      NULL /* Registration */, (VOID **)&mMmCpuIo);
>    if (EFI_ERROR (Status)) {
>      DEBUG ((DEBUG_ERROR, "%a: locate MmCpuIo: %r\n", __FUNCTION__, Status));
>      goto Fatal;
>    }
> +  Status = gMmst->MmLocateProtocol (&gEfiSmmCpuServiceProtocolGuid,
> +                    NULL /* Registration */, (VOID **)&mMmCpuService);
> +  if (EFI_ERROR (Status)) {
> +    DEBUG ((DEBUG_ERROR, "%a: locate MmCpuService: %r\n", __FUNCTION__,
> +      Status));
> +    goto Fatal;
> +  }
> +
> +  //
> +  // Our DEPEX on EFI_SMM_CPU_SERVICE_PROTOCOL guarantees that PiSmmCpuDxeSmm
> +  // has pointed PcdCpuHotPlugDataAddress to CPU_HOT_PLUG_DATA in SMRAM.
> +  //
> +  mCpuHotPlugData = (VOID *)(UINTN)PcdGet64 (PcdCpuHotPlugDataAddress);
> +  if (mCpuHotPlugData == NULL) {
> +    Status = EFI_NOT_FOUND;
> +    DEBUG ((DEBUG_ERROR, "%a: CPU_HOT_PLUG_DATA: %r\n", __FUNCTION__, Status));
> +    goto Fatal;
> +  }
> +  //
> +  // If the possible CPU count is 1, there's nothing for this driver to do.
> +  //
> +  if (mCpuHotPlugData->ArrayLength == 1) {
> +    return EFI_UNSUPPORTED;
> +  }
> +  //
> +  // Allocate the data structures that depend on the possible CPU count.
> +  //
> +  if (RETURN_ERROR (SafeUintnSub (mCpuHotPlugData->ArrayLength, 1, &Size)) ||
> +      RETURN_ERROR (SafeUintnMult (sizeof (APIC_ID), Size, &Size))) {
> +    Status = EFI_ABORTED;
> +    DEBUG ((DEBUG_ERROR, "%a: invalid CPU_HOT_PLUG_DATA\n", __FUNCTION__));
> +    goto Fatal;
> +  }
> +  Status = gMmst->MmAllocatePool (EfiRuntimeServicesData, Size,
> +                    (VOID **)&mPluggedApicIds);
> +  if (EFI_ERROR (Status)) {
> +    DEBUG ((DEBUG_ERROR, "%a: MmAllocatePool(): %r\n", __FUNCTION__, Status));
> +    goto Fatal;
> +  }
> +  Status = gMmst->MmAllocatePool (EfiRuntimeServicesData, Size,
> +                    (VOID **)&mToUnplugApicIds);
> +  if (EFI_ERROR (Status)) {
> +    DEBUG ((DEBUG_ERROR, "%a: MmAllocatePool(): %r\n", __FUNCTION__, Status));
> +    goto ReleasePluggedApicIds;
> +  }
>
>    //
>    // Sanity-check the CPU hotplug interface.
>    //
>    // Both of the following features are part of QEMU 5.0, introduced primarily
>    // in commit range 3e08b2b9cb64..3a61c8db9d25:
>    //
>    // (a) the QEMU_CPUHP_CMD_GET_ARCH_ID command of the modern CPU hotplug
>    //     interface,
>    //
>    // (b) the "SMRAM at default SMBASE" feature.
>    //
>    // From these, (b) is restricted to 5.0+ machine type versions, while (a)
>    // does not depend on machine type version. Because we ensured the stricter
>    // condition (b) through PcdQ35SmramAtDefaultSmbase above, the (a)
>    // QEMU_CPUHP_CMD_GET_ARCH_ID command must now be available too. While we
>    // can't verify the presence of precisely that command, we can still verify
>    // (sanity-check) that the modern interface is active, at least.
>    //
>    // Consult the "Typical usecases | Detecting and enabling modern CPU hotplug
>    // interface" section in QEMU's "docs/specs/acpi_cpu_hotplug.txt", on the
>    // following.
>    //
>    QemuCpuhpWriteCpuSelector (mMmCpuIo, 0);
>    QemuCpuhpWriteCpuSelector (mMmCpuIo, 0);
>    QemuCpuhpWriteCommand (mMmCpuIo, QEMU_CPUHP_CMD_GET_PENDING);
>    if (QemuCpuhpReadCommandData2 (mMmCpuIo) != 0) {
>      Status = EFI_NOT_FOUND;
>      DEBUG ((DEBUG_ERROR, "%a: modern CPU hotplug interface: %r\n",
>        __FUNCTION__, Status));
> -    goto Fatal;
> +    goto ReleaseToUnplugApicIds;
>    }
>
>    //
>    // Register the handler for the CPU Hotplug MMI.
>    //
>    Status = gMmst->MmiHandlerRegister (
>                      CpuHotplugMmi,
>                      NULL,            // HandlerType: root MMI handler
>                      &mDispatchHandle
>                      );
>    if (EFI_ERROR (Status)) {
>      DEBUG ((DEBUG_ERROR, "%a: MmiHandlerRegister(): %r\n", __FUNCTION__,
>        Status));
> -    goto Fatal;
> +    goto ReleaseToUnplugApicIds;
>    }
>
>    return EFI_SUCCESS;
>
> +ReleaseToUnplugApicIds:
> +  gMmst->MmFreePool (mToUnplugApicIds);
> +  mToUnplugApicIds = NULL;
> +
> +ReleasePluggedApicIds:
> +  gMmst->MmFreePool (mPluggedApicIds);
> +  mPluggedApicIds = NULL;
> +
>  Fatal:
>    ASSERT (FALSE);
>    CpuDeadLoop ();
>    return Status;
>  }
> --
> 2.19.1.3.g30247aa5d201
>
>