[PATCH v3 07/10] OvmfPkg/CpuHotplugSmm: introduce UnplugCpus()

["Ankur Arora" <ankur.a.arora@oracle.com>]

Introduce UnplugCpus() which, for each APIC ID being unplugged:

  * informs PiSmmCpuDxeSmm by calling
    EFI_SMM_CPU_SERVICE_PROTOCOL.RemoveProcessor()
  * stores the APIC ID in CPU_HOT_EJECT_DATA.ApicIdMap, such
    that it can later be used in the eject phase at SMI exit

Also add a holding area in CpuEject() where CPUs being ejected
can wait until ejected.

Cc: Laszlo Ersek <lersek@redhat.com>
Cc: Jordan Justen <jordan.l.justen@intel.com>
Cc: Ard Biesheuvel <ard.biesheuvel@arm.com>
Cc: Igor Mammedov <imammedo@redhat.com>
Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Cc: Aaron Young <aaron.young@oracle.com>
Ref: https://bugzilla.tianocore.org/show_bug.cgi?id=3132
Signed-off-by: Ankur Arora <ankur.a.arora@oracle.com>
---
 OvmfPkg/CpuHotplugSmm/CpuHotplug.c | 106 +++++++++++++++++++++++++++++++++++++
 1 file changed, 106 insertions(+)

diff --git a/OvmfPkg/CpuHotplugSmm/CpuHotplug.c b/OvmfPkg/CpuHotplugSmm/CpuHotplug.c
index f088049c7aef..4048490783e4 100644
--- a/OvmfPkg/CpuHotplugSmm/CpuHotplug.c
+++ b/OvmfPkg/CpuHotplugSmm/CpuHotplug.c
@@ -183,6 +183,99 @@ Fatal:
 }
 
 /**
+  CPU Hot-unplug handler function.
+
+  @param[in] mUnplugApicIds      List of APIC IDs to be unplugged.
+
+  @param[in] ToUnplugCount       Count of APIC IDs to be unplugged.
+
+  @retval EFI_SUCCESS            Some of the requested APIC IDs will be hot-unplugged.
+
+  @retval EFI_INTERRUPT_PENDING  Fatal error while hot-plugging.
+
+**/
+STATIC
+EFI_STATUS
+EFIAPI
+UnplugCpus(
+  IN APIC_ID                      *mUnplugApicIds,
+  IN UINT32                       ToUnplugCount
+  )
+{
+  EFI_STATUS Status = EFI_SUCCESS;
+  UINT32     ToUnplugIdx;
+  UINT32     EjectCount = 0;
+  UINTN      ProcessorNum;
+
+  ToUnplugIdx = 0;
+  while (ToUnplugIdx < ToUnplugCount) {
+    APIC_ID    RemoveApicId;
+
+    RemoveApicId = mUnplugApicIds[ToUnplugIdx];
+
+    //
+    // mCpuHotPlugData->ApicId maps ProcessorNum -> ApicId. Use it to find
+    // the ProcessorNum for the APIC ID to be removed.
+    //
+    for (ProcessorNum = 0;
+         ProcessorNum < mCpuHotPlugData->ArrayLength;
+         ProcessorNum++) {
+      if (mCpuHotPlugData->ApicId[ProcessorNum] == RemoveApicId) {
+        break;
+      }
+    }
+
+      //
+      // Ignore the unplug if APIC ID not found
+      //
+    if (ProcessorNum == mCpuHotPlugData->ArrayLength) {
+      DEBUG ((DEBUG_INFO, "%a: did not find APIC ID " FMT_APIC_ID " to unplug\n",
+        __FUNCTION__, RemoveApicId));
+      ToUnplugIdx++;
+      continue;
+    }
+
+    //
+    // Mark ProcessorNum for removal from SMM data structures
+    //
+    Status = mMmCpuService->RemoveProcessor (mMmCpuService, ProcessorNum);
+
+    if (EFI_ERROR(Status)) {
+      DEBUG ((DEBUG_ERROR, "%a: RemoveProcessor(" FMT_APIC_ID "): %r\n",
+        __FUNCTION__, RemoveApicId, Status));
+      goto Fatal;
+    } else {
+      //
+      // Stash the APIC IDs so we can do the actual unplug later.
+      //
+      if (mCpuHotEjectData->ApicIdMap[ProcessorNum] != CPU_EJECT_INVALID) {
+        //
+        // Since ProcessorNum and APIC-ID map 1-1, so a valid
+        // mCpuHotEjectData->ApicIdMap[ProcessorNum] means something
+        // is horribly wrong.
+        //
+        DEBUG ((DEBUG_ERROR, "%a: ProcessorNum %u maps to %llx, cannot map to " FMT_APIC_ID "\n",
+          __FUNCTION__, ProcessorNum, mCpuHotEjectData->ApicIdMap[ProcessorNum], RemoveApicId));
+        goto Fatal;
+      }
+
+      mCpuHotEjectData->ApicIdMap[ProcessorNum] = (UINT64)RemoveApicId;
+      EjectCount++;
+    }
+
+    ToUnplugIdx++;
+  }
+
+  //
+  // We've handled this unplug.
+  //
+  return EFI_SUCCESS;
+
+Fatal:
+  return EFI_INTERRUPT_PENDING;
+}
+
+/**
   CPU Hotplug MMI handler function.
 
   This is a root MMI handler.
@@ -298,6 +391,8 @@ CpuHotplugMmi (
 
   if (PluggedCount > 0) {
     Status = PlugCpus(mPluggedApicIds, PluggedCount);
+  } else if (ToUnplugCount > 0) {
+    Status = UnplugCpus(mToUnplugApicIds, ToUnplugCount);
   }
 
   if (EFI_ERROR(Status)) {
@@ -334,6 +429,17 @@ CpuEject(
   if (ApicId == CPU_EJECT_INVALID) {
     return;
   }
+
+  //
+  // CPU(s) being unplugged get here from SmmCpuFeaturesSmiRendezvousExit()
+  // after having been cleared to exit the SMI by the monarch and thus have
+  // no SMM processing remaining.
+  //
+  // Given that we cannot allow them to escape to the guest, we pen them
+  // here until the SMM monarch tells the HW to unplug them.
+  //
+
+  CpuDeadLoop ();
 }
 
 //
-- 
2.9.3