[PATCH 0/3] Xen PCI passthrough fixes

[PATCH 0/3] Xen PCI passthrough fixes

[Igor Druzhinin]

Igor Druzhinin (3):
  OvmfPkg/XenSupport: remove usage of prefetchable PCI host bridge
    aperture
  OvmfPkg/XenSupport: use a correct PCI host bridge aperture for BAR64
  OvmfPkg/XenSupport: turn off address decoding before BAR sizing

 OvmfPkg/Library/PciHostBridgeLib/XenSupport.c | 44 ++++++++++++++++++++++-----
 1 file changed, 37 insertions(+), 7 deletions(-)

-- 
2.7.4

Re: [PATCH 0/3] Xen PCI passthrough fixes

[Konrad Rzeszutek Wilk]

On Mon, Mar 04, 2019 at 03:04:05PM +0000, Igor Druzhinin wrote:
> Igor Druzhinin (3):

Could you also CC xen-devel please?
>   OvmfPkg/XenSupport: remove usage of prefetchable PCI host bridge
>     aperture
>   OvmfPkg/XenSupport: use a correct PCI host bridge aperture for BAR64
>   OvmfPkg/XenSupport: turn off address decoding before BAR sizing
> 
>  OvmfPkg/Library/PciHostBridgeLib/XenSupport.c | 44 ++++++++++++++++++++++-----
>  1 file changed, 37 insertions(+), 7 deletions(-)
> 
> -- 
> 2.7.4
> 
> _______________________________________________
> edk2-devel mailing list
> edk2-devel@lists.01.org
> https://lists.01.org/mailman/listinfo/edk2-devel

[PATCH 0/3] Xen PCI passthrough fixes

[Igor Druzhinin]

Igor Druzhinin (3):
  OvmfPkg/XenSupport: remove usage of prefetchable PCI host bridge
    aperture
  OvmfPkg/XenSupport: use a correct PCI host bridge aperture for BAR64
  OvmfPkg/XenSupport: turn off address decoding before BAR sizing

 OvmfPkg/Library/PciHostBridgeLib/XenSupport.c | 53 +++++++++++++++++----------
 1 file changed, 34 insertions(+), 19 deletions(-)

-- 
2.7.4

[PATCH v3 1/3] OvmfPkg/XenSupport: remove usage of prefetchable PCI host bridge aperture

[Igor Druzhinin]

This aperture doesn't exist in QEMU-XEN and hvmloader places BARs
in arbitrary order disregarding prefetchable bit. This makes
prefetchable and non-prefetchable BARs to follow each other that's
quite likely with PCI passthrough devices. In that case, the existing
code, that tries to work out aperture boundaries by reading hvmloader
BAR placement, will report a bogus prefetchable aperture which overlaps
with the regular one. It will eventually trigger an assertion in
DXE PCI initialization code.

Do the same thing as OVMF on QEMU-KVM and pass a non-existing aperture
there. It's not necessary to pass additional allocation flags as we set
ResourceAssigned flag on the root bridge which means they will be ignored.

Reviewed-by: Anthony PERARD <anthony.perard@citrix.com>
Signed-off-by: Igor Druzhinin <igor.druzhinin@citrix.com>
---
 OvmfPkg/Library/PciHostBridgeLib/XenSupport.c | 34 ++++++++++-----------------
 1 file changed, 12 insertions(+), 22 deletions(-)

diff --git a/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c b/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
index afb2c5e..354b0a5 100644
--- a/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
+++ b/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
@@ -60,9 +60,7 @@ PcatPciRootBridgeParseBars (
   IN UINTN                          BarOffsetEnd,
   IN PCI_ROOT_BRIDGE_APERTURE       *Io,
   IN PCI_ROOT_BRIDGE_APERTURE       *Mem,
-  IN PCI_ROOT_BRIDGE_APERTURE       *MemAbove4G,
-  IN PCI_ROOT_BRIDGE_APERTURE       *PMem,
-  IN PCI_ROOT_BRIDGE_APERTURE       *PMemAbove4G
+  IN PCI_ROOT_BRIDGE_APERTURE       *MemAbove4G
 
 )
 {
@@ -123,11 +121,7 @@ PcatPciRootBridgeParseBars (
           //
           Length = ((~Length) + 1) & 0xffffffff;
 
-          if ((Value & BIT3) == BIT3) {
-            MemAperture = PMem;
-          } else {
-            MemAperture = Mem;
-          }
+          MemAperture = Mem;
         } else {
           //
           // 64bit
@@ -143,11 +137,7 @@ PcatPciRootBridgeParseBars (
           Length = Length | LShiftU64 ((UINT64) UpperValue, 32);
           Length = (~Length) + 1;
 
-          if ((Value & BIT3) == BIT3) {
-            MemAperture = PMemAbove4G;
-          } else {
-            MemAperture = MemAbove4G;
-          }
+          MemAperture = MemAbove4G;
         }
 
         Limit = Base + Length - 1;
@@ -164,6 +154,8 @@ PcatPciRootBridgeParseBars (
   }
 }
 
+STATIC PCI_ROOT_BRIDGE_APERTURE mNonExistAperture = { MAX_UINT64, 0 };
+
 PCI_ROOT_BRIDGE *
 ScanForRootBridges (
   UINTN      *NumberOfRootBridges
@@ -180,7 +172,7 @@ ScanForRootBridges (
   UINT64     Base;
   UINT64     Limit;
   UINT64     Value;
-  PCI_ROOT_BRIDGE_APERTURE Io, Mem, MemAbove4G, PMem, PMemAbove4G, *MemAperture;
+  PCI_ROOT_BRIDGE_APERTURE Io, Mem, MemAbove4G, *MemAperture;
   PCI_ROOT_BRIDGE *RootBridges;
   UINTN      BarOffsetEnd;
 
@@ -200,9 +192,7 @@ ScanForRootBridges (
     ZeroMem (&Io, sizeof (Io));
     ZeroMem (&Mem, sizeof (Mem));
     ZeroMem (&MemAbove4G, sizeof (MemAbove4G));
-    ZeroMem (&PMem, sizeof (PMem));
-    ZeroMem (&PMemAbove4G, sizeof (PMemAbove4G));
-    Io.Base = Mem.Base = MemAbove4G.Base = PMem.Base = PMemAbove4G.Base = MAX_UINT64;
+    Io.Base = Mem.Base = MemAbove4G.Base = MAX_UINT64;
     //
     // Scan all the PCI devices on the primary bus of the PCI root bridge
     //
@@ -307,16 +297,17 @@ ScanForRootBridges (
 
           //
           // Get the Prefetchable Memory range that the PPB is decoding
+          // and merge it into Memory range
           //
           Value = Pci.Bridge.PrefetchableMemoryBase & 0x0f;
           Base = ((UINT32) Pci.Bridge.PrefetchableMemoryBase & 0xfff0) << 16;
           Limit = (((UINT32) Pci.Bridge.PrefetchableMemoryLimit & 0xfff0)
                    << 16) | 0xfffff;
-          MemAperture = &PMem;
+          MemAperture = &Mem;
           if (Value == BIT0) {
             Base |= LShiftU64 (Pci.Bridge.PrefetchableBaseUpper32, 32);
             Limit |= LShiftU64 (Pci.Bridge.PrefetchableLimitUpper32, 32);
-            MemAperture = &PMemAbove4G;
+            MemAperture = &MemAbove4G;
           }
           if (Base < Limit) {
             if (MemAperture->Base > Base) {
@@ -367,8 +358,7 @@ ScanForRootBridges (
           OFFSET_OF (PCI_TYPE00, Device.Bar),
           BarOffsetEnd,
           &Io,
-          &Mem, &MemAbove4G,
-          &PMem, &PMemAbove4G
+          &Mem, &MemAbove4G
         );
 
         //
@@ -440,7 +430,7 @@ ScanForRootBridges (
       InitRootBridge (
         Attributes, Attributes, 0,
         (UINT8) PrimaryBus, (UINT8) SubBus,
-        &Io, &Mem, &MemAbove4G, &PMem, &PMemAbove4G,
+        &Io, &Mem, &MemAbove4G, &mNonExistAperture, &mNonExistAperture,
         &RootBridges[*NumberOfRootBridges]
       );
       RootBridges[*NumberOfRootBridges].ResourceAssigned = TRUE;
-- 
2.7.4

[PATCH v3 2/3] OvmfPkg/XenSupport: use a correct PCI host bridge aperture for BAR64

[Igor Druzhinin]

In case BAR64 is placed below 4G choose the correct aperture.
This fixes a failed assertion down the code path.

Acked-by: Anthony PERARD <anthony.perard@citrix.com>
Signed-off-by: Igor Druzhinin <igor.druzhinin@citrix.com>
---
 OvmfPkg/Library/PciHostBridgeLib/XenSupport.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c b/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
index 354b0a5..76fca53 100644
--- a/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
+++ b/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
@@ -137,7 +137,11 @@ PcatPciRootBridgeParseBars (
           Length = Length | LShiftU64 ((UINT64) UpperValue, 32);
           Length = (~Length) + 1;
 
-          MemAperture = MemAbove4G;
+          if (Base < BASE_4GB) {
+            MemAperture = Mem;
+          } else {
+            MemAperture = MemAbove4G;
+          }
         }
 
         Limit = Base + Length - 1;
-- 
2.7.4

[PATCH v3 3/3] OvmfPkg/XenSupport: turn off address decoding before BAR sizing

[Igor Druzhinin]

On Xen, hvmloader firmware leaves address decoding enabled for
enumerated PCI device before jumping into OVMF. OVMF seems to
expect it to be disabled and tries to size PCI BARs in several places
without disabling it which causes BAR64, for example, being
incorrectly placed by QEMU.

Fix it by disabling PCI address decoding explicitly before the
first attempt to size BARs on Xen.

Signed-off-by: Igor Druzhinin <igor.druzhinin@citrix.com>
---
Changes in v3:
- dropped unused arguments and rename PcatPciRootBridgeDecoding function
- make mask application more clear as suggested
---
 OvmfPkg/Library/PciHostBridgeLib/XenSupport.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c b/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
index 76fca53..b41bd6b 100644
--- a/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
+++ b/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
@@ -49,6 +49,22 @@ PcatPciRootBridgeBarExisted (
   EnableInterrupts ();
 }
 
+#define PCI_COMMAND_DECODE ((UINT16)(EFI_PCI_COMMAND_IO_SPACE | \
+                                     EFI_PCI_COMMAND_MEMORY_SPACE))
+STATIC
+VOID
+PcatPciRootBridgeDecodingDisable (
+  IN  UINTN                          Address
+  )
+{
+  UINT16                             Value;
+
+  Value = PciRead16 (Address);
+  if (Value & PCI_COMMAND_DECODE) {
+    PciWrite16 (Address, Value & ~(UINT32)PCI_COMMAND_DECODE);
+  }
+}
+
 STATIC
 VOID
 PcatPciRootBridgeParseBars (
@@ -75,6 +91,11 @@ PcatPciRootBridgeParseBars (
   UINT64                            Limit;
   PCI_ROOT_BRIDGE_APERTURE          *MemAperture;
 
+  // Disable address decoding for every device before OVMF starts sizing it
+  PcatPciRootBridgeDecodingDisable (
+    PCI_LIB_ADDRESS (Bus, Device, Function, PCI_COMMAND_OFFSET)
+  );
+
   for (Offset = BarOffsetBase; Offset < BarOffsetEnd; Offset += sizeof (UINT32)) {
     PcatPciRootBridgeBarExisted (
       PCI_LIB_ADDRESS (Bus, Device, Function, Offset),
-- 
2.7.4

Re: [PATCH v3 3/3] OvmfPkg/XenSupport: turn off address decoding before BAR sizing

[Laszlo Ersek]

On 04/25/19 22:23, Igor Druzhinin wrote:
> On Xen, hvmloader firmware leaves address decoding enabled for
> enumerated PCI device before jumping into OVMF. OVMF seems to
> expect it to be disabled and tries to size PCI BARs in several places
> without disabling it which causes BAR64, for example, being
> incorrectly placed by QEMU.
> 
> Fix it by disabling PCI address decoding explicitly before the
> first attempt to size BARs on Xen.
> 
> Signed-off-by: Igor Druzhinin <igor.druzhinin@citrix.com>
> ---
> Changes in v3:
> - dropped unused arguments and rename PcatPciRootBridgeDecoding function
> - make mask application more clear as suggested
> ---
>  OvmfPkg/Library/PciHostBridgeLib/XenSupport.c | 21 +++++++++++++++++++++
>  1 file changed, 21 insertions(+)
> 
> diff --git a/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c b/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
> index 76fca53..b41bd6b 100644
> --- a/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
> +++ b/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
> @@ -49,6 +49,22 @@ PcatPciRootBridgeBarExisted (
>    EnableInterrupts ();
>  }
>  
> +#define PCI_COMMAND_DECODE ((UINT16)(EFI_PCI_COMMAND_IO_SPACE | \
> +                                     EFI_PCI_COMMAND_MEMORY_SPACE))
> +STATIC
> +VOID
> +PcatPciRootBridgeDecodingDisable (
> +  IN  UINTN                          Address
> +  )
> +{
> +  UINT16                             Value;
> +
> +  Value = PciRead16 (Address);
> +  if (Value & PCI_COMMAND_DECODE) {
> +    PciWrite16 (Address, Value & ~(UINT32)PCI_COMMAND_DECODE);
> +  }
> +}
> +
>  STATIC
>  VOID
>  PcatPciRootBridgeParseBars (
> @@ -75,6 +91,11 @@ PcatPciRootBridgeParseBars (
>    UINT64                            Limit;
>    PCI_ROOT_BRIDGE_APERTURE          *MemAperture;
>  
> +  // Disable address decoding for every device before OVMF starts sizing it
> +  PcatPciRootBridgeDecodingDisable (
> +    PCI_LIB_ADDRESS (Bus, Device, Function, PCI_COMMAND_OFFSET)
> +  );
> +
>    for (Offset = BarOffsetBase; Offset < BarOffsetEnd; Offset += sizeof (UINT32)) {
>      PcatPciRootBridgeBarExisted (
>        PCI_LIB_ADDRESS (Bus, Device, Function, Offset),
> 

Acked-by: Laszlo Ersek <lersek@redhat.com>

Someone from the Xen community please ACK this too, and then we can push
the series.

Thanks,
Laszlo

Re: [PATCH v3 3/3] OvmfPkg/XenSupport: turn off address decoding before BAR sizing

[Anthony PERARD]

On Fri, Apr 26, 2019 at 08:15:07PM +0200, Laszlo Ersek wrote:
> On 04/25/19 22:23, Igor Druzhinin wrote:
> > On Xen, hvmloader firmware leaves address decoding enabled for
> > enumerated PCI device before jumping into OVMF. OVMF seems to
> > expect it to be disabled and tries to size PCI BARs in several places
> > without disabling it which causes BAR64, for example, being
> > incorrectly placed by QEMU.
> > 
> > Fix it by disabling PCI address decoding explicitly before the
> > first attempt to size BARs on Xen.
> > 
> > Signed-off-by: Igor Druzhinin <igor.druzhinin@citrix.com>
> > ---
> > Changes in v3:
> > - dropped unused arguments and rename PcatPciRootBridgeDecoding function
> > - make mask application more clear as suggested
> > ---
> >  OvmfPkg/Library/PciHostBridgeLib/XenSupport.c | 21 +++++++++++++++++++++
> >  1 file changed, 21 insertions(+)
> > 
> > diff --git a/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c b/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
> > index 76fca53..b41bd6b 100644
> > --- a/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
> > +++ b/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
> > @@ -49,6 +49,22 @@ PcatPciRootBridgeBarExisted (
> >    EnableInterrupts ();
> >  }
> >  
> > +#define PCI_COMMAND_DECODE ((UINT16)(EFI_PCI_COMMAND_IO_SPACE | \
> > +                                     EFI_PCI_COMMAND_MEMORY_SPACE))
> > +STATIC
> > +VOID
> > +PcatPciRootBridgeDecodingDisable (
> > +  IN  UINTN                          Address
> > +  )
> > +{
> > +  UINT16                             Value;
> > +
> > +  Value = PciRead16 (Address);
> > +  if (Value & PCI_COMMAND_DECODE) {
> > +    PciWrite16 (Address, Value & ~(UINT32)PCI_COMMAND_DECODE);
> > +  }
> > +}
> > +
> >  STATIC
> >  VOID
> >  PcatPciRootBridgeParseBars (
> > @@ -75,6 +91,11 @@ PcatPciRootBridgeParseBars (
> >    UINT64                            Limit;
> >    PCI_ROOT_BRIDGE_APERTURE          *MemAperture;
> >  
> > +  // Disable address decoding for every device before OVMF starts sizing it
> > +  PcatPciRootBridgeDecodingDisable (
> > +    PCI_LIB_ADDRESS (Bus, Device, Function, PCI_COMMAND_OFFSET)
> > +  );
> > +
> >    for (Offset = BarOffsetBase; Offset < BarOffsetEnd; Offset += sizeof (UINT32)) {
> >      PcatPciRootBridgeBarExisted (
> >        PCI_LIB_ADDRESS (Bus, Device, Function, Offset),
> > 
> 
> Acked-by: Laszlo Ersek <lersek@redhat.com>
> 
> Someone from the Xen community please ACK this too, and then we can push
> the series.

Acked-by: Anthony PERARD <anthony.perard@citrix.com>

Thanks,

-- 
Anthony PERARD

Re: [edk2-devel] [PATCH v3 3/3] OvmfPkg/XenSupport: turn off address decoding before BAR sizing

[Laszlo Ersek]

On 04/30/19 18:35, Anthony PERARD wrote:
> On Fri, Apr 26, 2019 at 08:15:07PM +0200, Laszlo Ersek wrote:
>> On 04/25/19 22:23, Igor Druzhinin wrote:
>>> On Xen, hvmloader firmware leaves address decoding enabled for
>>> enumerated PCI device before jumping into OVMF. OVMF seems to
>>> expect it to be disabled and tries to size PCI BARs in several places
>>> without disabling it which causes BAR64, for example, being
>>> incorrectly placed by QEMU.
>>>
>>> Fix it by disabling PCI address decoding explicitly before the
>>> first attempt to size BARs on Xen.
>>>
>>> Signed-off-by: Igor Druzhinin <igor.druzhinin@citrix.com>
>>> ---
>>> Changes in v3:
>>> - dropped unused arguments and rename PcatPciRootBridgeDecoding function
>>> - make mask application more clear as suggested
>>> ---
>>>  OvmfPkg/Library/PciHostBridgeLib/XenSupport.c | 21 +++++++++++++++++++++
>>>  1 file changed, 21 insertions(+)
>>>
>>> diff --git a/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c b/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
>>> index 76fca53..b41bd6b 100644
>>> --- a/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
>>> +++ b/OvmfPkg/Library/PciHostBridgeLib/XenSupport.c
>>> @@ -49,6 +49,22 @@ PcatPciRootBridgeBarExisted (
>>>    EnableInterrupts ();
>>>  }
>>>  
>>> +#define PCI_COMMAND_DECODE ((UINT16)(EFI_PCI_COMMAND_IO_SPACE | \
>>> +                                     EFI_PCI_COMMAND_MEMORY_SPACE))
>>> +STATIC
>>> +VOID
>>> +PcatPciRootBridgeDecodingDisable (
>>> +  IN  UINTN                          Address
>>> +  )
>>> +{
>>> +  UINT16                             Value;
>>> +
>>> +  Value = PciRead16 (Address);
>>> +  if (Value & PCI_COMMAND_DECODE) {
>>> +    PciWrite16 (Address, Value & ~(UINT32)PCI_COMMAND_DECODE);
>>> +  }
>>> +}
>>> +
>>>  STATIC
>>>  VOID
>>>  PcatPciRootBridgeParseBars (
>>> @@ -75,6 +91,11 @@ PcatPciRootBridgeParseBars (
>>>    UINT64                            Limit;
>>>    PCI_ROOT_BRIDGE_APERTURE          *MemAperture;
>>>  
>>> +  // Disable address decoding for every device before OVMF starts sizing it
>>> +  PcatPciRootBridgeDecodingDisable (
>>> +    PCI_LIB_ADDRESS (Bus, Device, Function, PCI_COMMAND_OFFSET)
>>> +  );
>>> +
>>>    for (Offset = BarOffsetBase; Offset < BarOffsetEnd; Offset += sizeof (UINT32)) {
>>>      PcatPciRootBridgeBarExisted (
>>>        PCI_LIB_ADDRESS (Bus, Device, Function, Offset),
>>>
>>
>> Acked-by: Laszlo Ersek <lersek@redhat.com>
>>
>> Someone from the Xen community please ACK this too, and then we can push
>> the series.
> 
> Acked-by: Anthony PERARD <anthony.perard@citrix.com>

Series pushed as commit range 9fb2ce2f465d..0c40c9c925ca.

Thanks
Laszlo