[Patch V2] BaseTools/BinToPcd: Add support for multiple binary input files

[Patch V2] BaseTools/BinToPcd: Add support for multiple binary input files

[Michael D Kinney]

From: "Kinney, Michael D" <michael.d.kinney@intel.com>

https://bugzilla.tianocore.org/show_bug.cgi?id=890

There are use cases where a VOID * PCD needs to be generated from multiple
binary input files.  This can be in the form of an array of fixed size
elements or a set of variable sized elements.

Update BinToPcd to support multiple one or more -i INPUTFILE arguments.
By default, the contents of each binary input file are concatenated in
the order provided.  This supports generating a PCD that is an array of
fixed size elements

Add -x, --xdr flags to BinToPcd  to encodes the PCD using the
Variable-Length Opaque Data of RFC 4506 External Data Representation
Standard (XDR).

    https://tools.ietf.org/html/rfc4506
    https://tools.ietf.org/html/rfc4506#section-4.10

The data format from RFC 4506 meets the requirements for a PCD that is a
set of variable sized elements in the Variable-Length Opaque Data format.
The overhead of this format is a 32-bit length and 0 to 3 bytes of padding
to align the next element at a 32-bit boundary.

Cc: Sean Brogan <sean.brogan@microsoft.com>
Cc: Yonghong Zhu <yonghong.zhu@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Michael D Kinney <michael.d.kinney@intel.com>
---
 BaseTools/Scripts/BinToPcd.py | 83 ++++++++++++++++++++++++++++---------------
 1 file changed, 54 insertions(+), 29 deletions(-)

diff --git a/BaseTools/Scripts/BinToPcd.py b/BaseTools/Scripts/BinToPcd.py
index 68a7ac652d..f2485a27fa 100644
--- a/BaseTools/Scripts/BinToPcd.py
+++ b/BaseTools/Scripts/BinToPcd.py
@@ -1,7 +1,7 @@
 ## @file
 # Convert a binary file to a VOID* PCD value or DSC file VOID* PCD statement.
 #
-# Copyright (c) 2016, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2016 - 2018, Intel Corporation. All rights reserved.<BR>
 # This program and the accompanying materials
 # are licensed and made available under the terms and conditions of the BSD License
 # which accompanies this distribution.  The full text of the license may be found at
@@ -18,14 +18,15 @@ BinToPcd
 import sys
 import argparse
 import re
+import xdrlib
 
 #
 # Globals for help information
 #
 __prog__        = 'BinToPcd'
-__version__     = '%s Version %s' % (__prog__, '0.9 ')
-__copyright__   = 'Copyright (c) 2016, Intel Corporation. All rights reserved.'
-__description__ = 'Convert a binary file to a VOID* PCD value or DSC file VOID* PCD statement.\n'
+__version__     = '%s Version %s' % (__prog__, '0.91 ')
+__copyright__   = 'Copyright (c) 2016 - 2018, Intel Corporation. All rights reserved.'
+__description__ = 'Convert one or more binary files to a VOID* PCD value or DSC file VOID* PCD statement.\n'
 
 if __name__ == '__main__':
   def ValidateUnsignedInteger (Argument):
@@ -50,21 +51,35 @@ if __name__ == '__main__':
       Message = '%s is not a valid GUID C name' % (Argument)
       raise argparse.ArgumentTypeError(Message)
     return Argument
-    
-  def ByteArray (Buffer):
+
+  def ByteArray (Buffer, Xdr = False):
+    if Xdr:
+      #
+      # If Xdr flag is set then encode data using the Variable-Length Opaque
+      # Data format of RFC 4506 External Data Representation Standard (XDR).
+      #
+      XdrEncoder = xdrlib.Packer()
+      for Item in Buffer:
+        XdrEncoder.pack_bytes(Item)
+      Buffer = XdrEncoder.get_buffer()
+    else:
+      #
+      # If Xdr flag is not set, then concatenate all the data
+      #
+      Buffer = ''.join(Buffer)
     #
-    # Append byte array of values of the form '{0x01, 0x02, ...}'
+    # Return a PCD value of the form '{0x01, 0x02, ...}' along with the PCD length in bytes
     #
-    return '{%s}' % (', '.join(['0x%02x' % (ord(Item)) for Item in Buffer]))
-    
+    return '{%s}' % (', '.join(['0x%02x' % (ord(Item)) for Item in Buffer])), len (Buffer)
+
   #
   # Create command line argument parser object
   #
   parser = argparse.ArgumentParser(prog = __prog__, version = __version__,
                                    description = __description__ + __copyright__,
                                    conflict_handler = 'resolve')
-  parser.add_argument("-i", "--input", dest = 'InputFile', type = argparse.FileType('rb'),
-                      help = "Input binary filename", required = True)
+  parser.add_argument("-i", "--input", dest = 'InputFile', type = argparse.FileType('rb'), action='append', required = True,
+                      help = "Input binary filename.  Multiple input files are combined into a single PCD.")
   parser.add_argument("-o", "--output", dest = 'OutputFile', type = argparse.FileType('wb'),
                       help = "Output filename for PCD value or PCD statement")
   parser.add_argument("-p", "--pcd", dest = 'PcdName', type = ValidatePcdName,
@@ -79,6 +94,8 @@ if __name__ == '__main__':
                       help = "UEFI variable name.  Only used with --type HII.")
   parser.add_argument("-g", "--variable-guid", type = ValidateGuidName, dest = 'VariableGuid',
                       help = "UEFI variable GUID C name.  Only used with --type HII.")
+  parser.add_argument("-x", "--xdr", dest = 'Xdr', action = "store_true",
+                      help = "Encode PCD using the Variable-Length Opaque Data format of RFC 4506 External Data Representation Standard (XDR)")
   parser.add_argument("-v", "--verbose", dest = 'Verbose', action = "store_true",
                       help = "Increase output messages")
   parser.add_argument("-q", "--quiet", dest = 'Quiet', action = "store_true",
@@ -92,14 +109,22 @@ if __name__ == '__main__':
   args = parser.parse_args()
 
   #
-  # Read binary input file
+  # Read all binary input files
   #
-  try:
-    Buffer = args.InputFile.read()
-    args.InputFile.close()
-  except:
-    print 'BinToPcd: error: can not read binary input file'
-    sys.exit()
+  Buffer = []
+  for File in args.InputFile:
+    try:
+      Buffer.append(File.read())
+      File.close()
+    except:
+      print 'BinToPcd: error: can not read binary input file', File
+      sys.exit()
+
+  #
+  # Convert PCD to an encoded string of hex values and determine the size of
+  # the encoded PCD in bytes.
+  #
+  PcdValue, PcdSize = ByteArray (Buffer, args.Xdr)
 
   #
   # Convert binary buffer to a DSC file PCD statement
@@ -107,7 +132,8 @@ if __name__ == '__main__':
   if args.PcdName is None:
     #
     # If PcdName is None, then only a PCD value is being requested.
-    Pcd = ByteArray (Buffer)
+    #
+    Pcd = PcdValue
     if args.Verbose:
       print 'PcdToBin: Convert binary file to PCD Value'
   elif args.PcdType is None:
@@ -121,14 +147,13 @@ if __name__ == '__main__':
       # If --max-size is not provided, then do not generate the syntax that
       # includes the maximum size.
       #
-      Pcd = '  %s|%s' % (args.PcdName, ByteArray (Buffer))
-    elif args.MaxSize < len(Buffer):
+      Pcd = '  %s|%s' % (args.PcdName, PcdValue)
+    elif args.MaxSize < PcdSize:
       print 'BinToPcd: error: argument --max-size is smaller than input file.'
       sys.exit()
     else:
-      Pcd = '  %s|%s|VOID*|%d' % (args.PcdName, ByteArray (Buffer), args.MaxSize)
-      args.MaxSize = len(Buffer)
-    
+      Pcd = '  %s|%s|VOID*|%d' % (args.PcdName, PcdValue, args.MaxSize)
+
     if args.Verbose:
       print 'PcdToBin: Convert binary file to PCD statement compatible with PCD sections:'
       print '    [PcdsFixedAtBuild]'
@@ -141,8 +166,8 @@ if __name__ == '__main__':
       # If --max-size is not provided, then set maximum size to the size of the
       # binary input file
       #
-      args.MaxSize = len(Buffer)
-    if args.MaxSize < len(Buffer):
+      args.MaxSize = PcdSize
+    if args.MaxSize < PcdSize:
       print 'BinToPcd: error: argument --max-size is smaller than input file.'
       sys.exit()
     if args.Offset is None:
@@ -150,12 +175,12 @@ if __name__ == '__main__':
       # if --offset is not provided, then set offset field to '*' so build
       # tools will compute offset of PCD in VPD region.
       #
-      Pcd = '  %s|*|%d|%s' % (args.PcdName, args.MaxSize, ByteArray (Buffer))
+      Pcd = '  %s|*|%d|%s' % (args.PcdName, args.MaxSize, PcdValue)
     else:
       #
       # Use the --offset value provided.
       #
-      Pcd = '  %s|%d|%d|%s' % (args.PcdName, args.Offset, args.MaxSize, ByteArray (Buffer))
+      Pcd = '  %s|%d|%d|%s' % (args.PcdName, args.Offset, args.MaxSize, PcdValue)
     if args.Verbose:
       print 'PcdToBin: Convert binary file to PCD statement compatible with PCD sections'
       print '    [PcdsDynamicVpd]'
@@ -172,7 +197,7 @@ if __name__ == '__main__':
       # Use UEFI Variable offset of 0 if --offset is not provided
       #
       args.Offset = 0
-    Pcd = '  %s|L"%s"|%s|%d|%s' % (args.PcdName, args.VariableName, args.VariableGuid, args.Offset, ByteArray (Buffer))
+    Pcd = '  %s|L"%s"|%s|%d|%s' % (args.PcdName, args.VariableName, args.VariableGuid, args.Offset, PcdValue)
     if args.Verbose:
       print 'PcdToBin: Convert binary file to PCD statement compatible with PCD sections'
       print '    [PcdsDynamicHii]'
-- 
2.14.2.windows.3

Re: [Patch V2] BaseTools/BinToPcd: Add support for multiple binary input files

[Zhu, Yonghong]

Reviewed-by: Yonghong Zhu <yonghong.zhu@intel.com> 

Best Regards,
Zhu Yonghong


-----Original Message-----
From: Kinney, Michael D 
Sent: Thursday, March 22, 2018 6:54 AM
To: edk2-devel@lists.01.org
Cc: Kinney, Michael D <michael.d.kinney@intel.com>; Sean Brogan <sean.brogan@microsoft.com>; Zhu, Yonghong <yonghong.zhu@intel.com>; Gao, Liming <liming.gao@intel.com>
Subject: [Patch V2] BaseTools/BinToPcd: Add support for multiple binary input files

From: "Kinney, Michael D" <michael.d.kinney@intel.com>

https://bugzilla.tianocore.org/show_bug.cgi?id=890

There are use cases where a VOID * PCD needs to be generated from multiple binary input files.  This can be in the form of an array of fixed size elements or a set of variable sized elements.

Update BinToPcd to support multiple one or more -i INPUTFILE arguments.
By default, the contents of each binary input file are concatenated in the order provided.  This supports generating a PCD that is an array of fixed size elements

Add -x, --xdr flags to BinToPcd  to encodes the PCD using the Variable-Length Opaque Data of RFC 4506 External Data Representation Standard (XDR).

    https://tools.ietf.org/html/rfc4506
    https://tools.ietf.org/html/rfc4506#section-4.10

The data format from RFC 4506 meets the requirements for a PCD that is a set of variable sized elements in the Variable-Length Opaque Data format.
The overhead of this format is a 32-bit length and 0 to 3 bytes of padding to align the next element at a 32-bit boundary.

Cc: Sean Brogan <sean.brogan@microsoft.com>
Cc: Yonghong Zhu <yonghong.zhu@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Michael D Kinney <michael.d.kinney@intel.com>
---
 BaseTools/Scripts/BinToPcd.py | 83 ++++++++++++++++++++++++++++---------------
 1 file changed, 54 insertions(+), 29 deletions(-)

diff --git a/BaseTools/Scripts/BinToPcd.py b/BaseTools/Scripts/BinToPcd.py index 68a7ac652d..f2485a27fa 100644
--- a/BaseTools/Scripts/BinToPcd.py
+++ b/BaseTools/Scripts/BinToPcd.py
@@ -1,7 +1,7 @@
 ## @file
 # Convert a binary file to a VOID* PCD value or DSC file VOID* PCD statement.
 #
-# Copyright (c) 2016, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2016 - 2018, Intel Corporation. All rights 
+reserved.<BR>
 # This program and the accompanying materials  # are licensed and made available under the terms and conditions of the BSD License  # which accompanies this distribution.  The full text of the license may be found at @@ -18,14 +18,15 @@ BinToPcd  import sys  import argparse  import re
+import xdrlib
 
 #
 # Globals for help information
 #
 __prog__        = 'BinToPcd'
-__version__     = '%s Version %s' % (__prog__, '0.9 ')
-__copyright__   = 'Copyright (c) 2016, Intel Corporation. All rights reserved.'
-__description__ = 'Convert a binary file to a VOID* PCD value or DSC file VOID* PCD statement.\n'
+__version__     = '%s Version %s' % (__prog__, '0.91 ')
+__copyright__   = 'Copyright (c) 2016 - 2018, Intel Corporation. All rights reserved.'
+__description__ = 'Convert one or more binary files to a VOID* PCD value or DSC file VOID* PCD statement.\n'
 
 if __name__ == '__main__':
   def ValidateUnsignedInteger (Argument):
@@ -50,21 +51,35 @@ if __name__ == '__main__':
       Message = '%s is not a valid GUID C name' % (Argument)
       raise argparse.ArgumentTypeError(Message)
     return Argument
-    
-  def ByteArray (Buffer):
+
+  def ByteArray (Buffer, Xdr = False):
+    if Xdr:
+      #
+      # If Xdr flag is set then encode data using the Variable-Length Opaque
+      # Data format of RFC 4506 External Data Representation Standard (XDR).
+      #
+      XdrEncoder = xdrlib.Packer()
+      for Item in Buffer:
+        XdrEncoder.pack_bytes(Item)
+      Buffer = XdrEncoder.get_buffer()
+    else:
+      #
+      # If Xdr flag is not set, then concatenate all the data
+      #
+      Buffer = ''.join(Buffer)
     #
-    # Append byte array of values of the form '{0x01, 0x02, ...}'
+    # Return a PCD value of the form '{0x01, 0x02, ...}' along with the 
+ PCD length in bytes
     #
-    return '{%s}' % (', '.join(['0x%02x' % (ord(Item)) for Item in Buffer]))
-    
+    return '{%s}' % (', '.join(['0x%02x' % (ord(Item)) for Item in 
+ Buffer])), len (Buffer)
+
   #
   # Create command line argument parser object
   #
   parser = argparse.ArgumentParser(prog = __prog__, version = __version__,
                                    description = __description__ + __copyright__,
                                    conflict_handler = 'resolve')
-  parser.add_argument("-i", "--input", dest = 'InputFile', type = argparse.FileType('rb'),
-                      help = "Input binary filename", required = True)
+  parser.add_argument("-i", "--input", dest = 'InputFile', type = argparse.FileType('rb'), action='append', required = True,
+                      help = "Input binary filename.  Multiple input 
+ files are combined into a single PCD.")
   parser.add_argument("-o", "--output", dest = 'OutputFile', type = argparse.FileType('wb'),
                       help = "Output filename for PCD value or PCD statement")
   parser.add_argument("-p", "--pcd", dest = 'PcdName', type = ValidatePcdName, @@ -79,6 +94,8 @@ if __name__ == '__main__':
                       help = "UEFI variable name.  Only used with --type HII.")
   parser.add_argument("-g", "--variable-guid", type = ValidateGuidName, dest = 'VariableGuid',
                       help = "UEFI variable GUID C name.  Only used with --type HII.")
+  parser.add_argument("-x", "--xdr", dest = 'Xdr', action = "store_true",
+                      help = "Encode PCD using the Variable-Length 
+ Opaque Data format of RFC 4506 External Data Representation Standard 
+ (XDR)")
   parser.add_argument("-v", "--verbose", dest = 'Verbose', action = "store_true",
                       help = "Increase output messages")
   parser.add_argument("-q", "--quiet", dest = 'Quiet', action = "store_true", @@ -92,14 +109,22 @@ if __name__ == '__main__':
   args = parser.parse_args()
 
   #
-  # Read binary input file
+  # Read all binary input files
   #
-  try:
-    Buffer = args.InputFile.read()
-    args.InputFile.close()
-  except:
-    print 'BinToPcd: error: can not read binary input file'
-    sys.exit()
+  Buffer = []
+  for File in args.InputFile:
+    try:
+      Buffer.append(File.read())
+      File.close()
+    except:
+      print 'BinToPcd: error: can not read binary input file', File
+      sys.exit()
+
+  #
+  # Convert PCD to an encoded string of hex values and determine the 
+ size of  # the encoded PCD in bytes.
+  #
+  PcdValue, PcdSize = ByteArray (Buffer, args.Xdr)
 
   #
   # Convert binary buffer to a DSC file PCD statement @@ -107,7 +132,8 @@ if __name__ == '__main__':
   if args.PcdName is None:
     #
     # If PcdName is None, then only a PCD value is being requested.
-    Pcd = ByteArray (Buffer)
+    #
+    Pcd = PcdValue
     if args.Verbose:
       print 'PcdToBin: Convert binary file to PCD Value'
   elif args.PcdType is None:
@@ -121,14 +147,13 @@ if __name__ == '__main__':
       # If --max-size is not provided, then do not generate the syntax that
       # includes the maximum size.
       #
-      Pcd = '  %s|%s' % (args.PcdName, ByteArray (Buffer))
-    elif args.MaxSize < len(Buffer):
+      Pcd = '  %s|%s' % (args.PcdName, PcdValue)
+    elif args.MaxSize < PcdSize:
       print 'BinToPcd: error: argument --max-size is smaller than input file.'
       sys.exit()
     else:
-      Pcd = '  %s|%s|VOID*|%d' % (args.PcdName, ByteArray (Buffer), args.MaxSize)
-      args.MaxSize = len(Buffer)
-    
+      Pcd = '  %s|%s|VOID*|%d' % (args.PcdName, PcdValue, args.MaxSize)
+
     if args.Verbose:
       print 'PcdToBin: Convert binary file to PCD statement compatible with PCD sections:'
       print '    [PcdsFixedAtBuild]'
@@ -141,8 +166,8 @@ if __name__ == '__main__':
       # If --max-size is not provided, then set maximum size to the size of the
       # binary input file
       #
-      args.MaxSize = len(Buffer)
-    if args.MaxSize < len(Buffer):
+      args.MaxSize = PcdSize
+    if args.MaxSize < PcdSize:
       print 'BinToPcd: error: argument --max-size is smaller than input file.'
       sys.exit()
     if args.Offset is None:
@@ -150,12 +175,12 @@ if __name__ == '__main__':
       # if --offset is not provided, then set offset field to '*' so build
       # tools will compute offset of PCD in VPD region.
       #
-      Pcd = '  %s|*|%d|%s' % (args.PcdName, args.MaxSize, ByteArray (Buffer))
+      Pcd = '  %s|*|%d|%s' % (args.PcdName, args.MaxSize, PcdValue)
     else:
       #
       # Use the --offset value provided.
       #
-      Pcd = '  %s|%d|%d|%s' % (args.PcdName, args.Offset, args.MaxSize, ByteArray (Buffer))
+      Pcd = '  %s|%d|%d|%s' % (args.PcdName, args.Offset, args.MaxSize, 
+ PcdValue)
     if args.Verbose:
       print 'PcdToBin: Convert binary file to PCD statement compatible with PCD sections'
       print '    [PcdsDynamicVpd]'
@@ -172,7 +197,7 @@ if __name__ == '__main__':
       # Use UEFI Variable offset of 0 if --offset is not provided
       #
       args.Offset = 0
-    Pcd = '  %s|L"%s"|%s|%d|%s' % (args.PcdName, args.VariableName, args.VariableGuid, args.Offset, ByteArray (Buffer))
+    Pcd = '  %s|L"%s"|%s|%d|%s' % (args.PcdName, args.VariableName, 
+ args.VariableGuid, args.Offset, PcdValue)
     if args.Verbose:
       print 'PcdToBin: Convert binary file to PCD statement compatible with PCD sections'
       print '    [PcdsDynamicHii]'
--
2.14.2.windows.3