Re: [Patch] BaseTool: Variable Merge.

["Gao, Liming" <liming.gao@intel.com>]

Bob:
  Please update commit message title with the detail information. 

Thanks
Liming
>-----Original Message-----
>From: edk2-devel [mailto:edk2-devel-bounces@lists.01.org] On Behalf Of
>BobCF
>Sent: Friday, September 07, 2018 3:45 PM
>To: edk2-devel@lists.01.org
>Cc: Gao, Liming <liming.gao@intel.com>
>Subject: [edk2] [Patch] BaseTool: Variable Merge.
>
>If Structure PCD and Normal Pcd refer to the
>same variable, do variable merge.
>
>Enhance error message.
>
>Contributed-under: TianoCore Contribution Agreement 1.1
>Signed-off-by: Bob Feng <bob.c.feng@intel.com>
>Cc: Liming Gao <liming.gao@intel.com>
>---
> BaseTools/Source/Python/AutoGen/AutoGen.py         |  3 +-
> BaseTools/Source/Python/AutoGen/GenVar.py          | 71
>++++++++++++++++------
> .../Source/Python/Workspace/BuildClassObject.py    |  2 +
> BaseTools/Source/Python/Workspace/DscBuildData.py  |  3 +-
> 4 files changed, 60 insertions(+), 19 deletions(-)
>
>diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py
>b/BaseTools/Source/Python/AutoGen/AutoGen.py
>index 95370d1821..d1bd1b1d4c 100644
>--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
>+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
>@@ -1211,11 +1211,11 @@ class PlatformAutoGen(AutoGen):
>                     continue
>                 if len(Sku.VariableName) > 0:
>                     VariableGuidStructure = Sku.VariableGuidValue
>                     VariableGuid =
>GuidStructureStringToGuidString(VariableGuidStructure)
>                     for StorageName in Sku.DefaultStoreDict:
>-                        VariableInfo.append_variable(var_info(Index, pcdname,
>StorageName, SkuName, StringToArray(Sku.VariableName), VariableGuid,
>Sku.VariableOffset, Sku.VariableAttribute, Sku.HiiDefaultValue,
>Sku.DefaultStoreDict[StorageName], Pcd.DatumType))
>+                        VariableInfo.append_variable(var_info(Index, pcdname,
>StorageName, SkuName, StringToArray(Sku.VariableName), VariableGuid,
>Sku.VariableOffset, Sku.VariableAttribute, Sku.HiiDefaultValue,
>Sku.DefaultStoreDict[StorageName], Pcd.DatumType,
>Pcd.CustomAttribute['DscPosition'], Pcd.CustomAttribute.get('IsStru',False)))
>             Index += 1
>         return VariableInfo
>
>     def UpdateNVStoreMaxSize(self, OrgVpdFile):
>         if self.VariableInfo:
>@@ -2104,10 +2104,11 @@ class PlatformAutoGen(AutoGen):
>                 EdkLogger.error('build', FORMAT_INVALID, Cause, File=self.MetaFile,
>                                 ExtraData="%s.%s" % (ToPcd.TokenSpaceGuidCName,
>TokenCName))
>             ToPcd.validateranges = FromPcd.validateranges
>             ToPcd.validlists = FromPcd.validlists
>             ToPcd.expressions = FromPcd.expressions
>+            ToPcd.CustomAttribute = FromPcd.CustomAttribute
>
>         if FromPcd is not None and ToPcd.DatumType == TAB_VOID and not
>ToPcd.MaxDatumSize:
>             EdkLogger.debug(EdkLogger.DEBUG_9, "No MaxDatumSize specified
>for PCD %s.%s" \
>                             % (ToPcd.TokenSpaceGuidCName, TokenCName))
>             Value = ToPcd.DefaultValue
>diff --git a/BaseTools/Source/Python/AutoGen/GenVar.py
>b/BaseTools/Source/Python/AutoGen/GenVar.py
>index 75d455b407..036f00e2bb 100644
>--- a/BaseTools/Source/Python/AutoGen/GenVar.py
>+++ b/BaseTools/Source/Python/AutoGen/GenVar.py
>@@ -20,11 +20,11 @@ import copy
> from Common.VariableAttributes import VariableAttributes
> from Common.Misc import *
> import collections
> import Common.DataType as DataType
>
>-var_info = collections.namedtuple("uefi_var",
>"pcdindex,pcdname,defaultstoragename,skuname,var_name, var_guid,
>var_offset,var_attribute,pcd_default_value, default_value, data_type")
>+var_info = collections.namedtuple("uefi_var",
>"pcdindex,pcdname,defaultstoragename,skuname,var_name, var_guid,
>var_offset,var_attribute,pcd_default_value, default_value,
>data_type,PcdDscLine,StructurePcd")
> NvStorageHeaderSize = 28
> VariableHeaderSize = 32
>
> class VariableMgr(object):
>     def __init__(self, DefaultStoreMap, SkuIdMap):
>@@ -54,37 +54,74 @@ class VariableMgr(object):
>         value_str = "{"
>         default_var_bin_strip = [ data.strip("""'""") for data in default_var_bin]
>         value_str += ",".join(default_var_bin_strip)
>         value_str += "}"
>         return value_str
>+    def Do_combine(self,sku_var_info_offset_list):
>+        newvalue = {}
>+        for item in sku_var_info_offset_list:
>+            data_type = item.data_type
>+            value_list = item.default_value.strip("{").strip("}").split(",")
>+            if data_type in DataType.TAB_PCD_NUMERIC_TYPES:
>+                data_flag =
>DataType.PACK_CODE_BY_SIZE[MAX_SIZE_TYPE[data_type]]
>+                data = value_list[0]
>+                value_list = []
>+                for data_byte in pack(data_flag, int(data, 16) if
>data.upper().startswith('0X') else int(data)):
>+                    value_list.append(hex(unpack("B", data_byte)[0]))
>+            newvalue[int(item.var_offset, 16) if
>item.var_offset.upper().startswith("0X") else int(item.var_offset)] =
>value_list
>+        try:
>+            newvaluestr = "{" + ",".join(VariableMgr.assemble_variable(newvalue))
>+"}"
>+        except:
>+            EdkLogger.error("build", AUTOGEN_ERROR, "Variable offset conflict in
>PCDs: %s \n" % (" and ".join(item.pcdname for item in
>sku_var_info_offset_list)))
>+        return newvaluestr
>+    def Do_Merge(self,sku_var_info_offset_list):
>+        StructrurePcds = sorted([item for item in sku_var_info_offset_list if
>item.StructurePcd], key = lambda x: x.PcdDscLine, reverse =True )
>+        Base = StructrurePcds[0]
>+        BaseValue = Base.default_value.strip("{").strip("}").split(",")
>+        Override = [item for item in sku_var_info_offset_list if not
>item.StructurePcd and item.PcdDscLine > Base.PcdDscLine]
>+        newvalue = {}
>+        for item in Override:
>+            data_type = item.data_type
>+            value_list = item.default_value.strip("{").strip("}").split(",")
>+            if data_type in DataType.TAB_PCD_NUMERIC_TYPES:
>+                data_flag =
>DataType.PACK_CODE_BY_SIZE[MAX_SIZE_TYPE[data_type]]
>+                data = value_list[0]
>+                value_list = []
>+                for data_byte in pack(data_flag, int(data, 16) if
>data.upper().startswith('0X') else int(data)):
>+                    value_list.append(hex(unpack("B", data_byte)[0]))
>+            newvalue[int(item.var_offset, 16) if
>item.var_offset.upper().startswith("0X") else int(item.var_offset)] =
>(value_list,item.pcdname,item.PcdDscLine)
>+        for offset in newvalue:
>+            value_list,itemPcdname,itemPcdDscLine = newvalue[offset]
>+            if offset > len(BaseValue) or (offset + len(value_list) > len(BaseValue)):
>+                EdkLogger.error("build", AUTOGEN_ERROR, "The EFI Variable
>referred by PCD %s in line %s exceeds variable size: %s\n" %
>(itemPcdname,itemPcdDscLine,hex(len(BaseValue))))
>+            for i in xrange(len(value_list)):
>+                BaseValue[offset + i] = value_list[i]
>+        newvaluestr =  "{" + ",".join(BaseValue) +"}"
>+        return newvaluestr
>+    def NeedMerge(self,sku_var_info_offset_list):
>+        if [item for item in sku_var_info_offset_list if item.StructurePcd]:
>+            return True
>+        return False
>     def combine_variable(self):
>         indexedvarinfo = collections.OrderedDict()
>         for item in self.VarInfo:
>             if (item.skuname, item.defaultstoragename, item.var_name,
>item.var_guid) not in indexedvarinfo:
>                 indexedvarinfo[(item.skuname, item.defaultstoragename,
>item.var_name, item.var_guid) ] = []
>             indexedvarinfo[(item.skuname, item.defaultstoragename,
>item.var_name, item.var_guid)].append(item)
>         for key in indexedvarinfo:
>             sku_var_info_offset_list = indexedvarinfo[key]
>             if len(sku_var_info_offset_list) == 1:
>                 continue
>-            newvalue = {}
>-            for item in sku_var_info_offset_list:
>-                data_type = item.data_type
>-                value_list = item.default_value.strip("{").strip("}").split(",")
>-                if data_type in DataType.TAB_PCD_NUMERIC_TYPES:
>-                    data_flag =
>DataType.PACK_CODE_BY_SIZE[MAX_SIZE_TYPE[data_type]]
>-                    data = value_list[0]
>-                    value_list = []
>-                    for data_byte in pack(data_flag, int(data, 16) if
>data.upper().startswith('0X') else int(data)):
>-                        value_list.append(hex(unpack("B", data_byte)[0]))
>-                newvalue[int(item.var_offset, 16) if
>item.var_offset.upper().startswith("0X") else int(item.var_offset)] =
>value_list
>-            try:
>-                newvaluestr = "{" +
>",".join(VariableMgr.assemble_variable(newvalue)) +"}"
>-            except:
>-                EdkLogger.error("build", AUTOGEN_ERROR, "Variable offset conflict
>in PCDs: %s \n" % (" and ".join(item.pcdname for item in
>sku_var_info_offset_list)))
>+
>             n = sku_var_info_offset_list[0]
>-            indexedvarinfo[key] =  [var_info(n.pcdindex, n.pcdname,
>n.defaultstoragename, n.skuname, n.var_name, n.var_guid, "0x00",
>n.var_attribute, newvaluestr, newvaluestr, DataType.TAB_VOID)]
>+
>+            if self.NeedMerge(sku_var_info_offset_list):
>+                newvaluestr = self.Do_Merge(sku_var_info_offset_list)
>+            else:
>+                newvaluestr = self.Do_combine(sku_var_info_offset_list)
>+
>+            indexedvarinfo[key] =  [var_info(n.pcdindex, n.pcdname,
>n.defaultstoragename, n.skuname, n.var_name, n.var_guid, "0x00",
>n.var_attribute, newvaluestr, newvaluestr,
>DataType.TAB_VOID,n.PcdDscLine,n.StructurePcd)]
>         self.VarInfo = [item[0] for item in indexedvarinfo.values()]
>
>     @staticmethod
>     def assemble_variable(valuedict):
>         ordered_valuedict_keys = sorted(valuedict.keys())
>diff --git a/BaseTools/Source/Python/Workspace/BuildClassObject.py
>b/BaseTools/Source/Python/Workspace/BuildClassObject.py
>index 88465c59ea..c0dd1dd0cd 100644
>--- a/BaseTools/Source/Python/Workspace/BuildClassObject.py
>+++ b/BaseTools/Source/Python/Workspace/BuildClassObject.py
>@@ -65,10 +65,11 @@ class PcdClassObject(object):
>         if IsDsc:
>             self.DscDefaultValue = Value
>         self.PcdValueFromComm = ""
>         self.PcdValueFromFdf = ""
>         self.DefinitionPosition = ("","")
>+        self.CustomAttribute = {}
>
>     ## Get the maximum number of bytes
>     def GetPcdMaxSize(self):
>         if self.DatumType in TAB_PCD_NUMERIC_TYPES:
>             return MAX_SIZE_TYPE[self.DatumType]
>@@ -223,10 +224,11 @@ class StructurePcd(PcdClassObject):
>         self.expressions = PcdObject.expressions if PcdObject.expressions else
>self.expressions
>         self.DscRawValue = PcdObject.DscRawValue if PcdObject.DscRawValue
>else self.DscRawValue
>         self.PcdValueFromComm = PcdObject.PcdValueFromComm if
>PcdObject.PcdValueFromComm else self.PcdValueFromComm
>         self.PcdValueFromFdf = PcdObject.PcdValueFromFdf if
>PcdObject.PcdValueFromFdf else self.PcdValueFromFdf
>         self.DefinitionPosition = PcdObject.DefinitionPosition if
>PcdObject.DefinitionPosition else self.DefinitionPosition
>+        self.CustomAttribute = PcdObject.CustomAttribute if
>PcdObject.CustomAttribute else self.CustomAttribute
>         if isinstance(PcdObject, StructurePcd):
>             self.StructuredPcdIncludeFile = PcdObject.StructuredPcdIncludeFile if
>PcdObject.StructuredPcdIncludeFile else self.StructuredPcdIncludeFile
>             self.PackageDecs = PcdObject.PackageDecs if PcdObject.PackageDecs
>else self.PackageDecs
>             self.DefaultValues = PcdObject.DefaultValues if
>PcdObject.DefaultValues else self.DefaultValues
>             self.PcdMode = PcdObject.PcdMode if PcdObject.PcdMode else
>self.PcdMode
>diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py
>b/BaseTools/Source/Python/Workspace/DscBuildData.py
>index aaef404772..7a8885956e 100644
>--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
>+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
>@@ -1477,10 +1477,11 @@ class DscBuildData(PlatformBuildClassObject):
>
>             for str_pcd_obj in S_pcd_set.values():
>
>                 str_pcd_obj.MaxDatumSize =
>self.GetStructurePcdMaxSize(str_pcd_obj)
>                 Pcds[str_pcd_obj.TokenCName,
>str_pcd_obj.TokenSpaceGuidCName] = str_pcd_obj
>+                Pcds[str_pcd_obj.TokenCName,
>str_pcd_obj.TokenSpaceGuidCName].CustomAttribute['IsStru']=True
>
>             for pcdkey in Pcds:
>                 pcd = Pcds[pcdkey]
>                 if TAB_DEFAULT not in pcd.SkuInfoList and TAB_COMMON in
>pcd.SkuInfoList:
>                     pcd.SkuInfoList[TAB_DEFAULT] = pcd.SkuInfoList[TAB_COMMON]
>@@ -2616,11 +2617,11 @@ class DscBuildData(PlatformBuildClassObject):
>                                                 pcdDecObject.validateranges,
>                                                 pcdDecObject.validlists,
>                                                 pcdDecObject.expressions,
>                                                 IsDsc=True)
>
>-
>+                Pcds[PcdCName, TokenSpaceGuid].CustomAttribute['DscPosition'] =
>int(Dummy4)
>         for pcd in Pcds.values():
>             SkuInfoObj = pcd.SkuInfoList.values()[0]
>             pcdDecObject = self._DecPcds[pcd.TokenCName,
>pcd.TokenSpaceGuidCName]
>             pcd.DatumType = pcdDecObject.DatumType
>             # Only fix the value while no value provided in DSC file.
>--
>2.16.2.windows.1
>
>_______________________________________________
>edk2-devel mailing list
>edk2-devel@lists.01.org
>https://lists.01.org/mailman/listinfo/edk2-devel