From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=192.55.52.136; helo=mga12.intel.com; envelope-from=liming.gao@intel.com; receiver=edk2-devel@lists.01.org Received: from mga12.intel.com (mga12.intel.com [192.55.52.136]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ml01.01.org (Postfix) with ESMTPS id C9DAF21A00AE6 for ; Tue, 11 Sep 2018 22:56:29 -0700 (PDT) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga007.fm.intel.com ([10.253.24.52]) by fmsmga106.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 11 Sep 2018 22:56:29 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.53,363,1531810800"; d="scan'208";a="69331024" Received: from fmsmsx106.amr.corp.intel.com ([10.18.124.204]) by fmsmga007.fm.intel.com with ESMTP; 11 Sep 2018 22:56:17 -0700 Received: from fmsmsx120.amr.corp.intel.com (10.18.124.208) by FMSMSX106.amr.corp.intel.com (10.18.124.204) with Microsoft SMTP Server (TLS) id 14.3.319.2; Tue, 11 Sep 2018 22:56:17 -0700 Received: from shsmsx102.ccr.corp.intel.com (10.239.4.154) by fmsmsx120.amr.corp.intel.com (10.18.124.208) with Microsoft SMTP Server (TLS) id 14.3.319.2; Tue, 11 Sep 2018 22:56:16 -0700 Received: from shsmsx104.ccr.corp.intel.com ([169.254.5.143]) by shsmsx102.ccr.corp.intel.com ([169.254.2.226]) with mapi id 14.03.0319.002; Wed, 12 Sep 2018 13:55:53 +0800 From: "Gao, Liming" To: "Feng, Bob C" , "edk2-devel@lists.01.org" Thread-Topic: [edk2] [Patch] BaseTool: Variable Merge. Thread-Index: AQHURn61Gg4I2bRpA0KotcYL1m7mgKTsLWjQ Date: Wed, 12 Sep 2018 05:55:53 +0000 Message-ID: <4A89E2EF3DFEDB4C8BFDE51014F606A14E2F61FA@SHSMSX104.ccr.corp.intel.com> References: <20180907074443.173788-1-bob.c.feng@intel.com> In-Reply-To: <20180907074443.173788-1-bob.c.feng@intel.com> Accept-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-originating-ip: [10.239.127.40] MIME-Version: 1.0 Subject: Re: [Patch] BaseTool: Variable Merge. X-BeenThere: edk2-devel@lists.01.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: EDK II Development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 12 Sep 2018 05:56:30 -0000 Content-Language: en-US Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable Bob: Please update commit message title with the detail information.=20 Thanks Liming >-----Original Message----- >From: edk2-devel [mailto:edk2-devel-bounces@lists.01.org] On Behalf Of >BobCF >Sent: Friday, September 07, 2018 3:45 PM >To: edk2-devel@lists.01.org >Cc: Gao, Liming >Subject: [edk2] [Patch] BaseTool: Variable Merge. > >If Structure PCD and Normal Pcd refer to the >same variable, do variable merge. > >Enhance error message. > >Contributed-under: TianoCore Contribution Agreement 1.1 >Signed-off-by: Bob Feng >Cc: Liming Gao >--- > BaseTools/Source/Python/AutoGen/AutoGen.py | 3 +- > BaseTools/Source/Python/AutoGen/GenVar.py | 71 >++++++++++++++++------ > .../Source/Python/Workspace/BuildClassObject.py | 2 + > BaseTools/Source/Python/Workspace/DscBuildData.py | 3 +- > 4 files changed, 60 insertions(+), 19 deletions(-) > >diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py >b/BaseTools/Source/Python/AutoGen/AutoGen.py >index 95370d1821..d1bd1b1d4c 100644 >--- a/BaseTools/Source/Python/AutoGen/AutoGen.py >+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py >@@ -1211,11 +1211,11 @@ class PlatformAutoGen(AutoGen): > continue > if len(Sku.VariableName) > 0: > VariableGuidStructure =3D Sku.VariableGuidValue > VariableGuid =3D >GuidStructureStringToGuidString(VariableGuidStructure) > for StorageName in Sku.DefaultStoreDict: >- VariableInfo.append_variable(var_info(Index, pcdn= ame, >StorageName, SkuName, StringToArray(Sku.VariableName), VariableGuid, >Sku.VariableOffset, Sku.VariableAttribute, Sku.HiiDefaultValue, >Sku.DefaultStoreDict[StorageName], Pcd.DatumType)) >+ VariableInfo.append_variable(var_info(Index, pcdn= ame, >StorageName, SkuName, StringToArray(Sku.VariableName), VariableGuid, >Sku.VariableOffset, Sku.VariableAttribute, Sku.HiiDefaultValue, >Sku.DefaultStoreDict[StorageName], Pcd.DatumType, >Pcd.CustomAttribute['DscPosition'], Pcd.CustomAttribute.get('IsStru',False= ))) > Index +=3D 1 > return VariableInfo > > def UpdateNVStoreMaxSize(self, OrgVpdFile): > if self.VariableInfo: >@@ -2104,10 +2104,11 @@ class PlatformAutoGen(AutoGen): > EdkLogger.error('build', FORMAT_INVALID, Cause, File=3Dse= lf.MetaFile, > ExtraData=3D"%s.%s" % (ToPcd.TokenSpaceGu= idCName, >TokenCName)) > ToPcd.validateranges =3D FromPcd.validateranges > ToPcd.validlists =3D FromPcd.validlists > ToPcd.expressions =3D FromPcd.expressions >+ ToPcd.CustomAttribute =3D FromPcd.CustomAttribute > > if FromPcd is not None and ToPcd.DatumType =3D=3D TAB_VOID and no= t >ToPcd.MaxDatumSize: > EdkLogger.debug(EdkLogger.DEBUG_9, "No MaxDatumSize specified >for PCD %s.%s" \ > % (ToPcd.TokenSpaceGuidCName, TokenCName)) > Value =3D ToPcd.DefaultValue >diff --git a/BaseTools/Source/Python/AutoGen/GenVar.py >b/BaseTools/Source/Python/AutoGen/GenVar.py >index 75d455b407..036f00e2bb 100644 >--- a/BaseTools/Source/Python/AutoGen/GenVar.py >+++ b/BaseTools/Source/Python/AutoGen/GenVar.py >@@ -20,11 +20,11 @@ import copy > from Common.VariableAttributes import VariableAttributes > from Common.Misc import * > import collections > import Common.DataType as DataType > >-var_info =3D collections.namedtuple("uefi_var", >"pcdindex,pcdname,defaultstoragename,skuname,var_name, var_guid, >var_offset,var_attribute,pcd_default_value, default_value, data_type") >+var_info =3D collections.namedtuple("uefi_var", >"pcdindex,pcdname,defaultstoragename,skuname,var_name, var_guid, >var_offset,var_attribute,pcd_default_value, default_value, >data_type,PcdDscLine,StructurePcd") > NvStorageHeaderSize =3D 28 > VariableHeaderSize =3D 32 > > class VariableMgr(object): > def __init__(self, DefaultStoreMap, SkuIdMap): >@@ -54,37 +54,74 @@ class VariableMgr(object): > value_str =3D "{" > default_var_bin_strip =3D [ data.strip("""'""") for data in defau= lt_var_bin] > value_str +=3D ",".join(default_var_bin_strip) > value_str +=3D "}" > return value_str >+ def Do_combine(self,sku_var_info_offset_list): >+ newvalue =3D {} >+ for item in sku_var_info_offset_list: >+ data_type =3D item.data_type >+ value_list =3D item.default_value.strip("{").strip("}").split= (",") >+ if data_type in DataType.TAB_PCD_NUMERIC_TYPES: >+ data_flag =3D >DataType.PACK_CODE_BY_SIZE[MAX_SIZE_TYPE[data_type]] >+ data =3D value_list[0] >+ value_list =3D [] >+ for data_byte in pack(data_flag, int(data, 16) if >data.upper().startswith('0X') else int(data)): >+ value_list.append(hex(unpack("B", data_byte)[0])) >+ newvalue[int(item.var_offset, 16) if >item.var_offset.upper().startswith("0X") else int(item.var_offset)] =3D >value_list >+ try: >+ newvaluestr =3D "{" + ",".join(VariableMgr.assemble_variable(= newvalue)) >+"}" >+ except: >+ EdkLogger.error("build", AUTOGEN_ERROR, "Variable offset conf= lict in >PCDs: %s \n" % (" and ".join(item.pcdname for item in >sku_var_info_offset_list))) >+ return newvaluestr >+ def Do_Merge(self,sku_var_info_offset_list): >+ StructrurePcds =3D sorted([item for item in sku_var_info_offset_l= ist if >item.StructurePcd], key =3D lambda x: x.PcdDscLine, reverse =3DTrue ) >+ Base =3D StructrurePcds[0] >+ BaseValue =3D Base.default_value.strip("{").strip("}").split(",") >+ Override =3D [item for item in sku_var_info_offset_list if not >item.StructurePcd and item.PcdDscLine > Base.PcdDscLine] >+ newvalue =3D {} >+ for item in Override: >+ data_type =3D item.data_type >+ value_list =3D item.default_value.strip("{").strip("}").split= (",") >+ if data_type in DataType.TAB_PCD_NUMERIC_TYPES: >+ data_flag =3D >DataType.PACK_CODE_BY_SIZE[MAX_SIZE_TYPE[data_type]] >+ data =3D value_list[0] >+ value_list =3D [] >+ for data_byte in pack(data_flag, int(data, 16) if >data.upper().startswith('0X') else int(data)): >+ value_list.append(hex(unpack("B", data_byte)[0])) >+ newvalue[int(item.var_offset, 16) if >item.var_offset.upper().startswith("0X") else int(item.var_offset)] =3D >(value_list,item.pcdname,item.PcdDscLine) >+ for offset in newvalue: >+ value_list,itemPcdname,itemPcdDscLine =3D newvalue[offset] >+ if offset > len(BaseValue) or (offset + len(value_list) > len= (BaseValue)): >+ EdkLogger.error("build", AUTOGEN_ERROR, "The EFI Variable >referred by PCD %s in line %s exceeds variable size: %s\n" % >(itemPcdname,itemPcdDscLine,hex(len(BaseValue)))) >+ for i in xrange(len(value_list)): >+ BaseValue[offset + i] =3D value_list[i] >+ newvaluestr =3D "{" + ",".join(BaseValue) +"}" >+ return newvaluestr >+ def NeedMerge(self,sku_var_info_offset_list): >+ if [item for item in sku_var_info_offset_list if item.StructurePc= d]: >+ return True >+ return False > def combine_variable(self): > indexedvarinfo =3D collections.OrderedDict() > for item in self.VarInfo: > if (item.skuname, item.defaultstoragename, item.var_name, >item.var_guid) not in indexedvarinfo: > indexedvarinfo[(item.skuname, item.defaultstoragename, >item.var_name, item.var_guid) ] =3D [] > indexedvarinfo[(item.skuname, item.defaultstoragename, >item.var_name, item.var_guid)].append(item) > for key in indexedvarinfo: > sku_var_info_offset_list =3D indexedvarinfo[key] > if len(sku_var_info_offset_list) =3D=3D 1: > continue >- newvalue =3D {} >- for item in sku_var_info_offset_list: >- data_type =3D item.data_type >- value_list =3D item.default_value.strip("{").strip("}").s= plit(",") >- if data_type in DataType.TAB_PCD_NUMERIC_TYPES: >- data_flag =3D >DataType.PACK_CODE_BY_SIZE[MAX_SIZE_TYPE[data_type]] >- data =3D value_list[0] >- value_list =3D [] >- for data_byte in pack(data_flag, int(data, 16) if >data.upper().startswith('0X') else int(data)): >- value_list.append(hex(unpack("B", data_byte)[0])) >- newvalue[int(item.var_offset, 16) if >item.var_offset.upper().startswith("0X") else int(item.var_offset)] =3D >value_list >- try: >- newvaluestr =3D "{" + >",".join(VariableMgr.assemble_variable(newvalue)) +"}" >- except: >- EdkLogger.error("build", AUTOGEN_ERROR, "Variable offset = conflict >in PCDs: %s \n" % (" and ".join(item.pcdname for item in >sku_var_info_offset_list))) >+ > n =3D sku_var_info_offset_list[0] >- indexedvarinfo[key] =3D [var_info(n.pcdindex, n.pcdname, >n.defaultstoragename, n.skuname, n.var_name, n.var_guid, "0x00", >n.var_attribute, newvaluestr, newvaluestr, DataType.TAB_VOID)] >+ >+ if self.NeedMerge(sku_var_info_offset_list): >+ newvaluestr =3D self.Do_Merge(sku_var_info_offset_list) >+ else: >+ newvaluestr =3D self.Do_combine(sku_var_info_offset_list) >+ >+ indexedvarinfo[key] =3D [var_info(n.pcdindex, n.pcdname, >n.defaultstoragename, n.skuname, n.var_name, n.var_guid, "0x00", >n.var_attribute, newvaluestr, newvaluestr, >DataType.TAB_VOID,n.PcdDscLine,n.StructurePcd)] > self.VarInfo =3D [item[0] for item in indexedvarinfo.values()] > > @staticmethod > def assemble_variable(valuedict): > ordered_valuedict_keys =3D sorted(valuedict.keys()) >diff --git a/BaseTools/Source/Python/Workspace/BuildClassObject.py >b/BaseTools/Source/Python/Workspace/BuildClassObject.py >index 88465c59ea..c0dd1dd0cd 100644 >--- a/BaseTools/Source/Python/Workspace/BuildClassObject.py >+++ b/BaseTools/Source/Python/Workspace/BuildClassObject.py >@@ -65,10 +65,11 @@ class PcdClassObject(object): > if IsDsc: > self.DscDefaultValue =3D Value > self.PcdValueFromComm =3D "" > self.PcdValueFromFdf =3D "" > self.DefinitionPosition =3D ("","") >+ self.CustomAttribute =3D {} > > ## Get the maximum number of bytes > def GetPcdMaxSize(self): > if self.DatumType in TAB_PCD_NUMERIC_TYPES: > return MAX_SIZE_TYPE[self.DatumType] >@@ -223,10 +224,11 @@ class StructurePcd(PcdClassObject): > self.expressions =3D PcdObject.expressions if PcdObject.expressio= ns else >self.expressions > self.DscRawValue =3D PcdObject.DscRawValue if PcdObject.DscRawVal= ue >else self.DscRawValue > self.PcdValueFromComm =3D PcdObject.PcdValueFromComm if >PcdObject.PcdValueFromComm else self.PcdValueFromComm > self.PcdValueFromFdf =3D PcdObject.PcdValueFromFdf if >PcdObject.PcdValueFromFdf else self.PcdValueFromFdf > self.DefinitionPosition =3D PcdObject.DefinitionPosition if >PcdObject.DefinitionPosition else self.DefinitionPosition >+ self.CustomAttribute =3D PcdObject.CustomAttribute if >PcdObject.CustomAttribute else self.CustomAttribute > if isinstance(PcdObject, StructurePcd): > self.StructuredPcdIncludeFile =3D PcdObject.StructuredPcdIncl= udeFile if >PcdObject.StructuredPcdIncludeFile else self.StructuredPcdIncludeFile > self.PackageDecs =3D PcdObject.PackageDecs if PcdObject.Packa= geDecs >else self.PackageDecs > self.DefaultValues =3D PcdObject.DefaultValues if >PcdObject.DefaultValues else self.DefaultValues > self.PcdMode =3D PcdObject.PcdMode if PcdObject.PcdMode else >self.PcdMode >diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py >b/BaseTools/Source/Python/Workspace/DscBuildData.py >index aaef404772..7a8885956e 100644 >--- a/BaseTools/Source/Python/Workspace/DscBuildData.py >+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py >@@ -1477,10 +1477,11 @@ class DscBuildData(PlatformBuildClassObject): > > for str_pcd_obj in S_pcd_set.values(): > > str_pcd_obj.MaxDatumSize =3D >self.GetStructurePcdMaxSize(str_pcd_obj) > Pcds[str_pcd_obj.TokenCName, >str_pcd_obj.TokenSpaceGuidCName] =3D str_pcd_obj >+ Pcds[str_pcd_obj.TokenCName, >str_pcd_obj.TokenSpaceGuidCName].CustomAttribute['IsStru']=3DTrue > > for pcdkey in Pcds: > pcd =3D Pcds[pcdkey] > if TAB_DEFAULT not in pcd.SkuInfoList and TAB_COMMON in >pcd.SkuInfoList: > pcd.SkuInfoList[TAB_DEFAULT] =3D pcd.SkuInfoList[TAB_= COMMON] >@@ -2616,11 +2617,11 @@ class DscBuildData(PlatformBuildClassObject): > pcdDecObject.validaterang= es, > pcdDecObject.validlists, > pcdDecObject.expressions, > IsDsc=3DTrue) > >- >+ Pcds[PcdCName, TokenSpaceGuid].CustomAttribute['DscPositi= on'] =3D >int(Dummy4) > for pcd in Pcds.values(): > SkuInfoObj =3D pcd.SkuInfoList.values()[0] > pcdDecObject =3D self._DecPcds[pcd.TokenCName, >pcd.TokenSpaceGuidCName] > pcd.DatumType =3D pcdDecObject.DatumType > # Only fix the value while no value provided in DSC file. >-- >2.16.2.windows.1 > >_______________________________________________ >edk2-devel mailing list >edk2-devel@lists.01.org >https://lists.01.org/mailman/listinfo/edk2-devel