From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=134.134.136.65; helo=mga03.intel.com; envelope-from=bob.c.feng@intel.com; receiver=edk2-devel@lists.01.org Received: from mga03.intel.com (mga03.intel.com [134.134.136.65]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ml01.01.org (Postfix) with ESMTPS id 3CB702118B795 for ; Wed, 7 Nov 2018 22:03:42 -0800 (PST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga001.jf.intel.com ([10.7.209.18]) by orsmga103.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 07 Nov 2018 22:03:41 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.54,478,1534834800"; d="scan'208";a="106433863" Received: from shwdepsi1121.ccr.corp.intel.com ([10.239.158.47]) by orsmga001.jf.intel.com with ESMTP; 07 Nov 2018 22:03:40 -0800 From: BobCF To: edk2-devel@lists.01.org Cc: Liming Gao , Jaben Carsey Date: Thu, 8 Nov 2018 14:03:38 +0800 Message-Id: <20181108060338.40404-1-bob.c.feng@intel.com> X-Mailer: git-send-email 2.19.1.windows.1 MIME-Version: 1.0 Subject: [Patch] BaseTools: Customize deepcopy function. X-BeenThere: edk2-devel@lists.01.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: EDK II Development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 08 Nov 2018 06:03:42 -0000 Content-Transfer-Encoding: 8bit https://bugzilla.tianocore.org/show_bug.cgi?id=1288 This patch is one of build tool performance improvement series patches. This patch is going to customize the deepcopy function for SkuClass, PcdClassObject and python dictionary. python deepcopy copy everything of a object, but for our current usage we just need to copy the data we care about recursively. By implementing __deepcopy__ for SkuClass, PcdClassObject, we can customize deepcopy function for them. Contributed-under: TianoCore Contribution Agreement 1.1 Signed-off-by: BobCF Cc: Liming Gao Cc: Jaben Carsey --- BaseTools/Source/Python/Common/Expression.py | 4 +- BaseTools/Source/Python/Common/Misc.py | 16 ++++++ .../Python/CommonDataClass/CommonClass.py | 15 ++++++ .../Python/Workspace/BuildClassObject.py | 52 +++++++++++++++++++ .../Source/Python/Workspace/DscBuildData.py | 6 +-- 5 files changed, 88 insertions(+), 5 deletions(-) diff --git a/BaseTools/Source/Python/Common/Expression.py b/BaseTools/Source/Python/Common/Expression.py index a21ab5daa7..091c0f4296 100644 --- a/BaseTools/Source/Python/Common/Expression.py +++ b/BaseTools/Source/Python/Common/Expression.py @@ -15,11 +15,11 @@ from __future__ import print_function from __future__ import absolute_import from Common.GlobalData import * from CommonDataClass.Exceptions import BadExpression from CommonDataClass.Exceptions import WrnExpression -from .Misc import GuidStringToGuidStructureString, ParseFieldValue +from .Misc import GuidStringToGuidStructureString, ParseFieldValue,CopyDict import Common.EdkLogger as EdkLogger import copy from Common.DataType import * import sys from random import sample @@ -353,11 +353,11 @@ class ValueExpression(BaseExpression): raise BadExpression(ERR_EMPTY_EXPR) # # The symbol table including PCD and macro mapping # - self._Symb = copy.deepcopy(SymbolTable) + self._Symb = CopyDict(SymbolTable) self._Symb.update(self.LogicalOperators) self._Idx = 0 self._Len = len(self._Expr) self._Token = '' self._WarnExcept = None diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py index 3b8efb2e71..80236db160 100644 --- a/BaseTools/Source/Python/Common/Misc.py +++ b/BaseTools/Source/Python/Common/Misc.py @@ -39,10 +39,11 @@ from Common.LongFilePathSupport import OpenLongFilePath as open from Common.MultipleWorkspace import MultipleWorkspace as mws import uuid from CommonDataClass.Exceptions import BadExpression from Common.caching import cached_property import subprocess +from collections import OrderedDict ## Regular expression used to find out place holders in string template gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE) ## regular expressions for map file processing startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class") @@ -2129,10 +2130,25 @@ def PackByteFormatGUID(Guid): Guid[8], Guid[9], Guid[10], ) +## DeepCopy dict/OrderedDict recusively +# +# @param ori_dict a nested dict or ordereddict +# +# @retval new dict or orderdict +# +def CopyDict(ori_dict): + dict_type = ori_dict.__class__ + new_dict = dict_type() + for key in ori_dict: + if isinstance(ori_dict[key],(dict,OrderedDict)): + new_dict[key] = CopyDict(ori_dict[key]) + else: + new_dict[key] = ori_dict[key] + return new_dict ## # # This acts like the main() function for the script, unless it is 'import'ed into another # script. # diff --git a/BaseTools/Source/Python/CommonDataClass/CommonClass.py b/BaseTools/Source/Python/CommonDataClass/CommonClass.py index a98cf8a7c5..336bb11671 100644 --- a/BaseTools/Source/Python/CommonDataClass/CommonClass.py +++ b/BaseTools/Source/Python/CommonDataClass/CommonClass.py @@ -78,5 +78,20 @@ class SkuInfoClass(object): 'VariableOffset = ' + str(self.VariableOffset) + "," + \ 'HiiDefaultValue = ' + str(self.HiiDefaultValue) + "," + \ 'VpdOffset = ' + str(self.VpdOffset) + "," + \ 'DefaultValue = ' + str(self.DefaultValue) + "," return Rtn + + def __deepcopy__(self,memo): + new_sku = SkuInfoClass() + new_sku.SkuIdName = self.SkuIdName + new_sku.SkuId = self.SkuId + new_sku.VariableName = self.VariableName + new_sku.VariableGuid = self.VariableGuid + new_sku.VariableGuidValue = self.VariableGuidValue + new_sku.VariableOffset = self.VariableOffset + new_sku.HiiDefaultValue = self.HiiDefaultValue + new_sku.VariableAttribute = self.VariableAttribute + new_sku.DefaultStoreDict = {key:value for key,value in self.DefaultStoreDict.items()} + new_sku.VpdOffset = self.VpdOffset + new_sku.DefaultValue = self.DefaultValue + return new_sku diff --git a/BaseTools/Source/Python/Workspace/BuildClassObject.py b/BaseTools/Source/Python/Workspace/BuildClassObject.py index 95edc376fe..7d10391c1e 100644 --- a/BaseTools/Source/Python/Workspace/BuildClassObject.py +++ b/BaseTools/Source/Python/Workspace/BuildClassObject.py @@ -14,10 +14,12 @@ from collections import OrderedDict, namedtuple from Common.DataType import * import collections import re from collections import OrderedDict +from Common.Misc import CopyDict +import copy StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_\[\]]*$') ArrayIndex = re.compile("\[\s*\d{0,1}\s*\]") ## PcdClassObject # # This Class is used for PcdObject @@ -152,10 +154,41 @@ class PcdClassObject(object): # @retval truple() Key for hash table # def __hash__(self): return hash((self.TokenCName, self.TokenSpaceGuidCName)) + def sharedcopy(self,new_pcd): + new_pcd.TokenCName = self.TokenCName + new_pcd.TokenSpaceGuidCName = self.TokenSpaceGuidCName + new_pcd.TokenSpaceGuidValue = self.TokenSpaceGuidValue + new_pcd.Type = self.Type + new_pcd.DatumType = self.DatumType + new_pcd.DefaultValue = self.DefaultValue + new_pcd.TokenValue = self.TokenValue + new_pcd.MaxDatumSize = self.MaxDatumSize + + new_pcd.Phase = self.Phase + new_pcd.Pending = self.Pending + new_pcd.IsOverrided = self.IsOverrided + new_pcd.IsFromBinaryInf = self.IsFromBinaryInf + new_pcd.IsFromDsc = self.IsFromDsc + new_pcd.PcdValueFromComm = self.PcdValueFromComm + new_pcd.PcdValueFromFdf = self.PcdValueFromFdf + new_pcd.UserDefinedDefaultStoresFlag = self.UserDefinedDefaultStoresFlag + new_pcd.DscRawValue = self.DscRawValue + new_pcd.CustomAttribute = self.CustomAttribute + new_pcd.validateranges = [item for item in self.validateranges] + new_pcd.validlists = [item for item in self.validlists] + new_pcd.expressions = [item for item in self.expressions] + new_pcd.SkuInfoList = {key: copy.deepcopy(skuobj) for key,skuobj in self.SkuInfoList.items()} + return new_pcd + + def __deepcopy__(self,memo): + new_pcd = PcdClassObject() + self.sharedcopy(new_pcd) + return new_pcd + class StructurePcd(PcdClassObject): def __init__(self, StructuredPcdIncludeFile=None, Packages=None, Name=None, Guid=None, Type=None, DatumType=None, Value=None, Token=None, MaxDatumSize=None, SkuInfoList=None, IsOverrided=False, GuidValue=None, validateranges=None, validlists=None, expressions=None,default_store = TAB_DEFAULT_STORES_DEFAULT): if SkuInfoList is None: SkuInfoList = {} if validateranges is None: @@ -240,10 +273,29 @@ class StructurePcd(PcdClassObject): self.PkgPath = PcdObject.PkgPath if PcdObject.PkgPath else self.PkgPath self.ValueChain = PcdObject.ValueChain if PcdObject.ValueChain else self.ValueChain self.PcdFieldValueFromComm = PcdObject.PcdFieldValueFromComm if PcdObject.PcdFieldValueFromComm else self.PcdFieldValueFromComm self.PcdFieldValueFromFdf = PcdObject.PcdFieldValueFromFdf if PcdObject.PcdFieldValueFromFdf else self.PcdFieldValueFromFdf + def __deepcopy__(self,memo): + new_pcd = StructurePcd() + self.sharedcopy(new_pcd) + + new_pcd.DefaultValueFromDec = self.DefaultValueFromDec + new_pcd.PcdMode = self.PcdMode + new_pcd.StructName = self.DatumType + new_pcd.PcdDefineLineNo = self.PcdDefineLineNo + new_pcd.PkgPath = self.PkgPath + new_pcd.StructuredPcdIncludeFile = [item for item in self.StructuredPcdIncludeFile] + new_pcd.PackageDecs = [item for item in self.PackageDecs] + new_pcd.DefaultValues = CopyDict(self.DefaultValues) + new_pcd.DefaultFromDSC=CopyDict(self.DefaultFromDSC) + new_pcd.SkuOverrideValues = CopyDict(self.SkuOverrideValues) + new_pcd.PcdFieldValueFromComm = CopyDict(self.PcdFieldValueFromComm) + new_pcd.PcdFieldValueFromFdf = CopyDict(self.PcdFieldValueFromFdf) + new_pcd.ValueChain = {item for item in self.ValueChain} + return new_pcd + LibraryClassObject = namedtuple('LibraryClassObject', ['LibraryClass','SupModList'], verbose=False) ## ModuleBuildClassObject # # This Class defines ModuleBuildClass diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py index 11aa63fb26..5655034b88 100644 --- a/BaseTools/Source/Python/Workspace/DscBuildData.py +++ b/BaseTools/Source/Python/Workspace/DscBuildData.py @@ -970,11 +970,11 @@ class DscBuildData(PlatformBuildClassObject): for skuid in pcd.SkuInfoList: skuobj = pcd.SkuInfoList.get(skuid) if TAB_DEFAULT_STORES_DEFAULT not in skuobj.DefaultStoreDict: PcdDefaultStoreSet = set(defaultstorename for defaultstorename in skuobj.DefaultStoreDict) mindefaultstorename = DefaultStoreMgr.GetMin(PcdDefaultStoreSet) - skuobj.DefaultStoreDict[TAB_DEFAULT_STORES_DEFAULT] = copy.deepcopy(skuobj.DefaultStoreDict[mindefaultstorename]) + skuobj.DefaultStoreDict[TAB_DEFAULT_STORES_DEFAULT] = CopyDict(skuobj.DefaultStoreDict[mindefaultstorename]) return Pcds def RecoverCommandLinePcd(self): def UpdateCommandLineValue(pcd): if pcd.Type in [self._PCD_TYPE_STRING_[MODEL_PCD_FIXED_AT_BUILD], @@ -1501,11 +1501,11 @@ class DscBuildData(PlatformBuildClassObject): PcdDefaultStoreSet = set(defaultstorename for defaultstorename in stru_pcd.SkuOverrideValues[nextskuid]) mindefaultstorename = DefaultStoreMgr.GetMin(PcdDefaultStoreSet) for defaultstoreid in DefaultStores: if defaultstoreid not in stru_pcd.SkuOverrideValues[skuid]: - stru_pcd.SkuOverrideValues[skuid][defaultstoreid] = copy.deepcopy(stru_pcd.SkuOverrideValues[nextskuid][mindefaultstorename]) + stru_pcd.SkuOverrideValues[skuid][defaultstoreid] = CopyDict(stru_pcd.SkuOverrideValues[nextskuid][mindefaultstorename]) stru_pcd.ValueChain.add((skuid, defaultstoreid)) S_pcd_set = DscBuildData.OverrideByFdf(S_pcd_set,self.WorkspaceDir) S_pcd_set = DscBuildData.OverrideByComm(S_pcd_set) Str_Pcd_Values = self.GenerateByteArrayValue(S_pcd_set) if Str_Pcd_Values: @@ -2597,11 +2597,11 @@ class DscBuildData(PlatformBuildClassObject): for skuid in PcdObj.SkuInfoList: skuobj = PcdObj.SkuInfoList[skuid] mindefaultstorename = DefaultStoreObj.GetMin(set(defaultstorename for defaultstorename in skuobj.DefaultStoreDict)) for defaultstorename in DefaultStores: if defaultstorename not in skuobj.DefaultStoreDict: - skuobj.DefaultStoreDict[defaultstorename] = copy.deepcopy(skuobj.DefaultStoreDict[mindefaultstorename]) + skuobj.DefaultStoreDict[defaultstorename] = CopyDict(skuobj.DefaultStoreDict[mindefaultstorename]) skuobj.HiiDefaultValue = skuobj.DefaultStoreDict[mindefaultstorename] for skuname, skuid in SkuIds.items(): if skuname not in PcdObj.SkuInfoList: nextskuid = self.SkuIdMgr.GetNextSkuId(skuname) while nextskuid not in PcdObj.SkuInfoList: -- 2.19.1.windows.1