Re: [Patch] BaseTools: Customize deepcopy function.

["Carsey, Jaben" <jaben.carsey@intel.com>]

Reviewed-by: Jaben Carsey <jaben.carsey@intel.com>

> -----Original Message-----
> From: edk2-devel [mailto:edk2-devel-bounces@lists.01.org] On Behalf Of
> BobCF
> Sent: Wednesday, November 07, 2018 10:04 PM
> To: edk2-devel@lists.01.org
> Cc: Carsey, Jaben <jaben.carsey@intel.com>; Gao, Liming
> <liming.gao@intel.com>
> Subject: [edk2] [Patch] BaseTools: Customize deepcopy function.
> Importance: High
> 
> https://bugzilla.tianocore.org/show_bug.cgi?id=1288
> 
> This patch is one of build tool performance improvement
> series patches.
> 
> This patch is going to customize the deepcopy function for
> SkuClass, PcdClassObject and python dictionary.
> 
> python deepcopy copy everything of a object, but for our current
> usage we just need to copy the data we care about recursively.
> 
> By implementing __deepcopy__ for SkuClass, PcdClassObject, we can
> customize
> deepcopy function for them.
> 
> Contributed-under: TianoCore Contribution Agreement 1.1
> Signed-off-by: BobCF <bob.c.feng@intel.com>
> Cc: Liming Gao <liming.gao@intel.com>
> Cc: Jaben Carsey <jaben.carsey@intel.com>
> ---
>  BaseTools/Source/Python/Common/Expression.py  |  4 +-
>  BaseTools/Source/Python/Common/Misc.py        | 16 ++++++
>  .../Python/CommonDataClass/CommonClass.py     | 15 ++++++
>  .../Python/Workspace/BuildClassObject.py      | 52 +++++++++++++++++++
>  .../Source/Python/Workspace/DscBuildData.py   |  6 +--
>  5 files changed, 88 insertions(+), 5 deletions(-)
> 
> diff --git a/BaseTools/Source/Python/Common/Expression.py
> b/BaseTools/Source/Python/Common/Expression.py
> index a21ab5daa7..091c0f4296 100644
> --- a/BaseTools/Source/Python/Common/Expression.py
> +++ b/BaseTools/Source/Python/Common/Expression.py
> @@ -15,11 +15,11 @@
>  from __future__ import print_function
>  from __future__ import absolute_import
>  from Common.GlobalData import *
>  from CommonDataClass.Exceptions import BadExpression
>  from CommonDataClass.Exceptions import WrnExpression
> -from .Misc import GuidStringToGuidStructureString, ParseFieldValue
> +from .Misc import GuidStringToGuidStructureString,
> ParseFieldValue,CopyDict
>  import Common.EdkLogger as EdkLogger
>  import copy
>  from Common.DataType import *
>  import sys
>  from random import sample
> @@ -353,11 +353,11 @@ class ValueExpression(BaseExpression):
>              raise BadExpression(ERR_EMPTY_EXPR)
> 
>          #
>          # The symbol table including PCD and macro mapping
>          #
> -        self._Symb = copy.deepcopy(SymbolTable)
> +        self._Symb = CopyDict(SymbolTable)
>          self._Symb.update(self.LogicalOperators)
>          self._Idx = 0
>          self._Len = len(self._Expr)
>          self._Token = ''
>          self._WarnExcept = None
> diff --git a/BaseTools/Source/Python/Common/Misc.py
> b/BaseTools/Source/Python/Common/Misc.py
> index 3b8efb2e71..80236db160 100644
> --- a/BaseTools/Source/Python/Common/Misc.py
> +++ b/BaseTools/Source/Python/Common/Misc.py
> @@ -39,10 +39,11 @@ from Common.LongFilePathSupport import
> OpenLongFilePath as open
>  from Common.MultipleWorkspace import MultipleWorkspace as mws
>  import uuid
>  from CommonDataClass.Exceptions import BadExpression
>  from Common.caching import cached_property
>  import subprocess
> +from collections import OrderedDict
>  ## Regular expression used to find out place holders in string template
>  gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE |
> re.UNICODE)
> 
>  ## regular expressions for map file processing
>  startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
> @@ -2129,10 +2130,25 @@ def PackByteFormatGUID(Guid):
>                  Guid[8],
>                  Guid[9],
>                  Guid[10],
>                  )
> 
> +## DeepCopy dict/OrderedDict recusively
> +#
> +#   @param      ori_dict    a nested dict or ordereddict
> +#
> +#   @retval     new dict or orderdict
> +#
> +def CopyDict(ori_dict):
> +    dict_type = ori_dict.__class__
> +    new_dict = dict_type()
> +    for key in ori_dict:
> +        if isinstance(ori_dict[key],(dict,OrderedDict)):
> +            new_dict[key] = CopyDict(ori_dict[key])
> +        else:
> +            new_dict[key] = ori_dict[key]
> +    return new_dict
>  ##
>  #
>  # This acts like the main() function for the script, unless it is 'import'ed into
> another
>  # script.
>  #
> diff --git a/BaseTools/Source/Python/CommonDataClass/CommonClass.py
> b/BaseTools/Source/Python/CommonDataClass/CommonClass.py
> index a98cf8a7c5..336bb11671 100644
> --- a/BaseTools/Source/Python/CommonDataClass/CommonClass.py
> +++ b/BaseTools/Source/Python/CommonDataClass/CommonClass.py
> @@ -78,5 +78,20 @@ class SkuInfoClass(object):
>                      'VariableOffset = ' + str(self.VariableOffset) + "," + \
>                      'HiiDefaultValue = ' + str(self.HiiDefaultValue) + "," + \
>                      'VpdOffset = ' + str(self.VpdOffset) + "," + \
>                      'DefaultValue = ' + str(self.DefaultValue) + ","
>          return Rtn
> +
> +    def __deepcopy__(self,memo):
> +        new_sku = SkuInfoClass()
> +        new_sku.SkuIdName = self.SkuIdName
> +        new_sku.SkuId = self.SkuId
> +        new_sku.VariableName = self.VariableName
> +        new_sku.VariableGuid = self.VariableGuid
> +        new_sku.VariableGuidValue = self.VariableGuidValue
> +        new_sku.VariableOffset = self.VariableOffset
> +        new_sku.HiiDefaultValue = self.HiiDefaultValue
> +        new_sku.VariableAttribute = self.VariableAttribute
> +        new_sku.DefaultStoreDict = {key:value for key,value in
> self.DefaultStoreDict.items()}
> +        new_sku.VpdOffset = self.VpdOffset
> +        new_sku.DefaultValue = self.DefaultValue
> +        return new_sku
> diff --git a/BaseTools/Source/Python/Workspace/BuildClassObject.py
> b/BaseTools/Source/Python/Workspace/BuildClassObject.py
> index 95edc376fe..7d10391c1e 100644
> --- a/BaseTools/Source/Python/Workspace/BuildClassObject.py
> +++ b/BaseTools/Source/Python/Workspace/BuildClassObject.py
> @@ -14,10 +14,12 @@
>  from collections import OrderedDict, namedtuple
>  from Common.DataType import *
>  import collections
>  import re
>  from collections import OrderedDict
> +from Common.Misc import CopyDict
> +import copy
>  StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_\[\]]*$')
>  ArrayIndex = re.compile("\[\s*\d{0,1}\s*\]")
>  ## PcdClassObject
>  #
>  # This Class is used for PcdObject
> @@ -152,10 +154,41 @@ class PcdClassObject(object):
>      # @retval truple() Key for hash table
>      #
>      def __hash__(self):
>          return hash((self.TokenCName, self.TokenSpaceGuidCName))
> 
> +    def sharedcopy(self,new_pcd):
> +        new_pcd.TokenCName = self.TokenCName
> +        new_pcd.TokenSpaceGuidCName = self.TokenSpaceGuidCName
> +        new_pcd.TokenSpaceGuidValue = self.TokenSpaceGuidValue
> +        new_pcd.Type = self.Type
> +        new_pcd.DatumType = self.DatumType
> +        new_pcd.DefaultValue = self.DefaultValue
> +        new_pcd.TokenValue = self.TokenValue
> +        new_pcd.MaxDatumSize = self.MaxDatumSize
> +
> +        new_pcd.Phase = self.Phase
> +        new_pcd.Pending = self.Pending
> +        new_pcd.IsOverrided = self.IsOverrided
> +        new_pcd.IsFromBinaryInf = self.IsFromBinaryInf
> +        new_pcd.IsFromDsc = self.IsFromDsc
> +        new_pcd.PcdValueFromComm = self.PcdValueFromComm
> +        new_pcd.PcdValueFromFdf = self.PcdValueFromFdf
> +        new_pcd.UserDefinedDefaultStoresFlag =
> self.UserDefinedDefaultStoresFlag
> +        new_pcd.DscRawValue = self.DscRawValue
> +        new_pcd.CustomAttribute = self.CustomAttribute
> +        new_pcd.validateranges = [item for item in self.validateranges]
> +        new_pcd.validlists = [item for item in self.validlists]
> +        new_pcd.expressions = [item for item in self.expressions]
> +        new_pcd.SkuInfoList = {key: copy.deepcopy(skuobj) for key,skuobj in
> self.SkuInfoList.items()}
> +        return new_pcd
> +
> +    def __deepcopy__(self,memo):
> +        new_pcd = PcdClassObject()
> +        self.sharedcopy(new_pcd)
> +        return new_pcd
> +
>  class StructurePcd(PcdClassObject):
>      def __init__(self, StructuredPcdIncludeFile=None, Packages=None,
> Name=None, Guid=None, Type=None, DatumType=None, Value=None,
> Token=None, MaxDatumSize=None, SkuInfoList=None, IsOverrided=False,
> GuidValue=None, validateranges=None, validlists=None,
> expressions=None,default_store = TAB_DEFAULT_STORES_DEFAULT):
>          if SkuInfoList is None:
>              SkuInfoList = {}
>          if validateranges is None:
> @@ -240,10 +273,29 @@ class StructurePcd(PcdClassObject):
>              self.PkgPath = PcdObject.PkgPath if PcdObject.PkgPath else
> self.PkgPath
>              self.ValueChain = PcdObject.ValueChain if PcdObject.ValueChain else
> self.ValueChain
>              self.PcdFieldValueFromComm = PcdObject.PcdFieldValueFromComm
> if PcdObject.PcdFieldValueFromComm else self.PcdFieldValueFromComm
>              self.PcdFieldValueFromFdf = PcdObject.PcdFieldValueFromFdf if
> PcdObject.PcdFieldValueFromFdf else self.PcdFieldValueFromFdf
> 
> +    def __deepcopy__(self,memo):
> +        new_pcd = StructurePcd()
> +        self.sharedcopy(new_pcd)
> +
> +        new_pcd.DefaultValueFromDec = self.DefaultValueFromDec
> +        new_pcd.PcdMode = self.PcdMode
> +        new_pcd.StructName = self.DatumType
> +        new_pcd.PcdDefineLineNo = self.PcdDefineLineNo
> +        new_pcd.PkgPath = self.PkgPath
> +        new_pcd.StructuredPcdIncludeFile = [item for item in
> self.StructuredPcdIncludeFile]
> +        new_pcd.PackageDecs = [item for item in self.PackageDecs]
> +        new_pcd.DefaultValues = CopyDict(self.DefaultValues)
> +        new_pcd.DefaultFromDSC=CopyDict(self.DefaultFromDSC)
> +        new_pcd.SkuOverrideValues = CopyDict(self.SkuOverrideValues)
> +        new_pcd.PcdFieldValueFromComm =
> CopyDict(self.PcdFieldValueFromComm)
> +        new_pcd.PcdFieldValueFromFdf =
> CopyDict(self.PcdFieldValueFromFdf)
> +        new_pcd.ValueChain = {item for item in self.ValueChain}
> +        return new_pcd
> +
>  LibraryClassObject = namedtuple('LibraryClassObject',
> ['LibraryClass','SupModList'], verbose=False)
> 
>  ## ModuleBuildClassObject
>  #
>  # This Class defines ModuleBuildClass
> diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py
> b/BaseTools/Source/Python/Workspace/DscBuildData.py
> index 11aa63fb26..5655034b88 100644
> --- a/BaseTools/Source/Python/Workspace/DscBuildData.py
> +++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
> @@ -970,11 +970,11 @@ class DscBuildData(PlatformBuildClassObject):
>              for skuid in pcd.SkuInfoList:
>                  skuobj = pcd.SkuInfoList.get(skuid)
>                  if TAB_DEFAULT_STORES_DEFAULT not in skuobj.DefaultStoreDict:
>                      PcdDefaultStoreSet = set(defaultstorename  for defaultstorename
> in skuobj.DefaultStoreDict)
>                      mindefaultstorename =
> DefaultStoreMgr.GetMin(PcdDefaultStoreSet)
> -                    skuobj.DefaultStoreDict[TAB_DEFAULT_STORES_DEFAULT] =
> copy.deepcopy(skuobj.DefaultStoreDict[mindefaultstorename])
> +                    skuobj.DefaultStoreDict[TAB_DEFAULT_STORES_DEFAULT] =
> CopyDict(skuobj.DefaultStoreDict[mindefaultstorename])
>          return Pcds
> 
>      def RecoverCommandLinePcd(self):
>          def UpdateCommandLineValue(pcd):
>              if pcd.Type in
> [self._PCD_TYPE_STRING_[MODEL_PCD_FIXED_AT_BUILD],
> @@ -1501,11 +1501,11 @@ class DscBuildData(PlatformBuildClassObject):
>                      PcdDefaultStoreSet = set(defaultstorename  for defaultstorename
> in stru_pcd.SkuOverrideValues[nextskuid])
>                      mindefaultstorename =
> DefaultStoreMgr.GetMin(PcdDefaultStoreSet)
> 
>                      for defaultstoreid in DefaultStores:
>                          if defaultstoreid not in stru_pcd.SkuOverrideValues[skuid]:
> -                            stru_pcd.SkuOverrideValues[skuid][defaultstoreid] =
> copy.deepcopy(stru_pcd.SkuOverrideValues[nextskuid][mindefaultstorena
> me])
> +                            stru_pcd.SkuOverrideValues[skuid][defaultstoreid] =
> CopyDict(stru_pcd.SkuOverrideValues[nextskuid][mindefaultstorename])
>                              stru_pcd.ValueChain.add((skuid, defaultstoreid))
>          S_pcd_set = DscBuildData.OverrideByFdf(S_pcd_set,self.WorkspaceDir)
>          S_pcd_set = DscBuildData.OverrideByComm(S_pcd_set)
>          Str_Pcd_Values = self.GenerateByteArrayValue(S_pcd_set)
>          if Str_Pcd_Values:
> @@ -2597,11 +2597,11 @@ class DscBuildData(PlatformBuildClassObject):
>                  for skuid in PcdObj.SkuInfoList:
>                      skuobj = PcdObj.SkuInfoList[skuid]
>                      mindefaultstorename =
> DefaultStoreObj.GetMin(set(defaultstorename for defaultstorename in
> skuobj.DefaultStoreDict))
>                      for defaultstorename in DefaultStores:
>                          if defaultstorename not in skuobj.DefaultStoreDict:
> -                            skuobj.DefaultStoreDict[defaultstorename] =
> copy.deepcopy(skuobj.DefaultStoreDict[mindefaultstorename])
> +                            skuobj.DefaultStoreDict[defaultstorename] =
> CopyDict(skuobj.DefaultStoreDict[mindefaultstorename])
>                      skuobj.HiiDefaultValue =
> skuobj.DefaultStoreDict[mindefaultstorename]
>              for skuname, skuid in SkuIds.items():
>                  if skuname not in PcdObj.SkuInfoList:
>                      nextskuid = self.SkuIdMgr.GetNextSkuId(skuname)
>                      while nextskuid not in PcdObj.SkuInfoList:
> --
> 2.19.1.windows.1
> 
> _______________________________________________
> edk2-devel mailing list
> edk2-devel@lists.01.org
> https://lists.01.org/mailman/listinfo/edk2-devel