From mboxrd@z Thu Jan 1 00:00:00 1970 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 192.55.52.151, mailfrom: bob.c.feng@intel.com) Received: from mga17.intel.com (mga17.intel.com [192.55.52.151]) by groups.io with SMTP; Mon, 20 May 2019 18:56:42 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga008.fm.intel.com ([10.253.24.58]) by fmsmga107.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 20 May 2019 18:56:42 -0700 X-ExtLoop1: 1 Received: from fmsmsx108.amr.corp.intel.com ([10.18.124.206]) by fmsmga008.fm.intel.com with ESMTP; 20 May 2019 18:56:41 -0700 Received: from fmsmsx116.amr.corp.intel.com (10.18.116.20) by FMSMSX108.amr.corp.intel.com (10.18.124.206) with Microsoft SMTP Server (TLS) id 14.3.408.0; Mon, 20 May 2019 18:56:41 -0700 Received: from shsmsx103.ccr.corp.intel.com (10.239.4.69) by fmsmsx116.amr.corp.intel.com (10.18.116.20) with Microsoft SMTP Server (TLS) id 14.3.408.0; Mon, 20 May 2019 18:56:41 -0700 Received: from shsmsx101.ccr.corp.intel.com ([169.254.1.129]) by SHSMSX103.ccr.corp.intel.com ([169.254.4.70]) with mapi id 14.03.0415.000; Tue, 21 May 2019 09:56:39 +0800 From: "Bob Feng" To: "Rodriguez, Christian" , "devel@edk2.groups.io" CC: "Gao, Liming" , "Zhu, Yonghong" Subject: Re: [Patch V3] BaseTools: Library hashing fix and optimization for --hash feature Thread-Topic: [Patch V3] BaseTools: Library hashing fix and optimization for --hash feature Thread-Index: AQHVDxbYJ5kcBYLI0k+Ne4uNdyx47qZ00WBg Date: Tue, 21 May 2019 01:56:38 +0000 Message-ID: <08650203BA1BD64D8AD9B6D5D74A85D1601114DB@SHSMSX101.ccr.corp.intel.com> References: <20190520141754.12920-1-christian.rodriguez@intel.com> In-Reply-To: <20190520141754.12920-1-christian.rodriguez@intel.com> Accept-Language: zh-CN, en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-originating-ip: [10.239.127.40] MIME-Version: 1.0 Return-Path: bob.c.feng@intel.com Content-Language: en-US Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable This patch is good to me. Reviewed-by: Bob Feng -----Original Message----- From: Rodriguez, Christian=20 Sent: Monday, May 20, 2019 10:18 PM To: devel@edk2.groups.io Cc: Feng, Bob C ; Gao, Liming ;= Zhu, Yonghong Subject: [Patch V3] BaseTools: Library hashing fix and optimization for --h= ash feature BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=3D1788 In V3: Must generate hashes before attempting to copy from cache for hash v= erifcation In V2: Build failure caused by passing incorrect boolean paramet= er to SaveFileOnChange(). Fixed for patch instances. Library hashing is now supported by the --hash feature. The --hash feature = implementation assumed that the hashing could be done in place once per mod= ule, but that isn't true for libraries due to the fact that they are built = as dependencies. So on a clean build, we now generate the .hash after the l= ibrary dependencies are complete. Added early escape as optimization, if hash already exists in memory. Signed-off-by: Christian Rodriguez Cc: Bob Feng Cc: Liming Gao Cc: Yonghong Zhu --- BaseTools/Source/Python/AutoGen/AutoGen.py | 53 ++++++++++++++++++++++++= ++++++++++++++++------------- BaseTools/Source/Python/Common/GlobalData.py | 6 ++++++ BaseTools/Source/Python/build/build.py | 7 ++++++- 3 files changed, 52 insertions(+), 14 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/= Python/AutoGen/AutoGen.py index 31721a6f9f..81efc4f06c 100644 --- a/BaseTools/Source/Python/AutoGen/AutoGen.py +++ b/BaseTools/Source/Python/AutoGen/AutoGen.py @@ -3935,6 +3935,7 @@ class ModuleAutoGen(AutoGen): f =3D open(HashFile, 'r') CacheHash =3D f.read() f.close() + self.GenModuleHash() if GlobalData.gModuleHash[self.Arch][self.Name]: if CacheHash =3D=3D GlobalData.gModuleHash[self.Arch][self= .Name]: for root, dir, files in os.walk(FileDir): @@ -4093,13 +4094,20 @@ class ModuleAutoGen(AutoGen): return RetVal =20 def GenModuleHash(self): + # Initialize a dictionary for each arch type if self.Arch not in GlobalData.gModuleHash: GlobalData.gModuleHash[self.Arch] =3D {} - if self.Name in GlobalData.gModuleHash[self.Arch] and GlobalData.g= BinCacheSource and self.AttemptModuleCacheCopy(): - return False + + # Early exit if module or library has been hashed and is in memory + if self.Name in GlobalData.gModuleHash[self.Arch]: + return=20 + GlobalData.gModuleHash[self.Arch][self.Name].encode('utf-8') + + # Initialze hash object m =3D hashlib.md5() + # Add Platform level hash m.update(GlobalData.gPlatformHash.encode('utf-8')) + # Add Package level hash if self.DependentPackageList: for Pkg in sorted(self.DependentPackageList, key=3Dlambda x: x= .PackageName): @@ -4118,6 +4126,7 @@ class ModuleAutoGen(AutoGen): Content =3D f.read() f.close() m.update(Content) + # Add Module's source files if self.SourceFileList: for File in sorted(self.SourceFileList, key=3Dlambda x: str(x)= ): @@ -4126,27 +4135,45 @@ class ModuleAutoGen(AutoGen): f.close() m.update(Content) =20 - ModuleHashFile =3D path.join(self.BuildDir, self.Name + ".hash") - if self.Name not in GlobalData.gModuleHash[self.Arch]: - GlobalData.gModuleHash[self.Arch][self.Name] =3D m.hexdigest() - if GlobalData.gBinCacheSource and self.AttemptModuleCacheCopy(): - return False - return SaveFileOnChange(ModuleHashFile, m.hexdigest(), False) + GlobalData.gModuleHash[self.Arch][self.Name] =3D m.hexdigest() + + return=20 + GlobalData.gModuleHash[self.Arch][self.Name].encode('utf-8') =20 ## Decide whether we can skip the ModuleAutoGen process def CanSkipbyHash(self): + # Hashing feature is off + if not GlobalData.gUseHashCache: + return False + + # Initialize a dictionary for each arch type + if self.Arch not in GlobalData.gBuildHashSkipTracking: + GlobalData.gBuildHashSkipTracking[self.Arch] =3D dict() + # If library or Module is binary do not skip by hash if self.IsBinaryModule: return False + # .inc is contains binary information so do not skip by hash as we= ll for f_ext in self.SourceFileList: if '.inc' in str(f_ext): return False - if GlobalData.gUseHashCache: - # If there is a valid hash or function generated a valid hash;= function will return False - # and the statement below will return True - return not self.GenModuleHash() - return False + + # Use Cache, if exists and if Module has a copy in cache + if GlobalData.gBinCacheSource and self.AttemptModuleCacheCopy(): + return True + + # Early exit for libraries that haven't yet finished building + HashFile =3D path.join(self.BuildDir, self.Name + ".hash") + if self.IsLibrary and not os.path.exists(HashFile): + return False + + # Return a Boolean based on if can skip by hash, either from memor= y or from IO. + if self.Name not in GlobalData.gBuildHashSkipTracking[self.Arch]: + # If hashes are the same, SaveFileOnChange() will return False= . + GlobalData.gBuildHashSkipTracking[self.Arch][self.Name] =3D no= t SaveFileOnChange(HashFile, self.GenModuleHash(), True) + return GlobalData.gBuildHashSkipTracking[self.Arch][self.Name] + else: + return=20 + GlobalData.gBuildHashSkipTracking[self.Arch][self.Name] =20 ## Decide whether we can skip the ModuleAutoGen process # If any source file is newer than the module than we cannot skip dif= f --git a/BaseTools/Source/Python/Common/GlobalData.py b/BaseTools/Source/P= ython/Common/GlobalData.py index 79f23c892d..95e28a988f 100644 --- a/BaseTools/Source/Python/Common/GlobalData.py +++ b/BaseTools/Source/Python/Common/GlobalData.py @@ -112,3 +112,9 @@ gSikpAutoGenCache =3D set() # Dictionary for tracking = Module build status as success or failure # False -> Fail : True -> Succes= s gModuleBuildTracking =3D dict() + +# Dictionary of booleans that dictate whether a module or # library can=20 +be skiped +# Top Dict: Key: Arch Type Value: Dictionary +# Second Dict: Key: Module\Library Name Value: True\False +gBuildHashSkipTracking =3D dict() diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Pyth= on/build/build.py index 7271570d29..027061191c 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -593,7 +593,7 @@ class BuildTask: # def AddDependency(self, Dependency): for Dep in Dependency: - if not Dep.BuildObject.IsBinaryModule: + if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.= CanSkipbyHash(): self.DependencyList.append(BuildTask.New(Dep)) # BuildT= ask list =20 ## The thread wrapper of LaunchCommand function @@ -605,6 +605,11 @@ c= lass BuildTask: try: self.BuildItem.BuildObject.BuildTime =3D LaunchCommand(Command= , WorkingDir) self.CompleteFlag =3D True + + # Run hash operation post dependency, to account for libs + if GlobalData.gUseHashCache and self.BuildItem.BuildObject.IsL= ibrary: + HashFile =3D path.join(self.BuildItem.BuildObject.BuildDir= , self.BuildItem.BuildObject.Name + ".hash") + SaveFileOnChange(HashFile,=20 + self.BuildItem.BuildObject.GenModuleHash(), True) except: # # TRICK: hide the output of threads left running, so that the = user can -- 2.19.1.windows.1