From mboxrd@z Thu Jan 1 00:00:00 1970 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 192.55.52.88, mailfrom: bob.c.feng@intel.com) Received: from mga01.intel.com (mga01.intel.com [192.55.52.88]) by groups.io with SMTP; Wed, 29 May 2019 23:18:29 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga006.fm.intel.com ([10.253.24.20]) by fmsmga101.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 29 May 2019 23:18:28 -0700 X-ExtLoop1: 1 Received: from fmsmsx103.amr.corp.intel.com ([10.18.124.201]) by fmsmga006.fm.intel.com with ESMTP; 29 May 2019 23:18:29 -0700 Received: from fmsmsx158.amr.corp.intel.com (10.18.116.75) by FMSMSX103.amr.corp.intel.com (10.18.124.201) with Microsoft SMTP Server (TLS) id 14.3.408.0; Wed, 29 May 2019 23:18:28 -0700 Received: from shsmsx104.ccr.corp.intel.com (10.239.4.70) by fmsmsx158.amr.corp.intel.com (10.18.116.75) with Microsoft SMTP Server (TLS) id 14.3.408.0; Wed, 29 May 2019 23:18:28 -0700 Received: from shsmsx101.ccr.corp.intel.com ([169.254.1.10]) by SHSMSX104.ccr.corp.intel.com ([169.254.5.137]) with mapi id 14.03.0415.000; Thu, 30 May 2019 14:18:26 +0800 From: "Bob Feng" To: "Rodriguez, Christian" , "devel@edk2.groups.io" CC: "Gao, Liming" , "Zhu, Yonghong" Subject: Re: [Patch V4 2/2] BaseTools: Refactor hash tracking after checking for Sources section Thread-Topic: [Patch V4 2/2] BaseTools: Refactor hash tracking after checking for Sources section Thread-Index: AQHVFjtZa+KzvGEpEkK72TZtqNTY26aDMmrw Date: Thu, 30 May 2019 06:18:26 +0000 Message-ID: <08650203BA1BD64D8AD9B6D5D74A85D160128325@SHSMSX101.ccr.corp.intel.com> References: <20190529162649.3236-1-christian.rodriguez@intel.com> <20190529162649.3236-3-christian.rodriguez@intel.com> In-Reply-To: <20190529162649.3236-3-christian.rodriguez@intel.com> Accept-Language: zh-CN, en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-originating-ip: [10.239.127.40] MIME-Version: 1.0 Return-Path: bob.c.feng@intel.com Content-Language: en-US Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable Reviewed-by: Bob Feng -----Original Message----- From: Rodriguez, Christian=20 Sent: Thursday, May 30, 2019 12:27 AM To: devel@edk2.groups.io Cc: Feng, Bob C ; Gao, Liming ;= Zhu, Yonghong Subject: [Patch V4 2/2] BaseTools: Refactor hash tracking after checking fo= r Sources section BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=3D1804 After adding a check to see if [Sources] section lists all the header type = files of a module, track module and library hashes for --hash feature. If a= bove check is not in compilance for a library or module, force hash invalid= ation on that library or module. Signed-off-by: Christian Rodriguez Cc: Bob Feng Cc: Liming Gao Cc: Yonghong Zhu --- BaseTools/Source/Python/AutoGen/AutoGen.py | 6 +- BaseTools/Source/Python/AutoGen/GenMake.py | 6 ++ BaseTools/Source/Python/Common/GlobalData.py | 3 +- BaseTools/Source/Python/build/build.py | 65 ++++++++++++-------- 4 files changed, 53 insertions(+), 27 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/= Python/AutoGen/AutoGen.py index a5bef4f7c6..a376bc24d6 100644 --- a/BaseTools/Source/Python/AutoGen/AutoGen.py +++ b/BaseTools/Source/Python/AutoGen/AutoGen.py @@ -3989,7 +3989,8 @@ class ModuleAutoGen(AutoGen): for LibraryAutoGen in self.LibraryAutoGenList: LibraryAutoGen.CreateMakeFile() =20 - if self.CanSkip(): + # Don't enable if hash feature enabled, CanSkip uses timestamps to= determine build skipping + if not GlobalData.gUseHashCache and self.CanSkip(): return =20 if len(self.CustomMakefile) =3D=3D 0: @@ -4032,7 +4033,8 @@ class ModuleAutoGen(AutoGen): for LibraryAutoGen in self.LibraryAutoGenList: LibraryAutoGen.CreateCodeFile() =20 - if self.CanSkip(): + # Don't enable if hash feature enabled, CanSkip uses timestamps to= determine build skipping + if not GlobalData.gUseHashCache and self.CanSkip(): return =20 AutoGenList =3D [] diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/= Python/AutoGen/GenMake.py index 5c992d7c26..212ca0fa7f 100644 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py @@ -935,10 +935,16 @@ cleanlib: continue headerFileDependencySet.add(aFileName) =20 + # Ensure that gModuleBuildTracking has been initialized per archit= ecture + if self._AutoGenObject.Arch not in GlobalData.gModuleBuildTracking= : + GlobalData.gModuleBuildTracking[self._AutoGenObject.Arch] =3D= =20 + dict() + # Check if a module dependency header file is missing from the mod= ule's MetaFile for aFile in headerFileDependencySet: if aFile in headerFilesInMetaFileSet: continue + if GlobalData.gUseHashCache: + GlobalData.gModuleBuildTracking[self._AutoGenObject.Arch][= self._AutoGenObject] =3D 'FAIL_METAFILE' EdkLogger.warn("build","Module MetaFile [Sources] is missing l= ocal header!", ExtraData =3D "Local Header: " + aFile + " not fou= nd in " + self._AutoGenObject.MetaFile.Path ) diff --git a/BaseTools/Source/Python/Common/GlobalData.py b/BaseTools/Sourc= e/Python/Common/GlobalData.py index 95e28a988f..bd45a43728 100644 --- a/BaseTools/Source/Python/Common/GlobalData.py +++ b/BaseTools/Source/Python/Common/GlobalData.py @@ -110,7 +110,8 @@ gEnableGenfdsMultiThread =3D False gSikpAutoGenCache = =3D set() =20 # Dictionary for tracking Module build status as success or failure -# Fal= se -> Fail : True -> Success +# Top Dict: Key: Arch Type Value: Dictionary +# Second Dict: Key: AutoGen Obj Value: 'SUCCESS'\'FAIL'\'FAIL_METAFILE= ' gModuleBuildTracking =3D dict() =20 # Dictionary of booleans that dictate whether a module or diff --git a/Bas= eTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.p= y index 80ceb98310..0855d4561c 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -625,8 +625,16 @@ class BuildTask: BuildTask._ErrorFlag.set() BuildTask._ErrorMessage =3D "%s broken\n %s [%s]" % \ (threading.currentThread().getName()= , Command, WorkingDir) - if self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking a= nd not BuildTask._ErrorFlag.isSet(): - GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] = =3D True + + # Set the value used by hash invalidation flow in GlobalData.gModu= leBuildTracking to 'SUCCESS' + # If Module or Lib is being tracked, it did not fail header check = test, and built successfully + if (self.BuildItem.BuildObject.Arch in GlobalData.gModuleBuildTrac= king and + self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking[s= elf.BuildItem.BuildObject.Arch] and + GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch= ][self.BuildItem.BuildObject] !=3D 'FAIL_METAFILE' and + not BuildTask._ErrorFlag.isSet() + ): + GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arc= h][self.BuildItem.BuildObject] =3D 'SUCCESS' + # indicate there's a thread is available for another build task BuildTask._RunningQueueLock.acquire() BuildTask._RunningQueue.pop(self.BuildItem) @@ -1154,27 +1162,30 @@ class Build(): # # def invalidateHash(self): - # GlobalData.gModuleBuildTracking contains only modules that canno= t be skipped by hash - for moduleAutoGenObj in GlobalData.gModuleBuildTracking.keys(): - # False =3D=3D FAIL : True =3D=3D Success - # Skip invalidating for Successful module builds - if GlobalData.gModuleBuildTracking[moduleAutoGenObj] =3D=3D Tr= ue: - continue + # Only for hashing feature + if not GlobalData.gUseHashCache: + return + + # GlobalData.gModuleBuildTracking contains only modules or libs th= at cannot be skipped by hash + for moduleAutoGenObjArch in GlobalData.gModuleBuildTracking.keys()= : + for moduleAutoGenObj in GlobalData.gModuleBuildTracking[module= AutoGenObjArch].keys(): + # Skip invalidating for Successful Module/Lib builds + if GlobalData.gModuleBuildTracking[moduleAutoGenObjArch][m= oduleAutoGenObj] =3D=3D 'SUCCESS': + continue =20 - # The module failed to build or failed to start building, from= this point on + # The module failed to build, failed to start building,=20 + or failed the header check test from this point on =20 - # Remove .hash from build - if GlobalData.gUseHashCache: - ModuleHashFile =3D path.join(moduleAutoGenObj.BuildDir, mo= duleAutoGenObj.Name + ".hash") + # Remove .hash from build + ModuleHashFile =3D=20 + os.path.join(moduleAutoGenObj.BuildDir, moduleAutoGenObj.Name +=20 + ".hash") if os.path.exists(ModuleHashFile): os.remove(ModuleHashFile) =20 - # Remove .hash file from cache - if GlobalData.gBinCacheDest: - FileDir =3D path.join(GlobalData.gBinCacheDest, moduleAuto= GenObj.Arch, moduleAutoGenObj.SourceDir, moduleAutoGenObj.MetaFile.BaseName= ) - HashFile =3D path.join(FileDir, moduleAutoGenObj.Name + '.= hash') - if os.path.exists(HashFile): - os.remove(HashFile) + # Remove .hash file from cache + if GlobalData.gBinCacheDest: + FileDir =3D os.path.join(GlobalData.gBinCacheDest, mod= uleAutoGenObj.Arch, moduleAutoGenObj.SourceDir, moduleAutoGenObj.MetaFile.B= aseName) + HashFile =3D os.path.join(FileDir, moduleAutoGenObj.Na= me + '.hash') + if os.path.exists(HashFile): + os.remove(HashFile) =20 ## Build a module or platform # @@ -1825,9 +1836,11 @@ class Build(): if self.Target =3D=3D "genmake": return True self.BuildModules.append(Ma) - # Initialize all modules in tracking to False = (FAIL) - if Ma not in GlobalData.gModuleBuildTracking: - GlobalData.gModuleBuildTracking[Ma] =3D Fa= lse + # Initialize all modules in tracking to 'FAIL' + if Ma.Arch not in GlobalData.gModuleBuildTrack= ing: + GlobalData.gModuleBuildTracking[Ma.Arch] = =3D dict() + if Ma not in GlobalData.gModuleBuildTracking[M= a.Arch]: + GlobalData.gModuleBuildTracking[Ma.Arch][M= a] =3D 'FAIL' self.AutoGenTime +=3D int(round((time.time() - AutoGen= Start))) MakeStart =3D time.time() for Ma in self.BuildModules: @@ -1911,6 +1924,7 @@ class Build(): # Save MAP buffer into MAP file. # self._SaveMapFile (MapBuffer, Wa) + self.invalidateHash() =20 def _GenFfsCmd(self,ArchList): # convert dictionary of Cmd:(Inf,Arch) @@ -2009,9 +2023,11 @@ clas= s Build(): if self.Target =3D=3D "genmake": continue self.BuildModules.append(Ma) - # Initialize all modules in tracking to False (FAI= L) - if Ma not in GlobalData.gModuleBuildTracking: - GlobalData.gModuleBuildTracking[Ma] =3D False + # Initialize all modules in tracking to 'FAIL' + if Ma.Arch not in GlobalData.gModuleBuildTracking: + GlobalData.gModuleBuildTracking[Ma.Arch] =3D d= ict() + if Ma not in GlobalData.gModuleBuildTracking[Ma.Ar= ch]: + GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = =3D 'FAIL' self.Progress.Stop("done!") self.AutoGenTime +=3D int(round((time.time() - AutoGen= Start))) MakeStart =3D time.time() @@ -2099,6 +2115,7 @@ class = Build(): # Save MAP buffer into MAP file. # self._SaveMapFile(MapBuffer, Wa) + self.invalidateHash() =20 ## Generate GuidedSectionTools.txt in the FV directories. # -- 2.21.0.windows.1