From mboxrd@z Thu Jan 1 00:00:00 1970 Authentication-Results: mx.groups.io; dkim=missing; spf=fail (domain: intel.com, ip: , mailfrom: christian.rodriguez@intel.com) Received: from mga17.intel.com (mga17.intel.com []) by groups.io with SMTP; Wed, 29 May 2019 09:26:57 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga001.jf.intel.com ([10.7.209.18]) by fmsmga107.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 29 May 2019 09:26:57 -0700 X-ExtLoop1: 1 Received: from rodrigu3-desk.amr.corp.intel.com ([10.7.163.75]) by orsmga001.jf.intel.com with ESMTP; 29 May 2019 09:26:56 -0700 From: "Christian Rodriguez" To: devel@edk2.groups.io Cc: Bob Feng , Liming Gao , Yonghong Zhu Subject: [Patch V4 2/2] BaseTools: Refactor hash tracking after checking for Sources section Date: Wed, 29 May 2019 09:26:49 -0700 Message-Id: <20190529162649.3236-3-christian.rodriguez@intel.com> X-Mailer: git-send-email 2.21.0.windows.1 In-Reply-To: <20190529162649.3236-1-christian.rodriguez@intel.com> References: <20190529162649.3236-1-christian.rodriguez@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1804 After adding a check to see if [Sources] section lists all the header type files of a module, track module and library hashes for --hash feature. If above check is not in compilance for a library or module, force hash invalidation on that library or module. Signed-off-by: Christian Rodriguez Cc: Bob Feng Cc: Liming Gao Cc: Yonghong Zhu --- BaseTools/Source/Python/AutoGen/AutoGen.py | 6 +- BaseTools/Source/Python/AutoGen/GenMake.py | 6 ++ BaseTools/Source/Python/Common/GlobalData.py | 3 +- BaseTools/Source/Python/build/build.py | 65 ++++++++++++-------- 4 files changed, 53 insertions(+), 27 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/Python/AutoGen/AutoGen.py index a5bef4f7c6..a376bc24d6 100644 --- a/BaseTools/Source/Python/AutoGen/AutoGen.py +++ b/BaseTools/Source/Python/AutoGen/AutoGen.py @@ -3989,7 +3989,8 @@ class ModuleAutoGen(AutoGen): for LibraryAutoGen in self.LibraryAutoGenList: LibraryAutoGen.CreateMakeFile() - if self.CanSkip(): + # Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping + if not GlobalData.gUseHashCache and self.CanSkip(): return if len(self.CustomMakefile) == 0: @@ -4032,7 +4033,8 @@ class ModuleAutoGen(AutoGen): for LibraryAutoGen in self.LibraryAutoGenList: LibraryAutoGen.CreateCodeFile() - if self.CanSkip(): + # Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping + if not GlobalData.gUseHashCache and self.CanSkip(): return AutoGenList = [] diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py index 5c992d7c26..212ca0fa7f 100644 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py @@ -935,10 +935,16 @@ cleanlib: continue headerFileDependencySet.add(aFileName) + # Ensure that gModuleBuildTracking has been initialized per architecture + if self._AutoGenObject.Arch not in GlobalData.gModuleBuildTracking: + GlobalData.gModuleBuildTracking[self._AutoGenObject.Arch] = dict() + # Check if a module dependency header file is missing from the module's MetaFile for aFile in headerFileDependencySet: if aFile in headerFilesInMetaFileSet: continue + if GlobalData.gUseHashCache: + GlobalData.gModuleBuildTracking[self._AutoGenObject.Arch][self._AutoGenObject] = 'FAIL_METAFILE' EdkLogger.warn("build","Module MetaFile [Sources] is missing local header!", ExtraData = "Local Header: " + aFile + " not found in " + self._AutoGenObject.MetaFile.Path ) diff --git a/BaseTools/Source/Python/Common/GlobalData.py b/BaseTools/Source/Python/Common/GlobalData.py index 95e28a988f..bd45a43728 100644 --- a/BaseTools/Source/Python/Common/GlobalData.py +++ b/BaseTools/Source/Python/Common/GlobalData.py @@ -110,7 +110,8 @@ gEnableGenfdsMultiThread = False gSikpAutoGenCache = set() # Dictionary for tracking Module build status as success or failure -# False -> Fail : True -> Success +# Top Dict: Key: Arch Type Value: Dictionary +# Second Dict: Key: AutoGen Obj Value: 'SUCCESS'\'FAIL'\'FAIL_METAFILE' gModuleBuildTracking = dict() # Dictionary of booleans that dictate whether a module or diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index 80ceb98310..0855d4561c 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -625,8 +625,16 @@ class BuildTask: BuildTask._ErrorFlag.set() BuildTask._ErrorMessage = "%s broken\n %s [%s]" % \ (threading.currentThread().getName(), Command, WorkingDir) - if self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking and not BuildTask._ErrorFlag.isSet(): - GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] = True + + # Set the value used by hash invalidation flow in GlobalData.gModuleBuildTracking to 'SUCCESS' + # If Module or Lib is being tracked, it did not fail header check test, and built successfully + if (self.BuildItem.BuildObject.Arch in GlobalData.gModuleBuildTracking and + self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch] and + GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch][self.BuildItem.BuildObject] != 'FAIL_METAFILE' and + not BuildTask._ErrorFlag.isSet() + ): + GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch][self.BuildItem.BuildObject] = 'SUCCESS' + # indicate there's a thread is available for another build task BuildTask._RunningQueueLock.acquire() BuildTask._RunningQueue.pop(self.BuildItem) @@ -1154,27 +1162,30 @@ class Build(): # # def invalidateHash(self): - # GlobalData.gModuleBuildTracking contains only modules that cannot be skipped by hash - for moduleAutoGenObj in GlobalData.gModuleBuildTracking.keys(): - # False == FAIL : True == Success - # Skip invalidating for Successful module builds - if GlobalData.gModuleBuildTracking[moduleAutoGenObj] == True: - continue + # Only for hashing feature + if not GlobalData.gUseHashCache: + return + + # GlobalData.gModuleBuildTracking contains only modules or libs that cannot be skipped by hash + for moduleAutoGenObjArch in GlobalData.gModuleBuildTracking.keys(): + for moduleAutoGenObj in GlobalData.gModuleBuildTracking[moduleAutoGenObjArch].keys(): + # Skip invalidating for Successful Module/Lib builds + if GlobalData.gModuleBuildTracking[moduleAutoGenObjArch][moduleAutoGenObj] == 'SUCCESS': + continue - # The module failed to build or failed to start building, from this point on + # The module failed to build, failed to start building, or failed the header check test from this point on - # Remove .hash from build - if GlobalData.gUseHashCache: - ModuleHashFile = path.join(moduleAutoGenObj.BuildDir, moduleAutoGenObj.Name + ".hash") + # Remove .hash from build + ModuleHashFile = os.path.join(moduleAutoGenObj.BuildDir, moduleAutoGenObj.Name + ".hash") if os.path.exists(ModuleHashFile): os.remove(ModuleHashFile) - # Remove .hash file from cache - if GlobalData.gBinCacheDest: - FileDir = path.join(GlobalData.gBinCacheDest, moduleAutoGenObj.Arch, moduleAutoGenObj.SourceDir, moduleAutoGenObj.MetaFile.BaseName) - HashFile = path.join(FileDir, moduleAutoGenObj.Name + '.hash') - if os.path.exists(HashFile): - os.remove(HashFile) + # Remove .hash file from cache + if GlobalData.gBinCacheDest: + FileDir = os.path.join(GlobalData.gBinCacheDest, moduleAutoGenObj.Arch, moduleAutoGenObj.SourceDir, moduleAutoGenObj.MetaFile.BaseName) + HashFile = os.path.join(FileDir, moduleAutoGenObj.Name + '.hash') + if os.path.exists(HashFile): + os.remove(HashFile) ## Build a module or platform # @@ -1825,9 +1836,11 @@ class Build(): if self.Target == "genmake": return True self.BuildModules.append(Ma) - # Initialize all modules in tracking to False (FAIL) - if Ma not in GlobalData.gModuleBuildTracking: - GlobalData.gModuleBuildTracking[Ma] = False + # Initialize all modules in tracking to 'FAIL' + if Ma.Arch not in GlobalData.gModuleBuildTracking: + GlobalData.gModuleBuildTracking[Ma.Arch] = dict() + if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]: + GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL' self.AutoGenTime += int(round((time.time() - AutoGenStart))) MakeStart = time.time() for Ma in self.BuildModules: @@ -1911,6 +1924,7 @@ class Build(): # Save MAP buffer into MAP file. # self._SaveMapFile (MapBuffer, Wa) + self.invalidateHash() def _GenFfsCmd(self,ArchList): # convert dictionary of Cmd:(Inf,Arch) @@ -2009,9 +2023,11 @@ class Build(): if self.Target == "genmake": continue self.BuildModules.append(Ma) - # Initialize all modules in tracking to False (FAIL) - if Ma not in GlobalData.gModuleBuildTracking: - GlobalData.gModuleBuildTracking[Ma] = False + # Initialize all modules in tracking to 'FAIL' + if Ma.Arch not in GlobalData.gModuleBuildTracking: + GlobalData.gModuleBuildTracking[Ma.Arch] = dict() + if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]: + GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL' self.Progress.Stop("done!") self.AutoGenTime += int(round((time.time() - AutoGenStart))) MakeStart = time.time() @@ -2099,6 +2115,7 @@ class Build(): # Save MAP buffer into MAP file. # self._SaveMapFile(MapBuffer, Wa) + self.invalidateHash() ## Generate GuidedSectionTools.txt in the FV directories. # -- 2.21.0.windows.1