Re: [Patch V4 2/2] BaseTools: Refactor hash tracking after checking for Sources section

["Bob Feng" <bob.c.feng@intel.com>]

Reviewed-by: Bob Feng <bob.c.feng@intel.com>

-----Original Message-----
From: Rodriguez, Christian 
Sent: Thursday, May 30, 2019 12:27 AM
To: devel@edk2.groups.io
Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming <liming.gao@intel.com>; Zhu, Yonghong <yonghong.zhu@intel.com>
Subject: [Patch V4 2/2] BaseTools: Refactor hash tracking after checking for Sources section

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1804

After adding a check to see if [Sources] section lists all the header type files of a module, track module and library hashes for --hash feature. If above check is not in compilance for a library or module, force hash invalidation on that library or module.

Signed-off-by: Christian Rodriguez <christian.rodriguez@intel.com>
Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Cc: Yonghong Zhu <yonghong.zhu@intel.com>
---
 BaseTools/Source/Python/AutoGen/AutoGen.py   |  6 +-
 BaseTools/Source/Python/AutoGen/GenMake.py   |  6 ++
 BaseTools/Source/Python/Common/GlobalData.py |  3 +-
 BaseTools/Source/Python/build/build.py       | 65 ++++++++++++--------
 4 files changed, 53 insertions(+), 27 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/Python/AutoGen/AutoGen.py
index a5bef4f7c6..a376bc24d6 100644
--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
@@ -3989,7 +3989,8 @@ class ModuleAutoGen(AutoGen):
             for LibraryAutoGen in self.LibraryAutoGenList:
                 LibraryAutoGen.CreateMakeFile()
 
-        if self.CanSkip():
+        # Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping
+        if not GlobalData.gUseHashCache and self.CanSkip():
             return
 
         if len(self.CustomMakefile) == 0:
@@ -4032,7 +4033,8 @@ class ModuleAutoGen(AutoGen):
             for LibraryAutoGen in self.LibraryAutoGenList:
                 LibraryAutoGen.CreateCodeFile()
 
-        if self.CanSkip():
+        # Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping
+        if not GlobalData.gUseHashCache and self.CanSkip():
             return
 
         AutoGenList = []
diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py
index 5c992d7c26..212ca0fa7f 100644
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -935,10 +935,16 @@ cleanlib:
                     continue
                 headerFileDependencySet.add(aFileName)
 
+        # Ensure that gModuleBuildTracking has been initialized per architecture
+        if self._AutoGenObject.Arch not in GlobalData.gModuleBuildTracking:
+            GlobalData.gModuleBuildTracking[self._AutoGenObject.Arch] = 
+ dict()
+
         # Check if a module dependency header file is missing from the module's MetaFile
         for aFile in headerFileDependencySet:
             if aFile in headerFilesInMetaFileSet:
                 continue
+            if GlobalData.gUseHashCache:
+                GlobalData.gModuleBuildTracking[self._AutoGenObject.Arch][self._AutoGenObject] = 'FAIL_METAFILE'
             EdkLogger.warn("build","Module MetaFile [Sources] is missing local header!",
                         ExtraData = "Local Header: " + aFile + " not found in " + self._AutoGenObject.MetaFile.Path
                         )
diff --git a/BaseTools/Source/Python/Common/GlobalData.py b/BaseTools/Source/Python/Common/GlobalData.py
index 95e28a988f..bd45a43728 100644
--- a/BaseTools/Source/Python/Common/GlobalData.py
+++ b/BaseTools/Source/Python/Common/GlobalData.py
@@ -110,7 +110,8 @@ gEnableGenfdsMultiThread = False  gSikpAutoGenCache = set()
 
 # Dictionary for tracking Module build status as success or failure -# False -> Fail : True -> Success
+# Top Dict:     Key: Arch Type              Value: Dictionary
+# Second Dict:  Key: AutoGen Obj    Value: 'SUCCESS'\'FAIL'\'FAIL_METAFILE'
 gModuleBuildTracking = dict()
 
 # Dictionary of booleans that dictate whether a module or diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 80ceb98310..0855d4561c 100644
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -625,8 +625,16 @@ class BuildTask:
             BuildTask._ErrorFlag.set()
             BuildTask._ErrorMessage = "%s broken\n    %s [%s]" % \
                                       (threading.currentThread().getName(), Command, WorkingDir)
-        if self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking and not BuildTask._ErrorFlag.isSet():
-            GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] = True
+
+        # Set the value used by hash invalidation flow in GlobalData.gModuleBuildTracking to 'SUCCESS'
+        # If Module or Lib is being tracked, it did not fail header check test, and built successfully
+        if (self.BuildItem.BuildObject.Arch in GlobalData.gModuleBuildTracking and
+           self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch] and
+           GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch][self.BuildItem.BuildObject] != 'FAIL_METAFILE' and
+           not BuildTask._ErrorFlag.isSet()
+           ):
+            GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch][self.BuildItem.BuildObject] = 'SUCCESS'
+
         # indicate there's a thread is available for another build task
         BuildTask._RunningQueueLock.acquire()
         BuildTask._RunningQueue.pop(self.BuildItem)
@@ -1154,27 +1162,30 @@ class Build():
     #
     #
     def invalidateHash(self):
-        # GlobalData.gModuleBuildTracking contains only modules that cannot be skipped by hash
-        for moduleAutoGenObj in GlobalData.gModuleBuildTracking.keys():
-            # False == FAIL : True == Success
-            # Skip invalidating for Successful module builds
-            if GlobalData.gModuleBuildTracking[moduleAutoGenObj] == True:
-                continue
+        # Only for hashing feature
+        if not GlobalData.gUseHashCache:
+            return
+
+        # GlobalData.gModuleBuildTracking contains only modules or libs that cannot be skipped by hash
+        for moduleAutoGenObjArch in GlobalData.gModuleBuildTracking.keys():
+            for moduleAutoGenObj in GlobalData.gModuleBuildTracking[moduleAutoGenObjArch].keys():
+                # Skip invalidating for Successful Module/Lib builds
+                if GlobalData.gModuleBuildTracking[moduleAutoGenObjArch][moduleAutoGenObj] == 'SUCCESS':
+                    continue
 
-            # The module failed to build or failed to start building, from this point on
+                # The module failed to build, failed to start building, 
+ or failed the header check test from this point on
 
-            # Remove .hash from build
-            if GlobalData.gUseHashCache:
-                ModuleHashFile = path.join(moduleAutoGenObj.BuildDir, moduleAutoGenObj.Name + ".hash")
+                # Remove .hash from build
+                ModuleHashFile = 
+ os.path.join(moduleAutoGenObj.BuildDir, moduleAutoGenObj.Name + 
+ ".hash")
                 if os.path.exists(ModuleHashFile):
                     os.remove(ModuleHashFile)
 
-            # Remove .hash file from cache
-            if GlobalData.gBinCacheDest:
-                FileDir = path.join(GlobalData.gBinCacheDest, moduleAutoGenObj.Arch, moduleAutoGenObj.SourceDir, moduleAutoGenObj.MetaFile.BaseName)
-                HashFile = path.join(FileDir, moduleAutoGenObj.Name + '.hash')
-                if os.path.exists(HashFile):
-                    os.remove(HashFile)
+                # Remove .hash file from cache
+                if GlobalData.gBinCacheDest:
+                    FileDir = os.path.join(GlobalData.gBinCacheDest, moduleAutoGenObj.Arch, moduleAutoGenObj.SourceDir, moduleAutoGenObj.MetaFile.BaseName)
+                    HashFile = os.path.join(FileDir, moduleAutoGenObj.Name + '.hash')
+                    if os.path.exists(HashFile):
+                        os.remove(HashFile)
 
     ## Build a module or platform
     #
@@ -1825,9 +1836,11 @@ class Build():
                                 if self.Target == "genmake":
                                     return True
                             self.BuildModules.append(Ma)
-                            # Initialize all modules in tracking to False (FAIL)
-                            if Ma not in GlobalData.gModuleBuildTracking:
-                                GlobalData.gModuleBuildTracking[Ma] = False
+                            # Initialize all modules in tracking to 'FAIL'
+                            if Ma.Arch not in GlobalData.gModuleBuildTracking:
+                                GlobalData.gModuleBuildTracking[Ma.Arch] = dict()
+                            if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]:
+                                GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL'
                     self.AutoGenTime += int(round((time.time() - AutoGenStart)))
                     MakeStart = time.time()
                     for Ma in self.BuildModules:
@@ -1911,6 +1924,7 @@ class Build():
                     # Save MAP buffer into MAP file.
                     #
                     self._SaveMapFile (MapBuffer, Wa)
+        self.invalidateHash()
 
     def _GenFfsCmd(self,ArchList):
         # convert dictionary of Cmd:(Inf,Arch) @@ -2009,9 +2023,11 @@ class Build():
                             if self.Target == "genmake":
                                 continue
                         self.BuildModules.append(Ma)
-                        # Initialize all modules in tracking to False (FAIL)
-                        if Ma not in GlobalData.gModuleBuildTracking:
-                            GlobalData.gModuleBuildTracking[Ma] = False
+                        # Initialize all modules in tracking to 'FAIL'
+                        if Ma.Arch not in GlobalData.gModuleBuildTracking:
+                            GlobalData.gModuleBuildTracking[Ma.Arch] = dict()
+                        if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]:
+                            GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL'
                     self.Progress.Stop("done!")
                     self.AutoGenTime += int(round((time.time() - AutoGenStart)))
                     MakeStart = time.time() @@ -2099,6 +2115,7 @@ class Build():
                     # Save MAP buffer into MAP file.
                     #
                     self._SaveMapFile(MapBuffer, Wa)
+        self.invalidateHash()
 
     ## Generate GuidedSectionTools.txt in the FV directories.
     #
--
2.21.0.windows.1