From mboxrd@z Thu Jan 1 00:00:00 1970 Authentication-Results: mx.groups.io; dkim=missing; spf=fail (domain: intel.com, ip: , mailfrom: steven.shi@intel.com) Received: from mga05.intel.com (mga05.intel.com []) by groups.io with SMTP; Mon, 12 Aug 2019 20:47:05 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga003.jf.intel.com ([10.7.209.27]) by fmsmga105.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 12 Aug 2019 20:47:05 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.64,380,1559545200"; d="scan'208";a="178549507" Received: from jshi19-mobl.ccr.corp.intel.com ([10.254.211.59]) by orsmga003.jf.intel.com with ESMTP; 12 Aug 2019 20:47:03 -0700 From: "Steven Shi" To: devel@edk2.groups.io Cc: liming.gao@intel.com, bob.c.feng@intel.com, christian.rodriguez@intel.com, michael.johnson@intel.com, "Shi, Steven" Subject: [PATCH v2 2/4] BaseTools: Print first cache missing file for build cachle Date: Tue, 13 Aug 2019 11:46:52 +0800 Message-Id: <20190813034654.25716-3-steven.shi@intel.com> X-Mailer: git-send-email 2.17.1.windows.2 In-Reply-To: <20190813034654.25716-1-steven.shi@intel.com> References: <20190813034654.25716-1-steven.shi@intel.com> From: "Shi, Steven" BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1925 When a module build cache miss, add support to print the first cache missing file path and name. Cc: Liming Gao Cc: Bob Feng Signed-off-by: Steven Shi --- BaseTools/Source/Python/AutoGen/AutoGenWorker.py | 2 ++ BaseTools/Source/Python/AutoGen/ModuleAutoGen.py | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py index a84ed46f2e..30d2f96fc7 100755 --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -246,6 +246,8 @@ class AutoGenWorkerInProcess(mp.Process): Ma.GenMakeHash(GlobalData.gCacheIR) if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR): continue + else: + Ma.PrintFirstMakeCacheMissFile(GlobalData.gCacheIR) except Empty: pass except: diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py index 6cb7cae1c7..a73a8a53a0 100755 --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py @@ -2365,6 +2365,82 @@ class ModuleAutoGen(AutoGen): print("[cache hit]: checkpoint_Makefile:", self.MetaFile.Path, self.Arch) return True + ## Show the first file name which causes cache miss + def PrintFirstMakeCacheMissFile(self, gDict): + if not GlobalData.gBinCacheSource: + return + + # skip binary module + if self.IsBinaryModule: + return + + if not (self.MetaFile.Path, self.Arch) in gDict: + return + + # Only print cache miss file for the MakeCache not hit module + if gDict[(self.MetaFile.Path, self.Arch)].MakeCacheHit: + return + + if not gDict[(self.MetaFile.Path, self.Arch)].MakeHashChain: + EdkLogger.quiet("[cache insight]: MakeHashChain is missing for: %s[%s]" % (self.MetaFile.Path, self.Arch)) + return + + # Find the cache dir name through the .ModuleHashPair file info + FileDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.SourceDir, self.MetaFile.BaseName) + + ModuleHashPairList = [] # tuple list: [tuple(PreMakefileHash, MakeHash)] + ModuleHashPair = path.join(FileDir, self.Name + ".ModuleHashPair") + if not os.path.exists(ModuleHashPair): + EdkLogger.quiet("[cache insight]: Cannot find ModuleHashPair file for module: %s[%s]" % (self.MetaFile.Path, self.Arch)) + return + + try: + f = open(ModuleHashPair, 'r') + ModuleHashPairList = json.load(f) + f.close() + except: + EdkLogger.quiet("[cache insight]: Cannot load ModuleHashPair file for module: %s[%s]" % (self.MetaFile.Path, self.Arch)) + return + + MakeHashSet = set() + for idx, (PreMakefileHash, MakeHash) in enumerate (ModuleHashPairList): + TargetHashDir = path.join(FileDir, str(MakeHash)) + if os.path.exists(TargetHashDir): + MakeHashSet.add(MakeHash) + if not MakeHashSet: + EdkLogger.quiet("[cache insight]: Cannot find valid cache dir for module: %s[%s]" % (self.MetaFile.Path, self.Arch)) + return + + TargetHash = list(MakeHashSet)[0] + TargetHashDir = path.join(FileDir, str(TargetHash)) + if len(MakeHashSet) > 1 : + EdkLogger.quiet("[cache insight]: found multiple cache dirs for this module, random select dir '%s' to search the first cache miss file: %s[%s]" % (TargetHash, self.MetaFile.Path, self.Arch)) + + ListFile = path.join(TargetHashDir, self.Name + '.MakeHashChain') + if os.path.exists(ListFile): + try: + f = open(ListFile, 'r') + CachedList = json.load(f) + f.close() + except: + EdkLogger.quiet("[cache insight]: Cannot load MakeHashChain file: %s" % ListFile) + return + else: + EdkLogger.quiet("[cache insight]: Cannot find MakeHashChain file: %s" % ListFile) + return + + CurrentList = gDict[(self.MetaFile.Path, self.Arch)].MakeHashChain + for idx, (file, hash) in enumerate (CurrentList): + (filecached, hashcached) = CachedList[idx] + if file != filecached: + EdkLogger.quiet("[cache insight]: first different file in %s[%s] is %s, the cached one is %s" % (self.MetaFile.Path, self.Arch, file, filecached)) + break + if hash != hashcached: + EdkLogger.quiet("[cache insight]: first cache miss file in %s[%s] is %s" % (self.MetaFile.Path, self.Arch, file)) + break + + return True + ## Decide whether we can skip the ModuleAutoGen process def CanSkipbyCache(self, gDict): # Hashing feature is off -- 2.17.1