From mboxrd@z Thu Jan 1 00:00:00 1970 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 192.55.52.151, mailfrom: bob.c.feng@intel.com) Received: from mga17.intel.com (mga17.intel.com [192.55.52.151]) by groups.io with SMTP; Wed, 10 Apr 2019 18:39:10 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga003.fm.intel.com ([10.253.24.29]) by fmsmga107.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 10 Apr 2019 18:39:09 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,335,1549958400"; d="scan'208";a="148214056" Received: from fmsmsx104.amr.corp.intel.com ([10.18.124.202]) by FMSMGA003.fm.intel.com with ESMTP; 10 Apr 2019 18:39:10 -0700 Received: from fmsmsx113.amr.corp.intel.com (10.18.116.7) by fmsmsx104.amr.corp.intel.com (10.18.124.202) with Microsoft SMTP Server (TLS) id 14.3.408.0; Wed, 10 Apr 2019 18:39:10 -0700 Received: from shsmsx108.ccr.corp.intel.com (10.239.4.97) by FMSMSX113.amr.corp.intel.com (10.18.116.7) with Microsoft SMTP Server (TLS) id 14.3.408.0; Wed, 10 Apr 2019 18:39:09 -0700 Received: from shsmsx101.ccr.corp.intel.com ([169.254.1.164]) by SHSMSX108.ccr.corp.intel.com ([169.254.8.147]) with mapi id 14.03.0415.000; Thu, 11 Apr 2019 09:39:08 +0800 From: "BobCF" To: "Rodriguez, Christian" , "devel@edk2.groups.io" CC: "Gao, Liming" , "Zhu, Yonghong" Subject: Re: [PATCH] BaseTools: Hash false success with back to back builds Thread-Topic: [PATCH] BaseTools: Hash false success with back to back builds Thread-Index: AQHU77Hkadqc0aBUK02lINHRX4UVg6Y2KWKw Date: Thu, 11 Apr 2019 01:39:07 +0000 Message-ID: <08650203BA1BD64D8AD9B6D5D74A85D1600DD8C9@SHSMSX101.ccr.corp.intel.com> References: <20190410152705.11680-1-christian.rodriguez@intel.com> In-Reply-To: <20190410152705.11680-1-christian.rodriguez@intel.com> Accept-Language: zh-CN, en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-originating-ip: [10.239.127.40] MIME-Version: 1.0 Return-Path: bob.c.feng@intel.com Content-Language: en-US Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable The patch looks good for me. Reviewed-by: Bob Feng -Bob -----Original Message----- From: Rodriguez, Christian=20 Sent: Wednesday, April 10, 2019 11:27 PM To: devel@edk2.groups.io Cc: Feng, Bob C ; Gao, Liming ;= Zhu, Yonghong Subject: [PATCH] BaseTools: Hash false success with back to back builds BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=3D1692 Add error handling to the --hash feature so that hash files are invalidated= when a build error occurs. Signed-off-by: Christian Rodriguez Cc: Bob Feng Cc: Liming Gao Cc: Yonghong Zhu --- BaseTools/Source/Python/Common/GlobalData.py | 4 ++++ BaseTools/Source/Python/build/build.py | 45 ++++++++++++++++++++++++= +++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/BaseTools/Source/Python/Common/GlobalData.py b/BaseTools/Sourc= e/Python/Common/GlobalData.py index 1853f1d2f6..79f23c892d 100644 --- a/BaseTools/Source/Python/Common/GlobalData.py +++ b/BaseTools/Source/Python/Common/GlobalData.py @@ -108,3 +108,7 @@ gPackageHash =3D {} gModuleHash =3D {} gEnableGenfdsMultiThread =3D False gSikpAutoGenCache =3D set() + +# Dictionary for tracking Module build status as success or failure #=20 +False -> Fail : True -> Success gModuleBuildTracking =3D dict() diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Pyth= on/build/build.py index af8bba47b1..71478b7268 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -620,6 +620,8 @@ class BuildTask: BuildTask._ErrorFlag.set() BuildTask._ErrorMessage =3D "%s broken\n %s [%s]" % \ (threading.currentThread().getName()= , Command, WorkingDir) + if self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking a= nd not BuildTask._ErrorFlag.isSet(): + GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject]=20 + =3D True # indicate there's a thread is available for another build task BuildTask._RunningQueueLock.acquire() BuildTask._RunningQueue.pop(self.BuildItem) @@ -1138,6 +1140,37 @@ class Build(): if Process.returncode !=3D 0 : EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild p= rocess is not success!') EdkLogger.info("\n- Postbuild Done -\n") + + ## Error handling for hash feature + # + # On BuildTask error, iterate through the Module Build tracking + # dictionary to determine wheather a module failed to build. Invalidat= e + # the hash associated with that module by removing it from storage. + # + # + def invalidateHash(self): + # GlobalData.gModuleBuildTracking contains only modules that canno= t be skipped by hash + for moduleAutoGenObj in GlobalData.gModuleBuildTracking.keys(): + # False =3D=3D FAIL : True =3D=3D Success + # Skip invalidating for Successful module builds + if GlobalData.gModuleBuildTracking[moduleAutoGenObj] =3D=3D Tr= ue: + continue + + # The module failed to build or failed to start building,=20 + from this point on + + # Remove .hash from build + if GlobalData.gUseHashCache: + ModuleHashFile =3D path.join(moduleAutoGenObj.BuildDir, mo= duleAutoGenObj.Name + ".hash") + if os.path.exists(ModuleHashFile): + os.remove(ModuleHashFile) + + # Remove .hash file from cache + if GlobalData.gBinCacheSource: + FileDir =3D path.join(GlobalData.gBinCacheSource, moduleAu= toGenObj.Arch, moduleAutoGenObj.SourceDir, moduleAutoGenObj.MetaFile.BaseNa= me) + HashFile =3D path.join(FileDir, moduleAutoGenObj.Name + '.= hash') + if os.path.exists(HashFile): + os.remove(HashFile) + ## Build a module or platform # # Create autogen code and makefile for a module or platform, and the l= aunch @@ -1795,6 +1828,9 @@ class Build(): if self.Target =3D=3D "genmake": return True self.BuildModules.append(Ma) + # Initialize all modules in tracking to False = (FAIL) + if Ma not in GlobalData.gModuleBuildTracking: + GlobalData.gModuleBuildTracking[Ma] =3D=20 + False self.AutoGenTime +=3D int(round((time.time() - AutoGen= Start))) MakeStart =3D time.time() for Ma in self.BuildModules: @@ -1805,6 +1841,7 @@ class Build(): # we need a full version of makefile for platf= orm ExitFlag.set() BuildTask.WaitForComplete() + self.invalidateHash() Pa.CreateMakeFile(False) EdkLogger.error("build", BUILD_ERROR, "Failed = to build module", ExtraData=3DGlobalData.gBuildingModule) # Start task scheduler @@ -1814,6 +1851,7 @@ class= Build(): # in case there's an interruption. we need a full vers= ion of makefile for platform Pa.CreateMakeFile(False) if BuildTask.HasError(): + self.invalidateHash() EdkLogger.error("build", BUILD_ERROR, "Failed to b= uild module", ExtraData=3DGlobalData.gBuildingModule) self.MakeTime +=3D int(round((time.time() - MakeStart)= )) =20 @@ -1823,6 +1861,7 @@ class Build(): self.CreateAsBuiltInf() self.MakeTime +=3D int(round((time.time() - MakeContiue))) if BuildTask.HasError(): + self.invalidateHash() EdkLogger.error("build", BUILD_ERROR, "Failed to build= module", ExtraData=3DGlobalData.gBuildingModule) =20 self.BuildReport.AddPlatformReport(Wa, MaList) @@ -1973,6 = +2012,9 @@ class Build(): if self.Target =3D=3D "genmake": continue self.BuildModules.append(Ma) + # Initialize all modules in tracking to False (FAI= L) + if Ma not in GlobalData.gModuleBuildTracking: + GlobalData.gModuleBuildTracking[Ma] =3D False self.Progress.Stop("done!") self.AutoGenTime +=3D int(round((time.time() - AutoGen= Start))) MakeStart =3D time.time() @@ -1985,6 +2027,7 @@ class = Build(): # we need a full version of makefile for platf= orm ExitFlag.set() BuildTask.WaitForComplete() + self.invalidateHash() Pa.CreateMakeFile(False) EdkLogger.error("build", BUILD_ERROR, "Failed = to build module", ExtraData=3DGlobalData.gBuildingModule) # Start task scheduler @@ -1994,6 +2037,7 @@ class= Build(): # in case there's an interruption. we need a full vers= ion of makefile for platform Pa.CreateMakeFile(False) if BuildTask.HasError(): + self.invalidateHash() EdkLogger.error("build", BUILD_ERROR, "Failed to b= uild module", ExtraData=3DGlobalData.gBuildingModule) self.MakeTime +=3D int(round((time.time() - MakeStart)= )) =20 @@ -2013,6 +2057,7 @@ class Build(): # has been signaled. # if BuildTask.HasError(): + self.invalidateHash() EdkLogger.error("build", BUILD_ERROR, "Failed to build= module", ExtraData=3DGlobalData.gBuildingModule) =20 # Create MAP file when Load Fix Address is enabled. -- 2.19.1.windows.1