From mboxrd@z Thu Jan 1 00:00:00 1970 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 134.134.136.20, mailfrom: steven.shi@intel.com) Received: from mga02.intel.com (mga02.intel.com [134.134.136.20]) by groups.io with SMTP; Mon, 24 Jun 2019 00:20:48 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga006.fm.intel.com ([10.253.24.20]) by orsmga101.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 24 Jun 2019 00:19:35 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.63,411,1557212400"; d="scan'208";a="359924140" Received: from fmsmsx105.amr.corp.intel.com ([10.18.124.203]) by fmsmga006.fm.intel.com with ESMTP; 24 Jun 2019 00:19:34 -0700 Received: from fmsmsx116.amr.corp.intel.com (10.18.116.20) by FMSMSX105.amr.corp.intel.com (10.18.124.203) with Microsoft SMTP Server (TLS) id 14.3.439.0; Mon, 24 Jun 2019 00:19:34 -0700 Received: from shsmsx105.ccr.corp.intel.com (10.239.4.158) by fmsmsx116.amr.corp.intel.com (10.18.116.20) with Microsoft SMTP Server (TLS) id 14.3.439.0; Mon, 24 Jun 2019 00:19:33 -0700 Received: from shsmsx102.ccr.corp.intel.com ([169.254.2.33]) by SHSMSX105.ccr.corp.intel.com ([169.254.11.72]) with mapi id 14.03.0439.000; Mon, 24 Jun 2019 15:19:31 +0800 From: "Steven Shi" To: "Feng, Bob C" , "devel@edk2.groups.io" CC: "Gao, Liming" , "Rodriguez, Christian" Subject: Re: [Patch 2/2] BaseTools: Move Build Cache related function out of CreateAsBuiltInf Thread-Topic: [Patch 2/2] BaseTools: Move Build Cache related function out of CreateAsBuiltInf Thread-Index: AQHVKi9kmdlcxYMkdU6r/Jgd/g+TAaaqZeZQ Date: Mon, 24 Jun 2019 07:19:31 +0000 Message-ID: <06C8AB66E78EE34A949939824ABE2B314002D20F@shsmsx102.ccr.corp.intel.com> References: <20190624015144.31904-1-bob.c.feng@intel.com> In-Reply-To: <20190624015144.31904-1-bob.c.feng@intel.com> Accept-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-titus-metadata-40: eyJDYXRlZ29yeUxhYmVscyI6IiIsIk1ldGFkYXRhIjp7Im5zIjoiaHR0cDpcL1wvd3d3LnRpdHVzLmNvbVwvbnNcL0ludGVsMyIsImlkIjoiYzkwNDIxZDktNmFkYi00YzdkLTg0ODgtNzU3YTAwYzg1YjZjIiwicHJvcHMiOlt7Im4iOiJDVFBDbGFzc2lmaWNhdGlvbiIsInZhbHMiOlt7InZhbHVlIjoiQ1RQX05UIn1dfV19LCJTdWJqZWN0TGFiZWxzIjpbXSwiVE1DVmVyc2lvbiI6IjE3LjEwLjE4MDQuNDkiLCJUcnVzdGVkTGFiZWxIYXNoIjoiT0dna1wvNzNwd2lQZ2g2aldQK1BET0dta3huNDFcL0toQ0l2UzkrVlM0NWZrWFFVVTVMMXY4dTRxam43QlpETFk3In0= x-ctpclassification: CTP_NT dlp-product: dlpe-windows dlp-version: 11.2.0.6 dlp-reaction: no-action x-originating-ip: [10.239.127.40] MIME-Version: 1.0 Return-Path: steven.shi@intel.com Content-Language: en-US Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable Reviewed-by: Steven Shi > -----Original Message----- > From: Feng, Bob C > Sent: Monday, June 24, 2019 9:52 AM > To: devel@edk2.groups.io > Cc: Feng, Bob C ; Shi, Steven ; > Gao, Liming ; Rodriguez, Christian > > Subject: [Patch 2/2] BaseTools: Move Build Cache related function out of > CreateAsBuiltInf >=20 > BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=3D1932 > There are two functions in current CreateAsBuiltInf, Copy Binary files > to build cache folder and create asbuild inf file. >=20 > This patch is to separate UpdateBuildCache and CreateAsBuiltInf into > two functions. >=20 > Signed-off-by: Bob Feng > Cc: Steven Shi > Cc: Liming Gao > Cc: Christian Rodriguez > --- > BaseTools/Source/Python/AutoGen/AutoGen.py | 68 > +++++++++++++++++++++++++++++++++++++------------------------------- > BaseTools/Source/Python/build/build.py | 28 ++++++++++++++++++++++--= -- > -- > 2 files changed, 59 insertions(+), 37 deletions(-) >=20 > diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py > b/BaseTools/Source/Python/AutoGen/AutoGen.py > index 45b81ed660..e8e09dc8a3 100644 > --- a/BaseTools/Source/Python/AutoGen/AutoGen.py > +++ b/BaseTools/Source/Python/AutoGen/AutoGen.py > @@ -3577,23 +3577,41 @@ class ModuleAutoGen(AutoGen): >=20 > fStringIO.close () > fInputfile.close () > return OutputName >=20 > + @cached_property > + def OutputFile(self): > + retVal =3D set() > + OutputDir =3D self.OutputDir.replace('\\', '/').strip('/') > + DebugDir =3D self.DebugDir.replace('\\', '/').strip('/') > + for Item in self.CodaTargetList: > + File =3D Item.Target.Path.replace('\\', '/').strip('/').repl= ace(DebugDir, > '').replace(OutputDir, '').strip('/') > + retVal.add(File) > + if self.DepexGenerated: > + retVal.add(self.Name + '.depex') > + > + Bin =3D self._GenOffsetBin() > + if Bin: > + retVal.add(Bin) > + > + for Root, Dirs, Files in os.walk(OutputDir): > + for File in Files: > + if File.lower().endswith('.pdb'): > + retVal.add(File) > + > + return retVal > + > ## Create AsBuilt INF file the module > # > def CreateAsBuiltInf(self): > - self.OutputFile =3D set() >=20 > if self.IsAsBuiltInfCreated: > return >=20 > # Skip INF file generation for libraries > if self.IsLibrary: > - # Only store the library cache if needed > - if GlobalData.gBinCacheDest: > - self.CopyModuleToCache() > return >=20 > # Skip the following code for modules with no source files > if not self.SourceFileList: > return > @@ -3710,11 +3728,10 @@ class ModuleAutoGen(AutoGen): >=20 > OutputDir =3D self.OutputDir.replace('\\', '/').strip('/') > DebugDir =3D self.DebugDir.replace('\\', '/').strip('/') > for Item in self.CodaTargetList: > File =3D Item.Target.Path.replace('\\', '/').strip('/').repl= ace(DebugDir, > '').replace(OutputDir, '').strip('/') > - self.OutputFile.add(File) > if os.path.isabs(File): > File =3D File.replace('\\', '/').strip('/').replace(Outp= utDir, '').strip('/') > if Item.Target.Ext.lower() =3D=3D '.aml': > AsBuiltInfDict['binary_item'].append('ASL|' + File) > elif Item.Target.Ext.lower() =3D=3D '.acpi': > @@ -3726,28 +3743,25 @@ class ModuleAutoGen(AutoGen): > if not self.DepexGenerated: > DepexFile =3D os.path.join(self.OutputDir, self.Name + '.dep= ex') > if os.path.exists(DepexFile): > self.DepexGenerated =3D True > if self.DepexGenerated: > - self.OutputFile.add(self.Name + '.depex') > if self.ModuleType in [SUP_MODULE_PEIM]: > AsBuiltInfDict['binary_item'].append('PEI_DEPEX|' + self= .Name + > '.depex') > elif self.ModuleType in [SUP_MODULE_DXE_DRIVER, > SUP_MODULE_DXE_RUNTIME_DRIVER, SUP_MODULE_DXE_SAL_DRIVER, > SUP_MODULE_UEFI_DRIVER]: > AsBuiltInfDict['binary_item'].append('DXE_DEPEX|' + self= .Name + > '.depex') > elif self.ModuleType in [SUP_MODULE_DXE_SMM_DRIVER]: > AsBuiltInfDict['binary_item'].append('SMM_DEPEX|' + self= .Name + > '.depex') >=20 > Bin =3D self._GenOffsetBin() > if Bin: > AsBuiltInfDict['binary_item'].append('BIN|%s' % Bin) > - self.OutputFile.add(Bin) >=20 > for Root, Dirs, Files in os.walk(OutputDir): > for File in Files: > if File.lower().endswith('.pdb'): > AsBuiltInfDict['binary_item'].append('DISPOSABLE|' += File) > - self.OutputFile.add(File) > HeaderComments =3D self.Module.HeaderComments > StartPos =3D 0 > for Index in range(len(HeaderComments)): > if HeaderComments[Index].find('@BinaryHeader') !=3D -1: > HeaderComments[Index] =3D > HeaderComments[Index].replace('@BinaryHeader', '@file') > @@ -3912,43 +3926,35 @@ class ModuleAutoGen(AutoGen): > AsBuiltInf.Append(gAsBuiltInfHeaderString.Replace(AsBuiltInfDict= )) >=20 > SaveFileOnChange(os.path.join(self.OutputDir, self.Name + '.inf'= ), > str(AsBuiltInf), False) >=20 > self.IsAsBuiltInfCreated =3D True > - if GlobalData.gBinCacheDest: > - self.CopyModuleToCache() >=20 > def CopyModuleToCache(self): > FileDir =3D path.join(GlobalData.gBinCacheDest, > self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, sel= f.Arch, > self.SourceDir, self.MetaFile.BaseName) > CreateDirectory (FileDir) > HashFile =3D path.join(self.BuildDir, self.Name + '.hash') > if os.path.exists(HashFile): > CopyFileOnChange(HashFile, FileDir) > - if not self.IsLibrary: > - ModuleFile =3D path.join(self.OutputDir, self.Name + '.inf') > - if os.path.exists(ModuleFile): > - CopyFileOnChange(ModuleFile, FileDir) > - else: > - OutputDir =3D self.OutputDir.replace('\\', '/').strip('/') > - DebugDir =3D self.DebugDir.replace('\\', '/').strip('/') > - for Item in self.CodaTargetList: > - File =3D Item.Target.Path.replace('\\', '/').strip('/').= replace(DebugDir, > '').replace(OutputDir, '').strip('/') > - self.OutputFile.add(File) > + ModuleFile =3D path.join(self.OutputDir, self.Name + '.inf') > + if os.path.exists(ModuleFile): > + CopyFileOnChange(ModuleFile, FileDir) > + > if not self.OutputFile: > Ma =3D self.BuildDatabase[self.MetaFile, self.Arch, self.Bui= ldTarget, > self.ToolChain] > self.OutputFile =3D Ma.Binaries > - if self.OutputFile: > - for File in self.OutputFile: > - File =3D str(File) > - if not os.path.isabs(File): > - File =3D os.path.join(self.OutputDir, File) > - if os.path.exists(File): > - sub_dir =3D os.path.relpath(File, self.OutputDir) > - destination_file =3D os.path.join(FileDir, sub_dir) > - destination_dir =3D os.path.dirname(destination_file= ) > - CreateDirectory(destination_dir) > - CopyFileOnChange(File, destination_dir) > + > + for File in self.OutputFile: > + File =3D str(File) > + if not os.path.isabs(File): > + File =3D os.path.join(self.OutputDir, File) > + if os.path.exists(File): > + sub_dir =3D os.path.relpath(File, self.OutputDir) > + destination_file =3D os.path.join(FileDir, sub_dir) > + destination_dir =3D os.path.dirname(destination_file) > + CreateDirectory(destination_dir) > + CopyFileOnChange(File, destination_dir) >=20 > def AttemptModuleCacheCopy(self): > # If library or Module is binary do not skip by hash > if self.IsBinaryModule: > return False > diff --git a/BaseTools/Source/Python/build/build.py > b/BaseTools/Source/Python/build/build.py > index 3ece4d4c61..8c3315619a 100644 > --- a/BaseTools/Source/Python/build/build.py > +++ b/BaseTools/Source/Python/build/build.py > @@ -1248,10 +1248,13 @@ class Build(): > # build modules > if BuildModule: > BuildCommand =3D BuildCommand + [Target] > LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir) > self.CreateAsBuiltInf() > + if GlobalData.gBinCacheDest: > + self.UpdateBuildCache() > + self.BuildModules =3D [] > return True >=20 > # build library > if Target =3D=3D 'libraries': > for Lib in AutoGenObject.LibraryBuildDirectoryList: > @@ -1266,10 +1269,13 @@ class Build(): > LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir= ) > for Mod in AutoGenObject.ModuleBuildDirectoryList: > NewBuildCommand =3D BuildCommand + ['-f', > os.path.normpath(os.path.join(Mod, makefile)), 'pbuild'] > LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir= ) > self.CreateAsBuiltInf() > + if GlobalData.gBinCacheDest: > + self.UpdateBuildCache() > + self.BuildModules =3D [] > return True >=20 > # cleanlib > if Target =3D=3D 'cleanlib': > for Lib in AutoGenObject.LibraryBuildDirectoryList: > @@ -1359,10 +1365,13 @@ class Build(): > if BuildModule: > if Target !=3D 'fds': > BuildCommand =3D BuildCommand + [Target] > AutoGenObject.BuildTime =3D LaunchCommand(BuildCommand, > AutoGenObject.MakeFileDir) > self.CreateAsBuiltInf() > + if GlobalData.gBinCacheDest: > + self.UpdateBuildCache() > + self.BuildModules =3D [] > return True >=20 > # genfds > if Target =3D=3D 'fds': > if GenFdsApi(AutoGenObject.GenFdsCommandDict, self.Db): > @@ -1872,10 +1881,13 @@ class Build(): >=20 > MakeContiue =3D time.time() > ExitFlag.set() > BuildTask.WaitForComplete() > self.CreateAsBuiltInf() > + if GlobalData.gBinCacheDest: > + self.UpdateBuildCache() > + self.BuildModules =3D [] > self.MakeTime +=3D int(round((time.time() - MakeContiue)= )) > if BuildTask.HasError(): > self.invalidateHash() > EdkLogger.error("build", BUILD_ERROR, "Failed to bui= ld module", > ExtraData=3DGlobalData.gBuildingModule) >=20 > @@ -2072,10 +2084,13 @@ class Build(): > # to exit if all tasks are completed > # > ExitFlag.set() > BuildTask.WaitForComplete() > self.CreateAsBuiltInf() > + if GlobalData.gBinCacheDest: > + self.UpdateBuildCache() > + self.BuildModules =3D [] > self.MakeTime +=3D int(round((time.time() - MakeContiue)= )) > # > # Check for build error, and raise exception if one > # has been signaled. > # > @@ -2211,28 +2226,29 @@ class Build(): >=20 > if self.Target =3D=3D 'cleanall': > RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), T= rue) >=20 > def CreateAsBuiltInf(self): > + for Module in self.BuildModules: > + Module.CreateAsBuiltInf() > + > + def UpdateBuildCache(self): > all_lib_set =3D set() > all_mod_set =3D set() > for Module in self.BuildModules: > - Module.CreateAsBuiltInf() > + Module.CopyModuleToCache() > all_mod_set.add(Module) > for Module in self.HashSkipModules: > - if GlobalData.gBinCacheDest: > - Module.CopyModuleToCache() > + Module.CopyModuleToCache() > all_mod_set.add(Module) > for Module in all_mod_set: > for lib in Module.LibraryAutoGenList: > all_lib_set.add(lib) > for lib in all_lib_set: > - if GlobalData.gBinCacheDest: > - lib.CopyModuleToCache() > + lib.CopyModuleToCache() > all_lib_set.clear() > all_mod_set.clear() > - self.BuildModules =3D [] > self.HashSkipModules =3D [] > ## Do some clean-up works when error occurred > def Relinquish(self): > OldLogLevel =3D EdkLogger.GetLevel() > EdkLogger.SetLevel(EdkLogger.ERROR) > -- > 2.20.1.windows.1