From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=192.55.52.136; helo=mga12.intel.com; envelope-from=bob.c.feng@intel.com; receiver=edk2-devel@lists.01.org Received: from mga12.intel.com (mga12.intel.com [192.55.52.136]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ml01.01.org (Postfix) with ESMTPS id E12E3209574D3 for ; Tue, 27 Feb 2018 21:53:24 -0800 (PST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga005.jf.intel.com ([10.7.209.41]) by fmsmga106.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 27 Feb 2018 21:59:31 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.47,404,1515484800"; d="scan'208";a="204307339" Received: from shwdeopenpsi105.ccr.corp.intel.com ([10.239.9.129]) by orsmga005.jf.intel.com with ESMTP; 27 Feb 2018 21:59:30 -0800 From: BobCF To: edk2-devel@lists.01.org Cc: Bob Feng , Liming Gao Date: Wed, 28 Feb 2018 13:59:19 +0800 Message-Id: <20180228055921.35432-2-bob.c.feng@intel.com> X-Mailer: git-send-email 2.14.3.windows.1 In-Reply-To: <20180228055921.35432-1-bob.c.feng@intel.com> References: <20180228055921.35432-1-bob.c.feng@intel.com> Subject: [Patch 2/4] BaseTools: Improve build performance X-BeenThere: edk2-devel@lists.01.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: EDK II Development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 28 Feb 2018 05:53:25 -0000 Add cache for building PcdValueInit.c. Contributed-under: TianoCore Contribution Agreement 1.1 Signed-off-by: Bob Feng Cc: Liming Gao --- BaseTools/Source/Python/GenFds/GenFds.py | 1 + BaseTools/Source/Python/Workspace/DecBuildData.py | 12 +- BaseTools/Source/Python/Workspace/DscBuildData.py | 172 +++++++++++----------- 3 files changed, 101 insertions(+), 84 deletions(-) diff --git a/BaseTools/Source/Python/GenFds/GenFds.py b/BaseTools/Source/Python/GenFds/GenFds.py index dcba9f24cb..cd705630a3 100644 --- a/BaseTools/Source/Python/GenFds/GenFds.py +++ b/BaseTools/Source/Python/GenFds/GenFds.py @@ -324,10 +324,11 @@ def main(): EdkLogger.error("GenFds", FORMAT_INVALID, "The FV %s's region is specified in multiple FD with different value." %FvObj.UiFvName) else: FvObj.FvRegionInFD = RegionObj.Size RegionObj.BlockInfoOfRegion(FdObj.BlockSizeList, FvObj) + GlobalData.BuildOptionPcd = Options.OptionPcd if Options.OptionPcd else {} """Call GenFds""" GenFds.GenFd('', FdfParserObj, BuildWorkSpace, ArchList) """Generate GUID cross reference file""" GenFds.GenerateGuidXRefFile(BuildWorkSpace, ArchList, FdfParserObj) diff --git a/BaseTools/Source/Python/Workspace/DecBuildData.py b/BaseTools/Source/Python/Workspace/DecBuildData.py index 61f15086d0..18101a0add 100644 --- a/BaseTools/Source/Python/Workspace/DecBuildData.py +++ b/BaseTools/Source/Python/Workspace/DecBuildData.py @@ -93,10 +93,11 @@ class DecBuildData(PackageBuildClassObject): self._PkgUniFile = None self._Protocols = None self._Ppis = None self._Guids = None self._Includes = None + self._CommonIncludes = None self._LibraryClasses = None self._Pcds = None self.__Macros = None self._PrivateProtocols = None self._PrivatePpis = None @@ -294,11 +295,12 @@ class DecBuildData(PackageBuildClassObject): self._PrivateGuids[Name] = PrivateGuidDict[self._Arch, Name] return self._Guids ## Retrieve public include paths declared in this package def _GetInclude(self): - if self._Includes == None: + if self._Includes == None or self._CommonIncludes is None: + self._CommonIncludes = [] self._Includes = [] self._PrivateIncludes = [] PublicInclues = [] RecordList = self._RawData[MODEL_EFI_INCLUDE, self._Arch] Macros = self._Macros @@ -322,11 +324,12 @@ class DecBuildData(PackageBuildClassObject): else: if File not in PublicInclues: PublicInclues.append(File) if File in self._PrivateIncludes: EdkLogger.error('build', OPTION_CONFLICT, "Can't determine %s's attribute, it is both defined as Private and non-Private attribute in DEC file." % File, File=self.MetaFile, Line=LineNo) - + if Record[3] == "COMMON": + self._CommonIncludes.append(File) return self._Includes ## Retrieve library class declarations (not used in build at present) def _GetLibraryClass(self): if self._LibraryClasses == None: @@ -450,10 +453,15 @@ class DecBuildData(PackageBuildClassObject): StructurePcds = self.ProcessStructurePcd(StrPcdSet) for pcd in StructurePcds: Pcds[pcd.TokenCName, pcd.TokenSpaceGuidCName, self._PCD_TYPE_STRING_[Type]] = pcd return Pcds + @property + def CommonIncludes(self): + if self._CommonIncludes is None: + self.Includes + return self._CommonIncludes _Macros = property(_GetMacros) Arch = property(_GetArch, _SetArch) PackageName = property(_GetPackageName) diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py index 393ad0265f..59bfd9c8e0 100644 --- a/BaseTools/Source/Python/Workspace/DscBuildData.py +++ b/BaseTools/Source/Python/Workspace/DscBuildData.py @@ -35,10 +35,11 @@ from Common.Misc import ProcessDuplicatedInf import re from Common.Parsing import IsValidWord from Common.VariableAttributes import VariableAttributes import Common.GlobalData as GlobalData import subprocess +from Common.Misc import SaveFileOnChange from Workspace.BuildClassObject import PlatformBuildClassObject, StructurePcd, PcdClassObject, ModuleBuildClassObject # # Treat CHAR16 as a synonym for UINT16. CHAR16 support is required for VFR C structs # @@ -87,11 +88,10 @@ LIBS = $(LIB_PATH)\Common.lib !INCLUDE $(BASE_TOOLS_PATH)\Source\C\Makefiles\ms.app ''' PcdGccMakefile = ''' -ARCH ?= IA32 MAKEROOT ?= $(EDK_TOOLS_PATH)/Source/C LIBS = -lCommon ''' class DscBuildData(PlatformBuildClassObject): @@ -154,16 +154,19 @@ class DscBuildData(PlatformBuildClassObject): self._Target = Target self._Toolchain = Toolchain self._ToolChainFamily = None self._Clear() self._HandleOverridePath() - if os.getenv("WORKSPACE"): - self.OutputPath = os.path.join(os.getenv("WORKSPACE"), 'Build', PcdValueInitName) - else: - self.OutputPath = os.path.dirname(self.DscFile) + self.WorkspaceDir = os.getenv("WORKSPACE") if os.getenv("WORKSPACE") else "" self.DefaultStores = None self.SkuIdMgr = SkuClass(self.SkuName, self.SkuIds) + @property + def OutputPath(self): + if os.getenv("WORKSPACE"): + return os.path.join(os.getenv("WORKSPACE"), self.OutputDirectory, self._Target + "_" + self._Toolchain,PcdValueInitName) + else: + return os.path.dirname(self.DscFile) ## XXX[key] = value def __setitem__(self, key, value): self.__dict__[self._PROPERTY_[key]] = value @@ -1297,11 +1300,11 @@ class DscBuildData(PlatformBuildClassObject): str_pcd_obj_str.DefaultFromDSC = {skuname:{defaultstore: str_pcd_obj.SkuInfoList[skuname].DefaultStoreDict.get(defaultstore, str_pcd_obj.SkuInfoList[skuname].HiiDefaultValue) for defaultstore in DefaultStores} for skuname in str_pcd_obj.SkuInfoList} else: str_pcd_obj_str.DefaultFromDSC = {skuname:{defaultstore: str_pcd_obj.SkuInfoList[skuname].DefaultStoreDict.get(defaultstore, str_pcd_obj.SkuInfoList[skuname].DefaultValue) for defaultstore in DefaultStores} for skuname in str_pcd_obj.SkuInfoList} for str_pcd_data in StrPcdSet[str_pcd]: if str_pcd_data[3] in SkuIds: - str_pcd_obj_str.AddOverrideValue(str_pcd_data[2], str(str_pcd_data[6]), 'DEFAULT' if str_pcd_data[3] == 'COMMON' else str_pcd_data[3],'STANDARD' if str_pcd_data[4] == 'COMMON' else str_pcd_data[4], self.MetaFile.File,LineNo=str_pcd_data[5]) + str_pcd_obj_str.AddOverrideValue(str_pcd_data[2], str(str_pcd_data[6]), 'DEFAULT' if str_pcd_data[3] == 'COMMON' else str_pcd_data[3],'STANDARD' if str_pcd_data[4] == 'COMMON' else str_pcd_data[4], self.MetaFile.File if self.WorkspaceDir not in self.MetaFile.File else self.MetaFile.File[len(self.WorkspaceDir) if self.WorkspaceDir.endswith(os.path.sep) else len(self.WorkspaceDir)+1:],LineNo=str_pcd_data[5]) S_pcd_set[str_pcd[1], str_pcd[0]] = str_pcd_obj_str else: EdkLogger.error('build', PARSER_ERROR, "Pcd (%s.%s) defined in DSC is not declared in DEC files. Arch: ['%s']" % (str_pcd[0], str_pcd[1], self._Arch), File=self.MetaFile,Line = StrPcdSet[str_pcd][0][5]) @@ -1806,14 +1809,14 @@ class DscBuildData(PlatformBuildClassObject): for IncludeFile in Pcd.StructuredPcdIncludeFile: if IncludeFile not in Includes: Includes[IncludeFile] = True CApp = CApp + '#include <%s>\n' % (IncludeFile) CApp = CApp + '\n' - for PcdName in StructuredPcds: Pcd = StructuredPcds[PcdName] - if not Pcd.SkuOverrideValues: + if not Pcd.SkuOverrideValues or Pcd.Type in [self._PCD_TYPE_STRING_[MODEL_PCD_FIXED_AT_BUILD], + self._PCD_TYPE_STRING_[MODEL_PCD_PATCHABLE_IN_MODULE]]: InitByteValue, CApp = self.GenerateInitializeFunc(self.SkuIdMgr.SystemSkuId, 'STANDARD', Pcd, InitByteValue, CApp) else: for SkuName in self.SkuIdMgr.SkuOverrideOrder(): if SkuName not in Pcd.SkuOverrideValues: continue @@ -1825,11 +1828,11 @@ class DscBuildData(PlatformBuildClassObject): CApp = CApp + 'PcdEntryPoint(\n' CApp = CApp + ' VOID\n' CApp = CApp + ' )\n' CApp = CApp + '{\n' for Pcd in StructuredPcds.values(): - if not Pcd.SkuOverrideValues: + if not Pcd.SkuOverrideValues or Pcd.Type in [self._PCD_TYPE_STRING_[MODEL_PCD_FIXED_AT_BUILD],self._PCD_TYPE_STRING_[MODEL_PCD_PATCHABLE_IN_MODULE]]: CApp = CApp + ' Initialize_%s_%s_%s_%s();\n' % (self.SkuIdMgr.SystemSkuId, 'STANDARD', Pcd.TokenSpaceGuidCName, Pcd.TokenCName) else: for SkuName in self.SkuIdMgr.SkuOverrideOrder(): if SkuName not in Pcd.SkuOverrideValues: continue @@ -1840,17 +1843,15 @@ class DscBuildData(PlatformBuildClassObject): CApp = CApp + PcdMainCEntry + '\n' if not os.path.exists(self.OutputPath): os.makedirs(self.OutputPath) CAppBaseFileName = os.path.join(self.OutputPath, PcdValueInitName) - File = open (CAppBaseFileName + '.c', 'w') - File.write(CApp) - File.close() + SaveFileOnChange(CAppBaseFileName + '.c', CApp, False) MakeApp = PcdMakefileHeader if sys.platform == "win32": - MakeApp = MakeApp + 'ARCH = IA32\nAPPNAME = %s\n' % (PcdValueInitName) + 'OBJECTS = %s\%s.obj\n' % (self.OutputPath, PcdValueInitName) + 'INC = ' + MakeApp = MakeApp + 'APPNAME = %s\n' % (PcdValueInitName) + 'OBJECTS = %s\%s.obj\n' % (self.OutputPath, PcdValueInitName) + 'INC = ' else: MakeApp = MakeApp + PcdGccMakefile MakeApp = MakeApp + 'APPNAME = %s\n' % (PcdValueInitName) + 'OBJECTS = %s/%s.o\n' % (self.OutputPath, PcdValueInitName) + \ 'include $(MAKEROOT)/Makefiles/app.makefile\n' + 'INCLUDE +=' @@ -1858,11 +1859,11 @@ class DscBuildData(PlatformBuildClassObject): for Cache in self._Bdb._CACHE_.values(): if Cache.MetaFile.Ext.lower() != '.dec': continue if Cache.Includes: if str(Cache.MetaFile.Path) not in PlatformInc: - PlatformInc[str(Cache.MetaFile.Path)] = Cache.Includes + PlatformInc[str(Cache.MetaFile.Path)] = Cache.CommonIncludes PcdDependDEC = [] for Pcd in StructuredPcds.values(): for PackageDec in Pcd.PackageDecs: Package = os.path.normpath(mws.join(GlobalData.gWorkspace, PackageDec)) @@ -1922,94 +1923,101 @@ class DscBuildData(PlatformBuildClassObject): MakeApp += CC_FLAGS if sys.platform == "win32": MakeApp = MakeApp + PcdMakefileEnd MakeFileName = os.path.join(self.OutputPath, 'Makefile') - File = open (MakeFileName, 'w') - File.write(MakeApp) - File.close() + SaveFileOnChange(MakeFileName, MakeApp, False) InputValueFile = os.path.join(self.OutputPath, 'Input.txt') OutputValueFile = os.path.join(self.OutputPath, 'Output.txt') - File = open (InputValueFile, 'w') - File.write(InitByteValue) - File.close() - - Messages = '' - if sys.platform == "win32": - MakeCommand = 'nmake clean & nmake -f %s' % (MakeFileName) - returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand) - Messages = StdOut - else: - MakeCommand = 'make clean & make -f %s' % (MakeFileName) - returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand) - Messages = StdErr - Messages = Messages.split('\n') - MessageGroup = [] - if returncode <>0: - CAppBaseFileName = os.path.join(self.OutputPath, PcdValueInitName) - File = open (CAppBaseFileName + '.c', 'r') - FileData = File.readlines() - File.close() - for Message in Messages: - if " error" in Message or "warning" in Message: - FileInfo = Message.strip().split('(') - if len (FileInfo) > 1: - FileName = FileInfo [0] - FileLine = FileInfo [1].split (')')[0] - else: - FileInfo = Message.strip().split(':') - FileName = FileInfo [0] - FileLine = FileInfo [1] - if FileLine.isdigit(): - error_line = FileData[int (FileLine) - 1] - if r"//" in error_line: - c_line,dsc_line = error_line.split(r"//") - else: - dsc_line = error_line - message_itmes = Message.split(":") - Index = 0 - if "PcdValueInit.c" not in Message: - if not MessageGroup: - MessageGroup.append(Message) - break - else: - for item in message_itmes: - if "PcdValueInit.c" in item: - Index = message_itmes.index(item) - message_itmes[Index] = dsc_line.strip() - break - MessageGroup.append(":".join(message_itmes[Index:]).strip()) - continue - else: - MessageGroup.append(Message) - if MessageGroup: - EdkLogger.error("build", PCD_STRUCTURE_PCD_ERROR, "\n".join(MessageGroup) ) - else: - EdkLogger.error('Build', COMMAND_FAILURE, 'Can not execute command: %s' % MakeCommand) + SaveFileOnChange(InputValueFile, InitByteValue, False) PcdValueInitExe = PcdValueInitName if not sys.platform == "win32": PcdValueInitExe = os.path.join(os.getenv("EDK_TOOLS_PATH"), 'Source', 'C', 'bin', PcdValueInitName) - - Command = PcdValueInitExe + ' -i %s -o %s' % (InputValueFile, OutputValueFile) - returncode, StdOut, StdErr = self.ExecuteCommand (Command) - if returncode <> 0: - EdkLogger.warn('Build', COMMAND_FAILURE, 'Can not collect output from command: %s' % Command) - FileBuffer = [] else: - File = open (OutputValueFile, 'r') - FileBuffer = File.readlines() - File.close() + PcdValueInitExe = os.path.join(os.getenv("EDK_TOOLS_PATH"), 'Bin', 'Win32', PcdValueInitName) +".exe" + if not os.path.exists(PcdValueInitExe) or self.NeedUpdateOutput(OutputValueFile, CAppBaseFileName + '.c',MakeFileName,InputValueFile): + Messages = '' + if sys.platform == "win32": + MakeCommand = 'nmake clean & nmake -f %s' % (MakeFileName) + returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand) + Messages = StdOut + else: + MakeCommand = 'make clean & make -f %s' % (MakeFileName) + returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand) + Messages = StdErr + Messages = Messages.split('\n') + MessageGroup = [] + if returncode <>0: + CAppBaseFileName = os.path.join(self.OutputPath, PcdValueInitName) + File = open (CAppBaseFileName + '.c', 'r') + FileData = File.readlines() + File.close() + for Message in Messages: + if " error" in Message or "warning" in Message: + FileInfo = Message.strip().split('(') + if len (FileInfo) > 1: + FileName = FileInfo [0] + FileLine = FileInfo [1].split (')')[0] + else: + FileInfo = Message.strip().split(':') + FileName = FileInfo [0] + FileLine = FileInfo [1] + if FileLine.isdigit(): + error_line = FileData[int (FileLine) - 1] + if r"//" in error_line: + c_line,dsc_line = error_line.split(r"//") + else: + dsc_line = error_line + message_itmes = Message.split(":") + Index = 0 + if "PcdValueInit.c" not in Message: + if not MessageGroup: + MessageGroup.append(Message) + break + else: + for item in message_itmes: + if "PcdValueInit.c" in item: + Index = message_itmes.index(item) + message_itmes[Index] = dsc_line.strip() + break + MessageGroup.append(":".join(message_itmes[Index:]).strip()) + continue + else: + MessageGroup.append(Message) + if MessageGroup: + EdkLogger.error("build", PCD_STRUCTURE_PCD_ERROR, "\n".join(MessageGroup) ) + else: + EdkLogger.error('Build', COMMAND_FAILURE, 'Can not execute command: %s' % MakeCommand) + Command = PcdValueInitExe + ' -i %s -o %s' % (InputValueFile, OutputValueFile) + returncode, StdOut, StdErr = self.ExecuteCommand (Command) + if returncode <> 0: + EdkLogger.warn('Build', COMMAND_FAILURE, 'Can not collect output from command: %s' % Command) + + File = open (OutputValueFile, 'r') + FileBuffer = File.readlines() + File.close() StructurePcdSet = [] for Pcd in FileBuffer: PcdValue = Pcd.split ('|') PcdInfo = PcdValue[0].split ('.') StructurePcdSet.append((PcdInfo[0],PcdInfo[1], PcdInfo[2], PcdInfo[3], PcdValue[2].strip())) return StructurePcdSet + def NeedUpdateOutput(self,OutputFile, ValueCFile, MakeFile, StructureInput): + if not os.path.exists(OutputFile): + return True + if os.stat(OutputFile).st_mtime <= os.stat(ValueCFile).st_mtime: + return True + if os.stat(OutputFile).st_mtime <= os.stat(MakeFile).st_mtime: + return True + if os.stat(OutputFile).st_mtime <= os.stat(StructureInput).st_mtime: + return True + return False + ## Retrieve dynamic PCD settings # # @param Type PCD type # # @retval a dict object contains settings of given PCD type -- 2.14.3.windows.1