From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mga02.intel.com (mga02.intel.com []) by mx.groups.io with SMTP id smtpd.web09.325.1575613569984515505 for ; Thu, 05 Dec 2019 22:26:15 -0800 Authentication-Results: mx.groups.io; dkim=missing; spf=fail (domain: intel.com, ip: , mailfrom: bob.c.feng@intel.com) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga006.jf.intel.com ([10.7.209.51]) by orsmga101.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 05 Dec 2019 22:26:15 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.69,283,1571727600"; d="scan'208";a="214345838" Received: from shwdepsi1121.ccr.corp.intel.com ([10.239.158.47]) by orsmga006.jf.intel.com with ESMTP; 05 Dec 2019 22:26:13 -0800 From: "Bob Feng" To: devel@edk2.groups.io Cc: Liming Gao , Steven Shi Subject: [Patch 4/4 V5] BaseTools: Enhance Basetool for incremental build Date: Fri, 6 Dec 2019 14:26:05 +0800 Message-Id: <20191206062605.26000-5-bob.c.feng@intel.com> X-Mailer: git-send-email 2.20.1.windows.1 In-Reply-To: <20191206062605.26000-1-bob.c.feng@intel.com> References: <20191206062605.26000-1-bob.c.feng@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2311 Include dependency file in Makefile to enhance incremental build Cc: Liming Gao Cc: Steven Shi Signed-off-by: Bob Feng --- V4: Move .deps files to where corresponding .obj file locate. BaseTools/Source/Python/AutoGen/GenMake.py | 83 ++--- .../Source/Python/AutoGen/IncludesAutoGen.py | 284 ++++++++++++++++++ .../Source/Python/AutoGen/ModuleAutoGen.py | 23 ++ BaseTools/Source/Python/build/build.py | 63 +++- 4 files changed, 380 insertions(+), 73 deletions(-) create mode 100644 BaseTools/Source/Python/AutoGen/IncludesAutoGen.py diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py index 59a01a7f243a..fe94f9a4c232 100755 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py @@ -183,10 +183,16 @@ class BuildFile(object): EdkLogger.error("build", PARAMETER_INVALID, "Invalid build type [%s]" % FileType, ExtraData="[%s]" % str(self._AutoGenObject)) self._FileType = FileType FileContent = self._TEMPLATE_.Replace(self._TemplateDict) FileName = self._FILE_NAME_[FileType] + if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, "deps.txt")): + with open(os.path.join(self._AutoGenObject.MakeFileDir, "deps.txt"),"w+") as fd: + fd.write("") + if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, "dependency")): + with open(os.path.join(self._AutoGenObject.MakeFileDir, "dependency"),"w+") as fd: + fd.write("") return SaveFileOnChange(os.path.join(self._AutoGenObject.MakeFileDir, FileName), FileContent, False) ## Return a list of directory creation command string # # @param DirList The list of directory to be created @@ -302,13 +308,10 @@ MAKE_FILE = ${makefile_path} # Build Macro # ${BEGIN}${file_macro} ${END} -COMMON_DEPS = ${BEGIN}${common_dependency_file} \\ - ${END} - # # Overridable Target Macro Definitions # FORCE_REBUILD = force_build INIT_TARGET = init @@ -380,10 +383,12 @@ gen_libs: # gen_fds: \t@"$(MAKE)" $(MAKE_FLAGS) -f $(BUILD_DIR)${separator}${makefile_name} fds \t@cd $(MODULE_BUILD_DIR) +${INCLUDETAG} + # # Individual Object Build Targets # ${BEGIN}${file_build_target} ${END} @@ -513,13 +518,10 @@ cleanlib: if Tool == "MAKE": continue # Remove duplicated include path, if any if Attr == "FLAGS": Value = RemoveDupOption(Value, IncPrefix, MyAgo.IncludePathList) - if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Tool == 'CC' and '/GM' in Value: - Value = Value.replace(' /MP', '') - MyAgo.BuildOption[Tool][Attr] = Value if Tool == "OPTROM" and PCI_COMPRESS_Flag: ValueList = Value.split() if ValueList: for i, v in enumerate(ValueList): if '-e' == v: @@ -538,11 +540,11 @@ cleanlib: RespFile = os.path.join(MyAgo.OutputDir, str(Resp).lower() + '.txt') StrList = RespDict[Resp].split(' ') UnexpandMacro = [] NewStr = [] for Str in StrList: - if '$' in Str: + if '$' in Str or '-MMD' in Str or '-MF' in Str: UnexpandMacro.append(Str) else: NewStr.append(Str) UnexpandMacroStr = ' '.join(UnexpandMacro) NewRespStr = ' '.join(NewStr) @@ -588,14 +590,13 @@ cleanlib: "source_file" : IncludePathList } ) FileMacroList.append(FileMacro) # Add support when compiling .nasm source files - for File in self.FileCache.keys(): - if not str(File).endswith('.nasm'): - continue - IncludePathList = [] + IncludePathList = [] + asmsource = [item for item in MyAgo.SourceFileList if item.File.upper().endswith((".NASM",".ASM",".NASMB","S"))] + if asmsource: for P in MyAgo.IncludePathList: IncludePath = self._INC_FLAG_['NASM'] + self.PlaceMacro(P, self.Macros) if IncludePath.endswith(os.sep): IncludePath = IncludePath.rstrip(os.sep) # When compiling .nasm files, need to add a literal backslash at each path @@ -604,11 +605,10 @@ cleanlib: IncludePath = ''.join([IncludePath, '^', os.sep]) else: IncludePath = os.path.join(IncludePath, '') IncludePathList.append(IncludePath) FileMacroList.append(self._FILE_MACRO_TEMPLATE.Replace({"macro_name": "NASM_INC", "source_file": IncludePathList})) - break # Generate macros used to represent files containing list of input files for ListFileMacro in self.ListFileMacros: ListFileName = os.path.join(MyAgo.OutputDir, "%s.lst" % ListFileMacro.lower()[:len(ListFileMacro) - 5]) FileMacroList.append("%s = %s" % (ListFileMacro, ListFileName)) @@ -694,10 +694,11 @@ cleanlib: "dependent_library_build_directory" : self.LibraryBuildDirectoryList, "library_build_command" : LibraryMakeCommandList, "file_macro" : FileMacroList, "file_build_target" : self.BuildTargetList, "backward_compatible_target": BcTargetList, + "INCLUDETAG" : self._INCLUDE_CMD_[self._FileType] + " " + os.path.join("$(MODULE_BUILD_DIR)","dependency") } return MakefileTemplateDict def ParserGenerateFfsCmd(self): @@ -901,20 +902,14 @@ cleanlib: if OutPutFileList: for Item in OutPutFileList: if Item in SourceFileList: SourceFileList.remove(Item) - FileDependencyDict = self.GetFileDependency( - SourceFileList, - ForceIncludedFile, - self._AutoGenObject.IncludePathList + self._AutoGenObject.BuildOptionIncPathList - ) + FileDependencyDict = {item:ForceIncludedFile for item in SourceFileList} - - if FileDependencyDict: - for Dependency in FileDependencyDict.values(): - self.DependencyHeaderFileSet.update(set(Dependency)) + for Dependency in FileDependencyDict.values(): + self.DependencyHeaderFileSet.update(set(Dependency)) # Get a set of unique package includes from MetaFile parentMetaFileIncludes = set() for aInclude in self._AutoGenObject.PackageIncludePathList: aIncludeName = str(aInclude) @@ -970,46 +965,20 @@ cleanlib: GlobalData.gModuleBuildTracking[self._AutoGenObject] = 'FAIL_METAFILE' EdkLogger.warn("build","Module MetaFile [Sources] is missing local header!", ExtraData = "Local Header: " + aFile + " not found in " + self._AutoGenObject.MetaFile.Path ) - DepSet = None for File,Dependency in FileDependencyDict.items(): if not Dependency: - FileDependencyDict[File] = ['$(FORCE_REBUILD)'] continue self._AutoGenObject.AutoGenDepSet |= set(Dependency) - # skip non-C files - if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c": - continue - elif DepSet is None: - DepSet = set(Dependency) - else: - DepSet &= set(Dependency) - # in case nothing in SourceFileList - if DepSet is None: - DepSet = set() - # - # Extract common files list in the dependency files - # - for File in DepSet: - self.CommonFileDependency.append(self.PlaceMacro(File.Path, self.Macros)) - CmdSumDict = {} CmdTargetDict = {} CmdCppDict = {} DependencyDict = FileDependencyDict.copy() - for File in FileDependencyDict: - # skip non-C files - if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c": - continue - NewDepSet = set(FileDependencyDict[File]) - NewDepSet -= DepSet - FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet) - DependencyDict[File] = list(NewDepSet) # Convert target description object to target string in makefile if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and TAB_C_CODE_FILE in self._AutoGenObject.Targets: for T in self._AutoGenObject.Targets[TAB_C_CODE_FILE]: NewFile = self.PlaceMacro(str(T), self.Macros) @@ -1078,21 +1047,17 @@ cleanlib: if CmdCppDict.get(item.Target.SubDir): CmdCppDict[item.Target.SubDir].append(Path) else: CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path] if CppPath.Path in DependencyDict: - if '$(FORCE_REBUILD)' in DependencyDict[CppPath.Path]: - if '$(FORCE_REBUILD)' not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]): - CmdCppDict[item.Target.SubDir].append('$(FORCE_REBUILD)') - else: - for Temp in DependencyDict[CppPath.Path]: - try: - Path = self.PlaceMacro(Temp.Path, self.Macros) - except: - continue - if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]): - CmdCppDict[item.Target.SubDir].append(Path) + for Temp in DependencyDict[CppPath.Path]: + try: + Path = self.PlaceMacro(Temp.Path, self.Macros) + except: + continue + if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]): + CmdCppDict[item.Target.SubDir].append(Path) if T.Commands: CommandList = T.Commands[:] for Item in CommandList[:]: SingleCommandList = Item.split() if len(SingleCommandList) > 0 and self.CheckCCCmd(SingleCommandList): @@ -1107,11 +1072,11 @@ cleanlib: CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1]) Index = CommandList.index(Item) CommandList.pop(Index) if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign[3:].rsplit(TAB_SLASH, 1)[0]])): Cpplist = CmdCppDict[T.Target.SubDir] - Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir)) + Cpplist.insert(0, '$(OBJLIST_%d): ' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir)) T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign]) else: T.Commands.pop(Index) return T, CmdSumDict, CmdTargetDict, CmdCppDict diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py new file mode 100644 index 000000000000..bb6e883d84ca --- /dev/null +++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py @@ -0,0 +1,284 @@ +## @file +# Build cache intermediate result and state +# +# Copyright (c) 2019, Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +# +from Common.caching import cached_property +import Common.EdkLogger as EdkLogger +import Common.LongFilePathOs as os +from Common.BuildToolError import * +from Common.Misc import SaveFileOnChange, PathClass +from Common.Misc import TemplateString +import sys +gIsFileMap = {} +if sys.platform == "win32": + _INCLUDE_DEPS_TEMPLATE = TemplateString(''' +${BEGIN} +!IF EXIST(${deps_file}) +!INCLUDE ${deps_file} +!ENDIF +${END} + ''') +else: + _INCLUDE_DEPS_TEMPLATE = TemplateString(''' +${BEGIN} +-include ${deps_file} +${END} + ''') + +DEP_FILE_TAIL = "# Updated \n" + +class IncludesAutoGen(): + """ This class is to manage the dependent files witch are used in Makefile to support incremental build. + 1. C files: + 1. MSVS. + cl.exe has a build option /showIncludes to display include files on stdout. Build tool captures + that messages and generate dependency files, .deps files. + 2. CLANG and GCC + -MMD -MF build option are used to generate dependency files by compiler. Build tool updates the + .deps files. + 2. ASL files: + 1. Trim find out all the included files with asl specific include format and generate .trim.deps file. + 2. ASL PP use c preprocessor to find out all included files with #include format and generate a .deps file + 3. build tool updates the .deps file + 3. ASM files (.asm, .s or .nasm): + 1. Trim find out all the included files with asl specific include format and generate .trim.deps file. + 2. ASM PP use c preprocessor to find out all included files with #include format and generate a deps file + 3. build tool updates the .deps file + """ + def __init__(self, makefile_folder, ModuleAuto): + self.d_folder = makefile_folder + self.makefile_folder = makefile_folder + self.module_autogen = ModuleAuto + self.ToolChainFamily = ModuleAuto.ToolChainFamily + self.workspace = ModuleAuto.WorkspaceDir + + def CreateModuleDeps(self): + SaveFileOnChange(os.path.join(self.makefile_folder,"deps.txt"),"\n".join(self.DepsCollection),False) + + def CreateDepsInclude(self): + deps_file = {'deps_file':self.deps_files} + try: + deps_include_str = _INCLUDE_DEPS_TEMPLATE.Replace(deps_file) + except Exception as e: + print(e) + SaveFileOnChange(os.path.join(self.makefile_folder,"dependency"),deps_include_str,False) + + @cached_property + def deps_files(self): + """ Get all .deps file under module build folder. """ + deps_files = [] + for root, _, files in os.walk(self.d_folder, topdown=False): + for name in files: + if not name.endswith(".deps"): + continue + abspath = os.path.join(root, name) + deps_files.append(abspath) + return deps_files + + @cached_property + def DepsCollection(self): + """ Collect all the dependency files list from all .deps files under a module's build folder """ + includes = set() + targetname = [item[0].Name for item in self.TargetFileList.values()] + for abspath in self.deps_files: + try: + with open(abspath,"r") as fd: + lines = fd.readlines() + + firstlineitems = lines[0].split(": ") + dependency_file = firstlineitems[1].strip(" \\\n") + dependency_file = dependency_file.strip('''"''') + if dependency_file: + if os.path.normpath(dependency_file +".deps") == abspath: + continue + filename = os.path.basename(dependency_file).strip() + if filename not in self.SourceFileList and filename not in targetname: + includes.add(dependency_file.strip()) + + for item in lines[1:]: + if item == DEP_FILE_TAIL: + continue + dependency_file = item.strip(" \\\n") + dependency_file = dependency_file.strip('''"''') + if os.path.normpath(dependency_file +".deps") == abspath: + continue + filename = os.path.basename(dependency_file).strip() + if filename in self.SourceFileList: + continue + if filename in targetname: + continue + includes.add(dependency_file.strip()) + except Exception as e: + EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False) + continue + rt = sorted(list(set([item.strip(' " \\\n') for item in includes]))) + return rt + + @cached_property + def SourceFileList(self): + """ Get a map of module's source files name to module's source files path """ + source = {os.path.basename(item.File):item.Path for item in self.module_autogen.SourceFileList} + middle_file = {} + for afile in source: + if afile.upper().endswith(".VFR"): + middle_file.update({afile.split(".")[0]+".c":os.path.join(self.module_autogen.DebugDir,afile.split(".")[0]+".c")}) + if afile.upper().endswith((".S","ASM")): + middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")}) + if afile.upper().endswith(".ASL"): + middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")}) + source.update({"AutoGen.c":os.path.join(self.module_autogen.OutputDir,"AutoGen.c")}) + source.update(middle_file) + return source + + @cached_property + def HasNamesakeSourceFile(self): + source_base_name = set([os.path.basename(item.File) for item in self.module_autogen.SourceFileList]) + rt = len(source_base_name) != len(self.module_autogen.SourceFileList) + return rt + @cached_property + def CcPPCommandPathSet(self): + rt = set() + rt.add(self.module_autogen.BuildOption.get('CC',{}).get('PATH')) + rt.add(self.module_autogen.BuildOption.get('ASLCC',{}).get('PATH')) + rt.add(self.module_autogen.BuildOption.get('ASLPP',{}).get('PATH')) + rt.add(self.module_autogen.BuildOption.get('VFRPP',{}).get('PATH')) + rt.add(self.module_autogen.BuildOption.get('PP',{}).get('PATH')) + rt.add(self.module_autogen.BuildOption.get('APP',{}).get('PATH')) + rt.discard(None) + return rt + @cached_property + def TargetFileList(self): + """ Get a map of module's target name to a tuple of module's targets path and whose input file path """ + targets = {} + targets["AutoGen.obj"] = (PathClass(os.path.join(self.module_autogen.OutputDir,"AutoGen.obj")),PathClass(os.path.join(self.module_autogen.DebugDir,"AutoGen.c"))) + for item in self.module_autogen.Targets.values(): + for block in item: + targets[block.Target.Path] = (block.Target,block.Inputs[0]) + return targets + + def GetRealTarget(self,source_file_abs): + """ Get the final target file based on source file abspath """ + source_target_map = {item[1].Path:item[0].Path for item in self.TargetFileList.values()} + source_name_map = {item[1].File:item[0].Path for item in self.TargetFileList.values()} + target_abs = source_target_map.get(source_file_abs) + if target_abs is None: + if source_file_abs.strip().endswith(".i"): + sourcefilename = os.path.basename(source_file_abs.strip()) + for sourcefile in source_name_map: + if sourcefilename.split(".")[0] == sourcefile.split(".")[0]: + target_abs = source_name_map[sourcefile] + break + else: + target_abs = source_file_abs + else: + target_abs = source_file_abs + return target_abs + + def CreateDepsFileForMsvc(self, DepList): + """ Generate dependency files, .deps file from /showIncludes output message """ + if not DepList: + return + ModuleDepDict = {} + current_source = "" + SourceFileAbsPathMap = self.SourceFileList + for line in DepList: + line = line.strip() + if self.HasNamesakeSourceFile: + for cc_cmd in self.CcPPCommandPathSet: + if cc_cmd in line: + if '''"'''+cc_cmd+'''"''' in line: + cc_options = line[len(cc_cmd)+2:].split() + else: + cc_options = line[len(cc_cmd):].split() + SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)} + if line in SourceFileAbsPathMap: + current_source = line + if current_source not in ModuleDepDict: + ModuleDepDict[SourceFileAbsPathMap[current_source]] = [] + elif "Note: including file:" == line.lstrip()[:21]: + if not current_source: + EdkLogger.error("build",BUILD_ERROR, "Parse /showIncludes output failed. line: %s. \n" % line, RaiseError=False) + else: + ModuleDepDict[SourceFileAbsPathMap[current_source]].append(line.lstrip()[22:].strip()) + + for source_abs in ModuleDepDict: + if ModuleDepDict[source_abs]: + target_abs = self.GetRealTarget(source_abs) + dep_file_name = os.path.basename(source_abs) + ".deps" + SaveFileOnChange(os.path.join(os.path.dirname(target_abs),dep_file_name)," \\\n".join([target_abs+":"] + ['''"''' + item +'''"''' for item in ModuleDepDict[source_abs]]),False) + + def UpdateDepsFileforNonMsvc(self): + """ Update .deps files. + 1. Update target path to absolute path. + 2. Update middle target to final target. + """ + + for abspath in self.deps_files: + if abspath.endswith(".trim.deps"): + continue + try: + newcontent = [] + with open(abspath,"r") as fd: + lines = fd.readlines() + if lines[-1] == DEP_FILE_TAIL: + continue + firstlineitems = lines[0].strip().split(" ") + + if len(firstlineitems) > 2: + sourceitem = firstlineitems[1] + else: + sourceitem = lines[1].strip().split(" ")[0] + + source_abs = self.SourceFileList.get(sourceitem,sourceitem) + firstlineitems[0] = self.GetRealTarget(source_abs) + p_target = firstlineitems + if not p_target[0].strip().endswith(":"): + p_target[0] += ": " + + if len(p_target) == 2: + p_target[0] += lines[1] + newcontent.append(p_target[0]) + newcontent.extend(lines[2:]) + else: + line1 = " ".join(p_target).strip() + line1 += "\n" + newcontent.append(line1) + newcontent.extend(lines[1:]) + + newcontent.append("\n") + newcontent.append(DEP_FILE_TAIL) + with open(abspath,"w") as fw: + fw.write("".join(newcontent)) + except Exception as e: + EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False) + continue + + def UpdateDepsFileforTrim(self): + """ Update .deps file which generated by trim. """ + + for abspath in self.deps_files: + if not abspath.endswith(".trim.deps"): + continue + try: + newcontent = [] + with open(abspath,"r") as fd: + lines = fd.readlines() + if lines[-1] == DEP_FILE_TAIL: + continue + + source_abs = lines[0].strip().split(" ")[0] + targetitem = self.GetRealTarget(source_abs.strip(" :")) + + targetitem += ": " + targetitem += lines[1] + newcontent.append(targetitem) + newcontent.extend(lines[2:]) + newcontent.append("\n") + newcontent.append(DEP_FILE_TAIL) + with open(abspath,"w") as fw: + fw.write("".join(newcontent)) + except Exception as e: + EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False) + continue diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py index e6d6c43810ec..1111d5de25ac 100755 --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py @@ -1127,12 +1127,35 @@ class ModuleAutoGen(AutoGen): if not self.MetaFile.OriginalPath.Path.startswith(PackageDir): IncludesList = list(set(Package.Includes).difference(set(Package._PrivateIncludes))) for Inc in IncludesList: if Inc not in RetVal: RetVal.append(str(Inc)) + RetVal.extend(self.IncPathFromBuildOptions) return RetVal + @cached_property + def IncPathFromBuildOptions(self): + IncPathList = [] + for tool in self.BuildOption: + if 'FLAGS' in self.BuildOption[tool]: + flags = self.BuildOption[tool]['FLAGS'] + whitespace = False + for flag in flags.split(" "): + flag = flag.strip() + if flag.startswith(("/I","-I")): + if len(flag)>2: + if os.path.exists(flag[2:]): + IncPathList.append(flag[2:]) + else: + whitespace = True + continue + if whitespace and flag: + if os.path.exists(flag): + IncPathList.append(flag) + whitespace = False + return IncPathList + @cached_property def IncludePathLength(self): return sum(len(inc)+1 for inc in self.IncludePathList) ## Get the list of include paths from the packages diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index 4b31356a4253..6fcfc7ec7ab3 100755 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -55,11 +55,11 @@ from GenFds.GenFds import GenFds, GenFdsApi import multiprocessing as mp from multiprocessing import Manager from AutoGen.DataPipe import MemoryDataPipe from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo from GenFds.FdfParser import FdfParser - +from AutoGen.IncludesAutoGen import IncludesAutoGen ## standard targets of build command gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run'] ## build configuration file @@ -173,33 +173,46 @@ def NormFile(FilePath, Workspace): # # @param From The stream message read from # @param To The stream message put on # @param ExitFlag The flag used to indicate stopping reading # -def ReadMessage(From, To, ExitFlag): +def ReadMessage(From, To, ExitFlag,MemTo=None): while True: # read one line a time Line = From.readline() # empty string means "end" if Line is not None and Line != b"": - To(Line.rstrip().decode(encoding='utf-8', errors='ignore')) + LineStr = Line.rstrip().decode(encoding='utf-8', errors='ignore') + if MemTo is not None: + if "Note: including file:" == LineStr.lstrip()[:21]: + MemTo.append(LineStr) + else: + To(LineStr) + MemTo.append(LineStr) + else: + To(LineStr) else: break if ExitFlag.isSet(): break +class MakeSubProc(Popen): + def __init__(self,*args, **argv): + super(MakeSubProc,self).__init__(*args, **argv) + self.ProcOut = [] + ## Launch an external program # # This method will call subprocess.Popen to execute an external program with # given options in specified directory. Because of the dead-lock issue during # redirecting output of the external program, threads are used to to do the # redirection work. # # @param Command A list or string containing the call of the program # @param WorkingDir The directory in which the program will be running # -def LaunchCommand(Command, WorkingDir): +def LaunchCommand(Command, WorkingDir,ModuleAuto = None): BeginTime = time.time() # if working directory doesn't exist, Popen() will raise an exception if not os.path.isdir(WorkingDir): EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=WorkingDir) @@ -214,23 +227,23 @@ def LaunchCommand(Command, WorkingDir): Proc = None EndOfProcedure = None try: # launch the command - Proc = Popen(Command, stdout=PIPE, stderr=PIPE, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True) + Proc = MakeSubProc(Command, stdout=PIPE, stderr=PIPE, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True) # launch two threads to read the STDOUT and STDERR EndOfProcedure = Event() EndOfProcedure.clear() if Proc.stdout: - StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure)) + StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure,Proc.ProcOut)) StdOutThread.setName("STDOUT-Redirector") StdOutThread.setDaemon(False) StdOutThread.start() if Proc.stderr: - StdErrThread = Thread(target=ReadMessage, args=(Proc.stderr, EdkLogger.quiet, EndOfProcedure)) + StdErrThread = Thread(target=ReadMessage, args=(Proc.stderr, EdkLogger.quiet, EndOfProcedure,Proc.ProcOut)) StdErrThread.setName("STDERR-Redirector") StdErrThread.setDaemon(False) StdErrThread.start() # waiting for program exit @@ -261,10 +274,19 @@ def LaunchCommand(Command, WorkingDir): RespContent = f.read() f.close() EdkLogger.info(RespContent) EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir)) + if ModuleAuto: + iau = IncludesAutoGen(WorkingDir,ModuleAuto) + if ModuleAuto.ToolChainFamily == TAB_COMPILER_MSFT: + iau.CreateDepsFileForMsvc(Proc.ProcOut) + else: + iau.UpdateDepsFileforNonMsvc() + iau.UpdateDepsFileforTrim() + iau.CreateModuleDeps() + iau.CreateDepsInclude() return "%dms" % (int(round((time.time() - BeginTime) * 1000))) ## The smallest unit that can be built in multi-thread build mode # # This is the base class of build unit. The "Obj" parameter must provide @@ -606,11 +628,11 @@ class BuildTask: # @param Command A list or string contains the call of the command # @param WorkingDir The directory in which the program will be running # def _CommandThread(self, Command, WorkingDir): try: - self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir) + self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir,self.BuildItem.BuildObject) self.CompleteFlag = True # Run hash operation post dependency, to account for libs if GlobalData.gUseHashCache and self.BuildItem.BuildObject.IsLibrary: HashFile = path.join(self.BuildItem.BuildObject.BuildDir, self.BuildItem.BuildObject.Name + ".hash") @@ -1274,23 +1296,36 @@ class Build(): self.BuildModules = [] return True # build library if Target == 'libraries': - for Lib in AutoGenObject.LibraryBuildDirectoryList: + DirList = [] + for Lib in AutoGenObject.LibraryAutoGenList: + if not Lib.IsBinaryModule: + DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib)) + for Lib, LibAutoGen in DirList: NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild'] - LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir) + LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen) return True # build module if Target == 'modules': - for Lib in AutoGenObject.LibraryBuildDirectoryList: + DirList = [] + for Lib in AutoGenObject.LibraryAutoGenList: + if not Lib.IsBinaryModule: + DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib)) + for Lib, LibAutoGen in DirList: NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild'] - LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir) - for Mod in AutoGenObject.ModuleBuildDirectoryList: + LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen) + + DirList = [] + for ModuleAutoGen in AutoGenObject.ModuleAutoGenList: + if not ModuleAutoGen.IsBinaryModule: + DirList.append((os.path.join(AutoGenObject.BuildDir, ModuleAutoGen.BuildDir),ModuleAutoGen)) + for Mod,ModAutoGen in DirList: NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, makefile)), 'pbuild'] - LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir) + LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,ModAutoGen) self.CreateAsBuiltInf() if GlobalData.gBinCacheDest: self.UpdateBuildCache() self.BuildModules = [] return True -- 2.20.1.windows.1