[Patch 4/4 V6] BaseTools: Enhance Basetool for incremental build

["Bob Feng" <bob.c.feng@intel.com>]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2311

Include dependency file in Makefile to enhance
incremental build

Cc: Liming Gao <liming.gao@intel.com>
Cc: Steven Shi <steven.shi@intel.com>
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
---
 BaseTools/Source/Python/AutoGen/GenMake.py    |  83 ++---
 .../Source/Python/AutoGen/IncludesAutoGen.py  | 284 ++++++++++++++++++
 .../Source/Python/AutoGen/ModuleAutoGen.py    |  23 ++
 BaseTools/Source/Python/build/build.py        |  63 +++-
 4 files changed, 380 insertions(+), 73 deletions(-)
 create mode 100644 BaseTools/Source/Python/AutoGen/IncludesAutoGen.py

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py
index 59a01a7f24..fe94f9a4c2 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -183,10 +183,16 @@ class BuildFile(object):
             EdkLogger.error("build", PARAMETER_INVALID, "Invalid build type [%s]" % FileType,
                             ExtraData="[%s]" % str(self._AutoGenObject))
         self._FileType = FileType
         FileContent = self._TEMPLATE_.Replace(self._TemplateDict)
         FileName = self._FILE_NAME_[FileType]
+        if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, "deps.txt")):
+            with open(os.path.join(self._AutoGenObject.MakeFileDir, "deps.txt"),"w+") as fd:
+                fd.write("")
+        if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, "dependency")):
+            with open(os.path.join(self._AutoGenObject.MakeFileDir, "dependency"),"w+") as fd:
+                fd.write("")
         return SaveFileOnChange(os.path.join(self._AutoGenObject.MakeFileDir, FileName), FileContent, False)
 
     ## Return a list of directory creation command string
     #
     #   @param      DirList     The list of directory to be created
@@ -302,13 +308,10 @@ MAKE_FILE = ${makefile_path}
 # Build Macro
 #
 ${BEGIN}${file_macro}
 ${END}
 
-COMMON_DEPS = ${BEGIN}${common_dependency_file} \\
-              ${END}
-
 #
 # Overridable Target Macro Definitions
 #
 FORCE_REBUILD = force_build
 INIT_TARGET = init
@@ -380,10 +383,12 @@ gen_libs:
 #
 gen_fds:
 \t@"$(MAKE)" $(MAKE_FLAGS) -f $(BUILD_DIR)${separator}${makefile_name} fds
 \t@cd $(MODULE_BUILD_DIR)
 
+${INCLUDETAG}
+
 #
 # Individual Object Build Targets
 #
 ${BEGIN}${file_build_target}
 ${END}
@@ -513,13 +518,10 @@ cleanlib:
                     if Tool == "MAKE":
                         continue
                     # Remove duplicated include path, if any
                     if Attr == "FLAGS":
                         Value = RemoveDupOption(Value, IncPrefix, MyAgo.IncludePathList)
-                        if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Tool == 'CC' and '/GM' in Value:
-                            Value = Value.replace(' /MP', '')
-                            MyAgo.BuildOption[Tool][Attr] = Value
                         if Tool == "OPTROM" and PCI_COMPRESS_Flag:
                             ValueList = Value.split()
                             if ValueList:
                                 for i, v in enumerate(ValueList):
                                     if '-e' == v:
@@ -538,11 +540,11 @@ cleanlib:
                 RespFile = os.path.join(MyAgo.OutputDir, str(Resp).lower() + '.txt')
                 StrList = RespDict[Resp].split(' ')
                 UnexpandMacro = []
                 NewStr = []
                 for Str in StrList:
-                    if '$' in Str:
+                    if '$' in Str or '-MMD' in Str or '-MF' in Str:
                         UnexpandMacro.append(Str)
                     else:
                         NewStr.append(Str)
                 UnexpandMacroStr = ' '.join(UnexpandMacro)
                 NewRespStr = ' '.join(NewStr)
@@ -588,14 +590,13 @@ cleanlib:
                                                     "source_file" : IncludePathList
                                                 }
                                                 )
         FileMacroList.append(FileMacro)
         # Add support when compiling .nasm source files
-        for File in self.FileCache.keys():
-            if not str(File).endswith('.nasm'):
-                continue
-            IncludePathList = []
+        IncludePathList = []
+        asmsource = [item for item in MyAgo.SourceFileList if item.File.upper().endswith((".NASM",".ASM",".NASMB","S"))]
+        if asmsource:
             for P in  MyAgo.IncludePathList:
                 IncludePath = self._INC_FLAG_['NASM'] + self.PlaceMacro(P, self.Macros)
                 if IncludePath.endswith(os.sep):
                     IncludePath = IncludePath.rstrip(os.sep)
                 # When compiling .nasm files, need to add a literal backslash at each path
@@ -604,11 +605,10 @@ cleanlib:
                     IncludePath = ''.join([IncludePath, '^', os.sep])
                 else:
                     IncludePath = os.path.join(IncludePath, '')
                 IncludePathList.append(IncludePath)
             FileMacroList.append(self._FILE_MACRO_TEMPLATE.Replace({"macro_name": "NASM_INC", "source_file": IncludePathList}))
-            break
 
         # Generate macros used to represent files containing list of input files
         for ListFileMacro in self.ListFileMacros:
             ListFileName = os.path.join(MyAgo.OutputDir, "%s.lst" % ListFileMacro.lower()[:len(ListFileMacro) - 5])
             FileMacroList.append("%s = %s" % (ListFileMacro, ListFileName))
@@ -694,10 +694,11 @@ cleanlib:
             "dependent_library_build_directory" : self.LibraryBuildDirectoryList,
             "library_build_command"     : LibraryMakeCommandList,
             "file_macro"                : FileMacroList,
             "file_build_target"         : self.BuildTargetList,
             "backward_compatible_target": BcTargetList,
+            "INCLUDETAG"                   : self._INCLUDE_CMD_[self._FileType] + " " + os.path.join("$(MODULE_BUILD_DIR)","dependency")
         }
 
         return MakefileTemplateDict
 
     def ParserGenerateFfsCmd(self):
@@ -901,20 +902,14 @@ cleanlib:
         if OutPutFileList:
             for Item in OutPutFileList:
                 if Item in SourceFileList:
                     SourceFileList.remove(Item)
 
-        FileDependencyDict = self.GetFileDependency(
-                                    SourceFileList,
-                                    ForceIncludedFile,
-                                    self._AutoGenObject.IncludePathList + self._AutoGenObject.BuildOptionIncPathList
-                                    )
-
+        FileDependencyDict = {item:ForceIncludedFile for item in SourceFileList}
 
-        if FileDependencyDict:
-            for Dependency in FileDependencyDict.values():
-                self.DependencyHeaderFileSet.update(set(Dependency))
+        for Dependency in FileDependencyDict.values():
+            self.DependencyHeaderFileSet.update(set(Dependency))
 
         # Get a set of unique package includes from MetaFile
         parentMetaFileIncludes = set()
         for aInclude in self._AutoGenObject.PackageIncludePathList:
             aIncludeName = str(aInclude)
@@ -970,46 +965,20 @@ cleanlib:
                 GlobalData.gModuleBuildTracking[self._AutoGenObject] = 'FAIL_METAFILE'
             EdkLogger.warn("build","Module MetaFile [Sources] is missing local header!",
                         ExtraData = "Local Header: " + aFile + " not found in " + self._AutoGenObject.MetaFile.Path
                         )
 
-        DepSet = None
         for File,Dependency in FileDependencyDict.items():
             if not Dependency:
-                FileDependencyDict[File] = ['$(FORCE_REBUILD)']
                 continue
 
             self._AutoGenObject.AutoGenDepSet |= set(Dependency)
 
-            # skip non-C files
-            if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c":
-                continue
-            elif DepSet is None:
-                DepSet = set(Dependency)
-            else:
-                DepSet &= set(Dependency)
-        # in case nothing in SourceFileList
-        if DepSet is None:
-            DepSet = set()
-        #
-        # Extract common files list in the dependency files
-        #
-        for File in DepSet:
-            self.CommonFileDependency.append(self.PlaceMacro(File.Path, self.Macros))
-
         CmdSumDict = {}
         CmdTargetDict = {}
         CmdCppDict = {}
         DependencyDict = FileDependencyDict.copy()
-        for File in FileDependencyDict:
-            # skip non-C files
-            if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c":
-                continue
-            NewDepSet = set(FileDependencyDict[File])
-            NewDepSet -= DepSet
-            FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet)
-            DependencyDict[File] = list(NewDepSet)
 
         # Convert target description object to target string in makefile
         if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and TAB_C_CODE_FILE in self._AutoGenObject.Targets:
             for T in self._AutoGenObject.Targets[TAB_C_CODE_FILE]:
                 NewFile = self.PlaceMacro(str(T), self.Macros)
@@ -1078,21 +1047,17 @@ cleanlib:
                     if CmdCppDict.get(item.Target.SubDir):
                         CmdCppDict[item.Target.SubDir].append(Path)
                     else:
                         CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path]
                     if CppPath.Path in DependencyDict:
-                        if '$(FORCE_REBUILD)' in DependencyDict[CppPath.Path]:
-                            if '$(FORCE_REBUILD)' not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]):
-                                CmdCppDict[item.Target.SubDir].append('$(FORCE_REBUILD)')
-                        else:
-                            for Temp in DependencyDict[CppPath.Path]:
-                                try:
-                                    Path = self.PlaceMacro(Temp.Path, self.Macros)
-                                except:
-                                    continue
-                                if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]):
-                                    CmdCppDict[item.Target.SubDir].append(Path)
+                        for Temp in DependencyDict[CppPath.Path]:
+                            try:
+                                Path = self.PlaceMacro(Temp.Path, self.Macros)
+                            except:
+                                continue
+                            if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]):
+                                CmdCppDict[item.Target.SubDir].append(Path)
         if T.Commands:
             CommandList = T.Commands[:]
             for Item in CommandList[:]:
                 SingleCommandList = Item.split()
                 if len(SingleCommandList) > 0 and self.CheckCCCmd(SingleCommandList):
@@ -1107,11 +1072,11 @@ cleanlib:
                         CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1])
                     Index = CommandList.index(Item)
                     CommandList.pop(Index)
                     if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign[3:].rsplit(TAB_SLASH, 1)[0]])):
                         Cpplist = CmdCppDict[T.Target.SubDir]
-                        Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir))
+                        Cpplist.insert(0, '$(OBJLIST_%d): ' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir))
                         T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign])
                     else:
                         T.Commands.pop(Index)
         return T, CmdSumDict, CmdTargetDict, CmdCppDict
 
diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
new file mode 100644
index 0000000000..bb6e883d84
--- /dev/null
+++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
@@ -0,0 +1,284 @@
+## @file
+# Build cache intermediate result and state
+#
+# Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+from Common.caching import cached_property
+import Common.EdkLogger as EdkLogger
+import Common.LongFilePathOs as os
+from Common.BuildToolError import *
+from Common.Misc import SaveFileOnChange, PathClass
+from Common.Misc import TemplateString
+import sys
+gIsFileMap = {}
+if sys.platform == "win32":
+    _INCLUDE_DEPS_TEMPLATE = TemplateString('''
+${BEGIN}
+!IF EXIST(${deps_file})
+!INCLUDE ${deps_file}
+!ENDIF
+${END}
+        ''')
+else:
+    _INCLUDE_DEPS_TEMPLATE = TemplateString('''
+${BEGIN}
+-include ${deps_file}
+${END}
+        ''')
+
+DEP_FILE_TAIL = "# Updated \n"
+
+class IncludesAutoGen():
+    """ This class is to manage the dependent files witch are used in Makefile to support incremental build.
+        1. C files:
+            1. MSVS.
+               cl.exe has a build option /showIncludes to display include files on stdout. Build tool captures
+               that messages and generate dependency files, .deps files.
+            2. CLANG and GCC
+               -MMD -MF build option are used to generate dependency files by compiler. Build tool updates the
+               .deps files.
+        2. ASL files:
+            1. Trim find out all the included files with asl specific include format and generate .trim.deps file.
+            2. ASL PP use c preprocessor to find out all included files with #include format and generate a .deps file
+            3. build tool updates the .deps file
+        3. ASM files (.asm, .s or .nasm):
+            1. Trim find out all the included files with asl specific include format and generate .trim.deps file.
+            2. ASM PP use c preprocessor to find out all included files with #include format and generate a deps file
+            3. build tool updates the .deps file
+    """
+    def __init__(self, makefile_folder, ModuleAuto):
+        self.d_folder = makefile_folder
+        self.makefile_folder = makefile_folder
+        self.module_autogen = ModuleAuto
+        self.ToolChainFamily = ModuleAuto.ToolChainFamily
+        self.workspace = ModuleAuto.WorkspaceDir
+
+    def CreateModuleDeps(self):
+        SaveFileOnChange(os.path.join(self.makefile_folder,"deps.txt"),"\n".join(self.DepsCollection),False)
+
+    def CreateDepsInclude(self):
+        deps_file = {'deps_file':self.deps_files}
+        try:
+            deps_include_str = _INCLUDE_DEPS_TEMPLATE.Replace(deps_file)
+        except Exception as e:
+            print(e)
+        SaveFileOnChange(os.path.join(self.makefile_folder,"dependency"),deps_include_str,False)
+
+    @cached_property
+    def deps_files(self):
+        """ Get all .deps file under module build folder. """
+        deps_files = []
+        for root, _, files in os.walk(self.d_folder, topdown=False):
+            for name in files:
+                if not name.endswith(".deps"):
+                    continue
+                abspath = os.path.join(root, name)
+                deps_files.append(abspath)
+        return deps_files
+
+    @cached_property
+    def DepsCollection(self):
+        """ Collect all the dependency files list from all .deps files under a module's build folder """
+        includes = set()
+        targetname = [item[0].Name for item in self.TargetFileList.values()]
+        for abspath in self.deps_files:
+            try:
+                with open(abspath,"r") as fd:
+                    lines = fd.readlines()
+
+                firstlineitems = lines[0].split(": ")
+                dependency_file = firstlineitems[1].strip(" \\\n")
+                dependency_file = dependency_file.strip('''"''')
+                if dependency_file:
+                    if os.path.normpath(dependency_file +".deps") == abspath:
+                        continue
+                    filename = os.path.basename(dependency_file).strip()
+                    if filename not in self.SourceFileList and filename not in targetname:
+                        includes.add(dependency_file.strip())
+
+                for item in lines[1:]:
+                    if item == DEP_FILE_TAIL:
+                        continue
+                    dependency_file = item.strip(" \\\n")
+                    dependency_file = dependency_file.strip('''"''')
+                    if os.path.normpath(dependency_file +".deps") == abspath:
+                        continue
+                    filename = os.path.basename(dependency_file).strip()
+                    if filename in self.SourceFileList:
+                        continue
+                    if filename in targetname:
+                        continue
+                    includes.add(dependency_file.strip())
+            except Exception as e:
+                EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
+                continue
+        rt = sorted(list(set([item.strip(' " \\\n') for item in includes])))
+        return rt
+
+    @cached_property
+    def SourceFileList(self):
+        """ Get a map of module's source files name to module's source files path """
+        source = {os.path.basename(item.File):item.Path for item in self.module_autogen.SourceFileList}
+        middle_file = {}
+        for afile in source:
+            if afile.upper().endswith(".VFR"):
+                middle_file.update({afile.split(".")[0]+".c":os.path.join(self.module_autogen.DebugDir,afile.split(".")[0]+".c")})
+            if afile.upper().endswith((".S","ASM")):
+                middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")})
+            if afile.upper().endswith(".ASL"):
+                middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")})
+        source.update({"AutoGen.c":os.path.join(self.module_autogen.OutputDir,"AutoGen.c")})
+        source.update(middle_file)
+        return source
+
+    @cached_property
+    def HasNamesakeSourceFile(self):
+        source_base_name = set([os.path.basename(item.File) for item in self.module_autogen.SourceFileList])
+        rt = len(source_base_name) != len(self.module_autogen.SourceFileList)
+        return rt
+    @cached_property
+    def CcPPCommandPathSet(self):
+        rt = set()
+        rt.add(self.module_autogen.BuildOption.get('CC',{}).get('PATH'))
+        rt.add(self.module_autogen.BuildOption.get('ASLCC',{}).get('PATH'))
+        rt.add(self.module_autogen.BuildOption.get('ASLPP',{}).get('PATH'))
+        rt.add(self.module_autogen.BuildOption.get('VFRPP',{}).get('PATH'))
+        rt.add(self.module_autogen.BuildOption.get('PP',{}).get('PATH'))
+        rt.add(self.module_autogen.BuildOption.get('APP',{}).get('PATH'))
+        rt.discard(None)
+        return rt
+    @cached_property
+    def TargetFileList(self):
+        """ Get a map of module's target name to a tuple of module's targets path and whose input file path """
+        targets = {}
+        targets["AutoGen.obj"] = (PathClass(os.path.join(self.module_autogen.OutputDir,"AutoGen.obj")),PathClass(os.path.join(self.module_autogen.DebugDir,"AutoGen.c")))
+        for item in self.module_autogen.Targets.values():
+            for block in item:
+                targets[block.Target.Path] = (block.Target,block.Inputs[0])
+        return targets
+
+    def GetRealTarget(self,source_file_abs):
+        """ Get the final target file based on source file abspath """
+        source_target_map = {item[1].Path:item[0].Path for item in self.TargetFileList.values()}
+        source_name_map = {item[1].File:item[0].Path for item in self.TargetFileList.values()}
+        target_abs = source_target_map.get(source_file_abs)
+        if target_abs is None:
+            if source_file_abs.strip().endswith(".i"):
+                sourcefilename = os.path.basename(source_file_abs.strip())
+                for sourcefile in source_name_map:
+                    if sourcefilename.split(".")[0] == sourcefile.split(".")[0]:
+                        target_abs = source_name_map[sourcefile]
+                        break
+                else:
+                    target_abs = source_file_abs
+            else:
+                target_abs = source_file_abs
+        return target_abs
+
+    def CreateDepsFileForMsvc(self, DepList):
+        """ Generate dependency files, .deps file from /showIncludes output message """
+        if not DepList:
+            return
+        ModuleDepDict = {}
+        current_source = ""
+        SourceFileAbsPathMap = self.SourceFileList
+        for line in DepList:
+            line = line.strip()
+            if self.HasNamesakeSourceFile:
+                for cc_cmd in self.CcPPCommandPathSet:
+                    if cc_cmd in line:
+                        if '''"'''+cc_cmd+'''"''' in line:
+                            cc_options = line[len(cc_cmd)+2:].split()
+                        else:
+                            cc_options = line[len(cc_cmd):].split()
+                        SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)}
+            if line in SourceFileAbsPathMap:
+                current_source = line
+                if current_source not in ModuleDepDict:
+                    ModuleDepDict[SourceFileAbsPathMap[current_source]] = []
+            elif "Note: including file:" ==  line.lstrip()[:21]:
+                if not current_source:
+                    EdkLogger.error("build",BUILD_ERROR, "Parse /showIncludes output failed. line: %s. \n" % line, RaiseError=False)
+                else:
+                    ModuleDepDict[SourceFileAbsPathMap[current_source]].append(line.lstrip()[22:].strip())
+
+        for source_abs in ModuleDepDict:
+            if ModuleDepDict[source_abs]:
+                target_abs = self.GetRealTarget(source_abs)
+                dep_file_name = os.path.basename(source_abs) + ".deps"
+                SaveFileOnChange(os.path.join(os.path.dirname(target_abs),dep_file_name)," \\\n".join([target_abs+":"] + ['''"''' + item +'''"''' for item in ModuleDepDict[source_abs]]),False)
+
+    def UpdateDepsFileforNonMsvc(self):
+        """ Update .deps files.
+            1. Update target path to absolute path.
+            2. Update middle target to final target.
+        """
+
+        for abspath in self.deps_files:
+            if abspath.endswith(".trim.deps"):
+                continue
+            try:
+                newcontent = []
+                with open(abspath,"r") as fd:
+                    lines = fd.readlines()
+                if lines[-1] == DEP_FILE_TAIL:
+                    continue
+                firstlineitems = lines[0].strip().split(" ")
+
+                if len(firstlineitems) > 2:
+                    sourceitem = firstlineitems[1]
+                else:
+                    sourceitem = lines[1].strip().split(" ")[0]
+
+                source_abs = self.SourceFileList.get(sourceitem,sourceitem)
+                firstlineitems[0] = self.GetRealTarget(source_abs)
+                p_target = firstlineitems
+                if not p_target[0].strip().endswith(":"):
+                    p_target[0] += ": "
+
+                if len(p_target) == 2:
+                    p_target[0] += lines[1]
+                    newcontent.append(p_target[0])
+                    newcontent.extend(lines[2:])
+                else:
+                    line1 = " ".join(p_target).strip()
+                    line1 += "\n"
+                    newcontent.append(line1)
+                    newcontent.extend(lines[1:])
+
+                newcontent.append("\n")
+                newcontent.append(DEP_FILE_TAIL)
+                with open(abspath,"w") as fw:
+                    fw.write("".join(newcontent))
+            except Exception as e:
+                EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
+                continue
+
+    def UpdateDepsFileforTrim(self):
+        """ Update .deps file which generated by trim. """
+
+        for abspath in self.deps_files:
+            if not abspath.endswith(".trim.deps"):
+                continue
+            try:
+                newcontent = []
+                with open(abspath,"r") as fd:
+                    lines = fd.readlines()
+                if lines[-1] == DEP_FILE_TAIL:
+                    continue
+
+                source_abs = lines[0].strip().split(" ")[0]
+                targetitem = self.GetRealTarget(source_abs.strip(" :"))
+
+                targetitem += ": "
+                targetitem += lines[1]
+                newcontent.append(targetitem)
+                newcontent.extend(lines[2:])
+                newcontent.append("\n")
+                newcontent.append(DEP_FILE_TAIL)
+                with open(abspath,"w") as fw:
+                    fw.write("".join(newcontent))
+            except Exception as e:
+                EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
+                continue
diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
index e6d6c43810..1111d5de25 100755
--- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
@@ -1127,12 +1127,35 @@ class ModuleAutoGen(AutoGen):
                 if not self.MetaFile.OriginalPath.Path.startswith(PackageDir):
                     IncludesList = list(set(Package.Includes).difference(set(Package._PrivateIncludes)))
             for Inc in IncludesList:
                 if Inc not in RetVal:
                     RetVal.append(str(Inc))
+        RetVal.extend(self.IncPathFromBuildOptions)
         return RetVal
 
+    @cached_property
+    def IncPathFromBuildOptions(self):
+        IncPathList = []
+        for tool in self.BuildOption:
+            if 'FLAGS' in self.BuildOption[tool]:
+                flags = self.BuildOption[tool]['FLAGS']
+                whitespace = False
+                for flag in flags.split(" "):
+                    flag = flag.strip()
+                    if flag.startswith(("/I","-I")):
+                        if len(flag)>2:
+                            if os.path.exists(flag[2:]):
+                                IncPathList.append(flag[2:])
+                        else:
+                            whitespace = True
+                            continue
+                    if whitespace and flag:
+                        if os.path.exists(flag):
+                            IncPathList.append(flag)
+                            whitespace = False
+        return IncPathList
+
     @cached_property
     def IncludePathLength(self):
         return sum(len(inc)+1 for inc in self.IncludePathList)
 
     ## Get the list of include paths from the packages
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 4b31356a42..6fcfc7ec7a 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -55,11 +55,11 @@ from GenFds.GenFds import GenFds, GenFdsApi
 import multiprocessing as mp
 from multiprocessing import Manager
 from AutoGen.DataPipe import MemoryDataPipe
 from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo
 from GenFds.FdfParser import FdfParser
-
+from AutoGen.IncludesAutoGen import IncludesAutoGen
 
 ## standard targets of build command
 gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
 
 ## build configuration file
@@ -173,33 +173,46 @@ def NormFile(FilePath, Workspace):
 #
 # @param  From      The stream message read from
 # @param  To        The stream message put on
 # @param  ExitFlag  The flag used to indicate stopping reading
 #
-def ReadMessage(From, To, ExitFlag):
+def ReadMessage(From, To, ExitFlag,MemTo=None):
     while True:
         # read one line a time
         Line = From.readline()
         # empty string means "end"
         if Line is not None and Line != b"":
-            To(Line.rstrip().decode(encoding='utf-8', errors='ignore'))
+            LineStr = Line.rstrip().decode(encoding='utf-8', errors='ignore')
+            if MemTo is not None:
+                if "Note: including file:" ==  LineStr.lstrip()[:21]:
+                    MemTo.append(LineStr)
+                else:
+                    To(LineStr)
+                    MemTo.append(LineStr)
+            else:
+                To(LineStr)
         else:
             break
         if ExitFlag.isSet():
             break
 
+class MakeSubProc(Popen):
+    def __init__(self,*args, **argv):
+        super(MakeSubProc,self).__init__(*args, **argv)
+        self.ProcOut = []
+
 ## Launch an external program
 #
 # This method will call subprocess.Popen to execute an external program with
 # given options in specified directory. Because of the dead-lock issue during
 # redirecting output of the external program, threads are used to to do the
 # redirection work.
 #
 # @param  Command               A list or string containing the call of the program
 # @param  WorkingDir            The directory in which the program will be running
 #
-def LaunchCommand(Command, WorkingDir):
+def LaunchCommand(Command, WorkingDir,ModuleAuto = None):
     BeginTime = time.time()
     # if working directory doesn't exist, Popen() will raise an exception
     if not os.path.isdir(WorkingDir):
         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=WorkingDir)
 
@@ -214,23 +227,23 @@ def LaunchCommand(Command, WorkingDir):
 
     Proc = None
     EndOfProcedure = None
     try:
         # launch the command
-        Proc = Popen(Command, stdout=PIPE, stderr=PIPE, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True)
+        Proc = MakeSubProc(Command, stdout=PIPE, stderr=PIPE, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True)
 
         # launch two threads to read the STDOUT and STDERR
         EndOfProcedure = Event()
         EndOfProcedure.clear()
         if Proc.stdout:
-            StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure))
+            StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure,Proc.ProcOut))
             StdOutThread.setName("STDOUT-Redirector")
             StdOutThread.setDaemon(False)
             StdOutThread.start()
 
         if Proc.stderr:
-            StdErrThread = Thread(target=ReadMessage, args=(Proc.stderr, EdkLogger.quiet, EndOfProcedure))
+            StdErrThread = Thread(target=ReadMessage, args=(Proc.stderr, EdkLogger.quiet, EndOfProcedure,Proc.ProcOut))
             StdErrThread.setName("STDERR-Redirector")
             StdErrThread.setDaemon(False)
             StdErrThread.start()
 
         # waiting for program exit
@@ -261,10 +274,19 @@ def LaunchCommand(Command, WorkingDir):
             RespContent = f.read()
             f.close()
             EdkLogger.info(RespContent)
 
         EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir))
+    if ModuleAuto:
+        iau = IncludesAutoGen(WorkingDir,ModuleAuto)
+        if ModuleAuto.ToolChainFamily == TAB_COMPILER_MSFT:
+            iau.CreateDepsFileForMsvc(Proc.ProcOut)
+        else:
+            iau.UpdateDepsFileforNonMsvc()
+        iau.UpdateDepsFileforTrim()
+        iau.CreateModuleDeps()
+        iau.CreateDepsInclude()
     return "%dms" % (int(round((time.time() - BeginTime) * 1000)))
 
 ## The smallest unit that can be built in multi-thread build mode
 #
 # This is the base class of build unit. The "Obj" parameter must provide
@@ -606,11 +628,11 @@ class BuildTask:
     # @param  Command               A list or string contains the call of the command
     # @param  WorkingDir            The directory in which the program will be running
     #
     def _CommandThread(self, Command, WorkingDir):
         try:
-            self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir)
+            self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir,self.BuildItem.BuildObject)
             self.CompleteFlag = True
 
             # Run hash operation post dependency, to account for libs
             if GlobalData.gUseHashCache and self.BuildItem.BuildObject.IsLibrary:
                 HashFile = path.join(self.BuildItem.BuildObject.BuildDir, self.BuildItem.BuildObject.Name + ".hash")
@@ -1274,23 +1296,36 @@ class Build():
             self.BuildModules = []
             return True
 
         # build library
         if Target == 'libraries':
-            for Lib in AutoGenObject.LibraryBuildDirectoryList:
+            DirList = []
+            for Lib in AutoGenObject.LibraryAutoGenList:
+                if not Lib.IsBinaryModule:
+                    DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))
+            for Lib, LibAutoGen in DirList:
                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild']
-                LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
+                LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)
             return True
 
         # build module
         if Target == 'modules':
-            for Lib in AutoGenObject.LibraryBuildDirectoryList:
+            DirList = []
+            for Lib in AutoGenObject.LibraryAutoGenList:
+                if not Lib.IsBinaryModule:
+                    DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))
+            for Lib, LibAutoGen in DirList:
                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild']
-                LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
-            for Mod in AutoGenObject.ModuleBuildDirectoryList:
+                LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)
+
+            DirList = []
+            for ModuleAutoGen in AutoGenObject.ModuleAutoGenList:
+                if not ModuleAutoGen.IsBinaryModule:
+                    DirList.append((os.path.join(AutoGenObject.BuildDir, ModuleAutoGen.BuildDir),ModuleAutoGen))
+            for Mod,ModAutoGen in DirList:
                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, makefile)), 'pbuild']
-                LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
+                LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,ModAutoGen)
             self.CreateAsBuiltInf()
             if GlobalData.gBinCacheDest:
                 self.UpdateBuildCache()
             self.BuildModules = []
             return True
-- 
2.20.1.windows.1