public inbox for devel@edk2.groups.io
 help / color / mirror / Atom feed
* [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen
@ 2019-09-05  9:04 Bob Feng
  2019-09-05 13:11 ` [edk2-devel] " Liming Gao
  0 siblings, 1 reply; 6+ messages in thread
From: Bob Feng @ 2019-09-05  9:04 UTC (permalink / raw)
  To: devel; +Cc: Liming Gao, Steven Shi, Bob Feng

https://bugzilla.tianocore.org/show_bug.cgi?id=2080

This patch is to improve build -u option to re-use
GlobalVar_<platformguid>_<arch>.bin file which is
introduced by multiple-process-autogen feature.

Cc: Liming Gao <liming.gao@intel.com>
Cc: Steven Shi <steven.shi@intel.com>
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
---
 .../Source/Python/AutoGen/AutoGenWorker.py    |   2 +-
 .../Source/Python/AutoGen/ModuleAutoGen.py    |   4 +-
 .../Python/AutoGen/ModuleAutoGenHelper.py     |  29 ++
 .../Source/Python/AutoGen/PlatformAutoGen.py  |   2 +
 .../Source/Python/AutoGen/WorkspaceAutoGen.py |   1 -
 BaseTools/Source/Python/build/build.py        | 355 +++++++++++-------
 6 files changed, 255 insertions(+), 138 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
index 2e68538b1cb4..f488ae9d5f80 100755
--- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
@@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process):
                         Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
                 if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
                     Ma.GenModuleFilesHash(GlobalData.gCacheIR)
                     Ma.GenPreMakefileHash(GlobalData.gCacheIR)
                     if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
-                       continue
+                        continue
 
                 Ma.CreateCodeFile(False)
                 Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))
 
                 if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
index 3bb7e91154ac..5f28681e3146 100755
--- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
@@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen):
            not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
             self.GenModuleFilesHash(gDict)
 
         if not (self.MetaFile.Path, self.Arch) in gDict or \
            not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
-           EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, self.Arch))
-           return
+            EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, self.Arch))
+            return
 
         # Initialze hash object
         m = hashlib.md5()
 
         # Add Platform level hash
diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
index c7591253debd..9dd93b9beb12 100644
--- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
+++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
@@ -234,22 +234,51 @@ class AutoGenInfo(object):
 #
 #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in multiple thread
 #
 class WorkSpaceInfo(AutoGenInfo):
     def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch):
+        if not hasattr(self, "_Init"):
+            self.do_init(Workspace, MetaFile, Target, ToolChain, Arch)
+            self._Init = True
+    def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch):
         self._SrcTimeStamp = 0
         self.Db = BuildDB
         self.BuildDatabase = self.Db.BuildObject
         self.Target = Target
         self.ToolChain = ToolChain
         self.WorkspaceDir = Workspace
         self.ActivePlatform = MetaFile
         self.ArchList = Arch
+        self.AutoGenObjectList = []
+    @property
+    def BuildDir(self):
+        return self.AutoGenObjectList[0].BuildDir
 
+    @property
+    def Name(self):
+        return self.AutoGenObjectList[0].Platform.PlatformName
+
+    @property
+    def FlashDefinition(self):
+        return self.AutoGenObjectList[0].Platform.FlashDefinition
+    @property
+    def GenFdsCommandDict(self):
+        FdsCommandDict = self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict")
+        if FdsCommandDict:
+            return FdsCommandDict
+        return {}
+
+    @cached_property
+    def FvDir(self):
+        return os.path.join(self.BuildDir, TAB_FV_DIRECTORY)
 
 class PlatformInfo(AutoGenInfo):
     def __init__(self, Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
+        if not hasattr(self, "_Init"):
+            self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe)
+            self._Init = True
+    def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
         self.Wa = Workspace
         self.WorkspaceDir = self.Wa.WorkspaceDir
         self.MetaFile = MetaFile
         self.Arch = Arch
         self.Target = Target
diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
index 565424a95ead..1e17b6687129 100644
--- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
@@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen):
                 key = (Ma.MetaFile.File, self.Arch)
                 if key in FfsCommand:
                     Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key])
                 else:
                     Ma.CreateMakeFile(CreateModuleMakeFile)
+        self.CreateLibModuelDirs()
 
+    def CreateLibModuelDirs(self):
         # no need to create makefile for the platform more than once
         if self.IsMakeFileCreated:
             return
 
         # create library/module build dirs for platform
diff --git a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
index 24942674721f..f9d2c216169c 100644
--- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
@@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen):
         self._CheckPcdDefineAndType()
 
         self.CreateBuildOptionsFile()
         self.CreatePcdTokenNumberFile()
         self.CreateModuleHashInfo()
-        GlobalData.gAutoGenPhase = False
 
     #
     # Merge Arch
     #
     def MergeArch(self):
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 0406ac314b65..db57c3e526da 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import PatchBinaryFile
 
 import Common.GlobalData as GlobalData
 from GenFds.GenFds import GenFds, GenFdsApi
 import multiprocessing as mp
 from multiprocessing import Manager
+from AutoGen.DataPipe import MemoryDataPipe
+from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo
+from GenFds.FdfParser import FdfParser
 
 
 ## standard targets of build command
 gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
 
@@ -837,11 +840,11 @@ class Build():
                 for PcdMa in PcdMaList:
                     if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
                         PcdMa.GenModuleFilesHash(share_data)
                         PcdMa.GenPreMakefileHash(share_data)
                         if PcdMa.CanSkipbyPreMakefileCache(share_data):
-                           continue
+                            continue
 
                     PcdMa.CreateCodeFile(False)
                     PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[]))
 
                     if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
@@ -1218,18 +1221,28 @@ class Build():
             mqueue = mp.Queue()
             for m in AutoGenObject.GetAllModuleInfo:
                 mqueue.put(m)
 
             AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}
+            AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}
+            AutoGenObject.CreateLibModuelDirs()
+            AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
+            AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
+            AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": AutoGenObject.Workspace.GenFdsCommandDict}
             self.Progress.Start("Generating makefile and code")
             data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))
             AutoGenObject.DataPipe.dump(data_pipe_file)
             autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR)
+            AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
+            with open(AutoGenIdFile,"w") as fw:
+                fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList)))
+                fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir)
+                fw.write("PlatformGuid=%s\n" % str(AutoGenObject.Guid))
             self.Progress.Stop("done!")
             if not autogen_rt:
                 self.AutoGenMgr.TerminateWorkers()
-                self.AutoGenMgr.join(0.1)
+                self.AutoGenMgr.join(1)
                 raise FatalError(errorcode)
             AutoGenObject.CreateCodeFile(False)
             AutoGenObject.CreateMakeFile(False)
         else:
             # always recreate top/platform makefile when clean, just in case of inconsistency
@@ -1778,10 +1791,11 @@ class Build():
                         self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
                     #
                     # Save MAP buffer into MAP file.
                     #
                     self._SaveMapFile (MapBuffer, Wa)
+                self.CreateGuidedSectionToolsFile(Wa)
 
     ## Build active module for different build targets, different tool chains and different archs
     #
     def _BuildModule(self):
         for BuildTarget in self.BuildTargetList:
@@ -1852,13 +1866,13 @@ class Build():
 
                             if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
                                 Ma.GenModuleFilesHash(GlobalData.gCacheIR)
                                 Ma.GenPreMakefileHash(GlobalData.gCacheIR)
                                 if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
-                                   self.HashSkipModules.append(Ma)
-                                   EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
-                                   continue
+                                    self.HashSkipModules.append(Ma)
+                                    EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
+                                    continue
 
                             # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
                             if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
                                 # for target which must generate AutoGen code and makefile
                                 if not self.SkipAutoGen or self.Target == 'genc':
@@ -1987,125 +2001,221 @@ class Build():
         GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData)
         for Cmd in GenFfsDict:
             tmpInf, tmpArch = GenFfsDict[Cmd]
             CmdSetDict[tmpInf, tmpArch].add(Cmd)
         return CmdSetDict
+    def VerifyAutoGenFiles(self):
+        AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
+        try:
+            with open(AutoGenIdFile) as fd:
+                lines = fd.readlines()
+        except:
+            return None
+        for line in lines:
+            if "Arch" in line:
+                ArchList = line.strip().split("=")[1].split("|")
+            if "BuildDir" in line:
+                BuildDir = line.split("=")[1].strip()
+            if "PlatformGuid" in line:
+                PlatformGuid = line.split("=")[1].strip()
+        GlobalVarList = []
+        for arch in ArchList:
+            global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch))
+            if not os.path.exists(global_var):
+                return None
+            GlobalVarList.append(global_var)
+        for global_var in GlobalVarList:
+            data_pipe = MemoryDataPipe()
+            data_pipe.load(global_var)
+            target = data_pipe.Get("P_Info").get("Target")
+            toolchain = data_pipe.Get("P_Info").get("ToolChain")
+            archlist = data_pipe.Get("P_Info").get("ArchList")
+            Arch = data_pipe.Get("P_Info").get("Arch")
+            active_p = data_pipe.Get("P_Info").get("ActivePlatform")
+            workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir")
+            PackagesPath = os.getenv("PACKAGES_PATH")
+            mws.setWs(workspacedir, PackagesPath)
+            LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList")
+            ModuleBuildDirectoryList = data_pipe.Get("ModuleBuildDirectoryList")
+
+            for m_build_dir in LibraryBuildDirectoryList:
+                if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
+                    return None
+            for m_build_dir in ModuleBuildDirectoryList:
+                if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
+                    return None
+            Wa = WorkSpaceInfo(
+                workspacedir,active_p,target,toolchain,archlist
+                )
+            Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe)
+            Wa.AutoGenObjectList.append(Pa)
+        return Wa
+    def SetupMakeSetting(self,Wa):
+        BuildModules = []
+        for Pa in Wa.AutoGenObjectList:
+            for m in Pa._MbList:
+                ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe)
+                BuildModules.append(ma)
+        fdf_file = Wa.FlashDefinition
+        if fdf_file:
+            Fdf = FdfParser(fdf_file.Path)
+            Fdf.ParseFile()
+            GlobalData.gFdfParser = Fdf
+            if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:
+                FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]
+                for FdRegion in FdDict.RegionList:
+                    if str(FdRegion.RegionType) is 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):
+                        if int(FdRegion.Offset) % 8 != 0:
+                            EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' % (FdRegion.Offset))
+            Wa.FdfProfile = Fdf.Profile
+            self.Fdf = Fdf
+        else:
+            self.Fdf = None
+        return BuildModules
 
     ## Build a platform in multi-thread mode
     #
-    def _MultiThreadBuildPlatform(self):
-        SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
-        for BuildTarget in self.BuildTargetList:
-            GlobalData.gGlobalDefines['TARGET'] = BuildTarget
-            index = 0
-            for ToolChain in self.ToolChainList:
-                WorkspaceAutoGenTime = time.time()
-                GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
-                GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
-                GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
-                index += 1
-                Wa = WorkspaceAutoGen(
-                        self.WorkspaceDir,
-                        self.PlatformFile,
-                        BuildTarget,
-                        ToolChain,
-                        self.ArchList,
-                        self.BuildDatabase,
-                        self.TargetTxt,
-                        self.ToolDef,
-                        self.Fdf,
-                        self.FdList,
-                        self.FvList,
-                        self.CapList,
-                        self.SkuId,
-                        self.UniFlag,
-                        self.Progress
-                        )
-                self.Fdf = Wa.FdfFile
-                self.LoadFixAddress = Wa.Platform.LoadFixAddress
-                self.BuildReport.AddPlatformReport(Wa)
-                Wa.CreateMakeFile(False)
+    def PerformAutoGen(self,BuildTarget,ToolChain):
+        WorkspaceAutoGenTime = time.time()
+        Wa = WorkspaceAutoGen(
+                self.WorkspaceDir,
+                self.PlatformFile,
+                BuildTarget,
+                ToolChain,
+                self.ArchList,
+                self.BuildDatabase,
+                self.TargetTxt,
+                self.ToolDef,
+                self.Fdf,
+                self.FdList,
+                self.FvList,
+                self.CapList,
+                self.SkuId,
+                self.UniFlag,
+                self.Progress
+                )
+        self.Fdf = Wa.FdfFile
+        self.LoadFixAddress = Wa.Platform.LoadFixAddress
+        self.BuildReport.AddPlatformReport(Wa)
+        Wa.CreateMakeFile(False)
 
                 # Add ffs build to makefile
-                CmdListDict = {}
-                if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
-                    CmdListDict = self._GenFfsCmd(Wa.ArchList)
+        CmdListDict = {}
+        if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
+            CmdListDict = self._GenFfsCmd(Wa.ArchList)
 
-                # Add Platform and Package level hash in share_data for module hash calculation later
-                if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
-                    GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
-                    for PkgName in GlobalData.gPackageHash.keys():
-                        GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]
+        # Add Platform and Package level hash in share_data for module hash calculation later
+        if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
+            GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
+            for PkgName in GlobalData.gPackageHash.keys():
+                GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]
 
-                # multi-thread exit flag
-                ExitFlag = threading.Event()
-                ExitFlag.clear()
-                self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
-                self.BuildModules = []
-                TotalModules = []
-                for Arch in Wa.ArchList:
-                    PcdMaList    = []
-                    AutoGenStart = time.time()
-                    GlobalData.gGlobalDefines['ARCH'] = Arch
-                    Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
-                    if Pa is None:
-                        continue
-                    ModuleList = []
-                    for Inf in Pa.Platform.Modules:
-                        ModuleList.append(Inf)
+        self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
+        BuildModules = []
+        TotalModules = []
+        for Arch in Wa.ArchList:
+            PcdMaList    = []
+            AutoGenStart = time.time()
+            GlobalData.gGlobalDefines['ARCH'] = Arch
+            Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
+            if Pa is None:
+                continue
+            ModuleList = []
+            for Inf in Pa.Platform.Modules:
+                ModuleList.append(Inf)
                     # Add the INF only list in FDF
-                    if GlobalData.gFdfParser is not None:
-                        for InfName in GlobalData.gFdfParser.Profile.InfList:
-                            Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
-                            if Inf in Pa.Platform.Modules:
-                                continue
-                            ModuleList.append(Inf)
-                    Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
-                    Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
-                    Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
-                    for Module in ModuleList:
+            if GlobalData.gFdfParser is not None:
+                for InfName in GlobalData.gFdfParser.Profile.InfList:
+                    Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
+                    if Inf in Pa.Platform.Modules:
+                        continue
+                    ModuleList.append(Inf)
+            Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
+            Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
+            Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
+            Pa.CreateLibModuelDirs()
+            Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}
+            Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}
+            Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict}
+            ModuleCodaFile = {}
+            for ma in Pa.ModuleAutoGenList:
+                ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in ma.CodaTargetList]
+            Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}
+            for Module in ModuleList:
                         # Get ModuleAutoGen object to generate C code file and makefile
-                        Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
+                Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
 
-                        if Ma is None:
+                if Ma is None:
+                    continue
+                if Ma.PcdIsDriver:
+                    Ma.PlatformInfo = Pa
+                    Ma.Workspace = Wa
+                    PcdMaList.append(Ma)
+                TotalModules.append(Ma)
+                # Initialize all modules in tracking to 'FAIL'
+                GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
+
+
+            mqueue = mp.Queue()
+            for m in Pa.GetAllModuleInfo:
+                mqueue.put(m)
+            data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
+            Pa.DataPipe.dump(data_pipe_file)
+
+            autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR)
+
+            # Skip cache hit modules
+            if GlobalData.gBinCacheSource:
+                for Ma in TotalModules:
+                    if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
+                        GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:
+                            self.HashSkipModules.append(Ma)
+                            continue
+                    if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
+                        GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:
+                            self.HashSkipModules.append(Ma)
                             continue
-                        if Ma.PcdIsDriver:
-                            Ma.PlatformInfo = Pa
-                            Ma.Workspace = Wa
-                            PcdMaList.append(Ma)
-                        TotalModules.append(Ma)
-                        # Initialize all modules in tracking to 'FAIL'
-                        GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
+                    BuildModules.append(Ma)
+            else:
+                BuildModules.extend(TotalModules)
 
-                    mqueue = mp.Queue()
-                    for m in Pa.GetAllModuleInfo:
-                        mqueue.put(m)
-                    data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
-                    Pa.DataPipe.dump(data_pipe_file)
-                    autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,  GlobalData.gCacheIR)
+            if not autogen_rt:
+                self.AutoGenMgr.TerminateWorkers()
+                self.AutoGenMgr.join(1)
+                raise FatalError(errorcode)
+            self.AutoGenTime += int(round((time.time() - AutoGenStart)))
+        AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
+        with open(AutoGenIdFile,"w") as fw:
+            fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))
+            fw.write("BuildDir=%s\n" % Wa.BuildDir)
+            fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid))
+        self.Progress.Stop("done!")
+        return Wa, BuildModules
 
-                    # Skip cache hit modules
-                    if GlobalData.gBinCacheSource:
-                        for Ma in TotalModules:
-                            if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
-                                GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:
-                                    self.HashSkipModules.append(Ma)
-                                    continue
-                            if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
-                                GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:
-                                    self.HashSkipModules.append(Ma)
-                                    continue
-                            self.BuildModules.append(Ma)
+    def _MultiThreadBuildPlatform(self):
+        SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
+        for BuildTarget in self.BuildTargetList:
+            GlobalData.gGlobalDefines['TARGET'] = BuildTarget
+            index = 0
+            for ToolChain in self.ToolChainList:
+                GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
+                GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
+                GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
+                index += 1
+                ExitFlag = threading.Event()
+                ExitFlag.clear()
+                if self.SkipAutoGen:
+                    Wa = self.VerifyAutoGenFiles()
+                    if Wa is None:
+                        Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
                     else:
-                        self.BuildModules.extend(TotalModules)
-
-                    if not autogen_rt:
-                        self.AutoGenMgr.TerminateWorkers()
-                        self.AutoGenMgr.join(0.1)
-                        raise FatalError(errorcode)
-                self.AutoGenTime += int(round((time.time() - AutoGenStart)))
-                self.Progress.Stop("done!")
+                        GlobalData.gAutoGenPhase = True
+                        self.BuildModules = self.SetupMakeSetting(Wa)
+                else:
+                    Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
+                Pa = Wa.AutoGenObjectList[0]
+                GlobalData.gAutoGenPhase = False
 
                 if GlobalData.gBinCacheSource:
                     EdkLogger.quiet("Total cache hit driver num: %s, cache miss driver num: %s" % (len(set(self.HashSkipModules)), len(set(self.BuildModules))))
                     CacheHitMa = set()
                     CacheNotHitMa = set()
@@ -2136,18 +2246,17 @@ class Build():
                         # Start task scheduler
                         if not BuildTask.IsOnGoing():
                             BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
 
                     # in case there's an interruption. we need a full version of makefile for platform
-                    Pa.CreateMakeFile(False)
+
                     if BuildTask.HasError():
                         self.invalidateHash()
                         EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
                     self.MakeTime += int(round((time.time() - MakeStart)))
 
                 MakeContiue = time.time()
-
                 #
                 #
                 # All modules have been put in build tasks queue. Tell task scheduler
                 # to exit if all tasks are completed
                 #
@@ -2175,17 +2284,12 @@ class Build():
                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
                     #
                     # Get Module List
                     #
-                    ModuleList = {}
-                    for Pa in Wa.AutoGenObjectList:
-                        for Ma in Pa.ModuleAutoGenList:
-                            if Ma is None:
-                                continue
-                            if not Ma.IsLibrary:
-                                ModuleList[Ma.Guid.upper()] = Ma
+                    ModuleList = {ma.Guid.upper():ma for ma in self.BuildModules}
+
                     #
                     # Rebase module to the preferred memory address before GenFds
                     #
                     MapBuffer = []
                     if self.LoadFixAddress != 0:
@@ -2206,33 +2310,17 @@ class Build():
                         self.GenFdsTime += int(round((time.time() - GenFdsStart)))
                     #
                     # Save MAP buffer into MAP file.
                     #
                     self._SaveMapFile(MapBuffer, Wa)
+                self.CreateGuidedSectionToolsFile(Wa)
         self.invalidateHash()
-
     ## Generate GuidedSectionTools.txt in the FV directories.
     #
-    def CreateGuidedSectionToolsFile(self):
+    def CreateGuidedSectionToolsFile(self,Wa):
         for BuildTarget in self.BuildTargetList:
             for ToolChain in self.ToolChainList:
-                Wa = WorkspaceAutoGen(
-                        self.WorkspaceDir,
-                        self.PlatformFile,
-                        BuildTarget,
-                        ToolChain,
-                        self.ArchList,
-                        self.BuildDatabase,
-                        self.TargetTxt,
-                        self.ToolDef,
-                        self.Fdf,
-                        self.FdList,
-                        self.FvList,
-                        self.CapList,
-                        self.SkuId,
-                        self.UniFlag
-                        )
                 FvDir = Wa.FvDir
                 if not os.path.exists(FvDir):
                     continue
 
                 for Arch in self.ArchList:
@@ -2285,11 +2373,10 @@ class Build():
             if not self.SpawnMode or self.Target not in ["", "all"]:
                 self.SpawnMode = False
                 self._BuildPlatform()
             else:
                 self._MultiThreadBuildPlatform()
-            self.CreateGuidedSectionToolsFile()
         else:
             self.SpawnMode = False
             self._BuildModule()
 
         if self.Target == 'cleanall':
-- 
2.20.1.windows.1


^ permalink raw reply related	[flat|nested] 6+ messages in thread
end of thread, other threads:[~2019-09-09  7:15 UTC | newest]

Thread overview: 6+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2019-09-05  9:04 [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen Bob Feng
2019-09-05 13:11 ` [edk2-devel] " Liming Gao
2019-09-05 14:04   ` Bob Feng
     [not found]   ` <15C18FCC38905317.10639@groups.io>
2019-09-05 14:05     ` Bob Feng
2019-09-09  6:58       ` Liming Gao
     [not found]       ` <15C2B2E9C55F0B32.16248@groups.io>
2019-09-09  7:15         ` Liming Gao

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox