Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen

["Liming Gao" <liming.gao@intel.com>]

Liming Gao <liming.gao@intel.com>

>-----Original Message-----
>From: Feng, Bob C
>Sent: Thursday, September 05, 2019 10:06 PM
>To: devel@edk2.groups.io; Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming
><liming.gao@intel.com>
>Cc: Shi, Steven <steven.shi@intel.com>
>Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for
>IgnoreAutoGen
>
>Sorry for the incomplete mail.
>
>I verified this patch on Ovmf, Emulator, MinPlatforms and Intel Server
>Platform.
>
>Thanks,
>Bob
>
>-----Original Message-----
>From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Bob Feng
>Sent: Thursday, September 5, 2019 10:04 PM
>To: Gao, Liming <liming.gao@intel.com>; devel@edk2.groups.io
>Cc: Shi, Steven <steven.shi@intel.com>
>Subject: Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for
>IgnoreAutoGen
>
>Liming,
>
>Yes, this change only impact -u option.
>
>I separate _MultiThreadBuildPlatform function into two functions and create 2
>new functions specially for IgnoreAutoGen, so the there looks  big change for
>build.py in this patch. I verified this patch on Ovmf, E
>
>-Bob
>
>-----Original Message-----
>From: Gao, Liming
>Sent: Thursday, September 5, 2019 9:12 PM
>To: devel@edk2.groups.io; Feng, Bob C <bob.c.feng@intel.com>
>Cc: Shi, Steven <steven.shi@intel.com>
>Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for
>IgnoreAutoGen
>
>Bob:
> Does this change only impact -u option behavior?
>
>Thanks
>Liming
>> -----Original Message-----
>> From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of
>> Bob Feng
>> Sent: Thursday, September 5, 2019 5:05 PM
>> To: devel@edk2.groups.io
>> Cc: Gao, Liming <liming.gao@intel.com>; Shi, Steven
>> <steven.shi@intel.com>; Feng, Bob C <bob.c.feng@intel.com>
>> Subject: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for
>> IgnoreAutoGen
>>
>> https://bugzilla.tianocore.org/show_bug.cgi?id=2080
>>
>> This patch is to improve build -u option to re-use
>> GlobalVar_<platformguid>_<arch>.bin file which is introduced by
>> multiple-process-autogen feature.
>>
>> Cc: Liming Gao <liming.gao@intel.com>
>> Cc: Steven Shi <steven.shi@intel.com>
>> Signed-off-by: Bob Feng <bob.c.feng@intel.com>
>> ---
>>  .../Source/Python/AutoGen/AutoGenWorker.py    |   2 +-
>>  .../Source/Python/AutoGen/ModuleAutoGen.py    |   4 +-
>>  .../Python/AutoGen/ModuleAutoGenHelper.py     |  29 ++
>>  .../Source/Python/AutoGen/PlatformAutoGen.py  |   2 +
>>  .../Source/Python/AutoGen/WorkspaceAutoGen.py |   1 -
>>  BaseTools/Source/Python/build/build.py        | 355 +++++++++++-------
>>  6 files changed, 255 insertions(+), 138 deletions(-)
>>
>> diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
>> b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
>> index 2e68538b1cb4..f488ae9d5f80 100755
>> --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
>> +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
>> @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process):
>>                          Ma.ReferenceModules =
>Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
>>                  if GlobalData.gBinCacheSource and CommandTarget in [None, "",
>"all"]:
>>                      Ma.GenModuleFilesHash(GlobalData.gCacheIR)
>>                      Ma.GenPreMakefileHash(GlobalData.gCacheIR)
>>                      if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
>> -                       continue
>> +                        continue
>>
>>                  Ma.CreateCodeFile(False)
>>
>> Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File,
>> Ma.Arch),[]))
>>
>>                  if GlobalData.gBinCacheSource and CommandTarget in [None, "",
>"all"]:
>> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
>> b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
>> index 3bb7e91154ac..5f28681e3146 100755
>> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
>> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
>> @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen):
>>             not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
>>              self.GenModuleFilesHash(gDict)
>>
>>          if not (self.MetaFile.Path, self.Arch) in gDict or \
>>             not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
>> -           EdkLogger.quiet("[cache warning]: Cannot generate
>ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
>> self.Arch))
>> -           return
>> +            EdkLogger.quiet("[cache warning]: Cannot generate
>> + ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
>> self.Arch))
>> +            return
>>
>>          # Initialze hash object
>>          m = hashlib.md5()
>>
>>          # Add Platform level hash
>> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
>> b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
>> index c7591253debd..9dd93b9beb12 100644
>> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
>> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
>> @@ -234,22 +234,51 @@ class AutoGenInfo(object):
>>  #
>>  #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in
>> multiple thread  #  class WorkSpaceInfo(AutoGenInfo):
>>      def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch):
>> +        if not hasattr(self, "_Init"):
>> +            self.do_init(Workspace, MetaFile, Target, ToolChain, Arch)
>> +            self._Init = True
>> +    def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch):
>>          self._SrcTimeStamp = 0
>>          self.Db = BuildDB
>>          self.BuildDatabase = self.Db.BuildObject
>>          self.Target = Target
>>          self.ToolChain = ToolChain
>>          self.WorkspaceDir = Workspace
>>          self.ActivePlatform = MetaFile
>>          self.ArchList = Arch
>> +        self.AutoGenObjectList = []
>> +    @property
>> +    def BuildDir(self):
>> +        return self.AutoGenObjectList[0].BuildDir
>>
>> +    @property
>> +    def Name(self):
>> +        return self.AutoGenObjectList[0].Platform.PlatformName
>> +
>> +    @property
>> +    def FlashDefinition(self):
>> +        return self.AutoGenObjectList[0].Platform.FlashDefinition
>> +    @property
>> +    def GenFdsCommandDict(self):
>> +        FdsCommandDict =
>self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict")
>> +        if FdsCommandDict:
>> +            return FdsCommandDict
>> +        return {}
>> +
>> +    @cached_property
>> +    def FvDir(self):
>> +        return os.path.join(self.BuildDir, TAB_FV_DIRECTORY)
>>
>>  class PlatformInfo(AutoGenInfo):
>>      def __init__(self, Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
>> +        if not hasattr(self, "_Init"):
>> +            self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe)
>> +            self._Init = True
>> +    def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
>>          self.Wa = Workspace
>>          self.WorkspaceDir = self.Wa.WorkspaceDir
>>          self.MetaFile = MetaFile
>>          self.Arch = Arch
>>          self.Target = Target
>> diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
>> b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
>> index 565424a95ead..1e17b6687129 100644
>> --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
>> +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
>> @@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen):
>>                  key = (Ma.MetaFile.File, self.Arch)
>>                  if key in FfsCommand:
>>                      Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key])
>>                  else:
>>                      Ma.CreateMakeFile(CreateModuleMakeFile)
>> +        self.CreateLibModuelDirs()
>>
>> +    def CreateLibModuelDirs(self):
>>          # no need to create makefile for the platform more than once
>>          if self.IsMakeFileCreated:
>>              return
>>
>>          # create library/module build dirs for platform diff --git
>> a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
>> b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
>> index 24942674721f..f9d2c216169c 100644
>> --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
>> +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
>> @@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen):
>>          self._CheckPcdDefineAndType()
>>
>>          self.CreateBuildOptionsFile()
>>          self.CreatePcdTokenNumberFile()
>>          self.CreateModuleHashInfo()
>> -        GlobalData.gAutoGenPhase = False
>>
>>      #
>>      # Merge Arch
>>      #
>>      def MergeArch(self):
>> diff --git a/BaseTools/Source/Python/build/build.py
>> b/BaseTools/Source/Python/build/build.py
>> index 0406ac314b65..db57c3e526da 100755
>> --- a/BaseTools/Source/Python/build/build.py
>> +++ b/BaseTools/Source/Python/build/build.py
>> @@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import
>> PatchBinaryFile
>>
>>  import Common.GlobalData as GlobalData  from GenFds.GenFds import
>> GenFds, GenFdsApi  import multiprocessing as mp  from multiprocessing
>> import Manager
>> +from AutoGen.DataPipe import MemoryDataPipe from
>> +AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo
>from
>> +GenFds.FdfParser import FdfParser
>>
>>
>>  ## standard targets of build command
>>  gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries',
>> 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
>>
>> @@ -837,11 +840,11 @@ class Build():
>>                  for PcdMa in PcdMaList:
>>                      if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
>>                          PcdMa.GenModuleFilesHash(share_data)
>>                          PcdMa.GenPreMakefileHash(share_data)
>>                          if PcdMa.CanSkipbyPreMakefileCache(share_data):
>> -                           continue
>> +                            continue
>>
>>                      PcdMa.CreateCodeFile(False)
>>                      PcdMa.CreateMakeFile(False,GenFfsList =
>> DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File,
>> PcdMa.Arch),[]))
>>
>>                      if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
>> @@ -1218,18 +1221,28 @@ class Build():
>>              mqueue = mp.Queue()
>>              for m in AutoGenObject.GetAllModuleInfo:
>>                  mqueue.put(m)
>>
>>              AutoGenObject.DataPipe.DataContainer = {"CommandTarget":
>> self.Target}
>> +            AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp":
>AutoGenObject.Workspace._SrcTimeStamp}
>> +            AutoGenObject.CreateLibModuelDirs()
>> +            AutoGenObject.DataPipe.DataContainer =
>{"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
>> +            AutoGenObject.DataPipe.DataContainer =
>{"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
>> +            AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict":
>> + AutoGenObject.Workspace.GenFdsCommandDict}
>>              self.Progress.Start("Generating makefile and code")
>>              data_pipe_file = os.path.join(AutoGenObject.BuildDir,
>> "GlobalVar_%s_%s.bin" %
>> (str(AutoGenObject.Guid),AutoGenObject.Arch))
>>              AutoGenObject.DataPipe.dump(data_pipe_file)
>>              autogen_rt,errorcode = self.StartAutoGen(mqueue,
>> AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList,
>> GlobalData.gCacheIR)
>> +            AutoGenIdFile =
>os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
>> +            with open(AutoGenIdFile,"w") as fw:
>> +                fw.write("Arch=%s\n" %
>"|".join((AutoGenObject.Workspace.ArchList)))
>> +                fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir)
>> +                fw.write("PlatformGuid=%s\n" %
>> + str(AutoGenObject.Guid))
>>              self.Progress.Stop("done!")
>>              if not autogen_rt:
>>                  self.AutoGenMgr.TerminateWorkers()
>> -                self.AutoGenMgr.join(0.1)
>> +                self.AutoGenMgr.join(1)
>>                  raise FatalError(errorcode)
>>              AutoGenObject.CreateCodeFile(False)
>>              AutoGenObject.CreateMakeFile(False)
>>          else:
>>              # always recreate top/platform makefile when clean, just
>> in case of inconsistency @@ -1778,10 +1791,11 @@ class Build():
>>                          self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
>>                      #
>>                      # Save MAP buffer into MAP file.
>>                      #
>>                      self._SaveMapFile (MapBuffer, Wa)
>> +                self.CreateGuidedSectionToolsFile(Wa)
>>
>>      ## Build active module for different build targets, different tool chains
>and different archs
>>      #
>>      def _BuildModule(self):
>>          for BuildTarget in self.BuildTargetList:
>> @@ -1852,13 +1866,13 @@ class Build():
>>
>>                              if GlobalData.gBinCacheSource and self.Target in [None, "",
>"all"]:
>>                                  Ma.GenModuleFilesHash(GlobalData.gCacheIR)
>>                                  Ma.GenPreMakefileHash(GlobalData.gCacheIR)
>>                                  if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
>> -                                   self.HashSkipModules.append(Ma)
>> -                                   EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path,
>Ma.Arch))
>> -                                   continue
>> +                                    self.HashSkipModules.append(Ma)
>> +                                    EdkLogger.quiet("cache hit: %s[%s]" %
>(Ma.MetaFile.Path, Ma.Arch))
>> +                                    continue
>>
>>                              # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run',
>'fds'
>>                              if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
>>                                  # for target which must generate AutoGen code and
>makefile
>>                                  if not self.SkipAutoGen or self.Target == 'genc':
>> @@ -1987,125 +2001,221 @@ class Build():
>>          GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self,
>ArchList, GlobalData)
>>          for Cmd in GenFfsDict:
>>              tmpInf, tmpArch = GenFfsDict[Cmd]
>>              CmdSetDict[tmpInf, tmpArch].add(Cmd)
>>          return CmdSetDict
>> +    def VerifyAutoGenFiles(self):
>> +        AutoGenIdFile =
>os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
>> +        try:
>> +            with open(AutoGenIdFile) as fd:
>> +                lines = fd.readlines()
>> +        except:
>> +            return None
>> +        for line in lines:
>> +            if "Arch" in line:
>> +                ArchList = line.strip().split("=")[1].split("|")
>> +            if "BuildDir" in line:
>> +                BuildDir = line.split("=")[1].strip()
>> +            if "PlatformGuid" in line:
>> +                PlatformGuid = line.split("=")[1].strip()
>> +        GlobalVarList = []
>> +        for arch in ArchList:
>> +            global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" %
>(str(PlatformGuid),arch))
>> +            if not os.path.exists(global_var):
>> +                return None
>> +            GlobalVarList.append(global_var)
>> +        for global_var in GlobalVarList:
>> +            data_pipe = MemoryDataPipe()
>> +            data_pipe.load(global_var)
>> +            target = data_pipe.Get("P_Info").get("Target")
>> +            toolchain = data_pipe.Get("P_Info").get("ToolChain")
>> +            archlist = data_pipe.Get("P_Info").get("ArchList")
>> +            Arch = data_pipe.Get("P_Info").get("Arch")
>> +            active_p = data_pipe.Get("P_Info").get("ActivePlatform")
>> +            workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir")
>> +            PackagesPath = os.getenv("PACKAGES_PATH")
>> +            mws.setWs(workspacedir, PackagesPath)
>> +            LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList")
>> +            ModuleBuildDirectoryList =
>> + data_pipe.Get("ModuleBuildDirectoryList")
>> +
>> +            for m_build_dir in LibraryBuildDirectoryList:
>> +                if not
>os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[Ge
>nMake.gMakeType])):
>> +                    return None
>> +            for m_build_dir in ModuleBuildDirectoryList:
>> +                if not
>os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[Ge
>nMake.gMakeType])):
>> +                    return None
>> +            Wa = WorkSpaceInfo(
>> +                workspacedir,active_p,target,toolchain,archlist
>> +                )
>> +            Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe)
>> +            Wa.AutoGenObjectList.append(Pa)
>> +        return Wa
>> +    def SetupMakeSetting(self,Wa):
>> +        BuildModules = []
>> +        for Pa in Wa.AutoGenObjectList:
>> +            for m in Pa._MbList:
>> +                ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget,
>Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe)
>> +                BuildModules.append(ma)
>> +        fdf_file = Wa.FlashDefinition
>> +        if fdf_file:
>> +            Fdf = FdfParser(fdf_file.Path)
>> +            Fdf.ParseFile()
>> +            GlobalData.gFdfParser = Fdf
>> +            if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:
>> +                FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]
>> +                for FdRegion in FdDict.RegionList:
>> +                    if str(FdRegion.RegionType) is 'FILE' and
>self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):
>> +                        if int(FdRegion.Offset) % 8 != 0:
>> +                            EdkLogger.error("build", FORMAT_INVALID,
>> + 'The VPD Base Address %s must be 8-byte aligned.' %
>> (FdRegion.Offset))
>> +            Wa.FdfProfile = Fdf.Profile
>> +            self.Fdf = Fdf
>> +        else:
>> +            self.Fdf = None
>> +        return BuildModules
>>
>>      ## Build a platform in multi-thread mode
>>      #
>> -    def _MultiThreadBuildPlatform(self):
>> -        SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE
>auto-generated\n', False)
>> -        for BuildTarget in self.BuildTargetList:
>> -            GlobalData.gGlobalDefines['TARGET'] = BuildTarget
>> -            index = 0
>> -            for ToolChain in self.ToolChainList:
>> -                WorkspaceAutoGenTime = time.time()
>> -                GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
>> -                GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
>> -                GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
>> -                index += 1
>> -                Wa = WorkspaceAutoGen(
>> -                        self.WorkspaceDir,
>> -                        self.PlatformFile,
>> -                        BuildTarget,
>> -                        ToolChain,
>> -                        self.ArchList,
>> -                        self.BuildDatabase,
>> -                        self.TargetTxt,
>> -                        self.ToolDef,
>> -                        self.Fdf,
>> -                        self.FdList,
>> -                        self.FvList,
>> -                        self.CapList,
>> -                        self.SkuId,
>> -                        self.UniFlag,
>> -                        self.Progress
>> -                        )
>> -                self.Fdf = Wa.FdfFile
>> -                self.LoadFixAddress = Wa.Platform.LoadFixAddress
>> -                self.BuildReport.AddPlatformReport(Wa)
>> -                Wa.CreateMakeFile(False)
>> +    def PerformAutoGen(self,BuildTarget,ToolChain):
>> +        WorkspaceAutoGenTime = time.time()
>> +        Wa = WorkspaceAutoGen(
>> +                self.WorkspaceDir,
>> +                self.PlatformFile,
>> +                BuildTarget,
>> +                ToolChain,
>> +                self.ArchList,
>> +                self.BuildDatabase,
>> +                self.TargetTxt,
>> +                self.ToolDef,
>> +                self.Fdf,
>> +                self.FdList,
>> +                self.FvList,
>> +                self.CapList,
>> +                self.SkuId,
>> +                self.UniFlag,
>> +                self.Progress
>> +                )
>> +        self.Fdf = Wa.FdfFile
>> +        self.LoadFixAddress = Wa.Platform.LoadFixAddress
>> +        self.BuildReport.AddPlatformReport(Wa)
>> +        Wa.CreateMakeFile(False)
>>
>>                  # Add ffs build to makefile
>> -                CmdListDict = {}
>> -                if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
>> -                    CmdListDict = self._GenFfsCmd(Wa.ArchList)
>> +        CmdListDict = {}
>> +        if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
>> +            CmdListDict = self._GenFfsCmd(Wa.ArchList)
>>
>> -                # Add Platform and Package level hash in share_data for module
>hash calculation later
>> -                if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
>> -                    GlobalData.gCacheIR[('PlatformHash')] =
>GlobalData.gPlatformHash
>> -                    for PkgName in GlobalData.gPackageHash.keys():
>> -                        GlobalData.gCacheIR[(PkgName, 'PackageHash')] =
>GlobalData.gPackageHash[PkgName]
>> +        # Add Platform and Package level hash in share_data for module hash
>calculation later
>> +        if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
>> +            GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
>> +            for PkgName in GlobalData.gPackageHash.keys():
>> +                GlobalData.gCacheIR[(PkgName, 'PackageHash')] =
>> + GlobalData.gPackageHash[PkgName]
>>
>> -                # multi-thread exit flag
>> -                ExitFlag = threading.Event()
>> -                ExitFlag.clear()
>> -                self.AutoGenTime += int(round((time.time() -
>WorkspaceAutoGenTime)))
>> -                self.BuildModules = []
>> -                TotalModules = []
>> -                for Arch in Wa.ArchList:
>> -                    PcdMaList    = []
>> -                    AutoGenStart = time.time()
>> -                    GlobalData.gGlobalDefines['ARCH'] = Arch
>> -                    Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget,
>ToolChain, Arch)
>> -                    if Pa is None:
>> -                        continue
>> -                    ModuleList = []
>> -                    for Inf in Pa.Platform.Modules:
>> -                        ModuleList.append(Inf)
>> +        self.AutoGenTime += int(round((time.time() -
>WorkspaceAutoGenTime)))
>> +        BuildModules = []
>> +        TotalModules = []
>> +        for Arch in Wa.ArchList:
>> +            PcdMaList    = []
>> +            AutoGenStart = time.time()
>> +            GlobalData.gGlobalDefines['ARCH'] = Arch
>> +            Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain,
>Arch)
>> +            if Pa is None:
>> +                continue
>> +            ModuleList = []
>> +            for Inf in Pa.Platform.Modules:
>> +                ModuleList.append(Inf)
>>                      # Add the INF only list in FDF
>> -                    if GlobalData.gFdfParser is not None:
>> -                        for InfName in GlobalData.gFdfParser.Profile.InfList:
>> -                            Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
>> -                            if Inf in Pa.Platform.Modules:
>> -                                continue
>> -                            ModuleList.append(Inf)
>> -                    Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
>> -                    Pa.DataPipe.DataContainer = {"Workspace_timestamp":
>Wa._SrcTimeStamp}
>> -                    Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
>> -                    for Module in ModuleList:
>> +            if GlobalData.gFdfParser is not None:
>> +                for InfName in GlobalData.gFdfParser.Profile.InfList:
>> +                    Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
>> +                    if Inf in Pa.Platform.Modules:
>> +                        continue
>> +                    ModuleList.append(Inf)
>> +            Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
>> +            Pa.DataPipe.DataContainer = {"Workspace_timestamp":
>Wa._SrcTimeStamp}
>> +            Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
>> +            Pa.CreateLibModuelDirs()
>> +            Pa.DataPipe.DataContainer =
>{"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}
>> +            Pa.DataPipe.DataContainer =
>{"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}
>> +            Pa.DataPipe.DataContainer = {"FdsCommandDict":
>Wa.GenFdsCommandDict}
>> +            ModuleCodaFile = {}
>> +            for ma in Pa.ModuleAutoGenList:
>> +
>> + ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFil
>> + e.Path)] = [item.Target for item in
>> ma.CodaTargetList]
>> +            Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}
>> +            for Module in ModuleList:
>>                          # Get ModuleAutoGen object to generate C code file and
>makefile
>> -                        Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain,
>Arch, self.PlatformFile,Pa.DataPipe)
>> +                Ma = ModuleAutoGen(Wa, Module, BuildTarget,
>> + ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
>>
>> -                        if Ma is None:
>> +                if Ma is None:
>> +                    continue
>> +                if Ma.PcdIsDriver:
>> +                    Ma.PlatformInfo = Pa
>> +                    Ma.Workspace = Wa
>> +                    PcdMaList.append(Ma)
>> +                TotalModules.append(Ma)
>> +                # Initialize all modules in tracking to 'FAIL'
>> +                GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
>> +
>> +
>> +            mqueue = mp.Queue()
>> +            for m in Pa.GetAllModuleInfo:
>> +                mqueue.put(m)
>> +            data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" %
>(str(Pa.Guid),Pa.Arch))
>> +            Pa.DataPipe.dump(data_pipe_file)
>> +
>> +            autogen_rt, errorcode = self.StartAutoGen(mqueue,
>> + Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR)
>> +
>> +            # Skip cache hit modules
>> +            if GlobalData.gBinCacheSource:
>> +                for Ma in TotalModules:
>> +                    if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
>> +                        GlobalData.gCacheIR[(Ma.MetaFile.Path,
>Ma.Arch)].PreMakeCacheHit:
>> +                            self.HashSkipModules.append(Ma)
>> +                            continue
>> +                    if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
>> +                        GlobalData.gCacheIR[(Ma.MetaFile.Path,
>Ma.Arch)].MakeCacheHit:
>> +                            self.HashSkipModules.append(Ma)
>>                              continue
>> -                        if Ma.PcdIsDriver:
>> -                            Ma.PlatformInfo = Pa
>> -                            Ma.Workspace = Wa
>> -                            PcdMaList.append(Ma)
>> -                        TotalModules.append(Ma)
>> -                        # Initialize all modules in tracking to 'FAIL'
>> -                        GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
>> +                    BuildModules.append(Ma)
>> +            else:
>> +                BuildModules.extend(TotalModules)
>>
>> -                    mqueue = mp.Queue()
>> -                    for m in Pa.GetAllModuleInfo:
>> -                        mqueue.put(m)
>> -                    data_pipe_file = os.path.join(Pa.BuildDir,
>"GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
>> -                    Pa.DataPipe.dump(data_pipe_file)
>> -                    autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe,
>self.SkipAutoGen, PcdMaList,
>> GlobalData.gCacheIR)
>> +            if not autogen_rt:
>> +                self.AutoGenMgr.TerminateWorkers()
>> +                self.AutoGenMgr.join(1)
>> +                raise FatalError(errorcode)
>> +            self.AutoGenTime += int(round((time.time() - AutoGenStart)))
>> +        AutoGenIdFile =
>os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
>> +        with open(AutoGenIdFile,"w") as fw:
>> +            fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))
>> +            fw.write("BuildDir=%s\n" % Wa.BuildDir)
>> +            fw.write("PlatformGuid=%s\n" %
>str(Wa.AutoGenObjectList[0].Guid))
>> +        self.Progress.Stop("done!")
>> +        return Wa, BuildModules
>>
>> -                    # Skip cache hit modules
>> -                    if GlobalData.gBinCacheSource:
>> -                        for Ma in TotalModules:
>> -                            if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
>> -                                GlobalData.gCacheIR[(Ma.MetaFile.Path,
>Ma.Arch)].PreMakeCacheHit:
>> -                                    self.HashSkipModules.append(Ma)
>> -                                    continue
>> -                            if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
>> -                                GlobalData.gCacheIR[(Ma.MetaFile.Path,
>Ma.Arch)].MakeCacheHit:
>> -                                    self.HashSkipModules.append(Ma)
>> -                                    continue
>> -                            self.BuildModules.append(Ma)
>> +    def _MultiThreadBuildPlatform(self):
>> +        SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE
>auto-generated\n', False)
>> +        for BuildTarget in self.BuildTargetList:
>> +            GlobalData.gGlobalDefines['TARGET'] = BuildTarget
>> +            index = 0
>> +            for ToolChain in self.ToolChainList:
>> +                GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
>> +                GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
>> +                GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
>> +                index += 1
>> +                ExitFlag = threading.Event()
>> +                ExitFlag.clear()
>> +                if self.SkipAutoGen:
>> +                    Wa = self.VerifyAutoGenFiles()
>> +                    if Wa is None:
>> +                        Wa, self.BuildModules =
>> + self.PerformAutoGen(BuildTarget,ToolChain)
>>                      else:
>> -                        self.BuildModules.extend(TotalModules)
>> -
>> -                    if not autogen_rt:
>> -                        self.AutoGenMgr.TerminateWorkers()
>> -                        self.AutoGenMgr.join(0.1)
>> -                        raise FatalError(errorcode)
>> -                self.AutoGenTime += int(round((time.time() - AutoGenStart)))
>> -                self.Progress.Stop("done!")
>> +                        GlobalData.gAutoGenPhase = True
>> +                        self.BuildModules = self.SetupMakeSetting(Wa)
>> +                else:
>> +                    Wa, self.BuildModules =
>self.PerformAutoGen(BuildTarget,ToolChain)
>> +                Pa = Wa.AutoGenObjectList[0]
>> +                GlobalData.gAutoGenPhase = False
>>
>>                  if GlobalData.gBinCacheSource:
>>                      EdkLogger.quiet("Total cache hit driver num: %s,
>> cache miss driver num: %s" % (len(set(self.HashSkipModules)),
>> len(set(self.BuildModules))))
>>                      CacheHitMa = set()
>>                      CacheNotHitMa = set() @@ -2136,18 +2246,17 @@
>> class Build():
>>                          # Start task scheduler
>>                          if not BuildTask.IsOnGoing():
>>
>> BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
>>
>>                      # in case there's an interruption. we need a full version of
>makefile for platform
>> -                    Pa.CreateMakeFile(False)
>> +
>>                      if BuildTask.HasError():
>>                          self.invalidateHash()
>>                          EdkLogger.error("build", BUILD_ERROR, "Failed to build
>module", ExtraData=GlobalData.gBuildingModule)
>>                      self.MakeTime += int(round((time.time() -
>> MakeStart)))
>>
>>                  MakeContiue = time.time()
>> -
>>                  #
>>                  #
>>                  # All modules have been put in build tasks queue. Tell task
>scheduler
>>                  # to exit if all tasks are completed
>>                  #
>> @@ -2175,17 +2284,12 @@ class Build():
>>                          if (Arch == 'IA32' or Arch == 'ARM') and
>> self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >=
>0x100000000:
>>                              EdkLogger.error("build",
>> PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to
>larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
>>                      #
>>                      # Get Module List
>>                      #
>> -                    ModuleList = {}
>> -                    for Pa in Wa.AutoGenObjectList:
>> -                        for Ma in Pa.ModuleAutoGenList:
>> -                            if Ma is None:
>> -                                continue
>> -                            if not Ma.IsLibrary:
>> -                                ModuleList[Ma.Guid.upper()] = Ma
>> +                    ModuleList = {ma.Guid.upper():ma for ma in
>> + self.BuildModules}
>> +
>>                      #
>>                      # Rebase module to the preferred memory address before
>GenFds
>>                      #
>>                      MapBuffer = []
>>                      if self.LoadFixAddress != 0:
>> @@ -2206,33 +2310,17 @@ class Build():
>>                          self.GenFdsTime += int(round((time.time() - GenFdsStart)))
>>                      #
>>                      # Save MAP buffer into MAP file.
>>                      #
>>                      self._SaveMapFile(MapBuffer, Wa)
>> +                self.CreateGuidedSectionToolsFile(Wa)
>>          self.invalidateHash()
>> -
>>      ## Generate GuidedSectionTools.txt in the FV directories.
>>      #
>> -    def CreateGuidedSectionToolsFile(self):
>> +    def CreateGuidedSectionToolsFile(self,Wa):
>>          for BuildTarget in self.BuildTargetList:
>>              for ToolChain in self.ToolChainList:
>> -                Wa = WorkspaceAutoGen(
>> -                        self.WorkspaceDir,
>> -                        self.PlatformFile,
>> -                        BuildTarget,
>> -                        ToolChain,
>> -                        self.ArchList,
>> -                        self.BuildDatabase,
>> -                        self.TargetTxt,
>> -                        self.ToolDef,
>> -                        self.Fdf,
>> -                        self.FdList,
>> -                        self.FvList,
>> -                        self.CapList,
>> -                        self.SkuId,
>> -                        self.UniFlag
>> -                        )
>>                  FvDir = Wa.FvDir
>>                  if not os.path.exists(FvDir):
>>                      continue
>>
>>                  for Arch in self.ArchList:
>> @@ -2285,11 +2373,10 @@ class Build():
>>              if not self.SpawnMode or self.Target not in ["", "all"]:
>>                  self.SpawnMode = False
>>                  self._BuildPlatform()
>>              else:
>>                  self._MultiThreadBuildPlatform()
>> -            self.CreateGuidedSectionToolsFile()
>>          else:
>>              self.SpawnMode = False
>>              self._BuildModule()
>>
>>          if self.Target == 'cleanall':
>> --
>> 2.20.1.windows.1
>>
>>
>>
>
>
>