[PATCH] BaseTools: AutoGen and GenFds share the parser data.

[Zhaozh1x <zhiqiangx.zhao@intel.com>]

https://bugzilla.tianocore.org/show_bug.cgi?id=1288

Currently, AutoGen and GenFds run in different python interpreters. The
parser are duplicated. This patch is going to create new API for GenFds
and have the build to call that API instead of executing GenFds.py. As
such, the GenFds and build can share the parser data.

This patch is expected to save the time of GenFds about 2~3 seconds.
More details will be logged in BZ.

This is the summary measure data generated from python cProfile for
building Ovmf.

Currently:
8379147 function calls (8135450 primitive calls) in 12.580 seconds

After applying this patch:
3428712 function calls (3418881 primitive calls) in 8.944 seconds

Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: ZhiqiangX Zhao <zhiqiangx.zhao@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Cc: Carsey, Jaben <jaben.carsey@intel.com>
Cc: Bob Feng <bob.c.feng@intel.com>
---
 BaseTools/Source/Python/AutoGen/AutoGen.py |   4 +
 BaseTools/Source/Python/GenFds/GenFds.py   | 266 ++++++++++++++++++++++++++++-
 BaseTools/Source/Python/build/build.py     |   4 +-
 3 files changed, 271 insertions(+), 3 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/Python/AutoGen/AutoGen.py
index f3560bfc78..10ce7eb962 100644
--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
@@ -935,6 +935,10 @@ class WorkspaceAutoGen(AutoGen):
     def GenFdsCommand(self):
         return (GenMake.TopLevelMakefile(self)._TEMPLATE_.Replace(GenMake.TopLevelMakefile(self)._TemplateDict)).strip()
 
+    @property
+    def GenFdsCommandDict(self):
+        return GenMake.TopLevelMakefile(self)._TemplateDict
+
     ## Create makefile for the platform and modules in it
     #
     #   @param      CreateDepsMakeFile      Flag indicating if the makefile for
diff --git a/BaseTools/Source/Python/GenFds/GenFds.py b/BaseTools/Source/Python/GenFds/GenFds.py
index 0c8091b798..96158568b0 100644
--- a/BaseTools/Source/Python/GenFds/GenFds.py
+++ b/BaseTools/Source/Python/GenFds/GenFds.py
@@ -35,7 +35,7 @@ from Common.Misc import DirCache, PathClass, GuidStructureStringToGuidString
 from Common.Misc import SaveFileOnChange, ClearDuplicatedInf
 from Common.BuildVersion import gBUILD_VERSION
 from Common.MultipleWorkspace import MultipleWorkspace as mws
-from Common.BuildToolError import FatalError, GENFDS_ERROR, CODE_ERROR, FORMAT_INVALID, RESOURCE_NOT_AVAILABLE, FILE_NOT_FOUND, OPTION_MISSING, FORMAT_NOT_SUPPORTED,OPTION_VALUE_INVALID
+from Common.BuildToolError import FatalError, GENFDS_ERROR, CODE_ERROR, FORMAT_INVALID, RESOURCE_NOT_AVAILABLE, FILE_NOT_FOUND, OPTION_MISSING, FORMAT_NOT_SUPPORTED, OPTION_VALUE_INVALID, PARAMETER_INVALID
 from Workspace.WorkspaceDatabase import WorkspaceDatabase
 
 from .FdfParser import FdfParser, Warning
@@ -716,6 +716,270 @@ class GenFds(object):
             os.remove(GuidXRefFileName)
         GuidXRefFile.close()
 
+    @staticmethod
+    def GenFdsApi(FdsCommandDict, WorkSpaceDataBase):
+        global Workspace
+        Workspace = os.environ.get('WORKSPACE')
+        ArchList = None
+        ReturnCode = 0
+        try:
+            if not os.path.exists(Workspace):
+                EdkLogger.error("GenFds", PARAMETER_INVALID, "WORKSPACE is invalid",
+                                ExtraData="Please use '-w' switch to pass it or set the WORKSPACE environment variable.")
+            else:
+                Workspace = os.path.normcase(Workspace)
+                GenFdsGlobalVariable.WorkSpaceDir = Workspace
+                if 'EDK_SOURCE' in os.environ:
+                    GenFdsGlobalVariable.EdkSourceDir = os.path.normcase(os.environ['EDK_SOURCE'])
+            os.chdir(GenFdsGlobalVariable.WorkSpaceDir)
+
+            # set multiple workspace
+            PackagesPath = os.getenv("PACKAGES_PATH")
+            mws.setWs(GenFdsGlobalVariable.WorkSpaceDir, PackagesPath)
+
+            if FdsCommandDict["fdf_file"]:
+                FdfFilename = FdsCommandDict["fdf_file"][0]
+                FdfFilename = GenFdsGlobalVariable.ReplaceWorkspaceMacro(FdfFilename.Path)
+
+                if FdfFilename[0:2] == '..':
+                    FdfFilename = os.path.realpath(FdfFilename)
+                if not os.path.isabs(FdfFilename):
+                    FdfFilename = mws.join(GenFdsGlobalVariable.WorkSpaceDir, FdfFilename)
+                if not os.path.exists(FdfFilename):
+                    EdkLogger.error("GenFds", FILE_NOT_FOUND, ExtraData=FdfFilename)
+
+                GenFdsGlobalVariable.FdfFile = FdfFilename
+                GenFdsGlobalVariable.FdfFileTimeStamp = os.path.getmtime(FdfFilename)
+            else:
+                EdkLogger.error("GenFds", OPTION_MISSING, "Missing FDF filename")
+
+            if FdsCommandDict["build_target"]:
+                GenFdsGlobalVariable.TargetName = FdsCommandDict["build_target"]
+
+            if FdsCommandDict["toolchain_tag"]:
+                GenFdsGlobalVariable.ToolChainTag = FdsCommandDict["toolchain_tag"]
+
+            if FdsCommandDict["active_platform"]:
+                ActivePlatform = FdsCommandDict["active_platform"]
+                ActivePlatform = GenFdsGlobalVariable.ReplaceWorkspaceMacro(ActivePlatform)
+
+                if ActivePlatform[0:2] == '..':
+                    ActivePlatform = os.path.realpath(ActivePlatform)
+
+                if not os.path.isabs (ActivePlatform):
+                    ActivePlatform = mws.join(GenFdsGlobalVariable.WorkSpaceDir, ActivePlatform)
+
+                if not os.path.exists(ActivePlatform):
+                    EdkLogger.error("GenFds", FILE_NOT_FOUND, "ActivePlatform doesn't exist!")
+            else:
+                EdkLogger.error("GenFds", OPTION_MISSING, "Missing active platform")
+
+            GenFdsGlobalVariable.ActivePlatform = PathClass(NormPath(ActivePlatform))
+
+            if FdsCommandDict["conf_directory"]:
+                # Get alternate Conf location, if it is absolute, then just use the absolute directory name
+                ConfDirectoryPath = os.path.normpath(FdsCommandDict["conf_directory"])
+                if ConfDirectoryPath.startswith('"'):
+                    ConfDirectoryPath = ConfDirectoryPath[1:]
+                if ConfDirectoryPath.endswith('"'):
+                    ConfDirectoryPath = ConfDirectoryPath[:-1]
+                if not os.path.isabs(ConfDirectoryPath):
+                    # Since alternate directory name is not absolute, the alternate directory is located within the WORKSPACE
+                    # This also handles someone specifying the Conf directory in the workspace. Using --conf=Conf
+                    ConfDirectoryPath = os.path.join(GenFdsGlobalVariable.WorkSpaceDir, ConfDirectoryPath)
+            else:
+                if "CONF_PATH" in os.environ:
+                    ConfDirectoryPath = os.path.normcase(os.environ["CONF_PATH"])
+                else:
+                    # Get standard WORKSPACE/Conf, use the absolute path to the WORKSPACE/Conf
+                    ConfDirectoryPath = mws.join(GenFdsGlobalVariable.WorkSpaceDir, 'Conf')
+            GenFdsGlobalVariable.ConfDir = ConfDirectoryPath
+            if not GlobalData.gConfDirectory:
+                GlobalData.gConfDirectory = GenFdsGlobalVariable.ConfDir
+            BuildConfigurationFile = os.path.normpath(os.path.join(ConfDirectoryPath, "target.txt"))
+            if os.path.isfile(BuildConfigurationFile) == True:
+                TargetTxt = TargetTxtClassObject()
+                TargetTxt.LoadTargetTxtFile(BuildConfigurationFile)
+                # if no build target given in command line, get it from target.txt
+                if not GenFdsGlobalVariable.TargetName:
+                    BuildTargetList = TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TARGET]
+                    if len(BuildTargetList) != 1:
+                        EdkLogger.error("GenFds", OPTION_VALUE_INVALID, ExtraData="Only allows one instance for Target.")
+                    GenFdsGlobalVariable.TargetName = BuildTargetList[0]
+
+                # if no tool chain given in command line, get it from target.txt
+                if not GenFdsGlobalVariable.ToolChainTag:
+                    ToolChainList = TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TOOL_CHAIN_TAG]
+                    if ToolChainList is None or len(ToolChainList) == 0:
+                        EdkLogger.error("GenFds", RESOURCE_NOT_AVAILABLE, ExtraData="No toolchain given. Don't know how to build.")
+                    if len(ToolChainList) != 1:
+                        EdkLogger.error("GenFds", OPTION_VALUE_INVALID, ExtraData="Only allows one instance for ToolChain.")
+                    GenFdsGlobalVariable.ToolChainTag = ToolChainList[0]
+            else:
+                EdkLogger.error("GenFds", FILE_NOT_FOUND, ExtraData=BuildConfigurationFile)
+
+            if FdsCommandDict["macro"]:
+                for Pair in FdsCommandDict["macro"]:
+                    if Pair.startswith('"'):
+                        Pair = Pair[1:]
+                    if Pair.endswith('"'):
+                        Pair = Pair[:-1]
+                    List = Pair.split('=')
+                    if len(List) == 2:
+                        if not List[1].strip():
+                            EdkLogger.error("GenFds", OPTION_VALUE_INVALID, ExtraData="No Value given for Macro %s" %List[0])
+                        if List[0].strip() == "EFI_SOURCE":
+                            GlobalData.gEfiSource = List[1].strip()
+                            GlobalData.gGlobalDefines["EFI_SOURCE"] = GlobalData.gEfiSource
+                            continue
+                        elif List[0].strip() == "EDK_SOURCE":
+                            GlobalData.gEdkSource = List[1].strip()
+                            GlobalData.gGlobalDefines["EDK_SOURCE"] = GlobalData.gEdkSource
+                            continue
+                        elif List[0].strip() in ["WORKSPACE", "TARGET", "TOOLCHAIN"]:
+                            GlobalData.gGlobalDefines[List[0].strip()] = List[1].strip()
+                        else:
+                            GlobalData.gCommandLineDefines[List[0].strip()] = List[1].strip()
+                    else:
+                        GlobalData.gCommandLineDefines[List[0].strip()] = "TRUE"
+            os.environ["WORKSPACE"] = Workspace
+
+            # Use the -t and -b option as gGlobalDefines's TOOLCHAIN and TARGET if they are not defined
+            if "TARGET" not in GlobalData.gGlobalDefines:
+                GlobalData.gGlobalDefines["TARGET"] = GenFdsGlobalVariable.TargetName
+            if "TOOLCHAIN" not in GlobalData.gGlobalDefines:
+                GlobalData.gGlobalDefines["TOOLCHAIN"] = GenFdsGlobalVariable.ToolChainTag
+            if "TOOL_CHAIN_TAG" not in GlobalData.gGlobalDefines:
+                GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = GenFdsGlobalVariable.ToolChainTag
+
+            """call Workspace build create database"""
+            BuildWorkSpace = WorkSpaceDataBase
+
+            #
+            # Get files real name in workspace dir
+            #
+            GlobalData.gAllFiles = DirCache(Workspace)
+            GlobalData.gWorkspace = Workspace
+
+            if FdsCommandDict["build_architecture_list"]:
+                ArchList = FdsCommandDict["build_architecture_list"].split(',')
+            else:
+                ArchList = BuildWorkSpace.BuildObject[GenFdsGlobalVariable.ActivePlatform, TAB_COMMON, FdsCommandDict["build_target"], FdsCommandDict["toolchain_tag"]].SupArchList
+
+            TargetArchList = set(BuildWorkSpace.BuildObject[GenFdsGlobalVariable.ActivePlatform, TAB_COMMON, FdsCommandDict["build_target"], FdsCommandDict["toolchain_tag"]].SupArchList) & set(ArchList)
+            if len(TargetArchList) == 0:
+                EdkLogger.error("GenFds", GENFDS_ERROR, "Target ARCH %s not in platform supported ARCH %s" % (str(ArchList), str(BuildWorkSpace.BuildObject[GenFdsGlobalVariable.ActivePlatform, TAB_COMMON].SupArchList)))
+
+            for Arch in ArchList:
+                GenFdsGlobalVariable.OutputDirFromDscDict[Arch] = NormPath(BuildWorkSpace.BuildObject[GenFdsGlobalVariable.ActivePlatform, Arch, FdsCommandDict["build_target"], FdsCommandDict["toolchain_tag"]].OutputDirectory)
+
+            # assign platform name based on last entry in ArchList
+            GenFdsGlobalVariable.PlatformName = BuildWorkSpace.BuildObject[GenFdsGlobalVariable.ActivePlatform, ArchList[-1], FdsCommandDict["build_target"], FdsCommandDict["toolchain_tag"]].PlatformName
+
+            if FdsCommandDict["platform_build_directory"]:
+                OutputDirFromCommandLine = GenFdsGlobalVariable.ReplaceWorkspaceMacro(FdsCommandDict["platform_build_directory"])
+                if not os.path.isabs (OutputDirFromCommandLine):
+                    OutputDirFromCommandLine = os.path.join(GenFdsGlobalVariable.WorkSpaceDir, OutputDirFromCommandLine)
+                for Arch in ArchList:
+                    GenFdsGlobalVariable.OutputDirDict[Arch] = OutputDirFromCommandLine
+            else:
+                for Arch in ArchList:
+                    GenFdsGlobalVariable.OutputDirDict[Arch] = os.path.join(GenFdsGlobalVariable.OutputDirFromDscDict[Arch], GenFdsGlobalVariable.TargetName + '_' + GenFdsGlobalVariable.ToolChainTag)
+
+            for Key in GenFdsGlobalVariable.OutputDirDict:
+                OutputDir = GenFdsGlobalVariable.OutputDirDict[Key]
+                if OutputDir[0:2] == '..':
+                    OutputDir = os.path.realpath(OutputDir)
+
+                if OutputDir[1] != ':':
+                    OutputDir = os.path.join (GenFdsGlobalVariable.WorkSpaceDir, OutputDir)
+
+                if not os.path.exists(OutputDir):
+                    EdkLogger.error("GenFds", FILE_NOT_FOUND, ExtraData=OutputDir)
+                GenFdsGlobalVariable.OutputDirDict[Key] = OutputDir
+
+            """ Parse Fdf file, has to place after build Workspace as FDF may contain macros from DSC file """
+            FdfParserObj = GlobalData.gFdfParser
+
+            if FdfParserObj.CycleReferenceCheck():
+                EdkLogger.error("GenFds", FORMAT_NOT_SUPPORTED, "Cycle Reference Detected in FDF file")
+
+            if FdsCommandDict["fd"]:
+                if FdsCommandDict["fd"][0].upper() in FdfParserObj.Profile.FdDict:
+                    GenFds.OnlyGenerateThisFd = FdsCommandDict["fd"][0]
+                else:
+                    EdkLogger.error("GenFds", OPTION_VALUE_INVALID,
+                                    "No such an FD in FDF file: %s" % FdsCommandDict["fd"][0])
+
+            if FdsCommandDict["fv"]:
+                if FdsCommandDict["fv"][0].upper() in FdfParserObj.Profile.FvDict:
+                    GenFds.OnlyGenerateThisFv = FdsCommandDict["fv"][0]
+                else:
+                    EdkLogger.error("GenFds", OPTION_VALUE_INVALID,
+                                    "No such an FV in FDF file: %s" % FdsCommandDict["fv"][0])
+
+            if FdsCommandDict["cap"]:
+                if FdsCommandDict["cap"][0].upper() in FdfParserObj.Profile.CapsuleDict:
+                    GenFds.OnlyGenerateThisCap = FdsCommandDict["cap"][0]
+                else:
+                    EdkLogger.error("GenFds", OPTION_VALUE_INVALID,
+                                    "No such a Capsule in FDF file: %s" % FdsCommandDict["cap"][0])
+
+            GenFdsGlobalVariable.WorkSpace = BuildWorkSpace
+            if ArchList:
+                GenFdsGlobalVariable.ArchList = ArchList
+
+            # Dsc Build Data will handle Pcd Settings from CommandLine.
+
+            """Modify images from build output if the feature of loading driver at fixed address is on."""
+            if GenFdsGlobalVariable.FixedLoadAddress:
+                GenFds.PreprocessImage(BuildWorkSpace, GenFdsGlobalVariable.ActivePlatform)
+
+            # Record the FV Region info that may specific in the FD
+            if FdfParserObj.Profile.FvDict and FdfParserObj.Profile.FdDict:
+                for FvObj in FdfParserObj.Profile.FvDict.values():
+                    for FdObj in FdfParserObj.Profile.FdDict.values():
+                        for RegionObj in FdObj.RegionList:
+                            if RegionObj.RegionType != BINARY_FILE_TYPE_FV:
+                                continue
+                            for RegionData in RegionObj.RegionDataList:
+                                if FvObj.UiFvName.upper() == RegionData.upper():
+                                    if FvObj.FvRegionInFD:
+                                        if FvObj.FvRegionInFD != RegionObj.Size:
+                                            EdkLogger.error("GenFds", FORMAT_INVALID, "The FV %s's region is specified in multiple FD with different value." %FvObj.UiFvName)
+                                    else:
+                                        FvObj.FvRegionInFD = RegionObj.Size
+                                        RegionObj.BlockInfoOfRegion(FdObj.BlockSizeList, FvObj)
+
+            """Call GenFds"""
+            GenFds.GenFd('', FdfParserObj, BuildWorkSpace, ArchList)
+
+            """Generate GUID cross reference file"""
+            GenFds.GenerateGuidXRefFile(BuildWorkSpace, ArchList, FdfParserObj)
+
+            """Display FV space info."""
+            GenFds.DisplayFvSpaceInfo(FdfParserObj)
+
+        except Warning as X:
+            EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False)
+            ReturnCode = FORMAT_INVALID
+        except FatalError as X:
+            ReturnCode = X.args[0]
+        except:
+            import traceback
+            EdkLogger.error(
+                        "\nPython",
+                        CODE_ERROR,
+                        "Tools code failure",
+                        ExtraData="Please send email to edk2-devel@lists.01.org for help, attaching following call stack trace!\n",
+                        RaiseError=False
+                        )
+            EdkLogger.quiet(traceback.format_exc())
+            ReturnCode = CODE_ERROR
+        finally:
+            ClearDuplicatedInf()
+        return ReturnCode
+
 if __name__ == '__main__':
     r = main()
     ## 0-127 is a safe return range, and 1 is a standard default error
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index d74082fc26..71940b37e3 100644
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -1392,7 +1392,7 @@ class Build():
 
         # genfds
         if Target == 'fds':
-            LaunchCommand(AutoGenObject.GenFdsCommand, AutoGenObject.MakeFileDir)
+            GenFds.GenFdsApi(AutoGenObject.GenFdsCommandDict, self.Db)
             return True
 
         # run
@@ -2136,7 +2136,7 @@ class Build():
                         # Generate FD image if there's a FDF file found
                         #
                         GenFdsStart = time.time()
-                        LaunchCommand(Wa.GenFdsCommand, os.getcwd())
+                        GenFds.GenFdsApi(Wa.GenFdsCommandDict, self.Db)
 
                         #
                         # Create MAP file for all platform FVs after GenFds.
-- 
2.14.1.windows.1