From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=192.55.52.93; helo=mga11.intel.com; envelope-from=zhiqiangx.zhao@intel.com; receiver=edk2-devel@lists.01.org Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ml01.01.org (Postfix) with ESMTPS id 6E29021193763 for ; Tue, 20 Nov 2018 22:32:58 -0800 (PST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga001.fm.intel.com ([10.253.24.23]) by fmsmga102.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 20 Nov 2018 22:32:57 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.56,260,1539673200"; d="scan'208";a="109909335" Received: from tiano-zzq.ccr.corp.intel.com ([10.239.49.19]) by fmsmga001.fm.intel.com with ESMTP; 20 Nov 2018 22:32:56 -0800 From: Zhaozh1x To: edk2-devel@lists.01.org Cc: Zhaozh1x , Liming Gao , Carsey, Jaben , Bob Feng Date: Wed, 21 Nov 2018 14:32:53 +0800 Message-Id: <20181121063253.49824-1-zhiqiangx.zhao@intel.com> X-Mailer: git-send-email 2.14.1.windows.1 Subject: [PATCH] BaseTools: AutoGen and GenFds share the parser data. X-BeenThere: edk2-devel@lists.01.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: EDK II Development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 21 Nov 2018 06:32:58 -0000 https://bugzilla.tianocore.org/show_bug.cgi?id=1288 Currently, AutoGen and GenFds run in different python interpreters. The parser are duplicated. This patch is going to create new API for GenFds and have the build to call that API instead of executing GenFds.py. As such, the GenFds and build can share the parser data. This patch is expected to save the time of GenFds about 2~3 seconds. More details will be logged in BZ. This is the summary measure data generated from python cProfile for building Ovmf. Currently: 8379147 function calls (8135450 primitive calls) in 12.580 seconds After applying this patch: 3428712 function calls (3418881 primitive calls) in 8.944 seconds Contributed-under: TianoCore Contribution Agreement 1.1 Signed-off-by: ZhiqiangX Zhao Cc: Liming Gao Cc: Carsey, Jaben Cc: Bob Feng --- BaseTools/Source/Python/AutoGen/AutoGen.py | 4 + BaseTools/Source/Python/GenFds/GenFds.py | 266 ++++++++++++++++++++++++++++- BaseTools/Source/Python/build/build.py | 4 +- 3 files changed, 271 insertions(+), 3 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/Python/AutoGen/AutoGen.py index f3560bfc78..10ce7eb962 100644 --- a/BaseTools/Source/Python/AutoGen/AutoGen.py +++ b/BaseTools/Source/Python/AutoGen/AutoGen.py @@ -935,6 +935,10 @@ class WorkspaceAutoGen(AutoGen): def GenFdsCommand(self): return (GenMake.TopLevelMakefile(self)._TEMPLATE_.Replace(GenMake.TopLevelMakefile(self)._TemplateDict)).strip() + @property + def GenFdsCommandDict(self): + return GenMake.TopLevelMakefile(self)._TemplateDict + ## Create makefile for the platform and modules in it # # @param CreateDepsMakeFile Flag indicating if the makefile for diff --git a/BaseTools/Source/Python/GenFds/GenFds.py b/BaseTools/Source/Python/GenFds/GenFds.py index 0c8091b798..96158568b0 100644 --- a/BaseTools/Source/Python/GenFds/GenFds.py +++ b/BaseTools/Source/Python/GenFds/GenFds.py @@ -35,7 +35,7 @@ from Common.Misc import DirCache, PathClass, GuidStructureStringToGuidString from Common.Misc import SaveFileOnChange, ClearDuplicatedInf from Common.BuildVersion import gBUILD_VERSION from Common.MultipleWorkspace import MultipleWorkspace as mws -from Common.BuildToolError import FatalError, GENFDS_ERROR, CODE_ERROR, FORMAT_INVALID, RESOURCE_NOT_AVAILABLE, FILE_NOT_FOUND, OPTION_MISSING, FORMAT_NOT_SUPPORTED,OPTION_VALUE_INVALID +from Common.BuildToolError import FatalError, GENFDS_ERROR, CODE_ERROR, FORMAT_INVALID, RESOURCE_NOT_AVAILABLE, FILE_NOT_FOUND, OPTION_MISSING, FORMAT_NOT_SUPPORTED, OPTION_VALUE_INVALID, PARAMETER_INVALID from Workspace.WorkspaceDatabase import WorkspaceDatabase from .FdfParser import FdfParser, Warning @@ -716,6 +716,270 @@ class GenFds(object): os.remove(GuidXRefFileName) GuidXRefFile.close() + @staticmethod + def GenFdsApi(FdsCommandDict, WorkSpaceDataBase): + global Workspace + Workspace = os.environ.get('WORKSPACE') + ArchList = None + ReturnCode = 0 + try: + if not os.path.exists(Workspace): + EdkLogger.error("GenFds", PARAMETER_INVALID, "WORKSPACE is invalid", + ExtraData="Please use '-w' switch to pass it or set the WORKSPACE environment variable.") + else: + Workspace = os.path.normcase(Workspace) + GenFdsGlobalVariable.WorkSpaceDir = Workspace + if 'EDK_SOURCE' in os.environ: + GenFdsGlobalVariable.EdkSourceDir = os.path.normcase(os.environ['EDK_SOURCE']) + os.chdir(GenFdsGlobalVariable.WorkSpaceDir) + + # set multiple workspace + PackagesPath = os.getenv("PACKAGES_PATH") + mws.setWs(GenFdsGlobalVariable.WorkSpaceDir, PackagesPath) + + if FdsCommandDict["fdf_file"]: + FdfFilename = FdsCommandDict["fdf_file"][0] + FdfFilename = GenFdsGlobalVariable.ReplaceWorkspaceMacro(FdfFilename.Path) + + if FdfFilename[0:2] == '..': + FdfFilename = os.path.realpath(FdfFilename) + if not os.path.isabs(FdfFilename): + FdfFilename = mws.join(GenFdsGlobalVariable.WorkSpaceDir, FdfFilename) + if not os.path.exists(FdfFilename): + EdkLogger.error("GenFds", FILE_NOT_FOUND, ExtraData=FdfFilename) + + GenFdsGlobalVariable.FdfFile = FdfFilename + GenFdsGlobalVariable.FdfFileTimeStamp = os.path.getmtime(FdfFilename) + else: + EdkLogger.error("GenFds", OPTION_MISSING, "Missing FDF filename") + + if FdsCommandDict["build_target"]: + GenFdsGlobalVariable.TargetName = FdsCommandDict["build_target"] + + if FdsCommandDict["toolchain_tag"]: + GenFdsGlobalVariable.ToolChainTag = FdsCommandDict["toolchain_tag"] + + if FdsCommandDict["active_platform"]: + ActivePlatform = FdsCommandDict["active_platform"] + ActivePlatform = GenFdsGlobalVariable.ReplaceWorkspaceMacro(ActivePlatform) + + if ActivePlatform[0:2] == '..': + ActivePlatform = os.path.realpath(ActivePlatform) + + if not os.path.isabs (ActivePlatform): + ActivePlatform = mws.join(GenFdsGlobalVariable.WorkSpaceDir, ActivePlatform) + + if not os.path.exists(ActivePlatform): + EdkLogger.error("GenFds", FILE_NOT_FOUND, "ActivePlatform doesn't exist!") + else: + EdkLogger.error("GenFds", OPTION_MISSING, "Missing active platform") + + GenFdsGlobalVariable.ActivePlatform = PathClass(NormPath(ActivePlatform)) + + if FdsCommandDict["conf_directory"]: + # Get alternate Conf location, if it is absolute, then just use the absolute directory name + ConfDirectoryPath = os.path.normpath(FdsCommandDict["conf_directory"]) + if ConfDirectoryPath.startswith('"'): + ConfDirectoryPath = ConfDirectoryPath[1:] + if ConfDirectoryPath.endswith('"'): + ConfDirectoryPath = ConfDirectoryPath[:-1] + if not os.path.isabs(ConfDirectoryPath): + # Since alternate directory name is not absolute, the alternate directory is located within the WORKSPACE + # This also handles someone specifying the Conf directory in the workspace. Using --conf=Conf + ConfDirectoryPath = os.path.join(GenFdsGlobalVariable.WorkSpaceDir, ConfDirectoryPath) + else: + if "CONF_PATH" in os.environ: + ConfDirectoryPath = os.path.normcase(os.environ["CONF_PATH"]) + else: + # Get standard WORKSPACE/Conf, use the absolute path to the WORKSPACE/Conf + ConfDirectoryPath = mws.join(GenFdsGlobalVariable.WorkSpaceDir, 'Conf') + GenFdsGlobalVariable.ConfDir = ConfDirectoryPath + if not GlobalData.gConfDirectory: + GlobalData.gConfDirectory = GenFdsGlobalVariable.ConfDir + BuildConfigurationFile = os.path.normpath(os.path.join(ConfDirectoryPath, "target.txt")) + if os.path.isfile(BuildConfigurationFile) == True: + TargetTxt = TargetTxtClassObject() + TargetTxt.LoadTargetTxtFile(BuildConfigurationFile) + # if no build target given in command line, get it from target.txt + if not GenFdsGlobalVariable.TargetName: + BuildTargetList = TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TARGET] + if len(BuildTargetList) != 1: + EdkLogger.error("GenFds", OPTION_VALUE_INVALID, ExtraData="Only allows one instance for Target.") + GenFdsGlobalVariable.TargetName = BuildTargetList[0] + + # if no tool chain given in command line, get it from target.txt + if not GenFdsGlobalVariable.ToolChainTag: + ToolChainList = TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TOOL_CHAIN_TAG] + if ToolChainList is None or len(ToolChainList) == 0: + EdkLogger.error("GenFds", RESOURCE_NOT_AVAILABLE, ExtraData="No toolchain given. Don't know how to build.") + if len(ToolChainList) != 1: + EdkLogger.error("GenFds", OPTION_VALUE_INVALID, ExtraData="Only allows one instance for ToolChain.") + GenFdsGlobalVariable.ToolChainTag = ToolChainList[0] + else: + EdkLogger.error("GenFds", FILE_NOT_FOUND, ExtraData=BuildConfigurationFile) + + if FdsCommandDict["macro"]: + for Pair in FdsCommandDict["macro"]: + if Pair.startswith('"'): + Pair = Pair[1:] + if Pair.endswith('"'): + Pair = Pair[:-1] + List = Pair.split('=') + if len(List) == 2: + if not List[1].strip(): + EdkLogger.error("GenFds", OPTION_VALUE_INVALID, ExtraData="No Value given for Macro %s" %List[0]) + if List[0].strip() == "EFI_SOURCE": + GlobalData.gEfiSource = List[1].strip() + GlobalData.gGlobalDefines["EFI_SOURCE"] = GlobalData.gEfiSource + continue + elif List[0].strip() == "EDK_SOURCE": + GlobalData.gEdkSource = List[1].strip() + GlobalData.gGlobalDefines["EDK_SOURCE"] = GlobalData.gEdkSource + continue + elif List[0].strip() in ["WORKSPACE", "TARGET", "TOOLCHAIN"]: + GlobalData.gGlobalDefines[List[0].strip()] = List[1].strip() + else: + GlobalData.gCommandLineDefines[List[0].strip()] = List[1].strip() + else: + GlobalData.gCommandLineDefines[List[0].strip()] = "TRUE" + os.environ["WORKSPACE"] = Workspace + + # Use the -t and -b option as gGlobalDefines's TOOLCHAIN and TARGET if they are not defined + if "TARGET" not in GlobalData.gGlobalDefines: + GlobalData.gGlobalDefines["TARGET"] = GenFdsGlobalVariable.TargetName + if "TOOLCHAIN" not in GlobalData.gGlobalDefines: + GlobalData.gGlobalDefines["TOOLCHAIN"] = GenFdsGlobalVariable.ToolChainTag + if "TOOL_CHAIN_TAG" not in GlobalData.gGlobalDefines: + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = GenFdsGlobalVariable.ToolChainTag + + """call Workspace build create database""" + BuildWorkSpace = WorkSpaceDataBase + + # + # Get files real name in workspace dir + # + GlobalData.gAllFiles = DirCache(Workspace) + GlobalData.gWorkspace = Workspace + + if FdsCommandDict["build_architecture_list"]: + ArchList = FdsCommandDict["build_architecture_list"].split(',') + else: + ArchList = BuildWorkSpace.BuildObject[GenFdsGlobalVariable.ActivePlatform, TAB_COMMON, FdsCommandDict["build_target"], FdsCommandDict["toolchain_tag"]].SupArchList + + TargetArchList = set(BuildWorkSpace.BuildObject[GenFdsGlobalVariable.ActivePlatform, TAB_COMMON, FdsCommandDict["build_target"], FdsCommandDict["toolchain_tag"]].SupArchList) & set(ArchList) + if len(TargetArchList) == 0: + EdkLogger.error("GenFds", GENFDS_ERROR, "Target ARCH %s not in platform supported ARCH %s" % (str(ArchList), str(BuildWorkSpace.BuildObject[GenFdsGlobalVariable.ActivePlatform, TAB_COMMON].SupArchList))) + + for Arch in ArchList: + GenFdsGlobalVariable.OutputDirFromDscDict[Arch] = NormPath(BuildWorkSpace.BuildObject[GenFdsGlobalVariable.ActivePlatform, Arch, FdsCommandDict["build_target"], FdsCommandDict["toolchain_tag"]].OutputDirectory) + + # assign platform name based on last entry in ArchList + GenFdsGlobalVariable.PlatformName = BuildWorkSpace.BuildObject[GenFdsGlobalVariable.ActivePlatform, ArchList[-1], FdsCommandDict["build_target"], FdsCommandDict["toolchain_tag"]].PlatformName + + if FdsCommandDict["platform_build_directory"]: + OutputDirFromCommandLine = GenFdsGlobalVariable.ReplaceWorkspaceMacro(FdsCommandDict["platform_build_directory"]) + if not os.path.isabs (OutputDirFromCommandLine): + OutputDirFromCommandLine = os.path.join(GenFdsGlobalVariable.WorkSpaceDir, OutputDirFromCommandLine) + for Arch in ArchList: + GenFdsGlobalVariable.OutputDirDict[Arch] = OutputDirFromCommandLine + else: + for Arch in ArchList: + GenFdsGlobalVariable.OutputDirDict[Arch] = os.path.join(GenFdsGlobalVariable.OutputDirFromDscDict[Arch], GenFdsGlobalVariable.TargetName + '_' + GenFdsGlobalVariable.ToolChainTag) + + for Key in GenFdsGlobalVariable.OutputDirDict: + OutputDir = GenFdsGlobalVariable.OutputDirDict[Key] + if OutputDir[0:2] == '..': + OutputDir = os.path.realpath(OutputDir) + + if OutputDir[1] != ':': + OutputDir = os.path.join (GenFdsGlobalVariable.WorkSpaceDir, OutputDir) + + if not os.path.exists(OutputDir): + EdkLogger.error("GenFds", FILE_NOT_FOUND, ExtraData=OutputDir) + GenFdsGlobalVariable.OutputDirDict[Key] = OutputDir + + """ Parse Fdf file, has to place after build Workspace as FDF may contain macros from DSC file """ + FdfParserObj = GlobalData.gFdfParser + + if FdfParserObj.CycleReferenceCheck(): + EdkLogger.error("GenFds", FORMAT_NOT_SUPPORTED, "Cycle Reference Detected in FDF file") + + if FdsCommandDict["fd"]: + if FdsCommandDict["fd"][0].upper() in FdfParserObj.Profile.FdDict: + GenFds.OnlyGenerateThisFd = FdsCommandDict["fd"][0] + else: + EdkLogger.error("GenFds", OPTION_VALUE_INVALID, + "No such an FD in FDF file: %s" % FdsCommandDict["fd"][0]) + + if FdsCommandDict["fv"]: + if FdsCommandDict["fv"][0].upper() in FdfParserObj.Profile.FvDict: + GenFds.OnlyGenerateThisFv = FdsCommandDict["fv"][0] + else: + EdkLogger.error("GenFds", OPTION_VALUE_INVALID, + "No such an FV in FDF file: %s" % FdsCommandDict["fv"][0]) + + if FdsCommandDict["cap"]: + if FdsCommandDict["cap"][0].upper() in FdfParserObj.Profile.CapsuleDict: + GenFds.OnlyGenerateThisCap = FdsCommandDict["cap"][0] + else: + EdkLogger.error("GenFds", OPTION_VALUE_INVALID, + "No such a Capsule in FDF file: %s" % FdsCommandDict["cap"][0]) + + GenFdsGlobalVariable.WorkSpace = BuildWorkSpace + if ArchList: + GenFdsGlobalVariable.ArchList = ArchList + + # Dsc Build Data will handle Pcd Settings from CommandLine. + + """Modify images from build output if the feature of loading driver at fixed address is on.""" + if GenFdsGlobalVariable.FixedLoadAddress: + GenFds.PreprocessImage(BuildWorkSpace, GenFdsGlobalVariable.ActivePlatform) + + # Record the FV Region info that may specific in the FD + if FdfParserObj.Profile.FvDict and FdfParserObj.Profile.FdDict: + for FvObj in FdfParserObj.Profile.FvDict.values(): + for FdObj in FdfParserObj.Profile.FdDict.values(): + for RegionObj in FdObj.RegionList: + if RegionObj.RegionType != BINARY_FILE_TYPE_FV: + continue + for RegionData in RegionObj.RegionDataList: + if FvObj.UiFvName.upper() == RegionData.upper(): + if FvObj.FvRegionInFD: + if FvObj.FvRegionInFD != RegionObj.Size: + EdkLogger.error("GenFds", FORMAT_INVALID, "The FV %s's region is specified in multiple FD with different value." %FvObj.UiFvName) + else: + FvObj.FvRegionInFD = RegionObj.Size + RegionObj.BlockInfoOfRegion(FdObj.BlockSizeList, FvObj) + + """Call GenFds""" + GenFds.GenFd('', FdfParserObj, BuildWorkSpace, ArchList) + + """Generate GUID cross reference file""" + GenFds.GenerateGuidXRefFile(BuildWorkSpace, ArchList, FdfParserObj) + + """Display FV space info.""" + GenFds.DisplayFvSpaceInfo(FdfParserObj) + + except Warning as X: + EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False) + ReturnCode = FORMAT_INVALID + except FatalError as X: + ReturnCode = X.args[0] + except: + import traceback + EdkLogger.error( + "\nPython", + CODE_ERROR, + "Tools code failure", + ExtraData="Please send email to edk2-devel@lists.01.org for help, attaching following call stack trace!\n", + RaiseError=False + ) + EdkLogger.quiet(traceback.format_exc()) + ReturnCode = CODE_ERROR + finally: + ClearDuplicatedInf() + return ReturnCode + if __name__ == '__main__': r = main() ## 0-127 is a safe return range, and 1 is a standard default error diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index d74082fc26..71940b37e3 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -1392,7 +1392,7 @@ class Build(): # genfds if Target == 'fds': - LaunchCommand(AutoGenObject.GenFdsCommand, AutoGenObject.MakeFileDir) + GenFds.GenFdsApi(AutoGenObject.GenFdsCommandDict, self.Db) return True # run @@ -2136,7 +2136,7 @@ class Build(): # Generate FD image if there's a FDF file found # GenFdsStart = time.time() - LaunchCommand(Wa.GenFdsCommand, os.getcwd()) + GenFds.GenFdsApi(Wa.GenFdsCommandDict, self.Db) # # Create MAP file for all platform FVs after GenFds. -- 2.14.1.windows.1