From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=134.134.136.31; helo=mga06.intel.com; envelope-from=bob.c.feng@intel.com; receiver=edk2-devel@lists.01.org Received: from mga06.intel.com (mga06.intel.com [134.134.136.31]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ml01.01.org (Postfix) with ESMTPS id BF43C2095606A for ; Fri, 16 Mar 2018 01:06:44 -0700 (PDT) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga008.jf.intel.com ([10.7.209.65]) by orsmga104.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 16 Mar 2018 01:13:09 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.48,313,1517904000"; d="scan'208";a="25758022" Received: from shwdeopenpsi105.ccr.corp.intel.com ([10.239.9.129]) by orsmga008.jf.intel.com with ESMTP; 16 Mar 2018 01:13:07 -0700 From: BobCF To: edk2-devel@lists.01.org Cc: Bob Feng , Liming Gao Date: Fri, 16 Mar 2018 16:13:04 +0800 Message-Id: <20180316081304.11496-1-bob.c.feng@intel.com> X-Mailer: git-send-email 2.14.3.windows.1 Subject: [Patch] BaseTools: Detect structure pcd header file change. X-BeenThere: edk2-devel@lists.01.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: EDK II Development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 16 Mar 2018 08:06:45 -0000 Detect structure pcd header file change Contributed-under: TianoCore Contribution Agreement 1.1 Signed-off-by: Bob Feng Cc: Liming Gao --- BaseTools/Source/Python/Workspace/DscBuildData.py | 190 ++++++++++++++++------ 1 file changed, 136 insertions(+), 54 deletions(-) diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py index 9c43daca5e..af2f3d39ea 100644 --- a/BaseTools/Source/Python/Workspace/DscBuildData.py +++ b/BaseTools/Source/Python/Workspace/DscBuildData.py @@ -93,10 +93,74 @@ LIBS = $(LIB_PATH)\Common.lib PcdGccMakefile = ''' MAKEROOT ?= $(EDK_TOOLS_PATH)/Source/C LIBS = -lCommon ''' +## Regular expression for finding header file inclusions +from AutoGen.GenMake import gIncludePattern +## Regular expression for matching macro used in header file inclusion +from AutoGen.GenMake import gMacroPattern +from AutoGen.GenMake import gIncludeMacroConversion + +## Find dependencies for one source file +# +# By searching recursively "#include" directive in file, find out all the +# files needed by given source file. The dependecies will be only searched +# in given search path list. +# +# @param SearchPathList The list of search path +# +# @retval list The list of files the given source file depends on +# +def GetDependencyList(FileStack,SearchPathList): + DepDb = dict() + DependencySet = set(FileStack) + while len(FileStack) > 0: + F = FileStack.pop() + FullPathDependList = [] + CurrentFileDependencyList = [] + if F in DepDb: + CurrentFileDependencyList = DepDb[F] + else: + try: + Fd = open(F, 'r') + FileContent = Fd.read() + except BaseException, X: + EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=F + "\n\t" + str(X)) + finally: + if "Fd" in dir(locals()): + Fd.close() + + if len(FileContent) == 0: + continue + + if FileContent[0] == 0xff or FileContent[0] == 0xfe: + FileContent = unicode(FileContent, "utf-16") + IncludedFileList = gIncludePattern.findall(FileContent) + + for Inc in IncludedFileList: + Inc = Inc.strip() + Inc = os.path.normpath(Inc) + CurrentFileDependencyList.append(Inc) + DepDb[F] = CurrentFileDependencyList + + CurrentFilePath = os.path.dirname(F) + PathList = [CurrentFilePath] + SearchPathList + for Inc in CurrentFileDependencyList: + for SearchPath in PathList: + FilePath = os.path.join(SearchPath, Inc) + if not os.path.exists(FilePath): + continue + if FilePath not in DependencySet: + FileStack.append(FilePath) + FullPathDependList.append(FilePath) + break + DependencySet.update(FullPathDependList) + DependencyList = list(DependencySet) # remove duplicate ones + + return DependencyList + class DscBuildData(PlatformBuildClassObject): # dict used to convert PCD type in database to string used by build tool _PCD_TYPE_STRING_ = { MODEL_PCD_FIXED_AT_BUILD : "FixedAtBuild", MODEL_PCD_PATCHABLE_IN_MODULE : "PatchableInModule", @@ -1916,15 +1980,17 @@ class DscBuildData(PlatformBuildClassObject): InitByteValue = "" CApp = PcdMainCHeader Includes = {} + IncludeFiles = set() for PcdName in StructuredPcds: Pcd = StructuredPcds[PcdName] for IncludeFile in Pcd.StructuredPcdIncludeFile: if IncludeFile not in Includes: Includes[IncludeFile] = True + IncludeFiles.add((os.path.dirname(Pcd.PkgPath), IncludeFile)) CApp = CApp + '#include <%s>\n' % (IncludeFile) CApp = CApp + '\n' for PcdName in StructuredPcds: Pcd = StructuredPcds[PcdName] CApp = CApp + self.GenerateSizeFunction(Pcd) @@ -1979,10 +2045,11 @@ class DscBuildData(PlatformBuildClassObject): else: MakeApp = MakeApp + PcdGccMakefile MakeApp = MakeApp + 'APPNAME = %s\n' % (PcdValueInitName) + 'OBJECTS = %s/%s.o\n' % (self.OutputPath, PcdValueInitName) + \ 'include $(MAKEROOT)/Makefiles/app.makefile\n' + 'INCLUDE +=' + IncSearchList = [] PlatformInc = {} for Cache in self._Bdb._CACHE_.values(): if Cache.MetaFile.Ext.lower() != '.dec': continue if Cache.Includes: @@ -2001,10 +2068,11 @@ class DscBuildData(PlatformBuildClassObject): if PlatformInc and PcdDependDEC: for pkg in PcdDependDEC: if pkg in PlatformInc: for inc in PlatformInc[pkg]: MakeApp += '-I' + str(inc) + ' ' + IncSearchList.append(inc) MakeApp = MakeApp + '\n' CC_FLAGS = LinuxCFLAGS if sys.platform == "win32": CC_FLAGS = WindowsCFLAGS @@ -2048,11 +2116,25 @@ class DscBuildData(PlatformBuildClassObject): CC_FLAGS += ' ' + Item MakeApp += CC_FLAGS if sys.platform == "win32": MakeApp = MakeApp + PcdMakefileEnd + IncludeFileFullPaths = [] + for includefile in IncludeFiles: + for includepath in IncSearchList: + if includefile[0] in str(includepath): + includefullpath = os.path.join(str(includepath),includefile[1]) + if os.path.exists(includefullpath): + IncludeFileFullPaths.append(os.path.normpath(includefullpath)) + SearchPathList = [str(item) for item in IncSearchList] + SearchPathList.append(os.path.normpath(os.path.join(self.WorkspaceDir,r"BaseTools\Source\C\Include"))) + SearchPathList.append(os.path.normpath(os.path.join(self.WorkspaceDir,r"BaseTools\Source\C\Common"))) + IncFileList = GetDependencyList(IncludeFileFullPaths,SearchPathList) + for include_file in IncFileList: + MakeApp += "$(OBJECTS) : %s \n" % include_file MakeFileName = os.path.join(self.OutputPath, 'Makefile') + MakeApp += "$(OBJECTS) : %s \n" % MakeFileName SaveFileOnChange(MakeFileName, MakeApp, False) InputValueFile = os.path.join(self.OutputPath, 'Input.txt') OutputValueFile = os.path.join(self.OutputPath, 'Output.txt') SaveFileOnChange(InputValueFile, InitByteValue, False) @@ -2060,63 +2142,65 @@ class DscBuildData(PlatformBuildClassObject): PcdValueInitExe = PcdValueInitName if not sys.platform == "win32": PcdValueInitExe = os.path.join(os.getenv("EDK_TOOLS_PATH"), 'Source', 'C', 'bin', PcdValueInitName) else: PcdValueInitExe = os.path.join(os.getenv("EDK_TOOLS_PATH"), 'Bin', 'Win32', PcdValueInitName) +".exe" - if not os.path.exists(PcdValueInitExe) or self.NeedUpdateOutput(OutputValueFile, CAppBaseFileName + '.c',MakeFileName,InputValueFile): - Messages = '' - if sys.platform == "win32": - MakeCommand = 'nmake clean & nmake -f %s' % (MakeFileName) - returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand) - Messages = StdOut - else: - MakeCommand = 'make clean & make -f %s' % (MakeFileName) - returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand) - Messages = StdErr - Messages = Messages.split('\n') - MessageGroup = [] - if returncode <>0: - CAppBaseFileName = os.path.join(self.OutputPath, PcdValueInitName) - File = open (CAppBaseFileName + '.c', 'r') - FileData = File.readlines() - File.close() - for Message in Messages: - if " error" in Message or "warning" in Message: - FileInfo = Message.strip().split('(') - if len (FileInfo) > 1: - FileName = FileInfo [0] - FileLine = FileInfo [1].split (')')[0] + + Messages = '' + if sys.platform == "win32": + MakeCommand = 'nmake -f %s' % (MakeFileName) + returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand) + Messages = StdOut + else: + MakeCommand = 'make -f %s' % (MakeFileName) + returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand) + Messages = StdErr + Messages = Messages.split('\n') + MessageGroup = [] + if returncode <>0: + CAppBaseFileName = os.path.join(self.OutputPath, PcdValueInitName) + File = open (CAppBaseFileName + '.c', 'r') + FileData = File.readlines() + File.close() + for Message in Messages: + if " error" in Message or "warning" in Message: + FileInfo = Message.strip().split('(') + if len (FileInfo) > 1: + FileName = FileInfo [0] + FileLine = FileInfo [1].split (')')[0] + else: + FileInfo = Message.strip().split(':') + FileName = FileInfo [0] + FileLine = FileInfo [1] + if FileLine.isdigit(): + error_line = FileData[int (FileLine) - 1] + if r"//" in error_line: + c_line,dsc_line = error_line.split(r"//") else: - FileInfo = Message.strip().split(':') - FileName = FileInfo [0] - FileLine = FileInfo [1] - if FileLine.isdigit(): - error_line = FileData[int (FileLine) - 1] - if r"//" in error_line: - c_line,dsc_line = error_line.split(r"//") - else: - dsc_line = error_line - message_itmes = Message.split(":") - Index = 0 - if "PcdValueInit.c" not in Message: - if not MessageGroup: - MessageGroup.append(Message) - break - else: - for item in message_itmes: - if "PcdValueInit.c" in item: - Index = message_itmes.index(item) - message_itmes[Index] = dsc_line.strip() - break - MessageGroup.append(":".join(message_itmes[Index:]).strip()) - continue + dsc_line = error_line + message_itmes = Message.split(":") + Index = 0 + if "PcdValueInit.c" not in Message: + if not MessageGroup: + MessageGroup.append(Message) + break else: - MessageGroup.append(Message) - if MessageGroup: - EdkLogger.error("build", PCD_STRUCTURE_PCD_ERROR, "\n".join(MessageGroup) ) - else: - EdkLogger.error('Build', COMMAND_FAILURE, 'Can not execute command: %s' % MakeCommand) + for item in message_itmes: + if "PcdValueInit.c" in item: + Index = message_itmes.index(item) + message_itmes[Index] = dsc_line.strip() + break + MessageGroup.append(":".join(message_itmes[Index:]).strip()) + continue + else: + MessageGroup.append(Message) + if MessageGroup: + EdkLogger.error("build", PCD_STRUCTURE_PCD_ERROR, "\n".join(MessageGroup) ) + else: + EdkLogger.error('Build', COMMAND_FAILURE, 'Can not execute command: %s' % MakeCommand) + + if self.NeedUpdateOutput(OutputValueFile, PcdValueInitExe ,InputValueFile): Command = PcdValueInitExe + ' -i %s -o %s' % (InputValueFile, OutputValueFile) returncode, StdOut, StdErr = self.ExecuteCommand (Command) if returncode <> 0: EdkLogger.warn('Build', COMMAND_FAILURE, 'Can not collect output from command: %s' % Command) @@ -2129,17 +2213,15 @@ class DscBuildData(PlatformBuildClassObject): PcdValue = Pcd.split ('|') PcdInfo = PcdValue[0].split ('.') StructurePcdSet.append((PcdInfo[0],PcdInfo[1], PcdInfo[2], PcdInfo[3], PcdValue[2].strip())) return StructurePcdSet - def NeedUpdateOutput(self,OutputFile, ValueCFile, MakeFile, StructureInput): + def NeedUpdateOutput(self,OutputFile, ValueCFile, StructureInput): if not os.path.exists(OutputFile): return True if os.stat(OutputFile).st_mtime <= os.stat(ValueCFile).st_mtime: return True - if os.stat(OutputFile).st_mtime <= os.stat(MakeFile).st_mtime: - return True if os.stat(OutputFile).st_mtime <= os.stat(StructureInput).st_mtime: return True return False ## Retrieve dynamic PCD settings -- 2.14.3.windows.1