From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=192.55.52.93; helo=mga11.intel.com; envelope-from=liming.gao@intel.com; receiver=edk2-devel@lists.01.org Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ml01.01.org (Postfix) with ESMTPS id 277172246801D for ; Sat, 17 Mar 2018 07:13:40 -0700 (PDT) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga004.fm.intel.com ([10.253.24.48]) by fmsmga102.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 17 Mar 2018 07:20:06 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.48,321,1517904000"; d="scan'208";a="38276430" Received: from fmsmsx106.amr.corp.intel.com ([10.18.124.204]) by fmsmga004.fm.intel.com with ESMTP; 17 Mar 2018 07:20:05 -0700 Received: from FMSMSX110.amr.corp.intel.com (10.18.116.10) by FMSMSX106.amr.corp.intel.com (10.18.124.204) with Microsoft SMTP Server (TLS) id 14.3.319.2; Sat, 17 Mar 2018 07:20:06 -0700 Received: from shsmsx101.ccr.corp.intel.com (10.239.4.153) by fmsmsx110.amr.corp.intel.com (10.18.116.10) with Microsoft SMTP Server (TLS) id 14.3.319.2; Sat, 17 Mar 2018 07:20:03 -0700 Received: from shsmsx104.ccr.corp.intel.com ([169.254.5.226]) by SHSMSX101.ccr.corp.intel.com ([169.254.1.166]) with mapi id 14.03.0319.002; Sat, 17 Mar 2018 22:20:01 +0800 From: "Gao, Liming" To: "Feng, Bob C" , "edk2-devel@lists.01.org" Thread-Topic: [edk2] [Patch] BaseTools: Detect structure pcd header file change. Thread-Index: AQHTvQrOT2mK0jCsyU2o/Q3U6caff6PUe/YQ Date: Sat, 17 Mar 2018 14:20:01 +0000 Message-ID: <4A89E2EF3DFEDB4C8BFDE51014F606A14E1E8529@SHSMSX104.ccr.corp.intel.com> References: <20180316094000.25876-1-bob.c.feng@intel.com> In-Reply-To: <20180316094000.25876-1-bob.c.feng@intel.com> Accept-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: dlp-product: dlpe-windows dlp-version: 11.0.0.116 dlp-reaction: no-action x-originating-ip: [10.239.127.40] MIME-Version: 1.0 Subject: Re: [Patch] BaseTools: Detect structure pcd header file change. X-BeenThere: edk2-devel@lists.01.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: EDK II Development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 17 Mar 2018 14:13:42 -0000 Content-Language: en-US Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable Reviewed-by: Liming Gao > -----Original Message----- > From: edk2-devel [mailto:edk2-devel-bounces@lists.01.org] On Behalf Of Bo= bCF > Sent: Friday, March 16, 2018 5:40 PM > To: edk2-devel@lists.01.org > Cc: Gao, Liming > Subject: [edk2] [Patch] BaseTools: Detect structure pcd header file chang= e. >=20 > Detect structure pcd header file change >=20 > Contributed-under: TianoCore Contribution Agreement 1.1 > Signed-off-by: Bob Feng > Cc: Liming Gao > --- > BaseTools/Source/Python/Workspace/DscBuildData.py | 188 +++++++++++++++-= ------ > 1 file changed, 134 insertions(+), 54 deletions(-) >=20 > diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTool= s/Source/Python/Workspace/DscBuildData.py > index 9c43daca5e..9a1ed1b6e4 100644 > --- a/BaseTools/Source/Python/Workspace/DscBuildData.py > +++ b/BaseTools/Source/Python/Workspace/DscBuildData.py > @@ -93,10 +93,71 @@ LIBS =3D $(LIB_PATH)\Common.lib > PcdGccMakefile =3D ''' > MAKEROOT ?=3D $(EDK_TOOLS_PATH)/Source/C > LIBS =3D -lCommon > ''' >=20 > +## Regular expression for finding header file inclusions > +from AutoGen.GenMake import gIncludePattern > + > +## Find dependencies for one source file > +# > +# By searching recursively "#include" directive in file, find out all t= he > +# files needed by given source file. The dependecies will be only searc= hed > +# in given search path list. > +# > +# @param SearchPathList The list of search path > +# > +# @retval list The list of files the given source file = depends on > +# > +def GetDependencyList(FileStack,SearchPathList): > + DepDb =3D dict() > + DependencySet =3D set(FileStack) > + while len(FileStack) > 0: > + F =3D FileStack.pop() > + FullPathDependList =3D [] > + CurrentFileDependencyList =3D [] > + if F in DepDb: > + CurrentFileDependencyList =3D DepDb[F] > + else: > + try: > + Fd =3D open(F, 'r') > + FileContent =3D Fd.read() > + except BaseException, X: > + EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=3D= F + "\n\t" + str(X)) > + finally: > + if "Fd" in dir(locals()): > + Fd.close() > + > + if len(FileContent) =3D=3D 0: > + continue > + > + if FileContent[0] =3D=3D 0xff or FileContent[0] =3D=3D 0xfe: > + FileContent =3D unicode(FileContent, "utf-16") > + IncludedFileList =3D gIncludePattern.findall(FileContent) > + > + for Inc in IncludedFileList: > + Inc =3D Inc.strip() > + Inc =3D os.path.normpath(Inc) > + CurrentFileDependencyList.append(Inc) > + DepDb[F] =3D CurrentFileDependencyList > + > + CurrentFilePath =3D os.path.dirname(F) > + PathList =3D [CurrentFilePath] + SearchPathList > + for Inc in CurrentFileDependencyList: > + for SearchPath in PathList: > + FilePath =3D os.path.join(SearchPath, Inc) > + if not os.path.exists(FilePath): > + continue > + if FilePath not in DependencySet: > + FileStack.append(FilePath) > + FullPathDependList.append(FilePath) > + break > + DependencySet.update(FullPathDependList) > + DependencyList =3D list(DependencySet) # remove duplicate ones > + > + return DependencyList > + > class DscBuildData(PlatformBuildClassObject): > # dict used to convert PCD type in database to string used by build = tool > _PCD_TYPE_STRING_ =3D { > MODEL_PCD_FIXED_AT_BUILD : "FixedAtBuild", > MODEL_PCD_PATCHABLE_IN_MODULE : "PatchableInModule", > @@ -1916,15 +1977,17 @@ class DscBuildData(PlatformBuildClassObject): >=20 > InitByteValue =3D "" > CApp =3D PcdMainCHeader >=20 > Includes =3D {} > + IncludeFiles =3D set() > for PcdName in StructuredPcds: > Pcd =3D StructuredPcds[PcdName] > for IncludeFile in Pcd.StructuredPcdIncludeFile: > if IncludeFile not in Includes: > Includes[IncludeFile] =3D True > + IncludeFiles.add((os.path.dirname(Pcd.PkgPath), Incl= udeFile)) > CApp =3D CApp + '#include <%s>\n' % (IncludeFile) > CApp =3D CApp + '\n' > for PcdName in StructuredPcds: > Pcd =3D StructuredPcds[PcdName] > CApp =3D CApp + self.GenerateSizeFunction(Pcd) > @@ -1979,10 +2042,11 @@ class DscBuildData(PlatformBuildClassObject): > else: > MakeApp =3D MakeApp + PcdGccMakefile > MakeApp =3D MakeApp + 'APPNAME =3D %s\n' % (PcdValueInitName= ) + 'OBJECTS =3D %s/%s.o\n' % (self.OutputPath, > PcdValueInitName) + \ > 'include $(MAKEROOT)/Makefiles/app.makefile\n' + '= INCLUDE +=3D' >=20 > + IncSearchList =3D [] > PlatformInc =3D {} > for Cache in self._Bdb._CACHE_.values(): > if Cache.MetaFile.Ext.lower() !=3D '.dec': > continue > if Cache.Includes: > @@ -2001,10 +2065,11 @@ class DscBuildData(PlatformBuildClassObject): > if PlatformInc and PcdDependDEC: > for pkg in PcdDependDEC: > if pkg in PlatformInc: > for inc in PlatformInc[pkg]: > MakeApp +=3D '-I' + str(inc) + ' ' > + IncSearchList.append(inc) > MakeApp =3D MakeApp + '\n' >=20 > CC_FLAGS =3D LinuxCFLAGS > if sys.platform =3D=3D "win32": > CC_FLAGS =3D WindowsCFLAGS > @@ -2048,11 +2113,26 @@ class DscBuildData(PlatformBuildClassObject): > CC_FLAGS +=3D ' ' + Item > MakeApp +=3D CC_FLAGS >=20 > if sys.platform =3D=3D "win32": > MakeApp =3D MakeApp + PcdMakefileEnd > + IncludeFileFullPaths =3D [] > + for includefile in IncludeFiles: > + for includepath in IncSearchList: > + includefullpath =3D os.path.join(str(includepath),includ= efile[1]) > + if os.path.exists(includefullpath): > + IncludeFileFullPaths.append(os.path.normpath(include= fullpath)) > + break > + SearchPathList =3D [] > + SearchPathList.append(os.path.normpath(os.path.join(self.Workspa= ceDir,r"BaseTools\Source\C\Include"))) > + SearchPathList.append(os.path.normpath(os.path.join(self.Workspa= ceDir,r"BaseTools\Source\C\Common"))) > + SearchPathList.extend([str(item) for item in IncSearchList]) > + IncFileList =3D GetDependencyList(IncludeFileFullPaths,SearchPat= hList) > + for include_file in IncFileList: > + MakeApp +=3D "$(OBJECTS) : %s \n" % include_file > MakeFileName =3D os.path.join(self.OutputPath, 'Makefile') > + MakeApp +=3D "$(OBJECTS) : %s \n" % MakeFileName > SaveFileOnChange(MakeFileName, MakeApp, False) >=20 > InputValueFile =3D os.path.join(self.OutputPath, 'Input.txt') > OutputValueFile =3D os.path.join(self.OutputPath, 'Output.txt') > SaveFileOnChange(InputValueFile, InitByteValue, False) > @@ -2060,63 +2140,65 @@ class DscBuildData(PlatformBuildClassObject): > PcdValueInitExe =3D PcdValueInitName > if not sys.platform =3D=3D "win32": > PcdValueInitExe =3D os.path.join(os.getenv("EDK_TOOLS_PATH")= , 'Source', 'C', 'bin', PcdValueInitName) > else: > PcdValueInitExe =3D os.path.join(os.getenv("EDK_TOOLS_PATH")= , 'Bin', 'Win32', PcdValueInitName) +".exe" > - if not os.path.exists(PcdValueInitExe) or self.NeedUpdateOutput(= OutputValueFile, CAppBaseFileName + > '.c',MakeFileName,InputValueFile): > - Messages =3D '' > - if sys.platform =3D=3D "win32": > - MakeCommand =3D 'nmake clean & nmake -f %s' % (MakeFileN= ame) > - returncode, StdOut, StdErr =3D self.ExecuteCommand (Make= Command) > - Messages =3D StdOut > - else: > - MakeCommand =3D 'make clean & make -f %s' % (MakeFileNam= e) > - returncode, StdOut, StdErr =3D self.ExecuteCommand (Make= Command) > - Messages =3D StdErr > - Messages =3D Messages.split('\n') > - MessageGroup =3D [] > - if returncode <>0: > - CAppBaseFileName =3D os.path.join(self.OutputPath, PcdVa= lueInitName) > - File =3D open (CAppBaseFileName + '.c', 'r') > - FileData =3D File.readlines() > - File.close() > - for Message in Messages: > - if " error" in Message or "warning" in Message: > - FileInfo =3D Message.strip().split('(') > - if len (FileInfo) > 1: > - FileName =3D FileInfo [0] > - FileLine =3D FileInfo [1].split (')')[0] > + > + Messages =3D '' > + if sys.platform =3D=3D "win32": > + MakeCommand =3D 'nmake -f %s' % (MakeFileName) > + returncode, StdOut, StdErr =3D self.ExecuteCommand (MakeComm= and) > + Messages =3D StdOut > + else: > + MakeCommand =3D 'make -f %s' % (MakeFileName) > + returncode, StdOut, StdErr =3D self.ExecuteCommand (MakeComm= and) > + Messages =3D StdErr > + Messages =3D Messages.split('\n') > + MessageGroup =3D [] > + if returncode <>0: > + CAppBaseFileName =3D os.path.join(self.OutputPath, PcdValueI= nitName) > + File =3D open (CAppBaseFileName + '.c', 'r') > + FileData =3D File.readlines() > + File.close() > + for Message in Messages: > + if " error" in Message or "warning" in Message: > + FileInfo =3D Message.strip().split('(') > + if len (FileInfo) > 1: > + FileName =3D FileInfo [0] > + FileLine =3D FileInfo [1].split (')')[0] > + else: > + FileInfo =3D Message.strip().split(':') > + FileName =3D FileInfo [0] > + FileLine =3D FileInfo [1] > + if FileLine.isdigit(): > + error_line =3D FileData[int (FileLine) - 1] > + if r"//" in error_line: > + c_line,dsc_line =3D error_line.split(r"//") > else: > - FileInfo =3D Message.strip().split(':') > - FileName =3D FileInfo [0] > - FileLine =3D FileInfo [1] > - if FileLine.isdigit(): > - error_line =3D FileData[int (FileLine) - 1] > - if r"//" in error_line: > - c_line,dsc_line =3D error_line.split(r"/= /") > - else: > - dsc_line =3D error_line > - message_itmes =3D Message.split(":") > - Index =3D 0 > - if "PcdValueInit.c" not in Message: > - if not MessageGroup: > - MessageGroup.append(Message) > - break > - else: > - for item in message_itmes: > - if "PcdValueInit.c" in item: > - Index =3D message_itmes.index(it= em) > - message_itmes[Index] =3D dsc_lin= e.strip() > - break > - MessageGroup.append(":".join(message_itm= es[Index:]).strip()) > - continue > + dsc_line =3D error_line > + message_itmes =3D Message.split(":") > + Index =3D 0 > + if "PcdValueInit.c" not in Message: > + if not MessageGroup: > + MessageGroup.append(Message) > + break > else: > - MessageGroup.append(Message) > - if MessageGroup: > - EdkLogger.error("build", PCD_STRUCTURE_PCD_ERROR, "\= n".join(MessageGroup) ) > - else: > - EdkLogger.error('Build', COMMAND_FAILURE, 'Can not e= xecute command: %s' % MakeCommand) > + for item in message_itmes: > + if "PcdValueInit.c" in item: > + Index =3D message_itmes.index(item) > + message_itmes[Index] =3D dsc_line.st= rip() > + break > + MessageGroup.append(":".join(message_itmes[I= ndex:]).strip()) > + continue > + else: > + MessageGroup.append(Message) > + if MessageGroup: > + EdkLogger.error("build", PCD_STRUCTURE_PCD_ERROR, "\n".j= oin(MessageGroup) ) > + else: > + EdkLogger.error('Build', COMMAND_FAILURE, 'Can not execu= te command: %s' % MakeCommand) > + > + if self.NeedUpdateOutput(OutputValueFile, PcdValueInitExe ,Input= ValueFile): > Command =3D PcdValueInitExe + ' -i %s -o %s' % (InputValueFi= le, OutputValueFile) > returncode, StdOut, StdErr =3D self.ExecuteCommand (Command) > if returncode <> 0: > EdkLogger.warn('Build', COMMAND_FAILURE, 'Can not collec= t output from command: %s' % Command) >=20 > @@ -2129,17 +2211,15 @@ class DscBuildData(PlatformBuildClassObject): > PcdValue =3D Pcd.split ('|') > PcdInfo =3D PcdValue[0].split ('.') > StructurePcdSet.append((PcdInfo[0],PcdInfo[1], PcdInfo[2], P= cdInfo[3], PcdValue[2].strip())) > return StructurePcdSet >=20 > - def NeedUpdateOutput(self,OutputFile, ValueCFile, MakeFile, Structur= eInput): > + def NeedUpdateOutput(self,OutputFile, ValueCFile, StructureInput): > if not os.path.exists(OutputFile): > return True > if os.stat(OutputFile).st_mtime <=3D os.stat(ValueCFile).st_mtim= e: > return True > - if os.stat(OutputFile).st_mtime <=3D os.stat(MakeFile).st_mtime: > - return True > if os.stat(OutputFile).st_mtime <=3D os.stat(StructureInput).st_= mtime: > return True > return False >=20 > ## Retrieve dynamic PCD settings > -- > 2.14.3.windows.1 >=20 > _______________________________________________ > edk2-devel mailing list > edk2-devel@lists.01.org > https://lists.01.org/mailman/listinfo/edk2-devel