From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=134.134.136.65; helo=mga03.intel.com; envelope-from=liming.gao@intel.com; receiver=edk2-devel@lists.01.org Received: from mga03.intel.com (mga03.intel.com [134.134.136.65]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ml01.01.org (Postfix) with ESMTPS id 29F162119A44D for ; Mon, 17 Dec 2018 17:46:21 -0800 (PST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga003.jf.intel.com ([10.7.209.27]) by orsmga103.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 17 Dec 2018 17:46:20 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.56,367,1539673200"; d="scan'208";a="111361341" Received: from fmsmsx107.amr.corp.intel.com ([10.18.124.205]) by orsmga003.jf.intel.com with ESMTP; 17 Dec 2018 17:46:20 -0800 Received: from fmsmsx115.amr.corp.intel.com (10.18.116.19) by fmsmsx107.amr.corp.intel.com (10.18.124.205) with Microsoft SMTP Server (TLS) id 14.3.408.0; Mon, 17 Dec 2018 17:46:20 -0800 Received: from shsmsx152.ccr.corp.intel.com (10.239.6.52) by fmsmsx115.amr.corp.intel.com (10.18.116.19) with Microsoft SMTP Server (TLS) id 14.3.408.0; Mon, 17 Dec 2018 17:46:19 -0800 Received: from shsmsx104.ccr.corp.intel.com ([169.254.5.203]) by SHSMSX152.ccr.corp.intel.com ([169.254.6.222]) with mapi id 14.03.0415.000; Tue, 18 Dec 2018 09:46:17 +0800 From: "Gao, Liming" To: "Feng, Bob C" , "edk2-devel@lists.01.org" Thread-Topic: [Patch V2] BaseTools: Fixed metafile parser issues Thread-Index: AQHUleZEuiqoNdn5rU6gdPN1mgjDSaWDuwNQ Date: Tue, 18 Dec 2018 01:46:17 +0000 Message-ID: <4A89E2EF3DFEDB4C8BFDE51014F606A14E38DC5E@SHSMSX104.ccr.corp.intel.com> References: <20181217085507.21868-1-bob.c.feng@intel.com> In-Reply-To: <20181217085507.21868-1-bob.c.feng@intel.com> Accept-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-originating-ip: [10.239.127.40] MIME-Version: 1.0 Subject: Re: [Patch V2] BaseTools: Fixed metafile parser issues X-BeenThere: edk2-devel@lists.01.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: EDK II Development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 18 Dec 2018 01:46:21 -0000 Content-Language: en-US Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable Bob: Please remove Change-Id: I62c00e9f439e5d632c4b3f58cc5c4181d8acc52d The patch is good to me. Reviewed-by: Liming Gao Thanks Liming >-----Original Message----- >From: Feng, Bob C >Sent: Monday, December 17, 2018 4:55 PM >To: edk2-devel@lists.01.org >Cc: Feng, Bob C ; Gao, Liming >Subject: [Patch V2] BaseTools: Fixed metafile parser issues > >https://bugzilla.tianocore.org/show_bug.cgi?id=3D1406 >This patch is going to fix the regressions that >is introduced by commit 2f818ed0fb57d98985d151781a2ce9b8683129ee > >The internal array for storing the metadata info should be cached >so that the meta file is parsed only once in one build. > >Change-Id: I62c00e9f439e5d632c4b3f58cc5c4181d8acc52d >Contributed-under: TianoCore Contribution Agreement 1.1 >Signed-off-by: Bob Feng >Cc: Liming Gao >--- > .../Source/Python/Workspace/DscBuildData.py | 2 +- > .../Source/Python/Workspace/MetaFileParser.py | 25 +++++++------- > .../Source/Python/Workspace/MetaFileTable.py | 34 +++++++++++-------- > .../Python/Workspace/WorkspaceDatabase.py | 4 +-- > 4 files changed, 34 insertions(+), 31 deletions(-) > >diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py >b/BaseTools/Source/Python/Workspace/DscBuildData.py >index 4543ae7dc0..2c1e783d2e 100644 >--- a/BaseTools/Source/Python/Workspace/DscBuildData.py >+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py >@@ -879,11 +879,11 @@ class DscBuildData(PlatformBuildClassObject): > Library =3D self._Bdb[File, self._Arch, self._Target, sel= f._Toolchain] > self._LibraryClasses[Library.BaseName, ':dummy:'] =3D Lib= rary > return self._LibraryClasses > > def _ValidatePcd(self, PcdCName, TokenSpaceGuid, Setting, PcdType, >LineNo): >- if self._DecPcds is None: >+ if not self._DecPcds: > > FdfInfList =3D [] > if GlobalData.gFdfParser: > FdfInfList =3D GlobalData.gFdfParser.Profile.InfList > >diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py >b/BaseTools/Source/Python/Workspace/MetaFileParser.py >index eaedba0c12..032220813b 100644 >--- a/BaseTools/Source/Python/Workspace/MetaFileParser.py >+++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py >@@ -204,13 +204,11 @@ class MetaFileParser(object): > self._PostProcessed =3D False > > ## Set parsing complete flag in both class and table > def _Done(self): > self._Finished =3D True >- ## Do not set end flag when processing included files >- if self._From =3D=3D -1: >- self._Table.SetEndFlag() >+ self._Table.SetEndFlag() > > def _PostProcess(self): > self._PostProcessed =3D True > > ## Get the parse complete flag >@@ -239,17 +237,11 @@ class MetaFileParser(object): > def __getitem__(self, DataInfo): > if not isinstance(DataInfo, type(())): > DataInfo =3D (DataInfo,) > > # Parse the file first, if necessary >- if not self._Finished: >- if self._RawTable.IsIntegrity(): >- self._Finished =3D True >- else: >- self._Table =3D self._RawTable >- self._PostProcessed =3D False >- self.Start() >+ self.StartParse() > > # No specific ARCH or Platform given, use raw data > if self._RawTable and (len(DataInfo) =3D=3D 1 or DataInfo[1] is N= one): > return self._FilterRecordList(self._RawTable.Query(*DataInfo)= , >self._Arch) > >@@ -257,10 +249,18 @@ class MetaFileParser(object): > if not self._PostProcessed: > self._PostProcess() > > return self._FilterRecordList(self._Table.Query(*DataInfo), DataI= nfo[1]) > >+ def StartParse(self): >+ if not self._Finished: >+ if self._RawTable.IsIntegrity(): >+ self._Finished =3D True >+ else: >+ self._Table =3D self._RawTable >+ self._PostProcessed =3D False >+ self.Start() > ## Data parser for the common format in different type of file > # > # The common format in the meatfile is like > # > # xxx1 | xxx2 | xxx3 >@@ -915,10 +915,11 @@ class DscParser(MetaFileParser): > self._IdMapping =3D {-1:-1} > > self._PcdCodeValue =3D "" > self._PcdDataTypeCODE =3D False > self._CurrentPcdName =3D "" >+ self._Content =3D None > > ## Parser starter > def Start(self): > Content =3D '' > try: >@@ -1643,13 +1644,11 @@ class DscParser(MetaFileParser): > Parser._InSubsection =3D self._InSubsection > Parser._SectionType =3D self._SectionType > Parser._Scope =3D self._Scope > Parser._Enabled =3D self._Enabled > # Parse the included file >- Parser.Start() >- >- >+ Parser.StartParse() > # Insert all records in the table for the included file into = dsc file table > Records =3D IncludedFileTable.GetAll() > if Records: > self._Content[self._ContentIndex:self._ContentIndex] =3D = Records > self._Content.pop(self._ContentIndex - 1) >diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py >b/BaseTools/Source/Python/Workspace/MetaFileTable.py >index 081970dba8..004e9494c3 100644 >--- a/BaseTools/Source/Python/Workspace/MetaFileTable.py >+++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py >@@ -24,38 +24,42 @@ from CommonDataClass.DataClass import >MODEL_FILE_DSC, MODEL_FILE_DEC, MODEL_FILE > MODEL_FILE_OTHERS > from Common.DataType import * > > class MetaFileTable(): > # TRICK: use file ID as the part before '.' >- _ID_STEP_ =3D 0.00000001 >- _ID_MAX_ =3D 0.99999999 >+ _ID_STEP_ =3D 1 >+ _ID_MAX_ =3D 99999999 > > ## Constructor > def __init__(self, DB, MetaFile, FileType, Temporary, FromItem=3DNone= ): > self.MetaFile =3D MetaFile > self.TableName =3D "" > self.DB =3D DB > self._NumpyTab =3D None >- self.FileId =3D len(DB.TblFile) >- self.ID =3D self.FileId >+ > self.CurrentContent =3D [] > DB.TblFile.append([MetaFile.Name, > MetaFile.Ext, > MetaFile.Dir, > MetaFile.Path, > FileType, > MetaFile.TimeStamp, > FromItem]) >+ self.FileId =3D len(DB.TblFile) >+ self.ID =3D self.FileId * 10**8 > if Temporary: > self.TableName =3D "_%s_%s_%s" % (FileType, len(DB.TblFile), >uuid.uuid4().hex) > else: > self.TableName =3D "_%s_%s" % (FileType, len(DB.TblFile)) > > def IsIntegrity(self): > try: > TimeStamp =3D self.MetaFile.TimeStamp >- Result =3D int(self.CurrentContent[-1][0]) < 0 >+ if not self.CurrentContent: >+ Result =3D False >+ else: >+ Result =3D self.CurrentContent[-1][0] < 0 > if not Result: > # update the timestamp in database > self.DB.SetFileTimeStamp(self.FileId, TimeStamp) > return False > >@@ -70,16 +74,14 @@ class MetaFileTable(): > > def SetEndFlag(self): > self.CurrentContent.append(self._DUMMY_) > > def GetAll(self): >- return [item for item in self.CurrentContent if item[0] > 0 ] >+ return [item for item in self.CurrentContent if item[0] >=3D 0 ] > > ## Python class representation of table storing module data > class ModuleTable(MetaFileTable): >- _ID_STEP_ =3D 0.00000001 >- _ID_MAX_ =3D 0.99999999 > _COLUMN_ =3D ''' > ID REAL PRIMARY KEY, > Model INTEGER NOT NULL, > Value1 TEXT NOT NULL, > Value2 TEXT, >@@ -138,11 +140,10 @@ class ModuleTable(MetaFileTable): > Enabled > ] > self.CurrentContent.append(row) > return self.ID > >- > ## Query table > # > # @param Model: The Model of Record > # @param Arch: The Arch attribute of Record > # @param Platform The Platform attribute of Record >@@ -213,12 +214,10 @@ class PackageTable(MetaFileTable): > # > def Insert(self, Model, Value1, Value2, Value3, >Scope1=3DTAB_ARCH_COMMON, Scope2=3DTAB_COMMON, > BelongsToItem=3D-1, StartLine=3D-1, StartColumn=3D-1, EndL= ine=3D-1, >EndColumn=3D-1, Enabled=3D0): > (Value1, Value2, Value3, Scope1, Scope2) =3D (Value1.strip(), Val= ue2.strip(), >Value3.strip(), Scope1.strip(), Scope2.strip()) > self.ID =3D self.ID + self._ID_STEP_ >- if self.ID >=3D (MODEL_FILE_INF + self._ID_MAX_): >- self.ID =3D MODEL_FILE_INF + self._ID_STEP_ > > row =3D [ self.ID, > Model, > Value1, > Value2, >@@ -288,10 +287,11 @@ class PackageTable(MetaFileTable): > ValidType =3D "@Expression" > EdkLogger.error('Parser', FORMAT_INVALID, "The syntax for %s = of >PCD %s.%s is incorrect" % (ValidType, TokenSpaceGuid, PcdCName), > ExtraData=3Doricomment, File=3Dself.MetaFile,= Line=3DLineNum) > return set(), set(), set() > return set(validateranges), set(validlists), set(expressions) >+ > ## Python class representation of table storing platform data > class PlatformTable(MetaFileTable): > _COLUMN_ =3D ''' > ID REAL PRIMARY KEY, > Model INTEGER NOT NULL, >@@ -336,12 +336,10 @@ class PlatformTable(MetaFileTable): > # > def Insert(self, Model, Value1, Value2, Value3, >Scope1=3DTAB_ARCH_COMMON, Scope2=3DTAB_COMMON, >Scope3=3DTAB_DEFAULT_STORES_DEFAULT,BelongsToItem=3D-1, > FromItem=3D-1, StartLine=3D-1, StartColumn=3D-1, EndLine= =3D-1, >EndColumn=3D-1, Enabled=3D1): > (Value1, Value2, Value3, Scope1, Scope2, Scope3) =3D (Value1.stri= p(), >Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip(), Scope3.str= ip()) > self.ID =3D self.ID + self._ID_STEP_ >- if self.ID >=3D (MODEL_FILE_INF + self._ID_MAX_): >- self.ID =3D MODEL_FILE_INF + self._ID_STEP_ > > row =3D [ self.ID, > Model, > Value1, > Value2, >@@ -412,14 +410,17 @@ class MetaFileStorage(object): > _FILE_TYPE_ =3D { > ".inf" : MODEL_FILE_INF, > ".dec" : MODEL_FILE_DEC, > ".dsc" : MODEL_FILE_DSC, > } >- >+ _ObjectCache =3D {} > ## Constructor > def __new__(Class, Cursor, MetaFile, FileType=3DNone, Temporary=3DFal= se, >FromItem=3DNone): > # no type given, try to find one >+ key =3D (MetaFile.Path, FileType,Temporary,FromItem) >+ if key in Class._ObjectCache: >+ return Class._ObjectCache[key] > if not FileType: > if MetaFile.Type in self._FILE_TYPE_: > FileType =3D Class._FILE_TYPE_[MetaFile.Type] > else: > FileType =3D MODEL_FILE_OTHERS >@@ -431,7 +432,10 @@ class MetaFileStorage(object): > Args =3D (Cursor, MetaFile, Temporary) > if FromItem: > Args =3D Args + (FromItem,) > > # create the storage object and return it to caller >- return Class._FILE_TABLE_[FileType](*Args) >+ reval =3D Class._FILE_TABLE_[FileType](*Args) >+ if not Temporary: >+ Class._ObjectCache[key] =3D reval >+ return reval > >diff --git a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py >b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py >index 8dbf3ae97c..a6a292d15c 100644 >--- a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py >+++ b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py >@@ -161,14 +161,14 @@ class WorkspaceDatabase(object): > # conversion object for build or file format conversion purpose > self.BuildObject =3D WorkspaceDatabase.BuildObjectFactory(self) > self.TransformObject =3D WorkspaceDatabase.TransformObjectFactory= (self) > > def SetFileTimeStamp(self,FileId,TimeStamp): >- self.TblFile[FileId][6] =3D TimeStamp >+ self.TblFile[FileId-1][6] =3D TimeStamp > > def GetFileTimeStamp(self,FileId): >- return self.TblFile[FileId][6] >+ return self.TblFile[FileId-1][6] > > > ## Summarize all packages in the database > def GetPackageList(self, Platform, Arch, TargetName, ToolChainTag): > self.Platform =3D Platform >-- >2.19.1.windows.1