From: "Gao, Liming" <liming.gao@intel.com>
To: "Feng, Bob C" <bob.c.feng@intel.com>,
"edk2-devel@lists.01.org" <edk2-devel@lists.01.org>
Subject: Re: [Patch V2] BaseTools: Fixed metafile parser issues
Date: Tue, 18 Dec 2018 01:46:17 +0000 [thread overview]
Message-ID: <4A89E2EF3DFEDB4C8BFDE51014F606A14E38DC5E@SHSMSX104.ccr.corp.intel.com> (raw)
In-Reply-To: <20181217085507.21868-1-bob.c.feng@intel.com>
Bob:
Please remove Change-Id: I62c00e9f439e5d632c4b3f58cc5c4181d8acc52d
The patch is good to me. Reviewed-by: Liming Gao <liming.gao@intel.com>
Thanks
Liming
>-----Original Message-----
>From: Feng, Bob C
>Sent: Monday, December 17, 2018 4:55 PM
>To: edk2-devel@lists.01.org
>Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming <liming.gao@intel.com>
>Subject: [Patch V2] BaseTools: Fixed metafile parser issues
>
>https://bugzilla.tianocore.org/show_bug.cgi?id=1406
>This patch is going to fix the regressions that
>is introduced by commit 2f818ed0fb57d98985d151781a2ce9b8683129ee
>
>The internal array for storing the metadata info should be cached
>so that the meta file is parsed only once in one build.
>
>Change-Id: I62c00e9f439e5d632c4b3f58cc5c4181d8acc52d
>Contributed-under: TianoCore Contribution Agreement 1.1
>Signed-off-by: Bob Feng <bob.c.feng@intel.com>
>Cc: Liming Gao <liming.gao@intel.com>
>---
> .../Source/Python/Workspace/DscBuildData.py | 2 +-
> .../Source/Python/Workspace/MetaFileParser.py | 25 +++++++-------
> .../Source/Python/Workspace/MetaFileTable.py | 34 +++++++++++--------
> .../Python/Workspace/WorkspaceDatabase.py | 4 +--
> 4 files changed, 34 insertions(+), 31 deletions(-)
>
>diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py
>b/BaseTools/Source/Python/Workspace/DscBuildData.py
>index 4543ae7dc0..2c1e783d2e 100644
>--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
>+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
>@@ -879,11 +879,11 @@ class DscBuildData(PlatformBuildClassObject):
> Library = self._Bdb[File, self._Arch, self._Target, self._Toolchain]
> self._LibraryClasses[Library.BaseName, ':dummy:'] = Library
> return self._LibraryClasses
>
> def _ValidatePcd(self, PcdCName, TokenSpaceGuid, Setting, PcdType,
>LineNo):
>- if self._DecPcds is None:
>+ if not self._DecPcds:
>
> FdfInfList = []
> if GlobalData.gFdfParser:
> FdfInfList = GlobalData.gFdfParser.Profile.InfList
>
>diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py
>b/BaseTools/Source/Python/Workspace/MetaFileParser.py
>index eaedba0c12..032220813b 100644
>--- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
>+++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
>@@ -204,13 +204,11 @@ class MetaFileParser(object):
> self._PostProcessed = False
>
> ## Set parsing complete flag in both class and table
> def _Done(self):
> self._Finished = True
>- ## Do not set end flag when processing included files
>- if self._From == -1:
>- self._Table.SetEndFlag()
>+ self._Table.SetEndFlag()
>
> def _PostProcess(self):
> self._PostProcessed = True
>
> ## Get the parse complete flag
>@@ -239,17 +237,11 @@ class MetaFileParser(object):
> def __getitem__(self, DataInfo):
> if not isinstance(DataInfo, type(())):
> DataInfo = (DataInfo,)
>
> # Parse the file first, if necessary
>- if not self._Finished:
>- if self._RawTable.IsIntegrity():
>- self._Finished = True
>- else:
>- self._Table = self._RawTable
>- self._PostProcessed = False
>- self.Start()
>+ self.StartParse()
>
> # No specific ARCH or Platform given, use raw data
> if self._RawTable and (len(DataInfo) == 1 or DataInfo[1] is None):
> return self._FilterRecordList(self._RawTable.Query(*DataInfo),
>self._Arch)
>
>@@ -257,10 +249,18 @@ class MetaFileParser(object):
> if not self._PostProcessed:
> self._PostProcess()
>
> return self._FilterRecordList(self._Table.Query(*DataInfo), DataInfo[1])
>
>+ def StartParse(self):
>+ if not self._Finished:
>+ if self._RawTable.IsIntegrity():
>+ self._Finished = True
>+ else:
>+ self._Table = self._RawTable
>+ self._PostProcessed = False
>+ self.Start()
> ## Data parser for the common format in different type of file
> #
> # The common format in the meatfile is like
> #
> # xxx1 | xxx2 | xxx3
>@@ -915,10 +915,11 @@ class DscParser(MetaFileParser):
> self._IdMapping = {-1:-1}
>
> self._PcdCodeValue = ""
> self._PcdDataTypeCODE = False
> self._CurrentPcdName = ""
>+ self._Content = None
>
> ## Parser starter
> def Start(self):
> Content = ''
> try:
>@@ -1643,13 +1644,11 @@ class DscParser(MetaFileParser):
> Parser._InSubsection = self._InSubsection
> Parser._SectionType = self._SectionType
> Parser._Scope = self._Scope
> Parser._Enabled = self._Enabled
> # Parse the included file
>- Parser.Start()
>-
>-
>+ Parser.StartParse()
> # Insert all records in the table for the included file into dsc file table
> Records = IncludedFileTable.GetAll()
> if Records:
> self._Content[self._ContentIndex:self._ContentIndex] = Records
> self._Content.pop(self._ContentIndex - 1)
>diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py
>b/BaseTools/Source/Python/Workspace/MetaFileTable.py
>index 081970dba8..004e9494c3 100644
>--- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
>+++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
>@@ -24,38 +24,42 @@ from CommonDataClass.DataClass import
>MODEL_FILE_DSC, MODEL_FILE_DEC, MODEL_FILE
> MODEL_FILE_OTHERS
> from Common.DataType import *
>
> class MetaFileTable():
> # TRICK: use file ID as the part before '.'
>- _ID_STEP_ = 0.00000001
>- _ID_MAX_ = 0.99999999
>+ _ID_STEP_ = 1
>+ _ID_MAX_ = 99999999
>
> ## Constructor
> def __init__(self, DB, MetaFile, FileType, Temporary, FromItem=None):
> self.MetaFile = MetaFile
> self.TableName = ""
> self.DB = DB
> self._NumpyTab = None
>- self.FileId = len(DB.TblFile)
>- self.ID = self.FileId
>+
> self.CurrentContent = []
> DB.TblFile.append([MetaFile.Name,
> MetaFile.Ext,
> MetaFile.Dir,
> MetaFile.Path,
> FileType,
> MetaFile.TimeStamp,
> FromItem])
>+ self.FileId = len(DB.TblFile)
>+ self.ID = self.FileId * 10**8
> if Temporary:
> self.TableName = "_%s_%s_%s" % (FileType, len(DB.TblFile),
>uuid.uuid4().hex)
> else:
> self.TableName = "_%s_%s" % (FileType, len(DB.TblFile))
>
> def IsIntegrity(self):
> try:
> TimeStamp = self.MetaFile.TimeStamp
>- Result = int(self.CurrentContent[-1][0]) < 0
>+ if not self.CurrentContent:
>+ Result = False
>+ else:
>+ Result = self.CurrentContent[-1][0] < 0
> if not Result:
> # update the timestamp in database
> self.DB.SetFileTimeStamp(self.FileId, TimeStamp)
> return False
>
>@@ -70,16 +74,14 @@ class MetaFileTable():
>
> def SetEndFlag(self):
> self.CurrentContent.append(self._DUMMY_)
>
> def GetAll(self):
>- return [item for item in self.CurrentContent if item[0] > 0 ]
>+ return [item for item in self.CurrentContent if item[0] >= 0 ]
>
> ## Python class representation of table storing module data
> class ModuleTable(MetaFileTable):
>- _ID_STEP_ = 0.00000001
>- _ID_MAX_ = 0.99999999
> _COLUMN_ = '''
> ID REAL PRIMARY KEY,
> Model INTEGER NOT NULL,
> Value1 TEXT NOT NULL,
> Value2 TEXT,
>@@ -138,11 +140,10 @@ class ModuleTable(MetaFileTable):
> Enabled
> ]
> self.CurrentContent.append(row)
> return self.ID
>
>-
> ## Query table
> #
> # @param Model: The Model of Record
> # @param Arch: The Arch attribute of Record
> # @param Platform The Platform attribute of Record
>@@ -213,12 +214,10 @@ class PackageTable(MetaFileTable):
> #
> def Insert(self, Model, Value1, Value2, Value3,
>Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON,
> BelongsToItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1,
>EndColumn=-1, Enabled=0):
> (Value1, Value2, Value3, Scope1, Scope2) = (Value1.strip(), Value2.strip(),
>Value3.strip(), Scope1.strip(), Scope2.strip())
> self.ID = self.ID + self._ID_STEP_
>- if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
>- self.ID = MODEL_FILE_INF + self._ID_STEP_
>
> row = [ self.ID,
> Model,
> Value1,
> Value2,
>@@ -288,10 +287,11 @@ class PackageTable(MetaFileTable):
> ValidType = "@Expression"
> EdkLogger.error('Parser', FORMAT_INVALID, "The syntax for %s of
>PCD %s.%s is incorrect" % (ValidType, TokenSpaceGuid, PcdCName),
> ExtraData=oricomment, File=self.MetaFile, Line=LineNum)
> return set(), set(), set()
> return set(validateranges), set(validlists), set(expressions)
>+
> ## Python class representation of table storing platform data
> class PlatformTable(MetaFileTable):
> _COLUMN_ = '''
> ID REAL PRIMARY KEY,
> Model INTEGER NOT NULL,
>@@ -336,12 +336,10 @@ class PlatformTable(MetaFileTable):
> #
> def Insert(self, Model, Value1, Value2, Value3,
>Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON,
>Scope3=TAB_DEFAULT_STORES_DEFAULT,BelongsToItem=-1,
> FromItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1,
>EndColumn=-1, Enabled=1):
> (Value1, Value2, Value3, Scope1, Scope2, Scope3) = (Value1.strip(),
>Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip(), Scope3.strip())
> self.ID = self.ID + self._ID_STEP_
>- if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
>- self.ID = MODEL_FILE_INF + self._ID_STEP_
>
> row = [ self.ID,
> Model,
> Value1,
> Value2,
>@@ -412,14 +410,17 @@ class MetaFileStorage(object):
> _FILE_TYPE_ = {
> ".inf" : MODEL_FILE_INF,
> ".dec" : MODEL_FILE_DEC,
> ".dsc" : MODEL_FILE_DSC,
> }
>-
>+ _ObjectCache = {}
> ## Constructor
> def __new__(Class, Cursor, MetaFile, FileType=None, Temporary=False,
>FromItem=None):
> # no type given, try to find one
>+ key = (MetaFile.Path, FileType,Temporary,FromItem)
>+ if key in Class._ObjectCache:
>+ return Class._ObjectCache[key]
> if not FileType:
> if MetaFile.Type in self._FILE_TYPE_:
> FileType = Class._FILE_TYPE_[MetaFile.Type]
> else:
> FileType = MODEL_FILE_OTHERS
>@@ -431,7 +432,10 @@ class MetaFileStorage(object):
> Args = (Cursor, MetaFile, Temporary)
> if FromItem:
> Args = Args + (FromItem,)
>
> # create the storage object and return it to caller
>- return Class._FILE_TABLE_[FileType](*Args)
>+ reval = Class._FILE_TABLE_[FileType](*Args)
>+ if not Temporary:
>+ Class._ObjectCache[key] = reval
>+ return reval
>
>diff --git a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
>b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
>index 8dbf3ae97c..a6a292d15c 100644
>--- a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
>+++ b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
>@@ -161,14 +161,14 @@ class WorkspaceDatabase(object):
> # conversion object for build or file format conversion purpose
> self.BuildObject = WorkspaceDatabase.BuildObjectFactory(self)
> self.TransformObject = WorkspaceDatabase.TransformObjectFactory(self)
>
> def SetFileTimeStamp(self,FileId,TimeStamp):
>- self.TblFile[FileId][6] = TimeStamp
>+ self.TblFile[FileId-1][6] = TimeStamp
>
> def GetFileTimeStamp(self,FileId):
>- return self.TblFile[FileId][6]
>+ return self.TblFile[FileId-1][6]
>
>
> ## Summarize all packages in the database
> def GetPackageList(self, Platform, Arch, TargetName, ToolChainTag):
> self.Platform = Platform
>--
>2.19.1.windows.1
prev parent reply other threads:[~2018-12-18 1:46 UTC|newest]
Thread overview: 2+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-12-17 8:55 [Patch V2] BaseTools: Fixed metafile parser issues BobCF
2018-12-18 1:46 ` Gao, Liming [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-list from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=4A89E2EF3DFEDB4C8BFDE51014F606A14E38DC5E@SHSMSX104.ccr.corp.intel.com \
--to=devel@edk2.groups.io \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox