public inbox for devel@edk2.groups.io
 help / color / mirror / Atom feed
* [Patch V2] BaseTools: Fixed metafile parser issues
@ 2018-12-17  8:55 BobCF
  2018-12-18  1:46 ` Gao, Liming
  0 siblings, 1 reply; 2+ messages in thread
From: BobCF @ 2018-12-17  8:55 UTC (permalink / raw)
  To: edk2-devel; +Cc: Bob Feng, Liming Gao

https://bugzilla.tianocore.org/show_bug.cgi?id=1406
This patch is going to fix the regressions that
is introduced by commit 2f818ed0fb57d98985d151781a2ce9b8683129ee

The internal array for storing the metadata info should be cached
so that the meta file is parsed only once in one build.

Change-Id: I62c00e9f439e5d632c4b3f58cc5c4181d8acc52d
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
---
 .../Source/Python/Workspace/DscBuildData.py   |  2 +-
 .../Source/Python/Workspace/MetaFileParser.py | 25 +++++++-------
 .../Source/Python/Workspace/MetaFileTable.py  | 34 +++++++++++--------
 .../Python/Workspace/WorkspaceDatabase.py     |  4 +--
 4 files changed, 34 insertions(+), 31 deletions(-)

diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py
index 4543ae7dc0..2c1e783d2e 100644
--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
@@ -879,11 +879,11 @@ class DscBuildData(PlatformBuildClassObject):
                 Library = self._Bdb[File, self._Arch, self._Target, self._Toolchain]
                 self._LibraryClasses[Library.BaseName, ':dummy:'] = Library
         return self._LibraryClasses
 
     def _ValidatePcd(self, PcdCName, TokenSpaceGuid, Setting, PcdType, LineNo):
-        if self._DecPcds is None:
+        if not self._DecPcds:
 
             FdfInfList = []
             if GlobalData.gFdfParser:
                 FdfInfList = GlobalData.gFdfParser.Profile.InfList
 
diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py b/BaseTools/Source/Python/Workspace/MetaFileParser.py
index eaedba0c12..032220813b 100644
--- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
+++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
@@ -204,13 +204,11 @@ class MetaFileParser(object):
         self._PostProcessed = False
 
     ## Set parsing complete flag in both class and table
     def _Done(self):
         self._Finished = True
-        ## Do not set end flag when processing included files
-        if self._From == -1:
-            self._Table.SetEndFlag()
+        self._Table.SetEndFlag()
 
     def _PostProcess(self):
         self._PostProcessed = True
 
     ## Get the parse complete flag
@@ -239,17 +237,11 @@ class MetaFileParser(object):
     def __getitem__(self, DataInfo):
         if not isinstance(DataInfo, type(())):
             DataInfo = (DataInfo,)
 
         # Parse the file first, if necessary
-        if not self._Finished:
-            if self._RawTable.IsIntegrity():
-                self._Finished = True
-            else:
-                self._Table = self._RawTable
-                self._PostProcessed = False
-                self.Start()
+        self.StartParse()
 
         # No specific ARCH or Platform given, use raw data
         if self._RawTable and (len(DataInfo) == 1 or DataInfo[1] is None):
             return self._FilterRecordList(self._RawTable.Query(*DataInfo), self._Arch)
 
@@ -257,10 +249,18 @@ class MetaFileParser(object):
         if not self._PostProcessed:
             self._PostProcess()
 
         return self._FilterRecordList(self._Table.Query(*DataInfo), DataInfo[1])
 
+    def StartParse(self):
+        if not self._Finished:
+            if self._RawTable.IsIntegrity():
+                self._Finished = True
+            else:
+                self._Table = self._RawTable
+                self._PostProcessed = False
+                self.Start()
     ## Data parser for the common format in different type of file
     #
     #   The common format in the meatfile is like
     #
     #       xxx1 | xxx2 | xxx3
@@ -915,10 +915,11 @@ class DscParser(MetaFileParser):
         self._IdMapping = {-1:-1}
 
         self._PcdCodeValue = ""
         self._PcdDataTypeCODE = False
         self._CurrentPcdName = ""
+        self._Content = None
 
     ## Parser starter
     def Start(self):
         Content = ''
         try:
@@ -1643,13 +1644,11 @@ class DscParser(MetaFileParser):
             Parser._InSubsection = self._InSubsection
             Parser._SectionType = self._SectionType
             Parser._Scope = self._Scope
             Parser._Enabled = self._Enabled
             # Parse the included file
-            Parser.Start()
-
-
+            Parser.StartParse()
             # Insert all records in the table for the included file into dsc file table
             Records = IncludedFileTable.GetAll()
             if Records:
                 self._Content[self._ContentIndex:self._ContentIndex] = Records
                 self._Content.pop(self._ContentIndex - 1)
diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py b/BaseTools/Source/Python/Workspace/MetaFileTable.py
index 081970dba8..004e9494c3 100644
--- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
+++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
@@ -24,38 +24,42 @@ from CommonDataClass.DataClass import MODEL_FILE_DSC, MODEL_FILE_DEC, MODEL_FILE
                                       MODEL_FILE_OTHERS
 from Common.DataType import *
 
 class MetaFileTable():
     # TRICK: use file ID as the part before '.'
-    _ID_STEP_ = 0.00000001
-    _ID_MAX_ = 0.99999999
+    _ID_STEP_ = 1
+    _ID_MAX_ = 99999999
 
     ## Constructor
     def __init__(self, DB, MetaFile, FileType, Temporary, FromItem=None):
         self.MetaFile = MetaFile
         self.TableName = ""
         self.DB = DB
         self._NumpyTab = None
-        self.FileId = len(DB.TblFile)
-        self.ID = self.FileId
+
         self.CurrentContent = []
         DB.TblFile.append([MetaFile.Name,
                         MetaFile.Ext,
                         MetaFile.Dir,
                         MetaFile.Path,
                         FileType,
                         MetaFile.TimeStamp,
                         FromItem])
+        self.FileId = len(DB.TblFile)
+        self.ID = self.FileId * 10**8
         if Temporary:
             self.TableName = "_%s_%s_%s" % (FileType, len(DB.TblFile), uuid.uuid4().hex)
         else:
             self.TableName = "_%s_%s" % (FileType, len(DB.TblFile))
 
     def IsIntegrity(self):
         try:
             TimeStamp = self.MetaFile.TimeStamp
-            Result = int(self.CurrentContent[-1][0]) < 0
+            if not self.CurrentContent:
+                Result = False
+            else:
+                Result = self.CurrentContent[-1][0] < 0
             if not Result:
                 # update the timestamp in database
                 self.DB.SetFileTimeStamp(self.FileId, TimeStamp)
                 return False
 
@@ -70,16 +74,14 @@ class MetaFileTable():
 
     def SetEndFlag(self):
         self.CurrentContent.append(self._DUMMY_)
 
     def GetAll(self):
-        return [item for item in self.CurrentContent if item[0] > 0 ]
+        return [item for item in self.CurrentContent if item[0] >= 0 ]
 
 ## Python class representation of table storing module data
 class ModuleTable(MetaFileTable):
-    _ID_STEP_ = 0.00000001
-    _ID_MAX_  = 0.99999999
     _COLUMN_ = '''
         ID REAL PRIMARY KEY,
         Model INTEGER NOT NULL,
         Value1 TEXT NOT NULL,
         Value2 TEXT,
@@ -138,11 +140,10 @@ class ModuleTable(MetaFileTable):
                 Enabled
             ]
         self.CurrentContent.append(row)
         return self.ID
 
-
     ## Query table
     #
     # @param    Model:      The Model of Record
     # @param    Arch:       The Arch attribute of Record
     # @param    Platform    The Platform attribute of Record
@@ -213,12 +214,10 @@ class PackageTable(MetaFileTable):
     #
     def Insert(self, Model, Value1, Value2, Value3, Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON,
                BelongsToItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1, EndColumn=-1, Enabled=0):
         (Value1, Value2, Value3, Scope1, Scope2) = (Value1.strip(), Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip())
         self.ID = self.ID + self._ID_STEP_
-        if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
-            self.ID = MODEL_FILE_INF + self._ID_STEP_
 
         row = [ self.ID,
                 Model,
                 Value1,
                 Value2,
@@ -288,10 +287,11 @@ class PackageTable(MetaFileTable):
                 ValidType = "@Expression"
             EdkLogger.error('Parser', FORMAT_INVALID, "The syntax for %s of PCD %s.%s is incorrect" % (ValidType, TokenSpaceGuid, PcdCName),
                             ExtraData=oricomment, File=self.MetaFile, Line=LineNum)
             return set(), set(), set()
         return set(validateranges), set(validlists), set(expressions)
+
 ## Python class representation of table storing platform data
 class PlatformTable(MetaFileTable):
     _COLUMN_ = '''
         ID REAL PRIMARY KEY,
         Model INTEGER NOT NULL,
@@ -336,12 +336,10 @@ class PlatformTable(MetaFileTable):
     #
     def Insert(self, Model, Value1, Value2, Value3, Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON, Scope3=TAB_DEFAULT_STORES_DEFAULT,BelongsToItem=-1,
                FromItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1, EndColumn=-1, Enabled=1):
         (Value1, Value2, Value3, Scope1, Scope2, Scope3) = (Value1.strip(), Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip(), Scope3.strip())
         self.ID = self.ID + self._ID_STEP_
-        if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
-            self.ID = MODEL_FILE_INF + self._ID_STEP_
 
         row = [ self.ID,
                 Model,
                 Value1,
                 Value2,
@@ -412,14 +410,17 @@ class MetaFileStorage(object):
     _FILE_TYPE_ = {
         ".inf"  : MODEL_FILE_INF,
         ".dec"  : MODEL_FILE_DEC,
         ".dsc"  : MODEL_FILE_DSC,
     }
-
+    _ObjectCache = {}
     ## Constructor
     def __new__(Class, Cursor, MetaFile, FileType=None, Temporary=False, FromItem=None):
         # no type given, try to find one
+        key = (MetaFile.Path, FileType,Temporary,FromItem)
+        if key in Class._ObjectCache:
+            return Class._ObjectCache[key]
         if not FileType:
             if MetaFile.Type in self._FILE_TYPE_:
                 FileType = Class._FILE_TYPE_[MetaFile.Type]
             else:
                 FileType = MODEL_FILE_OTHERS
@@ -431,7 +432,10 @@ class MetaFileStorage(object):
             Args = (Cursor, MetaFile, Temporary)
         if FromItem:
             Args = Args + (FromItem,)
 
         # create the storage object and return it to caller
-        return Class._FILE_TABLE_[FileType](*Args)
+        reval = Class._FILE_TABLE_[FileType](*Args)
+        if not Temporary:
+            Class._ObjectCache[key] = reval
+        return reval
 
diff --git a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
index 8dbf3ae97c..a6a292d15c 100644
--- a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
+++ b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
@@ -161,14 +161,14 @@ class WorkspaceDatabase(object):
         # conversion object for build or file format conversion purpose
         self.BuildObject = WorkspaceDatabase.BuildObjectFactory(self)
         self.TransformObject = WorkspaceDatabase.TransformObjectFactory(self)
 
     def SetFileTimeStamp(self,FileId,TimeStamp):
-        self.TblFile[FileId][6] = TimeStamp
+        self.TblFile[FileId-1][6] = TimeStamp
 
     def GetFileTimeStamp(self,FileId):
-        return self.TblFile[FileId][6]
+        return self.TblFile[FileId-1][6]
 
 
     ## Summarize all packages in the database
     def GetPackageList(self, Platform, Arch, TargetName, ToolChainTag):
         self.Platform = Platform
-- 
2.19.1.windows.1



^ permalink raw reply related	[flat|nested] 2+ messages in thread
end of thread, other threads:[~2018-12-18  1:46 UTC | newest]

Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2018-12-17  8:55 [Patch V2] BaseTools: Fixed metafile parser issues BobCF
2018-12-18  1:46 ` Gao, Liming

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox