* [Patch V2] BaseTools: Replace the sqlite database with list
@ 2018-11-09 8:41 BobCF
2018-11-09 15:24 ` Carsey, Jaben
0 siblings, 1 reply; 3+ messages in thread
From: BobCF @ 2018-11-09 8:41 UTC (permalink / raw)
To: edk2-devel; +Cc: Liming Gao, Jaben Carsey
https://bugzilla.tianocore.org/show_bug.cgi?id=1288
[V2]
Optimize this patch so that it can be easy to review.
This patch is just apply the change to original files while
not create new similar files.
[V1]
This patch is one of build tool performance improvement
series patches.
This patch is going to use python list to store the parser data
instead of using sqlite database.
The replacement solution is as below:
SQL insert: list.append()
SQL select: list comprehension. for example:
Select * from table where field = “something”
->
[ item for item in table if item[3] == “something”]
SQL update: python map function. for example:
Update table set field1=newvalue where filed2 = “something”.
-> map(lambda x: x[1] = newvalue,
[item for item in table if item[2] == “something”])
SQL delete: list comprehension.
With this change, We can save the time of interpreting SQL statement
and the time of write database to file system
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: BobCF <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Cc: Jaben Carsey <jaben.carsey@intel.com>
---
BaseTools/Source/Python/GenFds/GenFds.py | 3 +-
.../Source/Python/Workspace/MetaDataTable.py | 93 ++-----
.../Source/Python/Workspace/MetaFileParser.py | 5 +-
.../Source/Python/Workspace/MetaFileTable.py | 248 ++++++++++--------
.../Python/Workspace/WorkspaceDatabase.py | 131 +--------
BaseTools/Source/Python/build/build.py | 27 +-
6 files changed, 186 insertions(+), 321 deletions(-)
diff --git a/BaseTools/Source/Python/GenFds/GenFds.py b/BaseTools/Source/Python/GenFds/GenFds.py
index 0c8091b798..4fd96706af 100644
--- a/BaseTools/Source/Python/GenFds/GenFds.py
+++ b/BaseTools/Source/Python/GenFds/GenFds.py
@@ -222,12 +222,11 @@ def main():
if "TOOL_CHAIN_TAG" not in GlobalData.gGlobalDefines:
GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = GenFdsGlobalVariable.ToolChainTag
"""call Workspace build create database"""
GlobalData.gDatabasePath = os.path.normpath(os.path.join(ConfDirectoryPath, GlobalData.gDatabasePath))
- BuildWorkSpace = WorkspaceDatabase(GlobalData.gDatabasePath)
- BuildWorkSpace.InitDatabase()
+ BuildWorkSpace = WorkspaceDatabase()
#
# Get files real name in workspace dir
#
GlobalData.gAllFiles = DirCache(Workspace)
diff --git a/BaseTools/Source/Python/Workspace/MetaDataTable.py b/BaseTools/Source/Python/Workspace/MetaDataTable.py
index bd751eadfb..8becddbe08 100644
--- a/BaseTools/Source/Python/Workspace/MetaDataTable.py
+++ b/BaseTools/Source/Python/Workspace/MetaDataTable.py
@@ -37,84 +37,58 @@ class Table(object):
_COLUMN_ = ''
_ID_STEP_ = 1
_ID_MAX_ = 0x80000000
_DUMMY_ = 0
- def __init__(self, Cursor, Name='', IdBase=0, Temporary=False):
- self.Cur = Cursor
+ def __init__(self, Db, Name='', IdBase=0, Temporary=False):
+ self.Db = Db
self.Table = Name
self.IdBase = int(IdBase)
self.ID = int(IdBase)
self.Temporary = Temporary
+ self.Contents = []
def __str__(self):
return self.Table
## Create table
#
# Create a table
#
def Create(self, NewTable=True):
- if NewTable:
- self.Drop()
-
- if self.Temporary:
- SqlCommand = """create temp table IF NOT EXISTS %s (%s)""" % (self.Table, self._COLUMN_)
- else:
- SqlCommand = """create table IF NOT EXISTS %s (%s)""" % (self.Table, self._COLUMN_)
- EdkLogger.debug(EdkLogger.DEBUG_8, SqlCommand)
- self.Cur.execute(SqlCommand)
+ self.Db.CreateEmptyTable(self.Table)
self.ID = self.GetId()
## Insert table
#
# Insert a record into a table
#
def Insert(self, *Args):
self.ID = self.ID + self._ID_STEP_
if self.ID >= (self.IdBase + self._ID_MAX_):
self.ID = self.IdBase + self._ID_STEP_
- Values = ", ".join(str(Arg) for Arg in Args)
- SqlCommand = "insert into %s values(%s, %s)" % (self.Table, self.ID, Values)
- EdkLogger.debug(EdkLogger.DEBUG_5, SqlCommand)
- self.Cur.execute(SqlCommand)
- return self.ID
+ row = [self.ID]
+ row.extend(Args)
+ self.Contents.append(row)
- ## Query table
- #
- # Query all records of the table
- #
- def Query(self):
- SqlCommand = """select * from %s""" % self.Table
- self.Cur.execute(SqlCommand)
- for Rs in self.Cur:
- EdkLogger.verbose(str(Rs))
- TotalCount = self.GetId()
+ return self.ID
- ## Drop a table
- #
- # Drop the table
- #
- def Drop(self):
- SqlCommand = """drop table IF EXISTS %s""" % self.Table
- self.Cur.execute(SqlCommand)
## Get count
#
# Get a count of all records of the table
#
# @retval Count: Total count of all records
#
def GetCount(self):
- SqlCommand = """select count(ID) from %s""" % self.Table
- Record = self.Cur.execute(SqlCommand).fetchall()
- return Record[0][0]
+ tab = self.Db.GetTable(self.Table)
+ return len(tab)
+
def GetId(self):
- SqlCommand = """select max(ID) from %s""" % self.Table
- Record = self.Cur.execute(SqlCommand).fetchall()
- Id = Record[0][0]
+ tab = self.Db.GetTable(self.Table)
+ Id = max([int(item[0]) for item in tab])
if Id is None:
Id = self.IdBase
return Id
## Init the ID of the table
@@ -132,29 +106,30 @@ class Table(object):
#
# @retval RecordSet: The result after executed
#
def Exec(self, SqlCommand):
EdkLogger.debug(EdkLogger.DEBUG_5, SqlCommand)
- self.Cur.execute(SqlCommand)
- RecordSet = self.Cur.fetchall()
+ self.Db.execute(SqlCommand)
+ RecordSet = self.Db.fetchall()
return RecordSet
def SetEndFlag(self):
- self.Exec("insert into %s values(%s)" % (self.Table, self._DUMMY_))
- #
- # Need to execution commit for table data changed.
- #
- self.Cur.connection.commit()
+ Tab = self.Db.GetTable(self.Table)
+ Tab.append(self._DUMMY_)
+
def IsIntegral(self):
- Result = self.Exec("select min(ID) from %s" % (self.Table))
- if Result[0][0] != -1:
+ tab = self.Db.GetTable(self.Table)
+ Id = min([int(item[0]) for item in tab])
+ if Id != -1:
return False
return True
def GetAll(self):
- return self.Exec("select * from %s where ID > 0 order by ID" % (self.Table))
+ tab = self.Db.GetTable(self.Table)
+ return tab
+
## TableFile
#
# This class defined a table used for file
#
@@ -225,26 +200,10 @@ class TableFile(Table):
File.Path,
Model,
File.TimeStamp
)
- ## Get ID of a given file
- #
- # @param FilePath Path of file
- #
- # @retval ID ID value of given file in the table
- #
- def GetFileId(self, File, FromItem=None):
- if FromItem:
- QueryScript = "select ID from %s where FullPath = '%s' and FromItem = %s" % (self.Table, str(File), str(FromItem))
- else:
- QueryScript = "select ID from %s where FullPath = '%s'" % (self.Table, str(File))
- RecordList = self.Exec(QueryScript)
- if len(RecordList) == 0:
- return None
- return RecordList[0][0]
-
## Get type of a given file
#
# @param FileId ID of a file
#
# @retval file_type Model value of given file in the table
@@ -343,11 +302,11 @@ class TableDataModel(Table):
# @retval CrossIndex: CrossIndex of the model
#
def GetCrossIndex(self, ModelName):
CrossIndex = -1
SqlCommand = """select CrossIndex from DataModel where name = '""" + ModelName + """'"""
- self.Cur.execute(SqlCommand)
- for Item in self.Cur:
+ self.Db.execute(SqlCommand)
+ for Item in self.Db:
CrossIndex = Item[0]
return CrossIndex
diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py b/BaseTools/Source/Python/Workspace/MetaFileParser.py
index 804a4aa5cb..a5ae057bd1 100644
--- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
+++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
@@ -1306,12 +1306,11 @@ class DscParser(MetaFileParser):
MODEL_UNKNOWN : self._Skip,
MODEL_META_DATA_USER_EXTENSION : self._SkipUserExtension,
MODEL_META_DATA_CONDITIONAL_STATEMENT_ERROR : self._ProcessError,
}
- self._Table = MetaFileStorage(self._RawTable.Cur, self.MetaFile, MODEL_FILE_DSC, True)
- self._Table.Create()
+ self._Table = MetaFileStorage(self._RawTable.DB, self.MetaFile, MODEL_FILE_DSC, True)
self._DirectiveStack = []
self._DirectiveEvalStack = []
self._FileWithError = self.MetaFile
self._FileLocalMacros = {}
self._SectionsMacroDict.clear()
@@ -1566,11 +1565,11 @@ class DscParser(MetaFileParser):
FromItem = self._Content[self._ContentIndex - 1][0]
if self._InSubsection:
Owner = self._Content[self._ContentIndex - 1][8]
else:
Owner = self._Content[self._ContentIndex - 1][0]
- IncludedFileTable = MetaFileStorage(self._Table.Cur, IncludedFile1, MODEL_FILE_DSC, False, FromItem=FromItem)
+ IncludedFileTable = MetaFileStorage(self._RawTable.DB, IncludedFile1, MODEL_FILE_DSC, False, FromItem=FromItem)
Parser = DscParser(IncludedFile1, self._FileType, self._Arch, IncludedFileTable,
Owner=Owner, From=FromItem)
self.IncludedFiles.add (IncludedFile1)
diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py b/BaseTools/Source/Python/Workspace/MetaFileTable.py
index e0a0b8d923..081970dba8 100644
--- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
+++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
@@ -18,59 +18,64 @@ from __future__ import absolute_import
import uuid
import Common.EdkLogger as EdkLogger
from Common.BuildToolError import FORMAT_INVALID
-from .MetaDataTable import Table, TableFile
-from .MetaDataTable import ConvertToSqlString
from CommonDataClass.DataClass import MODEL_FILE_DSC, MODEL_FILE_DEC, MODEL_FILE_INF, \
MODEL_FILE_OTHERS
from Common.DataType import *
-class MetaFileTable(Table):
+class MetaFileTable():
# TRICK: use file ID as the part before '.'
_ID_STEP_ = 0.00000001
_ID_MAX_ = 0.99999999
## Constructor
- def __init__(self, Cursor, MetaFile, FileType, Temporary, FromItem=None):
+ def __init__(self, DB, MetaFile, FileType, Temporary, FromItem=None):
self.MetaFile = MetaFile
-
- self._FileIndexTable = TableFile(Cursor)
- self._FileIndexTable.Create(False)
-
- FileId = self._FileIndexTable.GetFileId(MetaFile, FromItem)
- if not FileId:
- FileId = self._FileIndexTable.InsertFile(MetaFile, FileType, FromItem)
-
+ self.TableName = ""
+ self.DB = DB
+ self._NumpyTab = None
+ self.FileId = len(DB.TblFile)
+ self.ID = self.FileId
+ self.CurrentContent = []
+ DB.TblFile.append([MetaFile.Name,
+ MetaFile.Ext,
+ MetaFile.Dir,
+ MetaFile.Path,
+ FileType,
+ MetaFile.TimeStamp,
+ FromItem])
if Temporary:
- TableName = "_%s_%s_%s" % (FileType, FileId, uuid.uuid4().hex)
+ self.TableName = "_%s_%s_%s" % (FileType, len(DB.TblFile), uuid.uuid4().hex)
else:
- TableName = "_%s_%s" % (FileType, FileId)
-
- #Table.__init__(self, Cursor, TableName, FileId, False)
- Table.__init__(self, Cursor, TableName, FileId, Temporary)
- self.Create(not self.IsIntegrity())
+ self.TableName = "_%s_%s" % (FileType, len(DB.TblFile))
def IsIntegrity(self):
try:
TimeStamp = self.MetaFile.TimeStamp
- Result = self.Cur.execute("select ID from %s where ID<0" % (self.Table)).fetchall()
+ Result = int(self.CurrentContent[-1][0]) < 0
if not Result:
# update the timestamp in database
- self._FileIndexTable.SetFileTimeStamp(self.IdBase, TimeStamp)
+ self.DB.SetFileTimeStamp(self.FileId, TimeStamp)
return False
- if TimeStamp != self._FileIndexTable.GetFileTimeStamp(self.IdBase):
+ if TimeStamp != self.DB.GetFileTimeStamp(self.FileId):
# update the timestamp in database
- self._FileIndexTable.SetFileTimeStamp(self.IdBase, TimeStamp)
+ self.DB.SetFileTimeStamp(self.FileId, TimeStamp)
return False
except Exception as Exc:
EdkLogger.debug(EdkLogger.DEBUG_5, str(Exc))
return False
return True
+ def SetEndFlag(self):
+ self.CurrentContent.append(self._DUMMY_)
+
+ def GetAll(self):
+ return [item for item in self.CurrentContent if item[0] > 0 ]
+
## Python class representation of table storing module data
class ModuleTable(MetaFileTable):
_ID_STEP_ = 0.00000001
_ID_MAX_ = 0.99999999
_COLUMN_ = '''
@@ -87,15 +92,15 @@ class ModuleTable(MetaFileTable):
EndLine INTEGER NOT NULL,
EndColumn INTEGER NOT NULL,
Enabled INTEGER DEFAULT 0
'''
# used as table end flag, in case the changes to database is not committed to db file
- _DUMMY_ = "-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -1"
+ _DUMMY_ = [-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -1]
## Constructor
- def __init__(self, Cursor, MetaFile, Temporary):
- MetaFileTable.__init__(self, Cursor, MetaFile, MODEL_FILE_INF, Temporary)
+ def __init__(self, Db, MetaFile, Temporary):
+ MetaFileTable.__init__(self, Db, MetaFile, MODEL_FILE_INF, Temporary)
## Insert a record into table Inf
#
# @param Model: Model of a Inf item
# @param Value1: Value1 of a Inf item
@@ -110,48 +115,62 @@ class ModuleTable(MetaFileTable):
# @param EndColumn: EndColumn of a Inf item
# @param Enabled: If this item enabled
#
def Insert(self, Model, Value1, Value2, Value3, Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON,
BelongsToItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1, EndColumn=-1, Enabled=0):
- (Value1, Value2, Value3, Scope1, Scope2) = ConvertToSqlString((Value1, Value2, Value3, Scope1, Scope2))
- return Table.Insert(
- self,
- Model,
- Value1,
- Value2,
- Value3,
- Scope1,
- Scope2,
- BelongsToItem,
- StartLine,
- StartColumn,
- EndLine,
- EndColumn,
- Enabled
- )
+
+ (Value1, Value2, Value3, Scope1, Scope2) = (Value1.strip(), Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip())
+ self.ID = self.ID + self._ID_STEP_
+ if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
+ self.ID = MODEL_FILE_INF + self._ID_STEP_
+
+ row = [ self.ID,
+ Model,
+ Value1,
+ Value2,
+ Value3,
+ Scope1,
+ Scope2,
+ BelongsToItem,
+ StartLine,
+ StartColumn,
+ EndLine,
+ EndColumn,
+ Enabled
+ ]
+ self.CurrentContent.append(row)
+ return self.ID
+
## Query table
#
# @param Model: The Model of Record
# @param Arch: The Arch attribute of Record
# @param Platform The Platform attribute of Record
#
# @retval: A recordSet of all found records
#
def Query(self, Model, Arch=None, Platform=None, BelongsToItem=None):
- ConditionString = "Model=%s AND Enabled>=0" % Model
- ValueString = "Value1,Value2,Value3,Scope1,Scope2,ID,StartLine"
+
+ QueryTab = self.CurrentContent
+ result = [item for item in QueryTab if item[1] == Model and item[-1]>=0 ]
if Arch is not None and Arch != TAB_ARCH_COMMON:
- ConditionString += " AND (Scope1='%s' OR Scope1='COMMON')" % Arch
+ ArchList = set(['COMMON'])
+ ArchList.add(Arch)
+ result = [item for item in result if item[5] in ArchList]
+
if Platform is not None and Platform != TAB_COMMON:
- ConditionString += " AND (Scope2='%s' OR Scope2='COMMON' OR Scope2='DEFAULT')" % Platform
+ Platformlist = set( ['COMMON','DEFAULT'])
+ Platformlist.add(Platform)
+ result = [item for item in result if item[6] in Platformlist]
+
if BelongsToItem is not None:
- ConditionString += " AND BelongsToItem=%s" % BelongsToItem
+ result = [item for item in result if item[7] == BelongsToItem]
- SqlCommand = "SELECT %s FROM %s WHERE %s" % (ValueString, self.Table, ConditionString)
- return self.Exec(SqlCommand)
+ result = [ [r[2],r[3],r[4],r[5],r[6],r[0],r[9]] for r in result ]
+ return result
## Python class representation of table storing package data
class PackageTable(MetaFileTable):
_COLUMN_ = '''
ID REAL PRIMARY KEY,
@@ -167,11 +186,11 @@ class PackageTable(MetaFileTable):
EndLine INTEGER NOT NULL,
EndColumn INTEGER NOT NULL,
Enabled INTEGER DEFAULT 0
'''
# used as table end flag, in case the changes to database is not committed to db file
- _DUMMY_ = "-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -1"
+ _DUMMY_ = [-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -1]
## Constructor
def __init__(self, Cursor, MetaFile, Temporary):
MetaFileTable.__init__(self, Cursor, MetaFile, MODEL_FILE_DEC, Temporary)
@@ -192,52 +211,60 @@ class PackageTable(MetaFileTable):
# @param EndColumn: EndColumn of a Dec item
# @param Enabled: If this item enabled
#
def Insert(self, Model, Value1, Value2, Value3, Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON,
BelongsToItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1, EndColumn=-1, Enabled=0):
- (Value1, Value2, Value3, Scope1, Scope2) = ConvertToSqlString((Value1, Value2, Value3, Scope1, Scope2))
- return Table.Insert(
- self,
- Model,
- Value1,
- Value2,
- Value3,
- Scope1,
- Scope2,
- BelongsToItem,
- StartLine,
- StartColumn,
- EndLine,
- EndColumn,
- Enabled
- )
+ (Value1, Value2, Value3, Scope1, Scope2) = (Value1.strip(), Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip())
+ self.ID = self.ID + self._ID_STEP_
+ if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
+ self.ID = MODEL_FILE_INF + self._ID_STEP_
+
+ row = [ self.ID,
+ Model,
+ Value1,
+ Value2,
+ Value3,
+ Scope1,
+ Scope2,
+ BelongsToItem,
+ StartLine,
+ StartColumn,
+ EndLine,
+ EndColumn,
+ Enabled
+ ]
+ self.CurrentContent.append(row)
+ return self.ID
## Query table
#
# @param Model: The Model of Record
# @param Arch: The Arch attribute of Record
#
# @retval: A recordSet of all found records
#
def Query(self, Model, Arch=None):
- ConditionString = "Model=%s AND Enabled>=0" % Model
- ValueString = "Value1,Value2,Value3,Scope1,Scope2,ID,StartLine"
+
+ QueryTab = self.CurrentContent
+ result = [item for item in QueryTab if item[1] == Model and item[-1]>=0 ]
if Arch is not None and Arch != TAB_ARCH_COMMON:
- ConditionString += " AND (Scope1='%s' OR Scope1='COMMON')" % Arch
+ ArchList = set(['COMMON'])
+ ArchList.add(Arch)
+ result = [item for item in result if item[5] in ArchList]
- SqlCommand = "SELECT %s FROM %s WHERE %s" % (ValueString, self.Table, ConditionString)
- return self.Exec(SqlCommand)
+ return [[r[2], r[3], r[4], r[5], r[6], r[0], r[8]] for r in result]
def GetValidExpression(self, TokenSpaceGuid, PcdCName):
- SqlCommand = "select Value1,StartLine from %s WHERE Value2='%s' and Value3='%s'" % (self.Table, TokenSpaceGuid, PcdCName)
- self.Cur.execute(SqlCommand)
+
+ QueryTab = self.CurrentContent
+ result = [[item[2], item[8]] for item in QueryTab if item[3] == TokenSpaceGuid and item[4] == PcdCName]
validateranges = []
validlists = []
expressions = []
try:
- for row in self.Cur:
+ for row in result:
comment = row[0]
LineNum = row[1]
comment = comment.strip("#")
comment = comment.strip()
@@ -281,11 +308,11 @@ class PlatformTable(MetaFileTable):
EndLine INTEGER NOT NULL,
EndColumn INTEGER NOT NULL,
Enabled INTEGER DEFAULT 0
'''
# used as table end flag, in case the changes to database is not committed to db file
- _DUMMY_ = "-1, -1, '====', '====', '====', '====', '====','====', -1, -1, -1, -1, -1, -1, -1"
+ _DUMMY_ = [-1, -1, '====', '====', '====', '====', '====','====', -1, -1, -1, -1, -1, -1, -1]
## Constructor
def __init__(self, Cursor, MetaFile, Temporary, FromItem=0):
MetaFileTable.__init__(self, Cursor, MetaFile, MODEL_FILE_DSC, Temporary, FromItem)
@@ -307,28 +334,34 @@ class PlatformTable(MetaFileTable):
# @param EndColumn: EndColumn of a Dsc item
# @param Enabled: If this item enabled
#
def Insert(self, Model, Value1, Value2, Value3, Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON, Scope3=TAB_DEFAULT_STORES_DEFAULT,BelongsToItem=-1,
FromItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1, EndColumn=-1, Enabled=1):
- (Value1, Value2, Value3, Scope1, Scope2, Scope3) = ConvertToSqlString((Value1, Value2, Value3, Scope1, Scope2, Scope3))
- return Table.Insert(
- self,
- Model,
- Value1,
- Value2,
- Value3,
- Scope1,
- Scope2,
- Scope3,
- BelongsToItem,
- FromItem,
- StartLine,
- StartColumn,
- EndLine,
- EndColumn,
- Enabled
- )
+ (Value1, Value2, Value3, Scope1, Scope2, Scope3) = (Value1.strip(), Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip(), Scope3.strip())
+ self.ID = self.ID + self._ID_STEP_
+ if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
+ self.ID = MODEL_FILE_INF + self._ID_STEP_
+
+ row = [ self.ID,
+ Model,
+ Value1,
+ Value2,
+ Value3,
+ Scope1,
+ Scope2,
+ Scope3,
+ BelongsToItem,
+ FromItem,
+ StartLine,
+ StartColumn,
+ EndLine,
+ EndColumn,
+ Enabled
+ ]
+ self.CurrentContent.append(row)
+ return self.ID
+
## Query table
#
# @param Model: The Model of Record
# @param Scope1: Arch of a Dsc item
@@ -337,34 +370,37 @@ class PlatformTable(MetaFileTable):
# @param FromItem: The item belongs to which dsc file
#
# @retval: A recordSet of all found records
#
def Query(self, Model, Scope1=None, Scope2=None, BelongsToItem=None, FromItem=None):
- ConditionString = "Model=%s AND Enabled>0" % Model
- ValueString = "Value1,Value2,Value3,Scope1,Scope2,Scope3,ID,StartLine"
+
+ QueryTab = self.CurrentContent
+ result = [item for item in QueryTab if item[1] == Model and item[-1]>0 ]
if Scope1 is not None and Scope1 != TAB_ARCH_COMMON:
- ConditionString += " AND (Scope1='%s' OR Scope1='COMMON')" % Scope1
- if Scope2 is not None and Scope2 != TAB_COMMON:
- # Cover the case that CodeBase is 'COMMON' for BuildOptions section
+ Sc1 = set(['COMMON'])
+ Sc1.add(Scope1)
+ result = [item for item in result if item[5] in Sc1]
+ Sc2 = set( ['COMMON','DEFAULT'])
+ if Scope2 and Scope2 != TAB_COMMON:
if '.' in Scope2:
Index = Scope2.index('.')
NewScope = TAB_COMMON + Scope2[Index:]
- ConditionString += " AND (Scope2='%s' OR Scope2='COMMON' OR Scope2='DEFAULT' OR Scope2='%s')" % (Scope2, NewScope)
- else:
- ConditionString += " AND (Scope2='%s' OR Scope2='COMMON' OR Scope2='DEFAULT')" % Scope2
+ Sc2.add(NewScope)
+ Sc2.add(Scope2)
+ result = [item for item in result if item[6] in Sc2]
if BelongsToItem is not None:
- ConditionString += " AND BelongsToItem=%s" % BelongsToItem
+ result = [item for item in result if item[8] == BelongsToItem]
else:
- ConditionString += " AND BelongsToItem<0"
-
+ result = [item for item in result if item[8] < 0]
if FromItem is not None:
- ConditionString += " AND FromItem=%s" % FromItem
+ result = [item for item in result if item[9] == FromItem]
+
+ result = [ [r[2],r[3],r[4],r[5],r[6],r[7],r[0],r[9]] for r in result ]
+ return result
- SqlCommand = "SELECT %s FROM %s WHERE %s" % (ValueString, self.Table, ConditionString)
- return self.Exec(SqlCommand)
## Factory class to produce different storage for different type of meta-file
class MetaFileStorage(object):
_FILE_TABLE_ = {
MODEL_FILE_INF : ModuleTable,
diff --git a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
index 3bb287b8b2..bb6dafe9cc 100644
--- a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
+++ b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
@@ -14,11 +14,10 @@
##
# Import Modules
#
from __future__ import absolute_import
-import sqlite3
from Common.StringUtils import *
from Common.DataType import *
from Common.Misc import *
from types import *
@@ -116,11 +115,11 @@ class WorkspaceDatabase(object):
# get the parser ready for this file
MetaFile = self._FILE_PARSER_[FileType](
FilePath,
FileType,
Arch,
- MetaFileStorage(self.WorkspaceDb.Cur, FilePath, FileType)
+ MetaFileStorage(self.WorkspaceDb, FilePath, FileType)
)
# alwasy do post-process, in case of macros change
MetaFile.DoPostProcess()
# object the build is based on
BuildObject = self._GENERATOR_[FileType](
@@ -147,137 +146,27 @@ class WorkspaceDatabase(object):
#
# @param DbPath Path of database file
# @param GlobalMacros Global macros used for replacement during file parsing
# @prarm RenewDb=False Create new database file if it's already there
#
- def __init__(self, DbPath, RenewDb=False):
- self._DbClosedFlag = False
- if not DbPath:
- DbPath = os.path.normpath(mws.join(GlobalData.gWorkspace, 'Conf', GlobalData.gDatabasePath))
-
- # don't create necessary path for db in memory
- if DbPath != ':memory:':
- DbDir = os.path.split(DbPath)[0]
- if not os.path.exists(DbDir):
- os.makedirs(DbDir)
-
- # remove db file in case inconsistency between db and file in file system
- if self._CheckWhetherDbNeedRenew(RenewDb, DbPath):
- os.remove(DbPath)
-
- # create db with optimized parameters
- self.Conn = sqlite3.connect(DbPath, isolation_level='DEFERRED')
- self.Conn.execute("PRAGMA synchronous=OFF")
- self.Conn.execute("PRAGMA temp_store=MEMORY")
- self.Conn.execute("PRAGMA count_changes=OFF")
- self.Conn.execute("PRAGMA cache_size=8192")
- #self.Conn.execute("PRAGMA page_size=8192")
-
- # to avoid non-ascii character conversion issue
- self.Conn.text_factory = str
- self.Cur = self.Conn.cursor()
-
+ def __init__(self):
+ self.DB = dict()
# create table for internal uses
- self.TblDataModel = TableDataModel(self.Cur)
- self.TblFile = TableFile(self.Cur)
+ self.TblDataModel = DataClass.MODEL_LIST
+ self.TblFile = []
self.Platform = None
# conversion object for build or file format conversion purpose
self.BuildObject = WorkspaceDatabase.BuildObjectFactory(self)
self.TransformObject = WorkspaceDatabase.TransformObjectFactory(self)
- ## Check whether workspace database need to be renew.
- # The renew reason maybe:
- # 1) If user force to renew;
- # 2) If user do not force renew, and
- # a) If the time of last modified python source is newer than database file;
- # b) If the time of last modified frozen executable file is newer than database file;
- #
- # @param force User force renew database
- # @param DbPath The absolute path of workspace database file
- #
- # @return Bool value for whether need renew workspace databse
- #
- def _CheckWhetherDbNeedRenew (self, force, DbPath):
- # if database does not exist, we need do nothing
- if not os.path.exists(DbPath): return False
-
- # if user force to renew database, then not check whether database is out of date
- if force: return True
-
- #
- # Check the time of last modified source file or build.exe
- # if is newer than time of database, then database need to be re-created.
- #
- timeOfToolModified = 0
- if hasattr(sys, "frozen"):
- exePath = os.path.abspath(sys.executable)
- timeOfToolModified = os.stat(exePath).st_mtime
- else:
- curPath = os.path.dirname(__file__) # curPath is the path of WorkspaceDatabase.py
- rootPath = os.path.split(curPath)[0] # rootPath is root path of python source, such as /BaseTools/Source/Python
- if rootPath == "" or rootPath is None:
- EdkLogger.verbose("\nFail to find the root path of build.exe or python sources, so can not \
-determine whether database file is out of date!\n")
-
- # walk the root path of source or build's binary to get the time last modified.
-
- for root, dirs, files in os.walk (rootPath):
- for dir in dirs:
- # bypass source control folder
- if dir.lower() in [".svn", "_svn", "cvs"]:
- dirs.remove(dir)
+ def SetFileTimeStamp(self,FileId,TimeStamp):
+ self.TblFile[FileId][6] = TimeStamp
- for file in files:
- ext = os.path.splitext(file)[1]
- if ext.lower() == ".py": # only check .py files
- fd = os.stat(os.path.join(root, file))
- if timeOfToolModified < fd.st_mtime:
- timeOfToolModified = fd.st_mtime
- if timeOfToolModified > os.stat(DbPath).st_mtime:
- EdkLogger.verbose("\nWorkspace database is out of data!")
- return True
+ def GetFileTimeStamp(self,FileId):
+ return self.TblFile[FileId][6]
- return False
-
- ## Initialize build database
- def InitDatabase(self):
- EdkLogger.verbose("\nInitialize build database started ...")
-
- #
- # Create new tables
- #
- self.TblDataModel.Create(False)
- self.TblFile.Create(False)
-
- #
- # Initialize table DataModel
- #
- self.TblDataModel.InitTable()
- EdkLogger.verbose("Initialize build database ... DONE!")
-
- ## Query a table
- #
- # @param Table: The instance of the table to be queried
- #
- def QueryTable(self, Table):
- Table.Query()
-
- def __del__(self):
- self.Close()
-
- ## Close entire database
- #
- # Commit all first
- # Close the connection and cursor
- #
- def Close(self):
- if not self._DbClosedFlag:
- self.Conn.commit()
- self.Cur.close()
- self.Conn.close()
- self._DbClosedFlag = True
## Summarize all packages in the database
def GetPackageList(self, Platform, Arch, TargetName, ToolChainTag):
self.Platform = Platform
PackageList = []
@@ -302,11 +191,11 @@ determine whether database file is out of date!\n")
return PackageList
## Summarize all platforms in the database
def PlatformList(self):
RetVal = []
- for PlatformFile in self.TblFile.GetFileList(MODEL_FILE_DSC):
+ for PlatformFile in [item[3] for item in self.TblFile if item[5] == MODEL_FILE_DSC]:
try:
RetVal.append(self.BuildObject[PathClass(PlatformFile), TAB_COMMON])
except:
pass
return RetVal
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index d74082fc26..317d6a2728 100644
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -41,11 +41,11 @@ from Common.TargetTxtClassObject import *
from Common.ToolDefClassObject import *
from Common.DataType import *
from Common.BuildVersion import gBUILD_VERSION
from AutoGen.AutoGen import *
from Common.BuildToolError import *
-from Workspace.WorkspaceDatabase import *
+from Workspace.WorkspaceDatabase import WorkspaceDatabase
from Common.MultipleWorkspace import MultipleWorkspace as mws
from BuildReport import BuildReport
from GenPatchPcdTable.GenPatchPcdTable import *
from PatchPcdValue.PatchPcdValue import *
@@ -825,14 +825,11 @@ class Build():
# Get standard WORKSPACE/Conf use the absolute path to the WORKSPACE/Conf
ConfDirectoryPath = mws.join(self.WorkspaceDir, 'Conf')
GlobalData.gConfDirectory = ConfDirectoryPath
GlobalData.gDatabasePath = os.path.normpath(os.path.join(ConfDirectoryPath, GlobalData.gDatabasePath))
- if BuildOptions.DisableCache:
- self.Db = WorkspaceDatabase(":memory:")
- else:
- self.Db = WorkspaceDatabase(GlobalData.gDatabasePath, self.Reparse)
+ self.Db = WorkspaceDatabase()
self.BuildDatabase = self.Db.BuildObject
self.Platform = None
self.ToolChainFamily = None
self.LoadFixAddress = 0
self.UniFlag = BuildOptions.Flag
@@ -981,13 +978,10 @@ class Build():
# Allow case-insensitive for those from command line or configuration file
ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
if ErrorCode != 0:
EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
- # create metafile database
- if not self.Db_Flag:
- self.Db.InitDatabase()
def InitPreBuild(self):
self.LoadConfiguration()
ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
if ErrorCode != 0:
@@ -1002,11 +996,10 @@ class Build():
if self.ToolChainFamily:
GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[0]
if 'PREBUILD' in GlobalData.gCommandLineDefines:
self.Prebuild = GlobalData.gCommandLineDefines.get('PREBUILD')
else:
- self.Db.InitDatabase()
self.Db_Flag = True
Platform = self.Db.MapPlatform(str(self.PlatformFile))
self.Prebuild = str(Platform.Prebuild)
if self.Prebuild:
PrebuildList = []
@@ -2081,17 +2074,11 @@ class Build():
if BuildTask.HasError():
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
self.MakeTime += int(round((time.time() - MakeStart)))
MakeContiue = time.time()
- #
- # Save temp tables to a TmpTableDict.
- #
- for Key in Wa.BuildDatabase._CACHE_:
- if Wa.BuildDatabase._CACHE_[Key]._RawData and Wa.BuildDatabase._CACHE_[Key]._RawData._Table and Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table:
- if TemporaryTablePattern.match(Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table):
- TmpTableDict[Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table] = Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Cur
+
#
#
# All modules have been put in build tasks queue. Tell task scheduler
# to exit if all tasks are completed
#
@@ -2229,11 +2216,10 @@ class Build():
else:
self.SpawnMode = False
self._BuildModule()
if self.Target == 'cleanall':
- self.Db.Close()
RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True)
def CreateAsBuiltInf(self):
for Module in self.BuildModules:
Module.CreateAsBuiltInf()
@@ -2490,14 +2476,11 @@ def Main():
MyBuild = Build(Target, Workspace, Option)
GlobalData.gCommandLineDefines['ARCH'] = ' '.join(MyBuild.ArchList)
if not (MyBuild.LaunchPrebuildFlag and os.path.exists(MyBuild.PlatformBuildPath)):
MyBuild.Launch()
- # Drop temp tables to avoid database locked.
- for TmpTableName in TmpTableDict:
- SqlCommand = """drop table IF EXISTS %s""" % TmpTableName
- TmpTableDict[TmpTableName].execute(SqlCommand)
+
#MyBuild.DumpBuildData()
#
# All job done, no error found and no exception raised
#
BuildError = False
@@ -2565,11 +2548,11 @@ def Main():
else:
BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration)
if MyBuild is not None:
if not BuildError:
MyBuild.BuildReport.GenerateReport(BuildDurationStr, LogBuildTime(MyBuild.AutoGenTime), LogBuildTime(MyBuild.MakeTime), LogBuildTime(MyBuild.GenFdsTime))
- MyBuild.Db.Close()
+
EdkLogger.SetLevel(EdkLogger.QUIET)
EdkLogger.quiet("\n- %s -" % Conclusion)
EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d %Y", time.localtime()))
EdkLogger.quiet("Build total time: %s\n" % BuildDurationStr)
return ReturnCode
--
2.19.1.windows.1
^ permalink raw reply related [flat|nested] 3+ messages in thread
* Re: [Patch V2] BaseTools: Replace the sqlite database with list
2018-11-09 8:41 [Patch V2] BaseTools: Replace the sqlite database with list BobCF
@ 2018-11-09 15:24 ` Carsey, Jaben
2018-11-11 0:43 ` Feng, Bob C
0 siblings, 1 reply; 3+ messages in thread
From: Carsey, Jaben @ 2018-11-09 15:24 UTC (permalink / raw)
To: Feng, Bob C, edk2-devel@lists.01.org; +Cc: Gao, Liming
This is much easier to review! Thanks!
I am good with the code change.
I wonder: If we do not have the DB does it take longer if I do build, clean, build than it did before?
> -----Original Message-----
> From: Feng, Bob C
> Sent: Friday, November 09, 2018 12:42 AM
> To: edk2-devel@lists.01.org
> Cc: Gao, Liming <liming.gao@intel.com>; Carsey, Jaben
> <jaben.carsey@intel.com>
> Subject: [Patch V2] BaseTools: Replace the sqlite database with list
> Importance: High
>
> https://bugzilla.tianocore.org/show_bug.cgi?id=1288
>
> [V2]
> Optimize this patch so that it can be easy to review.
> This patch is just apply the change to original files while
> not create new similar files.
>
> [V1]
> This patch is one of build tool performance improvement
> series patches.
>
> This patch is going to use python list to store the parser data
> instead of using sqlite database.
>
> The replacement solution is as below:
>
> SQL insert: list.append()
> SQL select: list comprehension. for example:
> Select * from table where field = “something”
> ->
> [ item for item in table if item[3] == “something”]
>
> SQL update: python map function. for example:
> Update table set field1=newvalue where filed2 = “something”.
> -> map(lambda x: x[1] = newvalue,
> [item for item in table if item[2] == “something”])
>
> SQL delete: list comprehension.
>
> With this change, We can save the time of interpreting SQL statement
> and the time of write database to file system
>
> Contributed-under: TianoCore Contribution Agreement 1.1
> Signed-off-by: BobCF <bob.c.feng@intel.com>
> Cc: Liming Gao <liming.gao@intel.com>
> Cc: Jaben Carsey <jaben.carsey@intel.com>
> ---
> BaseTools/Source/Python/GenFds/GenFds.py | 3 +-
> .../Source/Python/Workspace/MetaDataTable.py | 93 ++-----
> .../Source/Python/Workspace/MetaFileParser.py | 5 +-
> .../Source/Python/Workspace/MetaFileTable.py | 248 ++++++++++--------
> .../Python/Workspace/WorkspaceDatabase.py | 131 +--------
> BaseTools/Source/Python/build/build.py | 27 +-
> 6 files changed, 186 insertions(+), 321 deletions(-)
>
> diff --git a/BaseTools/Source/Python/GenFds/GenFds.py
> b/BaseTools/Source/Python/GenFds/GenFds.py
> index 0c8091b798..4fd96706af 100644
> --- a/BaseTools/Source/Python/GenFds/GenFds.py
> +++ b/BaseTools/Source/Python/GenFds/GenFds.py
> @@ -222,12 +222,11 @@ def main():
> if "TOOL_CHAIN_TAG" not in GlobalData.gGlobalDefines:
> GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] =
> GenFdsGlobalVariable.ToolChainTag
>
> """call Workspace build create database"""
> GlobalData.gDatabasePath =
> os.path.normpath(os.path.join(ConfDirectoryPath,
> GlobalData.gDatabasePath))
> - BuildWorkSpace = WorkspaceDatabase(GlobalData.gDatabasePath)
> - BuildWorkSpace.InitDatabase()
> + BuildWorkSpace = WorkspaceDatabase()
>
> #
> # Get files real name in workspace dir
> #
> GlobalData.gAllFiles = DirCache(Workspace)
> diff --git a/BaseTools/Source/Python/Workspace/MetaDataTable.py
> b/BaseTools/Source/Python/Workspace/MetaDataTable.py
> index bd751eadfb..8becddbe08 100644
> --- a/BaseTools/Source/Python/Workspace/MetaDataTable.py
> +++ b/BaseTools/Source/Python/Workspace/MetaDataTable.py
> @@ -37,84 +37,58 @@ class Table(object):
> _COLUMN_ = ''
> _ID_STEP_ = 1
> _ID_MAX_ = 0x80000000
> _DUMMY_ = 0
>
> - def __init__(self, Cursor, Name='', IdBase=0, Temporary=False):
> - self.Cur = Cursor
> + def __init__(self, Db, Name='', IdBase=0, Temporary=False):
> + self.Db = Db
> self.Table = Name
> self.IdBase = int(IdBase)
> self.ID = int(IdBase)
> self.Temporary = Temporary
> + self.Contents = []
>
> def __str__(self):
> return self.Table
>
> ## Create table
> #
> # Create a table
> #
> def Create(self, NewTable=True):
> - if NewTable:
> - self.Drop()
> -
> - if self.Temporary:
> - SqlCommand = """create temp table IF NOT EXISTS %s (%s)""" %
> (self.Table, self._COLUMN_)
> - else:
> - SqlCommand = """create table IF NOT EXISTS %s (%s)""" % (self.Table,
> self._COLUMN_)
> - EdkLogger.debug(EdkLogger.DEBUG_8, SqlCommand)
> - self.Cur.execute(SqlCommand)
> + self.Db.CreateEmptyTable(self.Table)
> self.ID = self.GetId()
>
> ## Insert table
> #
> # Insert a record into a table
> #
> def Insert(self, *Args):
> self.ID = self.ID + self._ID_STEP_
> if self.ID >= (self.IdBase + self._ID_MAX_):
> self.ID = self.IdBase + self._ID_STEP_
> - Values = ", ".join(str(Arg) for Arg in Args)
> - SqlCommand = "insert into %s values(%s, %s)" % (self.Table, self.ID,
> Values)
> - EdkLogger.debug(EdkLogger.DEBUG_5, SqlCommand)
> - self.Cur.execute(SqlCommand)
> - return self.ID
> + row = [self.ID]
> + row.extend(Args)
> + self.Contents.append(row)
>
> - ## Query table
> - #
> - # Query all records of the table
> - #
> - def Query(self):
> - SqlCommand = """select * from %s""" % self.Table
> - self.Cur.execute(SqlCommand)
> - for Rs in self.Cur:
> - EdkLogger.verbose(str(Rs))
> - TotalCount = self.GetId()
> + return self.ID
>
> - ## Drop a table
> - #
> - # Drop the table
> - #
> - def Drop(self):
> - SqlCommand = """drop table IF EXISTS %s""" % self.Table
> - self.Cur.execute(SqlCommand)
>
> ## Get count
> #
> # Get a count of all records of the table
> #
> # @retval Count: Total count of all records
> #
> def GetCount(self):
> - SqlCommand = """select count(ID) from %s""" % self.Table
> - Record = self.Cur.execute(SqlCommand).fetchall()
> - return Record[0][0]
> + tab = self.Db.GetTable(self.Table)
> + return len(tab)
> +
>
> def GetId(self):
> - SqlCommand = """select max(ID) from %s""" % self.Table
> - Record = self.Cur.execute(SqlCommand).fetchall()
> - Id = Record[0][0]
> + tab = self.Db.GetTable(self.Table)
> + Id = max([int(item[0]) for item in tab])
> if Id is None:
> Id = self.IdBase
> return Id
>
> ## Init the ID of the table
> @@ -132,29 +106,30 @@ class Table(object):
> #
> # @retval RecordSet: The result after executed
> #
> def Exec(self, SqlCommand):
> EdkLogger.debug(EdkLogger.DEBUG_5, SqlCommand)
> - self.Cur.execute(SqlCommand)
> - RecordSet = self.Cur.fetchall()
> + self.Db.execute(SqlCommand)
> + RecordSet = self.Db.fetchall()
> return RecordSet
>
> def SetEndFlag(self):
> - self.Exec("insert into %s values(%s)" % (self.Table, self._DUMMY_))
> - #
> - # Need to execution commit for table data changed.
> - #
> - self.Cur.connection.commit()
> + Tab = self.Db.GetTable(self.Table)
> + Tab.append(self._DUMMY_)
> +
>
> def IsIntegral(self):
> - Result = self.Exec("select min(ID) from %s" % (self.Table))
> - if Result[0][0] != -1:
> + tab = self.Db.GetTable(self.Table)
> + Id = min([int(item[0]) for item in tab])
> + if Id != -1:
> return False
> return True
>
> def GetAll(self):
> - return self.Exec("select * from %s where ID > 0 order by ID" %
> (self.Table))
> + tab = self.Db.GetTable(self.Table)
> + return tab
> +
>
> ## TableFile
> #
> # This class defined a table used for file
> #
> @@ -225,26 +200,10 @@ class TableFile(Table):
> File.Path,
> Model,
> File.TimeStamp
> )
>
> - ## Get ID of a given file
> - #
> - # @param FilePath Path of file
> - #
> - # @retval ID ID value of given file in the table
> - #
> - def GetFileId(self, File, FromItem=None):
> - if FromItem:
> - QueryScript = "select ID from %s where FullPath = '%s' and FromItem
> = %s" % (self.Table, str(File), str(FromItem))
> - else:
> - QueryScript = "select ID from %s where FullPath = '%s'" % (self.Table,
> str(File))
> - RecordList = self.Exec(QueryScript)
> - if len(RecordList) == 0:
> - return None
> - return RecordList[0][0]
> -
> ## Get type of a given file
> #
> # @param FileId ID of a file
> #
> # @retval file_type Model value of given file in the table
> @@ -343,11 +302,11 @@ class TableDataModel(Table):
> # @retval CrossIndex: CrossIndex of the model
> #
> def GetCrossIndex(self, ModelName):
> CrossIndex = -1
> SqlCommand = """select CrossIndex from DataModel where name = '"""
> + ModelName + """'"""
> - self.Cur.execute(SqlCommand)
> - for Item in self.Cur:
> + self.Db.execute(SqlCommand)
> + for Item in self.Db:
> CrossIndex = Item[0]
>
> return CrossIndex
>
> diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py
> b/BaseTools/Source/Python/Workspace/MetaFileParser.py
> index 804a4aa5cb..a5ae057bd1 100644
> --- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
> +++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
> @@ -1306,12 +1306,11 @@ class DscParser(MetaFileParser):
> MODEL_UNKNOWN : self._Skip,
> MODEL_META_DATA_USER_EXTENSION :
> self._SkipUserExtension,
> MODEL_META_DATA_CONDITIONAL_STATEMENT_ERROR :
> self._ProcessError,
> }
>
> - self._Table = MetaFileStorage(self._RawTable.Cur, self.MetaFile,
> MODEL_FILE_DSC, True)
> - self._Table.Create()
> + self._Table = MetaFileStorage(self._RawTable.DB, self.MetaFile,
> MODEL_FILE_DSC, True)
> self._DirectiveStack = []
> self._DirectiveEvalStack = []
> self._FileWithError = self.MetaFile
> self._FileLocalMacros = {}
> self._SectionsMacroDict.clear()
> @@ -1566,11 +1565,11 @@ class DscParser(MetaFileParser):
> FromItem = self._Content[self._ContentIndex - 1][0]
> if self._InSubsection:
> Owner = self._Content[self._ContentIndex - 1][8]
> else:
> Owner = self._Content[self._ContentIndex - 1][0]
> - IncludedFileTable = MetaFileStorage(self._Table.Cur, IncludedFile1,
> MODEL_FILE_DSC, False, FromItem=FromItem)
> + IncludedFileTable = MetaFileStorage(self._RawTable.DB,
> IncludedFile1, MODEL_FILE_DSC, False, FromItem=FromItem)
> Parser = DscParser(IncludedFile1, self._FileType, self._Arch,
> IncludedFileTable,
> Owner=Owner, From=FromItem)
>
> self.IncludedFiles.add (IncludedFile1)
>
> diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py
> b/BaseTools/Source/Python/Workspace/MetaFileTable.py
> index e0a0b8d923..081970dba8 100644
> --- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
> +++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
> @@ -18,59 +18,64 @@ from __future__ import absolute_import
> import uuid
>
> import Common.EdkLogger as EdkLogger
> from Common.BuildToolError import FORMAT_INVALID
>
> -from .MetaDataTable import Table, TableFile
> -from .MetaDataTable import ConvertToSqlString
> from CommonDataClass.DataClass import MODEL_FILE_DSC,
> MODEL_FILE_DEC, MODEL_FILE_INF, \
> MODEL_FILE_OTHERS
> from Common.DataType import *
>
> -class MetaFileTable(Table):
> +class MetaFileTable():
> # TRICK: use file ID as the part before '.'
> _ID_STEP_ = 0.00000001
> _ID_MAX_ = 0.99999999
>
> ## Constructor
> - def __init__(self, Cursor, MetaFile, FileType, Temporary,
> FromItem=None):
> + def __init__(self, DB, MetaFile, FileType, Temporary, FromItem=None):
> self.MetaFile = MetaFile
> -
> - self._FileIndexTable = TableFile(Cursor)
> - self._FileIndexTable.Create(False)
> -
> - FileId = self._FileIndexTable.GetFileId(MetaFile, FromItem)
> - if not FileId:
> - FileId = self._FileIndexTable.InsertFile(MetaFile, FileType, FromItem)
> -
> + self.TableName = ""
> + self.DB = DB
> + self._NumpyTab = None
> + self.FileId = len(DB.TblFile)
> + self.ID = self.FileId
> + self.CurrentContent = []
> + DB.TblFile.append([MetaFile.Name,
> + MetaFile.Ext,
> + MetaFile.Dir,
> + MetaFile.Path,
> + FileType,
> + MetaFile.TimeStamp,
> + FromItem])
> if Temporary:
> - TableName = "_%s_%s_%s" % (FileType, FileId, uuid.uuid4().hex)
> + self.TableName = "_%s_%s_%s" % (FileType, len(DB.TblFile),
> uuid.uuid4().hex)
> else:
> - TableName = "_%s_%s" % (FileType, FileId)
> -
> - #Table.__init__(self, Cursor, TableName, FileId, False)
> - Table.__init__(self, Cursor, TableName, FileId, Temporary)
> - self.Create(not self.IsIntegrity())
> + self.TableName = "_%s_%s" % (FileType, len(DB.TblFile))
>
> def IsIntegrity(self):
> try:
> TimeStamp = self.MetaFile.TimeStamp
> - Result = self.Cur.execute("select ID from %s where ID<0" %
> (self.Table)).fetchall()
> + Result = int(self.CurrentContent[-1][0]) < 0
> if not Result:
> # update the timestamp in database
> - self._FileIndexTable.SetFileTimeStamp(self.IdBase, TimeStamp)
> + self.DB.SetFileTimeStamp(self.FileId, TimeStamp)
> return False
>
> - if TimeStamp != self._FileIndexTable.GetFileTimeStamp(self.IdBase):
> + if TimeStamp != self.DB.GetFileTimeStamp(self.FileId):
> # update the timestamp in database
> - self._FileIndexTable.SetFileTimeStamp(self.IdBase, TimeStamp)
> + self.DB.SetFileTimeStamp(self.FileId, TimeStamp)
> return False
> except Exception as Exc:
> EdkLogger.debug(EdkLogger.DEBUG_5, str(Exc))
> return False
> return True
>
> + def SetEndFlag(self):
> + self.CurrentContent.append(self._DUMMY_)
> +
> + def GetAll(self):
> + return [item for item in self.CurrentContent if item[0] > 0 ]
> +
> ## Python class representation of table storing module data
> class ModuleTable(MetaFileTable):
> _ID_STEP_ = 0.00000001
> _ID_MAX_ = 0.99999999
> _COLUMN_ = '''
> @@ -87,15 +92,15 @@ class ModuleTable(MetaFileTable):
> EndLine INTEGER NOT NULL,
> EndColumn INTEGER NOT NULL,
> Enabled INTEGER DEFAULT 0
> '''
> # used as table end flag, in case the changes to database is not committed
> to db file
> - _DUMMY_ = "-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -
> 1"
> + _DUMMY_ = [-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -
> 1]
>
> ## Constructor
> - def __init__(self, Cursor, MetaFile, Temporary):
> - MetaFileTable.__init__(self, Cursor, MetaFile, MODEL_FILE_INF,
> Temporary)
> + def __init__(self, Db, MetaFile, Temporary):
> + MetaFileTable.__init__(self, Db, MetaFile, MODEL_FILE_INF,
> Temporary)
>
> ## Insert a record into table Inf
> #
> # @param Model: Model of a Inf item
> # @param Value1: Value1 of a Inf item
> @@ -110,48 +115,62 @@ class ModuleTable(MetaFileTable):
> # @param EndColumn: EndColumn of a Inf item
> # @param Enabled: If this item enabled
> #
> def Insert(self, Model, Value1, Value2, Value3,
> Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON,
> BelongsToItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1,
> EndColumn=-1, Enabled=0):
> - (Value1, Value2, Value3, Scope1, Scope2) = ConvertToSqlString((Value1,
> Value2, Value3, Scope1, Scope2))
> - return Table.Insert(
> - self,
> - Model,
> - Value1,
> - Value2,
> - Value3,
> - Scope1,
> - Scope2,
> - BelongsToItem,
> - StartLine,
> - StartColumn,
> - EndLine,
> - EndColumn,
> - Enabled
> - )
> +
> + (Value1, Value2, Value3, Scope1, Scope2) = (Value1.strip(),
> Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip())
> + self.ID = self.ID + self._ID_STEP_
> + if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
> + self.ID = MODEL_FILE_INF + self._ID_STEP_
> +
> + row = [ self.ID,
> + Model,
> + Value1,
> + Value2,
> + Value3,
> + Scope1,
> + Scope2,
> + BelongsToItem,
> + StartLine,
> + StartColumn,
> + EndLine,
> + EndColumn,
> + Enabled
> + ]
> + self.CurrentContent.append(row)
> + return self.ID
> +
>
> ## Query table
> #
> # @param Model: The Model of Record
> # @param Arch: The Arch attribute of Record
> # @param Platform The Platform attribute of Record
> #
> # @retval: A recordSet of all found records
> #
> def Query(self, Model, Arch=None, Platform=None,
> BelongsToItem=None):
> - ConditionString = "Model=%s AND Enabled>=0" % Model
> - ValueString = "Value1,Value2,Value3,Scope1,Scope2,ID,StartLine"
> +
> + QueryTab = self.CurrentContent
> + result = [item for item in QueryTab if item[1] == Model and item[-1]>=0
> ]
>
> if Arch is not None and Arch != TAB_ARCH_COMMON:
> - ConditionString += " AND (Scope1='%s' OR Scope1='COMMON')" %
> Arch
> + ArchList = set(['COMMON'])
> + ArchList.add(Arch)
> + result = [item for item in result if item[5] in ArchList]
> +
> if Platform is not None and Platform != TAB_COMMON:
> - ConditionString += " AND (Scope2='%s' OR Scope2='COMMON' OR
> Scope2='DEFAULT')" % Platform
> + Platformlist = set( ['COMMON','DEFAULT'])
> + Platformlist.add(Platform)
> + result = [item for item in result if item[6] in Platformlist]
> +
> if BelongsToItem is not None:
> - ConditionString += " AND BelongsToItem=%s" % BelongsToItem
> + result = [item for item in result if item[7] == BelongsToItem]
>
> - SqlCommand = "SELECT %s FROM %s WHERE %s" % (ValueString,
> self.Table, ConditionString)
> - return self.Exec(SqlCommand)
> + result = [ [r[2],r[3],r[4],r[5],r[6],r[0],r[9]] for r in result ]
> + return result
>
> ## Python class representation of table storing package data
> class PackageTable(MetaFileTable):
> _COLUMN_ = '''
> ID REAL PRIMARY KEY,
> @@ -167,11 +186,11 @@ class PackageTable(MetaFileTable):
> EndLine INTEGER NOT NULL,
> EndColumn INTEGER NOT NULL,
> Enabled INTEGER DEFAULT 0
> '''
> # used as table end flag, in case the changes to database is not committed
> to db file
> - _DUMMY_ = "-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -
> 1"
> + _DUMMY_ = [-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -
> 1]
>
> ## Constructor
> def __init__(self, Cursor, MetaFile, Temporary):
> MetaFileTable.__init__(self, Cursor, MetaFile, MODEL_FILE_DEC,
> Temporary)
>
> @@ -192,52 +211,60 @@ class PackageTable(MetaFileTable):
> # @param EndColumn: EndColumn of a Dec item
> # @param Enabled: If this item enabled
> #
> def Insert(self, Model, Value1, Value2, Value3,
> Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON,
> BelongsToItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1,
> EndColumn=-1, Enabled=0):
> - (Value1, Value2, Value3, Scope1, Scope2) = ConvertToSqlString((Value1,
> Value2, Value3, Scope1, Scope2))
> - return Table.Insert(
> - self,
> - Model,
> - Value1,
> - Value2,
> - Value3,
> - Scope1,
> - Scope2,
> - BelongsToItem,
> - StartLine,
> - StartColumn,
> - EndLine,
> - EndColumn,
> - Enabled
> - )
> + (Value1, Value2, Value3, Scope1, Scope2) = (Value1.strip(),
> Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip())
> + self.ID = self.ID + self._ID_STEP_
> + if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
> + self.ID = MODEL_FILE_INF + self._ID_STEP_
> +
> + row = [ self.ID,
> + Model,
> + Value1,
> + Value2,
> + Value3,
> + Scope1,
> + Scope2,
> + BelongsToItem,
> + StartLine,
> + StartColumn,
> + EndLine,
> + EndColumn,
> + Enabled
> + ]
> + self.CurrentContent.append(row)
> + return self.ID
>
> ## Query table
> #
> # @param Model: The Model of Record
> # @param Arch: The Arch attribute of Record
> #
> # @retval: A recordSet of all found records
> #
> def Query(self, Model, Arch=None):
> - ConditionString = "Model=%s AND Enabled>=0" % Model
> - ValueString = "Value1,Value2,Value3,Scope1,Scope2,ID,StartLine"
> +
> + QueryTab = self.CurrentContent
> + result = [item for item in QueryTab if item[1] == Model and item[-1]>=0
> ]
>
> if Arch is not None and Arch != TAB_ARCH_COMMON:
> - ConditionString += " AND (Scope1='%s' OR Scope1='COMMON')" %
> Arch
> + ArchList = set(['COMMON'])
> + ArchList.add(Arch)
> + result = [item for item in result if item[5] in ArchList]
>
> - SqlCommand = "SELECT %s FROM %s WHERE %s" % (ValueString,
> self.Table, ConditionString)
> - return self.Exec(SqlCommand)
> + return [[r[2], r[3], r[4], r[5], r[6], r[0], r[8]] for r in result]
>
> def GetValidExpression(self, TokenSpaceGuid, PcdCName):
> - SqlCommand = "select Value1,StartLine from %s WHERE Value2='%s'
> and Value3='%s'" % (self.Table, TokenSpaceGuid, PcdCName)
> - self.Cur.execute(SqlCommand)
> +
> + QueryTab = self.CurrentContent
> + result = [[item[2], item[8]] for item in QueryTab if item[3] ==
> TokenSpaceGuid and item[4] == PcdCName]
> validateranges = []
> validlists = []
> expressions = []
> try:
> - for row in self.Cur:
> + for row in result:
> comment = row[0]
>
> LineNum = row[1]
> comment = comment.strip("#")
> comment = comment.strip()
> @@ -281,11 +308,11 @@ class PlatformTable(MetaFileTable):
> EndLine INTEGER NOT NULL,
> EndColumn INTEGER NOT NULL,
> Enabled INTEGER DEFAULT 0
> '''
> # used as table end flag, in case the changes to database is not committed
> to db file
> - _DUMMY_ = "-1, -1, '====', '====', '====', '====', '====','====', -1, -1, -1,
> -1, -1, -1, -1"
> + _DUMMY_ = [-1, -1, '====', '====', '====', '====', '====','====', -1, -1, -1,
> -1, -1, -1, -1]
>
> ## Constructor
> def __init__(self, Cursor, MetaFile, Temporary, FromItem=0):
> MetaFileTable.__init__(self, Cursor, MetaFile, MODEL_FILE_DSC,
> Temporary, FromItem)
>
> @@ -307,28 +334,34 @@ class PlatformTable(MetaFileTable):
> # @param EndColumn: EndColumn of a Dsc item
> # @param Enabled: If this item enabled
> #
> def Insert(self, Model, Value1, Value2, Value3,
> Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON,
> Scope3=TAB_DEFAULT_STORES_DEFAULT,BelongsToItem=-1,
> FromItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1,
> EndColumn=-1, Enabled=1):
> - (Value1, Value2, Value3, Scope1, Scope2, Scope3) =
> ConvertToSqlString((Value1, Value2, Value3, Scope1, Scope2, Scope3))
> - return Table.Insert(
> - self,
> - Model,
> - Value1,
> - Value2,
> - Value3,
> - Scope1,
> - Scope2,
> - Scope3,
> - BelongsToItem,
> - FromItem,
> - StartLine,
> - StartColumn,
> - EndLine,
> - EndColumn,
> - Enabled
> - )
> + (Value1, Value2, Value3, Scope1, Scope2, Scope3) = (Value1.strip(),
> Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip(), Scope3.strip())
> + self.ID = self.ID + self._ID_STEP_
> + if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
> + self.ID = MODEL_FILE_INF + self._ID_STEP_
> +
> + row = [ self.ID,
> + Model,
> + Value1,
> + Value2,
> + Value3,
> + Scope1,
> + Scope2,
> + Scope3,
> + BelongsToItem,
> + FromItem,
> + StartLine,
> + StartColumn,
> + EndLine,
> + EndColumn,
> + Enabled
> + ]
> + self.CurrentContent.append(row)
> + return self.ID
> +
>
> ## Query table
> #
> # @param Model: The Model of Record
> # @param Scope1: Arch of a Dsc item
> @@ -337,34 +370,37 @@ class PlatformTable(MetaFileTable):
> # @param FromItem: The item belongs to which dsc file
> #
> # @retval: A recordSet of all found records
> #
> def Query(self, Model, Scope1=None, Scope2=None,
> BelongsToItem=None, FromItem=None):
> - ConditionString = "Model=%s AND Enabled>0" % Model
> - ValueString =
> "Value1,Value2,Value3,Scope1,Scope2,Scope3,ID,StartLine"
> +
> + QueryTab = self.CurrentContent
> + result = [item for item in QueryTab if item[1] == Model and item[-1]>0 ]
>
> if Scope1 is not None and Scope1 != TAB_ARCH_COMMON:
> - ConditionString += " AND (Scope1='%s' OR Scope1='COMMON')" %
> Scope1
> - if Scope2 is not None and Scope2 != TAB_COMMON:
> - # Cover the case that CodeBase is 'COMMON' for BuildOptions section
> + Sc1 = set(['COMMON'])
> + Sc1.add(Scope1)
> + result = [item for item in result if item[5] in Sc1]
> + Sc2 = set( ['COMMON','DEFAULT'])
> + if Scope2 and Scope2 != TAB_COMMON:
> if '.' in Scope2:
> Index = Scope2.index('.')
> NewScope = TAB_COMMON + Scope2[Index:]
> - ConditionString += " AND (Scope2='%s' OR Scope2='COMMON' OR
> Scope2='DEFAULT' OR Scope2='%s')" % (Scope2, NewScope)
> - else:
> - ConditionString += " AND (Scope2='%s' OR Scope2='COMMON' OR
> Scope2='DEFAULT')" % Scope2
> + Sc2.add(NewScope)
> + Sc2.add(Scope2)
> + result = [item for item in result if item[6] in Sc2]
>
> if BelongsToItem is not None:
> - ConditionString += " AND BelongsToItem=%s" % BelongsToItem
> + result = [item for item in result if item[8] == BelongsToItem]
> else:
> - ConditionString += " AND BelongsToItem<0"
> -
> + result = [item for item in result if item[8] < 0]
> if FromItem is not None:
> - ConditionString += " AND FromItem=%s" % FromItem
> + result = [item for item in result if item[9] == FromItem]
> +
> + result = [ [r[2],r[3],r[4],r[5],r[6],r[7],r[0],r[9]] for r in result ]
> + return result
>
> - SqlCommand = "SELECT %s FROM %s WHERE %s" % (ValueString,
> self.Table, ConditionString)
> - return self.Exec(SqlCommand)
>
> ## Factory class to produce different storage for different type of meta-file
> class MetaFileStorage(object):
> _FILE_TABLE_ = {
> MODEL_FILE_INF : ModuleTable,
> diff --git a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
> b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
> index 3bb287b8b2..bb6dafe9cc 100644
> --- a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
> +++ b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
> @@ -14,11 +14,10 @@
>
> ##
> # Import Modules
> #
> from __future__ import absolute_import
> -import sqlite3
> from Common.StringUtils import *
> from Common.DataType import *
> from Common.Misc import *
> from types import *
>
> @@ -116,11 +115,11 @@ class WorkspaceDatabase(object):
> # get the parser ready for this file
> MetaFile = self._FILE_PARSER_[FileType](
> FilePath,
> FileType,
> Arch,
> - MetaFileStorage(self.WorkspaceDb.Cur, FilePath, FileType)
> + MetaFileStorage(self.WorkspaceDb, FilePath, FileType)
> )
> # alwasy do post-process, in case of macros change
> MetaFile.DoPostProcess()
> # object the build is based on
> BuildObject = self._GENERATOR_[FileType](
> @@ -147,137 +146,27 @@ class WorkspaceDatabase(object):
> #
> # @param DbPath Path of database file
> # @param GlobalMacros Global macros used for replacement during
> file parsing
> # @prarm RenewDb=False Create new database file if it's already there
> #
> - def __init__(self, DbPath, RenewDb=False):
> - self._DbClosedFlag = False
> - if not DbPath:
> - DbPath = os.path.normpath(mws.join(GlobalData.gWorkspace, 'Conf',
> GlobalData.gDatabasePath))
> -
> - # don't create necessary path for db in memory
> - if DbPath != ':memory:':
> - DbDir = os.path.split(DbPath)[0]
> - if not os.path.exists(DbDir):
> - os.makedirs(DbDir)
> -
> - # remove db file in case inconsistency between db and file in file
> system
> - if self._CheckWhetherDbNeedRenew(RenewDb, DbPath):
> - os.remove(DbPath)
> -
> - # create db with optimized parameters
> - self.Conn = sqlite3.connect(DbPath, isolation_level='DEFERRED')
> - self.Conn.execute("PRAGMA synchronous=OFF")
> - self.Conn.execute("PRAGMA temp_store=MEMORY")
> - self.Conn.execute("PRAGMA count_changes=OFF")
> - self.Conn.execute("PRAGMA cache_size=8192")
> - #self.Conn.execute("PRAGMA page_size=8192")
> -
> - # to avoid non-ascii character conversion issue
> - self.Conn.text_factory = str
> - self.Cur = self.Conn.cursor()
> -
> + def __init__(self):
> + self.DB = dict()
> # create table for internal uses
> - self.TblDataModel = TableDataModel(self.Cur)
> - self.TblFile = TableFile(self.Cur)
> + self.TblDataModel = DataClass.MODEL_LIST
> + self.TblFile = []
> self.Platform = None
>
> # conversion object for build or file format conversion purpose
> self.BuildObject = WorkspaceDatabase.BuildObjectFactory(self)
> self.TransformObject =
> WorkspaceDatabase.TransformObjectFactory(self)
>
> - ## Check whether workspace database need to be renew.
> - # The renew reason maybe:
> - # 1) If user force to renew;
> - # 2) If user do not force renew, and
> - # a) If the time of last modified python source is newer than database
> file;
> - # b) If the time of last modified frozen executable file is newer than
> database file;
> - #
> - # @param force User force renew database
> - # @param DbPath The absolute path of workspace database file
> - #
> - # @return Bool value for whether need renew workspace databse
> - #
> - def _CheckWhetherDbNeedRenew (self, force, DbPath):
> - # if database does not exist, we need do nothing
> - if not os.path.exists(DbPath): return False
> -
> - # if user force to renew database, then not check whether database is
> out of date
> - if force: return True
> -
> - #
> - # Check the time of last modified source file or build.exe
> - # if is newer than time of database, then database need to be re-
> created.
> - #
> - timeOfToolModified = 0
> - if hasattr(sys, "frozen"):
> - exePath = os.path.abspath(sys.executable)
> - timeOfToolModified = os.stat(exePath).st_mtime
> - else:
> - curPath = os.path.dirname(__file__) # curPath is the path of
> WorkspaceDatabase.py
> - rootPath = os.path.split(curPath)[0] # rootPath is root path of python
> source, such as /BaseTools/Source/Python
> - if rootPath == "" or rootPath is None:
> - EdkLogger.verbose("\nFail to find the root path of build.exe or
> python sources, so can not \
> -determine whether database file is out of date!\n")
> -
> - # walk the root path of source or build's binary to get the time last
> modified.
> -
> - for root, dirs, files in os.walk (rootPath):
> - for dir in dirs:
> - # bypass source control folder
> - if dir.lower() in [".svn", "_svn", "cvs"]:
> - dirs.remove(dir)
> + def SetFileTimeStamp(self,FileId,TimeStamp):
> + self.TblFile[FileId][6] = TimeStamp
>
> - for file in files:
> - ext = os.path.splitext(file)[1]
> - if ext.lower() == ".py": # only check .py files
> - fd = os.stat(os.path.join(root, file))
> - if timeOfToolModified < fd.st_mtime:
> - timeOfToolModified = fd.st_mtime
> - if timeOfToolModified > os.stat(DbPath).st_mtime:
> - EdkLogger.verbose("\nWorkspace database is out of data!")
> - return True
> + def GetFileTimeStamp(self,FileId):
> + return self.TblFile[FileId][6]
>
> - return False
> -
> - ## Initialize build database
> - def InitDatabase(self):
> - EdkLogger.verbose("\nInitialize build database started ...")
> -
> - #
> - # Create new tables
> - #
> - self.TblDataModel.Create(False)
> - self.TblFile.Create(False)
> -
> - #
> - # Initialize table DataModel
> - #
> - self.TblDataModel.InitTable()
> - EdkLogger.verbose("Initialize build database ... DONE!")
> -
> - ## Query a table
> - #
> - # @param Table: The instance of the table to be queried
> - #
> - def QueryTable(self, Table):
> - Table.Query()
> -
> - def __del__(self):
> - self.Close()
> -
> - ## Close entire database
> - #
> - # Commit all first
> - # Close the connection and cursor
> - #
> - def Close(self):
> - if not self._DbClosedFlag:
> - self.Conn.commit()
> - self.Cur.close()
> - self.Conn.close()
> - self._DbClosedFlag = True
>
> ## Summarize all packages in the database
> def GetPackageList(self, Platform, Arch, TargetName, ToolChainTag):
> self.Platform = Platform
> PackageList = []
> @@ -302,11 +191,11 @@ determine whether database file is out of date!\n")
> return PackageList
>
> ## Summarize all platforms in the database
> def PlatformList(self):
> RetVal = []
> - for PlatformFile in self.TblFile.GetFileList(MODEL_FILE_DSC):
> + for PlatformFile in [item[3] for item in self.TblFile if item[5] ==
> MODEL_FILE_DSC]:
> try:
> RetVal.append(self.BuildObject[PathClass(PlatformFile),
> TAB_COMMON])
> except:
> pass
> return RetVal
> diff --git a/BaseTools/Source/Python/build/build.py
> b/BaseTools/Source/Python/build/build.py
> index d74082fc26..317d6a2728 100644
> --- a/BaseTools/Source/Python/build/build.py
> +++ b/BaseTools/Source/Python/build/build.py
> @@ -41,11 +41,11 @@ from Common.TargetTxtClassObject import *
> from Common.ToolDefClassObject import *
> from Common.DataType import *
> from Common.BuildVersion import gBUILD_VERSION
> from AutoGen.AutoGen import *
> from Common.BuildToolError import *
> -from Workspace.WorkspaceDatabase import *
> +from Workspace.WorkspaceDatabase import WorkspaceDatabase
> from Common.MultipleWorkspace import MultipleWorkspace as mws
>
> from BuildReport import BuildReport
> from GenPatchPcdTable.GenPatchPcdTable import *
> from PatchPcdValue.PatchPcdValue import *
> @@ -825,14 +825,11 @@ class Build():
> # Get standard WORKSPACE/Conf use the absolute path to the
> WORKSPACE/Conf
> ConfDirectoryPath = mws.join(self.WorkspaceDir, 'Conf')
> GlobalData.gConfDirectory = ConfDirectoryPath
> GlobalData.gDatabasePath =
> os.path.normpath(os.path.join(ConfDirectoryPath,
> GlobalData.gDatabasePath))
>
> - if BuildOptions.DisableCache:
> - self.Db = WorkspaceDatabase(":memory:")
> - else:
> - self.Db = WorkspaceDatabase(GlobalData.gDatabasePath,
> self.Reparse)
> + self.Db = WorkspaceDatabase()
> self.BuildDatabase = self.Db.BuildObject
> self.Platform = None
> self.ToolChainFamily = None
> self.LoadFixAddress = 0
> self.UniFlag = BuildOptions.Flag
> @@ -981,13 +978,10 @@ class Build():
> # Allow case-insensitive for those from command line or configuration
> file
> ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
> if ErrorCode != 0:
> EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
>
> - # create metafile database
> - if not self.Db_Flag:
> - self.Db.InitDatabase()
>
> def InitPreBuild(self):
> self.LoadConfiguration()
> ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
> if ErrorCode != 0:
> @@ -1002,11 +996,10 @@ class Build():
> if self.ToolChainFamily:
> GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[0]
> if 'PREBUILD' in GlobalData.gCommandLineDefines:
> self.Prebuild = GlobalData.gCommandLineDefines.get('PREBUILD')
> else:
> - self.Db.InitDatabase()
> self.Db_Flag = True
> Platform = self.Db.MapPlatform(str(self.PlatformFile))
> self.Prebuild = str(Platform.Prebuild)
> if self.Prebuild:
> PrebuildList = []
> @@ -2081,17 +2074,11 @@ class Build():
> if BuildTask.HasError():
> EdkLogger.error("build", BUILD_ERROR, "Failed to build
> module", ExtraData=GlobalData.gBuildingModule)
> self.MakeTime += int(round((time.time() - MakeStart)))
>
> MakeContiue = time.time()
> - #
> - # Save temp tables to a TmpTableDict.
> - #
> - for Key in Wa.BuildDatabase._CACHE_:
> - if Wa.BuildDatabase._CACHE_[Key]._RawData and
> Wa.BuildDatabase._CACHE_[Key]._RawData._Table and
> Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table:
> - if
> TemporaryTablePattern.match(Wa.BuildDatabase._CACHE_[Key]._RawData.
> _Table.Table):
> -
> TmpTableDict[Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table] =
> Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Cur
> +
> #
> #
> # All modules have been put in build tasks queue. Tell task scheduler
> # to exit if all tasks are completed
> #
> @@ -2229,11 +2216,10 @@ class Build():
> else:
> self.SpawnMode = False
> self._BuildModule()
>
> if self.Target == 'cleanall':
> - self.Db.Close()
> RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True)
>
> def CreateAsBuiltInf(self):
> for Module in self.BuildModules:
> Module.CreateAsBuiltInf()
> @@ -2490,14 +2476,11 @@ def Main():
>
> MyBuild = Build(Target, Workspace, Option)
> GlobalData.gCommandLineDefines['ARCH'] = ' '.join(MyBuild.ArchList)
> if not (MyBuild.LaunchPrebuildFlag and
> os.path.exists(MyBuild.PlatformBuildPath)):
> MyBuild.Launch()
> - # Drop temp tables to avoid database locked.
> - for TmpTableName in TmpTableDict:
> - SqlCommand = """drop table IF EXISTS %s""" % TmpTableName
> - TmpTableDict[TmpTableName].execute(SqlCommand)
> +
> #MyBuild.DumpBuildData()
> #
> # All job done, no error found and no exception raised
> #
> BuildError = False
> @@ -2565,11 +2548,11 @@ def Main():
> else:
> BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration)
> if MyBuild is not None:
> if not BuildError:
> MyBuild.BuildReport.GenerateReport(BuildDurationStr,
> LogBuildTime(MyBuild.AutoGenTime), LogBuildTime(MyBuild.MakeTime),
> LogBuildTime(MyBuild.GenFdsTime))
> - MyBuild.Db.Close()
> +
> EdkLogger.SetLevel(EdkLogger.QUIET)
> EdkLogger.quiet("\n- %s -" % Conclusion)
> EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d %Y",
> time.localtime()))
> EdkLogger.quiet("Build total time: %s\n" % BuildDurationStr)
> return ReturnCode
> --
> 2.19.1.windows.1
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [Patch V2] BaseTools: Replace the sqlite database with list
2018-11-09 15:24 ` Carsey, Jaben
@ 2018-11-11 0:43 ` Feng, Bob C
0 siblings, 0 replies; 3+ messages in thread
From: Feng, Bob C @ 2018-11-11 0:43 UTC (permalink / raw)
To: Carsey, Jaben, edk2-devel@lists.01.org; +Cc: Gao, Liming
Hi Jaben,
It will be faster than before when you do build, clean, build...
_Bob
-----Original Message-----
From: Carsey, Jaben
Sent: Friday, November 9, 2018 11:25 PM
To: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org
Cc: Gao, Liming <liming.gao@intel.com>
Subject: RE: [Patch V2] BaseTools: Replace the sqlite database with list
This is much easier to review! Thanks!
I am good with the code change.
I wonder: If we do not have the DB does it take longer if I do build, clean, build than it did before?
> -----Original Message-----
> From: Feng, Bob C
> Sent: Friday, November 09, 2018 12:42 AM
> To: edk2-devel@lists.01.org
> Cc: Gao, Liming <liming.gao@intel.com>; Carsey, Jaben
> <jaben.carsey@intel.com>
> Subject: [Patch V2] BaseTools: Replace the sqlite database with list
> Importance: High
>
> https://bugzilla.tianocore.org/show_bug.cgi?id=1288
>
> [V2]
> Optimize this patch so that it can be easy to review.
> This patch is just apply the change to original files while not create
> new similar files.
>
> [V1]
> This patch is one of build tool performance improvement series
> patches.
>
> This patch is going to use python list to store the parser data
> instead of using sqlite database.
>
> The replacement solution is as below:
>
> SQL insert: list.append()
> SQL select: list comprehension. for example:
> Select * from table where field = “something”
> ->
> [ item for item in table if item[3] == “something”]
>
> SQL update: python map function. for example:
> Update table set field1=newvalue where filed2 = “something”.
> -> map(lambda x: x[1] = newvalue,
> [item for item in table if item[2] == “something”])
>
> SQL delete: list comprehension.
>
> With this change, We can save the time of interpreting SQL statement
> and the time of write database to file system
>
> Contributed-under: TianoCore Contribution Agreement 1.1
> Signed-off-by: BobCF <bob.c.feng@intel.com>
> Cc: Liming Gao <liming.gao@intel.com>
> Cc: Jaben Carsey <jaben.carsey@intel.com>
> ---
> BaseTools/Source/Python/GenFds/GenFds.py | 3 +-
> .../Source/Python/Workspace/MetaDataTable.py | 93 ++-----
> .../Source/Python/Workspace/MetaFileParser.py | 5 +-
> .../Source/Python/Workspace/MetaFileTable.py | 248 ++++++++++--------
> .../Python/Workspace/WorkspaceDatabase.py | 131 +--------
> BaseTools/Source/Python/build/build.py | 27 +-
> 6 files changed, 186 insertions(+), 321 deletions(-)
>
> diff --git a/BaseTools/Source/Python/GenFds/GenFds.py
> b/BaseTools/Source/Python/GenFds/GenFds.py
> index 0c8091b798..4fd96706af 100644
> --- a/BaseTools/Source/Python/GenFds/GenFds.py
> +++ b/BaseTools/Source/Python/GenFds/GenFds.py
> @@ -222,12 +222,11 @@ def main():
> if "TOOL_CHAIN_TAG" not in GlobalData.gGlobalDefines:
> GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] =
> GenFdsGlobalVariable.ToolChainTag
>
> """call Workspace build create database"""
> GlobalData.gDatabasePath =
> os.path.normpath(os.path.join(ConfDirectoryPath,
> GlobalData.gDatabasePath))
> - BuildWorkSpace = WorkspaceDatabase(GlobalData.gDatabasePath)
> - BuildWorkSpace.InitDatabase()
> + BuildWorkSpace = WorkspaceDatabase()
>
> #
> # Get files real name in workspace dir
> #
> GlobalData.gAllFiles = DirCache(Workspace) diff --git
> a/BaseTools/Source/Python/Workspace/MetaDataTable.py
> b/BaseTools/Source/Python/Workspace/MetaDataTable.py
> index bd751eadfb..8becddbe08 100644
> --- a/BaseTools/Source/Python/Workspace/MetaDataTable.py
> +++ b/BaseTools/Source/Python/Workspace/MetaDataTable.py
> @@ -37,84 +37,58 @@ class Table(object):
> _COLUMN_ = ''
> _ID_STEP_ = 1
> _ID_MAX_ = 0x80000000
> _DUMMY_ = 0
>
> - def __init__(self, Cursor, Name='', IdBase=0, Temporary=False):
> - self.Cur = Cursor
> + def __init__(self, Db, Name='', IdBase=0, Temporary=False):
> + self.Db = Db
> self.Table = Name
> self.IdBase = int(IdBase)
> self.ID = int(IdBase)
> self.Temporary = Temporary
> + self.Contents = []
>
> def __str__(self):
> return self.Table
>
> ## Create table
> #
> # Create a table
> #
> def Create(self, NewTable=True):
> - if NewTable:
> - self.Drop()
> -
> - if self.Temporary:
> - SqlCommand = """create temp table IF NOT EXISTS %s (%s)""" %
> (self.Table, self._COLUMN_)
> - else:
> - SqlCommand = """create table IF NOT EXISTS %s (%s)""" % (self.Table,
> self._COLUMN_)
> - EdkLogger.debug(EdkLogger.DEBUG_8, SqlCommand)
> - self.Cur.execute(SqlCommand)
> + self.Db.CreateEmptyTable(self.Table)
> self.ID = self.GetId()
>
> ## Insert table
> #
> # Insert a record into a table
> #
> def Insert(self, *Args):
> self.ID = self.ID + self._ID_STEP_
> if self.ID >= (self.IdBase + self._ID_MAX_):
> self.ID = self.IdBase + self._ID_STEP_
> - Values = ", ".join(str(Arg) for Arg in Args)
> - SqlCommand = "insert into %s values(%s, %s)" % (self.Table, self.ID,
> Values)
> - EdkLogger.debug(EdkLogger.DEBUG_5, SqlCommand)
> - self.Cur.execute(SqlCommand)
> - return self.ID
> + row = [self.ID]
> + row.extend(Args)
> + self.Contents.append(row)
>
> - ## Query table
> - #
> - # Query all records of the table
> - #
> - def Query(self):
> - SqlCommand = """select * from %s""" % self.Table
> - self.Cur.execute(SqlCommand)
> - for Rs in self.Cur:
> - EdkLogger.verbose(str(Rs))
> - TotalCount = self.GetId()
> + return self.ID
>
> - ## Drop a table
> - #
> - # Drop the table
> - #
> - def Drop(self):
> - SqlCommand = """drop table IF EXISTS %s""" % self.Table
> - self.Cur.execute(SqlCommand)
>
> ## Get count
> #
> # Get a count of all records of the table
> #
> # @retval Count: Total count of all records
> #
> def GetCount(self):
> - SqlCommand = """select count(ID) from %s""" % self.Table
> - Record = self.Cur.execute(SqlCommand).fetchall()
> - return Record[0][0]
> + tab = self.Db.GetTable(self.Table)
> + return len(tab)
> +
>
> def GetId(self):
> - SqlCommand = """select max(ID) from %s""" % self.Table
> - Record = self.Cur.execute(SqlCommand).fetchall()
> - Id = Record[0][0]
> + tab = self.Db.GetTable(self.Table)
> + Id = max([int(item[0]) for item in tab])
> if Id is None:
> Id = self.IdBase
> return Id
>
> ## Init the ID of the table
> @@ -132,29 +106,30 @@ class Table(object):
> #
> # @retval RecordSet: The result after executed
> #
> def Exec(self, SqlCommand):
> EdkLogger.debug(EdkLogger.DEBUG_5, SqlCommand)
> - self.Cur.execute(SqlCommand)
> - RecordSet = self.Cur.fetchall()
> + self.Db.execute(SqlCommand)
> + RecordSet = self.Db.fetchall()
> return RecordSet
>
> def SetEndFlag(self):
> - self.Exec("insert into %s values(%s)" % (self.Table, self._DUMMY_))
> - #
> - # Need to execution commit for table data changed.
> - #
> - self.Cur.connection.commit()
> + Tab = self.Db.GetTable(self.Table)
> + Tab.append(self._DUMMY_)
> +
>
> def IsIntegral(self):
> - Result = self.Exec("select min(ID) from %s" % (self.Table))
> - if Result[0][0] != -1:
> + tab = self.Db.GetTable(self.Table)
> + Id = min([int(item[0]) for item in tab])
> + if Id != -1:
> return False
> return True
>
> def GetAll(self):
> - return self.Exec("select * from %s where ID > 0 order by ID" %
> (self.Table))
> + tab = self.Db.GetTable(self.Table)
> + return tab
> +
>
> ## TableFile
> #
> # This class defined a table used for file # @@ -225,26 +200,10 @@
> class TableFile(Table):
> File.Path,
> Model,
> File.TimeStamp
> )
>
> - ## Get ID of a given file
> - #
> - # @param FilePath Path of file
> - #
> - # @retval ID ID value of given file in the table
> - #
> - def GetFileId(self, File, FromItem=None):
> - if FromItem:
> - QueryScript = "select ID from %s where FullPath = '%s' and FromItem
> = %s" % (self.Table, str(File), str(FromItem))
> - else:
> - QueryScript = "select ID from %s where FullPath = '%s'" % (self.Table,
> str(File))
> - RecordList = self.Exec(QueryScript)
> - if len(RecordList) == 0:
> - return None
> - return RecordList[0][0]
> -
> ## Get type of a given file
> #
> # @param FileId ID of a file
> #
> # @retval file_type Model value of given file in the table
> @@ -343,11 +302,11 @@ class TableDataModel(Table):
> # @retval CrossIndex: CrossIndex of the model
> #
> def GetCrossIndex(self, ModelName):
> CrossIndex = -1
> SqlCommand = """select CrossIndex from DataModel where name = '"""
> + ModelName + """'"""
> - self.Cur.execute(SqlCommand)
> - for Item in self.Cur:
> + self.Db.execute(SqlCommand)
> + for Item in self.Db:
> CrossIndex = Item[0]
>
> return CrossIndex
>
> diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py
> b/BaseTools/Source/Python/Workspace/MetaFileParser.py
> index 804a4aa5cb..a5ae057bd1 100644
> --- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
> +++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
> @@ -1306,12 +1306,11 @@ class DscParser(MetaFileParser):
> MODEL_UNKNOWN : self._Skip,
> MODEL_META_DATA_USER_EXTENSION :
> self._SkipUserExtension,
> MODEL_META_DATA_CONDITIONAL_STATEMENT_ERROR :
> self._ProcessError,
> }
>
> - self._Table = MetaFileStorage(self._RawTable.Cur, self.MetaFile,
> MODEL_FILE_DSC, True)
> - self._Table.Create()
> + self._Table = MetaFileStorage(self._RawTable.DB,
> + self.MetaFile,
> MODEL_FILE_DSC, True)
> self._DirectiveStack = []
> self._DirectiveEvalStack = []
> self._FileWithError = self.MetaFile
> self._FileLocalMacros = {}
> self._SectionsMacroDict.clear() @@ -1566,11 +1565,11 @@ class
> DscParser(MetaFileParser):
> FromItem = self._Content[self._ContentIndex - 1][0]
> if self._InSubsection:
> Owner = self._Content[self._ContentIndex - 1][8]
> else:
> Owner = self._Content[self._ContentIndex - 1][0]
> - IncludedFileTable = MetaFileStorage(self._Table.Cur, IncludedFile1,
> MODEL_FILE_DSC, False, FromItem=FromItem)
> + IncludedFileTable = MetaFileStorage(self._RawTable.DB,
> IncludedFile1, MODEL_FILE_DSC, False, FromItem=FromItem)
> Parser = DscParser(IncludedFile1, self._FileType,
> self._Arch, IncludedFileTable,
> Owner=Owner, From=FromItem)
>
> self.IncludedFiles.add (IncludedFile1)
>
> diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py
> b/BaseTools/Source/Python/Workspace/MetaFileTable.py
> index e0a0b8d923..081970dba8 100644
> --- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
> +++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
> @@ -18,59 +18,64 @@ from __future__ import absolute_import import
> uuid
>
> import Common.EdkLogger as EdkLogger
> from Common.BuildToolError import FORMAT_INVALID
>
> -from .MetaDataTable import Table, TableFile -from .MetaDataTable
> import ConvertToSqlString from CommonDataClass.DataClass import
> MODEL_FILE_DSC, MODEL_FILE_DEC, MODEL_FILE_INF, \
> MODEL_FILE_OTHERS from
> Common.DataType import *
>
> -class MetaFileTable(Table):
> +class MetaFileTable():
> # TRICK: use file ID as the part before '.'
> _ID_STEP_ = 0.00000001
> _ID_MAX_ = 0.99999999
>
> ## Constructor
> - def __init__(self, Cursor, MetaFile, FileType, Temporary,
> FromItem=None):
> + def __init__(self, DB, MetaFile, FileType, Temporary, FromItem=None):
> self.MetaFile = MetaFile
> -
> - self._FileIndexTable = TableFile(Cursor)
> - self._FileIndexTable.Create(False)
> -
> - FileId = self._FileIndexTable.GetFileId(MetaFile, FromItem)
> - if not FileId:
> - FileId = self._FileIndexTable.InsertFile(MetaFile, FileType, FromItem)
> -
> + self.TableName = ""
> + self.DB = DB
> + self._NumpyTab = None
> + self.FileId = len(DB.TblFile)
> + self.ID = self.FileId
> + self.CurrentContent = []
> + DB.TblFile.append([MetaFile.Name,
> + MetaFile.Ext,
> + MetaFile.Dir,
> + MetaFile.Path,
> + FileType,
> + MetaFile.TimeStamp,
> + FromItem])
> if Temporary:
> - TableName = "_%s_%s_%s" % (FileType, FileId, uuid.uuid4().hex)
> + self.TableName = "_%s_%s_%s" % (FileType,
> + len(DB.TblFile),
> uuid.uuid4().hex)
> else:
> - TableName = "_%s_%s" % (FileType, FileId)
> -
> - #Table.__init__(self, Cursor, TableName, FileId, False)
> - Table.__init__(self, Cursor, TableName, FileId, Temporary)
> - self.Create(not self.IsIntegrity())
> + self.TableName = "_%s_%s" % (FileType, len(DB.TblFile))
>
> def IsIntegrity(self):
> try:
> TimeStamp = self.MetaFile.TimeStamp
> - Result = self.Cur.execute("select ID from %s where ID<0" %
> (self.Table)).fetchall()
> + Result = int(self.CurrentContent[-1][0]) < 0
> if not Result:
> # update the timestamp in database
> - self._FileIndexTable.SetFileTimeStamp(self.IdBase, TimeStamp)
> + self.DB.SetFileTimeStamp(self.FileId, TimeStamp)
> return False
>
> - if TimeStamp != self._FileIndexTable.GetFileTimeStamp(self.IdBase):
> + if TimeStamp != self.DB.GetFileTimeStamp(self.FileId):
> # update the timestamp in database
> - self._FileIndexTable.SetFileTimeStamp(self.IdBase, TimeStamp)
> + self.DB.SetFileTimeStamp(self.FileId, TimeStamp)
> return False
> except Exception as Exc:
> EdkLogger.debug(EdkLogger.DEBUG_5, str(Exc))
> return False
> return True
>
> + def SetEndFlag(self):
> + self.CurrentContent.append(self._DUMMY_)
> +
> + def GetAll(self):
> + return [item for item in self.CurrentContent if item[0] > 0 ]
> +
> ## Python class representation of table storing module data class
> ModuleTable(MetaFileTable):
> _ID_STEP_ = 0.00000001
> _ID_MAX_ = 0.99999999
> _COLUMN_ = '''
> @@ -87,15 +92,15 @@ class ModuleTable(MetaFileTable):
> EndLine INTEGER NOT NULL,
> EndColumn INTEGER NOT NULL,
> Enabled INTEGER DEFAULT 0
> '''
> # used as table end flag, in case the changes to database is not
> committed to db file
> - _DUMMY_ = "-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -
> 1"
> + _DUMMY_ = [-1, -1, '====', '====', '====', '====', '====', -1,
> + -1, -1, -1, -1, -
> 1]
>
> ## Constructor
> - def __init__(self, Cursor, MetaFile, Temporary):
> - MetaFileTable.__init__(self, Cursor, MetaFile, MODEL_FILE_INF,
> Temporary)
> + def __init__(self, Db, MetaFile, Temporary):
> + MetaFileTable.__init__(self, Db, MetaFile, MODEL_FILE_INF,
> Temporary)
>
> ## Insert a record into table Inf
> #
> # @param Model: Model of a Inf item
> # @param Value1: Value1 of a Inf item
> @@ -110,48 +115,62 @@ class ModuleTable(MetaFileTable):
> # @param EndColumn: EndColumn of a Inf item
> # @param Enabled: If this item enabled
> #
> def Insert(self, Model, Value1, Value2, Value3,
> Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON,
> BelongsToItem=-1, StartLine=-1, StartColumn=-1,
> EndLine=-1, EndColumn=-1, Enabled=0):
> - (Value1, Value2, Value3, Scope1, Scope2) = ConvertToSqlString((Value1,
> Value2, Value3, Scope1, Scope2))
> - return Table.Insert(
> - self,
> - Model,
> - Value1,
> - Value2,
> - Value3,
> - Scope1,
> - Scope2,
> - BelongsToItem,
> - StartLine,
> - StartColumn,
> - EndLine,
> - EndColumn,
> - Enabled
> - )
> +
> + (Value1, Value2, Value3, Scope1, Scope2) = (Value1.strip(),
> Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip())
> + self.ID = self.ID + self._ID_STEP_
> + if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
> + self.ID = MODEL_FILE_INF + self._ID_STEP_
> +
> + row = [ self.ID,
> + Model,
> + Value1,
> + Value2,
> + Value3,
> + Scope1,
> + Scope2,
> + BelongsToItem,
> + StartLine,
> + StartColumn,
> + EndLine,
> + EndColumn,
> + Enabled
> + ]
> + self.CurrentContent.append(row)
> + return self.ID
> +
>
> ## Query table
> #
> # @param Model: The Model of Record
> # @param Arch: The Arch attribute of Record
> # @param Platform The Platform attribute of Record
> #
> # @retval: A recordSet of all found records
> #
> def Query(self, Model, Arch=None, Platform=None,
> BelongsToItem=None):
> - ConditionString = "Model=%s AND Enabled>=0" % Model
> - ValueString = "Value1,Value2,Value3,Scope1,Scope2,ID,StartLine"
> +
> + QueryTab = self.CurrentContent
> + result = [item for item in QueryTab if item[1] == Model and
> + item[-1]>=0
> ]
>
> if Arch is not None and Arch != TAB_ARCH_COMMON:
> - ConditionString += " AND (Scope1='%s' OR Scope1='COMMON')" %
> Arch
> + ArchList = set(['COMMON'])
> + ArchList.add(Arch)
> + result = [item for item in result if item[5] in ArchList]
> +
> if Platform is not None and Platform != TAB_COMMON:
> - ConditionString += " AND (Scope2='%s' OR Scope2='COMMON' OR
> Scope2='DEFAULT')" % Platform
> + Platformlist = set( ['COMMON','DEFAULT'])
> + Platformlist.add(Platform)
> + result = [item for item in result if item[6] in
> + Platformlist]
> +
> if BelongsToItem is not None:
> - ConditionString += " AND BelongsToItem=%s" % BelongsToItem
> + result = [item for item in result if item[7] ==
> + BelongsToItem]
>
> - SqlCommand = "SELECT %s FROM %s WHERE %s" % (ValueString,
> self.Table, ConditionString)
> - return self.Exec(SqlCommand)
> + result = [ [r[2],r[3],r[4],r[5],r[6],r[0],r[9]] for r in result ]
> + return result
>
> ## Python class representation of table storing package data class
> PackageTable(MetaFileTable):
> _COLUMN_ = '''
> ID REAL PRIMARY KEY,
> @@ -167,11 +186,11 @@ class PackageTable(MetaFileTable):
> EndLine INTEGER NOT NULL,
> EndColumn INTEGER NOT NULL,
> Enabled INTEGER DEFAULT 0
> '''
> # used as table end flag, in case the changes to database is not
> committed to db file
> - _DUMMY_ = "-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -
> 1"
> + _DUMMY_ = [-1, -1, '====', '====', '====', '====', '====', -1,
> + -1, -1, -1, -1, -
> 1]
>
> ## Constructor
> def __init__(self, Cursor, MetaFile, Temporary):
> MetaFileTable.__init__(self, Cursor, MetaFile,
> MODEL_FILE_DEC,
> Temporary)
>
> @@ -192,52 +211,60 @@ class PackageTable(MetaFileTable):
> # @param EndColumn: EndColumn of a Dec item
> # @param Enabled: If this item enabled
> #
> def Insert(self, Model, Value1, Value2, Value3,
> Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON,
> BelongsToItem=-1, StartLine=-1, StartColumn=-1,
> EndLine=-1, EndColumn=-1, Enabled=0):
> - (Value1, Value2, Value3, Scope1, Scope2) = ConvertToSqlString((Value1,
> Value2, Value3, Scope1, Scope2))
> - return Table.Insert(
> - self,
> - Model,
> - Value1,
> - Value2,
> - Value3,
> - Scope1,
> - Scope2,
> - BelongsToItem,
> - StartLine,
> - StartColumn,
> - EndLine,
> - EndColumn,
> - Enabled
> - )
> + (Value1, Value2, Value3, Scope1, Scope2) = (Value1.strip(),
> Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip())
> + self.ID = self.ID + self._ID_STEP_
> + if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
> + self.ID = MODEL_FILE_INF + self._ID_STEP_
> +
> + row = [ self.ID,
> + Model,
> + Value1,
> + Value2,
> + Value3,
> + Scope1,
> + Scope2,
> + BelongsToItem,
> + StartLine,
> + StartColumn,
> + EndLine,
> + EndColumn,
> + Enabled
> + ]
> + self.CurrentContent.append(row)
> + return self.ID
>
> ## Query table
> #
> # @param Model: The Model of Record
> # @param Arch: The Arch attribute of Record
> #
> # @retval: A recordSet of all found records
> #
> def Query(self, Model, Arch=None):
> - ConditionString = "Model=%s AND Enabled>=0" % Model
> - ValueString = "Value1,Value2,Value3,Scope1,Scope2,ID,StartLine"
> +
> + QueryTab = self.CurrentContent
> + result = [item for item in QueryTab if item[1] == Model and
> + item[-1]>=0
> ]
>
> if Arch is not None and Arch != TAB_ARCH_COMMON:
> - ConditionString += " AND (Scope1='%s' OR Scope1='COMMON')" %
> Arch
> + ArchList = set(['COMMON'])
> + ArchList.add(Arch)
> + result = [item for item in result if item[5] in ArchList]
>
> - SqlCommand = "SELECT %s FROM %s WHERE %s" % (ValueString,
> self.Table, ConditionString)
> - return self.Exec(SqlCommand)
> + return [[r[2], r[3], r[4], r[5], r[6], r[0], r[8]] for r in
> + result]
>
> def GetValidExpression(self, TokenSpaceGuid, PcdCName):
> - SqlCommand = "select Value1,StartLine from %s WHERE Value2='%s'
> and Value3='%s'" % (self.Table, TokenSpaceGuid, PcdCName)
> - self.Cur.execute(SqlCommand)
> +
> + QueryTab = self.CurrentContent
> + result = [[item[2], item[8]] for item in QueryTab if item[3]
> + ==
> TokenSpaceGuid and item[4] == PcdCName]
> validateranges = []
> validlists = []
> expressions = []
> try:
> - for row in self.Cur:
> + for row in result:
> comment = row[0]
>
> LineNum = row[1]
> comment = comment.strip("#")
> comment = comment.strip() @@ -281,11 +308,11 @@ class
> PlatformTable(MetaFileTable):
> EndLine INTEGER NOT NULL,
> EndColumn INTEGER NOT NULL,
> Enabled INTEGER DEFAULT 0
> '''
> # used as table end flag, in case the changes to database is not
> committed to db file
> - _DUMMY_ = "-1, -1, '====', '====', '====', '====', '====','====', -1, -1, -1,
> -1, -1, -1, -1"
> + _DUMMY_ = [-1, -1, '====', '====', '====', '====', '====','====',
> + -1, -1, -1,
> -1, -1, -1, -1]
>
> ## Constructor
> def __init__(self, Cursor, MetaFile, Temporary, FromItem=0):
> MetaFileTable.__init__(self, Cursor, MetaFile,
> MODEL_FILE_DSC, Temporary, FromItem)
>
> @@ -307,28 +334,34 @@ class PlatformTable(MetaFileTable):
> # @param EndColumn: EndColumn of a Dsc item
> # @param Enabled: If this item enabled
> #
> def Insert(self, Model, Value1, Value2, Value3,
> Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON,
> Scope3=TAB_DEFAULT_STORES_DEFAULT,BelongsToItem=-1,
> FromItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1,
> EndColumn=-1, Enabled=1):
> - (Value1, Value2, Value3, Scope1, Scope2, Scope3) =
> ConvertToSqlString((Value1, Value2, Value3, Scope1, Scope2, Scope3))
> - return Table.Insert(
> - self,
> - Model,
> - Value1,
> - Value2,
> - Value3,
> - Scope1,
> - Scope2,
> - Scope3,
> - BelongsToItem,
> - FromItem,
> - StartLine,
> - StartColumn,
> - EndLine,
> - EndColumn,
> - Enabled
> - )
> + (Value1, Value2, Value3, Scope1, Scope2, Scope3) =
> + (Value1.strip(),
> Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip(),
> Scope3.strip())
> + self.ID = self.ID + self._ID_STEP_
> + if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
> + self.ID = MODEL_FILE_INF + self._ID_STEP_
> +
> + row = [ self.ID,
> + Model,
> + Value1,
> + Value2,
> + Value3,
> + Scope1,
> + Scope2,
> + Scope3,
> + BelongsToItem,
> + FromItem,
> + StartLine,
> + StartColumn,
> + EndLine,
> + EndColumn,
> + Enabled
> + ]
> + self.CurrentContent.append(row)
> + return self.ID
> +
>
> ## Query table
> #
> # @param Model: The Model of Record
> # @param Scope1: Arch of a Dsc item
> @@ -337,34 +370,37 @@ class PlatformTable(MetaFileTable):
> # @param FromItem: The item belongs to which dsc file
> #
> # @retval: A recordSet of all found records
> #
> def Query(self, Model, Scope1=None, Scope2=None,
> BelongsToItem=None, FromItem=None):
> - ConditionString = "Model=%s AND Enabled>0" % Model
> - ValueString =
> "Value1,Value2,Value3,Scope1,Scope2,Scope3,ID,StartLine"
> +
> + QueryTab = self.CurrentContent
> + result = [item for item in QueryTab if item[1] == Model and
> + item[-1]>0 ]
>
> if Scope1 is not None and Scope1 != TAB_ARCH_COMMON:
> - ConditionString += " AND (Scope1='%s' OR Scope1='COMMON')" %
> Scope1
> - if Scope2 is not None and Scope2 != TAB_COMMON:
> - # Cover the case that CodeBase is 'COMMON' for BuildOptions section
> + Sc1 = set(['COMMON'])
> + Sc1.add(Scope1)
> + result = [item for item in result if item[5] in Sc1]
> + Sc2 = set( ['COMMON','DEFAULT'])
> + if Scope2 and Scope2 != TAB_COMMON:
> if '.' in Scope2:
> Index = Scope2.index('.')
> NewScope = TAB_COMMON + Scope2[Index:]
> - ConditionString += " AND (Scope2='%s' OR Scope2='COMMON' OR
> Scope2='DEFAULT' OR Scope2='%s')" % (Scope2, NewScope)
> - else:
> - ConditionString += " AND (Scope2='%s' OR Scope2='COMMON' OR
> Scope2='DEFAULT')" % Scope2
> + Sc2.add(NewScope)
> + Sc2.add(Scope2)
> + result = [item for item in result if item[6] in Sc2]
>
> if BelongsToItem is not None:
> - ConditionString += " AND BelongsToItem=%s" % BelongsToItem
> + result = [item for item in result if item[8] ==
> + BelongsToItem]
> else:
> - ConditionString += " AND BelongsToItem<0"
> -
> + result = [item for item in result if item[8] < 0]
> if FromItem is not None:
> - ConditionString += " AND FromItem=%s" % FromItem
> + result = [item for item in result if item[9] == FromItem]
> +
> + result = [ [r[2],r[3],r[4],r[5],r[6],r[7],r[0],r[9]] for r in result ]
> + return result
>
> - SqlCommand = "SELECT %s FROM %s WHERE %s" % (ValueString,
> self.Table, ConditionString)
> - return self.Exec(SqlCommand)
>
> ## Factory class to produce different storage for different type of
> meta-file class MetaFileStorage(object):
> _FILE_TABLE_ = {
> MODEL_FILE_INF : ModuleTable,
> diff --git a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
> b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
> index 3bb287b8b2..bb6dafe9cc 100644
> --- a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
> +++ b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
> @@ -14,11 +14,10 @@
>
> ##
> # Import Modules
> #
> from __future__ import absolute_import -import sqlite3 from
> Common.StringUtils import * from Common.DataType import * from
> Common.Misc import * from types import *
>
> @@ -116,11 +115,11 @@ class WorkspaceDatabase(object):
> # get the parser ready for this file
> MetaFile = self._FILE_PARSER_[FileType](
> FilePath,
> FileType,
> Arch,
> - MetaFileStorage(self.WorkspaceDb.Cur, FilePath, FileType)
> + MetaFileStorage(self.WorkspaceDb,
> + FilePath, FileType)
> )
> # alwasy do post-process, in case of macros change
> MetaFile.DoPostProcess()
> # object the build is based on
> BuildObject = self._GENERATOR_[FileType]( @@ -147,137
> +146,27 @@ class WorkspaceDatabase(object):
> #
> # @param DbPath Path of database file
> # @param GlobalMacros Global macros used for replacement during
> file parsing
> # @prarm RenewDb=False Create new database file if it's already there
> #
> - def __init__(self, DbPath, RenewDb=False):
> - self._DbClosedFlag = False
> - if not DbPath:
> - DbPath = os.path.normpath(mws.join(GlobalData.gWorkspace, 'Conf',
> GlobalData.gDatabasePath))
> -
> - # don't create necessary path for db in memory
> - if DbPath != ':memory:':
> - DbDir = os.path.split(DbPath)[0]
> - if not os.path.exists(DbDir):
> - os.makedirs(DbDir)
> -
> - # remove db file in case inconsistency between db and file in file
> system
> - if self._CheckWhetherDbNeedRenew(RenewDb, DbPath):
> - os.remove(DbPath)
> -
> - # create db with optimized parameters
> - self.Conn = sqlite3.connect(DbPath, isolation_level='DEFERRED')
> - self.Conn.execute("PRAGMA synchronous=OFF")
> - self.Conn.execute("PRAGMA temp_store=MEMORY")
> - self.Conn.execute("PRAGMA count_changes=OFF")
> - self.Conn.execute("PRAGMA cache_size=8192")
> - #self.Conn.execute("PRAGMA page_size=8192")
> -
> - # to avoid non-ascii character conversion issue
> - self.Conn.text_factory = str
> - self.Cur = self.Conn.cursor()
> -
> + def __init__(self):
> + self.DB = dict()
> # create table for internal uses
> - self.TblDataModel = TableDataModel(self.Cur)
> - self.TblFile = TableFile(self.Cur)
> + self.TblDataModel = DataClass.MODEL_LIST
> + self.TblFile = []
> self.Platform = None
>
> # conversion object for build or file format conversion purpose
> self.BuildObject = WorkspaceDatabase.BuildObjectFactory(self)
> self.TransformObject =
> WorkspaceDatabase.TransformObjectFactory(self)
>
> - ## Check whether workspace database need to be renew.
> - # The renew reason maybe:
> - # 1) If user force to renew;
> - # 2) If user do not force renew, and
> - # a) If the time of last modified python source is newer than database
> file;
> - # b) If the time of last modified frozen executable file is newer than
> database file;
> - #
> - # @param force User force renew database
> - # @param DbPath The absolute path of workspace database file
> - #
> - # @return Bool value for whether need renew workspace databse
> - #
> - def _CheckWhetherDbNeedRenew (self, force, DbPath):
> - # if database does not exist, we need do nothing
> - if not os.path.exists(DbPath): return False
> -
> - # if user force to renew database, then not check whether database is
> out of date
> - if force: return True
> -
> - #
> - # Check the time of last modified source file or build.exe
> - # if is newer than time of database, then database need to be re-
> created.
> - #
> - timeOfToolModified = 0
> - if hasattr(sys, "frozen"):
> - exePath = os.path.abspath(sys.executable)
> - timeOfToolModified = os.stat(exePath).st_mtime
> - else:
> - curPath = os.path.dirname(__file__) # curPath is the path of
> WorkspaceDatabase.py
> - rootPath = os.path.split(curPath)[0] # rootPath is root path of python
> source, such as /BaseTools/Source/Python
> - if rootPath == "" or rootPath is None:
> - EdkLogger.verbose("\nFail to find the root path of build.exe or
> python sources, so can not \
> -determine whether database file is out of date!\n")
> -
> - # walk the root path of source or build's binary to get the time last
> modified.
> -
> - for root, dirs, files in os.walk (rootPath):
> - for dir in dirs:
> - # bypass source control folder
> - if dir.lower() in [".svn", "_svn", "cvs"]:
> - dirs.remove(dir)
> + def SetFileTimeStamp(self,FileId,TimeStamp):
> + self.TblFile[FileId][6] = TimeStamp
>
> - for file in files:
> - ext = os.path.splitext(file)[1]
> - if ext.lower() == ".py": # only check .py files
> - fd = os.stat(os.path.join(root, file))
> - if timeOfToolModified < fd.st_mtime:
> - timeOfToolModified = fd.st_mtime
> - if timeOfToolModified > os.stat(DbPath).st_mtime:
> - EdkLogger.verbose("\nWorkspace database is out of data!")
> - return True
> + def GetFileTimeStamp(self,FileId):
> + return self.TblFile[FileId][6]
>
> - return False
> -
> - ## Initialize build database
> - def InitDatabase(self):
> - EdkLogger.verbose("\nInitialize build database started ...")
> -
> - #
> - # Create new tables
> - #
> - self.TblDataModel.Create(False)
> - self.TblFile.Create(False)
> -
> - #
> - # Initialize table DataModel
> - #
> - self.TblDataModel.InitTable()
> - EdkLogger.verbose("Initialize build database ... DONE!")
> -
> - ## Query a table
> - #
> - # @param Table: The instance of the table to be queried
> - #
> - def QueryTable(self, Table):
> - Table.Query()
> -
> - def __del__(self):
> - self.Close()
> -
> - ## Close entire database
> - #
> - # Commit all first
> - # Close the connection and cursor
> - #
> - def Close(self):
> - if not self._DbClosedFlag:
> - self.Conn.commit()
> - self.Cur.close()
> - self.Conn.close()
> - self._DbClosedFlag = True
>
> ## Summarize all packages in the database
> def GetPackageList(self, Platform, Arch, TargetName, ToolChainTag):
> self.Platform = Platform
> PackageList = []
> @@ -302,11 +191,11 @@ determine whether database file is out of date!\n")
> return PackageList
>
> ## Summarize all platforms in the database
> def PlatformList(self):
> RetVal = []
> - for PlatformFile in self.TblFile.GetFileList(MODEL_FILE_DSC):
> + for PlatformFile in [item[3] for item in self.TblFile if
> + item[5] ==
> MODEL_FILE_DSC]:
> try:
>
> RetVal.append(self.BuildObject[PathClass(PlatformFile),
> TAB_COMMON])
> except:
> pass
> return RetVal
> diff --git a/BaseTools/Source/Python/build/build.py
> b/BaseTools/Source/Python/build/build.py
> index d74082fc26..317d6a2728 100644
> --- a/BaseTools/Source/Python/build/build.py
> +++ b/BaseTools/Source/Python/build/build.py
> @@ -41,11 +41,11 @@ from Common.TargetTxtClassObject import * from
> Common.ToolDefClassObject import * from Common.DataType import *
> from Common.BuildVersion import gBUILD_VERSION from AutoGen.AutoGen
> import * from Common.BuildToolError import * -from
> Workspace.WorkspaceDatabase import *
> +from Workspace.WorkspaceDatabase import WorkspaceDatabase
> from Common.MultipleWorkspace import MultipleWorkspace as mws
>
> from BuildReport import BuildReport
> from GenPatchPcdTable.GenPatchPcdTable import * from
> PatchPcdValue.PatchPcdValue import * @@ -825,14 +825,11 @@ class
> Build():
> # Get standard WORKSPACE/Conf use the absolute path
> to the WORKSPACE/Conf
> ConfDirectoryPath = mws.join(self.WorkspaceDir, 'Conf')
> GlobalData.gConfDirectory = ConfDirectoryPath
> GlobalData.gDatabasePath =
> os.path.normpath(os.path.join(ConfDirectoryPath,
> GlobalData.gDatabasePath))
>
> - if BuildOptions.DisableCache:
> - self.Db = WorkspaceDatabase(":memory:")
> - else:
> - self.Db = WorkspaceDatabase(GlobalData.gDatabasePath,
> self.Reparse)
> + self.Db = WorkspaceDatabase()
> self.BuildDatabase = self.Db.BuildObject
> self.Platform = None
> self.ToolChainFamily = None
> self.LoadFixAddress = 0
> self.UniFlag = BuildOptions.Flag
> @@ -981,13 +978,10 @@ class Build():
> # Allow case-insensitive for those from command line or
> configuration file
> ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
> if ErrorCode != 0:
> EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
>
> - # create metafile database
> - if not self.Db_Flag:
> - self.Db.InitDatabase()
>
> def InitPreBuild(self):
> self.LoadConfiguration()
> ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
> if ErrorCode != 0:
> @@ -1002,11 +996,10 @@ class Build():
> if self.ToolChainFamily:
> GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[0]
> if 'PREBUILD' in GlobalData.gCommandLineDefines:
> self.Prebuild = GlobalData.gCommandLineDefines.get('PREBUILD')
> else:
> - self.Db.InitDatabase()
> self.Db_Flag = True
> Platform = self.Db.MapPlatform(str(self.PlatformFile))
> self.Prebuild = str(Platform.Prebuild)
> if self.Prebuild:
> PrebuildList = []
> @@ -2081,17 +2074,11 @@ class Build():
> if BuildTask.HasError():
> EdkLogger.error("build", BUILD_ERROR, "Failed
> to build module", ExtraData=GlobalData.gBuildingModule)
> self.MakeTime += int(round((time.time() -
> MakeStart)))
>
> MakeContiue = time.time()
> - #
> - # Save temp tables to a TmpTableDict.
> - #
> - for Key in Wa.BuildDatabase._CACHE_:
> - if Wa.BuildDatabase._CACHE_[Key]._RawData and
> Wa.BuildDatabase._CACHE_[Key]._RawData._Table and
> Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table:
> - if
> TemporaryTablePattern.match(Wa.BuildDatabase._CACHE_[Key]._RawData.
> _Table.Table):
> -
> TmpTableDict[Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table] =
> Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Cur
> +
> #
> #
> # All modules have been put in build tasks queue. Tell task scheduler
> # to exit if all tasks are completed
> #
> @@ -2229,11 +2216,10 @@ class Build():
> else:
> self.SpawnMode = False
> self._BuildModule()
>
> if self.Target == 'cleanall':
> - self.Db.Close()
>
> RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True)
>
> def CreateAsBuiltInf(self):
> for Module in self.BuildModules:
> Module.CreateAsBuiltInf() @@ -2490,14 +2476,11 @@ def
> Main():
>
> MyBuild = Build(Target, Workspace, Option)
> GlobalData.gCommandLineDefines['ARCH'] = ' '.join(MyBuild.ArchList)
> if not (MyBuild.LaunchPrebuildFlag and
> os.path.exists(MyBuild.PlatformBuildPath)):
> MyBuild.Launch()
> - # Drop temp tables to avoid database locked.
> - for TmpTableName in TmpTableDict:
> - SqlCommand = """drop table IF EXISTS %s""" % TmpTableName
> - TmpTableDict[TmpTableName].execute(SqlCommand)
> +
> #MyBuild.DumpBuildData()
> #
> # All job done, no error found and no exception raised
> #
> BuildError = False
> @@ -2565,11 +2548,11 @@ def Main():
> else:
> BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration)
> if MyBuild is not None:
> if not BuildError:
> MyBuild.BuildReport.GenerateReport(BuildDurationStr,
> LogBuildTime(MyBuild.AutoGenTime), LogBuildTime(MyBuild.MakeTime),
> LogBuildTime(MyBuild.GenFdsTime))
> - MyBuild.Db.Close()
> +
> EdkLogger.SetLevel(EdkLogger.QUIET)
> EdkLogger.quiet("\n- %s -" % Conclusion)
> EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d
> %Y",
> time.localtime()))
> EdkLogger.quiet("Build total time: %s\n" % BuildDurationStr)
> return ReturnCode
> --
> 2.19.1.windows.1
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2018-11-11 0:43 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2018-11-09 8:41 [Patch V2] BaseTools: Replace the sqlite database with list BobCF
2018-11-09 15:24 ` Carsey, Jaben
2018-11-11 0:43 ` Feng, Bob C
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox