Re: [Patch V2] BaseTools: Replace the sqlite database with list

["Carsey, Jaben" <jaben.carsey@intel.com>]

This is much easier to review! Thanks!

I am good with the code change.

I wonder: If we do not have the DB does it take longer if I do build, clean, build than it did before?

> -----Original Message-----
> From: Feng, Bob C
> Sent: Friday, November 09, 2018 12:42 AM
> To: edk2-devel@lists.01.org
> Cc: Gao, Liming <liming.gao@intel.com>; Carsey, Jaben
> <jaben.carsey@intel.com>
> Subject: [Patch V2] BaseTools: Replace the sqlite database with list
> Importance: High
> 
> https://bugzilla.tianocore.org/show_bug.cgi?id=1288
> 
> [V2]
> Optimize this patch so that it can be easy to review.
> This patch is just apply the change to original files while
> not create new similar files.
> 
> [V1]
> This patch is one of build tool performance improvement
> series patches.
> 
> This patch is going to use python list to store the parser data
> instead of using sqlite database.
> 
> The replacement solution is as below:
> 
> SQL insert: list.append()
> SQL select: list comprehension. for example:
> Select * from table where field = “something”
> ->
> [ item for item in table if item[3] == “something”]
> 
> SQL update: python map function. for example:
> Update table set field1=newvalue where filed2 = “something”.
> -> map(lambda x: x[1] = newvalue,
>    [item for item in table if item[2] == “something”])
> 
> SQL delete: list comprehension.
> 
> With this change, We can save the time of interpreting SQL statement
> and the time of write database to file system
> 
> Contributed-under: TianoCore Contribution Agreement 1.1
> Signed-off-by: BobCF <bob.c.feng@intel.com>
> Cc: Liming Gao <liming.gao@intel.com>
> Cc: Jaben Carsey <jaben.carsey@intel.com>
> ---
>  BaseTools/Source/Python/GenFds/GenFds.py      |   3 +-
>  .../Source/Python/Workspace/MetaDataTable.py  |  93 ++-----
>  .../Source/Python/Workspace/MetaFileParser.py |   5 +-
>  .../Source/Python/Workspace/MetaFileTable.py  | 248 ++++++++++--------
>  .../Python/Workspace/WorkspaceDatabase.py     | 131 +--------
>  BaseTools/Source/Python/build/build.py        |  27 +-
>  6 files changed, 186 insertions(+), 321 deletions(-)
> 
> diff --git a/BaseTools/Source/Python/GenFds/GenFds.py
> b/BaseTools/Source/Python/GenFds/GenFds.py
> index 0c8091b798..4fd96706af 100644
> --- a/BaseTools/Source/Python/GenFds/GenFds.py
> +++ b/BaseTools/Source/Python/GenFds/GenFds.py
> @@ -222,12 +222,11 @@ def main():
>          if "TOOL_CHAIN_TAG" not in GlobalData.gGlobalDefines:
>              GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] =
> GenFdsGlobalVariable.ToolChainTag
> 
>          """call Workspace build create database"""
>          GlobalData.gDatabasePath =
> os.path.normpath(os.path.join(ConfDirectoryPath,
> GlobalData.gDatabasePath))
> -        BuildWorkSpace = WorkspaceDatabase(GlobalData.gDatabasePath)
> -        BuildWorkSpace.InitDatabase()
> +        BuildWorkSpace = WorkspaceDatabase()
> 
>          #
>          # Get files real name in workspace dir
>          #
>          GlobalData.gAllFiles = DirCache(Workspace)
> diff --git a/BaseTools/Source/Python/Workspace/MetaDataTable.py
> b/BaseTools/Source/Python/Workspace/MetaDataTable.py
> index bd751eadfb..8becddbe08 100644
> --- a/BaseTools/Source/Python/Workspace/MetaDataTable.py
> +++ b/BaseTools/Source/Python/Workspace/MetaDataTable.py
> @@ -37,84 +37,58 @@ class Table(object):
>      _COLUMN_ = ''
>      _ID_STEP_ = 1
>      _ID_MAX_ = 0x80000000
>      _DUMMY_ = 0
> 
> -    def __init__(self, Cursor, Name='', IdBase=0, Temporary=False):
> -        self.Cur = Cursor
> +    def __init__(self, Db, Name='', IdBase=0, Temporary=False):
> +        self.Db = Db
>          self.Table = Name
>          self.IdBase = int(IdBase)
>          self.ID = int(IdBase)
>          self.Temporary = Temporary
> +        self.Contents = []
> 
>      def __str__(self):
>          return self.Table
> 
>      ## Create table
>      #
>      # Create a table
>      #
>      def Create(self, NewTable=True):
> -        if NewTable:
> -            self.Drop()
> -
> -        if self.Temporary:
> -            SqlCommand = """create temp table IF NOT EXISTS %s (%s)""" %
> (self.Table, self._COLUMN_)
> -        else:
> -            SqlCommand = """create table IF NOT EXISTS %s (%s)""" % (self.Table,
> self._COLUMN_)
> -        EdkLogger.debug(EdkLogger.DEBUG_8, SqlCommand)
> -        self.Cur.execute(SqlCommand)
> +        self.Db.CreateEmptyTable(self.Table)
>          self.ID = self.GetId()
> 
>      ## Insert table
>      #
>      # Insert a record into a table
>      #
>      def Insert(self, *Args):
>          self.ID = self.ID + self._ID_STEP_
>          if self.ID >= (self.IdBase + self._ID_MAX_):
>              self.ID = self.IdBase + self._ID_STEP_
> -        Values = ", ".join(str(Arg) for Arg in Args)
> -        SqlCommand = "insert into %s values(%s, %s)" % (self.Table, self.ID,
> Values)
> -        EdkLogger.debug(EdkLogger.DEBUG_5, SqlCommand)
> -        self.Cur.execute(SqlCommand)
> -        return self.ID
> +        row = [self.ID]
> +        row.extend(Args)
> +        self.Contents.append(row)
> 
> -    ## Query table
> -    #
> -    # Query all records of the table
> -    #
> -    def Query(self):
> -        SqlCommand = """select * from %s""" % self.Table
> -        self.Cur.execute(SqlCommand)
> -        for Rs in self.Cur:
> -            EdkLogger.verbose(str(Rs))
> -        TotalCount = self.GetId()
> +        return self.ID
> 
> -    ## Drop a table
> -    #
> -    # Drop the table
> -    #
> -    def Drop(self):
> -        SqlCommand = """drop table IF EXISTS %s""" % self.Table
> -        self.Cur.execute(SqlCommand)
> 
>      ## Get count
>      #
>      # Get a count of all records of the table
>      #
>      # @retval Count:  Total count of all records
>      #
>      def GetCount(self):
> -        SqlCommand = """select count(ID) from %s""" % self.Table
> -        Record = self.Cur.execute(SqlCommand).fetchall()
> -        return Record[0][0]
> +        tab = self.Db.GetTable(self.Table)
> +        return len(tab)
> +
> 
>      def GetId(self):
> -        SqlCommand = """select max(ID) from %s""" % self.Table
> -        Record = self.Cur.execute(SqlCommand).fetchall()
> -        Id = Record[0][0]
> +        tab = self.Db.GetTable(self.Table)
> +        Id = max([int(item[0]) for item in tab])
>          if Id is None:
>              Id = self.IdBase
>          return Id
> 
>      ## Init the ID of the table
> @@ -132,29 +106,30 @@ class Table(object):
>      #
>      # @retval RecordSet:  The result after executed
>      #
>      def Exec(self, SqlCommand):
>          EdkLogger.debug(EdkLogger.DEBUG_5, SqlCommand)
> -        self.Cur.execute(SqlCommand)
> -        RecordSet = self.Cur.fetchall()
> +        self.Db.execute(SqlCommand)
> +        RecordSet = self.Db.fetchall()
>          return RecordSet
> 
>      def SetEndFlag(self):
> -        self.Exec("insert into %s values(%s)" % (self.Table, self._DUMMY_))
> -        #
> -        # Need to execution commit for table data changed.
> -        #
> -        self.Cur.connection.commit()
> +        Tab = self.Db.GetTable(self.Table)
> +        Tab.append(self._DUMMY_)
> +
> 
>      def IsIntegral(self):
> -        Result = self.Exec("select min(ID) from %s" % (self.Table))
> -        if Result[0][0] != -1:
> +        tab = self.Db.GetTable(self.Table)
> +        Id = min([int(item[0]) for item in tab])
> +        if Id != -1:
>              return False
>          return True
> 
>      def GetAll(self):
> -        return self.Exec("select * from %s where ID > 0 order by ID" %
> (self.Table))
> +        tab = self.Db.GetTable(self.Table)
> +        return tab
> +
> 
>  ## TableFile
>  #
>  # This class defined a table used for file
>  #
> @@ -225,26 +200,10 @@ class TableFile(Table):
>                          File.Path,
>                          Model,
>                          File.TimeStamp
>                          )
> 
> -    ## Get ID of a given file
> -    #
> -    #   @param  FilePath    Path of file
> -    #
> -    #   @retval ID          ID value of given file in the table
> -    #
> -    def GetFileId(self, File, FromItem=None):
> -        if FromItem:
> -            QueryScript = "select ID from %s where FullPath = '%s' and FromItem
> = %s" % (self.Table, str(File), str(FromItem))
> -        else:
> -            QueryScript = "select ID from %s where FullPath = '%s'" % (self.Table,
> str(File))
> -        RecordList = self.Exec(QueryScript)
> -        if len(RecordList) == 0:
> -            return None
> -        return RecordList[0][0]
> -
>      ## Get type of a given file
>      #
>      #   @param  FileId      ID of a file
>      #
>      #   @retval file_type   Model value of given file in the table
> @@ -343,11 +302,11 @@ class TableDataModel(Table):
>      # @retval CrossIndex:  CrossIndex of the model
>      #
>      def GetCrossIndex(self, ModelName):
>          CrossIndex = -1
>          SqlCommand = """select CrossIndex from DataModel where name = '"""
> + ModelName + """'"""
> -        self.Cur.execute(SqlCommand)
> -        for Item in self.Cur:
> +        self.Db.execute(SqlCommand)
> +        for Item in self.Db:
>              CrossIndex = Item[0]
> 
>          return CrossIndex
> 
> diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py
> b/BaseTools/Source/Python/Workspace/MetaFileParser.py
> index 804a4aa5cb..a5ae057bd1 100644
> --- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
> +++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
> @@ -1306,12 +1306,11 @@ class DscParser(MetaFileParser):
>              MODEL_UNKNOWN                                   :   self._Skip,
>              MODEL_META_DATA_USER_EXTENSION                  :
> self._SkipUserExtension,
>              MODEL_META_DATA_CONDITIONAL_STATEMENT_ERROR     :
> self._ProcessError,
>          }
> 
> -        self._Table = MetaFileStorage(self._RawTable.Cur, self.MetaFile,
> MODEL_FILE_DSC, True)
> -        self._Table.Create()
> +        self._Table = MetaFileStorage(self._RawTable.DB, self.MetaFile,
> MODEL_FILE_DSC, True)
>          self._DirectiveStack = []
>          self._DirectiveEvalStack = []
>          self._FileWithError = self.MetaFile
>          self._FileLocalMacros = {}
>          self._SectionsMacroDict.clear()
> @@ -1566,11 +1565,11 @@ class DscParser(MetaFileParser):
>              FromItem = self._Content[self._ContentIndex - 1][0]
>              if self._InSubsection:
>                  Owner = self._Content[self._ContentIndex - 1][8]
>              else:
>                  Owner = self._Content[self._ContentIndex - 1][0]
> -            IncludedFileTable = MetaFileStorage(self._Table.Cur, IncludedFile1,
> MODEL_FILE_DSC, False, FromItem=FromItem)
> +            IncludedFileTable = MetaFileStorage(self._RawTable.DB,
> IncludedFile1, MODEL_FILE_DSC, False, FromItem=FromItem)
>              Parser = DscParser(IncludedFile1, self._FileType, self._Arch,
> IncludedFileTable,
>                                 Owner=Owner, From=FromItem)
> 
>              self.IncludedFiles.add (IncludedFile1)
> 
> diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py
> b/BaseTools/Source/Python/Workspace/MetaFileTable.py
> index e0a0b8d923..081970dba8 100644
> --- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
> +++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
> @@ -18,59 +18,64 @@ from __future__ import absolute_import
>  import uuid
> 
>  import Common.EdkLogger as EdkLogger
>  from Common.BuildToolError import FORMAT_INVALID
> 
> -from .MetaDataTable import Table, TableFile
> -from .MetaDataTable import ConvertToSqlString
>  from CommonDataClass.DataClass import MODEL_FILE_DSC,
> MODEL_FILE_DEC, MODEL_FILE_INF, \
>                                        MODEL_FILE_OTHERS
>  from Common.DataType import *
> 
> -class MetaFileTable(Table):
> +class MetaFileTable():
>      # TRICK: use file ID as the part before '.'
>      _ID_STEP_ = 0.00000001
>      _ID_MAX_ = 0.99999999
> 
>      ## Constructor
> -    def __init__(self, Cursor, MetaFile, FileType, Temporary,
> FromItem=None):
> +    def __init__(self, DB, MetaFile, FileType, Temporary, FromItem=None):
>          self.MetaFile = MetaFile
> -
> -        self._FileIndexTable = TableFile(Cursor)
> -        self._FileIndexTable.Create(False)
> -
> -        FileId = self._FileIndexTable.GetFileId(MetaFile, FromItem)
> -        if not FileId:
> -            FileId = self._FileIndexTable.InsertFile(MetaFile, FileType, FromItem)
> -
> +        self.TableName = ""
> +        self.DB = DB
> +        self._NumpyTab = None
> +        self.FileId = len(DB.TblFile)
> +        self.ID = self.FileId
> +        self.CurrentContent = []
> +        DB.TblFile.append([MetaFile.Name,
> +                        MetaFile.Ext,
> +                        MetaFile.Dir,
> +                        MetaFile.Path,
> +                        FileType,
> +                        MetaFile.TimeStamp,
> +                        FromItem])
>          if Temporary:
> -            TableName = "_%s_%s_%s" % (FileType, FileId, uuid.uuid4().hex)
> +            self.TableName = "_%s_%s_%s" % (FileType, len(DB.TblFile),
> uuid.uuid4().hex)
>          else:
> -            TableName = "_%s_%s" % (FileType, FileId)
> -
> -        #Table.__init__(self, Cursor, TableName, FileId, False)
> -        Table.__init__(self, Cursor, TableName, FileId, Temporary)
> -        self.Create(not self.IsIntegrity())
> +            self.TableName = "_%s_%s" % (FileType, len(DB.TblFile))
> 
>      def IsIntegrity(self):
>          try:
>              TimeStamp = self.MetaFile.TimeStamp
> -            Result = self.Cur.execute("select ID from %s where ID<0" %
> (self.Table)).fetchall()
> +            Result = int(self.CurrentContent[-1][0]) < 0
>              if not Result:
>                  # update the timestamp in database
> -                self._FileIndexTable.SetFileTimeStamp(self.IdBase, TimeStamp)
> +                self.DB.SetFileTimeStamp(self.FileId, TimeStamp)
>                  return False
> 
> -            if TimeStamp != self._FileIndexTable.GetFileTimeStamp(self.IdBase):
> +            if TimeStamp != self.DB.GetFileTimeStamp(self.FileId):
>                  # update the timestamp in database
> -                self._FileIndexTable.SetFileTimeStamp(self.IdBase, TimeStamp)
> +                self.DB.SetFileTimeStamp(self.FileId, TimeStamp)
>                  return False
>          except Exception as Exc:
>              EdkLogger.debug(EdkLogger.DEBUG_5, str(Exc))
>              return False
>          return True
> 
> +    def SetEndFlag(self):
> +        self.CurrentContent.append(self._DUMMY_)
> +
> +    def GetAll(self):
> +        return [item for item in self.CurrentContent if item[0] > 0 ]
> +
>  ## Python class representation of table storing module data
>  class ModuleTable(MetaFileTable):
>      _ID_STEP_ = 0.00000001
>      _ID_MAX_  = 0.99999999
>      _COLUMN_ = '''
> @@ -87,15 +92,15 @@ class ModuleTable(MetaFileTable):
>          EndLine INTEGER NOT NULL,
>          EndColumn INTEGER NOT NULL,
>          Enabled INTEGER DEFAULT 0
>          '''
>      # used as table end flag, in case the changes to database is not committed
> to db file
> -    _DUMMY_ = "-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -
> 1"
> +    _DUMMY_ = [-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -
> 1]
> 
>      ## Constructor
> -    def __init__(self, Cursor, MetaFile, Temporary):
> -        MetaFileTable.__init__(self, Cursor, MetaFile, MODEL_FILE_INF,
> Temporary)
> +    def __init__(self, Db, MetaFile, Temporary):
> +        MetaFileTable.__init__(self, Db, MetaFile, MODEL_FILE_INF,
> Temporary)
> 
>      ## Insert a record into table Inf
>      #
>      # @param Model:          Model of a Inf item
>      # @param Value1:         Value1 of a Inf item
> @@ -110,48 +115,62 @@ class ModuleTable(MetaFileTable):
>      # @param EndColumn:      EndColumn of a Inf item
>      # @param Enabled:        If this item enabled
>      #
>      def Insert(self, Model, Value1, Value2, Value3,
> Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON,
>                 BelongsToItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1,
> EndColumn=-1, Enabled=0):
> -        (Value1, Value2, Value3, Scope1, Scope2) = ConvertToSqlString((Value1,
> Value2, Value3, Scope1, Scope2))
> -        return Table.Insert(
> -                        self,
> -                        Model,
> -                        Value1,
> -                        Value2,
> -                        Value3,
> -                        Scope1,
> -                        Scope2,
> -                        BelongsToItem,
> -                        StartLine,
> -                        StartColumn,
> -                        EndLine,
> -                        EndColumn,
> -                        Enabled
> -                        )
> +
> +        (Value1, Value2, Value3, Scope1, Scope2) = (Value1.strip(),
> Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip())
> +        self.ID = self.ID + self._ID_STEP_
> +        if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
> +            self.ID = MODEL_FILE_INF + self._ID_STEP_
> +
> +        row = [ self.ID,
> +                Model,
> +                Value1,
> +                Value2,
> +                Value3,
> +                Scope1,
> +                Scope2,
> +                BelongsToItem,
> +                StartLine,
> +                StartColumn,
> +                EndLine,
> +                EndColumn,
> +                Enabled
> +            ]
> +        self.CurrentContent.append(row)
> +        return self.ID
> +
> 
>      ## Query table
>      #
>      # @param    Model:      The Model of Record
>      # @param    Arch:       The Arch attribute of Record
>      # @param    Platform    The Platform attribute of Record
>      #
>      # @retval:       A recordSet of all found records
>      #
>      def Query(self, Model, Arch=None, Platform=None,
> BelongsToItem=None):
> -        ConditionString = "Model=%s AND Enabled>=0" % Model
> -        ValueString = "Value1,Value2,Value3,Scope1,Scope2,ID,StartLine"
> +
> +        QueryTab = self.CurrentContent
> +        result = [item for item in QueryTab if item[1] == Model and item[-1]>=0
> ]
> 
>          if Arch is not None and Arch != TAB_ARCH_COMMON:
> -            ConditionString += " AND (Scope1='%s' OR Scope1='COMMON')" %
> Arch
> +            ArchList = set(['COMMON'])
> +            ArchList.add(Arch)
> +            result = [item for item in result if item[5] in ArchList]
> +
>          if Platform is not None and Platform != TAB_COMMON:
> -            ConditionString += " AND (Scope2='%s' OR Scope2='COMMON' OR
> Scope2='DEFAULT')" % Platform
> +            Platformlist = set( ['COMMON','DEFAULT'])
> +            Platformlist.add(Platform)
> +            result = [item for item in result if item[6] in Platformlist]
> +
>          if BelongsToItem is not None:
> -            ConditionString += " AND BelongsToItem=%s" % BelongsToItem
> +            result = [item for item in result if item[7] == BelongsToItem]
> 
> -        SqlCommand = "SELECT %s FROM %s WHERE %s" % (ValueString,
> self.Table, ConditionString)
> -        return self.Exec(SqlCommand)
> +        result = [ [r[2],r[3],r[4],r[5],r[6],r[0],r[9]] for r in result ]
> +        return result
> 
>  ## Python class representation of table storing package data
>  class PackageTable(MetaFileTable):
>      _COLUMN_ = '''
>          ID REAL PRIMARY KEY,
> @@ -167,11 +186,11 @@ class PackageTable(MetaFileTable):
>          EndLine INTEGER NOT NULL,
>          EndColumn INTEGER NOT NULL,
>          Enabled INTEGER DEFAULT 0
>          '''
>      # used as table end flag, in case the changes to database is not committed
> to db file
> -    _DUMMY_ = "-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -
> 1"
> +    _DUMMY_ = [-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -
> 1]
> 
>      ## Constructor
>      def __init__(self, Cursor, MetaFile, Temporary):
>          MetaFileTable.__init__(self, Cursor, MetaFile, MODEL_FILE_DEC,
> Temporary)
> 
> @@ -192,52 +211,60 @@ class PackageTable(MetaFileTable):
>      # @param EndColumn:      EndColumn of a Dec item
>      # @param Enabled:        If this item enabled
>      #
>      def Insert(self, Model, Value1, Value2, Value3,
> Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON,
>                 BelongsToItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1,
> EndColumn=-1, Enabled=0):
> -        (Value1, Value2, Value3, Scope1, Scope2) = ConvertToSqlString((Value1,
> Value2, Value3, Scope1, Scope2))
> -        return Table.Insert(
> -                        self,
> -                        Model,
> -                        Value1,
> -                        Value2,
> -                        Value3,
> -                        Scope1,
> -                        Scope2,
> -                        BelongsToItem,
> -                        StartLine,
> -                        StartColumn,
> -                        EndLine,
> -                        EndColumn,
> -                        Enabled
> -                        )
> +        (Value1, Value2, Value3, Scope1, Scope2) = (Value1.strip(),
> Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip())
> +        self.ID = self.ID + self._ID_STEP_
> +        if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
> +            self.ID = MODEL_FILE_INF + self._ID_STEP_
> +
> +        row = [ self.ID,
> +                Model,
> +                Value1,
> +                Value2,
> +                Value3,
> +                Scope1,
> +                Scope2,
> +                BelongsToItem,
> +                StartLine,
> +                StartColumn,
> +                EndLine,
> +                EndColumn,
> +                Enabled
> +            ]
> +        self.CurrentContent.append(row)
> +        return self.ID
> 
>      ## Query table
>      #
>      # @param    Model:  The Model of Record
>      # @param    Arch:   The Arch attribute of Record
>      #
>      # @retval:       A recordSet of all found records
>      #
>      def Query(self, Model, Arch=None):
> -        ConditionString = "Model=%s AND Enabled>=0" % Model
> -        ValueString = "Value1,Value2,Value3,Scope1,Scope2,ID,StartLine"
> +
> +        QueryTab = self.CurrentContent
> +        result = [item for item in QueryTab if item[1] == Model and item[-1]>=0
> ]
> 
>          if Arch is not None and Arch != TAB_ARCH_COMMON:
> -            ConditionString += " AND (Scope1='%s' OR Scope1='COMMON')" %
> Arch
> +            ArchList = set(['COMMON'])
> +            ArchList.add(Arch)
> +            result = [item for item in result if item[5] in ArchList]
> 
> -        SqlCommand = "SELECT %s FROM %s WHERE %s" % (ValueString,
> self.Table, ConditionString)
> -        return self.Exec(SqlCommand)
> +        return [[r[2], r[3], r[4], r[5], r[6], r[0], r[8]] for r in result]
> 
>      def GetValidExpression(self, TokenSpaceGuid, PcdCName):
> -        SqlCommand = "select Value1,StartLine from %s WHERE Value2='%s'
> and Value3='%s'" % (self.Table, TokenSpaceGuid, PcdCName)
> -        self.Cur.execute(SqlCommand)
> +
> +        QueryTab = self.CurrentContent
> +        result = [[item[2], item[8]] for item in QueryTab if item[3] ==
> TokenSpaceGuid and item[4] == PcdCName]
>          validateranges = []
>          validlists = []
>          expressions = []
>          try:
> -            for row in self.Cur:
> +            for row in result:
>                  comment = row[0]
> 
>                  LineNum = row[1]
>                  comment = comment.strip("#")
>                  comment = comment.strip()
> @@ -281,11 +308,11 @@ class PlatformTable(MetaFileTable):
>          EndLine INTEGER NOT NULL,
>          EndColumn INTEGER NOT NULL,
>          Enabled INTEGER DEFAULT 0
>          '''
>      # used as table end flag, in case the changes to database is not committed
> to db file
> -    _DUMMY_ = "-1, -1, '====', '====', '====', '====', '====','====', -1, -1, -1,
> -1, -1, -1, -1"
> +    _DUMMY_ = [-1, -1, '====', '====', '====', '====', '====','====', -1, -1, -1,
> -1, -1, -1, -1]
> 
>      ## Constructor
>      def __init__(self, Cursor, MetaFile, Temporary, FromItem=0):
>          MetaFileTable.__init__(self, Cursor, MetaFile, MODEL_FILE_DSC,
> Temporary, FromItem)
> 
> @@ -307,28 +334,34 @@ class PlatformTable(MetaFileTable):
>      # @param EndColumn:      EndColumn of a Dsc item
>      # @param Enabled:        If this item enabled
>      #
>      def Insert(self, Model, Value1, Value2, Value3,
> Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON,
> Scope3=TAB_DEFAULT_STORES_DEFAULT,BelongsToItem=-1,
>                 FromItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1,
> EndColumn=-1, Enabled=1):
> -        (Value1, Value2, Value3, Scope1, Scope2, Scope3) =
> ConvertToSqlString((Value1, Value2, Value3, Scope1, Scope2, Scope3))
> -        return Table.Insert(
> -                        self,
> -                        Model,
> -                        Value1,
> -                        Value2,
> -                        Value3,
> -                        Scope1,
> -                        Scope2,
> -                        Scope3,
> -                        BelongsToItem,
> -                        FromItem,
> -                        StartLine,
> -                        StartColumn,
> -                        EndLine,
> -                        EndColumn,
> -                        Enabled
> -                        )
> +        (Value1, Value2, Value3, Scope1, Scope2, Scope3) = (Value1.strip(),
> Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip(), Scope3.strip())
> +        self.ID = self.ID + self._ID_STEP_
> +        if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
> +            self.ID = MODEL_FILE_INF + self._ID_STEP_
> +
> +        row = [ self.ID,
> +                Model,
> +                Value1,
> +                Value2,
> +                Value3,
> +                Scope1,
> +                Scope2,
> +                Scope3,
> +                BelongsToItem,
> +                FromItem,
> +                StartLine,
> +                StartColumn,
> +                EndLine,
> +                EndColumn,
> +                Enabled
> +            ]
> +        self.CurrentContent.append(row)
> +        return self.ID
> +
> 
>      ## Query table
>      #
>      # @param Model:          The Model of Record
>      # @param Scope1:         Arch of a Dsc item
> @@ -337,34 +370,37 @@ class PlatformTable(MetaFileTable):
>      # @param FromItem:       The item belongs to which dsc file
>      #
>      # @retval:       A recordSet of all found records
>      #
>      def Query(self, Model, Scope1=None, Scope2=None,
> BelongsToItem=None, FromItem=None):
> -        ConditionString = "Model=%s AND Enabled>0" % Model
> -        ValueString =
> "Value1,Value2,Value3,Scope1,Scope2,Scope3,ID,StartLine"
> +
> +        QueryTab = self.CurrentContent
> +        result = [item for item in QueryTab if item[1] == Model and item[-1]>0 ]
> 
>          if Scope1 is not None and Scope1 != TAB_ARCH_COMMON:
> -            ConditionString += " AND (Scope1='%s' OR Scope1='COMMON')" %
> Scope1
> -        if Scope2 is not None and Scope2 != TAB_COMMON:
> -            # Cover the case that CodeBase is 'COMMON' for BuildOptions section
> +            Sc1 = set(['COMMON'])
> +            Sc1.add(Scope1)
> +            result = [item for item in result if item[5] in Sc1]
> +        Sc2 = set( ['COMMON','DEFAULT'])
> +        if Scope2 and Scope2 != TAB_COMMON:
>              if '.' in Scope2:
>                  Index = Scope2.index('.')
>                  NewScope = TAB_COMMON + Scope2[Index:]
> -                ConditionString += " AND (Scope2='%s' OR Scope2='COMMON' OR
> Scope2='DEFAULT' OR Scope2='%s')" % (Scope2, NewScope)
> -            else:
> -                ConditionString += " AND (Scope2='%s' OR Scope2='COMMON' OR
> Scope2='DEFAULT')" % Scope2
> +                Sc2.add(NewScope)
> +            Sc2.add(Scope2)
> +            result = [item for item in result if item[6] in Sc2]
> 
>          if BelongsToItem is not None:
> -            ConditionString += " AND BelongsToItem=%s" % BelongsToItem
> +            result = [item for item in result if item[8] == BelongsToItem]
>          else:
> -            ConditionString += " AND BelongsToItem<0"
> -
> +            result = [item for item in result if item[8] < 0]
>          if FromItem is not None:
> -            ConditionString += " AND FromItem=%s" % FromItem
> +            result = [item for item in result if item[9] == FromItem]
> +
> +        result = [ [r[2],r[3],r[4],r[5],r[6],r[7],r[0],r[9]] for r in result ]
> +        return result
> 
> -        SqlCommand = "SELECT %s FROM %s WHERE %s" % (ValueString,
> self.Table, ConditionString)
> -        return self.Exec(SqlCommand)
> 
>  ## Factory class to produce different storage for different type of meta-file
>  class MetaFileStorage(object):
>      _FILE_TABLE_ = {
>          MODEL_FILE_INF      :   ModuleTable,
> diff --git a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
> b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
> index 3bb287b8b2..bb6dafe9cc 100644
> --- a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
> +++ b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
> @@ -14,11 +14,10 @@
> 
>  ##
>  # Import Modules
>  #
>  from __future__ import absolute_import
> -import sqlite3
>  from Common.StringUtils import *
>  from Common.DataType import *
>  from Common.Misc import *
>  from types import *
> 
> @@ -116,11 +115,11 @@ class WorkspaceDatabase(object):
>              # get the parser ready for this file
>              MetaFile = self._FILE_PARSER_[FileType](
>                                  FilePath,
>                                  FileType,
>                                  Arch,
> -                                MetaFileStorage(self.WorkspaceDb.Cur, FilePath, FileType)
> +                                MetaFileStorage(self.WorkspaceDb, FilePath, FileType)
>                                  )
>              # alwasy do post-process, in case of macros change
>              MetaFile.DoPostProcess()
>              # object the build is based on
>              BuildObject = self._GENERATOR_[FileType](
> @@ -147,137 +146,27 @@ class WorkspaceDatabase(object):
>      #
>      # @param DbPath             Path of database file
>      # @param GlobalMacros       Global macros used for replacement during
> file parsing
>      # @prarm RenewDb=False      Create new database file if it's already there
>      #
> -    def __init__(self, DbPath, RenewDb=False):
> -        self._DbClosedFlag = False
> -        if not DbPath:
> -            DbPath = os.path.normpath(mws.join(GlobalData.gWorkspace, 'Conf',
> GlobalData.gDatabasePath))
> -
> -        # don't create necessary path for db in memory
> -        if DbPath != ':memory:':
> -            DbDir = os.path.split(DbPath)[0]
> -            if not os.path.exists(DbDir):
> -                os.makedirs(DbDir)
> -
> -            # remove db file in case inconsistency between db and file in file
> system
> -            if self._CheckWhetherDbNeedRenew(RenewDb, DbPath):
> -                os.remove(DbPath)
> -
> -        # create db with optimized parameters
> -        self.Conn = sqlite3.connect(DbPath, isolation_level='DEFERRED')
> -        self.Conn.execute("PRAGMA synchronous=OFF")
> -        self.Conn.execute("PRAGMA temp_store=MEMORY")
> -        self.Conn.execute("PRAGMA count_changes=OFF")
> -        self.Conn.execute("PRAGMA cache_size=8192")
> -        #self.Conn.execute("PRAGMA page_size=8192")
> -
> -        # to avoid non-ascii character conversion issue
> -        self.Conn.text_factory = str
> -        self.Cur = self.Conn.cursor()
> -
> +    def __init__(self):
> +        self.DB = dict()
>          # create table for internal uses
> -        self.TblDataModel = TableDataModel(self.Cur)
> -        self.TblFile = TableFile(self.Cur)
> +        self.TblDataModel = DataClass.MODEL_LIST
> +        self.TblFile = []
>          self.Platform = None
> 
>          # conversion object for build or file format conversion purpose
>          self.BuildObject = WorkspaceDatabase.BuildObjectFactory(self)
>          self.TransformObject =
> WorkspaceDatabase.TransformObjectFactory(self)
> 
> -    ## Check whether workspace database need to be renew.
> -    #  The renew reason maybe:
> -    #  1) If user force to renew;
> -    #  2) If user do not force renew, and
> -    #     a) If the time of last modified python source is newer than database
> file;
> -    #     b) If the time of last modified frozen executable file is newer than
> database file;
> -    #
> -    #  @param force     User force renew database
> -    #  @param DbPath    The absolute path of workspace database file
> -    #
> -    #  @return Bool value for whether need renew workspace databse
> -    #
> -    def _CheckWhetherDbNeedRenew (self, force, DbPath):
> -        # if database does not exist, we need do nothing
> -        if not os.path.exists(DbPath): return False
> -
> -        # if user force to renew database, then not check whether database is
> out of date
> -        if force: return True
> -
> -        #
> -        # Check the time of last modified source file or build.exe
> -        # if is newer than time of database, then database need to be re-
> created.
> -        #
> -        timeOfToolModified = 0
> -        if hasattr(sys, "frozen"):
> -            exePath             = os.path.abspath(sys.executable)
> -            timeOfToolModified  = os.stat(exePath).st_mtime
> -        else:
> -            curPath  = os.path.dirname(__file__) # curPath is the path of
> WorkspaceDatabase.py
> -            rootPath = os.path.split(curPath)[0] # rootPath is root path of python
> source, such as /BaseTools/Source/Python
> -            if rootPath == "" or rootPath is None:
> -                EdkLogger.verbose("\nFail to find the root path of build.exe or
> python sources, so can not \
> -determine whether database file is out of date!\n")
> -
> -            # walk the root path of source or build's binary to get the time last
> modified.
> -
> -            for root, dirs, files in os.walk (rootPath):
> -                for dir in dirs:
> -                    # bypass source control folder
> -                    if dir.lower() in [".svn", "_svn", "cvs"]:
> -                        dirs.remove(dir)
> +    def SetFileTimeStamp(self,FileId,TimeStamp):
> +        self.TblFile[FileId][6] = TimeStamp
> 
> -                for file in files:
> -                    ext = os.path.splitext(file)[1]
> -                    if ext.lower() == ".py":            # only check .py files
> -                        fd = os.stat(os.path.join(root, file))
> -                        if timeOfToolModified < fd.st_mtime:
> -                            timeOfToolModified = fd.st_mtime
> -        if timeOfToolModified > os.stat(DbPath).st_mtime:
> -            EdkLogger.verbose("\nWorkspace database is out of data!")
> -            return True
> +    def GetFileTimeStamp(self,FileId):
> +        return self.TblFile[FileId][6]
> 
> -        return False
> -
> -    ## Initialize build database
> -    def InitDatabase(self):
> -        EdkLogger.verbose("\nInitialize build database started ...")
> -
> -        #
> -        # Create new tables
> -        #
> -        self.TblDataModel.Create(False)
> -        self.TblFile.Create(False)
> -
> -        #
> -        # Initialize table DataModel
> -        #
> -        self.TblDataModel.InitTable()
> -        EdkLogger.verbose("Initialize build database ... DONE!")
> -
> -    ## Query a table
> -    #
> -    # @param Table:  The instance of the table to be queried
> -    #
> -    def QueryTable(self, Table):
> -        Table.Query()
> -
> -    def __del__(self):
> -        self.Close()
> -
> -    ## Close entire database
> -    #
> -    # Commit all first
> -    # Close the connection and cursor
> -    #
> -    def Close(self):
> -        if not self._DbClosedFlag:
> -            self.Conn.commit()
> -            self.Cur.close()
> -            self.Conn.close()
> -            self._DbClosedFlag = True
> 
>      ## Summarize all packages in the database
>      def GetPackageList(self, Platform, Arch, TargetName, ToolChainTag):
>          self.Platform = Platform
>          PackageList = []
> @@ -302,11 +191,11 @@ determine whether database file is out of date!\n")
>          return PackageList
> 
>      ## Summarize all platforms in the database
>      def PlatformList(self):
>          RetVal = []
> -        for PlatformFile in self.TblFile.GetFileList(MODEL_FILE_DSC):
> +        for PlatformFile in [item[3] for item in self.TblFile if item[5] ==
> MODEL_FILE_DSC]:
>              try:
>                  RetVal.append(self.BuildObject[PathClass(PlatformFile),
> TAB_COMMON])
>              except:
>                  pass
>          return RetVal
> diff --git a/BaseTools/Source/Python/build/build.py
> b/BaseTools/Source/Python/build/build.py
> index d74082fc26..317d6a2728 100644
> --- a/BaseTools/Source/Python/build/build.py
> +++ b/BaseTools/Source/Python/build/build.py
> @@ -41,11 +41,11 @@ from Common.TargetTxtClassObject import *
>  from Common.ToolDefClassObject import *
>  from Common.DataType import *
>  from Common.BuildVersion import gBUILD_VERSION
>  from AutoGen.AutoGen import *
>  from Common.BuildToolError import *
> -from Workspace.WorkspaceDatabase import *
> +from Workspace.WorkspaceDatabase import WorkspaceDatabase
>  from Common.MultipleWorkspace import MultipleWorkspace as mws
> 
>  from BuildReport import BuildReport
>  from GenPatchPcdTable.GenPatchPcdTable import *
>  from PatchPcdValue.PatchPcdValue import *
> @@ -825,14 +825,11 @@ class Build():
>                  # Get standard WORKSPACE/Conf use the absolute path to the
> WORKSPACE/Conf
>                  ConfDirectoryPath = mws.join(self.WorkspaceDir, 'Conf')
>          GlobalData.gConfDirectory = ConfDirectoryPath
>          GlobalData.gDatabasePath =
> os.path.normpath(os.path.join(ConfDirectoryPath,
> GlobalData.gDatabasePath))
> 
> -        if BuildOptions.DisableCache:
> -            self.Db         = WorkspaceDatabase(":memory:")
> -        else:
> -            self.Db = WorkspaceDatabase(GlobalData.gDatabasePath,
> self.Reparse)
> +        self.Db = WorkspaceDatabase()
>          self.BuildDatabase = self.Db.BuildObject
>          self.Platform = None
>          self.ToolChainFamily = None
>          self.LoadFixAddress = 0
>          self.UniFlag        = BuildOptions.Flag
> @@ -981,13 +978,10 @@ class Build():
>          # Allow case-insensitive for those from command line or configuration
> file
>          ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
>          if ErrorCode != 0:
>              EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
> 
> -        # create metafile database
> -        if not self.Db_Flag:
> -            self.Db.InitDatabase()
> 
>      def InitPreBuild(self):
>          self.LoadConfiguration()
>          ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
>          if ErrorCode != 0:
> @@ -1002,11 +996,10 @@ class Build():
>          if self.ToolChainFamily:
>              GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[0]
>          if 'PREBUILD' in GlobalData.gCommandLineDefines:
>              self.Prebuild   = GlobalData.gCommandLineDefines.get('PREBUILD')
>          else:
> -            self.Db.InitDatabase()
>              self.Db_Flag = True
>              Platform = self.Db.MapPlatform(str(self.PlatformFile))
>              self.Prebuild = str(Platform.Prebuild)
>          if self.Prebuild:
>              PrebuildList = []
> @@ -2081,17 +2074,11 @@ class Build():
>                      if BuildTask.HasError():
>                          EdkLogger.error("build", BUILD_ERROR, "Failed to build
> module", ExtraData=GlobalData.gBuildingModule)
>                      self.MakeTime += int(round((time.time() - MakeStart)))
> 
>                  MakeContiue = time.time()
> -                #
> -                # Save temp tables to a TmpTableDict.
> -                #
> -                for Key in Wa.BuildDatabase._CACHE_:
> -                    if Wa.BuildDatabase._CACHE_[Key]._RawData and
> Wa.BuildDatabase._CACHE_[Key]._RawData._Table and
> Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table:
> -                        if
> TemporaryTablePattern.match(Wa.BuildDatabase._CACHE_[Key]._RawData.
> _Table.Table):
> -
> TmpTableDict[Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table] =
> Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Cur
> +
>                  #
>                  #
>                  # All modules have been put in build tasks queue. Tell task scheduler
>                  # to exit if all tasks are completed
>                  #
> @@ -2229,11 +2216,10 @@ class Build():
>          else:
>              self.SpawnMode = False
>              self._BuildModule()
> 
>          if self.Target == 'cleanall':
> -            self.Db.Close()
>              RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True)
> 
>      def CreateAsBuiltInf(self):
>          for Module in self.BuildModules:
>              Module.CreateAsBuiltInf()
> @@ -2490,14 +2476,11 @@ def Main():
> 
>          MyBuild = Build(Target, Workspace, Option)
>          GlobalData.gCommandLineDefines['ARCH'] = ' '.join(MyBuild.ArchList)
>          if not (MyBuild.LaunchPrebuildFlag and
> os.path.exists(MyBuild.PlatformBuildPath)):
>              MyBuild.Launch()
> -        # Drop temp tables to avoid database locked.
> -        for TmpTableName in TmpTableDict:
> -            SqlCommand = """drop table IF EXISTS %s""" % TmpTableName
> -            TmpTableDict[TmpTableName].execute(SqlCommand)
> +
>          #MyBuild.DumpBuildData()
>          #
>          # All job done, no error found and no exception raised
>          #
>          BuildError = False
> @@ -2565,11 +2548,11 @@ def Main():
>      else:
>          BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration)
>      if MyBuild is not None:
>          if not BuildError:
>              MyBuild.BuildReport.GenerateReport(BuildDurationStr,
> LogBuildTime(MyBuild.AutoGenTime), LogBuildTime(MyBuild.MakeTime),
> LogBuildTime(MyBuild.GenFdsTime))
> -        MyBuild.Db.Close()
> +
>      EdkLogger.SetLevel(EdkLogger.QUIET)
>      EdkLogger.quiet("\n- %s -" % Conclusion)
>      EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d %Y",
> time.localtime()))
>      EdkLogger.quiet("Build total time: %s\n" % BuildDurationStr)
>      return ReturnCode
> --
> 2.19.1.windows.1