[Patch v1 0/4] Cleanup the Common.misc file

[Patch v1 0/4] Cleanup the Common.misc file

[Jaben Carsey]

move functions and remove unused content from common.misc

Cc: Bob Feng <Bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Jaben Carsey <jaben.carsey@intel.com>

Jaben Carsey (4):
  BaseTools/Common/Misc: move private functions
  BaseTools/Common/Misc: remove uncalled code
  BaseTools/Common/Misc: Cleanup the imports
  BaseTools/build/build: delete variable

 BaseTools/Source/Python/Common/Misc.py | 175 +++++++-------------
 BaseTools/Source/Python/build/build.py |  60 +------
 2 files changed, 64 insertions(+), 171 deletions(-)

-- 
2.16.2.windows.1

[Patch v1 1/4] BaseTools/Common/Misc: move private functions

[Jaben Carsey]

These 2 functions are only used internally.  move them to the smallest
scope and use them.

Cc: Bob Feng <Bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Jaben Carsey <jaben.carsey@intel.com>
---
 BaseTools/Source/Python/Common/Misc.py | 92 ++++++++++----------
 1 file changed, 46 insertions(+), 46 deletions(-)

diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py
index 8e4217a4f6e5..c547c2f8e43a 100644
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -566,32 +566,6 @@ def RealPath(File, Dir='', OverrideDir=''):
         NewFile = GlobalData.gAllFiles[NewFile]
     return NewFile
 
-def RealPath2(File, Dir='', OverrideDir=''):
-    NewFile = None
-    if OverrideDir:
-        NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
-        if NewFile:
-            if OverrideDir[-1] == os.path.sep:
-                return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
-            else:
-                return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
-    if GlobalData.gAllFiles:
-        NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
-    if not NewFile:
-        NewFile = os.path.normpath(os.path.join(Dir, File))
-        if not os.path.exists(NewFile):
-            return None, None
-    if NewFile:
-        if Dir:
-            if Dir[-1] == os.path.sep:
-                return NewFile[len(Dir):], NewFile[0:len(Dir)]
-            else:
-                return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
-        else:
-            return NewFile, ''
-
-    return None, None
-
 ## Get GUID value from given packages
 #
 #   @param      CName           The CName of the GUID
@@ -1189,27 +1163,27 @@ def AnalyzePcdExpression(Setting):
             FieldList[i] = ch.replace(RanStr,'\\\\')
     return FieldList
 
-def ParseDevPathValue (Value):
-    if '\\' in Value:
-        Value.replace('\\', '/').replace(' ', '')
-
-    Cmd = 'DevicePath ' + '"' + Value + '"'
-    try:
-        p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
-        out, err = p.communicate()
-    except Exception as X:
-        raise BadExpression("DevicePath: %s" % (str(X)) )
-    finally:
-        subprocess._cleanup()
-        p.stdout.close()
-        p.stderr.close()
-    if err:
-        raise BadExpression("DevicePath: %s" % str(err))
-    Size = len(out.split())
-    out = ','.join(out.split())
-    return '{' + out + '}', Size
-
 def ParseFieldValue (Value):
+    def ParseDevPathValue (Value):
+        if '\\' in Value:
+            Value.replace('\\', '/').replace(' ', '')
+
+        Cmd = 'DevicePath ' + '"' + Value + '"'
+        try:
+            p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+            out, err = p.communicate()
+        except Exception as X:
+            raise BadExpression("DevicePath: %s" % (str(X)) )
+        finally:
+            subprocess._cleanup()
+            p.stdout.close()
+            p.stderr.close()
+        if err:
+            raise BadExpression("DevicePath: %s" % str(err))
+        Size = len(out.split())
+        out = ','.join(out.split())
+        return '{' + out + '}', Size
+
     if "{CODE(" in Value:
         return Value, len(Value.split(","))
     if isinstance(Value, type(0)):
@@ -1625,6 +1599,32 @@ class PathClass(object):
         return os.stat(self.Path)[8]
 
     def Validate(self, Type='', CaseSensitive=True):
+        def RealPath2(File, Dir='', OverrideDir=''):
+            NewFile = None
+            if OverrideDir:
+                NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
+                if NewFile:
+                    if OverrideDir[-1] == os.path.sep:
+                        return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
+                    else:
+                        return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
+            if GlobalData.gAllFiles:
+                NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
+            if not NewFile:
+                NewFile = os.path.normpath(os.path.join(Dir, File))
+                if not os.path.exists(NewFile):
+                    return None, None
+            if NewFile:
+                if Dir:
+                    if Dir[-1] == os.path.sep:
+                        return NewFile[len(Dir):], NewFile[0:len(Dir)]
+                    else:
+                        return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
+                else:
+                    return NewFile, ''
+
+            return None, None
+
         if GlobalData.gCaseInsensitive:
             CaseSensitive = False
         if Type and Type.lower() != self.Type:
-- 
2.16.2.windows.1

[Patch v1 2/4] BaseTools/Common/Misc: remove uncalled code

[Jaben Carsey]

no use for this code content.

Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Jaben Carsey <jaben.carsey@intel.com>
---
 BaseTools/Source/Python/Common/Misc.py | 46 --------------------
 1 file changed, 46 deletions(-)

diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py
index c547c2f8e43a..65ccba36b2e9 100644
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -947,44 +947,6 @@ class sdict(IterableUserDict):
             for k, v in kwargs.items():
                 self[k] = v
 
-## Dictionary with restricted keys
-#
-class rdict(dict):
-    ## Constructor
-    def __init__(self, KeyList):
-        for Key in KeyList:
-            dict.__setitem__(self, Key, "")
-
-    ## []= operator
-    def __setitem__(self, key, value):
-        if key not in self:
-            EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
-                            ExtraData=", ".join(dict.keys(self)))
-        dict.__setitem__(self, key, value)
-
-    ## =[] operator
-    def __getitem__(self, key):
-        if key not in self:
-            return ""
-        return dict.__getitem__(self, key)
-
-    ## del operator
-    def __delitem__(self, key):
-        EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
-
-    ## Empty the dict
-    def clear(self):
-        for Key in self:
-            self.__setitem__(Key, "")
-
-    ## Return value related to a key, and remove the (key, value) from the dict
-    def pop(self, key, *dv):
-        EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
-
-    ## Return (key, value) pair, and remove the (key, value) from the dict
-    def popitem(self):
-        EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
-
 ## Dictionary using prioritized list as key
 #
 class tdict:
@@ -1987,11 +1949,3 @@ def CopyDict(ori_dict):
 #
 def RemoveCComments(ctext):
     return re.sub('//.*?\n|/\*.*?\*/', '\n', ctext, flags=re.S)
-##
-#
-# This acts like the main() function for the script, unless it is 'import'ed into another
-# script.
-#
-if __name__ == '__main__':
-    pass
-
-- 
2.16.2.windows.1

[Patch v1 3/4] BaseTools/Common/Misc: Cleanup the imports

[Jaben Carsey]

Refactor to 'dict' from 'IterableUserDict' which was only required for old
    python interpreter.
Sort imports according to PEP8
Remove those we dont need.

Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Jaben Carsey <jaben.carsey@intel.com>
---
 BaseTools/Source/Python/Common/Misc.py | 34 ++++++++++----------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py
index 65ccba36b2e9..e8be8f866511 100644
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -15,7 +15,7 @@
 # Import Modules
 #
 from __future__ import absolute_import
-import Common.LongFilePathOs as os
+
 import sys
 import string
 import threading
@@ -26,22 +26,22 @@ import array
 import shutil
 from random import sample
 from struct import pack
-from UserDict import IterableUserDict
-from UserList import UserList
+import uuid
+import subprocess
+from collections import OrderedDict
 
+import Common.LongFilePathOs as os
 from Common import EdkLogger as EdkLogger
 from Common import GlobalData as GlobalData
-from .DataType import *
-from .BuildToolError import *
+from Common.DataType import *
+from Common.BuildToolError import *
 from CommonDataClass.DataClass import *
-from .Parsing import GetSplitValueList
+from Common.Parsing import GetSplitValueList
 from Common.LongFilePathSupport import OpenLongFilePath as open
 from Common.MultipleWorkspace import MultipleWorkspace as mws
-import uuid
 from CommonDataClass.Exceptions import BadExpression
 from Common.caching import cached_property
-import subprocess
-from collections import OrderedDict
+
 ## Regular expression used to find out place holders in string template
 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
 
@@ -831,22 +831,22 @@ class Progressor:
 #  accessed in the order they are added into the dict. It guarantees the order
 #  by making use of an internal list to keep a copy of keys.
 #
-class sdict(IterableUserDict):
+class sdict(dict):
     ## Constructor
     def __init__(self):
-        IterableUserDict.__init__(self)
+        dict.__init__(self)
         self._key_list = []
 
     ## [] operator
     def __setitem__(self, key, value):
         if key not in self._key_list:
             self._key_list.append(key)
-        IterableUserDict.__setitem__(self, key, value)
+        dict.__setitem__(self, key, value)
 
     ## del operator
     def __delitem__(self, key):
         self._key_list.remove(key)
-        IterableUserDict.__delitem__(self, key)
+        dict.__delitem__(self, key)
 
     ## used in "for k in dict" loop to ensure the correct order
     def __iter__(self):
@@ -869,17 +869,17 @@ class sdict(IterableUserDict):
         index = self._key_list.index(key)
         if order == 'BEFORE':
             self._key_list.insert(index, newkey)
-            IterableUserDict.__setitem__(self, newkey, newvalue)
+            dict.__setitem__(self, newkey, newvalue)
         elif order == 'AFTER':
             self._key_list.insert(index + 1, newkey)
-            IterableUserDict.__setitem__(self, newkey, newvalue)
+            dict.__setitem__(self, newkey, newvalue)
 
     ## append support
     def append(self, sdict):
         for key in sdict:
             if key not in self._key_list:
                 self._key_list.append(key)
-            IterableUserDict.__setitem__(self, key, sdict[key])
+            dict.__setitem__(self, key, sdict[key])
 
     def has_key(self, key):
         return key in self._key_list
@@ -887,7 +887,7 @@ class sdict(IterableUserDict):
     ## Empty the dict
     def clear(self):
         self._key_list = []
-        IterableUserDict.clear(self)
+        dict.clear(self)
 
     ## Return a copy of keys
     def keys(self):
-- 
2.16.2.windows.1

[Patch v1 4/4] BaseTools/build/build: delete variable

[Jaben Carsey]

delete the shared global variable from Common.Misc
delete the uncalled users of the variable from build.build

Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Jaben Carsey <jaben.carsey@intel.com>
---
 BaseTools/Source/Python/Common/Misc.py |  3 -
 BaseTools/Source/Python/build/build.py | 60 +-------------------
 2 files changed, 1 insertion(+), 62 deletions(-)

diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py
index e8be8f866511..0b4dd7a7c162 100644
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -54,9 +54,6 @@ secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.
 
 StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$')
 
-## Dictionary used to store file time stamp for quick re-access
-gFileTimeStampCache = {}    # {file path : file time stamp}
-
 ## Dictionary used to store dependencies of files
 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
 
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 44ad86b780da..e79949fa28f9 100644
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -2,7 +2,7 @@
 # build a platform or a module
 #
 #  Copyright (c) 2014, Hewlett-Packard Development Company, L.P.<BR>
-#  Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
+#  Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.<BR>
 #  Copyright (c) 2018, Hewlett Packard Enterprise Development, L.P.<BR>
 #
 #  This program and the accompanying materials
@@ -72,43 +72,6 @@ gToolsDefinition = "tools_def.txt"
 TemporaryTablePattern = re.compile(r'^_\d+_\d+_[a-fA-F0-9]+$')
 TmpTableDict = {}
 
-## Make a Python object persistent on file system
-#
-#   @param      Data    The object to be stored in file
-#   @param      File    The path of file to store the object
-#
-def _DataDump(Data, File):
-    Fd = None
-    try:
-        Fd = open(File, 'wb')
-        pickle.dump(Data, Fd, pickle.HIGHEST_PROTOCOL)
-    except:
-        EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
-    finally:
-        if Fd is not None:
-            Fd.close()
-
-## Restore a Python object from a file
-#
-#   @param      File    The path of file stored the object
-#
-#   @retval     object  A python object
-#   @retval     None    If failure in file operation
-#
-def _DataRestore(File):
-    Data = None
-    Fd = None
-    try:
-        Fd = open(File, 'rb')
-        Data = pickle.load(Fd)
-    except Exception as e:
-        EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
-        Data = None
-    finally:
-        if Fd is not None:
-            Fd.close()
-    return Data
-
 ## Check environment PATH variable to make sure the specified tool is found
 #
 #   If the tool is found in the PATH, then True is returned
@@ -2192,31 +2155,11 @@ class Build():
     def Relinquish(self):
         OldLogLevel = EdkLogger.GetLevel()
         EdkLogger.SetLevel(EdkLogger.ERROR)
-        #self.DumpBuildData()
         Utils.Progressor.Abort()
         if self.SpawnMode == True:
             BuildTask.Abort()
         EdkLogger.SetLevel(OldLogLevel)
 
-    def DumpBuildData(self):
-        CacheDirectory = os.path.dirname(GlobalData.gDatabasePath)
-        Utils.CreateDirectory(CacheDirectory)
-        Utils._DataDump(Utils.gFileTimeStampCache, os.path.join(CacheDirectory, "gFileTimeStampCache"))
-        Utils._DataDump(Utils.gDependencyDatabase, os.path.join(CacheDirectory, "gDependencyDatabase"))
-
-    def RestoreBuildData(self):
-        FilePath = os.path.join(os.path.dirname(GlobalData.gDatabasePath), "gFileTimeStampCache")
-        if Utils.gFileTimeStampCache == {} and os.path.isfile(FilePath):
-            Utils.gFileTimeStampCache = Utils._DataRestore(FilePath)
-            if Utils.gFileTimeStampCache is None:
-                Utils.gFileTimeStampCache = {}
-
-        FilePath = os.path.join(os.path.dirname(GlobalData.gDatabasePath), "gDependencyDatabase")
-        if Utils.gDependencyDatabase == {} and os.path.isfile(FilePath):
-            Utils.gDependencyDatabase = Utils._DataRestore(FilePath)
-            if Utils.gDependencyDatabase is None:
-                Utils.gDependencyDatabase = {}
-
 def ParseDefines(DefineList=[]):
     DefineDict = {}
     if DefineList is not None:
@@ -2440,7 +2383,6 @@ def Main():
         if not (MyBuild.LaunchPrebuildFlag and os.path.exists(MyBuild.PlatformBuildPath)):
             MyBuild.Launch()
 
-        #MyBuild.DumpBuildData()
         #
         # All job done, no error found and no exception raised
         #
-- 
2.16.2.windows.1

Re: [Patch v1 2/4] BaseTools/Common/Misc: remove uncalled code

[Feng, Bob C]

Reviewed-by: Bob Feng <bob.c.feng@intel.com>

-----Original Message-----
From: Carsey, Jaben 
Sent: Friday, January 25, 2019 12:14 AM
To: edk2-devel@lists.01.org
Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming <liming.gao@intel.com>
Subject: [Patch v1 2/4] BaseTools/Common/Misc: remove uncalled code

no use for this code content.

Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Jaben Carsey <jaben.carsey@intel.com>
---
 BaseTools/Source/Python/Common/Misc.py | 46 --------------------
 1 file changed, 46 deletions(-)

diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py
index c547c2f8e43a..65ccba36b2e9 100644
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -947,44 +947,6 @@ class sdict(IterableUserDict):
             for k, v in kwargs.items():
                 self[k] = v
 
-## Dictionary with restricted keys
-#
-class rdict(dict):
-    ## Constructor
-    def __init__(self, KeyList):
-        for Key in KeyList:
-            dict.__setitem__(self, Key, "")
-
-    ## []= operator
-    def __setitem__(self, key, value):
-        if key not in self:
-            EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
-                            ExtraData=", ".join(dict.keys(self)))
-        dict.__setitem__(self, key, value)
-
-    ## =[] operator
-    def __getitem__(self, key):
-        if key not in self:
-            return ""
-        return dict.__getitem__(self, key)
-
-    ## del operator
-    def __delitem__(self, key):
-        EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
-
-    ## Empty the dict
-    def clear(self):
-        for Key in self:
-            self.__setitem__(Key, "")
-
-    ## Return value related to a key, and remove the (key, value) from the dict
-    def pop(self, key, *dv):
-        EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
-
-    ## Return (key, value) pair, and remove the (key, value) from the dict
-    def popitem(self):
-        EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
-
 ## Dictionary using prioritized list as key  #  class tdict:
@@ -1987,11 +1949,3 @@ def CopyDict(ori_dict):
 #
 def RemoveCComments(ctext):
     return re.sub('//.*?\n|/\*.*?\*/', '\n', ctext, flags=re.S) -## -# -# This acts like the main() function for the script, unless it is 'import'ed into another -# script.
-#
-if __name__ == '__main__':
-    pass
-
--
2.16.2.windows.1

Re: [Patch v1 3/4] BaseTools/Common/Misc: Cleanup the imports

[Feng, Bob C]

Reviewed-by: Bob Feng <bob.c.feng@intel.com>

-----Original Message-----
From: Carsey, Jaben 
Sent: Friday, January 25, 2019 12:14 AM
To: edk2-devel@lists.01.org
Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming <liming.gao@intel.com>
Subject: [Patch v1 3/4] BaseTools/Common/Misc: Cleanup the imports

Refactor to 'dict' from 'IterableUserDict' which was only required for old
    python interpreter.
Sort imports according to PEP8
Remove those we dont need.

Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Jaben Carsey <jaben.carsey@intel.com>
---
 BaseTools/Source/Python/Common/Misc.py | 34 ++++++++++----------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py
index 65ccba36b2e9..e8be8f866511 100644
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -15,7 +15,7 @@
 # Import Modules
 #
 from __future__ import absolute_import
-import Common.LongFilePathOs as os
+
 import sys
 import string
 import threading
@@ -26,22 +26,22 @@ import array
 import shutil
 from random import sample
 from struct import pack
-from UserDict import IterableUserDict
-from UserList import UserList
+import uuid
+import subprocess
+from collections import OrderedDict
 
+import Common.LongFilePathOs as os
 from Common import EdkLogger as EdkLogger  from Common import GlobalData as GlobalData -from .DataType import * -from .BuildToolError import *
+from Common.DataType import *
+from Common.BuildToolError import *
 from CommonDataClass.DataClass import * -from .Parsing import GetSplitValueList
+from Common.Parsing import GetSplitValueList
 from Common.LongFilePathSupport import OpenLongFilePath as open  from Common.MultipleWorkspace import MultipleWorkspace as mws -import uuid  from CommonDataClass.Exceptions import BadExpression  from Common.caching import cached_property -import subprocess -from collections import OrderedDict
+
 ## Regular expression used to find out place holders in string template  gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
 
@@ -831,22 +831,22 @@ class Progressor:
 #  accessed in the order they are added into the dict. It guarantees the order  #  by making use of an internal list to keep a copy of keys.
 #
-class sdict(IterableUserDict):
+class sdict(dict):
     ## Constructor
     def __init__(self):
-        IterableUserDict.__init__(self)
+        dict.__init__(self)
         self._key_list = []
 
     ## [] operator
     def __setitem__(self, key, value):
         if key not in self._key_list:
             self._key_list.append(key)
-        IterableUserDict.__setitem__(self, key, value)
+        dict.__setitem__(self, key, value)
 
     ## del operator
     def __delitem__(self, key):
         self._key_list.remove(key)
-        IterableUserDict.__delitem__(self, key)
+        dict.__delitem__(self, key)
 
     ## used in "for k in dict" loop to ensure the correct order
     def __iter__(self):
@@ -869,17 +869,17 @@ class sdict(IterableUserDict):
         index = self._key_list.index(key)
         if order == 'BEFORE':
             self._key_list.insert(index, newkey)
-            IterableUserDict.__setitem__(self, newkey, newvalue)
+            dict.__setitem__(self, newkey, newvalue)
         elif order == 'AFTER':
             self._key_list.insert(index + 1, newkey)
-            IterableUserDict.__setitem__(self, newkey, newvalue)
+            dict.__setitem__(self, newkey, newvalue)
 
     ## append support
     def append(self, sdict):
         for key in sdict:
             if key not in self._key_list:
                 self._key_list.append(key)
-            IterableUserDict.__setitem__(self, key, sdict[key])
+            dict.__setitem__(self, key, sdict[key])
 
     def has_key(self, key):
         return key in self._key_list
@@ -887,7 +887,7 @@ class sdict(IterableUserDict):
     ## Empty the dict
     def clear(self):
         self._key_list = []
-        IterableUserDict.clear(self)
+        dict.clear(self)
 
     ## Return a copy of keys
     def keys(self):
--
2.16.2.windows.1

Re: [Patch v1 4/4] BaseTools/build/build: delete variable

[Feng, Bob C]

Reviewed-by: Bob Feng <bob.c.feng@intel.com>

-----Original Message-----
From: Carsey, Jaben 
Sent: Friday, January 25, 2019 12:14 AM
To: edk2-devel@lists.01.org
Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming <liming.gao@intel.com>
Subject: [Patch v1 4/4] BaseTools/build/build: delete variable

delete the shared global variable from Common.Misc delete the uncalled users of the variable from build.build

Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Jaben Carsey <jaben.carsey@intel.com>
---
 BaseTools/Source/Python/Common/Misc.py |  3 -  BaseTools/Source/Python/build/build.py | 60 +-------------------
 2 files changed, 1 insertion(+), 62 deletions(-)

diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py
index e8be8f866511..0b4dd7a7c162 100644
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -54,9 +54,6 @@ secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.
 
 StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$')
 
-## Dictionary used to store file time stamp for quick re-access
-gFileTimeStampCache = {}    # {file path : file time stamp}
-
 ## Dictionary used to store dependencies of files
 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
 
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 44ad86b780da..e79949fa28f9 100644
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -2,7 +2,7 @@
 # build a platform or a module
 #
 #  Copyright (c) 2014, Hewlett-Packard Development Company, L.P.<BR> -#  Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
+#  Copyright (c) 2007 - 2019, Intel Corporation. All rights 
+reserved.<BR>
 #  Copyright (c) 2018, Hewlett Packard Enterprise Development, L.P.<BR>  #  #  This program and the accompanying materials @@ -72,43 +72,6 @@ gToolsDefinition = "tools_def.txt"
 TemporaryTablePattern = re.compile(r'^_\d+_\d+_[a-fA-F0-9]+$')
 TmpTableDict = {}
 
-## Make a Python object persistent on file system -#
-#   @param      Data    The object to be stored in file
-#   @param      File    The path of file to store the object
-#
-def _DataDump(Data, File):
-    Fd = None
-    try:
-        Fd = open(File, 'wb')
-        pickle.dump(Data, Fd, pickle.HIGHEST_PROTOCOL)
-    except:
-        EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
-    finally:
-        if Fd is not None:
-            Fd.close()
-
-## Restore a Python object from a file
-#
-#   @param      File    The path of file stored the object
-#
-#   @retval     object  A python object
-#   @retval     None    If failure in file operation
-#
-def _DataRestore(File):
-    Data = None
-    Fd = None
-    try:
-        Fd = open(File, 'rb')
-        Data = pickle.load(Fd)
-    except Exception as e:
-        EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
-        Data = None
-    finally:
-        if Fd is not None:
-            Fd.close()
-    return Data
-
 ## Check environment PATH variable to make sure the specified tool is found  #
 #   If the tool is found in the PATH, then True is returned
@@ -2192,31 +2155,11 @@ class Build():
     def Relinquish(self):
         OldLogLevel = EdkLogger.GetLevel()
         EdkLogger.SetLevel(EdkLogger.ERROR)
-        #self.DumpBuildData()
         Utils.Progressor.Abort()
         if self.SpawnMode == True:
             BuildTask.Abort()
         EdkLogger.SetLevel(OldLogLevel)
 
-    def DumpBuildData(self):
-        CacheDirectory = os.path.dirname(GlobalData.gDatabasePath)
-        Utils.CreateDirectory(CacheDirectory)
-        Utils._DataDump(Utils.gFileTimeStampCache, os.path.join(CacheDirectory, "gFileTimeStampCache"))
-        Utils._DataDump(Utils.gDependencyDatabase, os.path.join(CacheDirectory, "gDependencyDatabase"))
-
-    def RestoreBuildData(self):
-        FilePath = os.path.join(os.path.dirname(GlobalData.gDatabasePath), "gFileTimeStampCache")
-        if Utils.gFileTimeStampCache == {} and os.path.isfile(FilePath):
-            Utils.gFileTimeStampCache = Utils._DataRestore(FilePath)
-            if Utils.gFileTimeStampCache is None:
-                Utils.gFileTimeStampCache = {}
-
-        FilePath = os.path.join(os.path.dirname(GlobalData.gDatabasePath), "gDependencyDatabase")
-        if Utils.gDependencyDatabase == {} and os.path.isfile(FilePath):
-            Utils.gDependencyDatabase = Utils._DataRestore(FilePath)
-            if Utils.gDependencyDatabase is None:
-                Utils.gDependencyDatabase = {}
-
 def ParseDefines(DefineList=[]):
     DefineDict = {}
     if DefineList is not None:
@@ -2440,7 +2383,6 @@ def Main():
         if not (MyBuild.LaunchPrebuildFlag and os.path.exists(MyBuild.PlatformBuildPath)):
             MyBuild.Launch()
 
-        #MyBuild.DumpBuildData()
         #
         # All job done, no error found and no exception raised
         #
--
2.16.2.windows.1

Re: [Patch v1 1/4] BaseTools/Common/Misc: move private functions

[Feng, Bob C]

Reviewed-by: Bob Feng <bob.c.feng@intel.com>

-----Original Message-----
From: Carsey, Jaben 
Sent: Friday, January 25, 2019 12:14 AM
To: edk2-devel@lists.01.org
Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming <liming.gao@intel.com>
Subject: [Patch v1 1/4] BaseTools/Common/Misc: move private functions

These 2 functions are only used internally.  move them to the smallest scope and use them.

Cc: Bob Feng <Bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Jaben Carsey <jaben.carsey@intel.com>
---
 BaseTools/Source/Python/Common/Misc.py | 92 ++++++++++----------
 1 file changed, 46 insertions(+), 46 deletions(-)

diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py
index 8e4217a4f6e5..c547c2f8e43a 100644
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -566,32 +566,6 @@ def RealPath(File, Dir='', OverrideDir=''):
         NewFile = GlobalData.gAllFiles[NewFile]
     return NewFile
 
-def RealPath2(File, Dir='', OverrideDir=''):
-    NewFile = None
-    if OverrideDir:
-        NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
-        if NewFile:
-            if OverrideDir[-1] == os.path.sep:
-                return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
-            else:
-                return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
-    if GlobalData.gAllFiles:
-        NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
-    if not NewFile:
-        NewFile = os.path.normpath(os.path.join(Dir, File))
-        if not os.path.exists(NewFile):
-            return None, None
-    if NewFile:
-        if Dir:
-            if Dir[-1] == os.path.sep:
-                return NewFile[len(Dir):], NewFile[0:len(Dir)]
-            else:
-                return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
-        else:
-            return NewFile, ''
-
-    return None, None
-
 ## Get GUID value from given packages
 #
 #   @param      CName           The CName of the GUID
@@ -1189,27 +1163,27 @@ def AnalyzePcdExpression(Setting):
             FieldList[i] = ch.replace(RanStr,'\\\\')
     return FieldList
 
-def ParseDevPathValue (Value):
-    if '\\' in Value:
-        Value.replace('\\', '/').replace(' ', '')
-
-    Cmd = 'DevicePath ' + '"' + Value + '"'
-    try:
-        p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
-        out, err = p.communicate()
-    except Exception as X:
-        raise BadExpression("DevicePath: %s" % (str(X)) )
-    finally:
-        subprocess._cleanup()
-        p.stdout.close()
-        p.stderr.close()
-    if err:
-        raise BadExpression("DevicePath: %s" % str(err))
-    Size = len(out.split())
-    out = ','.join(out.split())
-    return '{' + out + '}', Size
-
 def ParseFieldValue (Value):
+    def ParseDevPathValue (Value):
+        if '\\' in Value:
+            Value.replace('\\', '/').replace(' ', '')
+
+        Cmd = 'DevicePath ' + '"' + Value + '"'
+        try:
+            p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+            out, err = p.communicate()
+        except Exception as X:
+            raise BadExpression("DevicePath: %s" % (str(X)) )
+        finally:
+            subprocess._cleanup()
+            p.stdout.close()
+            p.stderr.close()
+        if err:
+            raise BadExpression("DevicePath: %s" % str(err))
+        Size = len(out.split())
+        out = ','.join(out.split())
+        return '{' + out + '}', Size
+
     if "{CODE(" in Value:
         return Value, len(Value.split(","))
     if isinstance(Value, type(0)):
@@ -1625,6 +1599,32 @@ class PathClass(object):
         return os.stat(self.Path)[8]
 
     def Validate(self, Type='', CaseSensitive=True):
+        def RealPath2(File, Dir='', OverrideDir=''):
+            NewFile = None
+            if OverrideDir:
+                NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
+                if NewFile:
+                    if OverrideDir[-1] == os.path.sep:
+                        return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
+                    else:
+                        return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
+            if GlobalData.gAllFiles:
+                NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
+            if not NewFile:
+                NewFile = os.path.normpath(os.path.join(Dir, File))
+                if not os.path.exists(NewFile):
+                    return None, None
+            if NewFile:
+                if Dir:
+                    if Dir[-1] == os.path.sep:
+                        return NewFile[len(Dir):], NewFile[0:len(Dir)]
+                    else:
+                        return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
+                else:
+                    return NewFile, ''
+
+            return None, None
+
         if GlobalData.gCaseInsensitive:
             CaseSensitive = False
         if Type and Type.lower() != self.Type:
--
2.16.2.windows.1