* [PATCH] BaseTools:Add DetectNotUsedItem.py to Edk2\BaseTools\Scripts
@ 2019-05-28 9:32 Fan, ZhijuX
2019-06-11 2:08 ` Bob Feng
0 siblings, 1 reply; 2+ messages in thread
From: Fan, ZhijuX @ 2019-05-28 9:32 UTC (permalink / raw)
To: devel@edk2.groups.io; +Cc: Gao, Liming, Feng, Bob C
[-- Attachment #1: Type: text/plain, Size: 7714 bytes --]
BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=1850
This script is used to Detect unreferenced PCD and GUID/Protocols/PPIs.
The input parameters are Dec file and package directory.
This script can be run in both Py2 and Py3.
Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com>
---
BaseTools/Scripts/DetectNotUsedItem.py | 185 +++++++++++++++++++++++++++++++++
1 file changed, 185 insertions(+)
create mode 100644 BaseTools/Scripts/DetectNotUsedItem.py
diff --git a/BaseTools/Scripts/DetectNotUsedItem.py b/BaseTools/Scripts/DetectNotUsedItem.py
new file mode 100644
index 0000000000..655fb65a96
--- /dev/null
+++ b/BaseTools/Scripts/DetectNotUsedItem.py
@@ -0,0 +1,185 @@
+## @file
+# Detect unreferenced PCD and GUID/Protocols/PPIs.
+#
+# Copyright (c) 2019, Intel Corporation. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+
+'''
+DetectNotUsedItem
+'''
+import re
+import os
+import sys
+import argparse
+
+#
+# Globals for help information
+#
+__prog__ = 'DetectNotUsedItem'
+__version__ = '%s Version %s' % (__prog__, '0.1')
+__copyright__ = 'Copyright (c) 2019, Intel Corporation. All rights reserved.'
+__description__ = "Detect unreferenced PCD and GUID/Protocols/PPIs.\n"
+
+SectionList = ["LibraryClasses", "Guids", "Ppis", "Protocols", "Pcd"]
+
+class PROCESS(object):
+
+ def __init__(self, DecPath, InfDirs):
+ self.Dec = DecPath
+ self.InfPath = InfDirs
+ self.Log = []
+
+ def ParserDscFdfInfFile(self):
+ AllContentList = []
+ for File in (self.SearchbyExt(".dsc") + self.SearchbyExt(".fdf") + self.SearchbyExt(".inf")):
+ AllContentList += self.ParseDscFdfInfContent(File)
+ return AllContentList
+
+ #Search File by extension name
+ def SearchbyExt(self, Ext):
+ FileList = []
+ for path in self.InfPath:
+ for root, _, files in os.walk(path, topdown=True, followlinks=False):
+ for filename in files:
+ if filename.endswith(Ext):
+ FileList.append(os.path.join(root, filename))
+ return FileList
+
+ # Parse DEC file to get Line number and Name
+ # return section name, the Item Name and comments line number
+ def ParseDecContent(self):
+ SectionRE = re.compile(r'\[(.*)\]')
+ Flag = False
+ Comments ={}
+ Comment_Line = []
+ ItemName = {}
+ with open(self.Dec, 'r') as F:
+ for Index, content in enumerate(F):
+ NotComment = not content.strip().startswith("#")
+ Section = SectionRE.findall(content)
+ if Section and NotComment:
+ Flag = self.IsNeedParseSection(Section[0])
+ if Flag:
+ Comment_Line.append(Index)
+ if NotComment:
+ if content != "\n" and content != "\r\n":
+ ItemName[Index] = content.split('=')[0].split('|')[0].split('#')[0].strip()
+ Comments[Index] = Comment_Line
+ Comment_Line = []
+ return ItemName, Comments
+
+ def IsNeedParseSection(self, SectionName):
+ for item in SectionList:
+ if item in SectionName:
+ return True
+ return False
+
+ #Parse DSC, FDF, INF File, remove comments, return Lines list
+ def ParseDscFdfInfContent(self, File):
+ with open(File,'r') as F:
+ lines = F.readlines()
+ for Index in range(len(lines)-1, -1, -1):
+ if lines[Index].strip().startswith("#") or lines[Index] == "\n" or lines[Index] == "\r\n":
+ lines.remove(lines[Index])
+ elif "#" in lines[Index]:
+ lines[Index] = lines[Index].split("#")[0].strip()
+ else:
+ lines[Index] = lines[Index].strip()
+ return lines
+
+ def DetectNotUsedItem(self):
+ NotUsedItem = {}
+ DecItem, DecComments = self.ParseDecContent()
+ InfDscFdfContent = self.ParserDscFdfInfFile()
+ for LineNum in list(DecItem.keys()):
+ DecItemName = DecItem[LineNum]
+ Match_reg = re.compile("(?<![a-zA-Z0-9_-])%s(?![a-zA-Z0-9_-])" % DecItemName)
+ MatchFlag = False
+ for Line in InfDscFdfContent:
+ if Match_reg.search(Line):
+ MatchFlag = True
+ break
+ if not MatchFlag:
+ NotUsedItem[LineNum] = DecItemName
+ self.Display(NotUsedItem)
+ return NotUsedItem, DecComments
+
+ def Display(self,UnuseDict):
+ print("DEC File:\n%s\n%s%s" % (self.Dec, "{:<15}".format("Line Number"),"{:<0}".format("Unused Item")))
+ self.Log.append("DEC File:\n%s\n%s%s\n" % (self.Dec, "{:<15}".format("Line Number"),"{:<0}".format("Unused Item")))
+ for num in list(sorted(UnuseDict.keys())):
+ ItemName = UnuseDict[num]
+ print("%s%s%s" % (" "*3,"{:<12}".format(num + 1), "{:<1}".format(ItemName)))
+ self.Log.append(("%s%s%s\n" % (" "*3,"{:<12}".format(num + 1), "{:<1}".format(ItemName))))
+
+ def Clean(self, UnUseDict, Comments):
+ removednum = []
+ for num in list(UnUseDict.keys()):
+ if num in list(Comments.keys()):
+ removednum += Comments[num]
+ with open(self.Dec, 'r') as Dec:
+ lines = Dec.readlines()
+ try:
+ with open(self.Dec, 'w+') as T:
+ for linenum in range(len(lines)):
+ if linenum in removednum:
+ continue
+ else:
+ T.write(lines[linenum])
+ print("DEC File has been clean: %s"%(self.Dec))
+ except Exception as err:
+ print(err)
+
+class Main(object):
+
+ def mainprocess(self, Dec, Dirs, Isclean, LogPath):
+ for dir in Dirs:
+ if not os.path.exists(dir):
+ print("Error: Invalid path for '--dirs': %s" % dir)
+ sys.exit(1)
+ Pa = PROCESS(Dec, Dirs)
+ unuse, comment = Pa.DetectNotUsedItem()
+ if Isclean:
+ Pa.Clean(unuse, comment)
+ self.Logging(Pa.Log, LogPath)
+
+ def Logging(self, content, LogPath):
+ if LogPath:
+ try:
+ if os.path.isdir(LogPath):
+ FilePath = os.path.dirname(LogPath)
+ if not os.path.exists(FilePath):
+ os.makedirs(FilePath)
+ with open(LogPath, 'w+') as log:
+ for line in content:
+ log.write(line)
+ print("Log save to file: %s" % LogPath)
+ log.close()
+ except Exception as e:
+ print("Save log Error: %s" % e)
+
+def main():
+ parser = argparse.ArgumentParser(prog=__prog__,
+ description=__description__ + __copyright__,
+ conflict_handler='resolve')
+ parser.add_argument('-i', '--input', metavar="", dest='InputDec', help="Input DEC file name.")
+ parser.add_argument('--dirs', metavar="", action='append', dest='Dirs',
+ help="The package directory. To specify more directories, please repeat this option.")
+ parser.add_argument('--clean', action = 'store_true', default=False, dest='Clean', help="Clean the unreferenced items from DEC file.")
+ parser.add_argument('--log', metavar="", dest="Logfile", default=False, help="Put log in specified file as well as on console.")
+ options = parser.parse_args()
+ if options.InputDec:
+ if not (os.path.exists(options.InputDec) and options.InputDec.endswith(".dec")):
+ print("Error: Invalid DEC file input: %s" % options.InputDec)
+ if options.Dirs:
+ M = Main()
+ M.mainprocess(options.InputDec, options.Dirs, options.Clean, options.Logfile)
+ else:
+ print("Error: the following argument is required:'--dirs'.")
+ else:
+ print("Error: the following argument is required:'-i/--input'.")
+
+if __name__ == '__main__':
+ main()
--
2.14.1.windows.1
[-- Attachment #2: winmail.dat --]
[-- Type: application/ms-tnef, Size: 6557 bytes --]
^ permalink raw reply related [flat|nested] 2+ messages in thread
* Re: [PATCH] BaseTools:Add DetectNotUsedItem.py to Edk2\BaseTools\Scripts
2019-05-28 9:32 [PATCH] BaseTools:Add DetectNotUsedItem.py to Edk2\BaseTools\Scripts Fan, ZhijuX
@ 2019-06-11 2:08 ` Bob Feng
0 siblings, 0 replies; 2+ messages in thread
From: Bob Feng @ 2019-06-11 2:08 UTC (permalink / raw)
To: Fan, ZhijuX, devel@edk2.groups.io; +Cc: Gao, Liming
Hi Zhiju,
+ #Search File by extension name
+ def SearchbyExt(self, Ext):
+ FileList = []
+ for path in self.InfPath:
+ for root, _, files in os.walk(path, topdown=True, followlinks=False):
+ for filename in files:
+ if filename.endswith(Ext):
+ FileList.append(os.path.join(root, filename))
+ return FileList
I think this function can support to search by multiple file extensions, that would be more efficient.
Thanks,
Bob
-----Original Message-----
From: Fan, ZhijuX
Sent: Tuesday, May 28, 2019 5:33 PM
To: devel@edk2.groups.io
Cc: Gao, Liming <liming.gao@intel.com>; Feng, Bob C <bob.c.feng@intel.com>
Subject: [PATCH] BaseTools:Add DetectNotUsedItem.py to Edk2\BaseTools\Scripts
BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=1850
This script is used to Detect unreferenced PCD and GUID/Protocols/PPIs.
The input parameters are Dec file and package directory.
This script can be run in both Py2 and Py3.
Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com>
---
BaseTools/Scripts/DetectNotUsedItem.py | 185 +++++++++++++++++++++++++++++++++
1 file changed, 185 insertions(+)
create mode 100644 BaseTools/Scripts/DetectNotUsedItem.py
diff --git a/BaseTools/Scripts/DetectNotUsedItem.py b/BaseTools/Scripts/DetectNotUsedItem.py
new file mode 100644
index 0000000000..655fb65a96
--- /dev/null
+++ b/BaseTools/Scripts/DetectNotUsedItem.py
@@ -0,0 +1,185 @@
+## @file
+# Detect unreferenced PCD and GUID/Protocols/PPIs.
+#
+# Copyright (c) 2019, Intel Corporation. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-2-Clause-Patent #
+
+'''
+DetectNotUsedItem
+'''
+import re
+import os
+import sys
+import argparse
+
+#
+# Globals for help information
+#
+__prog__ = 'DetectNotUsedItem'
+__version__ = '%s Version %s' % (__prog__, '0.1')
+__copyright__ = 'Copyright (c) 2019, Intel Corporation. All rights reserved.'
+__description__ = "Detect unreferenced PCD and GUID/Protocols/PPIs.\n"
+
+SectionList = ["LibraryClasses", "Guids", "Ppis", "Protocols", "Pcd"]
+
+class PROCESS(object):
+
+ def __init__(self, DecPath, InfDirs):
+ self.Dec = DecPath
+ self.InfPath = InfDirs
+ self.Log = []
+
+ def ParserDscFdfInfFile(self):
+ AllContentList = []
+ for File in (self.SearchbyExt(".dsc") + self.SearchbyExt(".fdf") + self.SearchbyExt(".inf")):
+ AllContentList += self.ParseDscFdfInfContent(File)
+ return AllContentList
+
+ #Search File by extension name
+ def SearchbyExt(self, Ext):
+ FileList = []
+ for path in self.InfPath:
+ for root, _, files in os.walk(path, topdown=True, followlinks=False):
+ for filename in files:
+ if filename.endswith(Ext):
+ FileList.append(os.path.join(root, filename))
+ return FileList
+
+ # Parse DEC file to get Line number and Name # return section name,
+ the Item Name and comments line number def ParseDecContent(self):
+ SectionRE = re.compile(r'\[(.*)\]')
+ Flag = False
+ Comments ={}
+ Comment_Line = []
+ ItemName = {}
+ with open(self.Dec, 'r') as F:
+ for Index, content in enumerate(F):
+ NotComment = not content.strip().startswith("#")
+ Section = SectionRE.findall(content)
+ if Section and NotComment:
+ Flag = self.IsNeedParseSection(Section[0])
+ if Flag:
+ Comment_Line.append(Index)
+ if NotComment:
+ if content != "\n" and content != "\r\n":
+ ItemName[Index] = content.split('=')[0].split('|')[0].split('#')[0].strip()
+ Comments[Index] = Comment_Line
+ Comment_Line = []
+ return ItemName, Comments
+
+ def IsNeedParseSection(self, SectionName):
+ for item in SectionList:
+ if item in SectionName:
+ return True
+ return False
+
+ #Parse DSC, FDF, INF File, remove comments, return Lines list def
+ ParseDscFdfInfContent(self, File):
+ with open(File,'r') as F:
+ lines = F.readlines()
+ for Index in range(len(lines)-1, -1, -1):
+ if lines[Index].strip().startswith("#") or lines[Index] == "\n" or lines[Index] == "\r\n":
+ lines.remove(lines[Index])
+ elif "#" in lines[Index]:
+ lines[Index] = lines[Index].split("#")[0].strip()
+ else:
+ lines[Index] = lines[Index].strip()
+ return lines
+
+ def DetectNotUsedItem(self):
+ NotUsedItem = {}
+ DecItem, DecComments = self.ParseDecContent()
+ InfDscFdfContent = self.ParserDscFdfInfFile()
+ for LineNum in list(DecItem.keys()):
+ DecItemName = DecItem[LineNum]
+ Match_reg = re.compile("(?<![a-zA-Z0-9_-])%s(?![a-zA-Z0-9_-])" % DecItemName)
+ MatchFlag = False
+ for Line in InfDscFdfContent:
+ if Match_reg.search(Line):
+ MatchFlag = True
+ break
+ if not MatchFlag:
+ NotUsedItem[LineNum] = DecItemName
+ self.Display(NotUsedItem)
+ return NotUsedItem, DecComments
+
+ def Display(self,UnuseDict):
+ print("DEC File:\n%s\n%s%s" % (self.Dec, "{:<15}".format("Line Number"),"{:<0}".format("Unused Item")))
+ self.Log.append("DEC File:\n%s\n%s%s\n" % (self.Dec, "{:<15}".format("Line Number"),"{:<0}".format("Unused Item")))
+ for num in list(sorted(UnuseDict.keys())):
+ ItemName = UnuseDict[num]
+ print("%s%s%s" % (" "*3,"{:<12}".format(num + 1), "{:<1}".format(ItemName)))
+ self.Log.append(("%s%s%s\n" % (" "*3,"{:<12}".format(num + 1),
+ "{:<1}".format(ItemName))))
+
+ def Clean(self, UnUseDict, Comments):
+ removednum = []
+ for num in list(UnUseDict.keys()):
+ if num in list(Comments.keys()):
+ removednum += Comments[num]
+ with open(self.Dec, 'r') as Dec:
+ lines = Dec.readlines()
+ try:
+ with open(self.Dec, 'w+') as T:
+ for linenum in range(len(lines)):
+ if linenum in removednum:
+ continue
+ else:
+ T.write(lines[linenum])
+ print("DEC File has been clean: %s"%(self.Dec))
+ except Exception as err:
+ print(err)
+
+class Main(object):
+
+ def mainprocess(self, Dec, Dirs, Isclean, LogPath):
+ for dir in Dirs:
+ if not os.path.exists(dir):
+ print("Error: Invalid path for '--dirs': %s" % dir)
+ sys.exit(1)
+ Pa = PROCESS(Dec, Dirs)
+ unuse, comment = Pa.DetectNotUsedItem()
+ if Isclean:
+ Pa.Clean(unuse, comment)
+ self.Logging(Pa.Log, LogPath)
+
+ def Logging(self, content, LogPath):
+ if LogPath:
+ try:
+ if os.path.isdir(LogPath):
+ FilePath = os.path.dirname(LogPath)
+ if not os.path.exists(FilePath):
+ os.makedirs(FilePath)
+ with open(LogPath, 'w+') as log:
+ for line in content:
+ log.write(line)
+ print("Log save to file: %s" % LogPath)
+ log.close()
+ except Exception as e:
+ print("Save log Error: %s" % e)
+
+def main():
+ parser = argparse.ArgumentParser(prog=__prog__,
+ description=__description__ + __copyright__,
+ conflict_handler='resolve')
+ parser.add_argument('-i', '--input', metavar="", dest='InputDec',
+help="Input DEC file name.")
+ parser.add_argument('--dirs', metavar="", action='append', dest='Dirs',
+ help="The package directory. To specify more
+directories, please repeat this option.")
+ parser.add_argument('--clean', action = 'store_true', default=False,
+dest='Clean', help="Clean the unreferenced items from DEC file.")
+ parser.add_argument('--log', metavar="", dest="Logfile",
+default=False, help="Put log in specified file as well as on console.")
+ options = parser.parse_args()
+ if options.InputDec:
+ if not (os.path.exists(options.InputDec) and options.InputDec.endswith(".dec")):
+ print("Error: Invalid DEC file input: %s" % options.InputDec)
+ if options.Dirs:
+ M = Main()
+ M.mainprocess(options.InputDec, options.Dirs, options.Clean, options.Logfile)
+ else:
+ print("Error: the following argument is required:'--dirs'.")
+ else:
+ print("Error: the following argument is required:'-i/--input'.")
+
+if __name__ == '__main__':
+ main()
--
2.14.1.windows.1
^ permalink raw reply related [flat|nested] 2+ messages in thread
end of thread, other threads:[~2019-06-11 2:09 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2019-05-28 9:32 [PATCH] BaseTools:Add DetectNotUsedItem.py to Edk2\BaseTools\Scripts Fan, ZhijuX
2019-06-11 2:08 ` Bob Feng
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox