From mboxrd@z Thu Jan  1 00:00:00 1970
Authentication-Results: mx.groups.io;
 dkim=missing; spf=pass (domain: intel.com, ip: 192.55.52.115, mailfrom: bob.c.feng@intel.com)
Received: from mga14.intel.com (mga14.intel.com [192.55.52.115])
 by groups.io with SMTP; Mon, 10 Jun 2019 19:09:32 -0700
X-Amp-Result: SKIPPED(no attachment in message)
X-Amp-File-Uploaded: False
Received: from orsmga006.jf.intel.com ([10.7.209.51])
  by fmsmga103.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 10 Jun 2019 19:09:31 -0700
X-ExtLoop1: 1
Received: from fmsmsx107.amr.corp.intel.com ([10.18.124.205])
  by orsmga006.jf.intel.com with ESMTP; 10 Jun 2019 19:09:30 -0700
Received: from fmsmsx113.amr.corp.intel.com (10.18.116.7) by
 fmsmsx107.amr.corp.intel.com (10.18.124.205) with Microsoft SMTP Server (TLS)
 id 14.3.408.0; Mon, 10 Jun 2019 19:09:31 -0700
Received: from shsmsx105.ccr.corp.intel.com (10.239.4.158) by
 FMSMSX113.amr.corp.intel.com (10.18.116.7) with Microsoft SMTP Server (TLS)
 id 14.3.408.0; Mon, 10 Jun 2019 19:09:30 -0700
Received: from shsmsx101.ccr.corp.intel.com ([169.254.1.10]) by
 SHSMSX105.ccr.corp.intel.com ([169.254.11.153]) with mapi id 14.03.0415.000;
 Tue, 11 Jun 2019 10:08:24 +0800
From: "Bob Feng" <bob.c.feng@intel.com>
To: "Fan, ZhijuX" <zhijux.fan@intel.com>, "devel@edk2.groups.io"
	<devel@edk2.groups.io>
CC: "Gao, Liming" <liming.gao@intel.com>
Subject: Re: [PATCH] BaseTools:Add DetectNotUsedItem.py to Edk2\BaseTools\Scripts
Thread-Topic: [PATCH] BaseTools:Add DetectNotUsedItem.py to
 Edk2\BaseTools\Scripts
Thread-Index: AdUVODvoQioEnMiHSEKe1Y9pnbE06QKwdSfQ
Date: Tue, 11 Jun 2019 02:08:23 +0000
Message-ID: <08650203BA1BD64D8AD9B6D5D74A85D16013546B@SHSMSX101.ccr.corp.intel.com>
References: <FAD0D7E0AE0FA54D987F6E72435CAFD50AF62DAA@SHSMSX101.ccr.corp.intel.com>
In-Reply-To: <FAD0D7E0AE0FA54D987F6E72435CAFD50AF62DAA@SHSMSX101.ccr.corp.intel.com>
Accept-Language: zh-CN, en-US
X-MS-Has-Attach: 
X-MS-TNEF-Correlator: 
x-originating-ip: [10.239.127.40]
MIME-Version: 1.0
Return-Path: bob.c.feng@intel.com
Content-Language: en-US
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: quoted-printable

Hi Zhiju,

+  #Search File by extension name
+  def SearchbyExt(self, Ext):
+    FileList =3D []
+    for path in self.InfPath:
+      for root, _, files in os.walk(path, topdown=3DTrue, followlinks=3DFa=
lse):
+        for filename in files:
+          if filename.endswith(Ext):
+            FileList.append(os.path.join(root, filename))
+    return FileList

I think this function can support to search by multiple file extensions, th=
at would be more efficient.=20

Thanks,
Bob

-----Original Message-----
From: Fan, ZhijuX=20
Sent: Tuesday, May 28, 2019 5:33 PM
To: devel@edk2.groups.io
Cc: Gao, Liming <liming.gao@intel.com>; Feng, Bob C <bob.c.feng@intel.com>
Subject: [PATCH] BaseTools:Add DetectNotUsedItem.py to Edk2\BaseTools\Scrip=
ts

BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=3D1850

This script is used to Detect unreferenced PCD and GUID/Protocols/PPIs.
The input parameters are Dec file and package directory.

This script can be run in both Py2 and Py3.

Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com>
---
 BaseTools/Scripts/DetectNotUsedItem.py | 185 +++++++++++++++++++++++++++++=
++++
 1 file changed, 185 insertions(+)
 create mode 100644 BaseTools/Scripts/DetectNotUsedItem.py

diff --git a/BaseTools/Scripts/DetectNotUsedItem.py b/BaseTools/Scripts/Det=
ectNotUsedItem.py
new file mode 100644
index 0000000000..655fb65a96
--- /dev/null
+++ b/BaseTools/Scripts/DetectNotUsedItem.py
@@ -0,0 +1,185 @@
+## @file
+# Detect unreferenced PCD and GUID/Protocols/PPIs.
+#
+# Copyright (c) 2019, Intel Corporation. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-2-Clause-Patent #
+
+'''
+DetectNotUsedItem
+'''
+import re
+import os
+import sys
+import argparse
+
+#
+# Globals for help information
+#
+__prog__        =3D 'DetectNotUsedItem'
+__version__     =3D '%s Version %s' % (__prog__, '0.1')
+__copyright__   =3D 'Copyright (c) 2019, Intel Corporation. All rights res=
erved.'
+__description__ =3D "Detect unreferenced PCD and GUID/Protocols/PPIs.\n"
+
+SectionList =3D ["LibraryClasses", "Guids", "Ppis", "Protocols", "Pcd"]
+
+class PROCESS(object):
+
+  def __init__(self, DecPath, InfDirs):
+    self.Dec =3D DecPath
+    self.InfPath =3D InfDirs
+    self.Log =3D []
+
+  def ParserDscFdfInfFile(self):
+    AllContentList =3D []
+    for File in (self.SearchbyExt(".dsc") + self.SearchbyExt(".fdf") + sel=
f.SearchbyExt(".inf")):
+      AllContentList +=3D self.ParseDscFdfInfContent(File)
+    return AllContentList
+
+  #Search File by extension name
+  def SearchbyExt(self, Ext):
+    FileList =3D []
+    for path in self.InfPath:
+      for root, _, files in os.walk(path, topdown=3DTrue, followlinks=3DFa=
lse):
+        for filename in files:
+          if filename.endswith(Ext):
+            FileList.append(os.path.join(root, filename))
+    return FileList
+
+  # Parse DEC file to get Line number and Name  # return section name,=20
+ the Item Name and comments line number  def ParseDecContent(self):
+    SectionRE =3D re.compile(r'\[(.*)\]')
+    Flag =3D False
+    Comments =3D{}
+    Comment_Line =3D []
+    ItemName =3D {}
+    with open(self.Dec, 'r') as F:
+      for Index, content in enumerate(F):
+        NotComment =3D not content.strip().startswith("#")
+        Section =3D SectionRE.findall(content)
+        if Section and NotComment:
+          Flag =3D self.IsNeedParseSection(Section[0])
+        if Flag:
+          Comment_Line.append(Index)
+          if NotComment:
+            if content !=3D "\n" and content !=3D "\r\n":
+              ItemName[Index] =3D content.split('=3D')[0].split('|')[0].sp=
lit('#')[0].strip()
+              Comments[Index] =3D Comment_Line
+              Comment_Line =3D []
+    return ItemName, Comments
+
+  def IsNeedParseSection(self, SectionName):
+    for item in SectionList:
+      if item in SectionName:
+        return True
+    return False
+
+  #Parse DSC, FDF, INF File, remove comments, return Lines list  def=20
+ ParseDscFdfInfContent(self, File):
+    with open(File,'r') as F:
+      lines =3D F.readlines()
+    for Index in range(len(lines)-1, -1, -1):
+      if lines[Index].strip().startswith("#") or lines[Index] =3D=3D "\n" =
or lines[Index] =3D=3D "\r\n":
+        lines.remove(lines[Index])
+      elif "#" in lines[Index]:
+        lines[Index] =3D lines[Index].split("#")[0].strip()
+      else:
+        lines[Index] =3D lines[Index].strip()
+    return lines
+
+  def DetectNotUsedItem(self):
+    NotUsedItem =3D {}
+    DecItem, DecComments =3D self.ParseDecContent()
+    InfDscFdfContent =3D self.ParserDscFdfInfFile()
+    for LineNum in list(DecItem.keys()):
+      DecItemName =3D DecItem[LineNum]
+      Match_reg =3D re.compile("(?<![a-zA-Z0-9_-])%s(?![a-zA-Z0-9_-])" % D=
ecItemName)
+      MatchFlag =3D False
+      for Line in InfDscFdfContent:
+        if Match_reg.search(Line):
+          MatchFlag =3D True
+          break
+      if not MatchFlag:
+        NotUsedItem[LineNum] =3D DecItemName
+    self.Display(NotUsedItem)
+    return NotUsedItem, DecComments
+
+  def Display(self,UnuseDict):
+    print("DEC File:\n%s\n%s%s" % (self.Dec, "{:<15}".format("Line Number"=
),"{:<0}".format("Unused Item")))
+    self.Log.append("DEC File:\n%s\n%s%s\n" % (self.Dec, "{:<15}".format("=
Line Number"),"{:<0}".format("Unused Item")))
+    for num in list(sorted(UnuseDict.keys())):
+      ItemName =3D UnuseDict[num]
+      print("%s%s%s" % (" "*3,"{:<12}".format(num + 1), "{:<1}".format(Ite=
mName)))
+      self.Log.append(("%s%s%s\n" % (" "*3,"{:<12}".format(num + 1),=20
+ "{:<1}".format(ItemName))))
+
+  def Clean(self, UnUseDict, Comments):
+    removednum =3D []
+    for num in list(UnUseDict.keys()):
+      if num in list(Comments.keys()):
+        removednum +=3D Comments[num]
+    with open(self.Dec, 'r') as Dec:
+      lines =3D Dec.readlines()
+    try:
+      with open(self.Dec, 'w+') as T:
+        for linenum in range(len(lines)):
+          if linenum in removednum:
+            continue
+          else:
+            T.write(lines[linenum])
+      print("DEC File has been clean: %s"%(self.Dec))
+    except Exception as err:
+      print(err)
+
+class Main(object):
+
+  def mainprocess(self, Dec, Dirs, Isclean, LogPath):
+    for dir in Dirs:
+      if not os.path.exists(dir):
+        print("Error: Invalid path for '--dirs': %s" % dir)
+        sys.exit(1)
+    Pa =3D PROCESS(Dec, Dirs)
+    unuse, comment =3D Pa.DetectNotUsedItem()
+    if Isclean:
+      Pa.Clean(unuse, comment)
+    self.Logging(Pa.Log, LogPath)
+
+  def Logging(self, content, LogPath):
+    if LogPath:
+      try:
+        if os.path.isdir(LogPath):
+          FilePath =3D os.path.dirname(LogPath)
+          if not os.path.exists(FilePath):
+            os.makedirs(FilePath)
+        with open(LogPath, 'w+') as log:
+          for line in content:
+            log.write(line)
+        print("Log save to file: %s" % LogPath)
+        log.close()
+      except Exception as e:
+        print("Save log Error: %s" % e)
+
+def main():
+  parser =3D argparse.ArgumentParser(prog=3D__prog__,
+                                description=3D__description__ + __copyrigh=
t__,
+                                conflict_handler=3D'resolve')
+  parser.add_argument('-i', '--input', metavar=3D"", dest=3D'InputDec',=20
+help=3D"Input DEC file name.")
+  parser.add_argument('--dirs', metavar=3D"", action=3D'append', dest=3D'D=
irs',
+                        help=3D"The package directory. To specify more=20
+directories, please repeat this option.")
+  parser.add_argument('--clean', action =3D 'store_true', default=3DFalse,=
=20
+dest=3D'Clean', help=3D"Clean the unreferenced items from DEC file.")
+  parser.add_argument('--log', metavar=3D"", dest=3D"Logfile",=20
+default=3DFalse, help=3D"Put log in specified file as well as on console."=
)
+  options =3D parser.parse_args()
+  if options.InputDec:
+    if not (os.path.exists(options.InputDec) and options.InputDec.endswith=
(".dec")):
+      print("Error: Invalid DEC file input: %s" % options.InputDec)
+    if options.Dirs:
+      M =3D Main()
+      M.mainprocess(options.InputDec, options.Dirs, options.Clean, options=
.Logfile)
+    else:
+      print("Error: the following argument is required:'--dirs'.")
+  else:
+    print("Error: the following argument is required:'-i/--input'.")
+
+if __name__ =3D=3D '__main__':
+  main()
--
2.14.1.windows.1