From mboxrd@z Thu Jan 1 00:00:00 1970 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 134.134.136.100, mailfrom: liming.gao@intel.com) Received: from mga07.intel.com (mga07.intel.com [134.134.136.100]) by groups.io with SMTP; Thu, 20 Jun 2019 18:28:13 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by orsmga105.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 20 Jun 2019 18:28:12 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.63,398,1557212400"; d="scan'208";a="187008687" Received: from fmsmsx106.amr.corp.intel.com ([10.18.124.204]) by fmsmga002.fm.intel.com with ESMTP; 20 Jun 2019 18:28:12 -0700 Received: from fmsmsx115.amr.corp.intel.com (10.18.116.19) by FMSMSX106.amr.corp.intel.com (10.18.124.204) with Microsoft SMTP Server (TLS) id 14.3.439.0; Thu, 20 Jun 2019 18:28:12 -0700 Received: from shsmsx108.ccr.corp.intel.com (10.239.4.97) by fmsmsx115.amr.corp.intel.com (10.18.116.19) with Microsoft SMTP Server (TLS) id 14.3.439.0; Thu, 20 Jun 2019 18:28:11 -0700 Received: from shsmsx104.ccr.corp.intel.com ([169.254.5.185]) by SHSMSX108.ccr.corp.intel.com ([169.254.8.236]) with mapi id 14.03.0439.000; Fri, 21 Jun 2019 09:28:09 +0800 From: "Liming Gao" To: "Fan, ZhijuX" , "devel@edk2.groups.io" CC: "Feng, Bob C" Subject: Re: [PATCH V3] BaseTools:Add DetectNotUsedItem.py to Edk2\BaseTools\Scripts Thread-Topic: [PATCH V3] BaseTools:Add DetectNotUsedItem.py to Edk2\BaseTools\Scripts Thread-Index: AdUm/6D9MqCvoJniSmeNtuQNBeSgIwA0M9ig Date: Fri, 21 Jun 2019 01:28:09 +0000 Message-ID: <4A89E2EF3DFEDB4C8BFDE51014F606A14E487EFB@SHSMSX104.ccr.corp.intel.com> References: In-Reply-To: Accept-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-ctpclassification: CTP_NT x-titus-metadata-40: eyJDYXRlZ29yeUxhYmVscyI6IiIsIk1ldGFkYXRhIjp7Im5zIjoiaHR0cDpcL1wvd3d3LnRpdHVzLmNvbVwvbnNcL0ludGVsMyIsImlkIjoiNWVkZWU4NjYtZWY5ZS00ZWI2LTlhY2EtZjZiMjc1YTUzNmQ0IiwicHJvcHMiOlt7Im4iOiJDVFBDbGFzc2lmaWNhdGlvbiIsInZhbHMiOlt7InZhbHVlIjoiQ1RQX05UIn1dfV19LCJTdWJqZWN0TGFiZWxzIjpbXSwiVE1DVmVyc2lvbiI6IjE3LjEwLjE4MDQuNDkiLCJUcnVzdGVkTGFiZWxIYXNoIjoiN21OUW0wQThRWm9pV0dzMEVXMUxmNjBMcE1MNDFCNlg3TG9rcW1cL01mWG9TTG9cL1R2dFpXaXFUb1crcFwvZHJDRCJ9 dlp-product: dlpe-windows dlp-version: 11.0.600.7 dlp-reaction: no-action x-originating-ip: [10.239.127.40] MIME-Version: 1.0 Return-Path: liming.gao@intel.com Content-Language: en-US Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable Reviewed-by: Liming Gao > -----Original Message----- > From: Fan, ZhijuX > Sent: Thursday, June 20, 2019 8:32 AM > To: devel@edk2.groups.io > Cc: Gao, Liming ; Feng, Bob C > Subject: [PATCH V3] BaseTools:Add DetectNotUsedItem.py to Edk2\BaseTools\= Scripts >=20 > BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=3D1850 >=20 > This script is used to Detect unreferenced PCD and GUID/Protocols/PPIs. > The input parameters are Dec file and package directory. >=20 > This script can be run in both Py2 and Py3. >=20 > Cc: Bob Feng > Cc: Liming Gao > Signed-off-by: Zhiju.Fan > --- > BaseTools/Scripts/DetectNotUsedItem.py | 185 +++++++++++++++++++++++++++= ++++++ > 1 file changed, 185 insertions(+) > create mode 100644 BaseTools/Scripts/DetectNotUsedItem.py >=20 > diff --git a/BaseTools/Scripts/DetectNotUsedItem.py b/BaseTools/Scripts/D= etectNotUsedItem.py > new file mode 100644 > index 0000000000..655fb65a96 > --- /dev/null > +++ b/BaseTools/Scripts/DetectNotUsedItem.py > @@ -0,0 +1,185 @@ > +## @file > +# Detect unreferenced PCD and GUID/Protocols/PPIs. > +# > +# Copyright (c) 2019, Intel Corporation. All rights reserved. > +# > +# SPDX-License-Identifier: BSD-2-Clause-Patent > +# > + > +''' > +DetectNotUsedItem > +''' > +import re > +import os > +import sys > +import argparse > + > +# > +# Globals for help information > +# > +__prog__ =3D 'DetectNotUsedItem' > +__version__ =3D '%s Version %s' % (__prog__, '0.1') > +__copyright__ =3D 'Copyright (c) 2019, Intel Corporation. All rights r= eserved.' > +__description__ =3D "Detect unreferenced PCD and GUID/Protocols/PPIs.\n" > + > +SectionList =3D ["LibraryClasses", "Guids", "Ppis", "Protocols", "Pcd"] > + > +class PROCESS(object): > + > + def __init__(self, DecPath, InfDirs): > + self.Dec =3D DecPath > + self.InfPath =3D InfDirs > + self.Log =3D [] > + > + def ParserDscFdfInfFile(self): > + AllContentList =3D [] > + for File in (self.SearchbyExt(".dsc") + self.SearchbyExt(".fdf") + s= elf.SearchbyExt(".inf")): > + AllContentList +=3D self.ParseDscFdfInfContent(File) > + return AllContentList > + > + #Search File by extension name > + def SearchbyExt(self, Ext): > + FileList =3D [] > + for path in self.InfPath: > + for root, _, files in os.walk(path, topdown=3DTrue, followlinks=3D= False): > + for filename in files: > + if filename.endswith(Ext): > + FileList.append(os.path.join(root, filename)) > + return FileList > + > + # Parse DEC file to get Line number and Name > + # return section name, the Item Name and comments line number > + def ParseDecContent(self): > + SectionRE =3D re.compile(r'\[(.*)\]') > + Flag =3D False > + Comments =3D{} > + Comment_Line =3D [] > + ItemName =3D {} > + with open(self.Dec, 'r') as F: > + for Index, content in enumerate(F): > + NotComment =3D not content.strip().startswith("#") > + Section =3D SectionRE.findall(content) > + if Section and NotComment: > + Flag =3D self.IsNeedParseSection(Section[0]) > + if Flag: > + Comment_Line.append(Index) > + if NotComment: > + if content !=3D "\n" and content !=3D "\r\n": > + ItemName[Index] =3D content.split('=3D')[0].split('|')[0].= split('#')[0].strip() > + Comments[Index] =3D Comment_Line > + Comment_Line =3D [] > + return ItemName, Comments > + > + def IsNeedParseSection(self, SectionName): > + for item in SectionList: > + if item in SectionName: > + return True > + return False > + > + #Parse DSC, FDF, INF File, remove comments, return Lines list > + def ParseDscFdfInfContent(self, File): > + with open(File,'r') as F: > + lines =3D F.readlines() > + for Index in range(len(lines)-1, -1, -1): > + if lines[Index].strip().startswith("#") or lines[Index] =3D=3D "\n= " or lines[Index] =3D=3D "\r\n": > + lines.remove(lines[Index]) > + elif "#" in lines[Index]: > + lines[Index] =3D lines[Index].split("#")[0].strip() > + else: > + lines[Index] =3D lines[Index].strip() > + return lines > + > + def DetectNotUsedItem(self): > + NotUsedItem =3D {} > + DecItem, DecComments =3D self.ParseDecContent() > + InfDscFdfContent =3D self.ParserDscFdfInfFile() > + for LineNum in list(DecItem.keys()): > + DecItemName =3D DecItem[LineNum] > + Match_reg =3D re.compile("(? + MatchFlag =3D False > + for Line in InfDscFdfContent: > + if Match_reg.search(Line): > + MatchFlag =3D True > + break > + if not MatchFlag: > + NotUsedItem[LineNum] =3D DecItemName > + self.Display(NotUsedItem) > + return NotUsedItem, DecComments > + > + def Display(self,UnuseDict): > + print("DEC File:\n%s\n%s%s" % (self.Dec, "{:<15}".format("Line Numbe= r"),"{:<0}".format("Unused Item"))) > + self.Log.append("DEC File:\n%s\n%s%s\n" % (self.Dec, "{:<15}".format= ("Line Number"),"{:<0}".format("Unused Item"))) > + for num in list(sorted(UnuseDict.keys())): > + ItemName =3D UnuseDict[num] > + print("%s%s%s" % (" "*3,"{:<12}".format(num + 1), "{:<1}".format(I= temName))) > + self.Log.append(("%s%s%s\n" % (" "*3,"{:<12}".format(num + 1), "{:= <1}".format(ItemName)))) > + > + def Clean(self, UnUseDict, Comments): > + removednum =3D [] > + for num in list(UnUseDict.keys()): > + if num in list(Comments.keys()): > + removednum +=3D Comments[num] > + with open(self.Dec, 'r') as Dec: > + lines =3D Dec.readlines() > + try: > + with open(self.Dec, 'w+') as T: > + for linenum in range(len(lines)): > + if linenum in removednum: > + continue > + else: > + T.write(lines[linenum]) > + print("DEC File has been clean: %s"%(self.Dec)) > + except Exception as err: > + print(err) > + > +class Main(object): > + > + def mainprocess(self, Dec, Dirs, Isclean, LogPath): > + for dir in Dirs: > + if not os.path.exists(dir): > + print("Error: Invalid path for '--dirs': %s" % dir) > + sys.exit(1) > + Pa =3D PROCESS(Dec, Dirs) > + unuse, comment =3D Pa.DetectNotUsedItem() > + if Isclean: > + Pa.Clean(unuse, comment) > + self.Logging(Pa.Log, LogPath) > + > + def Logging(self, content, LogPath): > + if LogPath: > + try: > + if os.path.isdir(LogPath): > + FilePath =3D os.path.dirname(LogPath) > + if not os.path.exists(FilePath): > + os.makedirs(FilePath) > + with open(LogPath, 'w+') as log: > + for line in content: > + log.write(line) > + print("Log save to file: %s" % LogPath) > + log.close() > + except Exception as e: > + print("Save log Error: %s" % e) > + > +def main(): > + parser =3D argparse.ArgumentParser(prog=3D__prog__, > + description=3D__description__ + __copyri= ght__, > + conflict_handler=3D'resolve') > + parser.add_argument('-i', '--input', metavar=3D"", dest=3D'InputDec', = help=3D"Input DEC file name.") > + parser.add_argument('--dirs', metavar=3D"", action=3D'append', dest=3D= 'Dirs', > + help=3D"The package directory. To specify more d= irectories, please repeat this option.") > + parser.add_argument('--clean', action =3D 'store_true', default=3DFals= e, dest=3D'Clean', help=3D"Clean the unreferenced items from DEC file.") > + parser.add_argument('--log', metavar=3D"", dest=3D"Logfile", default= =3DFalse, help=3D"Put log in specified file as well as on console.") > + options =3D parser.parse_args() > + if options.InputDec: > + if not (os.path.exists(options.InputDec) and options.InputDec.endswi= th(".dec")): > + print("Error: Invalid DEC file input: %s" % options.InputDec) > + if options.Dirs: > + M =3D Main() > + M.mainprocess(options.InputDec, options.Dirs, options.Clean, optio= ns.Logfile) > + else: > + print("Error: the following argument is required:'--dirs'.") > + else: > + print("Error: the following argument is required:'-i/--input'.") > + > +if __name__ =3D=3D '__main__': > + main() > -- > 2.14.1.windows.1