From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mga01.intel.com (mga01.intel.com []) by mx.groups.io with SMTP id smtpd.web10.9648.1591174092723471280 for ; Wed, 03 Jun 2020 01:48:17 -0700 Authentication-Results: mx.groups.io; dkim=missing; spf=fail (domain: intel.com, ip: , mailfrom: shenglei.zhang@intel.com) IronPort-SDR: XXS1y91Ya/g9+QKqvx6CNXhftf4pURwLDB+DGXQ6MK5Ah0xVyOG/18tuvRyB1Nth8/pYEv2TNP 6Zz0lSYa+tWQ== X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga006.fm.intel.com ([10.253.24.20]) by fmsmga101.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 03 Jun 2020 01:48:17 -0700 IronPort-SDR: LzNm5OC5L/47xQe1lsNqtMX59a6/AxqCNcsPQZCcDIo/DyB3CUHYyhs3xmVbY9RDRgukB6eZfY yOM6lgZY04Lw== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.73,467,1583222400"; d="scan'208";a="470975837" Received: from shenglei-dev.ccr.corp.intel.com ([10.239.158.52]) by fmsmga006.fm.intel.com with ESMTP; 03 Jun 2020 01:48:15 -0700 From: "Zhang, Shenglei" To: devel@edk2.groups.io Cc: Bob Feng , Liming Gao Subject: [PATCH v2 2/5] BaseTools/Scripts: Add EccCheck.py Date: Wed, 3 Jun 2020 16:48:04 +0800 Message-Id: <20200603084807.24484-3-shenglei.zhang@intel.com> X-Mailer: git-send-email 2.18.0.windows.1 In-Reply-To: <20200603084807.24484-1-shenglei.zhang@intel.com> References: <20200603084807.24484-1-shenglei.zhang@intel.com> REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2606 EccCheck.py is a tool to report Ecc issues for commits, which will be run on open ci. But note not each kind of issue could be reported out. It can only handle the issues, whose line number in CSV report accurately map with their code in source code files. And comment issues can also be handled. Its usage is similar to PatchCheck.py. Type EccCheck.py -h and then learn how to use it. If a patch passes EccCheck, "Ecc Pass" will show up. Otherwise, "Ecc error detected" alerts the users and the details are also presented. Cc: Bob Feng Cc: Liming Gao Signed-off-by: Shenglei Zhang --- v2: Update 2/5, fix the bug that the script can't hanlde multiple commits. BaseTools/Scripts/EccCheck.py | 433 ++++++++++++++++++++++++++++++++++ 1 file changed, 433 insertions(+) create mode 100644 BaseTools/Scripts/EccCheck.py diff --git a/BaseTools/Scripts/EccCheck.py b/BaseTools/Scripts/EccCheck.py new file mode 100644 index 000000000000..034a9965a79b --- /dev/null +++ b/BaseTools/Scripts/EccCheck.py @@ -0,0 +1,433 @@ +## @file +# Check a patch for various format issues +# +# Copyright (c) 2020, Intel Corporation. All rights reserved.
+# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# + +import os +import re +import csv +import subprocess +import argparse +import sys +import yaml +import xml.dom.minidom + +__copyright__ = "Copyright (c) 2020, Intel Corporation All rights reserved." +ReModifyFile = re.compile(r'[B-Q,S-Z]+[\d]*\t(.*?)\n') +FindModifyFile = re.compile(r'\+\+\+ b\/(.*)') +LineScopePattern = (r'@@ -\d*\,*\d* \+\d*\,*\d* @@.*') +LineNumRange = re.compile(r'@@ -\d*\,*\d* \+(\d*)\,*(\d*) @@.*') + +EnvList = os.environ +GlobalSymbol = {} + +def AppendException(ExceptionList, ExceptionFile): + error_code_list = ExceptionList[::2] + keyword_list = ExceptionList[1::2] + domTree = xml.dom.minidom.parse(ExceptionFile) + rootNode = domTree.documentElement + for error_code, keyword in zip(error_code_list, keyword_list): + customer_node = domTree.createElement("Exception") + keyword_node = domTree.createElement("KeyWord") + keyword_node_text_value = domTree.createTextNode(keyword) + keyword_node.appendChild(keyword_node_text_value) + customer_node.appendChild(keyword_node) + error_code_node = domTree.createElement("ErrorID") + error_code_text_value = domTree.createTextNode(error_code) + error_code_node.appendChild(error_code_text_value) + customer_node.appendChild(error_code_node) + rootNode.appendChild(customer_node) + + with open(ExceptionFile, 'w') as f: + domTree.writexml(f,indent='',addindent='',newl='\n',encoding='UTF-8') + +def GetPkgList(EnvList): + WORKDIR = EnvList['WORKDIR'] + dirs = os.listdir(WORKDIR) + PkgList = [] + for directory in dirs: + if directory.endswith('Pkg'): + PkgList.append(directory) + return PkgList + +def GenerateEccReport(EnvList, ModifyDirList, EccDiffRange): + IsECCNeed = False + IsTrue = True + PkgList = GetPkgList(EnvList) + + for Line in ModifyDirList: + print('echo Run ECC tool for the commit in %s' % Line) + GlobalSymbol['GenerateEccReport'] = True + for Pkg in PkgList: + if Pkg in Line: + IsECCNeed = True + Ecc_cmd = ["ecc", "-c", "%WORKDIR%\BaseTools\Source\Python\Ecc\config.ini", + "-e", "%WORKDIR%\BaseTools\Source\Python\Ecc\exception.xml", + "-t", "%WORKDIR%\{}".format(Line), + "-r", "%WORKDIR%\Ecc.csv"] + _, _, result, return_code = ExecuteScript(Ecc_cmd, EnvList, shell=True) + if return_code != 0: + IsTrue = False + break + + if not IsTrue: + print('Fail to run ECC tool') + GlobalSymbol['SCRIPT_ERROR'] = True + EndDelFile(EnvList) + + if GlobalSymbol.get('GenerateEccReport'): + ParseEccReport(EnvList, EccDiffRange) + else: + print("Patch check tool or ECC tool don't detect error") + + if IsECCNeed: + revert_cmd = ["git", "checkout", "--", "%WORKDIR%\BaseTools\Source\Python\Ecc\exception.xml"] + _, _, result, return_code = ExecuteScript(revert_cmd, EnvList, shell=True) + else: + print("Doesn't need run ECC check") + return + +def ParseEccReport(EnvList, EccDiffRange): + WorkDir = EnvList['WORKDIR'] + EccLog = os.path.join(WorkDir, "Ecc.log") + EccsCsv = "Ecc.csv" + FileList = os.listdir(WorkDir) + rowLines = [] + if EccsCsv in FileList: + with open(EccsCsv) as Csv: + reader = csv.reader(Csv) + for row in reader: + for ModifyFile in EccDiffRange: + if ModifyFile in row[3]: + for i in EccDiffRange[ModifyFile]: + LineNo = int(row[4]) + if i[0] <= LineNo <= i[1]: + row[0] = '\nEFI coding style error' + row[1] = 'Error code: ' + row[1] + row[3] = 'File: ' + row[3] + row[4] = 'Line number: ' + row[4] + rowLine = '\n *'.join(row) + rowLines.append(rowLine) + break + break + if rowLines: + GlobalSymbol['ECC_PASS'] = False + + with open(EccLog, 'a') as EccFile: + AllLine = '\n'.join(rowLines) + AllLine = AllLine + '\n' + EccFile.writelines(AllLine) + +def RemoveFile(File): + if os.path.exists(File): + os.remove(File) + +def ExecuteScript(command, env_variables, collect_env=False, + enable_std_pipe=False, shell=True): + """launches a process that executes a script/shell command passed to it + + :param command: The command/script with its commandline + arguments to be executed + :type command: List:String + :param env_variables: Environment variables passed to the process + :type env_variables: String + :param collect_env: Enables the collection of evironment variables + when process execution is done + :type collect_env: Boolean + :param enable_std_pipe: Enables process out to be piped to + :type enable_std_pipe: String + :returns: a tuple of std_out, stderr , environment variables, + return code + :rtype: Tuple: (std_out, stderr , enVar, return_code) + """ + + env_marker = '-----env-----' + env = {} + kwarg = {"env": env_variables, + "universal_newlines": True, + "shell": shell, + "cwd": env_variables["WORKSPACE"]} + + if enable_std_pipe or collect_env: + kwarg["stdout"] = subprocess.PIPE + kwarg["stderr"] = subprocess.PIPE + + if collect_env: + # get the binary that prints environment variables based on os + if os.name == 'nt': + get_var_command = "set" + else: + get_var_command = "env" + # modify the command to print the environment variables + if isinstance(command, list): + command += ["&&", "echo", env_marker, "&&", + get_var_command, "&&", "echo", env_marker] + else: + command += " " + " ".join(["&&", "echo", env_marker, + "&&", get_var_command, + "&&", "echo", env_marker]) + + # execute the command + execute = subprocess.Popen(command, **kwarg) + std_out, stderr = execute.communicate() + code = execute.returncode + + # wait for process to be done + execute.wait() + + # if collect enviroment variables + if collect_env: + # get the new environment variables + std_out, env = GetEnvironmentVariables(std_out, env_marker) + return (std_out, stderr, env, code) + +def GetEnvironmentVariables(std_out_str, marker): + """Gets the environment variables from a process + + :param std_out_str: The std_out pipe + :type std_out_str: String + :param marker: A begining and end mark of environment + variables printed to std_out + :type marker: String + :returns: The environment variables read from the process' std_out pipe + :rtype: Tuple + """ + start_env_update = False + environment_vars = {} + out_put = "" + for line in std_out_str.split("\n"): + if start_env_update and len(line.split("=")) == 2: + key, value = line.split("=") + environment_vars[key] = value + else: + out_put += "\n" + line.replace(marker, "") + + if marker in line: + if start_env_update: + start_env_update = False + else: + start_env_update = True + return (out_put, environment_vars) + +def EndDelFile(EnvList): + WORKDIR = EnvList['WORKDIR'] + ModifyFileListLog = os.path.join(WORKDIR, 'PatchModifyFiles.log') + RemoveFile(ModifyFileListLog) + PatchLog = os.path.join(WORKDIR, 'PatchFile.log') + RemoveFile(PatchLog) + FileLog = os.path.join(WORKDIR, "File.log") + RemoveFile(FileLog) + if GlobalSymbol.get('GenerateEccReport'): + FileList = os.listdir(WORKDIR) + CsvList = [os.path.join(WORKDIR, File) for File in FileList if File.endswith('.csv')] + for Csv in CsvList: + RemoveFile(Csv) + if GlobalSymbol.get('SCRIPT_ERROR'): + print('ECC tool detect error') + exit(1) + +def EdksetupRebuild(EnvList, WORKDIR): + EnvList['WORKDIR'] = WORKDIR + EnvList['WORKSPACE'] = WORKDIR + edk2_setup_cmd = ["edksetup", "Rebuild"] + _, _, result, return_code = ExecuteScript(edk2_setup_cmd, EnvList, collect_env=True, shell=True) + if return_code == 0: + for Key in result: + EnvList[Key] = result[Key] + return True + return False + +def GetDiffrange(EnvList, commit): + WORKDIR = EnvList['WORKDIR'] + ModifyFileListLog = EnvList['ModifyFileListLog'] + RangeDirectory = {} + PatchLog = os.path.join(WORKDIR, 'PatchFile.log') + Format_Patch_cmd = ["git", "show", str(commit), "--unified=0", ">", PatchLog] + _, _, result, return_code = ExecuteScript(Format_Patch_cmd, EnvList, shell=True) + if return_code != 0: + print('Fail to run GIT') + GlobalSymbol['SCRIPT_ERROR'] = True + EndDelFile(EnvList) + with open(PatchLog, encoding='utf8') as PatchLogFile: + Filelines = PatchLogFile.readlines() + IsDelete = True + StartCheck = False + for line in Filelines: + ModifyFile = FindModifyFile.findall(line) + if ModifyFile and not StartCheck and os.path.isfile(ModifyFile[0]): + ModifyFileCommentDic = GetCommentRange(EnvList, ModifyFile[0]) + IsDelete = False + StartCheck = True + ModifyFilieDic = ModifyFile[0] + ModifyFilieDic = ModifyFilieDic.replace("/","\\") + RangeDirectory[ModifyFilieDic] = [] + elif line.startswith('--- '): + StartCheck = False + elif re.match(LineScopePattern, line, re.I) and not IsDelete and StartCheck: + startline = LineNumRange.search(line).group(1) + linerange = LineNumRange.search(line).group(2) + if not linerange: + linerange = '1' + RangeDirectory[ModifyFilieDic].append((int(startline), int(startline) + int(linerange) - 1)) + for i in ModifyFileCommentDic: + if i[0] <= int(startline) <= i[1]: + RangeDirectory[ModifyFilieDic].append(i) + return RangeDirectory + +def GetCommentRange(EnvList, ModifyFile): + WORKDIR = EnvList['WORKDIR'] + ModifyFilePath = os.path.join(WORKDIR, ModifyFile) + with open(ModifyFilePath) as f: + LineNo = 1 + CommentRange = [] + Start = False + for line in f: + if line.startswith('/**'): + startno = LineNo + Start = True + if line.startswith('**/') and Start: + endno = LineNo + Start = False + CommentRange.append((int(startno), int(endno))) + LineNo += 1 + + if CommentRange and CommentRange[0][0] == 1: + del CommentRange[0] + return CommentRange + +def GetModifyDir(EnvList, commit): + WORKDIR = EnvList['WORKDIR'] + ModifyDirList = [] + ModifyFileListLog = os.path.join(WORKDIR, 'PatchModifyFiles.log') + EnvList['ModifyFileListLog'] = ModifyFileListLog + Patch_Modify_cmd = ["git", "diff", "--name-status", str(commit), str(commit)+"~1", ">", ModifyFileListLog] + _, _, result, return_code = ExecuteScript(Patch_Modify_cmd, EnvList, shell=True) + if return_code != 0: + print('Fail to run GIT') + GlobalSymbol['SCRIPT_ERROR'] = True + EndDelFile(EnvList) + with open(ModifyFileListLog) as ModifyFile: + Filelines = ModifyFile.readlines() + for Line in Filelines: + FilePath = ReModifyFile.findall(Line) + if FilePath: + FileDir = os.path.dirname(FilePath[0]) + else: + continue + PkgList = GetPkgList(EnvList) + if FileDir in PkgList or not FileDir: + continue + else: + ModifyDirList.append('%s'%FileDir) + + ModifyDirList = list(set(ModifyDirList)) + return ModifyDirList + +def ApplyConfig(EnvList, ModifyDirList, EccDiffRange): + WORKDIR = EnvList['WORKDIR'] + ModifyPkgList = [] + for ModifyDir in ModifyDirList: + ModifyPkg = ModifyDir.split("/")[0] + ModifyPkgList.append(ModifyPkg) + ModifyPkgList = list(set(ModifyPkgList)) + for ModifyPkg in ModifyPkgList: + pkg_config_file = os.path.join(WORKDIR, ModifyPkg, ModifyPkg + ".ci.yaml") + if os.path.exists(pkg_config_file): + with open(pkg_config_file, 'r') as f: + pkg_config = yaml.safe_load(f) + if "EccCheck" in pkg_config: + EccConfig = pkg_config["EccCheck"] + # + # Add exceptions + # + ExceptionList = EccConfig["ExceptionList"] + ExceptionFile = os.path.join(WORKDIR, "BaseTools", "Source", "Python", "Ecc", "exception.xml") + if os.path.exists(ExceptionFile): + AppendException(ExceptionList, ExceptionFile) + # + # Exclude ignored files + # + IgnoreFilesList = EccConfig['IgnoreFiles'] + for ignore_file in IgnoreFilesList: + ignore_file = ignore_file.replace("/", "\\") + ignore_file = os.path.join(ModifyPkg, ignore_file) + if ignore_file in EccDiffRange: + del EccDiffRange[ignore_file] + return + +def CheckOneCommit(commit): + WORKDIR = os.getcwd() + EdksetupRebuild(EnvList, WORKDIR) + ModifyDirList = GetModifyDir(EnvList, commit) + EccDiffRange = GetDiffrange(EnvList, commit) + ApplyConfig(EnvList, ModifyDirList, EccDiffRange) + GenerateEccReport(EnvList, ModifyDirList, EccDiffRange) + EndDelFile(EnvList) + +def parse_options(): + parser = argparse.ArgumentParser(description=__copyright__) + parser.add_argument('commits', nargs='*', + help='[commit(s) ID | number of commits, like "-3" means check first 3 commits]') + args = parser.parse_args() + return args + +def process_one_arg(self, arg): + if len(arg) >= 2 and arg[0] == '-': + count = int(arg[1:]) + CheckOneArg(arg, self.count).ok + + +def read_commit_list_from_git(start_commit, count): + cmd = ['git', 'rev-list', '--abbrev-commit', '--no-walk' ] + if count is not None: + cmd.append('--max-count=' + str(count)) + cmd.append(start_commit) + p = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + result = p.communicate() + out = result[0].decode('utf-8', 'ignore') if result[0] and result[0].find(b"fatal")!=0 else None + return out.split() if out else [] + +def ReleaseReport(EnvList): + WORKDIR = EnvList['WORKDIR'] + EccLog = os.path.join(WORKDIR, "ECC.log") + if GlobalSymbol['ECC_PASS']: + print('\n===================Ecc pass===================') + RemoveFile(EccLog) + return 0 + else: + print('\n===================Ecc error detected===================') + with open(EccLog) as output: + print(output.read()) + RemoveFile(EccLog) + return -1 + +def main(): + commits = parse_options().commits + + if len(commits) == 0: + commits = [ 'HEAD' ] + + if len(commits[0]) >= 2 and commits[0][0] == '-': + count = int(commits[0][1:]) + commits = read_commit_list_from_git('HEAD', count) + + """ + This 'if' block is only used for creating pull request. + """ + if ".." in commits[0]: + commits = read_commit_list_from_git(commits[0], None) + + GlobalSymbol['ECC_PASS'] = True + for commit in commits: + CheckOneCommit(commit) + + retval = ReleaseReport(EnvList) + return retval + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file -- 2.18.0.windows.1