[PATCH 2/5] BaseTools/Scripts: Add EccCheck.py

["Zhang, Shenglei" <shenglei.zhang@intel.com>]

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2606
EccCheck.py is a tool to report Ecc issues for commits, which will
be run on open ci.
But note not each kind of issue could be reported out.
It can only handle the issues, whose line number in CSV report
accurately map with their code in source code files. And comment
issues can also be handled.

Its usage is similar to PatchCheck.py. Type EccCheck.py -h and then
learn how to use it.

If a patch passes EccCheck, "Ecc Pass" will show up.
Otherwise, "Ecc error detected" alerts the users and the details
are also presented.

Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Signed-off-by: Shenglei Zhang <shenglei.zhang@intel.com>
---
 BaseTools/Scripts/EccCheck.py | 426 ++++++++++++++++++++++++++++++++++
 1 file changed, 426 insertions(+)
 create mode 100644 BaseTools/Scripts/EccCheck.py

diff --git a/BaseTools/Scripts/EccCheck.py b/BaseTools/Scripts/EccCheck.py
new file mode 100644
index 000000000000..479d2af8eac2
--- /dev/null
+++ b/BaseTools/Scripts/EccCheck.py
@@ -0,0 +1,426 @@
+## @file
+#  Check a patch for various format issues
+#
+#  Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
+#
+#  SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+
+import os
+import re
+import csv
+import subprocess
+import argparse
+import sys
+import yaml
+import xml.dom.minidom
+
+__copyright__ = "Copyright (c) 2020, Intel Corporation  All rights reserved."
+ReModifyFile = re.compile(r'[B-Q,S-Z]+[\d]*\t(.*?)\n')
+FindModifyFile = re.compile(r'\+\+\+ b\/(.*)')
+LineScopePattern = (r'@@ -\d*\,*\d* \+\d*\,*\d* @@.*')
+LineNumRange = re.compile(r'@@ -\d*\,*\d* \+(\d*)\,*(\d*) @@.*')
+
+EnvList = os.environ
+GlobalSymbol = {}
+
+def AppendException(ExceptionList, ExceptionFile):
+    error_code_list = ExceptionList[::2]
+    keyword_list = ExceptionList[1::2]
+    domTree = xml.dom.minidom.parse(ExceptionFile)
+    rootNode = domTree.documentElement
+    for error_code, keyword in zip(error_code_list, keyword_list):
+        customer_node = domTree.createElement("Exception")
+        keyword_node = domTree.createElement("KeyWord")
+        keyword_node_text_value = domTree.createTextNode(keyword)
+        keyword_node.appendChild(keyword_node_text_value)
+        customer_node.appendChild(keyword_node)
+        error_code_node = domTree.createElement("ErrorID")
+        error_code_text_value = domTree.createTextNode(error_code)
+        error_code_node.appendChild(error_code_text_value)
+        customer_node.appendChild(error_code_node)
+        rootNode.appendChild(customer_node)
+
+    with open(ExceptionFile, 'w') as f:
+        domTree.writexml(f,indent='',addindent='',newl='\n',encoding='UTF-8')
+
+def GetPkgList(EnvList):
+    WORKDIR = EnvList['WORKDIR']
+    dirs = os.listdir(WORKDIR)
+    PkgList = []
+    for directory in dirs:
+        if directory.endswith('Pkg'):
+            PkgList.append(directory)
+    return PkgList
+
+def GenerateEccReport(EnvList, ModifyDirList, EccDiffRange):
+    IsECCNeed = False
+    IsTrue = True
+    PkgList = GetPkgList(EnvList)
+
+    for Line in ModifyDirList:
+        print('echo Run ECC tool for the commit in %s' % Line)
+        GlobalSymbol['GenerateEccReport'] = True
+        for Pkg in PkgList:
+            if Pkg in Line:
+                IsECCNeed = True
+                Ecc_cmd = ["ecc", "-c", "%WORKDIR%\BaseTools\Source\Python\Ecc\config.ini",
+                                  "-e", "%WORKDIR%\BaseTools\Source\Python\Ecc\exception.xml",
+                                  "-t", "%WORKDIR%\{}".format(Line),
+                                  "-r", "%WORKDIR%\Ecc.csv"]
+                _, _, result, return_code = ExecuteScript(Ecc_cmd, EnvList, shell=True)
+                if return_code != 0:
+                    IsTrue = False
+                    break
+
+        if not IsTrue:
+            print('Fail to run ECC tool')
+            GlobalSymbol['SCRIPT_ERROR'] = True
+            EndDelFile(EnvList)
+
+        if GlobalSymbol.get('GenerateEccReport'):
+            ParseEccReport(EnvList, EccDiffRange)
+        else:
+            print("Patch check tool or ECC tool don't detect error")
+
+    if IsECCNeed:
+        revert_cmd = ["git", "checkout", "--", "%WORKDIR%\BaseTools\Source\Python\Ecc\exception.xml"]
+        _, _, result, return_code = ExecuteScript(revert_cmd, EnvList, shell=True)
+    else:
+        print("Doesn't need run ECC check")
+        return
+
+def ParseEccReport(EnvList, EccDiffRange):
+    WorkDir = EnvList['WORKDIR']
+    EccLog = os.path.join(WorkDir, "Ecc.log")
+    EccsCsv = "Ecc.csv"
+    FileList = os.listdir(WorkDir)
+    rowLines = []
+    if EccsCsv in FileList:
+        with open(EccsCsv) as Csv:
+            reader = csv.reader(Csv)
+            for row in reader:
+                for ModifyFile in EccDiffRange:
+                    if ModifyFile in row[3]:
+                        for i in EccDiffRange[ModifyFile]:
+                            LineNo = int(row[4])
+                            if i[0] <= LineNo <= i[1]:
+                                row[0] = '\nEFI coding style error'
+                                row[1] = 'Error code: ' + row[1]
+                                row[3] = 'File: ' + row[3]
+                                row[4] = 'Line number: ' + row[4]
+                                rowLine = '\n  *'.join(row)
+                                rowLines.append(rowLine)
+                                break
+                        break
+    if rowLines:
+        GlobalSymbol['ECC_PASS'] = False
+
+    with open(EccLog, 'a') as EccFile:
+            AllLine = '\n'.join(rowLines)
+            AllLine = AllLine + '\n'
+            EccFile.writelines(AllLine)
+
+def RemoveFile(File):
+    if os.path.exists(File):
+        os.remove(File)
+
+def ExecuteScript(command, env_variables, collect_env=False,
+                  enable_std_pipe=False, shell=True):
+    """launches a process that executes a script/shell command passed to it
+
+        :param command: The command/script with its commandline
+            arguments to be executed
+        :type command:  List:String
+        :param env_variables: Environment variables passed to the process
+        :type env_variables: String
+        :param collect_env: Enables the collection of evironment variables
+            when process execution is done
+        :type collect_env: Boolean
+        :param enable_std_pipe: Enables process out to be piped to
+        :type enable_std_pipe: String
+        :returns: a tuple of std_out, stderr , environment variables,
+            return code
+        :rtype: Tuple: (std_out, stderr , enVar, return_code)
+    """
+
+    env_marker = '-----env-----'
+    env = {}
+    kwarg = {"env": env_variables,
+             "universal_newlines": True,
+             "shell": shell,
+             "cwd": env_variables["WORKSPACE"]}
+
+    if enable_std_pipe or collect_env:
+        kwarg["stdout"] = subprocess.PIPE
+        kwarg["stderr"] = subprocess.PIPE
+
+    if collect_env:
+        # get the binary that prints environment variables based on os
+        if os.name == 'nt':
+            get_var_command = "set"
+        else:
+            get_var_command = "env"
+        # modify the command to print the environment variables
+        if isinstance(command, list):
+            command += ["&&", "echo", env_marker, "&&",
+                        get_var_command, "&&", "echo", env_marker]
+        else:
+            command += " " + " ".join(["&&", "echo", env_marker,
+                                       "&&", get_var_command,
+                                       "&&", "echo", env_marker])
+
+    # execute the command
+    execute = subprocess.Popen(command, **kwarg)
+    std_out, stderr = execute.communicate()
+    code = execute.returncode
+
+    # wait for process to be done
+    execute.wait()
+
+    # if collect enviroment variables
+    if collect_env:
+        # get the new environment variables
+        std_out, env = GetEnvironmentVariables(std_out, env_marker)
+    return (std_out, stderr, env, code)
+
+def GetEnvironmentVariables(std_out_str, marker):
+    """Gets the environment variables from a process
+
+        :param std_out_str: The std_out pipe
+        :type std_out_str: String
+        :param marker: A begining and end mark of environment
+            variables printed to std_out
+        :type marker: String
+        :returns: The environment variables read from the process' std_out pipe
+        :rtype: Tuple
+    """
+    start_env_update = False
+    environment_vars = {}
+    out_put = ""
+    for line in std_out_str.split("\n"):
+        if start_env_update and len(line.split("=")) == 2:
+            key, value = line.split("=")
+            environment_vars[key] = value
+        else:
+            out_put += "\n" + line.replace(marker, "")
+
+        if marker in line:
+            if start_env_update:
+                start_env_update = False
+            else:
+                start_env_update = True
+    return (out_put, environment_vars)
+
+def EndDelFile(EnvList):
+    WORKDIR = EnvList['WORKDIR']
+    ModifyFileListLog = os.path.join(WORKDIR, 'PatchModifyFiles.log')
+    RemoveFile(ModifyFileListLog)
+    PatchLog = os.path.join(WORKDIR, 'PatchFile.log')
+    RemoveFile(PatchLog)
+    FileLog = os.path.join(WORKDIR, "File.log")
+    RemoveFile(FileLog)
+    if GlobalSymbol.get('GenerateEccReport'):
+        FileList = os.listdir(WORKDIR)
+        CsvList = [os.path.join(WORKDIR, File) for File in FileList if File.endswith('.csv')]
+        for Csv in CsvList:
+            RemoveFile(Csv)
+    if GlobalSymbol.get('SCRIPT_ERROR'):
+        print('ECC tool detect error')
+        exit(1)
+
+def EdksetupRebuild(EnvList, WORKDIR):
+    EnvList['WORKDIR'] = WORKDIR
+    EnvList['WORKSPACE'] = WORKDIR
+    edk2_setup_cmd = ["edksetup", "Rebuild"]
+    _, _, result, return_code = ExecuteScript(edk2_setup_cmd, EnvList, collect_env=True, shell=True)
+    if return_code == 0:
+        for Key in result:
+            EnvList[Key] = result[Key]
+        return True
+    return False
+
+def GetDiffrange(EnvList, commit):
+    WORKDIR = EnvList['WORKDIR']
+    ModifyFileListLog = EnvList['ModifyFileListLog']
+    RangeDirectory = {}
+    PatchLog = os.path.join(WORKDIR, 'PatchFile.log')
+    Format_Patch_cmd = ["git", "show", str(commit), "--unified=0", ">", PatchLog]
+    _, _, result, return_code = ExecuteScript(Format_Patch_cmd, EnvList, shell=True)
+    if return_code != 0:
+        print('Fail to run GIT')
+        GlobalSymbol['SCRIPT_ERROR'] = True
+        EndDelFile(EnvList)
+    with open(PatchLog, encoding='utf8') as PatchLogFile:
+        Filelines = PatchLogFile.readlines()
+        IsDelete = True
+        StartCheck = False
+        for line in Filelines:
+            ModifyFile = FindModifyFile.findall(line)
+            if ModifyFile and not StartCheck and os.path.isfile(ModifyFile[0]):
+                ModifyFileCommentDic = GetCommentRange(EnvList, ModifyFile[0])
+                IsDelete = False
+                StartCheck = True
+                ModifyFilieDic = ModifyFile[0]
+                ModifyFilieDic = ModifyFilieDic.replace("/","\\")
+                RangeDirectory[ModifyFilieDic] = []
+            elif line.startswith('--- '):
+                StartCheck = False
+            elif re.match(LineScopePattern, line, re.I) and not IsDelete and StartCheck:
+                startline = LineNumRange.search(line).group(1)
+                linerange = LineNumRange.search(line).group(2)
+                if not linerange:
+                    linerange = '1'
+                RangeDirectory[ModifyFilieDic].append((int(startline), int(startline) + int(linerange) - 1))
+                for i in ModifyFileCommentDic:
+                    if i[0] <= int(startline) <= i[1]:
+                        RangeDirectory[ModifyFilieDic].append(i)
+    return RangeDirectory
+
+def GetCommentRange(EnvList, ModifyFile):
+    WORKDIR = EnvList['WORKDIR']
+    ModifyFilePath = os.path.join(WORKDIR, ModifyFile)
+    with open(ModifyFilePath) as f:
+        LineNo = 1
+        CommentRange = []
+        Start = False
+        for line in f:
+            if line.startswith('/**'):
+                startno = LineNo
+                Start = True
+            if line.startswith('**/') and Start:
+                endno = LineNo
+                Start = False
+                CommentRange.append((int(startno), int(endno)))
+            LineNo += 1
+
+    if CommentRange and CommentRange[0][0] == 1:
+        del CommentRange[0]
+    return CommentRange
+
+def GetModifyDir(EnvList, commit):
+    WORKDIR = EnvList['WORKDIR']
+    ModifyDirList = []
+    ModifyFileListLog = os.path.join(WORKDIR, 'PatchModifyFiles.log')
+    EnvList['ModifyFileListLog'] = ModifyFileListLog
+    Patch_Modify_cmd = ["git", "diff", "--name-status", str(commit), str(commit)+"~1", ">", ModifyFileListLog]
+    _, _, result, return_code = ExecuteScript(Patch_Modify_cmd, EnvList, shell=True)
+    if return_code != 0:
+        print('Fail to run GIT')
+        GlobalSymbol['SCRIPT_ERROR'] = True
+        EndDelFile(EnvList)
+    with open(ModifyFileListLog) as ModifyFile:
+        Filelines = ModifyFile.readlines()
+    for Line in Filelines:
+        FilePath = ReModifyFile.findall(Line)
+        if FilePath:
+            FileDir = os.path.dirname(FilePath[0])
+        else:
+            continue
+        PkgList = GetPkgList(EnvList)
+        if FileDir in PkgList or not FileDir:
+            continue
+        else:
+            ModifyDirList.append('%s'%FileDir)
+
+    ModifyDirList = list(set(ModifyDirList))
+    return ModifyDirList
+
+def ApplyConfig(EnvList, ModifyDirList, EccDiffRange):
+    WORKDIR = EnvList['WORKDIR']
+    ModifyPkgList = []
+    for ModifyDir in ModifyDirList:
+        ModifyPkg = ModifyDir.split("/")[0]
+        ModifyPkgList.append(ModifyPkg)
+    ModifyPkgList = list(set(ModifyPkgList))
+    for ModifyPkg in ModifyPkgList:
+        pkg_config_file = os.path.join(WORKDIR, ModifyPkg, ModifyPkg + ".ci.yaml")
+        if os.path.exists(pkg_config_file):
+            with open(pkg_config_file, 'r') as f:
+                pkg_config = yaml.safe_load(f)
+            if "EccCheck" in pkg_config:
+                EccConfig = pkg_config["EccCheck"]
+                #
+                # Add exceptions
+                #
+                ExceptionList = EccConfig["ExceptionList"]
+                ExceptionFile = os.path.join(WORKDIR, "BaseTools", "Source", "Python", "Ecc", "exception.xml")
+                if os.path.exists(ExceptionFile):
+                    AppendException(ExceptionList, ExceptionFile)
+                #
+                # Exclude ignored files
+                #
+                IgnoreFilesList = EccConfig['IgnoreFiles']
+                for ignore_file in IgnoreFilesList:
+                    ignore_file = ignore_file.replace("/", "\\")
+                    ignore_file = os.path.join(ModifyPkg, ignore_file)
+                    if ignore_file in EccDiffRange:
+                        del EccDiffRange[ignore_file]
+    return
+
+def CheckOneCommit(commit):
+    WORKDIR = os.getcwd()
+    EdksetupRebuild(EnvList, WORKDIR)
+    ModifyDirList = GetModifyDir(EnvList, commit)
+    EccDiffRange = GetDiffrange(EnvList, commit)
+    ApplyConfig(EnvList, ModifyDirList, EccDiffRange)
+    GenerateEccReport(EnvList, ModifyDirList, EccDiffRange)
+    EndDelFile(EnvList)
+
+def parse_options():
+    parser = argparse.ArgumentParser(description=__copyright__)
+    parser.add_argument('commits', nargs='*',
+                        help='[commit(s) ID | number of commits, like "-3" means check first 3 commits]')
+    args = parser.parse_args()
+    return args
+
+def process_one_arg(self, arg):
+        if len(arg) >= 2 and arg[0] == '-':
+            count = int(arg[1:])
+        CheckOneArg(arg, self.count).ok
+
+
+def read_commit_list_from_git(start_commit, count):
+    cmd = ['git', 'rev-list', '--abbrev-commit', '--no-walk' ]
+    cmd.append('--max-count=' + str(count))
+    cmd.append(start_commit)
+    p = subprocess.Popen(cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT)
+    result = p.communicate()
+    out = result[0].decode('utf-8', 'ignore') if result[0] and result[0].find(b"fatal")!=0 else None
+    return out.split() if out else []
+
+def ReleaseReport(EnvList):
+        WORKDIR = EnvList['WORKDIR']
+        EccLog = os.path.join(WORKDIR, "ECC.log")
+        if GlobalSymbol['ECC_PASS']:
+            print('\n===================Ecc pass===================')
+            RemoveFile(EccLog)
+            return 0
+        else:
+            print('\n===================Ecc error detected===================')
+            with open(EccLog) as output:
+                print(output.read())
+            RemoveFile(EccLog)
+            return -1
+
+def main():
+    commits = parse_options().commits
+
+    if len(commits) == 0:
+        commits = [ 'HEAD' ]
+
+    if len(commits[0]) >= 2 and commits[0][0] == '-':
+        count = int(commits[0][1:])
+        commits = read_commit_list_from_git('HEAD', count)
+
+    GlobalSymbol['ECC_PASS'] = True
+    for commit in commits:
+        CheckOneCommit(commit)
+
+    retval = ReleaseReport(EnvList)
+    return retval
+
+if __name__ == "__main__":
+    sys.exit(main())
\ No newline at end of file
-- 
2.18.0.windows.1