Re: [PATCH V3] BaseTools:Trim will trig exception when input asl UTF8 format file

["Gao, Liming" <liming.gao@intel.com>]

Reviewed-by: Liming Gao <liming.gao@intel.com>

> -----Original Message-----
> From: Fan, ZhijuX
> Sent: Thursday, March 28, 2019 11:21 AM
> To: edk2-devel@lists.01.org
> Cc: Gao, Liming <liming.gao@intel.com>; Feng, Bob C <bob.c.feng@intel.com>
> Subject: [edk2][PATCH V3] BaseTools:Trim will trig exception when input asl UTF8 format file
> 
> BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=1641
> 
> The command trim --asl-file -o test.i UTF8.asl will trig the exception.
> There's a problem with the encoding of the file,it only appears in python3.
> I changed the way I opened it to support reading this file
> 
> Cc: Bob Feng <bob.c.feng@intel.com>
> Cc: Liming Gao <liming.gao@intel.com>
> Contributed-under: TianoCore Contribution Agreement 1.1
> Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com>
> ---
>  BaseTools/Source/Python/Trim/Trim.py | 44 +++++++++++++++++-------------------
>  1 file changed, 21 insertions(+), 23 deletions(-)
> 
> diff --git a/BaseTools/Source/Python/Trim/Trim.py b/BaseTools/Source/Python/Trim/Trim.py
> index 228779b5a9..05feab6abe 100644
> --- a/BaseTools/Source/Python/Trim/Trim.py
> +++ b/BaseTools/Source/Python/Trim/Trim.py
> @@ -18,7 +18,7 @@ import Common.LongFilePathOs as os
>  import sys
>  import re
>  from io import BytesIO
> -
> +import codecs
>  from optparse import OptionParser
>  from optparse import make_option
>  from Common.BuildToolError import *
> @@ -77,14 +77,11 @@ gIncludedAslFile = []
>  def TrimPreprocessedFile(Source, Target, ConvertHex, TrimLong):
>      CreateDirectory(os.path.dirname(Target))
>      try:
> -        f = open (Source, 'r')
> +        with open(Source, "r") as File:
> +            Lines = File.readlines()
>      except:
>          EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
> 
> -    # read whole file
> -    Lines = f.readlines()
> -    f.close()
> -
>      PreprocessedFile = ""
>      InjectedFile = ""
>      LineIndexOfOriginalFile = None
> @@ -181,11 +178,10 @@ def TrimPreprocessedFile(Source, Target, ConvertHex, TrimLong):
> 
>      # save to file
>      try:
> -        f = open (Target, 'w')
> +        with open(Target, 'w') as File:
> +            File.writelines(NewLines)
>      except:
>          EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
> -    f.writelines(NewLines)
> -    f.close()
> 
>  ## Trim preprocessed VFR file
>  #
> @@ -199,12 +195,11 @@ def TrimPreprocessedVfr(Source, Target):
>      CreateDirectory(os.path.dirname(Target))
> 
>      try:
> -        f = open (Source, 'r')
> +        with open(Source, "r") as File:
> +            Lines = File.readlines()
>      except:
>          EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
>      # read whole file
> -    Lines = f.readlines()
> -    f.close()
> 
>      FoundTypedef = False
>      Brace = 0
> @@ -248,11 +243,10 @@ def TrimPreprocessedVfr(Source, Target):
> 
>      # save all lines trimmed
>      try:
> -        f = open (Target, 'w')
> +        with open(Target, 'w') as File:
> +            File.writelines(Lines)
>      except:
>          EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
> -    f.writelines(Lines)
> -    f.close()
> 
>  ## Read the content  ASL file, including ASL included, recursively
>  #
> @@ -278,7 +272,12 @@ def DoInclude(Source, Indent='', IncludePathList=[], LocalSearchPath=None):
>          for IncludePath in SearchPathList:
>              IncludeFile = os.path.join(IncludePath, Source)
>              if os.path.isfile(IncludeFile):
> -                F = open(IncludeFile, "r")
> +                try:
> +                    with open(IncludeFile, "r") as File:
> +                        F = File.readlines()
> +                except:
> +                    with codecs.open(IncludeFile, "r", encoding='utf-8') as File:
> +                        F = File.readlines()
>                  break
>          else:
>              EdkLogger.error("Trim", "Failed to find include file %s" % Source)
> @@ -313,7 +312,6 @@ def DoInclude(Source, Indent='', IncludePathList=[], LocalSearchPath=None):
>          NewFileContent.append("\n")
> 
>      gIncludedAslFile.pop()
> -    F.close()
> 
>      return NewFileContent
> 
> @@ -345,7 +343,9 @@ def TrimAslFile(Source, Target, IncludePathFile):
>      if IncludePathFile:
>          try:
>              LineNum = 0
> -            for Line in open(IncludePathFile, 'r'):
> +            with open(IncludePathFile, 'r') as File:
> +                FileLines = File.readlines()
> +            for Line in FileLines:
>                  LineNum += 1
>                  if Line.startswith("/I") or Line.startswith ("-I"):
>                      IncludePathList.append(Line[2:].strip())
> @@ -363,13 +363,11 @@ def TrimAslFile(Source, Target, IncludePathFile):
> 
>      # save all lines trimmed
>      try:
> -        f = open (Target, 'w')
> +        with open(Target, 'w') as File:
> +            File.writelines(Lines)
>      except:
>          EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
> 
> -    f.writelines(Lines)
> -    f.close()
> -
>  def GenerateVfrBinSec(ModuleName, DebugDir, OutputFile):
>      VfrNameList = []
>      if os.path.isdir(DebugDir):
> @@ -389,7 +387,7 @@ def GenerateVfrBinSec(ModuleName, DebugDir, OutputFile):
>          return
> 
>      try:
> -        fInputfile = open(OutputFile, "wb+", 0)
> +        fInputfile = open(OutputFile, "wb+")
>      except:
>          EdkLogger.error("Trim", FILE_OPEN_FAILURE, "File open failed for %s" %OutputFile, None)
> 
> --
> 2.14.1.windows.1