* [PATCH V3] BaseTools:Trim will trig exception when input asl UTF8 format file
@ 2019-03-28 3:20 Fan, ZhijuX
2019-03-29 12:30 ` Gao, Liming
0 siblings, 1 reply; 2+ messages in thread
From: Fan, ZhijuX @ 2019-03-28 3:20 UTC (permalink / raw)
To: edk2-devel@lists.01.org; +Cc: Gao, Liming, Feng, Bob C
BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=1641
The command trim --asl-file -o test.i UTF8.asl will trig the exception.
There's a problem with the encoding of the file,it only appears in python3.
I changed the way I opened it to support reading this file
Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com>
---
BaseTools/Source/Python/Trim/Trim.py | 44 +++++++++++++++++-------------------
1 file changed, 21 insertions(+), 23 deletions(-)
diff --git a/BaseTools/Source/Python/Trim/Trim.py b/BaseTools/Source/Python/Trim/Trim.py
index 228779b5a9..05feab6abe 100644
--- a/BaseTools/Source/Python/Trim/Trim.py
+++ b/BaseTools/Source/Python/Trim/Trim.py
@@ -18,7 +18,7 @@ import Common.LongFilePathOs as os
import sys
import re
from io import BytesIO
-
+import codecs
from optparse import OptionParser
from optparse import make_option
from Common.BuildToolError import *
@@ -77,14 +77,11 @@ gIncludedAslFile = []
def TrimPreprocessedFile(Source, Target, ConvertHex, TrimLong):
CreateDirectory(os.path.dirname(Target))
try:
- f = open (Source, 'r')
+ with open(Source, "r") as File:
+ Lines = File.readlines()
except:
EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
- # read whole file
- Lines = f.readlines()
- f.close()
-
PreprocessedFile = ""
InjectedFile = ""
LineIndexOfOriginalFile = None
@@ -181,11 +178,10 @@ def TrimPreprocessedFile(Source, Target, ConvertHex, TrimLong):
# save to file
try:
- f = open (Target, 'w')
+ with open(Target, 'w') as File:
+ File.writelines(NewLines)
except:
EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
- f.writelines(NewLines)
- f.close()
## Trim preprocessed VFR file
#
@@ -199,12 +195,11 @@ def TrimPreprocessedVfr(Source, Target):
CreateDirectory(os.path.dirname(Target))
try:
- f = open (Source, 'r')
+ with open(Source, "r") as File:
+ Lines = File.readlines()
except:
EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
# read whole file
- Lines = f.readlines()
- f.close()
FoundTypedef = False
Brace = 0
@@ -248,11 +243,10 @@ def TrimPreprocessedVfr(Source, Target):
# save all lines trimmed
try:
- f = open (Target, 'w')
+ with open(Target, 'w') as File:
+ File.writelines(Lines)
except:
EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
- f.writelines(Lines)
- f.close()
## Read the content ASL file, including ASL included, recursively
#
@@ -278,7 +272,12 @@ def DoInclude(Source, Indent='', IncludePathList=[], LocalSearchPath=None):
for IncludePath in SearchPathList:
IncludeFile = os.path.join(IncludePath, Source)
if os.path.isfile(IncludeFile):
- F = open(IncludeFile, "r")
+ try:
+ with open(IncludeFile, "r") as File:
+ F = File.readlines()
+ except:
+ with codecs.open(IncludeFile, "r", encoding='utf-8') as File:
+ F = File.readlines()
break
else:
EdkLogger.error("Trim", "Failed to find include file %s" % Source)
@@ -313,7 +312,6 @@ def DoInclude(Source, Indent='', IncludePathList=[], LocalSearchPath=None):
NewFileContent.append("\n")
gIncludedAslFile.pop()
- F.close()
return NewFileContent
@@ -345,7 +343,9 @@ def TrimAslFile(Source, Target, IncludePathFile):
if IncludePathFile:
try:
LineNum = 0
- for Line in open(IncludePathFile, 'r'):
+ with open(IncludePathFile, 'r') as File:
+ FileLines = File.readlines()
+ for Line in FileLines:
LineNum += 1
if Line.startswith("/I") or Line.startswith ("-I"):
IncludePathList.append(Line[2:].strip())
@@ -363,13 +363,11 @@ def TrimAslFile(Source, Target, IncludePathFile):
# save all lines trimmed
try:
- f = open (Target, 'w')
+ with open(Target, 'w') as File:
+ File.writelines(Lines)
except:
EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
- f.writelines(Lines)
- f.close()
-
def GenerateVfrBinSec(ModuleName, DebugDir, OutputFile):
VfrNameList = []
if os.path.isdir(DebugDir):
@@ -389,7 +387,7 @@ def GenerateVfrBinSec(ModuleName, DebugDir, OutputFile):
return
try:
- fInputfile = open(OutputFile, "wb+", 0)
+ fInputfile = open(OutputFile, "wb+")
except:
EdkLogger.error("Trim", FILE_OPEN_FAILURE, "File open failed for %s" %OutputFile, None)
--
2.14.1.windows.1
^ permalink raw reply related [flat|nested] 2+ messages in thread
* Re: [PATCH V3] BaseTools:Trim will trig exception when input asl UTF8 format file
2019-03-28 3:20 [PATCH V3] BaseTools:Trim will trig exception when input asl UTF8 format file Fan, ZhijuX
@ 2019-03-29 12:30 ` Gao, Liming
0 siblings, 0 replies; 2+ messages in thread
From: Gao, Liming @ 2019-03-29 12:30 UTC (permalink / raw)
To: Fan, ZhijuX, edk2-devel@lists.01.org
Reviewed-by: Liming Gao <liming.gao@intel.com>
> -----Original Message-----
> From: Fan, ZhijuX
> Sent: Thursday, March 28, 2019 11:21 AM
> To: edk2-devel@lists.01.org
> Cc: Gao, Liming <liming.gao@intel.com>; Feng, Bob C <bob.c.feng@intel.com>
> Subject: [edk2][PATCH V3] BaseTools:Trim will trig exception when input asl UTF8 format file
>
> BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=1641
>
> The command trim --asl-file -o test.i UTF8.asl will trig the exception.
> There's a problem with the encoding of the file,it only appears in python3.
> I changed the way I opened it to support reading this file
>
> Cc: Bob Feng <bob.c.feng@intel.com>
> Cc: Liming Gao <liming.gao@intel.com>
> Contributed-under: TianoCore Contribution Agreement 1.1
> Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com>
> ---
> BaseTools/Source/Python/Trim/Trim.py | 44 +++++++++++++++++-------------------
> 1 file changed, 21 insertions(+), 23 deletions(-)
>
> diff --git a/BaseTools/Source/Python/Trim/Trim.py b/BaseTools/Source/Python/Trim/Trim.py
> index 228779b5a9..05feab6abe 100644
> --- a/BaseTools/Source/Python/Trim/Trim.py
> +++ b/BaseTools/Source/Python/Trim/Trim.py
> @@ -18,7 +18,7 @@ import Common.LongFilePathOs as os
> import sys
> import re
> from io import BytesIO
> -
> +import codecs
> from optparse import OptionParser
> from optparse import make_option
> from Common.BuildToolError import *
> @@ -77,14 +77,11 @@ gIncludedAslFile = []
> def TrimPreprocessedFile(Source, Target, ConvertHex, TrimLong):
> CreateDirectory(os.path.dirname(Target))
> try:
> - f = open (Source, 'r')
> + with open(Source, "r") as File:
> + Lines = File.readlines()
> except:
> EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
>
> - # read whole file
> - Lines = f.readlines()
> - f.close()
> -
> PreprocessedFile = ""
> InjectedFile = ""
> LineIndexOfOriginalFile = None
> @@ -181,11 +178,10 @@ def TrimPreprocessedFile(Source, Target, ConvertHex, TrimLong):
>
> # save to file
> try:
> - f = open (Target, 'w')
> + with open(Target, 'w') as File:
> + File.writelines(NewLines)
> except:
> EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
> - f.writelines(NewLines)
> - f.close()
>
> ## Trim preprocessed VFR file
> #
> @@ -199,12 +195,11 @@ def TrimPreprocessedVfr(Source, Target):
> CreateDirectory(os.path.dirname(Target))
>
> try:
> - f = open (Source, 'r')
> + with open(Source, "r") as File:
> + Lines = File.readlines()
> except:
> EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
> # read whole file
> - Lines = f.readlines()
> - f.close()
>
> FoundTypedef = False
> Brace = 0
> @@ -248,11 +243,10 @@ def TrimPreprocessedVfr(Source, Target):
>
> # save all lines trimmed
> try:
> - f = open (Target, 'w')
> + with open(Target, 'w') as File:
> + File.writelines(Lines)
> except:
> EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
> - f.writelines(Lines)
> - f.close()
>
> ## Read the content ASL file, including ASL included, recursively
> #
> @@ -278,7 +272,12 @@ def DoInclude(Source, Indent='', IncludePathList=[], LocalSearchPath=None):
> for IncludePath in SearchPathList:
> IncludeFile = os.path.join(IncludePath, Source)
> if os.path.isfile(IncludeFile):
> - F = open(IncludeFile, "r")
> + try:
> + with open(IncludeFile, "r") as File:
> + F = File.readlines()
> + except:
> + with codecs.open(IncludeFile, "r", encoding='utf-8') as File:
> + F = File.readlines()
> break
> else:
> EdkLogger.error("Trim", "Failed to find include file %s" % Source)
> @@ -313,7 +312,6 @@ def DoInclude(Source, Indent='', IncludePathList=[], LocalSearchPath=None):
> NewFileContent.append("\n")
>
> gIncludedAslFile.pop()
> - F.close()
>
> return NewFileContent
>
> @@ -345,7 +343,9 @@ def TrimAslFile(Source, Target, IncludePathFile):
> if IncludePathFile:
> try:
> LineNum = 0
> - for Line in open(IncludePathFile, 'r'):
> + with open(IncludePathFile, 'r') as File:
> + FileLines = File.readlines()
> + for Line in FileLines:
> LineNum += 1
> if Line.startswith("/I") or Line.startswith ("-I"):
> IncludePathList.append(Line[2:].strip())
> @@ -363,13 +363,11 @@ def TrimAslFile(Source, Target, IncludePathFile):
>
> # save all lines trimmed
> try:
> - f = open (Target, 'w')
> + with open(Target, 'w') as File:
> + File.writelines(Lines)
> except:
> EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
>
> - f.writelines(Lines)
> - f.close()
> -
> def GenerateVfrBinSec(ModuleName, DebugDir, OutputFile):
> VfrNameList = []
> if os.path.isdir(DebugDir):
> @@ -389,7 +387,7 @@ def GenerateVfrBinSec(ModuleName, DebugDir, OutputFile):
> return
>
> try:
> - fInputfile = open(OutputFile, "wb+", 0)
> + fInputfile = open(OutputFile, "wb+")
> except:
> EdkLogger.error("Trim", FILE_OPEN_FAILURE, "File open failed for %s" %OutputFile, None)
>
> --
> 2.14.1.windows.1
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2019-03-29 12:30 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2019-03-28 3:20 [PATCH V3] BaseTools:Trim will trig exception when input asl UTF8 format file Fan, ZhijuX
2019-03-29 12:30 ` Gao, Liming
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox