From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=134.134.136.31; helo=mga06.intel.com; envelope-from=zhijux.fan@intel.com; receiver=edk2-devel@lists.01.org Received: from mga06.intel.com (mga06.intel.com [134.134.136.31]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ml01.01.org (Postfix) with ESMTPS id 3C555211E9595 for ; Wed, 27 Mar 2019 20:20:54 -0700 (PDT) X-Amp-Result: UNSCANNABLE X-Amp-File-Uploaded: False Received: from fmsmga003.fm.intel.com ([10.253.24.29]) by orsmga104.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 27 Mar 2019 20:20:54 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,278,1549958400"; d="dat'59?scan'59,208,59";a="144502437" Received: from fmsmsx106.amr.corp.intel.com ([10.18.124.204]) by FMSMGA003.fm.intel.com with ESMTP; 27 Mar 2019 20:20:54 -0700 Received: from FMSMSX110.amr.corp.intel.com (10.18.116.10) by FMSMSX106.amr.corp.intel.com (10.18.124.204) with Microsoft SMTP Server (TLS) id 14.3.408.0; Wed, 27 Mar 2019 20:20:53 -0700 Received: from shsmsx105.ccr.corp.intel.com (10.239.4.158) by fmsmsx110.amr.corp.intel.com (10.18.116.10) with Microsoft SMTP Server (TLS) id 14.3.408.0; Wed, 27 Mar 2019 20:20:53 -0700 Received: from shsmsx101.ccr.corp.intel.com ([169.254.1.158]) by SHSMSX105.ccr.corp.intel.com ([169.254.11.113]) with mapi id 14.03.0415.000; Thu, 28 Mar 2019 11:20:51 +0800 From: "Fan, ZhijuX" To: "edk2-devel@lists.01.org" CC: "Gao, Liming" , "Feng, Bob C" Thread-Topic: [edk2][PATCH V3] BaseTools:Trim will trig exception when input asl UTF8 format file Thread-Index: AdTlFSvnOgCPcRRUQeOgkgNlPm2ssA== Date: Thu, 28 Mar 2019 03:20:50 +0000 Message-ID: Accept-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: dlp-product: dlpe-windows dlp-version: 11.0.400.15 dlp-reaction: no-action x-originating-ip: [10.239.127.40] MIME-Version: 1.0 X-Content-Filtered-By: Mailman/MimeDel 2.1.29 Subject: [PATCH V3] BaseTools:Trim will trig exception when input asl UTF8 format file X-BeenThere: edk2-devel@lists.01.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: EDK II Development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 28 Mar 2019 03:20:55 -0000 Content-Language: en-US Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=3D1641 The command trim --asl-file -o test.i UTF8.asl will trig the exception. There's a problem with the encoding of the file,it only appears in python3. I changed the way I opened it to support reading this file Cc: Bob Feng Cc: Liming Gao Contributed-under: TianoCore Contribution Agreement 1.1 Signed-off-by: Zhiju.Fan --- BaseTools/Source/Python/Trim/Trim.py | 44 +++++++++++++++++---------------= ---- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/BaseTools/Source/Python/Trim/Trim.py b/BaseTools/Source/Python= /Trim/Trim.py index 228779b5a9..05feab6abe 100644 --- a/BaseTools/Source/Python/Trim/Trim.py +++ b/BaseTools/Source/Python/Trim/Trim.py @@ -18,7 +18,7 @@ import Common.LongFilePathOs as os import sys import re from io import BytesIO - +import codecs from optparse import OptionParser from optparse import make_option from Common.BuildToolError import * @@ -77,14 +77,11 @@ gIncludedAslFile =3D [] def TrimPreprocessedFile(Source, Target, ConvertHex, TrimLong): CreateDirectory(os.path.dirname(Target)) try: - f =3D open (Source, 'r') + with open(Source, "r") as File: + Lines =3D File.readlines() except: EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=3DSource) =20 - # read whole file - Lines =3D f.readlines() - f.close() - PreprocessedFile =3D "" InjectedFile =3D "" LineIndexOfOriginalFile =3D None @@ -181,11 +178,10 @@ def TrimPreprocessedFile(Source, Target, ConvertHex, = TrimLong): =20 # save to file try: - f =3D open (Target, 'w') + with open(Target, 'w') as File: + File.writelines(NewLines) except: EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=3DTarget) - f.writelines(NewLines) - f.close() =20 ## Trim preprocessed VFR file # @@ -199,12 +195,11 @@ def TrimPreprocessedVfr(Source, Target): CreateDirectory(os.path.dirname(Target)) =20 try: - f =3D open (Source, 'r') + with open(Source, "r") as File: + Lines =3D File.readlines() except: EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=3DSource) # read whole file - Lines =3D f.readlines() - f.close() =20 FoundTypedef =3D False Brace =3D 0 @@ -248,11 +243,10 @@ def TrimPreprocessedVfr(Source, Target): =20 # save all lines trimmed try: - f =3D open (Target, 'w') + with open(Target, 'w') as File: + File.writelines(Lines) except: EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=3DTarget) - f.writelines(Lines) - f.close() =20 ## Read the content ASL file, including ASL included, recursively # @@ -278,7 +272,12 @@ def DoInclude(Source, Indent=3D'', IncludePathList=3D[= ], LocalSearchPath=3DNone): for IncludePath in SearchPathList: IncludeFile =3D os.path.join(IncludePath, Source) if os.path.isfile(IncludeFile): - F =3D open(IncludeFile, "r") + try: + with open(IncludeFile, "r") as File: + F =3D File.readlines() + except: + with codecs.open(IncludeFile, "r", encoding=3D'utf-8')= as File: + F =3D File.readlines() break else: EdkLogger.error("Trim", "Failed to find include file %s" % Sou= rce) @@ -313,7 +312,6 @@ def DoInclude(Source, Indent=3D'', IncludePathList=3D[]= , LocalSearchPath=3DNone): NewFileContent.append("\n") =20 gIncludedAslFile.pop() - F.close() =20 return NewFileContent =20 @@ -345,7 +343,9 @@ def TrimAslFile(Source, Target, IncludePathFile): if IncludePathFile: try: LineNum =3D 0 - for Line in open(IncludePathFile, 'r'): + with open(IncludePathFile, 'r') as File: + FileLines =3D File.readlines() + for Line in FileLines: LineNum +=3D 1 if Line.startswith("/I") or Line.startswith ("-I"): IncludePathList.append(Line[2:].strip()) @@ -363,13 +363,11 @@ def TrimAslFile(Source, Target, IncludePathFile): =20 # save all lines trimmed try: - f =3D open (Target, 'w') + with open(Target, 'w') as File: + File.writelines(Lines) except: EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=3DTarget) =20 - f.writelines(Lines) - f.close() - def GenerateVfrBinSec(ModuleName, DebugDir, OutputFile): VfrNameList =3D [] if os.path.isdir(DebugDir): @@ -389,7 +387,7 @@ def GenerateVfrBinSec(ModuleName, DebugDir, OutputFile)= : return =20 try: - fInputfile =3D open(OutputFile, "wb+", 0) + fInputfile =3D open(OutputFile, "wb+") except: EdkLogger.error("Trim", FILE_OPEN_FAILURE, "File open failed for %= s" %OutputFile, None) =20 --=20 2.14.1.windows.1