From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=134.134.136.65; helo=mga03.intel.com; envelope-from=zhijux.fan@intel.com; receiver=edk2-devel@lists.01.org Received: from mga03.intel.com (mga03.intel.com [134.134.136.65]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ml01.01.org (Postfix) with ESMTPS id 18749211E7437 for ; Fri, 22 Mar 2019 00:55:07 -0700 (PDT) X-Amp-Result: UNSCANNABLE X-Amp-File-Uploaded: False Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by orsmga103.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 22 Mar 2019 00:55:06 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,256,1549958400"; d="dat'59?scan'59,208,59";a="330950635" Received: from fmsmsx104.amr.corp.intel.com ([10.18.124.202]) by fmsmga005.fm.intel.com with ESMTP; 22 Mar 2019 00:55:06 -0700 Received: from shsmsx104.ccr.corp.intel.com (10.239.4.70) by fmsmsx104.amr.corp.intel.com (10.18.124.202) with Microsoft SMTP Server (TLS) id 14.3.408.0; Fri, 22 Mar 2019 00:55:06 -0700 Received: from shsmsx101.ccr.corp.intel.com ([169.254.1.158]) by SHSMSX104.ccr.corp.intel.com ([169.254.5.74]) with mapi id 14.03.0415.000; Fri, 22 Mar 2019 15:55:04 +0800 From: "Fan, ZhijuX" To: "edk2-devel@lists.01.org" CC: "Gao, Liming" , "Feng, Bob C" Thread-Topic: [edk2][PATCH] BaseTools:Trim will trig exception when input asl UTF8 format file Thread-Index: AdTghI0MfPf4/Y1cR/mEjpCLqsCt9g== Date: Fri, 22 Mar 2019 07:55:02 +0000 Message-ID: Accept-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: dlp-product: dlpe-windows dlp-version: 11.0.400.15 dlp-reaction: no-action x-originating-ip: [10.239.127.40] MIME-Version: 1.0 X-Content-Filtered-By: Mailman/MimeDel 2.1.29 Subject: [PATCH] BaseTools:Trim will trig exception when input asl UTF8 format file X-BeenThere: edk2-devel@lists.01.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: EDK II Development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 22 Mar 2019 07:55:08 -0000 Content-Language: en-US Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=3D1641 The command trim --asl-file -o test.i UTF8.asl will trig the exception. Trim tool should report error message for unsupported UTF8 file instead of the exception. Cc: Bob Feng Cc: Liming Gao Contributed-under: TianoCore Contribution Agreement 1.1 Signed-off-by: Zhiju.Fan --- BaseTools/Source/Python/Trim/Trim.py | 40 +++++++++++++++-----------------= ---- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/BaseTools/Source/Python/Trim/Trim.py b/BaseTools/Source/Python= /Trim/Trim.py index 825ed3e5d5..13d537f5aa 100644 --- a/BaseTools/Source/Python/Trim/Trim.py +++ b/BaseTools/Source/Python/Trim/Trim.py @@ -18,7 +18,7 @@ import Common.LongFilePathOs as os import sys import re from io import BytesIO - +import codecs from optparse import OptionParser from optparse import make_option from Common.BuildToolError import * @@ -141,14 +141,11 @@ gIncludedAslFile =3D [] def TrimPreprocessedFile(Source, Target, ConvertHex, TrimLong): CreateDirectory(os.path.dirname(Target)) try: - f =3D open (Source, 'r') + with codecs.open(Source, "r", encoding=3D"utf-8") as File: + Lines =3D File.readlines() except: EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=3DSource) =20 - # read whole file - Lines =3D f.readlines() - f.close() - PreprocessedFile =3D "" InjectedFile =3D "" LineIndexOfOriginalFile =3D None @@ -245,11 +242,10 @@ def TrimPreprocessedFile(Source, Target, ConvertHex, = TrimLong): =20 # save to file try: - f =3D open (Target, 'w') + with codecs.open(Target, 'w', encoding=3D"utf-8") as File: + File.writelines(Lines) except: EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=3DTarget) - f.writelines(NewLines) - f.close() =20 ## Trim preprocessed VFR file # @@ -263,12 +259,11 @@ def TrimPreprocessedVfr(Source, Target): CreateDirectory(os.path.dirname(Target)) =20 try: - f =3D open (Source, 'r') + with codecs.open(Source, "r", encoding=3D"utf-8") as File: + Lines =3D File.readlines() except: EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=3DSource) # read whole file - Lines =3D f.readlines() - f.close() =20 FoundTypedef =3D False Brace =3D 0 @@ -312,11 +307,10 @@ def TrimPreprocessedVfr(Source, Target): =20 # save all lines trimmed try: - f =3D open (Target, 'w') + with codecs.open(Target, 'w', encoding=3D"utf-8") as File: + File.writelines(Lines) except: EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=3DTarget) - f.writelines(Lines) - f.close() =20 ## Read the content ASL file, including ASL included, recursively # @@ -342,7 +336,8 @@ def DoInclude(Source, Indent=3D'', IncludePathList=3D[]= , LocalSearchPath=3DNone): for IncludePath in SearchPathList: IncludeFile =3D os.path.join(IncludePath, Source) if os.path.isfile(IncludeFile): - F =3D open(IncludeFile, "r") + with codecs.open(IncludeFile, "r", encoding=3D"utf-8") as = File: + F =3D File.readlines() break else: EdkLogger.error("Trim", "Failed to find include file %s" % Sou= rce) @@ -377,7 +372,6 @@ def DoInclude(Source, Indent=3D'', IncludePathList=3D[]= , LocalSearchPath=3DNone): NewFileContent.append("\n") =20 gIncludedAslFile.pop() - F.close() =20 return NewFileContent =20 @@ -409,7 +403,9 @@ def TrimAslFile(Source, Target, IncludePathFile): if IncludePathFile: try: LineNum =3D 0 - for Line in open(IncludePathFile, 'r'): + with codecs.open(IncludePathFile, 'r', encoding=3D"utf-8") as = File: + FileLines =3D File.readlines() + for Line in FileLines: LineNum +=3D 1 if Line.startswith("/I") or Line.startswith ("-I"): IncludePathList.append(Line[2:].strip()) @@ -427,13 +423,11 @@ def TrimAslFile(Source, Target, IncludePathFile): =20 # save all lines trimmed try: - f =3D open (Target, 'w') + with codecs.open(Target, 'w', encoding=3D"utf-8") as File: + File.writelines(Lines) except: EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=3DTarget) =20 - f.writelines(Lines) - f.close() - def GenerateVfrBinSec(ModuleName, DebugDir, OutputFile): VfrNameList =3D [] if os.path.isdir(DebugDir): @@ -453,7 +447,7 @@ def GenerateVfrBinSec(ModuleName, DebugDir, OutputFile)= : return =20 try: - fInputfile =3D open(OutputFile, "wb+", 0) + fInputfile =3D open(OutputFile, "wb+") except: EdkLogger.error("Trim", FILE_OPEN_FAILURE, "File open failed for %= s" %OutputFile, None) =20 --=20 2.14.1.windows.1