From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=134.134.136.65; helo=mga03.intel.com; envelope-from=liming.gao@intel.com; receiver=edk2-devel@lists.01.org Received: from mga03.intel.com (mga03.intel.com [134.134.136.65]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ml01.01.org (Postfix) with ESMTPS id 3B6FE211DE90C for ; Wed, 27 Mar 2019 19:28:39 -0700 (PDT) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga003.fm.intel.com ([10.253.24.29]) by orsmga103.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 27 Mar 2019 19:28:39 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,278,1549958400"; d="scan'208";a="144493510" Received: from fmsmsx104.amr.corp.intel.com ([10.18.124.202]) by FMSMGA003.fm.intel.com with ESMTP; 27 Mar 2019 19:28:38 -0700 Received: from fmsmsx116.amr.corp.intel.com (10.18.116.20) by fmsmsx104.amr.corp.intel.com (10.18.124.202) with Microsoft SMTP Server (TLS) id 14.3.408.0; Wed, 27 Mar 2019 19:28:38 -0700 Received: from shsmsx101.ccr.corp.intel.com (10.239.4.153) by fmsmsx116.amr.corp.intel.com (10.18.116.20) with Microsoft SMTP Server (TLS) id 14.3.408.0; Wed, 27 Mar 2019 19:28:38 -0700 Received: from shsmsx104.ccr.corp.intel.com ([169.254.5.74]) by SHSMSX101.ccr.corp.intel.com ([169.254.1.158]) with mapi id 14.03.0415.000; Thu, 28 Mar 2019 10:28:36 +0800 From: "Gao, Liming" To: "Fan, ZhijuX" , "edk2-devel@lists.01.org" Thread-Topic: [edk2][PATCH V2] BaseTools:Trim will trig exception when input asl UTF8 format file Thread-Index: AdTlC7HkCa5emQtJRhSHoSEHlLQWLQAAjllA Date: Thu, 28 Mar 2019 02:28:35 +0000 Message-ID: <4A89E2EF3DFEDB4C8BFDE51014F606A14E40CE02@SHSMSX104.ccr.corp.intel.com> References: In-Reply-To: Accept-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-originating-ip: [10.239.127.40] MIME-Version: 1.0 Subject: Re: [PATCH V2] BaseTools:Trim will trig exception when input asl UTF8 format file X-BeenThere: edk2-devel@lists.01.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: EDK II Development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 28 Mar 2019 02:28:40 -0000 Content-Language: en-US Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable What error message is reported? >-----Original Message----- >From: Fan, ZhijuX >Sent: Thursday, March 28, 2019 10:13 AM >To: edk2-devel@lists.01.org >Cc: Gao, Liming ; Feng, Bob C >Subject: [edk2][PATCH V2] BaseTools:Trim will trig exception when input as= l >UTF8 format file > >BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=3D1641 > >The command trim --asl-file -o test.i UTF8.asl will trig the exception. >Trim tool should report error message for unsupported UTF8 file instead >of the exception. > >Cc: Bob Feng >Cc: Liming Gao >Contributed-under: TianoCore Contribution Agreement 1.1 >Signed-off-by: Zhiju.Fan >--- > BaseTools/Source/Python/Trim/Trim.py | 44 +++++++++++++++++------------ >------- > 1 file changed, 21 insertions(+), 23 deletions(-) > >diff --git a/BaseTools/Source/Python/Trim/Trim.py >b/BaseTools/Source/Python/Trim/Trim.py >index 228779b5a9..05feab6abe 100644 >--- a/BaseTools/Source/Python/Trim/Trim.py >+++ b/BaseTools/Source/Python/Trim/Trim.py >@@ -18,7 +18,7 @@ import Common.LongFilePathOs as os > import sys > import re > from io import BytesIO >- >+import codecs > from optparse import OptionParser > from optparse import make_option > from Common.BuildToolError import * >@@ -77,14 +77,11 @@ gIncludedAslFile =3D [] > def TrimPreprocessedFile(Source, Target, ConvertHex, TrimLong): > CreateDirectory(os.path.dirname(Target)) > try: >- f =3D open (Source, 'r') >+ with open(Source, "r") as File: >+ Lines =3D File.readlines() > except: > EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=3DSource) > >- # read whole file >- Lines =3D f.readlines() >- f.close() >- > PreprocessedFile =3D "" > InjectedFile =3D "" > LineIndexOfOriginalFile =3D None >@@ -181,11 +178,10 @@ def TrimPreprocessedFile(Source, Target, >ConvertHex, TrimLong): > > # save to file > try: >- f =3D open (Target, 'w') >+ with open(Target, 'w') as File: >+ File.writelines(NewLines) > except: > EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=3DTarget) >- f.writelines(NewLines) >- f.close() > > ## Trim preprocessed VFR file > # >@@ -199,12 +195,11 @@ def TrimPreprocessedVfr(Source, Target): > CreateDirectory(os.path.dirname(Target)) > > try: >- f =3D open (Source, 'r') >+ with open(Source, "r") as File: >+ Lines =3D File.readlines() > except: > EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=3DSource) > # read whole file >- Lines =3D f.readlines() >- f.close() > > FoundTypedef =3D False > Brace =3D 0 >@@ -248,11 +243,10 @@ def TrimPreprocessedVfr(Source, Target): > > # save all lines trimmed > try: >- f =3D open (Target, 'w') >+ with open(Target, 'w') as File: >+ File.writelines(Lines) > except: > EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=3DTarget) >- f.writelines(Lines) >- f.close() > > ## Read the content ASL file, including ASL included, recursively > # >@@ -278,7 +272,12 @@ def DoInclude(Source, Indent=3D'', IncludePathList=3D= [], >LocalSearchPath=3DNone): > for IncludePath in SearchPathList: > IncludeFile =3D os.path.join(IncludePath, Source) > if os.path.isfile(IncludeFile): >- F =3D open(IncludeFile, "r") >+ try: >+ with open(IncludeFile, "r") as File: >+ F =3D File.readlines() >+ except: >+ with codecs.open(IncludeFile, "r", encoding=3D'utf-8'= ) as File: >+ F =3D File.readlines() > break > else: > EdkLogger.error("Trim", "Failed to find include file %s" % So= urce) >@@ -313,7 +312,6 @@ def DoInclude(Source, Indent=3D'', IncludePathList=3D[= ], >LocalSearchPath=3DNone): > NewFileContent.append("\n") > > gIncludedAslFile.pop() >- F.close() > > return NewFileContent > >@@ -345,7 +343,9 @@ def TrimAslFile(Source, Target, IncludePathFile): > if IncludePathFile: > try: > LineNum =3D 0 >- for Line in open(IncludePathFile, 'r'): >+ with open(IncludePathFile, 'r') as File: >+ FileLines =3D File.readlines() >+ for Line in FileLines: > LineNum +=3D 1 > if Line.startswith("/I") or Line.startswith ("-I"): > IncludePathList.append(Line[2:].strip()) >@@ -363,13 +363,11 @@ def TrimAslFile(Source, Target, IncludePathFile): > > # save all lines trimmed > try: >- f =3D open (Target, 'w') >+ with open(Target, 'w') as File: >+ File.writelines(Lines) > except: > EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=3DTarget) > >- f.writelines(Lines) >- f.close() >- > def GenerateVfrBinSec(ModuleName, DebugDir, OutputFile): > VfrNameList =3D [] > if os.path.isdir(DebugDir): >@@ -389,7 +387,7 @@ def GenerateVfrBinSec(ModuleName, DebugDir, >OutputFile): > return > > try: >- fInputfile =3D open(OutputFile, "wb+", 0) >+ fInputfile =3D open(OutputFile, "wb+") > except: > EdkLogger.error("Trim", FILE_OPEN_FAILURE, "File open failed >for %s" %OutputFile, None) > >-- >2.14.1.windows.1