From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail05.groups.io (mail05.groups.io [45.79.224.7]) by spool.mail.gandi.net (Postfix) with ESMTPS id B7E4D740032 for ; Fri, 6 Dec 2024 01:41:12 +0000 (UTC) DKIM-Signature: a=rsa-sha256; bh=5m0Cz/LvmuRZFxat3InyptuHb1tKVbjmdJS1+Y78Oik=; c=relaxed/simple; d=groups.io; h=From:To:References:In-Reply-To:Subject:Date:Message-ID:MIME-Version:Thread-Index:Precedence:List-Subscribe:List-Help:Sender:List-Id:Mailing-List:Delivered-To:Resent-Date:Resent-From:Reply-To:List-Unsubscribe-Post:List-Unsubscribe:Content-Type:Content-Language; s=20240830; t=1733449272; v=1; x=1733708471; b=PpZFAzBHta8MqKty+1gvgswSyDkGlxBg4MyDMSffGd1hYsh8Bdafkl2mhW8vGehVHnQ67+ew g6+h2UC9RMD4TtvNVK3zdYabataucBZK8V/iWfVhtdp3Pc6DavGrSoJEQb790U/IvvcceXgRTRF PFKSXbHSS+4BhsClBpPRplgAu8EOufcMVLmPgTjwTsV3BEchnbpTVW+YEpv0XYR9o4OVZZVTIpZ 45EGejVtCgS3KE+k0vzg1BTk7wQ7qk0yATfRiMyECDIpZs4pk+zfhAP7RPmM9JU7don2U24GU65 Vt7eMA3GfSEzpxy7TeXuwMsgWeL1Zoj0UOx1rYFTFksVQ== X-Received: by 127.0.0.2 with SMTP id lSTXYY7687511xBm9ix42mAX; Thu, 05 Dec 2024 17:41:11 -0800 X-Received: from zrleap.intel-email.com (zrleap.intel-email.com [114.80.218.36]) by mx.groups.io with SMTP id smtpd.web11.27829.1733449269565810957 for ; Thu, 05 Dec 2024 17:41:10 -0800 X-Received: from zrleap.intel-email.com (localhost [127.0.0.1]) by zrleap.intel-email.com (Postfix) with ESMTP id BB654A32E180 for ; Fri, 6 Dec 2024 09:41:06 +0800 (CST) X-Received: from localhost (localhost [127.0.0.1]) by zrleap.intel-email.com (Postfix) with ESMTP id 9A25CA32E17D for ; Fri, 6 Dec 2024 09:41:06 +0800 (CST) X-Received: from mail.byosoft.com.cn (mail.byosoft.com.cn [58.240.74.242]) by zrleap.intel-email.com (Postfix) with SMTP id CC420A32E122 for ; Fri, 6 Dec 2024 09:41:03 +0800 (CST) X-Received: from DESKTOPS6D0PVI ([58.246.60.130]) (envelope-sender ) by 192.168.6.13 with ESMTP(SSL) for ; Fri, 06 Dec 2024 09:41:01 +0800 X-WM-Sender: gaoliming@byosoft.com.cn X-Originating-IP: 58.246.60.130 X-WM-AuthFlag: YES X-WM-AuthUser: gaoliming@byosoft.com.cn From: "gaoliming via groups.io" To: , References: <018e01db46af$bd125d00$37371700$@byosoft.com.cn> <15637.1733446486644216677@groups.io> In-Reply-To: <15637.1733446486644216677@groups.io> Subject: =?UTF-8?B?5Zue5aSNOiBbZWRrMi1kZXZlbF0g5Zue5aSNOiBbZWRrMi1kZXZlbF0gRkRGIHBhcnNlciBwZXJmb3JtYW5jZSBkZWdyYWRlcyByYXBpZGx5IG9uIG5vbi10cml2aWFsbHkgc2l6ZWQgaW5wdXRz?= Date: Fri, 6 Dec 2024 09:41:02 +0800 Message-ID: <006f01db477f$e8de19d0$ba9a4d70$@byosoft.com.cn> MIME-Version: 1.0 Thread-Index: AQHPNDK2Im9Kr/ZX4KCZbsaBF4NdSgHSJr4AsuIGKmA= Precedence: Bulk List-Subscribe: List-Help: Sender: devel@edk2.groups.io List-Id: Mailing-List: list devel@edk2.groups.io; contact devel+owner@edk2.groups.io Resent-Date: Thu, 05 Dec 2024 17:41:10 -0800 Resent-From: gaoliming@byosoft.com.cn Reply-To: devel@edk2.groups.io,gaoliming@byosoft.com.cn List-Unsubscribe-Post: List-Unsubscribe=One-Click List-Unsubscribe: X-Gm-Message-State: 7eMYOjdr5gaLDCeFnV9WGQLMx7686176AA= Content-Type: multipart/alternative; boundary="----=_NextPart_000_0070_01DB47C2.F703CAD0" Content-Language: zh-cn X-GND-Status: LEGIT Authentication-Results: spool.mail.gandi.net; dkim=pass header.d=groups.io header.s=20240830 header.b=PpZFAzBH; dmarc=pass (policy=none) header.from=groups.io; spf=pass (spool.mail.gandi.net: domain of bounce@groups.io designates 45.79.224.7 as permitted sender) smtp.mailfrom=bounce@groups.io ------=_NextPart_000_0070_01DB47C2.F703CAD0 Content-Type: text/plain; charset="UTF-8" Content-Transfer-Encoding: quoted-printable Thanks. I will try it.=20 =20 =E5=8F=91=E4=BB=B6=E4=BA=BA: via groups.io =20 =E5=8F=91=E9=80=81=E6=97=B6=E9=97=B4: 2024=E5=B9=B412=E6=9C=886=E6=97=A5 8:= 55 =E6=94=B6=E4=BB=B6=E4=BA=BA: gaoliming ; devel@ed= k2.groups.io =E4=B8=BB=E9=A2=98: Re: [edk2-devel] =E5=9B=9E=E5=A4=8D: [edk2-devel] FDF p= arser performance degrades rapidly on non-trivially sized inputs =20 Here is the code I was testing with. You can probably reduce the changes do= wn to just _SkipWhiteSpace(), but I haven't tried that yet. =20 diff --git a/BaseTools/Source/Python/GenFds/FdfParser.py b/BaseTools/Source= /Python/GenFds/FdfParser.py index feb4c72779..8c57720116 100644 --- a/BaseTools/Source/Python/GenFds/FdfParser.py +++ b/BaseTools/Source/Python/GenFds/FdfParser.py @@ -12,6 +12,7 @@ # from __future__ import print_function from __future__ import absolute_import +from io import StringIO from re import compile, DOTALL from string import hexdigits from uuid import UUID @@ -253,7 +254,7 @@ class FdfParser: self.CurrentFdName =3D None self.CurrentFvName =3D None self._Token =3D "" - self._SkippedChars =3D "" + self._SkippedChars =3D StringIO() GlobalData.gFdfParser =3D self =20 # Used to section info @@ -276,7 +277,7 @@ class FdfParser: def _SkipWhiteSpace(self): while not self._EndOfFile(): if self._CurrentChar() in {TAB_PRINTCHAR_NUL, T_CHAR_CR, TAB_L= INE_BREAK, TAB_SPACE_SPLIT, T_CHAR_TAB}: - self._SkippedChars +=3D str(self._CurrentChar()) + self._SkippedChars.write(str(self._CurrentChar())) self._GetOneChar() else: return @@ -696,7 +697,7 @@ class FdfParser: Header =3D self._Token if not self._Token.endswith(TAB_SECTION_END): self._SkipToToken(TAB_SECTION_END) - Header +=3D self._SkippedChars + Header +=3D self._SkippedChars.getvalue() if Header.find('$(') !=3D -1: raise Warning("macro cannot be used in section hea= der", self.FileName, self.CurrentLineNumber) self._SectionHeaderParser(Header) @@ -1226,7 +1227,7 @@ class FdfParser: raise Warning(QuoteToUse, self.FileName, self.CurrentLineNumbe= r) if currentLineNumber !=3D self.CurrentLineNumber: raise Warning(QuoteToUse, self.FileName, self.CurrentLineNumbe= r) - self._Token =3D self._SkippedChars.rstrip(QuoteToUse) + self._Token =3D self._SkippedChars.getvalue().rstrip(QuoteToUse) return True =20 ## _SkipToToken() method @@ -1243,7 +1244,7 @@ class FdfParser: def _SkipToToken(self, String, IgnoreCase =3D False): StartPos =3D self.GetFileBufferPos() =20 - self._SkippedChars =3D "" + self._SkippedChars =3D StringIO() while not self._EndOfFile(): index =3D -1 if IgnoreCase: @@ -1252,13 +1253,13 @@ class FdfParser: index =3D self._CurrentLine()[self.CurrentOffsetWithinLine= : ].find(String) if index =3D=3D 0: self.CurrentOffsetWithinLine +=3D len(String) - self._SkippedChars +=3D String + self._SkippedChars.write(String) return True - self._SkippedChars +=3D str(self._CurrentChar()) + self._SkippedChars.write(str(self._CurrentChar())) self._GetOneChar() =20 self.SetFileBufferPos(StartPos) - self._SkippedChars =3D "" + self._SkippedChars =3D StringIO() return False =20 ## GetFileBufferPos() method @@ -2890,7 +2891,7 @@ class FdfParser: if not self._SkipToToken(T_CHAR_BRACE_R): raise Warning.Expected("Depex expression ending '}'", self= .FileName, self.CurrentLineNumber) =20 - DepexSectionObj.Expression =3D self._SkippedChars.rstrip(T_CHA= R_BRACE_R) + DepexSectionObj.Expression =3D self._SkippedChars.getvalue().r= strip(T_CHAR_BRACE_R) Obj.SectionList.append(DepexSectionObj) =20 elif self._IsKeyword("SUBTYPE_GUID"): @@ -3525,7 +3526,7 @@ class FdfParser: if not self._SkipToToken(TAB_SPLIT): raise Warning.Expected("'.'", self.FileName, self.CurrentLineN= umber) =20 - Arch =3D self._SkippedChars.rstrip(TAB_SPLIT) + Arch =3D self._SkippedChars.getvalue().rstrip(TAB_SPLIT) =20 ModuleType =3D self._GetModuleType() =20 =20 =20 Also, if you would like to profile the build process you can apply this pat= ch. =20 diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Pyth= on/build/build.py index 51fb1f433e..396729efd9 100755 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -2778,12 +2778,18 @@ def Main(): Log_Agent.join() return ReturnCode =20 +import cProfile + if __name__ =3D=3D '__main__': try: mp.set_start_method('spawn') except: pass - r =3D Main() + + with cProfile.Profile() as pr: + r =3D Main() + pr.print_stats('tottime') + ## 0-127 is a safe return range, and 1 is a standard default error if r < 0 or r > 127: r =3D 1 sys.exit(r) -=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D- Groups.io Links: You receive all messages sent to this group. View/Reply Online (#120885): https://edk2.groups.io/g/devel/message/120885 Mute This Topic: https://groups.io/mt/109951333/7686176 Group Owner: devel+owner@edk2.groups.io Unsubscribe: https://edk2.groups.io/g/devel/unsub [rebecca@openfw.io] -=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D- ------=_NextPart_000_0070_01DB47C2.F703CAD0 Content-Type: text/html; charset="UTF-8" Content-Transfer-Encoding: quoted-printable

Thanks. I will try it.

 

=E5=8F=91=E4=BB=B6=E4=BA=BA:= via groups.io <davidr=3Dghs.com@groups.io>
=E5=8F=91=E9=80=81=E6= =97=B6=E9=97=B4: 2024=E5=B9=B412=E6=9C=886=E6=97=A5 8:55
=E6=94=B6=E4=BB=B6=E4=BA=BA:= gaoliming <gaoliming@byosoft.com.cn>; = devel@edk2.groups.io
=E4=B8=BB=E9=A2=98: Re: [edk2-devel] =E5=9B=9E=E5=A4=8D: [edk2-devel] FDF parser performance degrades rapidly on non= -trivially sized inputs

 

Here is the code I was testing with. You can pro= bably reduce the changes down to just _SkipWhiteSpace(), but I haven't trie= d that yet.

 

diff --git a/BaseTools/Source/Python/GenFds/FdfParser.py b/Bas= eTools/Source/Python/GenFds/FdfParser.py
index feb4c72779..8c57720116 10= 0644
--- a/BaseTools/Source/Python/GenFds/FdfParser.py
+++ b/BaseTool= s/Source/Python/GenFds/FdfParser.py
@@ -12,6 +12,7 @@
 #
&nbs= p;from __future__ import print_function
 from __future__ import abs= olute_import
+from io import StringIO
 from re import compile, D= OTALL
 from string import hexdigits
 from uuid import UUID<= br>@@ -253,7 +254,7 @@ class FdfParser:
        &nbs= p;self.CurrentFdName =3D None
         self.Cur= rentFvName =3D None
         self._Token =3D &q= uot;"
-        self._SkippedChars =3D "&qu= ot;
+        self._SkippedChars =3D StringIO()
&n= bsp;        GlobalData.gFdfParser =3D self
          # Used to section info
@@ -276,7 +277= ,7 @@ class FdfParser:
     def _SkipWhiteSpace(self):         while not self._EndOfFile():
  &= nbsp;          if self._CurrentChar() in {TAB_PRIN= TCHAR_NUL, T_CHAR_CR, TAB_LINE_BREAK, TAB_SPACE_SPLIT, T_CHAR_TAB}:
- &n= bsp;              self._SkippedChars += =3D str(self._CurrentChar())
+            =    self._SkippedChars.write(str(self._CurrentChar()))
  =                self._GetOneChar()             else:
    &n= bsp;            return
@@ -696,7 +697,7 @@= class FdfParser:
               = ;      Header =3D self._Token
       =              if not self._Token.endswit= h(TAB_SECTION_END):
              &nb= sp;          self._SkipToToken(TAB_SECTION_END)-                     &n= bsp;  Header +=3D self._SkippedChars
+         =                Header +=3D self._Sk= ippedChars.getvalue()
              &= nbsp;      if Header.find('$(') !=3D -1:
    &n= bsp;                    r= aise Warning("macro cannot be used in section header", self.FileN= ame, self.CurrentLineNumber)
            &= nbsp;        self._SectionHeaderParser(Header)
@@ -1= 226,7 +1227,7 @@ class FdfParser:
          &nb= sp;  raise Warning(QuoteToUse, self.FileName, self.CurrentLineNumber)<= br>         if currentLineNumber !=3D self.Current= LineNumber:
             raise Warnin= g(QuoteToUse, self.FileName, self.CurrentLineNumber)
-     &nb= sp;  self._Token =3D self._SkippedChars.rstrip(QuoteToUse)
+  =      self._Token =3D self._SkippedChars.getvalue().rstrip(Q= uoteToUse)
         return True
 
&n= bsp;    ## _SkipToToken() method
@@ -1243,7 +1244,7 @@ class F= dfParser:
     def _SkipToToken(self, String, IgnoreCase = =3D False):
         StartPos =3D self.GetFileB= ufferPos()
 
-        self._SkippedChars =3D= ""
+        self._SkippedChars =3D String= IO()
         while not self._EndOfFile():
&= nbsp;            index =3D -1
   = ;          if IgnoreCase:
@@ -1252,13 +1253,13 = @@ class FdfParser:
              &nb= sp;  index =3D self._CurrentLine()[self.CurrentOffsetWithinLine: ].fin= d(String)
             if index =3D= =3D 0:
                 sel= f.CurrentOffsetWithinLine +=3D len(String)
-        =        self._SkippedChars +=3D String
+   &nbs= p;            self._SkippedChars.write(String= )
                 return T= rue
-            self._SkippedChars +=3D s= tr(self._CurrentChar())
+            self.= _SkippedChars.write(str(self._CurrentChar()))
       = ;      self._GetOneChar()
 
      =    self.SetFileBufferPos(StartPos)
-       &nbs= p;self._SkippedChars =3D ""
+        self.= _SkippedChars =3D StringIO()
         return Fa= lse
 
     ## GetFileBufferPos() method
@@ -28= 90,7 +2891,7 @@ class FdfParser:
          &nbs= p;  if not self._SkipToToken(T_CHAR_BRACE_R):
      =            raise Warning.Expected("Depex= expression ending '}'", self.FileName, self.CurrentLineNumber)
&nb= sp;
-            DepexSectionObj.Expressio= n =3D self._SkippedChars.rstrip(T_CHAR_BRACE_R)
+       &= nbsp;    DepexSectionObj.Expression =3D self._SkippedChars.getval= ue().rstrip(T_CHAR_BRACE_R)
            &n= bsp;Obj.SectionList.append(DepexSectionObj)
 
    &nbs= p;    elif self._IsKeyword("SUBTYPE_GUID"):
@@ -3525= ,7 +3526,7 @@ class FdfParser:
         if not = self._SkipToToken(TAB_SPLIT):
            =  raise Warning.Expected("'.'", self.FileName, self.CurrentLi= neNumber)
 
-        Arch =3D self._SkippedC= hars.rstrip(TAB_SPLIT)
+        Arch =3D self._Skipp= edChars.getvalue().rstrip(TAB_SPLIT)
 
      &nbs= p;  ModuleType =3D self._GetModuleType()
 

 

 <= /span>

Also, if you w= ould like to profile the build process you can apply this patch.=

 

diff --gi= t a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/= build.py
index 51fb1f433e..396729efd9 100755
--- a/BaseTools/Source/P= ython/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ = -2778,12 +2778,18 @@ def Main():
     Log_Agent.join()     return ReturnCode
 
+import cProfile
+ if __name__ =3D=3D '__main__':
     try:
 = ;        mp.set_start_method('spawn')
    =  except:
         pass
-    r= =3D Main()
+
+    with cProfile.Profile() as pr:
+ &nbs= p;      r =3D Main()
+        pr.prin= t_stats('tottime')
+
     ## 0-127 is a safe return ra= nge, and 1 is a standard default error
     if r < 0 o= r r > 127: r =3D 1
     sys.exit(r)<= /p>

_._,_._,_
Groups.io Links:

=20 You receive all messages sent to this group. =20 =20

View/Reply Online (#120885) | =20 | Mute= This Topic | New Topic
Your Subscriptio= n | Contact Group Owner | Unsubscribe [rebecca@openfw.io]

_._,_._,_
------=_NextPart_000_0070_01DB47C2.F703CAD0--