* [Patch] BaseTools: Move CreateAsBuiltInf into AutoGenWorker for parallel @ 2020-08-12 6:09 Bob Feng 2020-08-12 14:35 ` Liming Gao 2020-08-12 21:54 ` [edk2-devel] " Carsey, Jaben 0 siblings, 2 replies; 5+ messages in thread From: Bob Feng @ 2020-08-12 6:09 UTC (permalink / raw) To: devel; +Cc: Liming Gao, Yuwei Chen AsBuiltInf can be created during AutoGen phase. Move CreateAsBuiltInf into AutoGenWorker to make this step run in parallel. Signed-off-by: Bob Feng <bob.c.feng@intel.com> Cc: Liming Gao <liming.gao@intel.com> Cc: Yuwei Chen <yuwei.chen@intel.com> --- BaseTools/Source/Python/AutoGen/AutoGenWorker.py | 2 +- BaseTools/Source/Python/build/build.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py index 017f676399..0425f1ab0b 100755 --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -278,11 +278,11 @@ class AutoGenWorkerInProcess(mp.Process): else: self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False)) Ma.CreateCodeFile(False) Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[])) - + Ma.CreateAsBuiltInf() if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: try: CacheResult = Ma.CanSkipbyMakeCache() except: CacheResult = False diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index 59ceacfed0..c4cfe38ad9 100755 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -871,11 +871,11 @@ class Build(): if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]: cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "PreMakeCache", False)) PcdMa.CreateCodeFile(False) PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[])) - + PcdMa.CreateAsBuiltInf() # Force cache miss for PCD driver if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "MakeCache", False)) self.AutoGenMgr.join() @@ -1263,11 +1263,10 @@ class Build(): # build modules if BuildModule: BuildCommand = BuildCommand + [Target] LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir) - self.CreateAsBuiltInf() if GlobalData.gBinCacheDest: self.GenDestCache() elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource: # Only for --hash # Update PreMakeCacheChain files @@ -2274,11 +2273,10 @@ class Build(): # All modules have been put in build tasks queue. Tell task scheduler # to exit if all tasks are completed # ExitFlag.set() BuildTask.WaitForComplete() - self.CreateAsBuiltInf() if GlobalData.gBinCacheDest: self.GenDestCache() elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource: # Only for --hash # Update PreMakeCacheChain files -- 2.20.1.windows.1 ^ permalink raw reply related [flat|nested] 5+ messages in thread
* Re: [Patch] BaseTools: Move CreateAsBuiltInf into AutoGenWorker for parallel 2020-08-12 6:09 [Patch] BaseTools: Move CreateAsBuiltInf into AutoGenWorker for parallel Bob Feng @ 2020-08-12 14:35 ` Liming Gao 2020-08-13 9:01 ` Bob Feng 2020-08-12 21:54 ` [edk2-devel] " Carsey, Jaben 1 sibling, 1 reply; 5+ messages in thread From: Liming Gao @ 2020-08-12 14:35 UTC (permalink / raw) To: Feng, Bob C, devel@edk2.groups.io; +Cc: Chen, Christine I like this idea. Is there any build performance improvement? > -----Original Message----- > From: Feng, Bob C <bob.c.feng@intel.com> > Sent: Wednesday, August 12, 2020 2:09 PM > To: devel@edk2.groups.io > Cc: Gao, Liming <liming.gao@intel.com>; Chen, Christine <yuwei.chen@intel.com> > Subject: [Patch] BaseTools: Move CreateAsBuiltInf into AutoGenWorker for parallel > > AsBuiltInf can be created during AutoGen phase. Move CreateAsBuiltInf > into AutoGenWorker to make this step run in parallel. > > Signed-off-by: Bob Feng <bob.c.feng@intel.com> > Cc: Liming Gao <liming.gao@intel.com> > Cc: Yuwei Chen <yuwei.chen@intel.com> > --- > BaseTools/Source/Python/AutoGen/AutoGenWorker.py | 2 +- > BaseTools/Source/Python/build/build.py | 4 +--- > 2 files changed, 2 insertions(+), 4 deletions(-) > > diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > index 017f676399..0425f1ab0b 100755 > --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > @@ -278,11 +278,11 @@ class AutoGenWorkerInProcess(mp.Process): > else: > self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False)) > > Ma.CreateCodeFile(False) > Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[])) > - > + Ma.CreateAsBuiltInf() > if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: > try: > CacheResult = Ma.CanSkipbyMakeCache() > except: > CacheResult = False > diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py > index 59ceacfed0..c4cfe38ad9 100755 > --- a/BaseTools/Source/Python/build/build.py > +++ b/BaseTools/Source/Python/build/build.py > @@ -871,11 +871,11 @@ class Build(): > if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]: > cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "PreMakeCache", False)) > > PcdMa.CreateCodeFile(False) > PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[])) > - > + PcdMa.CreateAsBuiltInf() > # Force cache miss for PCD driver > if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: > cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "MakeCache", False)) > > self.AutoGenMgr.join() > @@ -1263,11 +1263,10 @@ class Build(): > > # build modules > if BuildModule: > BuildCommand = BuildCommand + [Target] > LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir) > - self.CreateAsBuiltInf() > if GlobalData.gBinCacheDest: > self.GenDestCache() > elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource: > # Only for --hash > # Update PreMakeCacheChain files > @@ -2274,11 +2273,10 @@ class Build(): > # All modules have been put in build tasks queue. Tell task scheduler > # to exit if all tasks are completed > # > ExitFlag.set() > BuildTask.WaitForComplete() > - self.CreateAsBuiltInf() > if GlobalData.gBinCacheDest: > self.GenDestCache() > elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource: > # Only for --hash > # Update PreMakeCacheChain files > -- > 2.20.1.windows.1 ^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [Patch] BaseTools: Move CreateAsBuiltInf into AutoGenWorker for parallel 2020-08-12 14:35 ` Liming Gao @ 2020-08-13 9:01 ` Bob Feng 2020-08-13 9:12 ` Liming Gao 0 siblings, 1 reply; 5+ messages in thread From: Bob Feng @ 2020-08-13 9:01 UTC (permalink / raw) To: Gao, Liming, devel@edk2.groups.io; +Cc: Chen, Christine Liming, I did the test on Ovmf, there is some improvement but not much. Thanks, Bob -----Original Message----- From: Gao, Liming <liming.gao@intel.com> Sent: Wednesday, August 12, 2020 10:35 PM To: Feng, Bob C <bob.c.feng@intel.com>; devel@edk2.groups.io Cc: Chen, Christine <yuwei.chen@intel.com> Subject: RE: [Patch] BaseTools: Move CreateAsBuiltInf into AutoGenWorker for parallel I like this idea. Is there any build performance improvement? > -----Original Message----- > From: Feng, Bob C <bob.c.feng@intel.com> > Sent: Wednesday, August 12, 2020 2:09 PM > To: devel@edk2.groups.io > Cc: Gao, Liming <liming.gao@intel.com>; Chen, Christine > <yuwei.chen@intel.com> > Subject: [Patch] BaseTools: Move CreateAsBuiltInf into AutoGenWorker > for parallel > > AsBuiltInf can be created during AutoGen phase. Move CreateAsBuiltInf > into AutoGenWorker to make this step run in parallel. > > Signed-off-by: Bob Feng <bob.c.feng@intel.com> > Cc: Liming Gao <liming.gao@intel.com> > Cc: Yuwei Chen <yuwei.chen@intel.com> > --- > BaseTools/Source/Python/AutoGen/AutoGenWorker.py | 2 +- > BaseTools/Source/Python/build/build.py | 4 +--- > 2 files changed, 2 insertions(+), 4 deletions(-) > > diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > index 017f676399..0425f1ab0b 100755 > --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > @@ -278,11 +278,11 @@ class AutoGenWorkerInProcess(mp.Process): > else: > self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, > "PreMakeCache", False)) > > Ma.CreateCodeFile(False) > > Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, > Ma.Arch),[])) > - > + Ma.CreateAsBuiltInf() > if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: > try: > CacheResult = Ma.CanSkipbyMakeCache() > except: > CacheResult = False diff --git > a/BaseTools/Source/Python/build/build.py > b/BaseTools/Source/Python/build/build.py > index 59ceacfed0..c4cfe38ad9 100755 > --- a/BaseTools/Source/Python/build/build.py > +++ b/BaseTools/Source/Python/build/build.py > @@ -871,11 +871,11 @@ class Build(): > if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]: > cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, > "PreMakeCache", False)) > > PcdMa.CreateCodeFile(False) > PcdMa.CreateMakeFile(False,GenFfsList = > DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[])) > - > + PcdMa.CreateAsBuiltInf() > # Force cache miss for PCD driver > if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: > cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, > "MakeCache", False)) > > self.AutoGenMgr.join() > @@ -1263,11 +1263,10 @@ class Build(): > > # build modules > if BuildModule: > BuildCommand = BuildCommand + [Target] > LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir) > - self.CreateAsBuiltInf() > if GlobalData.gBinCacheDest: > self.GenDestCache() > elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource: > # Only for --hash > # Update PreMakeCacheChain files @@ -2274,11 +2273,10 > @@ class Build(): > # All modules have been put in build tasks queue. Tell task scheduler > # to exit if all tasks are completed > # > ExitFlag.set() > BuildTask.WaitForComplete() > - self.CreateAsBuiltInf() > if GlobalData.gBinCacheDest: > self.GenDestCache() > elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource: > # Only for --hash > # Update PreMakeCacheChain files > -- > 2.20.1.windows.1 ^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [Patch] BaseTools: Move CreateAsBuiltInf into AutoGenWorker for parallel 2020-08-13 9:01 ` Bob Feng @ 2020-08-13 9:12 ` Liming Gao 0 siblings, 0 replies; 5+ messages in thread From: Liming Gao @ 2020-08-13 9:12 UTC (permalink / raw) To: Feng, Bob C, devel@edk2.groups.io; +Cc: Chen, Christine A little better is also OK. Reviewed-by: Liming Gao <liming.gao@intel.com> -----Original Message----- From: Feng, Bob C <bob.c.feng@intel.com> Sent: 2020年8月13日 17:01 To: Gao, Liming <liming.gao@intel.com>; devel@edk2.groups.io Cc: Chen, Christine <yuwei.chen@intel.com> Subject: RE: [Patch] BaseTools: Move CreateAsBuiltInf into AutoGenWorker for parallel Liming, I did the test on Ovmf, there is some improvement but not much. Thanks, Bob -----Original Message----- From: Gao, Liming <liming.gao@intel.com> Sent: Wednesday, August 12, 2020 10:35 PM To: Feng, Bob C <bob.c.feng@intel.com>; devel@edk2.groups.io Cc: Chen, Christine <yuwei.chen@intel.com> Subject: RE: [Patch] BaseTools: Move CreateAsBuiltInf into AutoGenWorker for parallel I like this idea. Is there any build performance improvement? > -----Original Message----- > From: Feng, Bob C <bob.c.feng@intel.com> > Sent: Wednesday, August 12, 2020 2:09 PM > To: devel@edk2.groups.io > Cc: Gao, Liming <liming.gao@intel.com>; Chen, Christine > <yuwei.chen@intel.com> > Subject: [Patch] BaseTools: Move CreateAsBuiltInf into AutoGenWorker > for parallel > > AsBuiltInf can be created during AutoGen phase. Move CreateAsBuiltInf > into AutoGenWorker to make this step run in parallel. > > Signed-off-by: Bob Feng <bob.c.feng@intel.com> > Cc: Liming Gao <liming.gao@intel.com> > Cc: Yuwei Chen <yuwei.chen@intel.com> > --- > BaseTools/Source/Python/AutoGen/AutoGenWorker.py | 2 +- > BaseTools/Source/Python/build/build.py | 4 +--- > 2 files changed, 2 insertions(+), 4 deletions(-) > > diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > index 017f676399..0425f1ab0b 100755 > --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > @@ -278,11 +278,11 @@ class AutoGenWorkerInProcess(mp.Process): > else: > self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, > "PreMakeCache", False)) > > Ma.CreateCodeFile(False) > > Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, > Ma.Arch),[])) > - > + Ma.CreateAsBuiltInf() > if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: > try: > CacheResult = Ma.CanSkipbyMakeCache() > except: > CacheResult = False diff --git > a/BaseTools/Source/Python/build/build.py > b/BaseTools/Source/Python/build/build.py > index 59ceacfed0..c4cfe38ad9 100755 > --- a/BaseTools/Source/Python/build/build.py > +++ b/BaseTools/Source/Python/build/build.py > @@ -871,11 +871,11 @@ class Build(): > if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]: > cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, > "PreMakeCache", False)) > > PcdMa.CreateCodeFile(False) > PcdMa.CreateMakeFile(False,GenFfsList = > DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[])) > - > + PcdMa.CreateAsBuiltInf() > # Force cache miss for PCD driver > if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: > cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, > "MakeCache", False)) > > self.AutoGenMgr.join() > @@ -1263,11 +1263,10 @@ class Build(): > > # build modules > if BuildModule: > BuildCommand = BuildCommand + [Target] > LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir) > - self.CreateAsBuiltInf() > if GlobalData.gBinCacheDest: > self.GenDestCache() > elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource: > # Only for --hash > # Update PreMakeCacheChain files @@ -2274,11 +2273,10 > @@ class Build(): > # All modules have been put in build tasks queue. Tell task scheduler > # to exit if all tasks are completed > # > ExitFlag.set() > BuildTask.WaitForComplete() > - self.CreateAsBuiltInf() > if GlobalData.gBinCacheDest: > self.GenDestCache() > elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource: > # Only for --hash > # Update PreMakeCacheChain files > -- > 2.20.1.windows.1 ^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [edk2-devel] [Patch] BaseTools: Move CreateAsBuiltInf into AutoGenWorker for parallel 2020-08-12 6:09 [Patch] BaseTools: Move CreateAsBuiltInf into AutoGenWorker for parallel Bob Feng 2020-08-12 14:35 ` Liming Gao @ 2020-08-12 21:54 ` Carsey, Jaben 1 sibling, 0 replies; 5+ messages in thread From: Carsey, Jaben @ 2020-08-12 21:54 UTC (permalink / raw) To: devel@edk2.groups.io, Feng, Bob C; +Cc: Gao, Liming, Chen, Christine Reviewed-by: Jaben Carsey <jaben.carsey@intel.com> > -----Original Message----- > From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Bob > Feng > Sent: Tuesday, August 11, 2020 11:09 PM > To: devel@edk2.groups.io > Cc: Gao, Liming <liming.gao@intel.com>; Chen, Christine > <yuwei.chen@intel.com> > Subject: [edk2-devel] [Patch] BaseTools: Move CreateAsBuiltInf into > AutoGenWorker for parallel > > AsBuiltInf can be created during AutoGen phase. Move CreateAsBuiltInf into > AutoGenWorker to make this step run in parallel. > > Signed-off-by: Bob Feng <bob.c.feng@intel.com> > Cc: Liming Gao <liming.gao@intel.com> > Cc: Yuwei Chen <yuwei.chen@intel.com> > --- > BaseTools/Source/Python/AutoGen/AutoGenWorker.py | 2 +- > BaseTools/Source/Python/build/build.py | 4 +--- > 2 files changed, 2 insertions(+), 4 deletions(-) > > diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > index 017f676399..0425f1ab0b 100755 > --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > @@ -278,11 +278,11 @@ class AutoGenWorkerInProcess(mp.Process): > else: > self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, > "PreMakeCache", False)) > > Ma.CreateCodeFile(False) > > Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, > Ma.Arch),[])) > - > + Ma.CreateAsBuiltInf() > if GlobalData.gBinCacheSource and CommandTarget in [None, "", > "all"]: > try: > CacheResult = Ma.CanSkipbyMakeCache() > except: > CacheResult = False diff --git > a/BaseTools/Source/Python/build/build.py > b/BaseTools/Source/Python/build/build.py > index 59ceacfed0..c4cfe38ad9 100755 > --- a/BaseTools/Source/Python/build/build.py > +++ b/BaseTools/Source/Python/build/build.py > @@ -871,11 +871,11 @@ class Build(): > if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest > and self.Target in [None, "", "all"]: > cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, > "PreMakeCache", False)) > > PcdMa.CreateCodeFile(False) > PcdMa.CreateMakeFile(False,GenFfsList = > DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[])) > - > + PcdMa.CreateAsBuiltInf() > # Force cache miss for PCD driver > if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: > cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "MakeCache", > False)) > > self.AutoGenMgr.join() > @@ -1263,11 +1263,10 @@ class Build(): > > # build modules > if BuildModule: > BuildCommand = BuildCommand + [Target] > LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir) > - self.CreateAsBuiltInf() > if GlobalData.gBinCacheDest: > self.GenDestCache() > elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource: > # Only for --hash > # Update PreMakeCacheChain files @@ -2274,11 +2273,10 @@ class > Build(): > # All modules have been put in build tasks queue. Tell task scheduler > # to exit if all tasks are completed > # > ExitFlag.set() > BuildTask.WaitForComplete() > - self.CreateAsBuiltInf() > if GlobalData.gBinCacheDest: > self.GenDestCache() > elif GlobalData.gUseHashCache and not > GlobalData.gBinCacheSource: > # Only for --hash > # Update PreMakeCacheChain files > -- > 2.20.1.windows.1 > > > ^ permalink raw reply [flat|nested] 5+ messages in thread
end of thread, other threads:[~2020-08-13 9:12 UTC | newest] Thread overview: 5+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2020-08-12 6:09 [Patch] BaseTools: Move CreateAsBuiltInf into AutoGenWorker for parallel Bob Feng 2020-08-12 14:35 ` Liming Gao 2020-08-13 9:01 ` Bob Feng 2020-08-13 9:12 ` Liming Gao 2020-08-12 21:54 ` [edk2-devel] " Carsey, Jaben
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox