* [Patch] BaseTools: Enable component override functionality
@ 2019-01-11 2:38 BobCF
2019-01-14 3:00 ` Gao, Liming
2019-02-22 22:06 ` Felix Polyudov
0 siblings, 2 replies; 10+ messages in thread
From: BobCF @ 2019-01-11 2:38 UTC (permalink / raw)
To: edk2-devel; +Cc: Bob Feng, Liming Gao, Carsey Jaben
https://bugzilla.tianocore.org/show_bug.cgi?id=1449
This patch enable build tools to recognize that
when two given files have the same GUID, file path and ARCH in Dsc,
The later one's definition will be used.
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Cc: Carsey Jaben <jaben.carsey@intel.com>
---
.../Source/Python/Workspace/DscBuildData.py | 24 ++++++++++++-------
.../Source/Python/Workspace/MetaFileParser.py | 5 ++++
.../Source/Python/Workspace/MetaFileTable.py | 7 ++++--
3 files changed, 25 insertions(+), 11 deletions(-)
diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py
index 7e82e8e934..f9805f58f5 100644
--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
@@ -704,36 +704,44 @@ class DscBuildData(PlatformBuildClassObject):
if TAB_DEFAULT_STORES_DEFAULT not in self.DefaultStores:
self.DefaultStores[TAB_DEFAULT_STORES_DEFAULT] = (0, TAB_DEFAULT_STORES_DEFAULT)
GlobalData.gDefaultStores = sorted(self.DefaultStores.keys())
return self.DefaultStores
+ def OverrideDuplicateModule(self):
+ RecordList = self._RawData[MODEL_META_DATA_COMPONENT, self._Arch]
+ Macros = self._Macros
+ Macros["EDK_SOURCE"] = GlobalData.gEcpSource
+ Components = {}
+ for Record in RecordList:
+ ModuleId = Record[6]
+ file_guid = self._RawData[MODEL_META_DATA_HEADER, self._Arch, None, ModuleId]
+ file_guid_str = file_guid[0][2] if file_guid else "NULL"
+ ModuleFile = PathClass(NormPath(Record[0], Macros), GlobalData.gWorkspace, Arch=self._Arch)
+ if self._Arch != TAB_ARCH_COMMON and (file_guid_str,str(ModuleFile)) in Components:
+ self._RawData.DisableOverrideComponent(Components[(file_guid_str,str(ModuleFile))])
+ Components[(file_guid_str,str(ModuleFile))] = ModuleId
+ self._RawData._PostProcessed = False
## Retrieve [Components] section information
@property
def Modules(self):
if self._Modules is not None:
return self._Modules
-
+ self.OverrideDuplicateModule()
self._Modules = OrderedDict()
RecordList = self._RawData[MODEL_META_DATA_COMPONENT, self._Arch]
Macros = self._Macros
Macros["EDK_SOURCE"] = GlobalData.gEcpSource
for Record in RecordList:
- DuplicatedFile = False
-
ModuleFile = PathClass(NormPath(Record[0], Macros), GlobalData.gWorkspace, Arch=self._Arch)
ModuleId = Record[6]
LineNo = Record[7]
# check the file validation
ErrorCode, ErrorInfo = ModuleFile.Validate('.inf')
if ErrorCode != 0:
EdkLogger.error('build', ErrorCode, File=self.MetaFile, Line=LineNo,
ExtraData=ErrorInfo)
- # Check duplication
- # If arch is COMMON, no duplicate module is checked since all modules in all component sections are selected
- if self._Arch != TAB_ARCH_COMMON and ModuleFile in self._Modules:
- DuplicatedFile = True
Module = ModuleBuildClassObject()
Module.MetaFile = ModuleFile
# get module private library instance
@@ -792,12 +800,10 @@ class DscBuildData(PlatformBuildClassObject):
else:
OptionString = Module.BuildOptions[ToolChainFamily, ToolChain]
Module.BuildOptions[ToolChainFamily, ToolChain] = OptionString + " " + Option
RecordList = self._RawData[MODEL_META_DATA_HEADER, self._Arch, None, ModuleId]
- if DuplicatedFile and not RecordList:
- EdkLogger.error('build', FILE_DUPLICATED, File=self.MetaFile, ExtraData=str(ModuleFile), Line=LineNo)
if RecordList:
if len(RecordList) != 1:
EdkLogger.error('build', OPTION_UNKNOWN, 'Only FILE_GUID can be listed in <Defines> section.',
File=self.MetaFile, ExtraData=str(ModuleFile), Line=LineNo)
ModuleFile = ProcessDuplicatedInf(ModuleFile, RecordList[0][2], GlobalData.gWorkspace)
diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py b/BaseTools/Source/Python/Workspace/MetaFileParser.py
index 032220813b..a52e9229df 100644
--- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
+++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
@@ -1711,10 +1711,15 @@ class DscParser(MetaFileParser):
def __ProcessBuildOption(self):
self._ValueList = [ReplaceMacro(Value, self._Macros, RaiseError=False)
for Value in self._ValueList]
+ def DisableOverrideComponent(self,module_id):
+ for ori_id in self._IdMapping:
+ if self._IdMapping[ori_id] == module_id:
+ self._RawTable.DisableComponent(ori_id)
+
_SectionParser = {
MODEL_META_DATA_HEADER : _DefineParser,
MODEL_EFI_SKU_ID : _SkuIdParser,
MODEL_EFI_DEFAULT_STORES : _DefaultStoresParser,
MODEL_EFI_LIBRARY_INSTANCE : _LibraryInstanceParser,
diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py b/BaseTools/Source/Python/Workspace/MetaFileTable.py
index 004e9494c3..823a87e057 100644
--- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
+++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
@@ -74,11 +74,11 @@ class MetaFileTable():
def SetEndFlag(self):
self.CurrentContent.append(self._DUMMY_)
def GetAll(self):
- return [item for item in self.CurrentContent if item[0] >= 0 ]
+ return [item for item in self.CurrentContent if item[0] >= 0 and item[-1]>=0]
## Python class representation of table storing module data
class ModuleTable(MetaFileTable):
_COLUMN_ = '''
ID REAL PRIMARY KEY,
@@ -371,11 +371,10 @@ class PlatformTable(MetaFileTable):
#
def Query(self, Model, Scope1=None, Scope2=None, BelongsToItem=None, FromItem=None):
QueryTab = self.CurrentContent
result = [item for item in QueryTab if item[1] == Model and item[-1]>0 ]
-
if Scope1 is not None and Scope1 != TAB_ARCH_COMMON:
Sc1 = set(['COMMON'])
Sc1.add(Scope1)
result = [item for item in result if item[5] in Sc1]
Sc2 = set( ['COMMON','DEFAULT'])
@@ -395,10 +394,14 @@ class PlatformTable(MetaFileTable):
result = [item for item in result if item[9] == FromItem]
result = [ [r[2],r[3],r[4],r[5],r[6],r[7],r[0],r[9]] for r in result ]
return result
+ def DisableComponent(self,comp_id):
+ for item in self.CurrentContent:
+ if item[0] == comp_id or item[8] == comp_id:
+ item[-1] = -1
## Factory class to produce different storage for different type of meta-file
class MetaFileStorage(object):
_FILE_TABLE_ = {
MODEL_FILE_INF : ModuleTable,
--
2.19.1.windows.1
^ permalink raw reply related [flat|nested] 10+ messages in thread
* Re: [Patch] BaseTools: Enable component override functionality
2019-01-11 2:38 [Patch] BaseTools: Enable component override functionality BobCF
@ 2019-01-14 3:00 ` Gao, Liming
2019-01-14 6:57 ` Ard Biesheuvel
2019-02-22 22:06 ` Felix Polyudov
1 sibling, 1 reply; 10+ messages in thread
From: Gao, Liming @ 2019-01-14 3:00 UTC (permalink / raw)
To: Feng, Bob C, edk2-devel@lists.01.org; +Cc: Carsey, Jaben
Reviewed-by: Liming Gao <liming.gao@intel.com>
>-----Original Message-----
>From: Feng, Bob C
>Sent: Friday, January 11, 2019 10:39 AM
>To: edk2-devel@lists.01.org
>Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming
><liming.gao@intel.com>; Carsey, Jaben <jaben.carsey@intel.com>
>Subject: [Patch] BaseTools: Enable component override functionality
>
>https://bugzilla.tianocore.org/show_bug.cgi?id=1449
>This patch enable build tools to recognize that
>when two given files have the same GUID, file path and ARCH in Dsc,
>The later one's definition will be used.
>
>Contributed-under: TianoCore Contribution Agreement 1.1
>Signed-off-by: Bob Feng <bob.c.feng@intel.com>
>Cc: Liming Gao <liming.gao@intel.com>
>Cc: Carsey Jaben <jaben.carsey@intel.com>
>---
> .../Source/Python/Workspace/DscBuildData.py | 24 ++++++++++++-------
> .../Source/Python/Workspace/MetaFileParser.py | 5 ++++
> .../Source/Python/Workspace/MetaFileTable.py | 7 ++++--
> 3 files changed, 25 insertions(+), 11 deletions(-)
>
>diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py
>b/BaseTools/Source/Python/Workspace/DscBuildData.py
>index 7e82e8e934..f9805f58f5 100644
>--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
>+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
>@@ -704,36 +704,44 @@ class DscBuildData(PlatformBuildClassObject):
> if TAB_DEFAULT_STORES_DEFAULT not in self.DefaultStores:
> self.DefaultStores[TAB_DEFAULT_STORES_DEFAULT] = (0,
>TAB_DEFAULT_STORES_DEFAULT)
> GlobalData.gDefaultStores = sorted(self.DefaultStores.keys())
> return self.DefaultStores
>
>+ def OverrideDuplicateModule(self):
>+ RecordList = self._RawData[MODEL_META_DATA_COMPONENT,
>self._Arch]
>+ Macros = self._Macros
>+ Macros["EDK_SOURCE"] = GlobalData.gEcpSource
>+ Components = {}
>+ for Record in RecordList:
>+ ModuleId = Record[6]
>+ file_guid = self._RawData[MODEL_META_DATA_HEADER, self._Arch,
>None, ModuleId]
>+ file_guid_str = file_guid[0][2] if file_guid else "NULL"
>+ ModuleFile = PathClass(NormPath(Record[0], Macros),
>GlobalData.gWorkspace, Arch=self._Arch)
>+ if self._Arch != TAB_ARCH_COMMON and
>(file_guid_str,str(ModuleFile)) in Components:
>+
>self._RawData.DisableOverrideComponent(Components[(file_guid_str,str(M
>oduleFile))])
>+ Components[(file_guid_str,str(ModuleFile))] = ModuleId
>+ self._RawData._PostProcessed = False
> ## Retrieve [Components] section information
> @property
> def Modules(self):
> if self._Modules is not None:
> return self._Modules
>-
>+ self.OverrideDuplicateModule()
> self._Modules = OrderedDict()
> RecordList = self._RawData[MODEL_META_DATA_COMPONENT,
>self._Arch]
> Macros = self._Macros
> Macros["EDK_SOURCE"] = GlobalData.gEcpSource
> for Record in RecordList:
>- DuplicatedFile = False
>-
> ModuleFile = PathClass(NormPath(Record[0], Macros),
>GlobalData.gWorkspace, Arch=self._Arch)
> ModuleId = Record[6]
> LineNo = Record[7]
>
> # check the file validation
> ErrorCode, ErrorInfo = ModuleFile.Validate('.inf')
> if ErrorCode != 0:
> EdkLogger.error('build', ErrorCode, File=self.MetaFile, Line=LineNo,
> ExtraData=ErrorInfo)
>- # Check duplication
>- # If arch is COMMON, no duplicate module is checked since all modules
>in all component sections are selected
>- if self._Arch != TAB_ARCH_COMMON and ModuleFile in self._Modules:
>- DuplicatedFile = True
>
> Module = ModuleBuildClassObject()
> Module.MetaFile = ModuleFile
>
> # get module private library instance
>@@ -792,12 +800,10 @@ class DscBuildData(PlatformBuildClassObject):
> else:
> OptionString = Module.BuildOptions[ToolChainFamily, ToolChain]
> Module.BuildOptions[ToolChainFamily, ToolChain] = OptionString +
>" " + Option
>
> RecordList = self._RawData[MODEL_META_DATA_HEADER, self._Arch,
>None, ModuleId]
>- if DuplicatedFile and not RecordList:
>- EdkLogger.error('build', FILE_DUPLICATED, File=self.MetaFile,
>ExtraData=str(ModuleFile), Line=LineNo)
> if RecordList:
> if len(RecordList) != 1:
> EdkLogger.error('build', OPTION_UNKNOWN, 'Only FILE_GUID can
>be listed in <Defines> section.',
> File=self.MetaFile, ExtraData=str(ModuleFile), Line=LineNo)
> ModuleFile = ProcessDuplicatedInf(ModuleFile, RecordList[0][2],
>GlobalData.gWorkspace)
>diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py
>b/BaseTools/Source/Python/Workspace/MetaFileParser.py
>index 032220813b..a52e9229df 100644
>--- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
>+++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
>@@ -1711,10 +1711,15 @@ class DscParser(MetaFileParser):
>
> def __ProcessBuildOption(self):
> self._ValueList = [ReplaceMacro(Value, self._Macros, RaiseError=False)
> for Value in self._ValueList]
>
>+ def DisableOverrideComponent(self,module_id):
>+ for ori_id in self._IdMapping:
>+ if self._IdMapping[ori_id] == module_id:
>+ self._RawTable.DisableComponent(ori_id)
>+
> _SectionParser = {
> MODEL_META_DATA_HEADER : _DefineParser,
> MODEL_EFI_SKU_ID : _SkuIdParser,
> MODEL_EFI_DEFAULT_STORES : _DefaultStoresParser,
> MODEL_EFI_LIBRARY_INSTANCE : _LibraryInstanceParser,
>diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py
>b/BaseTools/Source/Python/Workspace/MetaFileTable.py
>index 004e9494c3..823a87e057 100644
>--- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
>+++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
>@@ -74,11 +74,11 @@ class MetaFileTable():
>
> def SetEndFlag(self):
> self.CurrentContent.append(self._DUMMY_)
>
> def GetAll(self):
>- return [item for item in self.CurrentContent if item[0] >= 0 ]
>+ return [item for item in self.CurrentContent if item[0] >= 0 and item[-
>1]>=0]
>
> ## Python class representation of table storing module data
> class ModuleTable(MetaFileTable):
> _COLUMN_ = '''
> ID REAL PRIMARY KEY,
>@@ -371,11 +371,10 @@ class PlatformTable(MetaFileTable):
> #
> def Query(self, Model, Scope1=None, Scope2=None,
>BelongsToItem=None, FromItem=None):
>
> QueryTab = self.CurrentContent
> result = [item for item in QueryTab if item[1] == Model and item[-1]>0 ]
>-
> if Scope1 is not None and Scope1 != TAB_ARCH_COMMON:
> Sc1 = set(['COMMON'])
> Sc1.add(Scope1)
> result = [item for item in result if item[5] in Sc1]
> Sc2 = set( ['COMMON','DEFAULT'])
>@@ -395,10 +394,14 @@ class PlatformTable(MetaFileTable):
> result = [item for item in result if item[9] == FromItem]
>
> result = [ [r[2],r[3],r[4],r[5],r[6],r[7],r[0],r[9]] for r in result ]
> return result
>
>+ def DisableComponent(self,comp_id):
>+ for item in self.CurrentContent:
>+ if item[0] == comp_id or item[8] == comp_id:
>+ item[-1] = -1
>
> ## Factory class to produce different storage for different type of meta-file
> class MetaFileStorage(object):
> _FILE_TABLE_ = {
> MODEL_FILE_INF : ModuleTable,
>--
>2.19.1.windows.1
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [Patch] BaseTools: Enable component override functionality
2019-01-14 3:00 ` Gao, Liming
@ 2019-01-14 6:57 ` Ard Biesheuvel
2019-01-14 7:01 ` Gao, Liming
0 siblings, 1 reply; 10+ messages in thread
From: Ard Biesheuvel @ 2019-01-14 6:57 UTC (permalink / raw)
To: Gao, Liming; +Cc: Feng, Bob C, edk2-devel@lists.01.org, Carsey, Jaben
On Mon, 14 Jan 2019 at 04:00, Gao, Liming <liming.gao@intel.com> wrote:
>
> Reviewed-by: Liming Gao <liming.gao@intel.com>
>
This patch breaks the build on AArch64/GGC:
(Python 2.7.13 on linux2) Traceback (most recent call last):
File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Python/build/build.py",>
line 2403, in Main
MyBuild.Launch()
File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Python/build/build.py",>
line 2137, in Launch
self._MultiThreadBuildPlatform()
File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Python/build/build.py",>
line 1917, in _MultiThreadBuildPlatform
self.Progress
File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/AutoGen/AutoGen.py",>
line 239, in __init__
self._InitWorker(Workspace, MetaFile, Target, Toolchain, Arch,
*args, **kwargs)
File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/AutoGen/AutoGen.py",>
line 403, in _InitWorker
PlatformPcds = Platform.Pcds
File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
line 1194, in Pcds
self._Pcds = self.UpdateStructuredPcds(MODEL_PCD_TYPE_LIST, self._Pcds)
File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
line 1486, in UpdateStructuredPcds
for Pcd in self.DecPcds:
File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
line 3183, in DecPcds
self._DecPcds, self._GuidDict = GetDeclaredPcd(self, self._Bdb,
self._Arch, self._Target, self._Toolchain, PkgSet)
File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/Workspace/WorkspaceCommon.py",>
line 62, in GetDeclaredPcd
PkgList = GetPackageList(Platform, BuildDatabase, Arch, Target, Toolchain)
File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/Workspace/WorkspaceCommon.py",>
line 44, in GetPackageList
for ModuleFile in Platform.Modules:
File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
line 728, in Modules
self.OverrideDuplicateModule()
File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
line 712, in OverrideDuplicateModule
Macros["EDK_SOURCE"] = GlobalData.gEcpSource
AttributeError: 'module' object has no attribute 'gEcpSource'
> >-----Original Message-----
> >From: Feng, Bob C
> >Sent: Friday, January 11, 2019 10:39 AM
> >To: edk2-devel@lists.01.org
> >Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming
> ><liming.gao@intel.com>; Carsey, Jaben <jaben.carsey@intel.com>
> >Subject: [Patch] BaseTools: Enable component override functionality
> >
> >https://bugzilla.tianocore.org/show_bug.cgi?id=1449
> >This patch enable build tools to recognize that
> >when two given files have the same GUID, file path and ARCH in Dsc,
> >The later one's definition will be used.
> >
> >Contributed-under: TianoCore Contribution Agreement 1.1
> >Signed-off-by: Bob Feng <bob.c.feng@intel.com>
> >Cc: Liming Gao <liming.gao@intel.com>
> >Cc: Carsey Jaben <jaben.carsey@intel.com>
> >---
> > .../Source/Python/Workspace/DscBuildData.py | 24 ++++++++++++-------
> > .../Source/Python/Workspace/MetaFileParser.py | 5 ++++
> > .../Source/Python/Workspace/MetaFileTable.py | 7 ++++--
> > 3 files changed, 25 insertions(+), 11 deletions(-)
> >
> >diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py
> >b/BaseTools/Source/Python/Workspace/DscBuildData.py
> >index 7e82e8e934..f9805f58f5 100644
> >--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
> >+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
> >@@ -704,36 +704,44 @@ class DscBuildData(PlatformBuildClassObject):
> > if TAB_DEFAULT_STORES_DEFAULT not in self.DefaultStores:
> > self.DefaultStores[TAB_DEFAULT_STORES_DEFAULT] = (0,
> >TAB_DEFAULT_STORES_DEFAULT)
> > GlobalData.gDefaultStores = sorted(self.DefaultStores.keys())
> > return self.DefaultStores
> >
> >+ def OverrideDuplicateModule(self):
> >+ RecordList = self._RawData[MODEL_META_DATA_COMPONENT,
> >self._Arch]
> >+ Macros = self._Macros
> >+ Macros["EDK_SOURCE"] = GlobalData.gEcpSource
> >+ Components = {}
> >+ for Record in RecordList:
> >+ ModuleId = Record[6]
> >+ file_guid = self._RawData[MODEL_META_DATA_HEADER, self._Arch,
> >None, ModuleId]
> >+ file_guid_str = file_guid[0][2] if file_guid else "NULL"
> >+ ModuleFile = PathClass(NormPath(Record[0], Macros),
> >GlobalData.gWorkspace, Arch=self._Arch)
> >+ if self._Arch != TAB_ARCH_COMMON and
> >(file_guid_str,str(ModuleFile)) in Components:
> >+
> >self._RawData.DisableOverrideComponent(Components[(file_guid_str,str(M
> >oduleFile))])
> >+ Components[(file_guid_str,str(ModuleFile))] = ModuleId
> >+ self._RawData._PostProcessed = False
> > ## Retrieve [Components] section information
> > @property
> > def Modules(self):
> > if self._Modules is not None:
> > return self._Modules
> >-
> >+ self.OverrideDuplicateModule()
> > self._Modules = OrderedDict()
> > RecordList = self._RawData[MODEL_META_DATA_COMPONENT,
> >self._Arch]
> > Macros = self._Macros
> > Macros["EDK_SOURCE"] = GlobalData.gEcpSource
> > for Record in RecordList:
> >- DuplicatedFile = False
> >-
> > ModuleFile = PathClass(NormPath(Record[0], Macros),
> >GlobalData.gWorkspace, Arch=self._Arch)
> > ModuleId = Record[6]
> > LineNo = Record[7]
> >
> > # check the file validation
> > ErrorCode, ErrorInfo = ModuleFile.Validate('.inf')
> > if ErrorCode != 0:
> > EdkLogger.error('build', ErrorCode, File=self.MetaFile, Line=LineNo,
> > ExtraData=ErrorInfo)
> >- # Check duplication
> >- # If arch is COMMON, no duplicate module is checked since all modules
> >in all component sections are selected
> >- if self._Arch != TAB_ARCH_COMMON and ModuleFile in self._Modules:
> >- DuplicatedFile = True
> >
> > Module = ModuleBuildClassObject()
> > Module.MetaFile = ModuleFile
> >
> > # get module private library instance
> >@@ -792,12 +800,10 @@ class DscBuildData(PlatformBuildClassObject):
> > else:
> > OptionString = Module.BuildOptions[ToolChainFamily, ToolChain]
> > Module.BuildOptions[ToolChainFamily, ToolChain] = OptionString +
> >" " + Option
> >
> > RecordList = self._RawData[MODEL_META_DATA_HEADER, self._Arch,
> >None, ModuleId]
> >- if DuplicatedFile and not RecordList:
> >- EdkLogger.error('build', FILE_DUPLICATED, File=self.MetaFile,
> >ExtraData=str(ModuleFile), Line=LineNo)
> > if RecordList:
> > if len(RecordList) != 1:
> > EdkLogger.error('build', OPTION_UNKNOWN, 'Only FILE_GUID can
> >be listed in <Defines> section.',
> > File=self.MetaFile, ExtraData=str(ModuleFile), Line=LineNo)
> > ModuleFile = ProcessDuplicatedInf(ModuleFile, RecordList[0][2],
> >GlobalData.gWorkspace)
> >diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >b/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >index 032220813b..a52e9229df 100644
> >--- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >+++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >@@ -1711,10 +1711,15 @@ class DscParser(MetaFileParser):
> >
> > def __ProcessBuildOption(self):
> > self._ValueList = [ReplaceMacro(Value, self._Macros, RaiseError=False)
> > for Value in self._ValueList]
> >
> >+ def DisableOverrideComponent(self,module_id):
> >+ for ori_id in self._IdMapping:
> >+ if self._IdMapping[ori_id] == module_id:
> >+ self._RawTable.DisableComponent(ori_id)
> >+
> > _SectionParser = {
> > MODEL_META_DATA_HEADER : _DefineParser,
> > MODEL_EFI_SKU_ID : _SkuIdParser,
> > MODEL_EFI_DEFAULT_STORES : _DefaultStoresParser,
> > MODEL_EFI_LIBRARY_INSTANCE : _LibraryInstanceParser,
> >diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >b/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >index 004e9494c3..823a87e057 100644
> >--- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >+++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >@@ -74,11 +74,11 @@ class MetaFileTable():
> >
> > def SetEndFlag(self):
> > self.CurrentContent.append(self._DUMMY_)
> >
> > def GetAll(self):
> >- return [item for item in self.CurrentContent if item[0] >= 0 ]
> >+ return [item for item in self.CurrentContent if item[0] >= 0 and item[-
> >1]>=0]
> >
> > ## Python class representation of table storing module data
> > class ModuleTable(MetaFileTable):
> > _COLUMN_ = '''
> > ID REAL PRIMARY KEY,
> >@@ -371,11 +371,10 @@ class PlatformTable(MetaFileTable):
> > #
> > def Query(self, Model, Scope1=None, Scope2=None,
> >BelongsToItem=None, FromItem=None):
> >
> > QueryTab = self.CurrentContent
> > result = [item for item in QueryTab if item[1] == Model and item[-1]>0 ]
> >-
> > if Scope1 is not None and Scope1 != TAB_ARCH_COMMON:
> > Sc1 = set(['COMMON'])
> > Sc1.add(Scope1)
> > result = [item for item in result if item[5] in Sc1]
> > Sc2 = set( ['COMMON','DEFAULT'])
> >@@ -395,10 +394,14 @@ class PlatformTable(MetaFileTable):
> > result = [item for item in result if item[9] == FromItem]
> >
> > result = [ [r[2],r[3],r[4],r[5],r[6],r[7],r[0],r[9]] for r in result ]
> > return result
> >
> >+ def DisableComponent(self,comp_id):
> >+ for item in self.CurrentContent:
> >+ if item[0] == comp_id or item[8] == comp_id:
> >+ item[-1] = -1
> >
> > ## Factory class to produce different storage for different type of meta-file
> > class MetaFileStorage(object):
> > _FILE_TABLE_ = {
> > MODEL_FILE_INF : ModuleTable,
> >--
> >2.19.1.windows.1
>
> _______________________________________________
> edk2-devel mailing list
> edk2-devel@lists.01.org
> https://lists.01.org/mailman/listinfo/edk2-devel
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [Patch] BaseTools: Enable component override functionality
2019-01-14 6:57 ` Ard Biesheuvel
@ 2019-01-14 7:01 ` Gao, Liming
2019-01-14 7:10 ` Ard Biesheuvel
0 siblings, 1 reply; 10+ messages in thread
From: Gao, Liming @ 2019-01-14 7:01 UTC (permalink / raw)
To: Ard Biesheuvel; +Cc: Feng, Bob C, edk2-devel@lists.01.org, Carsey, Jaben
Ard:
Sorry for this break, Bob just sent hot fix for this issue in https://lists.01.org/pipermail/edk2-devel/2019-January/034947.html. Could you verify it?
In fact, it is caused by previous patch to remove EDK support in BaseTools Python code.
Thanks
Liming
>-----Original Message-----
>From: Ard Biesheuvel [mailto:ard.biesheuvel@linaro.org]
>Sent: Monday, January 14, 2019 2:58 PM
>To: Gao, Liming <liming.gao@intel.com>
>Cc: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org; Carsey,
>Jaben <jaben.carsey@intel.com>
>Subject: Re: [edk2] [Patch] BaseTools: Enable component override
>functionality
>
>On Mon, 14 Jan 2019 at 04:00, Gao, Liming <liming.gao@intel.com> wrote:
>>
>> Reviewed-by: Liming Gao <liming.gao@intel.com>
>>
>
>This patch breaks the build on AArch64/GGC:
>
>(Python 2.7.13 on linux2) Traceback (most recent call last):
> File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Python/b
>uild/build.py",>
>line 2403, in Main
> MyBuild.Launch()
> File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Python/b
>uild/build.py",>
>line 2137, in Launch
> self._MultiThreadBuildPlatform()
> File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Python/b
>uild/build.py",>
>line 1917, in _MultiThreadBuildPlatform
> self.Progress
> File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>upstream/ws/edk2/BaseTools/Source/Python/AutoGen/AutoGen.py",>
>line 239, in __init__
> self._InitWorker(Workspace, MetaFile, Target, Toolchain, Arch,
>*args, **kwargs)
> File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>upstream/ws/edk2/BaseTools/Source/Python/AutoGen/AutoGen.py",>
>line 403, in _InitWorker
> PlatformPcds = Platform.Pcds
> File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
>line 1194, in Pcds
> self._Pcds = self.UpdateStructuredPcds(MODEL_PCD_TYPE_LIST, self._Pcds)
> File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
>line 1486, in UpdateStructuredPcds
> for Pcd in self.DecPcds:
> File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
>line 3183, in DecPcds
> self._DecPcds, self._GuidDict = GetDeclaredPcd(self, self._Bdb,
>self._Arch, self._Target, self._Toolchain, PkgSet)
> File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>upstream/ws/edk2/BaseTools/Source/Python/Workspace/WorkspaceComm
>on.py",>
>line 62, in GetDeclaredPcd
> PkgList = GetPackageList(Platform, BuildDatabase, Arch, Target, Toolchain)
> File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>upstream/ws/edk2/BaseTools/Source/Python/Workspace/WorkspaceComm
>on.py",>
>line 44, in GetPackageList
> for ModuleFile in Platform.Modules:
> File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
>line 728, in Modules
> self.OverrideDuplicateModule()
> File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
>line 712, in OverrideDuplicateModule
> Macros["EDK_SOURCE"] = GlobalData.gEcpSource
>AttributeError: 'module' object has no attribute 'gEcpSource'
>
>
>
>> >-----Original Message-----
>> >From: Feng, Bob C
>> >Sent: Friday, January 11, 2019 10:39 AM
>> >To: edk2-devel@lists.01.org
>> >Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming
>> ><liming.gao@intel.com>; Carsey, Jaben <jaben.carsey@intel.com>
>> >Subject: [Patch] BaseTools: Enable component override functionality
>> >
>> >https://bugzilla.tianocore.org/show_bug.cgi?id=1449
>> >This patch enable build tools to recognize that
>> >when two given files have the same GUID, file path and ARCH in Dsc,
>> >The later one's definition will be used.
>> >
>> >Contributed-under: TianoCore Contribution Agreement 1.1
>> >Signed-off-by: Bob Feng <bob.c.feng@intel.com>
>> >Cc: Liming Gao <liming.gao@intel.com>
>> >Cc: Carsey Jaben <jaben.carsey@intel.com>
>> >---
>> > .../Source/Python/Workspace/DscBuildData.py | 24 ++++++++++++------
>-
>> > .../Source/Python/Workspace/MetaFileParser.py | 5 ++++
>> > .../Source/Python/Workspace/MetaFileTable.py | 7 ++++--
>> > 3 files changed, 25 insertions(+), 11 deletions(-)
>> >
>> >diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py
>> >b/BaseTools/Source/Python/Workspace/DscBuildData.py
>> >index 7e82e8e934..f9805f58f5 100644
>> >--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
>> >+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
>> >@@ -704,36 +704,44 @@ class DscBuildData(PlatformBuildClassObject):
>> > if TAB_DEFAULT_STORES_DEFAULT not in self.DefaultStores:
>> > self.DefaultStores[TAB_DEFAULT_STORES_DEFAULT] = (0,
>> >TAB_DEFAULT_STORES_DEFAULT)
>> > GlobalData.gDefaultStores = sorted(self.DefaultStores.keys())
>> > return self.DefaultStores
>> >
>> >+ def OverrideDuplicateModule(self):
>> >+ RecordList = self._RawData[MODEL_META_DATA_COMPONENT,
>> >self._Arch]
>> >+ Macros = self._Macros
>> >+ Macros["EDK_SOURCE"] = GlobalData.gEcpSource
>> >+ Components = {}
>> >+ for Record in RecordList:
>> >+ ModuleId = Record[6]
>> >+ file_guid = self._RawData[MODEL_META_DATA_HEADER,
>self._Arch,
>> >None, ModuleId]
>> >+ file_guid_str = file_guid[0][2] if file_guid else "NULL"
>> >+ ModuleFile = PathClass(NormPath(Record[0], Macros),
>> >GlobalData.gWorkspace, Arch=self._Arch)
>> >+ if self._Arch != TAB_ARCH_COMMON and
>> >(file_guid_str,str(ModuleFile)) in Components:
>> >+
>> >self._RawData.DisableOverrideComponent(Components[(file_guid_str,str
>(M
>> >oduleFile))])
>> >+ Components[(file_guid_str,str(ModuleFile))] = ModuleId
>> >+ self._RawData._PostProcessed = False
>> > ## Retrieve [Components] section information
>> > @property
>> > def Modules(self):
>> > if self._Modules is not None:
>> > return self._Modules
>> >-
>> >+ self.OverrideDuplicateModule()
>> > self._Modules = OrderedDict()
>> > RecordList = self._RawData[MODEL_META_DATA_COMPONENT,
>> >self._Arch]
>> > Macros = self._Macros
>> > Macros["EDK_SOURCE"] = GlobalData.gEcpSource
>> > for Record in RecordList:
>> >- DuplicatedFile = False
>> >-
>> > ModuleFile = PathClass(NormPath(Record[0], Macros),
>> >GlobalData.gWorkspace, Arch=self._Arch)
>> > ModuleId = Record[6]
>> > LineNo = Record[7]
>> >
>> > # check the file validation
>> > ErrorCode, ErrorInfo = ModuleFile.Validate('.inf')
>> > if ErrorCode != 0:
>> > EdkLogger.error('build', ErrorCode, File=self.MetaFile,
>Line=LineNo,
>> > ExtraData=ErrorInfo)
>> >- # Check duplication
>> >- # If arch is COMMON, no duplicate module is checked since all
>modules
>> >in all component sections are selected
>> >- if self._Arch != TAB_ARCH_COMMON and ModuleFile in
>self._Modules:
>> >- DuplicatedFile = True
>> >
>> > Module = ModuleBuildClassObject()
>> > Module.MetaFile = ModuleFile
>> >
>> > # get module private library instance
>> >@@ -792,12 +800,10 @@ class DscBuildData(PlatformBuildClassObject):
>> > else:
>> > OptionString = Module.BuildOptions[ToolChainFamily, ToolChain]
>> > Module.BuildOptions[ToolChainFamily, ToolChain] =
>OptionString +
>> >" " + Option
>> >
>> > RecordList = self._RawData[MODEL_META_DATA_HEADER,
>self._Arch,
>> >None, ModuleId]
>> >- if DuplicatedFile and not RecordList:
>> >- EdkLogger.error('build', FILE_DUPLICATED, File=self.MetaFile,
>> >ExtraData=str(ModuleFile), Line=LineNo)
>> > if RecordList:
>> > if len(RecordList) != 1:
>> > EdkLogger.error('build', OPTION_UNKNOWN, 'Only FILE_GUID
>can
>> >be listed in <Defines> section.',
>> > File=self.MetaFile, ExtraData=str(ModuleFile),
>Line=LineNo)
>> > ModuleFile = ProcessDuplicatedInf(ModuleFile, RecordList[0][2],
>> >GlobalData.gWorkspace)
>> >diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py
>> >b/BaseTools/Source/Python/Workspace/MetaFileParser.py
>> >index 032220813b..a52e9229df 100644
>> >--- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
>> >+++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
>> >@@ -1711,10 +1711,15 @@ class DscParser(MetaFileParser):
>> >
>> > def __ProcessBuildOption(self):
>> > self._ValueList = [ReplaceMacro(Value, self._Macros, RaiseError=False)
>> > for Value in self._ValueList]
>> >
>> >+ def DisableOverrideComponent(self,module_id):
>> >+ for ori_id in self._IdMapping:
>> >+ if self._IdMapping[ori_id] == module_id:
>> >+ self._RawTable.DisableComponent(ori_id)
>> >+
>> > _SectionParser = {
>> > MODEL_META_DATA_HEADER : _DefineParser,
>> > MODEL_EFI_SKU_ID : _SkuIdParser,
>> > MODEL_EFI_DEFAULT_STORES : _DefaultStoresParser,
>> > MODEL_EFI_LIBRARY_INSTANCE : _LibraryInstanceParser,
>> >diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py
>> >b/BaseTools/Source/Python/Workspace/MetaFileTable.py
>> >index 004e9494c3..823a87e057 100644
>> >--- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
>> >+++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
>> >@@ -74,11 +74,11 @@ class MetaFileTable():
>> >
>> > def SetEndFlag(self):
>> > self.CurrentContent.append(self._DUMMY_)
>> >
>> > def GetAll(self):
>> >- return [item for item in self.CurrentContent if item[0] >= 0 ]
>> >+ return [item for item in self.CurrentContent if item[0] >= 0 and item[-
>> >1]>=0]
>> >
>> > ## Python class representation of table storing module data
>> > class ModuleTable(MetaFileTable):
>> > _COLUMN_ = '''
>> > ID REAL PRIMARY KEY,
>> >@@ -371,11 +371,10 @@ class PlatformTable(MetaFileTable):
>> > #
>> > def Query(self, Model, Scope1=None, Scope2=None,
>> >BelongsToItem=None, FromItem=None):
>> >
>> > QueryTab = self.CurrentContent
>> > result = [item for item in QueryTab if item[1] == Model and item[-1]>0 ]
>> >-
>> > if Scope1 is not None and Scope1 != TAB_ARCH_COMMON:
>> > Sc1 = set(['COMMON'])
>> > Sc1.add(Scope1)
>> > result = [item for item in result if item[5] in Sc1]
>> > Sc2 = set( ['COMMON','DEFAULT'])
>> >@@ -395,10 +394,14 @@ class PlatformTable(MetaFileTable):
>> > result = [item for item in result if item[9] == FromItem]
>> >
>> > result = [ [r[2],r[3],r[4],r[5],r[6],r[7],r[0],r[9]] for r in result ]
>> > return result
>> >
>> >+ def DisableComponent(self,comp_id):
>> >+ for item in self.CurrentContent:
>> >+ if item[0] == comp_id or item[8] == comp_id:
>> >+ item[-1] = -1
>> >
>> > ## Factory class to produce different storage for different type of meta-
>file
>> > class MetaFileStorage(object):
>> > _FILE_TABLE_ = {
>> > MODEL_FILE_INF : ModuleTable,
>> >--
>> >2.19.1.windows.1
>>
>> _______________________________________________
>> edk2-devel mailing list
>> edk2-devel@lists.01.org
>> https://lists.01.org/mailman/listinfo/edk2-devel
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [Patch] BaseTools: Enable component override functionality
2019-01-14 7:01 ` Gao, Liming
@ 2019-01-14 7:10 ` Ard Biesheuvel
2019-01-14 7:18 ` Feng, Bob C
0 siblings, 1 reply; 10+ messages in thread
From: Ard Biesheuvel @ 2019-01-14 7:10 UTC (permalink / raw)
To: Gao, Liming; +Cc: Feng, Bob C, edk2-devel@lists.01.org, Carsey, Jaben
On Mon, 14 Jan 2019 at 08:01, Gao, Liming <liming.gao@intel.com> wrote:
>
> Ard:
> Sorry for this break, Bob just sent hot fix for this issue in https://lists.01.org/pipermail/edk2-devel/2019-January/034947.html. Could you verify it?
> In fact, it is caused by previous patch to remove EDK support in BaseTools Python code.
>
OK, that patch fixes it for me.
But may I kindly suggest that Bob tests his code before pushing it? Thanks.
> >-----Original Message-----
> >From: Ard Biesheuvel [mailto:ard.biesheuvel@linaro.org]
> >Sent: Monday, January 14, 2019 2:58 PM
> >To: Gao, Liming <liming.gao@intel.com>
> >Cc: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org; Carsey,
> >Jaben <jaben.carsey@intel.com>
> >Subject: Re: [edk2] [Patch] BaseTools: Enable component override
> >functionality
> >
> >On Mon, 14 Jan 2019 at 04:00, Gao, Liming <liming.gao@intel.com> wrote:
> >>
> >> Reviewed-by: Liming Gao <liming.gao@intel.com>
> >>
> >
> >This patch breaks the build on AArch64/GGC:
> >
> >(Python 2.7.13 on linux2) Traceback (most recent call last):
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Python/b
> >uild/build.py",>
> >line 2403, in Main
> > MyBuild.Launch()
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Python/b
> >uild/build.py",>
> >line 2137, in Launch
> > self._MultiThreadBuildPlatform()
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Python/b
> >uild/build.py",>
> >line 1917, in _MultiThreadBuildPlatform
> > self.Progress
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/AutoGen/AutoGen.py",>
> >line 239, in __init__
> > self._InitWorker(Workspace, MetaFile, Target, Toolchain, Arch,
> >*args, **kwargs)
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/AutoGen/AutoGen.py",>
> >line 403, in _InitWorker
> > PlatformPcds = Platform.Pcds
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
> >line 1194, in Pcds
> > self._Pcds = self.UpdateStructuredPcds(MODEL_PCD_TYPE_LIST, self._Pcds)
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
> >line 1486, in UpdateStructuredPcds
> > for Pcd in self.DecPcds:
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
> >line 3183, in DecPcds
> > self._DecPcds, self._GuidDict = GetDeclaredPcd(self, self._Bdb,
> >self._Arch, self._Target, self._Toolchain, PkgSet)
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/WorkspaceComm
> >on.py",>
> >line 62, in GetDeclaredPcd
> > PkgList = GetPackageList(Platform, BuildDatabase, Arch, Target, Toolchain)
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/WorkspaceComm
> >on.py",>
> >line 44, in GetPackageList
> > for ModuleFile in Platform.Modules:
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
> >line 728, in Modules
> > self.OverrideDuplicateModule()
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
> >line 712, in OverrideDuplicateModule
> > Macros["EDK_SOURCE"] = GlobalData.gEcpSource
> >AttributeError: 'module' object has no attribute 'gEcpSource'
> >
> >
> >
> >> >-----Original Message-----
> >> >From: Feng, Bob C
> >> >Sent: Friday, January 11, 2019 10:39 AM
> >> >To: edk2-devel@lists.01.org
> >> >Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming
> >> ><liming.gao@intel.com>; Carsey, Jaben <jaben.carsey@intel.com>
> >> >Subject: [Patch] BaseTools: Enable component override functionality
> >> >
> >> >https://bugzilla.tianocore.org/show_bug.cgi?id=1449
> >> >This patch enable build tools to recognize that
> >> >when two given files have the same GUID, file path and ARCH in Dsc,
> >> >The later one's definition will be used.
> >> >
> >> >Contributed-under: TianoCore Contribution Agreement 1.1
> >> >Signed-off-by: Bob Feng <bob.c.feng@intel.com>
> >> >Cc: Liming Gao <liming.gao@intel.com>
> >> >Cc: Carsey Jaben <jaben.carsey@intel.com>
> >> >---
> >> > .../Source/Python/Workspace/DscBuildData.py | 24 ++++++++++++------
> >-
> >> > .../Source/Python/Workspace/MetaFileParser.py | 5 ++++
> >> > .../Source/Python/Workspace/MetaFileTable.py | 7 ++++--
> >> > 3 files changed, 25 insertions(+), 11 deletions(-)
> >> >
> >> >diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py
> >> >b/BaseTools/Source/Python/Workspace/DscBuildData.py
> >> >index 7e82e8e934..f9805f58f5 100644
> >> >--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
> >> >+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
> >> >@@ -704,36 +704,44 @@ class DscBuildData(PlatformBuildClassObject):
> >> > if TAB_DEFAULT_STORES_DEFAULT not in self.DefaultStores:
> >> > self.DefaultStores[TAB_DEFAULT_STORES_DEFAULT] = (0,
> >> >TAB_DEFAULT_STORES_DEFAULT)
> >> > GlobalData.gDefaultStores = sorted(self.DefaultStores.keys())
> >> > return self.DefaultStores
> >> >
> >> >+ def OverrideDuplicateModule(self):
> >> >+ RecordList = self._RawData[MODEL_META_DATA_COMPONENT,
> >> >self._Arch]
> >> >+ Macros = self._Macros
> >> >+ Macros["EDK_SOURCE"] = GlobalData.gEcpSource
> >> >+ Components = {}
> >> >+ for Record in RecordList:
> >> >+ ModuleId = Record[6]
> >> >+ file_guid = self._RawData[MODEL_META_DATA_HEADER,
> >self._Arch,
> >> >None, ModuleId]
> >> >+ file_guid_str = file_guid[0][2] if file_guid else "NULL"
> >> >+ ModuleFile = PathClass(NormPath(Record[0], Macros),
> >> >GlobalData.gWorkspace, Arch=self._Arch)
> >> >+ if self._Arch != TAB_ARCH_COMMON and
> >> >(file_guid_str,str(ModuleFile)) in Components:
> >> >+
> >> >self._RawData.DisableOverrideComponent(Components[(file_guid_str,str
> >(M
> >> >oduleFile))])
> >> >+ Components[(file_guid_str,str(ModuleFile))] = ModuleId
> >> >+ self._RawData._PostProcessed = False
> >> > ## Retrieve [Components] section information
> >> > @property
> >> > def Modules(self):
> >> > if self._Modules is not None:
> >> > return self._Modules
> >> >-
> >> >+ self.OverrideDuplicateModule()
> >> > self._Modules = OrderedDict()
> >> > RecordList = self._RawData[MODEL_META_DATA_COMPONENT,
> >> >self._Arch]
> >> > Macros = self._Macros
> >> > Macros["EDK_SOURCE"] = GlobalData.gEcpSource
> >> > for Record in RecordList:
> >> >- DuplicatedFile = False
> >> >-
> >> > ModuleFile = PathClass(NormPath(Record[0], Macros),
> >> >GlobalData.gWorkspace, Arch=self._Arch)
> >> > ModuleId = Record[6]
> >> > LineNo = Record[7]
> >> >
> >> > # check the file validation
> >> > ErrorCode, ErrorInfo = ModuleFile.Validate('.inf')
> >> > if ErrorCode != 0:
> >> > EdkLogger.error('build', ErrorCode, File=self.MetaFile,
> >Line=LineNo,
> >> > ExtraData=ErrorInfo)
> >> >- # Check duplication
> >> >- # If arch is COMMON, no duplicate module is checked since all
> >modules
> >> >in all component sections are selected
> >> >- if self._Arch != TAB_ARCH_COMMON and ModuleFile in
> >self._Modules:
> >> >- DuplicatedFile = True
> >> >
> >> > Module = ModuleBuildClassObject()
> >> > Module.MetaFile = ModuleFile
> >> >
> >> > # get module private library instance
> >> >@@ -792,12 +800,10 @@ class DscBuildData(PlatformBuildClassObject):
> >> > else:
> >> > OptionString = Module.BuildOptions[ToolChainFamily, ToolChain]
> >> > Module.BuildOptions[ToolChainFamily, ToolChain] =
> >OptionString +
> >> >" " + Option
> >> >
> >> > RecordList = self._RawData[MODEL_META_DATA_HEADER,
> >self._Arch,
> >> >None, ModuleId]
> >> >- if DuplicatedFile and not RecordList:
> >> >- EdkLogger.error('build', FILE_DUPLICATED, File=self.MetaFile,
> >> >ExtraData=str(ModuleFile), Line=LineNo)
> >> > if RecordList:
> >> > if len(RecordList) != 1:
> >> > EdkLogger.error('build', OPTION_UNKNOWN, 'Only FILE_GUID
> >can
> >> >be listed in <Defines> section.',
> >> > File=self.MetaFile, ExtraData=str(ModuleFile),
> >Line=LineNo)
> >> > ModuleFile = ProcessDuplicatedInf(ModuleFile, RecordList[0][2],
> >> >GlobalData.gWorkspace)
> >> >diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >> >b/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >> >index 032220813b..a52e9229df 100644
> >> >--- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >> >+++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >> >@@ -1711,10 +1711,15 @@ class DscParser(MetaFileParser):
> >> >
> >> > def __ProcessBuildOption(self):
> >> > self._ValueList = [ReplaceMacro(Value, self._Macros, RaiseError=False)
> >> > for Value in self._ValueList]
> >> >
> >> >+ def DisableOverrideComponent(self,module_id):
> >> >+ for ori_id in self._IdMapping:
> >> >+ if self._IdMapping[ori_id] == module_id:
> >> >+ self._RawTable.DisableComponent(ori_id)
> >> >+
> >> > _SectionParser = {
> >> > MODEL_META_DATA_HEADER : _DefineParser,
> >> > MODEL_EFI_SKU_ID : _SkuIdParser,
> >> > MODEL_EFI_DEFAULT_STORES : _DefaultStoresParser,
> >> > MODEL_EFI_LIBRARY_INSTANCE : _LibraryInstanceParser,
> >> >diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >> >b/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >> >index 004e9494c3..823a87e057 100644
> >> >--- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >> >+++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >> >@@ -74,11 +74,11 @@ class MetaFileTable():
> >> >
> >> > def SetEndFlag(self):
> >> > self.CurrentContent.append(self._DUMMY_)
> >> >
> >> > def GetAll(self):
> >> >- return [item for item in self.CurrentContent if item[0] >= 0 ]
> >> >+ return [item for item in self.CurrentContent if item[0] >= 0 and item[-
> >> >1]>=0]
> >> >
> >> > ## Python class representation of table storing module data
> >> > class ModuleTable(MetaFileTable):
> >> > _COLUMN_ = '''
> >> > ID REAL PRIMARY KEY,
> >> >@@ -371,11 +371,10 @@ class PlatformTable(MetaFileTable):
> >> > #
> >> > def Query(self, Model, Scope1=None, Scope2=None,
> >> >BelongsToItem=None, FromItem=None):
> >> >
> >> > QueryTab = self.CurrentContent
> >> > result = [item for item in QueryTab if item[1] == Model and item[-1]>0 ]
> >> >-
> >> > if Scope1 is not None and Scope1 != TAB_ARCH_COMMON:
> >> > Sc1 = set(['COMMON'])
> >> > Sc1.add(Scope1)
> >> > result = [item for item in result if item[5] in Sc1]
> >> > Sc2 = set( ['COMMON','DEFAULT'])
> >> >@@ -395,10 +394,14 @@ class PlatformTable(MetaFileTable):
> >> > result = [item for item in result if item[9] == FromItem]
> >> >
> >> > result = [ [r[2],r[3],r[4],r[5],r[6],r[7],r[0],r[9]] for r in result ]
> >> > return result
> >> >
> >> >+ def DisableComponent(self,comp_id):
> >> >+ for item in self.CurrentContent:
> >> >+ if item[0] == comp_id or item[8] == comp_id:
> >> >+ item[-1] = -1
> >> >
> >> > ## Factory class to produce different storage for different type of meta-
> >file
> >> > class MetaFileStorage(object):
> >> > _FILE_TABLE_ = {
> >> > MODEL_FILE_INF : ModuleTable,
> >> >--
> >> >2.19.1.windows.1
> >>
> >> _______________________________________________
> >> edk2-devel mailing list
> >> edk2-devel@lists.01.org
> >> https://lists.01.org/mailman/listinfo/edk2-devel
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [Patch] BaseTools: Enable component override functionality
2019-01-14 7:10 ` Ard Biesheuvel
@ 2019-01-14 7:18 ` Feng, Bob C
2019-01-14 7:22 ` Gao, Liming
0 siblings, 1 reply; 10+ messages in thread
From: Feng, Bob C @ 2019-01-14 7:18 UTC (permalink / raw)
To: Ard Biesheuvel, Gao, Liming; +Cc: edk2-devel@lists.01.org, Carsey, Jaben
Sorry for break your build.
Yes. I'll double test the code before pushing it next time.
-Bob
-----Original Message-----
From: Ard Biesheuvel [mailto:ard.biesheuvel@linaro.org]
Sent: Monday, January 14, 2019 3:11 PM
To: Gao, Liming <liming.gao@intel.com>
Cc: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org; Carsey, Jaben <jaben.carsey@intel.com>
Subject: Re: [edk2] [Patch] BaseTools: Enable component override functionality
On Mon, 14 Jan 2019 at 08:01, Gao, Liming <liming.gao@intel.com> wrote:
>
> Ard:
> Sorry for this break, Bob just sent hot fix for this issue in https://lists.01.org/pipermail/edk2-devel/2019-January/034947.html. Could you verify it?
> In fact, it is caused by previous patch to remove EDK support in BaseTools Python code.
>
OK, that patch fixes it for me.
But may I kindly suggest that Bob tests his code before pushing it? Thanks.
> >-----Original Message-----
> >From: Ard Biesheuvel [mailto:ard.biesheuvel@linaro.org]
> >Sent: Monday, January 14, 2019 2:58 PM
> >To: Gao, Liming <liming.gao@intel.com>
> >Cc: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org;
> >Carsey, Jaben <jaben.carsey@intel.com>
> >Subject: Re: [edk2] [Patch] BaseTools: Enable component override
> >functionality
> >
> >On Mon, 14 Jan 2019 at 04:00, Gao, Liming <liming.gao@intel.com> wrote:
> >>
> >> Reviewed-by: Liming Gao <liming.gao@intel.com>
> >>
> >
> >This patch breaks the build on AArch64/GGC:
> >
> >(Python 2.7.13 on linux2) Traceback (most recent call last):
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Python/
> >b
> >uild/build.py",>
> >line 2403, in Main
> > MyBuild.Launch()
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Python/
> >b
> >uild/build.py",>
> >line 2137, in Launch
> > self._MultiThreadBuildPlatform()
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Python/
> >b
> >uild/build.py",>
> >line 1917, in _MultiThreadBuildPlatform
> > self.Progress
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/AutoGen/AutoGen.py",>
> >line 239, in __init__
> > self._InitWorker(Workspace, MetaFile, Target, Toolchain, Arch,
> >*args, **kwargs)
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/AutoGen/AutoGen.py",>
> >line 403, in _InitWorker
> > PlatformPcds = Platform.Pcds
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
> >line 1194, in Pcds
> > self._Pcds = self.UpdateStructuredPcds(MODEL_PCD_TYPE_LIST,
> >self._Pcds)
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
> >line 1486, in UpdateStructuredPcds
> > for Pcd in self.DecPcds:
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
> >line 3183, in DecPcds
> > self._DecPcds, self._GuidDict = GetDeclaredPcd(self, self._Bdb,
> >self._Arch, self._Target, self._Toolchain, PkgSet)
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/WorkspaceComm
> >on.py",>
> >line 62, in GetDeclaredPcd
> > PkgList = GetPackageList(Platform, BuildDatabase, Arch, Target,
> >Toolchain)
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/WorkspaceComm
> >on.py",>
> >line 44, in GetPackageList
> > for ModuleFile in Platform.Modules:
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
> >line 728, in Modules
> > self.OverrideDuplicateModule()
> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
> >line 712, in OverrideDuplicateModule
> > Macros["EDK_SOURCE"] = GlobalData.gEcpSource
> >AttributeError: 'module' object has no attribute 'gEcpSource'
> >
> >
> >
> >> >-----Original Message-----
> >> >From: Feng, Bob C
> >> >Sent: Friday, January 11, 2019 10:39 AM
> >> >To: edk2-devel@lists.01.org
> >> >Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming
> >> ><liming.gao@intel.com>; Carsey, Jaben <jaben.carsey@intel.com>
> >> >Subject: [Patch] BaseTools: Enable component override
> >> >functionality
> >> >
> >> >https://bugzilla.tianocore.org/show_bug.cgi?id=1449
> >> >This patch enable build tools to recognize that when two given
> >> >files have the same GUID, file path and ARCH in Dsc, The later
> >> >one's definition will be used.
> >> >
> >> >Contributed-under: TianoCore Contribution Agreement 1.1
> >> >Signed-off-by: Bob Feng <bob.c.feng@intel.com>
> >> >Cc: Liming Gao <liming.gao@intel.com>
> >> >Cc: Carsey Jaben <jaben.carsey@intel.com>
> >> >---
> >> > .../Source/Python/Workspace/DscBuildData.py | 24 ++++++++++++------
> >-
> >> > .../Source/Python/Workspace/MetaFileParser.py | 5 ++++
> >> > .../Source/Python/Workspace/MetaFileTable.py | 7 ++++--
> >> > 3 files changed, 25 insertions(+), 11 deletions(-)
> >> >
> >> >diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py
> >> >b/BaseTools/Source/Python/Workspace/DscBuildData.py
> >> >index 7e82e8e934..f9805f58f5 100644
> >> >--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
> >> >+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
> >> >@@ -704,36 +704,44 @@ class DscBuildData(PlatformBuildClassObject):
> >> > if TAB_DEFAULT_STORES_DEFAULT not in self.DefaultStores:
> >> > self.DefaultStores[TAB_DEFAULT_STORES_DEFAULT] =
> >> >(0,
> >> >TAB_DEFAULT_STORES_DEFAULT)
> >> > GlobalData.gDefaultStores = sorted(self.DefaultStores.keys())
> >> > return self.DefaultStores
> >> >
> >> >+ def OverrideDuplicateModule(self):
> >> >+ RecordList = self._RawData[MODEL_META_DATA_COMPONENT,
> >> >self._Arch]
> >> >+ Macros = self._Macros
> >> >+ Macros["EDK_SOURCE"] = GlobalData.gEcpSource
> >> >+ Components = {}
> >> >+ for Record in RecordList:
> >> >+ ModuleId = Record[6]
> >> >+ file_guid = self._RawData[MODEL_META_DATA_HEADER,
> >self._Arch,
> >> >None, ModuleId]
> >> >+ file_guid_str = file_guid[0][2] if file_guid else "NULL"
> >> >+ ModuleFile = PathClass(NormPath(Record[0], Macros),
> >> >GlobalData.gWorkspace, Arch=self._Arch)
> >> >+ if self._Arch != TAB_ARCH_COMMON and
> >> >(file_guid_str,str(ModuleFile)) in Components:
> >> >+
> >> >self._RawData.DisableOverrideComponent(Components[(file_guid_str,s
> >> >tr
> >(M
> >> >oduleFile))])
> >> >+ Components[(file_guid_str,str(ModuleFile))] = ModuleId
> >> >+ self._RawData._PostProcessed = False
> >> > ## Retrieve [Components] section information
> >> > @property
> >> > def Modules(self):
> >> > if self._Modules is not None:
> >> > return self._Modules
> >> >-
> >> >+ self.OverrideDuplicateModule()
> >> > self._Modules = OrderedDict()
> >> > RecordList = self._RawData[MODEL_META_DATA_COMPONENT,
> >> >self._Arch]
> >> > Macros = self._Macros
> >> > Macros["EDK_SOURCE"] = GlobalData.gEcpSource
> >> > for Record in RecordList:
> >> >- DuplicatedFile = False
> >> >-
> >> > ModuleFile = PathClass(NormPath(Record[0], Macros),
> >> >GlobalData.gWorkspace, Arch=self._Arch)
> >> > ModuleId = Record[6]
> >> > LineNo = Record[7]
> >> >
> >> > # check the file validation
> >> > ErrorCode, ErrorInfo = ModuleFile.Validate('.inf')
> >> > if ErrorCode != 0:
> >> > EdkLogger.error('build', ErrorCode,
> >> > File=self.MetaFile,
> >Line=LineNo,
> >> > ExtraData=ErrorInfo)
> >> >- # Check duplication
> >> >- # If arch is COMMON, no duplicate module is checked since all
> >modules
> >> >in all component sections are selected
> >> >- if self._Arch != TAB_ARCH_COMMON and ModuleFile in
> >self._Modules:
> >> >- DuplicatedFile = True
> >> >
> >> > Module = ModuleBuildClassObject()
> >> > Module.MetaFile = ModuleFile
> >> >
> >> > # get module private library instance @@ -792,12
> >> >+800,10 @@ class DscBuildData(PlatformBuildClassObject):
> >> > else:
> >> > OptionString = Module.BuildOptions[ToolChainFamily, ToolChain]
> >> > Module.BuildOptions[ToolChainFamily,
> >> >ToolChain] =
> >OptionString +
> >> >" " + Option
> >> >
> >> > RecordList = self._RawData[MODEL_META_DATA_HEADER,
> >self._Arch,
> >> >None, ModuleId]
> >> >- if DuplicatedFile and not RecordList:
> >> >- EdkLogger.error('build', FILE_DUPLICATED, File=self.MetaFile,
> >> >ExtraData=str(ModuleFile), Line=LineNo)
> >> > if RecordList:
> >> > if len(RecordList) != 1:
> >> > EdkLogger.error('build', OPTION_UNKNOWN,
> >> >'Only FILE_GUID
> >can
> >> >be listed in <Defines> section.',
> >> > File=self.MetaFile,
> >> >ExtraData=str(ModuleFile),
> >Line=LineNo)
> >> > ModuleFile = ProcessDuplicatedInf(ModuleFile,
> >> >RecordList[0][2],
> >> >GlobalData.gWorkspace)
> >> >diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >> >b/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >> >index 032220813b..a52e9229df 100644
> >> >--- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >> >+++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >> >@@ -1711,10 +1711,15 @@ class DscParser(MetaFileParser):
> >> >
> >> > def __ProcessBuildOption(self):
> >> > self._ValueList = [ReplaceMacro(Value, self._Macros, RaiseError=False)
> >> > for Value in self._ValueList]
> >> >
> >> >+ def DisableOverrideComponent(self,module_id):
> >> >+ for ori_id in self._IdMapping:
> >> >+ if self._IdMapping[ori_id] == module_id:
> >> >+ self._RawTable.DisableComponent(ori_id)
> >> >+
> >> > _SectionParser = {
> >> > MODEL_META_DATA_HEADER : _DefineParser,
> >> > MODEL_EFI_SKU_ID : _SkuIdParser,
> >> > MODEL_EFI_DEFAULT_STORES : _DefaultStoresParser,
> >> > MODEL_EFI_LIBRARY_INSTANCE : _LibraryInstanceParser,
> >> >diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >> >b/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >> >index 004e9494c3..823a87e057 100644
> >> >--- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >> >+++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >> >@@ -74,11 +74,11 @@ class MetaFileTable():
> >> >
> >> > def SetEndFlag(self):
> >> > self.CurrentContent.append(self._DUMMY_)
> >> >
> >> > def GetAll(self):
> >> >- return [item for item in self.CurrentContent if item[0] >= 0 ]
> >> >+ return [item for item in self.CurrentContent if item[0]
> >> >+ >= 0 and item[-
> >> >1]>=0]
> >> >
> >> > ## Python class representation of table storing module data
> >> >class ModuleTable(MetaFileTable):
> >> > _COLUMN_ = '''
> >> > ID REAL PRIMARY KEY,
> >> >@@ -371,11 +371,10 @@ class PlatformTable(MetaFileTable):
> >> > #
> >> > def Query(self, Model, Scope1=None, Scope2=None,
> >> >BelongsToItem=None, FromItem=None):
> >> >
> >> > QueryTab = self.CurrentContent
> >> > result = [item for item in QueryTab if item[1] == Model
> >> >and item[-1]>0 ]
> >> >-
> >> > if Scope1 is not None and Scope1 != TAB_ARCH_COMMON:
> >> > Sc1 = set(['COMMON'])
> >> > Sc1.add(Scope1)
> >> > result = [item for item in result if item[5] in Sc1]
> >> > Sc2 = set( ['COMMON','DEFAULT']) @@ -395,10 +394,14 @@
> >> >class PlatformTable(MetaFileTable):
> >> > result = [item for item in result if item[9] ==
> >> >FromItem]
> >> >
> >> > result = [ [r[2],r[3],r[4],r[5],r[6],r[7],r[0],r[9]] for r in result ]
> >> > return result
> >> >
> >> >+ def DisableComponent(self,comp_id):
> >> >+ for item in self.CurrentContent:
> >> >+ if item[0] == comp_id or item[8] == comp_id:
> >> >+ item[-1] = -1
> >> >
> >> > ## Factory class to produce different storage for different type
> >> > of meta-
> >file
> >> > class MetaFileStorage(object):
> >> > _FILE_TABLE_ = {
> >> > MODEL_FILE_INF : ModuleTable,
> >> >--
> >> >2.19.1.windows.1
> >>
> >> _______________________________________________
> >> edk2-devel mailing list
> >> edk2-devel@lists.01.org
> >> https://lists.01.org/mailman/listinfo/edk2-devel
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [Patch] BaseTools: Enable component override functionality
2019-01-14 7:18 ` Feng, Bob C
@ 2019-01-14 7:22 ` Gao, Liming
2019-01-14 7:27 ` Ard Biesheuvel
0 siblings, 1 reply; 10+ messages in thread
From: Gao, Liming @ 2019-01-14 7:22 UTC (permalink / raw)
To: Feng, Bob C, Ard Biesheuvel; +Cc: edk2-devel@lists.01.org, Carsey, Jaben
Ard:
Could you let me know which platform do you use? We will verify it for the base tools change.
Thanks
Liming
>-----Original Message-----
>From: Feng, Bob C
>Sent: Monday, January 14, 2019 3:19 PM
>To: Ard Biesheuvel <ard.biesheuvel@linaro.org>; Gao, Liming
><liming.gao@intel.com>
>Cc: edk2-devel@lists.01.org; Carsey, Jaben <jaben.carsey@intel.com>
>Subject: RE: [edk2] [Patch] BaseTools: Enable component override
>functionality
>
>Sorry for break your build.
>
>Yes. I'll double test the code before pushing it next time.
>
>-Bob
>
>-----Original Message-----
>From: Ard Biesheuvel [mailto:ard.biesheuvel@linaro.org]
>Sent: Monday, January 14, 2019 3:11 PM
>To: Gao, Liming <liming.gao@intel.com>
>Cc: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org; Carsey,
>Jaben <jaben.carsey@intel.com>
>Subject: Re: [edk2] [Patch] BaseTools: Enable component override
>functionality
>
>On Mon, 14 Jan 2019 at 08:01, Gao, Liming <liming.gao@intel.com> wrote:
>>
>> Ard:
>> Sorry for this break, Bob just sent hot fix for this issue in
>https://lists.01.org/pipermail/edk2-devel/2019-January/034947.html. Could
>you verify it?
>> In fact, it is caused by previous patch to remove EDK support in BaseTools
>Python code.
>>
>
>OK, that patch fixes it for me.
>
>But may I kindly suggest that Bob tests his code before pushing it? Thanks.
>
>
>> >-----Original Message-----
>> >From: Ard Biesheuvel [mailto:ard.biesheuvel@linaro.org]
>> >Sent: Monday, January 14, 2019 2:58 PM
>> >To: Gao, Liming <liming.gao@intel.com>
>> >Cc: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org;
>> >Carsey, Jaben <jaben.carsey@intel.com>
>> >Subject: Re: [edk2] [Patch] BaseTools: Enable component override
>> >functionality
>> >
>> >On Mon, 14 Jan 2019 at 04:00, Gao, Liming <liming.gao@intel.com> wrote:
>> >>
>> >> Reviewed-by: Liming Gao <liming.gao@intel.com>
>> >>
>> >
>> >This patch breaks the build on AArch64/GGC:
>> >
>> >(Python 2.7.13 on linux2) Traceback (most recent call last):
>> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>> >upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Pytho
>n/
>> >b
>> >uild/build.py",>
>> >line 2403, in Main
>> > MyBuild.Launch()
>> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>> >upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Pytho
>n/
>> >b
>> >uild/build.py",>
>> >line 2137, in Launch
>> > self._MultiThreadBuildPlatform()
>> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>> >upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Pytho
>n/
>> >b
>> >uild/build.py",>
>> >line 1917, in _MultiThreadBuildPlatform
>> > self.Progress
>> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>> >upstream/ws/edk2/BaseTools/Source/Python/AutoGen/AutoGen.py",>
>> >line 239, in __init__
>> > self._InitWorker(Workspace, MetaFile, Target, Toolchain, Arch,
>> >*args, **kwargs)
>> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>> >upstream/ws/edk2/BaseTools/Source/Python/AutoGen/AutoGen.py",>
>> >line 403, in _InitWorker
>> > PlatformPcds = Platform.Pcds
>> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.p
>y",>
>> >line 1194, in Pcds
>> > self._Pcds = self.UpdateStructuredPcds(MODEL_PCD_TYPE_LIST,
>> >self._Pcds)
>> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.p
>y",>
>> >line 1486, in UpdateStructuredPcds
>> > for Pcd in self.DecPcds:
>> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.p
>y",>
>> >line 3183, in DecPcds
>> > self._DecPcds, self._GuidDict = GetDeclaredPcd(self, self._Bdb,
>> >self._Arch, self._Target, self._Toolchain, PkgSet)
>> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/WorkspaceCo
>mm
>> >on.py",>
>> >line 62, in GetDeclaredPcd
>> > PkgList = GetPackageList(Platform, BuildDatabase, Arch, Target,
>> >Toolchain)
>> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/WorkspaceCo
>mm
>> >on.py",>
>> >line 44, in GetPackageList
>> > for ModuleFile in Platform.Modules:
>> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.p
>y",>
>> >line 728, in Modules
>> > self.OverrideDuplicateModule()
>> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
>> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.p
>y",>
>> >line 712, in OverrideDuplicateModule
>> > Macros["EDK_SOURCE"] = GlobalData.gEcpSource
>> >AttributeError: 'module' object has no attribute 'gEcpSource'
>> >
>> >
>> >
>> >> >-----Original Message-----
>> >> >From: Feng, Bob C
>> >> >Sent: Friday, January 11, 2019 10:39 AM
>> >> >To: edk2-devel@lists.01.org
>> >> >Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming
>> >> ><liming.gao@intel.com>; Carsey, Jaben <jaben.carsey@intel.com>
>> >> >Subject: [Patch] BaseTools: Enable component override
>> >> >functionality
>> >> >
>> >> >https://bugzilla.tianocore.org/show_bug.cgi?id=1449
>> >> >This patch enable build tools to recognize that when two given
>> >> >files have the same GUID, file path and ARCH in Dsc, The later
>> >> >one's definition will be used.
>> >> >
>> >> >Contributed-under: TianoCore Contribution Agreement 1.1
>> >> >Signed-off-by: Bob Feng <bob.c.feng@intel.com>
>> >> >Cc: Liming Gao <liming.gao@intel.com>
>> >> >Cc: Carsey Jaben <jaben.carsey@intel.com>
>> >> >---
>> >> > .../Source/Python/Workspace/DscBuildData.py | 24 ++++++++++++---
>---
>> >-
>> >> > .../Source/Python/Workspace/MetaFileParser.py | 5 ++++
>> >> > .../Source/Python/Workspace/MetaFileTable.py | 7 ++++--
>> >> > 3 files changed, 25 insertions(+), 11 deletions(-)
>> >> >
>> >> >diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py
>> >> >b/BaseTools/Source/Python/Workspace/DscBuildData.py
>> >> >index 7e82e8e934..f9805f58f5 100644
>> >> >--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
>> >> >+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
>> >> >@@ -704,36 +704,44 @@ class DscBuildData(PlatformBuildClassObject):
>> >> > if TAB_DEFAULT_STORES_DEFAULT not in self.DefaultStores:
>> >> > self.DefaultStores[TAB_DEFAULT_STORES_DEFAULT] =
>> >> >(0,
>> >> >TAB_DEFAULT_STORES_DEFAULT)
>> >> > GlobalData.gDefaultStores = sorted(self.DefaultStores.keys())
>> >> > return self.DefaultStores
>> >> >
>> >> >+ def OverrideDuplicateModule(self):
>> >> >+ RecordList = self._RawData[MODEL_META_DATA_COMPONENT,
>> >> >self._Arch]
>> >> >+ Macros = self._Macros
>> >> >+ Macros["EDK_SOURCE"] = GlobalData.gEcpSource
>> >> >+ Components = {}
>> >> >+ for Record in RecordList:
>> >> >+ ModuleId = Record[6]
>> >> >+ file_guid = self._RawData[MODEL_META_DATA_HEADER,
>> >self._Arch,
>> >> >None, ModuleId]
>> >> >+ file_guid_str = file_guid[0][2] if file_guid else "NULL"
>> >> >+ ModuleFile = PathClass(NormPath(Record[0], Macros),
>> >> >GlobalData.gWorkspace, Arch=self._Arch)
>> >> >+ if self._Arch != TAB_ARCH_COMMON and
>> >> >(file_guid_str,str(ModuleFile)) in Components:
>> >> >+
>> >> >self._RawData.DisableOverrideComponent(Components[(file_guid_str,
>s
>> >> >tr
>> >(M
>> >> >oduleFile))])
>> >> >+ Components[(file_guid_str,str(ModuleFile))] = ModuleId
>> >> >+ self._RawData._PostProcessed = False
>> >> > ## Retrieve [Components] section information
>> >> > @property
>> >> > def Modules(self):
>> >> > if self._Modules is not None:
>> >> > return self._Modules
>> >> >-
>> >> >+ self.OverrideDuplicateModule()
>> >> > self._Modules = OrderedDict()
>> >> > RecordList = self._RawData[MODEL_META_DATA_COMPONENT,
>> >> >self._Arch]
>> >> > Macros = self._Macros
>> >> > Macros["EDK_SOURCE"] = GlobalData.gEcpSource
>> >> > for Record in RecordList:
>> >> >- DuplicatedFile = False
>> >> >-
>> >> > ModuleFile = PathClass(NormPath(Record[0], Macros),
>> >> >GlobalData.gWorkspace, Arch=self._Arch)
>> >> > ModuleId = Record[6]
>> >> > LineNo = Record[7]
>> >> >
>> >> > # check the file validation
>> >> > ErrorCode, ErrorInfo = ModuleFile.Validate('.inf')
>> >> > if ErrorCode != 0:
>> >> > EdkLogger.error('build', ErrorCode,
>> >> > File=self.MetaFile,
>> >Line=LineNo,
>> >> > ExtraData=ErrorInfo)
>> >> >- # Check duplication
>> >> >- # If arch is COMMON, no duplicate module is checked since all
>> >modules
>> >> >in all component sections are selected
>> >> >- if self._Arch != TAB_ARCH_COMMON and ModuleFile in
>> >self._Modules:
>> >> >- DuplicatedFile = True
>> >> >
>> >> > Module = ModuleBuildClassObject()
>> >> > Module.MetaFile = ModuleFile
>> >> >
>> >> > # get module private library instance @@ -792,12
>> >> >+800,10 @@ class DscBuildData(PlatformBuildClassObject):
>> >> > else:
>> >> > OptionString = Module.BuildOptions[ToolChainFamily,
>ToolChain]
>> >> > Module.BuildOptions[ToolChainFamily,
>> >> >ToolChain] =
>> >OptionString +
>> >> >" " + Option
>> >> >
>> >> > RecordList = self._RawData[MODEL_META_DATA_HEADER,
>> >self._Arch,
>> >> >None, ModuleId]
>> >> >- if DuplicatedFile and not RecordList:
>> >> >- EdkLogger.error('build', FILE_DUPLICATED, File=self.MetaFile,
>> >> >ExtraData=str(ModuleFile), Line=LineNo)
>> >> > if RecordList:
>> >> > if len(RecordList) != 1:
>> >> > EdkLogger.error('build', OPTION_UNKNOWN,
>> >> >'Only FILE_GUID
>> >can
>> >> >be listed in <Defines> section.',
>> >> > File=self.MetaFile,
>> >> >ExtraData=str(ModuleFile),
>> >Line=LineNo)
>> >> > ModuleFile = ProcessDuplicatedInf(ModuleFile,
>> >> >RecordList[0][2],
>> >> >GlobalData.gWorkspace)
>> >> >diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py
>> >> >b/BaseTools/Source/Python/Workspace/MetaFileParser.py
>> >> >index 032220813b..a52e9229df 100644
>> >> >--- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
>> >> >+++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
>> >> >@@ -1711,10 +1711,15 @@ class DscParser(MetaFileParser):
>> >> >
>> >> > def __ProcessBuildOption(self):
>> >> > self._ValueList = [ReplaceMacro(Value, self._Macros,
>RaiseError=False)
>> >> > for Value in self._ValueList]
>> >> >
>> >> >+ def DisableOverrideComponent(self,module_id):
>> >> >+ for ori_id in self._IdMapping:
>> >> >+ if self._IdMapping[ori_id] == module_id:
>> >> >+ self._RawTable.DisableComponent(ori_id)
>> >> >+
>> >> > _SectionParser = {
>> >> > MODEL_META_DATA_HEADER : _DefineParser,
>> >> > MODEL_EFI_SKU_ID : _SkuIdParser,
>> >> > MODEL_EFI_DEFAULT_STORES : _DefaultStoresParser,
>> >> > MODEL_EFI_LIBRARY_INSTANCE :
>_LibraryInstanceParser,
>> >> >diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py
>> >> >b/BaseTools/Source/Python/Workspace/MetaFileTable.py
>> >> >index 004e9494c3..823a87e057 100644
>> >> >--- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
>> >> >+++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
>> >> >@@ -74,11 +74,11 @@ class MetaFileTable():
>> >> >
>> >> > def SetEndFlag(self):
>> >> > self.CurrentContent.append(self._DUMMY_)
>> >> >
>> >> > def GetAll(self):
>> >> >- return [item for item in self.CurrentContent if item[0] >= 0 ]
>> >> >+ return [item for item in self.CurrentContent if item[0]
>> >> >+ >= 0 and item[-
>> >> >1]>=0]
>> >> >
>> >> > ## Python class representation of table storing module data
>> >> >class ModuleTable(MetaFileTable):
>> >> > _COLUMN_ = '''
>> >> > ID REAL PRIMARY KEY,
>> >> >@@ -371,11 +371,10 @@ class PlatformTable(MetaFileTable):
>> >> > #
>> >> > def Query(self, Model, Scope1=None, Scope2=None,
>> >> >BelongsToItem=None, FromItem=None):
>> >> >
>> >> > QueryTab = self.CurrentContent
>> >> > result = [item for item in QueryTab if item[1] == Model
>> >> >and item[-1]>0 ]
>> >> >-
>> >> > if Scope1 is not None and Scope1 != TAB_ARCH_COMMON:
>> >> > Sc1 = set(['COMMON'])
>> >> > Sc1.add(Scope1)
>> >> > result = [item for item in result if item[5] in Sc1]
>> >> > Sc2 = set( ['COMMON','DEFAULT']) @@ -395,10 +394,14 @@
>> >> >class PlatformTable(MetaFileTable):
>> >> > result = [item for item in result if item[9] ==
>> >> >FromItem]
>> >> >
>> >> > result = [ [r[2],r[3],r[4],r[5],r[6],r[7],r[0],r[9]] for r in result ]
>> >> > return result
>> >> >
>> >> >+ def DisableComponent(self,comp_id):
>> >> >+ for item in self.CurrentContent:
>> >> >+ if item[0] == comp_id or item[8] == comp_id:
>> >> >+ item[-1] = -1
>> >> >
>> >> > ## Factory class to produce different storage for different type
>> >> > of meta-
>> >file
>> >> > class MetaFileStorage(object):
>> >> > _FILE_TABLE_ = {
>> >> > MODEL_FILE_INF : ModuleTable,
>> >> >--
>> >> >2.19.1.windows.1
>> >>
>> >> _______________________________________________
>> >> edk2-devel mailing list
>> >> edk2-devel@lists.01.org
>> >> https://lists.01.org/mailman/listinfo/edk2-devel
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [Patch] BaseTools: Enable component override functionality
2019-01-14 7:22 ` Gao, Liming
@ 2019-01-14 7:27 ` Ard Biesheuvel
0 siblings, 0 replies; 10+ messages in thread
From: Ard Biesheuvel @ 2019-01-14 7:27 UTC (permalink / raw)
To: Gao, Liming; +Cc: Feng, Bob C, edk2-devel@lists.01.org, Carsey, Jaben
On Mon, 14 Jan 2019 at 08:22, Gao, Liming <liming.gao@intel.com> wrote:
>
> Ard:
> Could you let me know which platform do you use? We will verify it for the base tools change.
>
I use a CI job here
https://ci.linaro.org/view/leg-ci/job/leg-virt-tianocore-edk2-upstream/
that builds {ArmPkg,ArmPlatformPkg,EmbeddedPkg,ArmVirtQemu,ArmVirtXen,ArmVirtQemuKernel}
x {DEBUG,RELEASE,NOOPT} x {GCC48,GCC49,GCC5,CLANG35,CLANG38} and
{OvmfIa32,OvmfIa32X64,OvmfX64} x {DEBUG,RELEASE,NOOPT} with GCC5 only.
If anyone wants to be added to cc for the failure reports, I can
easily add them.
Thanks,
Ard.
> Thanks
> Liming
> >-----Original Message-----
> >From: Feng, Bob C
> >Sent: Monday, January 14, 2019 3:19 PM
> >To: Ard Biesheuvel <ard.biesheuvel@linaro.org>; Gao, Liming
> ><liming.gao@intel.com>
> >Cc: edk2-devel@lists.01.org; Carsey, Jaben <jaben.carsey@intel.com>
> >Subject: RE: [edk2] [Patch] BaseTools: Enable component override
> >functionality
> >
> >Sorry for break your build.
> >
> >Yes. I'll double test the code before pushing it next time.
> >
> >-Bob
> >
> >-----Original Message-----
> >From: Ard Biesheuvel [mailto:ard.biesheuvel@linaro.org]
> >Sent: Monday, January 14, 2019 3:11 PM
> >To: Gao, Liming <liming.gao@intel.com>
> >Cc: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org; Carsey,
> >Jaben <jaben.carsey@intel.com>
> >Subject: Re: [edk2] [Patch] BaseTools: Enable component override
> >functionality
> >
> >On Mon, 14 Jan 2019 at 08:01, Gao, Liming <liming.gao@intel.com> wrote:
> >>
> >> Ard:
> >> Sorry for this break, Bob just sent hot fix for this issue in
> >https://lists.01.org/pipermail/edk2-devel/2019-January/034947.html. Could
> >you verify it?
> >> In fact, it is caused by previous patch to remove EDK support in BaseTools
> >Python code.
> >>
> >
> >OK, that patch fixes it for me.
> >
> >But may I kindly suggest that Bob tests his code before pushing it? Thanks.
> >
> >
> >> >-----Original Message-----
> >> >From: Ard Biesheuvel [mailto:ard.biesheuvel@linaro.org]
> >> >Sent: Monday, January 14, 2019 2:58 PM
> >> >To: Gao, Liming <liming.gao@intel.com>
> >> >Cc: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org;
> >> >Carsey, Jaben <jaben.carsey@intel.com>
> >> >Subject: Re: [edk2] [Patch] BaseTools: Enable component override
> >> >functionality
> >> >
> >> >On Mon, 14 Jan 2019 at 04:00, Gao, Liming <liming.gao@intel.com> wrote:
> >> >>
> >> >> Reviewed-by: Liming Gao <liming.gao@intel.com>
> >> >>
> >> >
> >> >This patch breaks the build on AArch64/GGC:
> >> >
> >> >(Python 2.7.13 on linux2) Traceback (most recent call last):
> >> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >> >upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Pytho
> >n/
> >> >b
> >> >uild/build.py",>
> >> >line 2403, in Main
> >> > MyBuild.Launch()
> >> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >> >upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Pytho
> >n/
> >> >b
> >> >uild/build.py",>
> >> >line 2137, in Launch
> >> > self._MultiThreadBuildPlatform()
> >> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >> >upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Pytho
> >n/
> >> >b
> >> >uild/build.py",>
> >> >line 1917, in _MultiThreadBuildPlatform
> >> > self.Progress
> >> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >> >upstream/ws/edk2/BaseTools/Source/Python/AutoGen/AutoGen.py",>
> >> >line 239, in __init__
> >> > self._InitWorker(Workspace, MetaFile, Target, Toolchain, Arch,
> >> >*args, **kwargs)
> >> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >> >upstream/ws/edk2/BaseTools/Source/Python/AutoGen/AutoGen.py",>
> >> >line 403, in _InitWorker
> >> > PlatformPcds = Platform.Pcds
> >> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.p
> >y",>
> >> >line 1194, in Pcds
> >> > self._Pcds = self.UpdateStructuredPcds(MODEL_PCD_TYPE_LIST,
> >> >self._Pcds)
> >> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.p
> >y",>
> >> >line 1486, in UpdateStructuredPcds
> >> > for Pcd in self.DecPcds:
> >> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.p
> >y",>
> >> >line 3183, in DecPcds
> >> > self._DecPcds, self._GuidDict = GetDeclaredPcd(self, self._Bdb,
> >> >self._Arch, self._Target, self._Toolchain, PkgSet)
> >> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/WorkspaceCo
> >mm
> >> >on.py",>
> >> >line 62, in GetDeclaredPcd
> >> > PkgList = GetPackageList(Platform, BuildDatabase, Arch, Target,
> >> >Toolchain)
> >> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/WorkspaceCo
> >mm
> >> >on.py",>
> >> >line 44, in GetPackageList
> >> > for ModuleFile in Platform.Modules:
> >> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.p
> >y",>
> >> >line 728, in Modules
> >> > self.OverrideDuplicateModule()
> >> > File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-
> >> >upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.p
> >y",>
> >> >line 712, in OverrideDuplicateModule
> >> > Macros["EDK_SOURCE"] = GlobalData.gEcpSource
> >> >AttributeError: 'module' object has no attribute 'gEcpSource'
> >> >
> >> >
> >> >
> >> >> >-----Original Message-----
> >> >> >From: Feng, Bob C
> >> >> >Sent: Friday, January 11, 2019 10:39 AM
> >> >> >To: edk2-devel@lists.01.org
> >> >> >Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming
> >> >> ><liming.gao@intel.com>; Carsey, Jaben <jaben.carsey@intel.com>
> >> >> >Subject: [Patch] BaseTools: Enable component override
> >> >> >functionality
> >> >> >
> >> >> >https://bugzilla.tianocore.org/show_bug.cgi?id=1449
> >> >> >This patch enable build tools to recognize that when two given
> >> >> >files have the same GUID, file path and ARCH in Dsc, The later
> >> >> >one's definition will be used.
> >> >> >
> >> >> >Contributed-under: TianoCore Contribution Agreement 1.1
> >> >> >Signed-off-by: Bob Feng <bob.c.feng@intel.com>
> >> >> >Cc: Liming Gao <liming.gao@intel.com>
> >> >> >Cc: Carsey Jaben <jaben.carsey@intel.com>
> >> >> >---
> >> >> > .../Source/Python/Workspace/DscBuildData.py | 24 ++++++++++++---
> >---
> >> >-
> >> >> > .../Source/Python/Workspace/MetaFileParser.py | 5 ++++
> >> >> > .../Source/Python/Workspace/MetaFileTable.py | 7 ++++--
> >> >> > 3 files changed, 25 insertions(+), 11 deletions(-)
> >> >> >
> >> >> >diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py
> >> >> >b/BaseTools/Source/Python/Workspace/DscBuildData.py
> >> >> >index 7e82e8e934..f9805f58f5 100644
> >> >> >--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
> >> >> >+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
> >> >> >@@ -704,36 +704,44 @@ class DscBuildData(PlatformBuildClassObject):
> >> >> > if TAB_DEFAULT_STORES_DEFAULT not in self.DefaultStores:
> >> >> > self.DefaultStores[TAB_DEFAULT_STORES_DEFAULT] =
> >> >> >(0,
> >> >> >TAB_DEFAULT_STORES_DEFAULT)
> >> >> > GlobalData.gDefaultStores = sorted(self.DefaultStores.keys())
> >> >> > return self.DefaultStores
> >> >> >
> >> >> >+ def OverrideDuplicateModule(self):
> >> >> >+ RecordList = self._RawData[MODEL_META_DATA_COMPONENT,
> >> >> >self._Arch]
> >> >> >+ Macros = self._Macros
> >> >> >+ Macros["EDK_SOURCE"] = GlobalData.gEcpSource
> >> >> >+ Components = {}
> >> >> >+ for Record in RecordList:
> >> >> >+ ModuleId = Record[6]
> >> >> >+ file_guid = self._RawData[MODEL_META_DATA_HEADER,
> >> >self._Arch,
> >> >> >None, ModuleId]
> >> >> >+ file_guid_str = file_guid[0][2] if file_guid else "NULL"
> >> >> >+ ModuleFile = PathClass(NormPath(Record[0], Macros),
> >> >> >GlobalData.gWorkspace, Arch=self._Arch)
> >> >> >+ if self._Arch != TAB_ARCH_COMMON and
> >> >> >(file_guid_str,str(ModuleFile)) in Components:
> >> >> >+
> >> >> >self._RawData.DisableOverrideComponent(Components[(file_guid_str,
> >s
> >> >> >tr
> >> >(M
> >> >> >oduleFile))])
> >> >> >+ Components[(file_guid_str,str(ModuleFile))] = ModuleId
> >> >> >+ self._RawData._PostProcessed = False
> >> >> > ## Retrieve [Components] section information
> >> >> > @property
> >> >> > def Modules(self):
> >> >> > if self._Modules is not None:
> >> >> > return self._Modules
> >> >> >-
> >> >> >+ self.OverrideDuplicateModule()
> >> >> > self._Modules = OrderedDict()
> >> >> > RecordList = self._RawData[MODEL_META_DATA_COMPONENT,
> >> >> >self._Arch]
> >> >> > Macros = self._Macros
> >> >> > Macros["EDK_SOURCE"] = GlobalData.gEcpSource
> >> >> > for Record in RecordList:
> >> >> >- DuplicatedFile = False
> >> >> >-
> >> >> > ModuleFile = PathClass(NormPath(Record[0], Macros),
> >> >> >GlobalData.gWorkspace, Arch=self._Arch)
> >> >> > ModuleId = Record[6]
> >> >> > LineNo = Record[7]
> >> >> >
> >> >> > # check the file validation
> >> >> > ErrorCode, ErrorInfo = ModuleFile.Validate('.inf')
> >> >> > if ErrorCode != 0:
> >> >> > EdkLogger.error('build', ErrorCode,
> >> >> > File=self.MetaFile,
> >> >Line=LineNo,
> >> >> > ExtraData=ErrorInfo)
> >> >> >- # Check duplication
> >> >> >- # If arch is COMMON, no duplicate module is checked since all
> >> >modules
> >> >> >in all component sections are selected
> >> >> >- if self._Arch != TAB_ARCH_COMMON and ModuleFile in
> >> >self._Modules:
> >> >> >- DuplicatedFile = True
> >> >> >
> >> >> > Module = ModuleBuildClassObject()
> >> >> > Module.MetaFile = ModuleFile
> >> >> >
> >> >> > # get module private library instance @@ -792,12
> >> >> >+800,10 @@ class DscBuildData(PlatformBuildClassObject):
> >> >> > else:
> >> >> > OptionString = Module.BuildOptions[ToolChainFamily,
> >ToolChain]
> >> >> > Module.BuildOptions[ToolChainFamily,
> >> >> >ToolChain] =
> >> >OptionString +
> >> >> >" " + Option
> >> >> >
> >> >> > RecordList = self._RawData[MODEL_META_DATA_HEADER,
> >> >self._Arch,
> >> >> >None, ModuleId]
> >> >> >- if DuplicatedFile and not RecordList:
> >> >> >- EdkLogger.error('build', FILE_DUPLICATED, File=self.MetaFile,
> >> >> >ExtraData=str(ModuleFile), Line=LineNo)
> >> >> > if RecordList:
> >> >> > if len(RecordList) != 1:
> >> >> > EdkLogger.error('build', OPTION_UNKNOWN,
> >> >> >'Only FILE_GUID
> >> >can
> >> >> >be listed in <Defines> section.',
> >> >> > File=self.MetaFile,
> >> >> >ExtraData=str(ModuleFile),
> >> >Line=LineNo)
> >> >> > ModuleFile = ProcessDuplicatedInf(ModuleFile,
> >> >> >RecordList[0][2],
> >> >> >GlobalData.gWorkspace)
> >> >> >diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >> >> >b/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >> >> >index 032220813b..a52e9229df 100644
> >> >> >--- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >> >> >+++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >> >> >@@ -1711,10 +1711,15 @@ class DscParser(MetaFileParser):
> >> >> >
> >> >> > def __ProcessBuildOption(self):
> >> >> > self._ValueList = [ReplaceMacro(Value, self._Macros,
> >RaiseError=False)
> >> >> > for Value in self._ValueList]
> >> >> >
> >> >> >+ def DisableOverrideComponent(self,module_id):
> >> >> >+ for ori_id in self._IdMapping:
> >> >> >+ if self._IdMapping[ori_id] == module_id:
> >> >> >+ self._RawTable.DisableComponent(ori_id)
> >> >> >+
> >> >> > _SectionParser = {
> >> >> > MODEL_META_DATA_HEADER : _DefineParser,
> >> >> > MODEL_EFI_SKU_ID : _SkuIdParser,
> >> >> > MODEL_EFI_DEFAULT_STORES : _DefaultStoresParser,
> >> >> > MODEL_EFI_LIBRARY_INSTANCE :
> >_LibraryInstanceParser,
> >> >> >diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >> >> >b/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >> >> >index 004e9494c3..823a87e057 100644
> >> >> >--- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >> >> >+++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >> >> >@@ -74,11 +74,11 @@ class MetaFileTable():
> >> >> >
> >> >> > def SetEndFlag(self):
> >> >> > self.CurrentContent.append(self._DUMMY_)
> >> >> >
> >> >> > def GetAll(self):
> >> >> >- return [item for item in self.CurrentContent if item[0] >= 0 ]
> >> >> >+ return [item for item in self.CurrentContent if item[0]
> >> >> >+ >= 0 and item[-
> >> >> >1]>=0]
> >> >> >
> >> >> > ## Python class representation of table storing module data
> >> >> >class ModuleTable(MetaFileTable):
> >> >> > _COLUMN_ = '''
> >> >> > ID REAL PRIMARY KEY,
> >> >> >@@ -371,11 +371,10 @@ class PlatformTable(MetaFileTable):
> >> >> > #
> >> >> > def Query(self, Model, Scope1=None, Scope2=None,
> >> >> >BelongsToItem=None, FromItem=None):
> >> >> >
> >> >> > QueryTab = self.CurrentContent
> >> >> > result = [item for item in QueryTab if item[1] == Model
> >> >> >and item[-1]>0 ]
> >> >> >-
> >> >> > if Scope1 is not None and Scope1 != TAB_ARCH_COMMON:
> >> >> > Sc1 = set(['COMMON'])
> >> >> > Sc1.add(Scope1)
> >> >> > result = [item for item in result if item[5] in Sc1]
> >> >> > Sc2 = set( ['COMMON','DEFAULT']) @@ -395,10 +394,14 @@
> >> >> >class PlatformTable(MetaFileTable):
> >> >> > result = [item for item in result if item[9] ==
> >> >> >FromItem]
> >> >> >
> >> >> > result = [ [r[2],r[3],r[4],r[5],r[6],r[7],r[0],r[9]] for r in result ]
> >> >> > return result
> >> >> >
> >> >> >+ def DisableComponent(self,comp_id):
> >> >> >+ for item in self.CurrentContent:
> >> >> >+ if item[0] == comp_id or item[8] == comp_id:
> >> >> >+ item[-1] = -1
> >> >> >
> >> >> > ## Factory class to produce different storage for different type
> >> >> > of meta-
> >> >file
> >> >> > class MetaFileStorage(object):
> >> >> > _FILE_TABLE_ = {
> >> >> > MODEL_FILE_INF : ModuleTable,
> >> >> >--
> >> >> >2.19.1.windows.1
> >> >>
> >> >> _______________________________________________
> >> >> edk2-devel mailing list
> >> >> edk2-devel@lists.01.org
> >> >> https://lists.01.org/mailman/listinfo/edk2-devel
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [Patch] BaseTools: Enable component override functionality
2019-01-11 2:38 [Patch] BaseTools: Enable component override functionality BobCF
2019-01-14 3:00 ` Gao, Liming
@ 2019-02-22 22:06 ` Felix Polyudov
2019-02-23 8:37 ` Feng, Bob C
1 sibling, 1 reply; 10+ messages in thread
From: Felix Polyudov @ 2019-02-22 22:06 UTC (permalink / raw)
To: 'BobCF', edk2-devel@lists.01.org; +Cc: Carsey Jaben, Liming Gao
Bob,
Do you know if DSC specification has been updated to document this new behavior?
If not, it should be updated.
-----Original Message-----
From: edk2-devel [mailto:edk2-devel-bounces@lists.01.org] On Behalf Of BobCF
Sent: Thursday, January 10, 2019 9:39 PM
To: edk2-devel@lists.01.org
Cc: Carsey Jaben; Liming Gao
Subject: [edk2] [Patch] BaseTools: Enable component override functionality
https://bugzilla.tianocore.org/show_bug.cgi?id=1449
This patch enable build tools to recognize that
when two given files have the same GUID, file path and ARCH in Dsc,
The later one's definition will be used.
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Cc: Carsey Jaben <jaben.carsey@intel.com>
---
.../Source/Python/Workspace/DscBuildData.py | 24 ++++++++++++-------
.../Source/Python/Workspace/MetaFileParser.py | 5 ++++
.../Source/Python/Workspace/MetaFileTable.py | 7 ++++--
3 files changed, 25 insertions(+), 11 deletions(-)
diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py
index 7e82e8e934..f9805f58f5 100644
--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
@@ -704,36 +704,44 @@ class DscBuildData(PlatformBuildClassObject):
if TAB_DEFAULT_STORES_DEFAULT not in self.DefaultStores:
self.DefaultStores[TAB_DEFAULT_STORES_DEFAULT] = (0, TAB_DEFAULT_STORES_DEFAULT)
GlobalData.gDefaultStores = sorted(self.DefaultStores.keys())
return self.DefaultStores
+ def OverrideDuplicateModule(self):
+ RecordList = self._RawData[MODEL_META_DATA_COMPONENT, self._Arch]
+ Macros = self._Macros
+ Macros["EDK_SOURCE"] = GlobalData.gEcpSource
+ Components = {}
+ for Record in RecordList:
+ ModuleId = Record[6]
+ file_guid = self._RawData[MODEL_META_DATA_HEADER, self._Arch, None, ModuleId]
+ file_guid_str = file_guid[0][2] if file_guid else "NULL"
+ ModuleFile = PathClass(NormPath(Record[0], Macros), GlobalData.gWorkspace, Arch=self._Arch)
+ if self._Arch != TAB_ARCH_COMMON and (file_guid_str,str(ModuleFile)) in Components:
+ self._RawData.DisableOverrideComponent(Components[(file_guid_str,str(ModuleFile))])
+ Components[(file_guid_str,str(ModuleFile))] = ModuleId
+ self._RawData._PostProcessed = False
## Retrieve [Components] section information
@property
def Modules(self):
if self._Modules is not None:
return self._Modules
-
+ self.OverrideDuplicateModule()
self._Modules = OrderedDict()
RecordList = self._RawData[MODEL_META_DATA_COMPONENT, self._Arch]
Macros = self._Macros
Macros["EDK_SOURCE"] = GlobalData.gEcpSource
for Record in RecordList:
- DuplicatedFile = False
-
ModuleFile = PathClass(NormPath(Record[0], Macros), GlobalData.gWorkspace, Arch=self._Arch)
ModuleId = Record[6]
LineNo = Record[7]
# check the file validation
ErrorCode, ErrorInfo = ModuleFile.Validate('.inf')
if ErrorCode != 0:
EdkLogger.error('build', ErrorCode, File=self.MetaFile, Line=LineNo,
ExtraData=ErrorInfo)
- # Check duplication
- # If arch is COMMON, no duplicate module is checked since all modules in all component sections are selected
- if self._Arch != TAB_ARCH_COMMON and ModuleFile in self._Modules:
- DuplicatedFile = True
Module = ModuleBuildClassObject()
Module.MetaFile = ModuleFile
# get module private library instance
@@ -792,12 +800,10 @@ class DscBuildData(PlatformBuildClassObject):
else:
OptionString = Module.BuildOptions[ToolChainFamily, ToolChain]
Module.BuildOptions[ToolChainFamily, ToolChain] = OptionString + " " + Option
RecordList = self._RawData[MODEL_META_DATA_HEADER, self._Arch, None, ModuleId]
- if DuplicatedFile and not RecordList:
- EdkLogger.error('build', FILE_DUPLICATED, File=self.MetaFile, ExtraData=str(ModuleFile), Line=LineNo)
if RecordList:
if len(RecordList) != 1:
EdkLogger.error('build', OPTION_UNKNOWN, 'Only FILE_GUID can be listed in <Defines> section.',
File=self.MetaFile, ExtraData=str(ModuleFile), Line=LineNo)
ModuleFile = ProcessDuplicatedInf(ModuleFile, RecordList[0][2], GlobalData.gWorkspace)
diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py b/BaseTools/Source/Python/Workspace/MetaFileParser.py
index 032220813b..a52e9229df 100644
--- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
+++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
@@ -1711,10 +1711,15 @@ class DscParser(MetaFileParser):
def __ProcessBuildOption(self):
self._ValueList = [ReplaceMacro(Value, self._Macros, RaiseError=False)
for Value in self._ValueList]
+ def DisableOverrideComponent(self,module_id):
+ for ori_id in self._IdMapping:
+ if self._IdMapping[ori_id] == module_id:
+ self._RawTable.DisableComponent(ori_id)
+
_SectionParser = {
MODEL_META_DATA_HEADER : _DefineParser,
MODEL_EFI_SKU_ID : _SkuIdParser,
MODEL_EFI_DEFAULT_STORES : _DefaultStoresParser,
MODEL_EFI_LIBRARY_INSTANCE : _LibraryInstanceParser,
diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py b/BaseTools/Source/Python/Workspace/MetaFileTable.py
index 004e9494c3..823a87e057 100644
--- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
+++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
@@ -74,11 +74,11 @@ class MetaFileTable():
def SetEndFlag(self):
self.CurrentContent.append(self._DUMMY_)
def GetAll(self):
- return [item for item in self.CurrentContent if item[0] >= 0 ]
+ return [item for item in self.CurrentContent if item[0] >= 0 and item[-1]>=0]
## Python class representation of table storing module data
class ModuleTable(MetaFileTable):
_COLUMN_ = '''
ID REAL PRIMARY KEY,
@@ -371,11 +371,10 @@ class PlatformTable(MetaFileTable):
#
def Query(self, Model, Scope1=None, Scope2=None, BelongsToItem=None, FromItem=None):
QueryTab = self.CurrentContent
result = [item for item in QueryTab if item[1] == Model and item[-1]>0 ]
-
if Scope1 is not None and Scope1 != TAB_ARCH_COMMON:
Sc1 = set(['COMMON'])
Sc1.add(Scope1)
result = [item for item in result if item[5] in Sc1]
Sc2 = set( ['COMMON','DEFAULT'])
@@ -395,10 +394,14 @@ class PlatformTable(MetaFileTable):
result = [item for item in result if item[9] == FromItem]
result = [ [r[2],r[3],r[4],r[5],r[6],r[7],r[0],r[9]] for r in result ]
return result
+ def DisableComponent(self,comp_id):
+ for item in self.CurrentContent:
+ if item[0] == comp_id or item[8] == comp_id:
+ item[-1] = -1
## Factory class to produce different storage for different type of meta-file
class MetaFileStorage(object):
_FILE_TABLE_ = {
MODEL_FILE_INF : ModuleTable,
--
2.19.1.windows.1
_______________________________________________
edk2-devel mailing list
edk2-devel@lists.01.org
https://lists.01.org/mailman/listinfo/edk2-devel
Please consider the environment before printing this email.
The information contained in this message may be confidential and proprietary to American Megatrends, Inc. This communication is intended to be read only by the individual or entity to whom it is addressed or by their designee. If the reader of this message is not the intended recipient, you are on notice that any distribution of this message, in any form, is strictly prohibited. Please promptly notify the sender by reply e-mail or by telephone at 770-246-8600, and then delete or destroy all copies of the transmission.
^ permalink raw reply related [flat|nested] 10+ messages in thread
* Re: [Patch] BaseTools: Enable component override functionality
2019-02-22 22:06 ` Felix Polyudov
@ 2019-02-23 8:37 ` Feng, Bob C
0 siblings, 0 replies; 10+ messages in thread
From: Feng, Bob C @ 2019-02-23 8:37 UTC (permalink / raw)
To: Felix Polyudov, edk2-devel@lists.01.org; +Cc: Carsey, Jaben, Gao, Liming
Felix,
I agree. I filed a new BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1557 to track the document update.
I think this new behavior would be better to be recorded in the Build spec.
Thanks,
Bob
-----Original Message-----
From: Felix Polyudov [mailto:Felixp@ami.com]
Sent: Saturday, February 23, 2019 6:06 AM
To: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org
Cc: Carsey, Jaben <jaben.carsey@intel.com>; Gao, Liming <liming.gao@intel.com>
Subject: RE: [edk2] [Patch] BaseTools: Enable component override functionality
Bob,
Do you know if DSC specification has been updated to document this new behavior?
If not, it should be updated.
-----Original Message-----
From: edk2-devel [mailto:edk2-devel-bounces@lists.01.org] On Behalf Of BobCF
Sent: Thursday, January 10, 2019 9:39 PM
To: edk2-devel@lists.01.org
Cc: Carsey Jaben; Liming Gao
Subject: [edk2] [Patch] BaseTools: Enable component override functionality
https://bugzilla.tianocore.org/show_bug.cgi?id=1449
This patch enable build tools to recognize that when two given files have the same GUID, file path and ARCH in Dsc, The later one's definition will be used.
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Cc: Carsey Jaben <jaben.carsey@intel.com>
---
.../Source/Python/Workspace/DscBuildData.py | 24 ++++++++++++-------
.../Source/Python/Workspace/MetaFileParser.py | 5 ++++ .../Source/Python/Workspace/MetaFileTable.py | 7 ++++--
3 files changed, 25 insertions(+), 11 deletions(-)
diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py
index 7e82e8e934..f9805f58f5 100644
--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
@@ -704,36 +704,44 @@ class DscBuildData(PlatformBuildClassObject):
if TAB_DEFAULT_STORES_DEFAULT not in self.DefaultStores:
self.DefaultStores[TAB_DEFAULT_STORES_DEFAULT] = (0, TAB_DEFAULT_STORES_DEFAULT)
GlobalData.gDefaultStores = sorted(self.DefaultStores.keys())
return self.DefaultStores
+ def OverrideDuplicateModule(self):
+ RecordList = self._RawData[MODEL_META_DATA_COMPONENT, self._Arch]
+ Macros = self._Macros
+ Macros["EDK_SOURCE"] = GlobalData.gEcpSource
+ Components = {}
+ for Record in RecordList:
+ ModuleId = Record[6]
+ file_guid = self._RawData[MODEL_META_DATA_HEADER, self._Arch, None, ModuleId]
+ file_guid_str = file_guid[0][2] if file_guid else "NULL"
+ ModuleFile = PathClass(NormPath(Record[0], Macros), GlobalData.gWorkspace, Arch=self._Arch)
+ if self._Arch != TAB_ARCH_COMMON and (file_guid_str,str(ModuleFile)) in Components:
+ self._RawData.DisableOverrideComponent(Components[(file_guid_str,str(ModuleFile))])
+ Components[(file_guid_str,str(ModuleFile))] = ModuleId
+ self._RawData._PostProcessed = False
## Retrieve [Components] section information
@property
def Modules(self):
if self._Modules is not None:
return self._Modules
-
+ self.OverrideDuplicateModule()
self._Modules = OrderedDict()
RecordList = self._RawData[MODEL_META_DATA_COMPONENT, self._Arch]
Macros = self._Macros
Macros["EDK_SOURCE"] = GlobalData.gEcpSource
for Record in RecordList:
- DuplicatedFile = False
-
ModuleFile = PathClass(NormPath(Record[0], Macros), GlobalData.gWorkspace, Arch=self._Arch)
ModuleId = Record[6]
LineNo = Record[7]
# check the file validation
ErrorCode, ErrorInfo = ModuleFile.Validate('.inf')
if ErrorCode != 0:
EdkLogger.error('build', ErrorCode, File=self.MetaFile, Line=LineNo,
ExtraData=ErrorInfo)
- # Check duplication
- # If arch is COMMON, no duplicate module is checked since all modules in all component sections are selected
- if self._Arch != TAB_ARCH_COMMON and ModuleFile in self._Modules:
- DuplicatedFile = True
Module = ModuleBuildClassObject()
Module.MetaFile = ModuleFile
# get module private library instance @@ -792,12 +800,10 @@ class DscBuildData(PlatformBuildClassObject):
else:
OptionString = Module.BuildOptions[ToolChainFamily, ToolChain]
Module.BuildOptions[ToolChainFamily, ToolChain] = OptionString + " " + Option
RecordList = self._RawData[MODEL_META_DATA_HEADER, self._Arch, None, ModuleId]
- if DuplicatedFile and not RecordList:
- EdkLogger.error('build', FILE_DUPLICATED, File=self.MetaFile, ExtraData=str(ModuleFile), Line=LineNo)
if RecordList:
if len(RecordList) != 1:
EdkLogger.error('build', OPTION_UNKNOWN, 'Only FILE_GUID can be listed in <Defines> section.',
File=self.MetaFile, ExtraData=str(ModuleFile), Line=LineNo)
ModuleFile = ProcessDuplicatedInf(ModuleFile, RecordList[0][2], GlobalData.gWorkspace) diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py b/BaseTools/Source/Python/Workspace/MetaFileParser.py
index 032220813b..a52e9229df 100644
--- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
+++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
@@ -1711,10 +1711,15 @@ class DscParser(MetaFileParser):
def __ProcessBuildOption(self):
self._ValueList = [ReplaceMacro(Value, self._Macros, RaiseError=False)
for Value in self._ValueList]
+ def DisableOverrideComponent(self,module_id):
+ for ori_id in self._IdMapping:
+ if self._IdMapping[ori_id] == module_id:
+ self._RawTable.DisableComponent(ori_id)
+
_SectionParser = {
MODEL_META_DATA_HEADER : _DefineParser,
MODEL_EFI_SKU_ID : _SkuIdParser,
MODEL_EFI_DEFAULT_STORES : _DefaultStoresParser,
MODEL_EFI_LIBRARY_INSTANCE : _LibraryInstanceParser,
diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py b/BaseTools/Source/Python/Workspace/MetaFileTable.py
index 004e9494c3..823a87e057 100644
--- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
+++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
@@ -74,11 +74,11 @@ class MetaFileTable():
def SetEndFlag(self):
self.CurrentContent.append(self._DUMMY_)
def GetAll(self):
- return [item for item in self.CurrentContent if item[0] >= 0 ]
+ return [item for item in self.CurrentContent if item[0] >= 0
+ and item[-1]>=0]
## Python class representation of table storing module data class ModuleTable(MetaFileTable):
_COLUMN_ = '''
ID REAL PRIMARY KEY,
@@ -371,11 +371,10 @@ class PlatformTable(MetaFileTable):
#
def Query(self, Model, Scope1=None, Scope2=None, BelongsToItem=None, FromItem=None):
QueryTab = self.CurrentContent
result = [item for item in QueryTab if item[1] == Model and item[-1]>0 ]
-
if Scope1 is not None and Scope1 != TAB_ARCH_COMMON:
Sc1 = set(['COMMON'])
Sc1.add(Scope1)
result = [item for item in result if item[5] in Sc1]
Sc2 = set( ['COMMON','DEFAULT']) @@ -395,10 +394,14 @@ class PlatformTable(MetaFileTable):
result = [item for item in result if item[9] == FromItem]
result = [ [r[2],r[3],r[4],r[5],r[6],r[7],r[0],r[9]] for r in result ]
return result
+ def DisableComponent(self,comp_id):
+ for item in self.CurrentContent:
+ if item[0] == comp_id or item[8] == comp_id:
+ item[-1] = -1
## Factory class to produce different storage for different type of meta-file class MetaFileStorage(object):
_FILE_TABLE_ = {
MODEL_FILE_INF : ModuleTable,
--
2.19.1.windows.1
_______________________________________________
edk2-devel mailing list
edk2-devel@lists.01.org
https://lists.01.org/mailman/listinfo/edk2-devel
Please consider the environment before printing this email.
The information contained in this message may be confidential and proprietary to American Megatrends, Inc. This communication is intended to be read only by the individual or entity to whom it is addressed or by their designee. If the reader of this message is not the intended recipient, you are on notice that any distribution of this message, in any form, is strictly prohibited. Please promptly notify the sender by reply e-mail or by telephone at 770-246-8600, and then delete or destroy all copies of the transmission.
^ permalink raw reply related [flat|nested] 10+ messages in thread
end of thread, other threads:[~2019-02-23 8:37 UTC | newest]
Thread overview: 10+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2019-01-11 2:38 [Patch] BaseTools: Enable component override functionality BobCF
2019-01-14 3:00 ` Gao, Liming
2019-01-14 6:57 ` Ard Biesheuvel
2019-01-14 7:01 ` Gao, Liming
2019-01-14 7:10 ` Ard Biesheuvel
2019-01-14 7:18 ` Feng, Bob C
2019-01-14 7:22 ` Gao, Liming
2019-01-14 7:27 ` Ard Biesheuvel
2019-02-22 22:06 ` Felix Polyudov
2019-02-23 8:37 ` Feng, Bob C
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox