Re: [Patch] BaseTools: Enable component override functionality

[Ard Biesheuvel <ard.biesheuvel@linaro.org>]

On Mon, 14 Jan 2019 at 04:00, Gao, Liming <liming.gao@intel.com> wrote:
>
> Reviewed-by: Liming Gao <liming.gao@intel.com>
>

This patch breaks the build on AArch64/GGC:

(Python 2.7.13 on linux2) Traceback (most recent call last):
  File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Python/build/build.py",>
line 2403, in Main
    MyBuild.Launch()
  File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Python/build/build.py",>
line 2137, in Launch
    self._MultiThreadBuildPlatform()
  File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/BinWrappers/PosixLike/../../Source/Python/build/build.py",>
line 1917, in _MultiThreadBuildPlatform
    self.Progress
  File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/AutoGen/AutoGen.py",>
line 239, in __init__
    self._InitWorker(Workspace, MetaFile, Target, Toolchain, Arch,
*args, **kwargs)
  File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/AutoGen/AutoGen.py",>
line 403, in _InitWorker
    PlatformPcds = Platform.Pcds
  File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
line 1194, in Pcds
    self._Pcds = self.UpdateStructuredPcds(MODEL_PCD_TYPE_LIST, self._Pcds)
  File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
line 1486, in UpdateStructuredPcds
    for Pcd in self.DecPcds:
  File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
line 3183, in DecPcds
    self._DecPcds, self._GuidDict = GetDeclaredPcd(self, self._Bdb,
self._Arch, self._Target, self._Toolchain, PkgSet)
  File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/Workspace/WorkspaceCommon.py",>
line 62, in GetDeclaredPcd
    PkgList = GetPackageList(Platform, BuildDatabase, Arch, Target, Toolchain)
  File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/Workspace/WorkspaceCommon.py",>
line 44, in GetPackageList
    for ModuleFile in Platform.Modules:
  File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
line 728, in Modules
    self.OverrideDuplicateModule()
  File "<https://ci.linaro.org/job/leg-virt-tianocore-edk2-upstream/ws/edk2/BaseTools/Source/Python/Workspace/DscBuildData.py",>
line 712, in OverrideDuplicateModule
    Macros["EDK_SOURCE"] = GlobalData.gEcpSource
AttributeError: 'module' object has no attribute 'gEcpSource'



> >-----Original Message-----
> >From: Feng, Bob C
> >Sent: Friday, January 11, 2019 10:39 AM
> >To: edk2-devel@lists.01.org
> >Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming
> ><liming.gao@intel.com>; Carsey, Jaben <jaben.carsey@intel.com>
> >Subject: [Patch] BaseTools: Enable component override functionality
> >
> >https://bugzilla.tianocore.org/show_bug.cgi?id=1449
> >This patch enable build tools to recognize that
> >when two given files have the same GUID, file path and ARCH in Dsc,
> >The later one's definition will be used.
> >
> >Contributed-under: TianoCore Contribution Agreement 1.1
> >Signed-off-by: Bob Feng <bob.c.feng@intel.com>
> >Cc: Liming Gao <liming.gao@intel.com>
> >Cc: Carsey Jaben <jaben.carsey@intel.com>
> >---
> > .../Source/Python/Workspace/DscBuildData.py   | 24 ++++++++++++-------
> > .../Source/Python/Workspace/MetaFileParser.py |  5 ++++
> > .../Source/Python/Workspace/MetaFileTable.py  |  7 ++++--
> > 3 files changed, 25 insertions(+), 11 deletions(-)
> >
> >diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py
> >b/BaseTools/Source/Python/Workspace/DscBuildData.py
> >index 7e82e8e934..f9805f58f5 100644
> >--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
> >+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
> >@@ -704,36 +704,44 @@ class DscBuildData(PlatformBuildClassObject):
> >             if TAB_DEFAULT_STORES_DEFAULT not in self.DefaultStores:
> >                 self.DefaultStores[TAB_DEFAULT_STORES_DEFAULT] = (0,
> >TAB_DEFAULT_STORES_DEFAULT)
> >             GlobalData.gDefaultStores = sorted(self.DefaultStores.keys())
> >         return self.DefaultStores
> >
> >+    def OverrideDuplicateModule(self):
> >+        RecordList = self._RawData[MODEL_META_DATA_COMPONENT,
> >self._Arch]
> >+        Macros = self._Macros
> >+        Macros["EDK_SOURCE"] = GlobalData.gEcpSource
> >+        Components = {}
> >+        for Record in RecordList:
> >+            ModuleId = Record[6]
> >+            file_guid = self._RawData[MODEL_META_DATA_HEADER, self._Arch,
> >None, ModuleId]
> >+            file_guid_str = file_guid[0][2] if file_guid else "NULL"
> >+            ModuleFile = PathClass(NormPath(Record[0], Macros),
> >GlobalData.gWorkspace, Arch=self._Arch)
> >+            if self._Arch != TAB_ARCH_COMMON and
> >(file_guid_str,str(ModuleFile)) in Components:
> >+
> >self._RawData.DisableOverrideComponent(Components[(file_guid_str,str(M
> >oduleFile))])
> >+            Components[(file_guid_str,str(ModuleFile))] = ModuleId
> >+        self._RawData._PostProcessed = False
> >     ## Retrieve [Components] section information
> >     @property
> >     def Modules(self):
> >         if self._Modules is not None:
> >             return self._Modules
> >-
> >+        self.OverrideDuplicateModule()
> >         self._Modules = OrderedDict()
> >         RecordList = self._RawData[MODEL_META_DATA_COMPONENT,
> >self._Arch]
> >         Macros = self._Macros
> >         Macros["EDK_SOURCE"] = GlobalData.gEcpSource
> >         for Record in RecordList:
> >-            DuplicatedFile = False
> >-
> >             ModuleFile = PathClass(NormPath(Record[0], Macros),
> >GlobalData.gWorkspace, Arch=self._Arch)
> >             ModuleId = Record[6]
> >             LineNo = Record[7]
> >
> >             # check the file validation
> >             ErrorCode, ErrorInfo = ModuleFile.Validate('.inf')
> >             if ErrorCode != 0:
> >                 EdkLogger.error('build', ErrorCode, File=self.MetaFile, Line=LineNo,
> >                                 ExtraData=ErrorInfo)
> >-            # Check duplication
> >-            # If arch is COMMON, no duplicate module is checked since all modules
> >in all component sections are selected
> >-            if self._Arch != TAB_ARCH_COMMON and ModuleFile in self._Modules:
> >-                DuplicatedFile = True
> >
> >             Module = ModuleBuildClassObject()
> >             Module.MetaFile = ModuleFile
> >
> >             # get module private library instance
> >@@ -792,12 +800,10 @@ class DscBuildData(PlatformBuildClassObject):
> >                 else:
> >                     OptionString = Module.BuildOptions[ToolChainFamily, ToolChain]
> >                     Module.BuildOptions[ToolChainFamily, ToolChain] = OptionString +
> >" " + Option
> >
> >             RecordList = self._RawData[MODEL_META_DATA_HEADER, self._Arch,
> >None, ModuleId]
> >-            if DuplicatedFile and not RecordList:
> >-                EdkLogger.error('build', FILE_DUPLICATED, File=self.MetaFile,
> >ExtraData=str(ModuleFile), Line=LineNo)
> >             if RecordList:
> >                 if len(RecordList) != 1:
> >                     EdkLogger.error('build', OPTION_UNKNOWN, 'Only FILE_GUID can
> >be listed in <Defines> section.',
> >                                     File=self.MetaFile, ExtraData=str(ModuleFile), Line=LineNo)
> >                 ModuleFile = ProcessDuplicatedInf(ModuleFile, RecordList[0][2],
> >GlobalData.gWorkspace)
> >diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >b/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >index 032220813b..a52e9229df 100644
> >--- a/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >+++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py
> >@@ -1711,10 +1711,15 @@ class DscParser(MetaFileParser):
> >
> >     def __ProcessBuildOption(self):
> >         self._ValueList = [ReplaceMacro(Value, self._Macros, RaiseError=False)
> >                            for Value in self._ValueList]
> >
> >+    def DisableOverrideComponent(self,module_id):
> >+        for ori_id in self._IdMapping:
> >+            if self._IdMapping[ori_id] == module_id:
> >+                self._RawTable.DisableComponent(ori_id)
> >+
> >     _SectionParser = {
> >         MODEL_META_DATA_HEADER                          :   _DefineParser,
> >         MODEL_EFI_SKU_ID                                :   _SkuIdParser,
> >         MODEL_EFI_DEFAULT_STORES                        :   _DefaultStoresParser,
> >         MODEL_EFI_LIBRARY_INSTANCE                      :   _LibraryInstanceParser,
> >diff --git a/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >b/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >index 004e9494c3..823a87e057 100644
> >--- a/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >+++ b/BaseTools/Source/Python/Workspace/MetaFileTable.py
> >@@ -74,11 +74,11 @@ class MetaFileTable():
> >
> >     def SetEndFlag(self):
> >         self.CurrentContent.append(self._DUMMY_)
> >
> >     def GetAll(self):
> >-        return [item for item in self.CurrentContent if item[0] >= 0 ]
> >+        return [item for item in self.CurrentContent if item[0] >= 0 and item[-
> >1]>=0]
> >
> > ## Python class representation of table storing module data
> > class ModuleTable(MetaFileTable):
> >     _COLUMN_ = '''
> >         ID REAL PRIMARY KEY,
> >@@ -371,11 +371,10 @@ class PlatformTable(MetaFileTable):
> >     #
> >     def Query(self, Model, Scope1=None, Scope2=None,
> >BelongsToItem=None, FromItem=None):
> >
> >         QueryTab = self.CurrentContent
> >         result = [item for item in QueryTab if item[1] == Model and item[-1]>0 ]
> >-
> >         if Scope1 is not None and Scope1 != TAB_ARCH_COMMON:
> >             Sc1 = set(['COMMON'])
> >             Sc1.add(Scope1)
> >             result = [item for item in result if item[5] in Sc1]
> >         Sc2 = set( ['COMMON','DEFAULT'])
> >@@ -395,10 +394,14 @@ class PlatformTable(MetaFileTable):
> >             result = [item for item in result if item[9] == FromItem]
> >
> >         result = [ [r[2],r[3],r[4],r[5],r[6],r[7],r[0],r[9]] for r in result ]
> >         return result
> >
> >+    def DisableComponent(self,comp_id):
> >+        for item in self.CurrentContent:
> >+            if item[0] == comp_id or item[8] == comp_id:
> >+                item[-1] = -1
> >
> > ## Factory class to produce different storage for different type of meta-file
> > class MetaFileStorage(object):
> >     _FILE_TABLE_ = {
> >         MODEL_FILE_INF      :   ModuleTable,
> >--
> >2.19.1.windows.1
>
> _______________________________________________
> edk2-devel mailing list
> edk2-devel@lists.01.org
> https://lists.01.org/mailman/listinfo/edk2-devel