public inbox for devel@edk2.groups.io
 help / color / mirror / Atom feed
* [Patch] BaseTools: Detect structure pcd header file change.
@ 2018-03-16  8:13 BobCF
  0 siblings, 0 replies; 3+ messages in thread
From: BobCF @ 2018-03-16  8:13 UTC (permalink / raw)
  To: edk2-devel; +Cc: Bob Feng, Liming Gao

Detect structure pcd header file change

Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
---
 BaseTools/Source/Python/Workspace/DscBuildData.py | 190 ++++++++++++++++------
 1 file changed, 136 insertions(+), 54 deletions(-)

diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py
index 9c43daca5e..af2f3d39ea 100644
--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
@@ -93,10 +93,74 @@ LIBS = $(LIB_PATH)\Common.lib
 PcdGccMakefile = '''
 MAKEROOT ?= $(EDK_TOOLS_PATH)/Source/C
 LIBS = -lCommon
 '''
 
+## Regular expression for finding header file inclusions
+from AutoGen.GenMake import gIncludePattern
+## Regular expression for matching macro used in header file inclusion
+from AutoGen.GenMake import gMacroPattern
+from AutoGen.GenMake import gIncludeMacroConversion
+
+## Find dependencies for one source file
+#
+#  By searching recursively "#include" directive in file, find out all the
+#  files needed by given source file. The dependecies will be only searched
+#  in given search path list.
+#
+#   @param      SearchPathList  The list of search path
+#
+#   @retval     list            The list of files the given source file depends on
+#
+def GetDependencyList(FileStack,SearchPathList):
+    DepDb = dict()
+    DependencySet = set(FileStack)
+    while len(FileStack) > 0:
+        F = FileStack.pop()
+        FullPathDependList = []
+        CurrentFileDependencyList = []
+        if F in DepDb:
+            CurrentFileDependencyList = DepDb[F]
+        else:
+            try:
+                Fd = open(F, 'r')
+                FileContent = Fd.read()
+            except BaseException, X:
+                EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=F + "\n\t" + str(X))
+            finally:
+                if "Fd" in dir(locals()):
+                    Fd.close()
+
+            if len(FileContent) == 0:
+                continue
+
+            if FileContent[0] == 0xff or FileContent[0] == 0xfe:
+                FileContent = unicode(FileContent, "utf-16")
+            IncludedFileList = gIncludePattern.findall(FileContent)
+
+            for Inc in IncludedFileList:
+                Inc = Inc.strip()
+                Inc = os.path.normpath(Inc)
+                CurrentFileDependencyList.append(Inc)
+            DepDb[F] = CurrentFileDependencyList
+
+        CurrentFilePath = os.path.dirname(F)
+        PathList = [CurrentFilePath] + SearchPathList
+        for Inc in CurrentFileDependencyList:
+            for SearchPath in PathList:
+                FilePath = os.path.join(SearchPath, Inc)
+                if not os.path.exists(FilePath):
+                    continue
+                if FilePath not in DependencySet:
+                    FileStack.append(FilePath)
+                    FullPathDependList.append(FilePath)
+                break
+        DependencySet.update(FullPathDependList)
+    DependencyList = list(DependencySet)  # remove duplicate ones
+
+    return DependencyList
+
 class DscBuildData(PlatformBuildClassObject):
     # dict used to convert PCD type in database to string used by build tool
     _PCD_TYPE_STRING_ = {
         MODEL_PCD_FIXED_AT_BUILD        :   "FixedAtBuild",
         MODEL_PCD_PATCHABLE_IN_MODULE   :   "PatchableInModule",
@@ -1916,15 +1980,17 @@ class DscBuildData(PlatformBuildClassObject):
 
         InitByteValue = ""
         CApp = PcdMainCHeader
 
         Includes = {}
+        IncludeFiles = set()
         for PcdName in StructuredPcds:
             Pcd = StructuredPcds[PcdName]
             for IncludeFile in Pcd.StructuredPcdIncludeFile:
                 if IncludeFile not in Includes:
                     Includes[IncludeFile] = True
+                    IncludeFiles.add((os.path.dirname(Pcd.PkgPath), IncludeFile))
                     CApp = CApp + '#include <%s>\n' % (IncludeFile)
         CApp = CApp + '\n'
         for PcdName in StructuredPcds:
             Pcd = StructuredPcds[PcdName]
             CApp = CApp + self.GenerateSizeFunction(Pcd)
@@ -1979,10 +2045,11 @@ class DscBuildData(PlatformBuildClassObject):
         else:
             MakeApp = MakeApp + PcdGccMakefile
             MakeApp = MakeApp + 'APPNAME = %s\n' % (PcdValueInitName) + 'OBJECTS = %s/%s.o\n' % (self.OutputPath, PcdValueInitName) + \
                       'include $(MAKEROOT)/Makefiles/app.makefile\n' + 'INCLUDE +='
 
+        IncSearchList = []
         PlatformInc = {}
         for Cache in self._Bdb._CACHE_.values():
             if Cache.MetaFile.Ext.lower() != '.dec':
                 continue
             if Cache.Includes:
@@ -2001,10 +2068,11 @@ class DscBuildData(PlatformBuildClassObject):
         if PlatformInc and PcdDependDEC:
             for pkg in PcdDependDEC:
                 if pkg in PlatformInc:
                     for inc in PlatformInc[pkg]:
                         MakeApp += '-I'  + str(inc) + ' '
+                        IncSearchList.append(inc)
         MakeApp = MakeApp + '\n'
 
         CC_FLAGS = LinuxCFLAGS
         if sys.platform == "win32":
             CC_FLAGS = WindowsCFLAGS
@@ -2048,11 +2116,25 @@ class DscBuildData(PlatformBuildClassObject):
                                     CC_FLAGS += ' ' + Item
         MakeApp += CC_FLAGS
 
         if sys.platform == "win32":
             MakeApp = MakeApp + PcdMakefileEnd
+        IncludeFileFullPaths = []
+        for includefile in IncludeFiles:
+            for includepath in IncSearchList:
+                if includefile[0] in str(includepath):
+                    includefullpath = os.path.join(str(includepath),includefile[1])
+                    if os.path.exists(includefullpath):
+                        IncludeFileFullPaths.append(os.path.normpath(includefullpath))
+        SearchPathList = [str(item) for item in IncSearchList]
+        SearchPathList.append(os.path.normpath(os.path.join(self.WorkspaceDir,r"BaseTools\Source\C\Include")))
+        SearchPathList.append(os.path.normpath(os.path.join(self.WorkspaceDir,r"BaseTools\Source\C\Common")))
+        IncFileList = GetDependencyList(IncludeFileFullPaths,SearchPathList)
+        for include_file in IncFileList:
+            MakeApp += "$(OBJECTS) : %s \n" % include_file
         MakeFileName = os.path.join(self.OutputPath, 'Makefile')
+        MakeApp += "$(OBJECTS) : %s \n" % MakeFileName
         SaveFileOnChange(MakeFileName, MakeApp, False)
 
         InputValueFile = os.path.join(self.OutputPath, 'Input.txt')
         OutputValueFile = os.path.join(self.OutputPath, 'Output.txt')
         SaveFileOnChange(InputValueFile, InitByteValue, False)
@@ -2060,63 +2142,65 @@ class DscBuildData(PlatformBuildClassObject):
         PcdValueInitExe = PcdValueInitName
         if not sys.platform == "win32":
             PcdValueInitExe = os.path.join(os.getenv("EDK_TOOLS_PATH"), 'Source', 'C', 'bin', PcdValueInitName)
         else:
             PcdValueInitExe = os.path.join(os.getenv("EDK_TOOLS_PATH"), 'Bin', 'Win32', PcdValueInitName) +".exe"
-        if not os.path.exists(PcdValueInitExe) or self.NeedUpdateOutput(OutputValueFile, CAppBaseFileName + '.c',MakeFileName,InputValueFile):
-            Messages = ''
-            if sys.platform == "win32":
-                MakeCommand = 'nmake clean & nmake -f %s' % (MakeFileName)
-                returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand)
-                Messages = StdOut
-            else:
-                MakeCommand = 'make clean & make -f %s' % (MakeFileName)
-                returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand)
-                Messages = StdErr
-            Messages = Messages.split('\n')
-            MessageGroup = []
-            if returncode <>0:
-                CAppBaseFileName = os.path.join(self.OutputPath, PcdValueInitName)
-                File = open (CAppBaseFileName + '.c', 'r')
-                FileData = File.readlines()
-                File.close()
-                for Message in Messages:
-                    if " error" in Message or "warning" in Message:
-                        FileInfo = Message.strip().split('(')
-                        if len (FileInfo) > 1:
-                            FileName = FileInfo [0]
-                            FileLine = FileInfo [1].split (')')[0]
+
+        Messages = ''
+        if sys.platform == "win32":
+            MakeCommand = 'nmake -f %s' % (MakeFileName)
+            returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand)
+            Messages = StdOut
+        else:
+            MakeCommand = 'make -f %s' % (MakeFileName)
+            returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand)
+            Messages = StdErr
+        Messages = Messages.split('\n')
+        MessageGroup = []
+        if returncode <>0:
+            CAppBaseFileName = os.path.join(self.OutputPath, PcdValueInitName)
+            File = open (CAppBaseFileName + '.c', 'r')
+            FileData = File.readlines()
+            File.close()
+            for Message in Messages:
+                if " error" in Message or "warning" in Message:
+                    FileInfo = Message.strip().split('(')
+                    if len (FileInfo) > 1:
+                        FileName = FileInfo [0]
+                        FileLine = FileInfo [1].split (')')[0]
+                    else:
+                        FileInfo = Message.strip().split(':')
+                        FileName = FileInfo [0]
+                        FileLine = FileInfo [1]
+                    if FileLine.isdigit():
+                        error_line = FileData[int (FileLine) - 1]
+                        if r"//" in error_line:
+                            c_line,dsc_line = error_line.split(r"//")
                         else:
-                            FileInfo = Message.strip().split(':')
-                            FileName = FileInfo [0]
-                            FileLine = FileInfo [1]
-                        if FileLine.isdigit():
-                            error_line = FileData[int (FileLine) - 1]
-                            if r"//" in error_line:
-                                c_line,dsc_line = error_line.split(r"//")
-                            else:
-                                dsc_line = error_line
-                            message_itmes = Message.split(":")
-                            Index = 0
-                            if "PcdValueInit.c" not in Message:
-                                if not MessageGroup:
-                                    MessageGroup.append(Message)
-                                break
-                            else:
-                                for item in message_itmes:
-                                    if "PcdValueInit.c" in item:
-                                        Index = message_itmes.index(item)
-                                        message_itmes[Index] = dsc_line.strip()
-                                        break
-                                MessageGroup.append(":".join(message_itmes[Index:]).strip())
-                                continue
+                            dsc_line = error_line
+                        message_itmes = Message.split(":")
+                        Index = 0
+                        if "PcdValueInit.c" not in Message:
+                            if not MessageGroup:
+                                MessageGroup.append(Message)
+                            break
                         else:
-                            MessageGroup.append(Message)
-                if MessageGroup:
-                    EdkLogger.error("build", PCD_STRUCTURE_PCD_ERROR, "\n".join(MessageGroup) )
-                else:
-                    EdkLogger.error('Build', COMMAND_FAILURE, 'Can not execute command: %s' % MakeCommand)
+                            for item in message_itmes:
+                                if "PcdValueInit.c" in item:
+                                    Index = message_itmes.index(item)
+                                    message_itmes[Index] = dsc_line.strip()
+                                    break
+                            MessageGroup.append(":".join(message_itmes[Index:]).strip())
+                            continue
+                    else:
+                        MessageGroup.append(Message)
+            if MessageGroup:
+                EdkLogger.error("build", PCD_STRUCTURE_PCD_ERROR, "\n".join(MessageGroup) )
+            else:
+                EdkLogger.error('Build', COMMAND_FAILURE, 'Can not execute command: %s' % MakeCommand)
+
+        if self.NeedUpdateOutput(OutputValueFile, PcdValueInitExe ,InputValueFile):
             Command = PcdValueInitExe + ' -i %s -o %s' % (InputValueFile, OutputValueFile)
             returncode, StdOut, StdErr = self.ExecuteCommand (Command)
             if returncode <> 0:
                 EdkLogger.warn('Build', COMMAND_FAILURE, 'Can not collect output from command: %s' % Command)
 
@@ -2129,17 +2213,15 @@ class DscBuildData(PlatformBuildClassObject):
             PcdValue = Pcd.split ('|')
             PcdInfo = PcdValue[0].split ('.')
             StructurePcdSet.append((PcdInfo[0],PcdInfo[1], PcdInfo[2], PcdInfo[3], PcdValue[2].strip()))
         return StructurePcdSet
 
-    def NeedUpdateOutput(self,OutputFile, ValueCFile, MakeFile, StructureInput):
+    def NeedUpdateOutput(self,OutputFile, ValueCFile, StructureInput):
         if not os.path.exists(OutputFile):
             return True
         if os.stat(OutputFile).st_mtime <= os.stat(ValueCFile).st_mtime:
             return True
-        if os.stat(OutputFile).st_mtime <= os.stat(MakeFile).st_mtime:
-            return True
         if os.stat(OutputFile).st_mtime <= os.stat(StructureInput).st_mtime:
             return True
         return False
 
     ## Retrieve dynamic PCD settings
-- 
2.14.3.windows.1



^ permalink raw reply related	[flat|nested] 3+ messages in thread

* [Patch] BaseTools: Detect structure pcd header file change.
@ 2018-03-16  9:40 BobCF
  2018-03-17 14:20 ` Gao, Liming
  0 siblings, 1 reply; 3+ messages in thread
From: BobCF @ 2018-03-16  9:40 UTC (permalink / raw)
  To: edk2-devel; +Cc: Bob Feng, Liming Gao

Detect structure pcd header file change

Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
---
 BaseTools/Source/Python/Workspace/DscBuildData.py | 188 +++++++++++++++-------
 1 file changed, 134 insertions(+), 54 deletions(-)

diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py
index 9c43daca5e..9a1ed1b6e4 100644
--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
@@ -93,10 +93,71 @@ LIBS = $(LIB_PATH)\Common.lib
 PcdGccMakefile = '''
 MAKEROOT ?= $(EDK_TOOLS_PATH)/Source/C
 LIBS = -lCommon
 '''
 
+## Regular expression for finding header file inclusions
+from AutoGen.GenMake import gIncludePattern
+
+## Find dependencies for one source file
+#
+#  By searching recursively "#include" directive in file, find out all the
+#  files needed by given source file. The dependecies will be only searched
+#  in given search path list.
+#
+#   @param      SearchPathList  The list of search path
+#
+#   @retval     list            The list of files the given source file depends on
+#
+def GetDependencyList(FileStack,SearchPathList):
+    DepDb = dict()
+    DependencySet = set(FileStack)
+    while len(FileStack) > 0:
+        F = FileStack.pop()
+        FullPathDependList = []
+        CurrentFileDependencyList = []
+        if F in DepDb:
+            CurrentFileDependencyList = DepDb[F]
+        else:
+            try:
+                Fd = open(F, 'r')
+                FileContent = Fd.read()
+            except BaseException, X:
+                EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=F + "\n\t" + str(X))
+            finally:
+                if "Fd" in dir(locals()):
+                    Fd.close()
+
+            if len(FileContent) == 0:
+                continue
+
+            if FileContent[0] == 0xff or FileContent[0] == 0xfe:
+                FileContent = unicode(FileContent, "utf-16")
+            IncludedFileList = gIncludePattern.findall(FileContent)
+
+            for Inc in IncludedFileList:
+                Inc = Inc.strip()
+                Inc = os.path.normpath(Inc)
+                CurrentFileDependencyList.append(Inc)
+            DepDb[F] = CurrentFileDependencyList
+
+        CurrentFilePath = os.path.dirname(F)
+        PathList = [CurrentFilePath] + SearchPathList
+        for Inc in CurrentFileDependencyList:
+            for SearchPath in PathList:
+                FilePath = os.path.join(SearchPath, Inc)
+                if not os.path.exists(FilePath):
+                    continue
+                if FilePath not in DependencySet:
+                    FileStack.append(FilePath)
+                    FullPathDependList.append(FilePath)
+                break
+        DependencySet.update(FullPathDependList)
+    DependencyList = list(DependencySet)  # remove duplicate ones
+
+    return DependencyList
+
 class DscBuildData(PlatformBuildClassObject):
     # dict used to convert PCD type in database to string used by build tool
     _PCD_TYPE_STRING_ = {
         MODEL_PCD_FIXED_AT_BUILD        :   "FixedAtBuild",
         MODEL_PCD_PATCHABLE_IN_MODULE   :   "PatchableInModule",
@@ -1916,15 +1977,17 @@ class DscBuildData(PlatformBuildClassObject):
 
         InitByteValue = ""
         CApp = PcdMainCHeader
 
         Includes = {}
+        IncludeFiles = set()
         for PcdName in StructuredPcds:
             Pcd = StructuredPcds[PcdName]
             for IncludeFile in Pcd.StructuredPcdIncludeFile:
                 if IncludeFile not in Includes:
                     Includes[IncludeFile] = True
+                    IncludeFiles.add((os.path.dirname(Pcd.PkgPath), IncludeFile))
                     CApp = CApp + '#include <%s>\n' % (IncludeFile)
         CApp = CApp + '\n'
         for PcdName in StructuredPcds:
             Pcd = StructuredPcds[PcdName]
             CApp = CApp + self.GenerateSizeFunction(Pcd)
@@ -1979,10 +2042,11 @@ class DscBuildData(PlatformBuildClassObject):
         else:
             MakeApp = MakeApp + PcdGccMakefile
             MakeApp = MakeApp + 'APPNAME = %s\n' % (PcdValueInitName) + 'OBJECTS = %s/%s.o\n' % (self.OutputPath, PcdValueInitName) + \
                       'include $(MAKEROOT)/Makefiles/app.makefile\n' + 'INCLUDE +='
 
+        IncSearchList = []
         PlatformInc = {}
         for Cache in self._Bdb._CACHE_.values():
             if Cache.MetaFile.Ext.lower() != '.dec':
                 continue
             if Cache.Includes:
@@ -2001,10 +2065,11 @@ class DscBuildData(PlatformBuildClassObject):
         if PlatformInc and PcdDependDEC:
             for pkg in PcdDependDEC:
                 if pkg in PlatformInc:
                     for inc in PlatformInc[pkg]:
                         MakeApp += '-I'  + str(inc) + ' '
+                        IncSearchList.append(inc)
         MakeApp = MakeApp + '\n'
 
         CC_FLAGS = LinuxCFLAGS
         if sys.platform == "win32":
             CC_FLAGS = WindowsCFLAGS
@@ -2048,11 +2113,26 @@ class DscBuildData(PlatformBuildClassObject):
                                     CC_FLAGS += ' ' + Item
         MakeApp += CC_FLAGS
 
         if sys.platform == "win32":
             MakeApp = MakeApp + PcdMakefileEnd
+        IncludeFileFullPaths = []
+        for includefile in IncludeFiles:
+            for includepath in IncSearchList:
+                includefullpath = os.path.join(str(includepath),includefile[1])
+                if os.path.exists(includefullpath):
+                    IncludeFileFullPaths.append(os.path.normpath(includefullpath))
+                    break
+        SearchPathList = []
+        SearchPathList.append(os.path.normpath(os.path.join(self.WorkspaceDir,r"BaseTools\Source\C\Include")))
+        SearchPathList.append(os.path.normpath(os.path.join(self.WorkspaceDir,r"BaseTools\Source\C\Common")))
+        SearchPathList.extend([str(item) for item in IncSearchList])
+        IncFileList = GetDependencyList(IncludeFileFullPaths,SearchPathList)
+        for include_file in IncFileList:
+            MakeApp += "$(OBJECTS) : %s \n" % include_file
         MakeFileName = os.path.join(self.OutputPath, 'Makefile')
+        MakeApp += "$(OBJECTS) : %s \n" % MakeFileName
         SaveFileOnChange(MakeFileName, MakeApp, False)
 
         InputValueFile = os.path.join(self.OutputPath, 'Input.txt')
         OutputValueFile = os.path.join(self.OutputPath, 'Output.txt')
         SaveFileOnChange(InputValueFile, InitByteValue, False)
@@ -2060,63 +2140,65 @@ class DscBuildData(PlatformBuildClassObject):
         PcdValueInitExe = PcdValueInitName
         if not sys.platform == "win32":
             PcdValueInitExe = os.path.join(os.getenv("EDK_TOOLS_PATH"), 'Source', 'C', 'bin', PcdValueInitName)
         else:
             PcdValueInitExe = os.path.join(os.getenv("EDK_TOOLS_PATH"), 'Bin', 'Win32', PcdValueInitName) +".exe"
-        if not os.path.exists(PcdValueInitExe) or self.NeedUpdateOutput(OutputValueFile, CAppBaseFileName + '.c',MakeFileName,InputValueFile):
-            Messages = ''
-            if sys.platform == "win32":
-                MakeCommand = 'nmake clean & nmake -f %s' % (MakeFileName)
-                returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand)
-                Messages = StdOut
-            else:
-                MakeCommand = 'make clean & make -f %s' % (MakeFileName)
-                returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand)
-                Messages = StdErr
-            Messages = Messages.split('\n')
-            MessageGroup = []
-            if returncode <>0:
-                CAppBaseFileName = os.path.join(self.OutputPath, PcdValueInitName)
-                File = open (CAppBaseFileName + '.c', 'r')
-                FileData = File.readlines()
-                File.close()
-                for Message in Messages:
-                    if " error" in Message or "warning" in Message:
-                        FileInfo = Message.strip().split('(')
-                        if len (FileInfo) > 1:
-                            FileName = FileInfo [0]
-                            FileLine = FileInfo [1].split (')')[0]
+
+        Messages = ''
+        if sys.platform == "win32":
+            MakeCommand = 'nmake -f %s' % (MakeFileName)
+            returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand)
+            Messages = StdOut
+        else:
+            MakeCommand = 'make -f %s' % (MakeFileName)
+            returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand)
+            Messages = StdErr
+        Messages = Messages.split('\n')
+        MessageGroup = []
+        if returncode <>0:
+            CAppBaseFileName = os.path.join(self.OutputPath, PcdValueInitName)
+            File = open (CAppBaseFileName + '.c', 'r')
+            FileData = File.readlines()
+            File.close()
+            for Message in Messages:
+                if " error" in Message or "warning" in Message:
+                    FileInfo = Message.strip().split('(')
+                    if len (FileInfo) > 1:
+                        FileName = FileInfo [0]
+                        FileLine = FileInfo [1].split (')')[0]
+                    else:
+                        FileInfo = Message.strip().split(':')
+                        FileName = FileInfo [0]
+                        FileLine = FileInfo [1]
+                    if FileLine.isdigit():
+                        error_line = FileData[int (FileLine) - 1]
+                        if r"//" in error_line:
+                            c_line,dsc_line = error_line.split(r"//")
                         else:
-                            FileInfo = Message.strip().split(':')
-                            FileName = FileInfo [0]
-                            FileLine = FileInfo [1]
-                        if FileLine.isdigit():
-                            error_line = FileData[int (FileLine) - 1]
-                            if r"//" in error_line:
-                                c_line,dsc_line = error_line.split(r"//")
-                            else:
-                                dsc_line = error_line
-                            message_itmes = Message.split(":")
-                            Index = 0
-                            if "PcdValueInit.c" not in Message:
-                                if not MessageGroup:
-                                    MessageGroup.append(Message)
-                                break
-                            else:
-                                for item in message_itmes:
-                                    if "PcdValueInit.c" in item:
-                                        Index = message_itmes.index(item)
-                                        message_itmes[Index] = dsc_line.strip()
-                                        break
-                                MessageGroup.append(":".join(message_itmes[Index:]).strip())
-                                continue
+                            dsc_line = error_line
+                        message_itmes = Message.split(":")
+                        Index = 0
+                        if "PcdValueInit.c" not in Message:
+                            if not MessageGroup:
+                                MessageGroup.append(Message)
+                            break
                         else:
-                            MessageGroup.append(Message)
-                if MessageGroup:
-                    EdkLogger.error("build", PCD_STRUCTURE_PCD_ERROR, "\n".join(MessageGroup) )
-                else:
-                    EdkLogger.error('Build', COMMAND_FAILURE, 'Can not execute command: %s' % MakeCommand)
+                            for item in message_itmes:
+                                if "PcdValueInit.c" in item:
+                                    Index = message_itmes.index(item)
+                                    message_itmes[Index] = dsc_line.strip()
+                                    break
+                            MessageGroup.append(":".join(message_itmes[Index:]).strip())
+                            continue
+                    else:
+                        MessageGroup.append(Message)
+            if MessageGroup:
+                EdkLogger.error("build", PCD_STRUCTURE_PCD_ERROR, "\n".join(MessageGroup) )
+            else:
+                EdkLogger.error('Build', COMMAND_FAILURE, 'Can not execute command: %s' % MakeCommand)
+
+        if self.NeedUpdateOutput(OutputValueFile, PcdValueInitExe ,InputValueFile):
             Command = PcdValueInitExe + ' -i %s -o %s' % (InputValueFile, OutputValueFile)
             returncode, StdOut, StdErr = self.ExecuteCommand (Command)
             if returncode <> 0:
                 EdkLogger.warn('Build', COMMAND_FAILURE, 'Can not collect output from command: %s' % Command)
 
@@ -2129,17 +2211,15 @@ class DscBuildData(PlatformBuildClassObject):
             PcdValue = Pcd.split ('|')
             PcdInfo = PcdValue[0].split ('.')
             StructurePcdSet.append((PcdInfo[0],PcdInfo[1], PcdInfo[2], PcdInfo[3], PcdValue[2].strip()))
         return StructurePcdSet
 
-    def NeedUpdateOutput(self,OutputFile, ValueCFile, MakeFile, StructureInput):
+    def NeedUpdateOutput(self,OutputFile, ValueCFile, StructureInput):
         if not os.path.exists(OutputFile):
             return True
         if os.stat(OutputFile).st_mtime <= os.stat(ValueCFile).st_mtime:
             return True
-        if os.stat(OutputFile).st_mtime <= os.stat(MakeFile).st_mtime:
-            return True
         if os.stat(OutputFile).st_mtime <= os.stat(StructureInput).st_mtime:
             return True
         return False
 
     ## Retrieve dynamic PCD settings
-- 
2.14.3.windows.1



^ permalink raw reply related	[flat|nested] 3+ messages in thread

* Re: [Patch] BaseTools: Detect structure pcd header file change.
  2018-03-16  9:40 [Patch] BaseTools: Detect structure pcd header file change BobCF
@ 2018-03-17 14:20 ` Gao, Liming
  0 siblings, 0 replies; 3+ messages in thread
From: Gao, Liming @ 2018-03-17 14:20 UTC (permalink / raw)
  To: Feng, Bob C, edk2-devel@lists.01.org

Reviewed-by: Liming Gao <liming.gao@intel.com>

> -----Original Message-----
> From: edk2-devel [mailto:edk2-devel-bounces@lists.01.org] On Behalf Of BobCF
> Sent: Friday, March 16, 2018 5:40 PM
> To: edk2-devel@lists.01.org
> Cc: Gao, Liming <liming.gao@intel.com>
> Subject: [edk2] [Patch] BaseTools: Detect structure pcd header file change.
> 
> Detect structure pcd header file change
> 
> Contributed-under: TianoCore Contribution Agreement 1.1
> Signed-off-by: Bob Feng <bob.c.feng@intel.com>
> Cc: Liming Gao <liming.gao@intel.com>
> ---
>  BaseTools/Source/Python/Workspace/DscBuildData.py | 188 +++++++++++++++-------
>  1 file changed, 134 insertions(+), 54 deletions(-)
> 
> diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py
> index 9c43daca5e..9a1ed1b6e4 100644
> --- a/BaseTools/Source/Python/Workspace/DscBuildData.py
> +++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
> @@ -93,10 +93,71 @@ LIBS = $(LIB_PATH)\Common.lib
>  PcdGccMakefile = '''
>  MAKEROOT ?= $(EDK_TOOLS_PATH)/Source/C
>  LIBS = -lCommon
>  '''
> 
> +## Regular expression for finding header file inclusions
> +from AutoGen.GenMake import gIncludePattern
> +
> +## Find dependencies for one source file
> +#
> +#  By searching recursively "#include" directive in file, find out all the
> +#  files needed by given source file. The dependecies will be only searched
> +#  in given search path list.
> +#
> +#   @param      SearchPathList  The list of search path
> +#
> +#   @retval     list            The list of files the given source file depends on
> +#
> +def GetDependencyList(FileStack,SearchPathList):
> +    DepDb = dict()
> +    DependencySet = set(FileStack)
> +    while len(FileStack) > 0:
> +        F = FileStack.pop()
> +        FullPathDependList = []
> +        CurrentFileDependencyList = []
> +        if F in DepDb:
> +            CurrentFileDependencyList = DepDb[F]
> +        else:
> +            try:
> +                Fd = open(F, 'r')
> +                FileContent = Fd.read()
> +            except BaseException, X:
> +                EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=F + "\n\t" + str(X))
> +            finally:
> +                if "Fd" in dir(locals()):
> +                    Fd.close()
> +
> +            if len(FileContent) == 0:
> +                continue
> +
> +            if FileContent[0] == 0xff or FileContent[0] == 0xfe:
> +                FileContent = unicode(FileContent, "utf-16")
> +            IncludedFileList = gIncludePattern.findall(FileContent)
> +
> +            for Inc in IncludedFileList:
> +                Inc = Inc.strip()
> +                Inc = os.path.normpath(Inc)
> +                CurrentFileDependencyList.append(Inc)
> +            DepDb[F] = CurrentFileDependencyList
> +
> +        CurrentFilePath = os.path.dirname(F)
> +        PathList = [CurrentFilePath] + SearchPathList
> +        for Inc in CurrentFileDependencyList:
> +            for SearchPath in PathList:
> +                FilePath = os.path.join(SearchPath, Inc)
> +                if not os.path.exists(FilePath):
> +                    continue
> +                if FilePath not in DependencySet:
> +                    FileStack.append(FilePath)
> +                    FullPathDependList.append(FilePath)
> +                break
> +        DependencySet.update(FullPathDependList)
> +    DependencyList = list(DependencySet)  # remove duplicate ones
> +
> +    return DependencyList
> +
>  class DscBuildData(PlatformBuildClassObject):
>      # dict used to convert PCD type in database to string used by build tool
>      _PCD_TYPE_STRING_ = {
>          MODEL_PCD_FIXED_AT_BUILD        :   "FixedAtBuild",
>          MODEL_PCD_PATCHABLE_IN_MODULE   :   "PatchableInModule",
> @@ -1916,15 +1977,17 @@ class DscBuildData(PlatformBuildClassObject):
> 
>          InitByteValue = ""
>          CApp = PcdMainCHeader
> 
>          Includes = {}
> +        IncludeFiles = set()
>          for PcdName in StructuredPcds:
>              Pcd = StructuredPcds[PcdName]
>              for IncludeFile in Pcd.StructuredPcdIncludeFile:
>                  if IncludeFile not in Includes:
>                      Includes[IncludeFile] = True
> +                    IncludeFiles.add((os.path.dirname(Pcd.PkgPath), IncludeFile))
>                      CApp = CApp + '#include <%s>\n' % (IncludeFile)
>          CApp = CApp + '\n'
>          for PcdName in StructuredPcds:
>              Pcd = StructuredPcds[PcdName]
>              CApp = CApp + self.GenerateSizeFunction(Pcd)
> @@ -1979,10 +2042,11 @@ class DscBuildData(PlatformBuildClassObject):
>          else:
>              MakeApp = MakeApp + PcdGccMakefile
>              MakeApp = MakeApp + 'APPNAME = %s\n' % (PcdValueInitName) + 'OBJECTS = %s/%s.o\n' % (self.OutputPath,
> PcdValueInitName) + \
>                        'include $(MAKEROOT)/Makefiles/app.makefile\n' + 'INCLUDE +='
> 
> +        IncSearchList = []
>          PlatformInc = {}
>          for Cache in self._Bdb._CACHE_.values():
>              if Cache.MetaFile.Ext.lower() != '.dec':
>                  continue
>              if Cache.Includes:
> @@ -2001,10 +2065,11 @@ class DscBuildData(PlatformBuildClassObject):
>          if PlatformInc and PcdDependDEC:
>              for pkg in PcdDependDEC:
>                  if pkg in PlatformInc:
>                      for inc in PlatformInc[pkg]:
>                          MakeApp += '-I'  + str(inc) + ' '
> +                        IncSearchList.append(inc)
>          MakeApp = MakeApp + '\n'
> 
>          CC_FLAGS = LinuxCFLAGS
>          if sys.platform == "win32":
>              CC_FLAGS = WindowsCFLAGS
> @@ -2048,11 +2113,26 @@ class DscBuildData(PlatformBuildClassObject):
>                                      CC_FLAGS += ' ' + Item
>          MakeApp += CC_FLAGS
> 
>          if sys.platform == "win32":
>              MakeApp = MakeApp + PcdMakefileEnd
> +        IncludeFileFullPaths = []
> +        for includefile in IncludeFiles:
> +            for includepath in IncSearchList:
> +                includefullpath = os.path.join(str(includepath),includefile[1])
> +                if os.path.exists(includefullpath):
> +                    IncludeFileFullPaths.append(os.path.normpath(includefullpath))
> +                    break
> +        SearchPathList = []
> +        SearchPathList.append(os.path.normpath(os.path.join(self.WorkspaceDir,r"BaseTools\Source\C\Include")))
> +        SearchPathList.append(os.path.normpath(os.path.join(self.WorkspaceDir,r"BaseTools\Source\C\Common")))
> +        SearchPathList.extend([str(item) for item in IncSearchList])
> +        IncFileList = GetDependencyList(IncludeFileFullPaths,SearchPathList)
> +        for include_file in IncFileList:
> +            MakeApp += "$(OBJECTS) : %s \n" % include_file
>          MakeFileName = os.path.join(self.OutputPath, 'Makefile')
> +        MakeApp += "$(OBJECTS) : %s \n" % MakeFileName
>          SaveFileOnChange(MakeFileName, MakeApp, False)
> 
>          InputValueFile = os.path.join(self.OutputPath, 'Input.txt')
>          OutputValueFile = os.path.join(self.OutputPath, 'Output.txt')
>          SaveFileOnChange(InputValueFile, InitByteValue, False)
> @@ -2060,63 +2140,65 @@ class DscBuildData(PlatformBuildClassObject):
>          PcdValueInitExe = PcdValueInitName
>          if not sys.platform == "win32":
>              PcdValueInitExe = os.path.join(os.getenv("EDK_TOOLS_PATH"), 'Source', 'C', 'bin', PcdValueInitName)
>          else:
>              PcdValueInitExe = os.path.join(os.getenv("EDK_TOOLS_PATH"), 'Bin', 'Win32', PcdValueInitName) +".exe"
> -        if not os.path.exists(PcdValueInitExe) or self.NeedUpdateOutput(OutputValueFile, CAppBaseFileName +
> '.c',MakeFileName,InputValueFile):
> -            Messages = ''
> -            if sys.platform == "win32":
> -                MakeCommand = 'nmake clean & nmake -f %s' % (MakeFileName)
> -                returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand)
> -                Messages = StdOut
> -            else:
> -                MakeCommand = 'make clean & make -f %s' % (MakeFileName)
> -                returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand)
> -                Messages = StdErr
> -            Messages = Messages.split('\n')
> -            MessageGroup = []
> -            if returncode <>0:
> -                CAppBaseFileName = os.path.join(self.OutputPath, PcdValueInitName)
> -                File = open (CAppBaseFileName + '.c', 'r')
> -                FileData = File.readlines()
> -                File.close()
> -                for Message in Messages:
> -                    if " error" in Message or "warning" in Message:
> -                        FileInfo = Message.strip().split('(')
> -                        if len (FileInfo) > 1:
> -                            FileName = FileInfo [0]
> -                            FileLine = FileInfo [1].split (')')[0]
> +
> +        Messages = ''
> +        if sys.platform == "win32":
> +            MakeCommand = 'nmake -f %s' % (MakeFileName)
> +            returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand)
> +            Messages = StdOut
> +        else:
> +            MakeCommand = 'make -f %s' % (MakeFileName)
> +            returncode, StdOut, StdErr = self.ExecuteCommand (MakeCommand)
> +            Messages = StdErr
> +        Messages = Messages.split('\n')
> +        MessageGroup = []
> +        if returncode <>0:
> +            CAppBaseFileName = os.path.join(self.OutputPath, PcdValueInitName)
> +            File = open (CAppBaseFileName + '.c', 'r')
> +            FileData = File.readlines()
> +            File.close()
> +            for Message in Messages:
> +                if " error" in Message or "warning" in Message:
> +                    FileInfo = Message.strip().split('(')
> +                    if len (FileInfo) > 1:
> +                        FileName = FileInfo [0]
> +                        FileLine = FileInfo [1].split (')')[0]
> +                    else:
> +                        FileInfo = Message.strip().split(':')
> +                        FileName = FileInfo [0]
> +                        FileLine = FileInfo [1]
> +                    if FileLine.isdigit():
> +                        error_line = FileData[int (FileLine) - 1]
> +                        if r"//" in error_line:
> +                            c_line,dsc_line = error_line.split(r"//")
>                          else:
> -                            FileInfo = Message.strip().split(':')
> -                            FileName = FileInfo [0]
> -                            FileLine = FileInfo [1]
> -                        if FileLine.isdigit():
> -                            error_line = FileData[int (FileLine) - 1]
> -                            if r"//" in error_line:
> -                                c_line,dsc_line = error_line.split(r"//")
> -                            else:
> -                                dsc_line = error_line
> -                            message_itmes = Message.split(":")
> -                            Index = 0
> -                            if "PcdValueInit.c" not in Message:
> -                                if not MessageGroup:
> -                                    MessageGroup.append(Message)
> -                                break
> -                            else:
> -                                for item in message_itmes:
> -                                    if "PcdValueInit.c" in item:
> -                                        Index = message_itmes.index(item)
> -                                        message_itmes[Index] = dsc_line.strip()
> -                                        break
> -                                MessageGroup.append(":".join(message_itmes[Index:]).strip())
> -                                continue
> +                            dsc_line = error_line
> +                        message_itmes = Message.split(":")
> +                        Index = 0
> +                        if "PcdValueInit.c" not in Message:
> +                            if not MessageGroup:
> +                                MessageGroup.append(Message)
> +                            break
>                          else:
> -                            MessageGroup.append(Message)
> -                if MessageGroup:
> -                    EdkLogger.error("build", PCD_STRUCTURE_PCD_ERROR, "\n".join(MessageGroup) )
> -                else:
> -                    EdkLogger.error('Build', COMMAND_FAILURE, 'Can not execute command: %s' % MakeCommand)
> +                            for item in message_itmes:
> +                                if "PcdValueInit.c" in item:
> +                                    Index = message_itmes.index(item)
> +                                    message_itmes[Index] = dsc_line.strip()
> +                                    break
> +                            MessageGroup.append(":".join(message_itmes[Index:]).strip())
> +                            continue
> +                    else:
> +                        MessageGroup.append(Message)
> +            if MessageGroup:
> +                EdkLogger.error("build", PCD_STRUCTURE_PCD_ERROR, "\n".join(MessageGroup) )
> +            else:
> +                EdkLogger.error('Build', COMMAND_FAILURE, 'Can not execute command: %s' % MakeCommand)
> +
> +        if self.NeedUpdateOutput(OutputValueFile, PcdValueInitExe ,InputValueFile):
>              Command = PcdValueInitExe + ' -i %s -o %s' % (InputValueFile, OutputValueFile)
>              returncode, StdOut, StdErr = self.ExecuteCommand (Command)
>              if returncode <> 0:
>                  EdkLogger.warn('Build', COMMAND_FAILURE, 'Can not collect output from command: %s' % Command)
> 
> @@ -2129,17 +2211,15 @@ class DscBuildData(PlatformBuildClassObject):
>              PcdValue = Pcd.split ('|')
>              PcdInfo = PcdValue[0].split ('.')
>              StructurePcdSet.append((PcdInfo[0],PcdInfo[1], PcdInfo[2], PcdInfo[3], PcdValue[2].strip()))
>          return StructurePcdSet
> 
> -    def NeedUpdateOutput(self,OutputFile, ValueCFile, MakeFile, StructureInput):
> +    def NeedUpdateOutput(self,OutputFile, ValueCFile, StructureInput):
>          if not os.path.exists(OutputFile):
>              return True
>          if os.stat(OutputFile).st_mtime <= os.stat(ValueCFile).st_mtime:
>              return True
> -        if os.stat(OutputFile).st_mtime <= os.stat(MakeFile).st_mtime:
> -            return True
>          if os.stat(OutputFile).st_mtime <= os.stat(StructureInput).st_mtime:
>              return True
>          return False
> 
>      ## Retrieve dynamic PCD settings
> --
> 2.14.3.windows.1
> 
> _______________________________________________
> edk2-devel mailing list
> edk2-devel@lists.01.org
> https://lists.01.org/mailman/listinfo/edk2-devel


^ permalink raw reply	[flat|nested] 3+ messages in thread

end of thread, other threads:[~2018-03-17 14:13 UTC | newest]

Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2018-03-16  9:40 [Patch] BaseTools: Detect structure pcd header file change BobCF
2018-03-17 14:20 ` Gao, Liming
  -- strict thread matches above, loose matches on Subject: below --
2018-03-16  8:13 BobCF

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox