[Patch v2 1/1] BaseTools: Collect full Header files for struct finding.

[Patch v2 1/1] BaseTools: Collect full Header files for struct finding.

[Yuwei Chen]

Currently, only parts of the Header files can be collected which
caused some struct definition can not be found. To solve this issue,
Header files full collection has been added in this file to support
the struct finding.

Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <gaoliming@byosoft.com.cn>
Signed-off-by: Yuwei Chen <yuwei.chen@intel.com>
---
 BaseTools/Scripts/ConvertFceToStructurePcd.py | 25 ++++++++++++++-----
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/BaseTools/Scripts/ConvertFceToStructurePcd.py b/BaseTools/Scripts/ConvertFceToStructurePcd.py
index 89e6a727a201..aeda3ff26dfe 100644
--- a/BaseTools/Scripts/ConvertFceToStructurePcd.py
+++ b/BaseTools/Scripts/ConvertFceToStructurePcd.py
@@ -370,7 +370,7 @@ class PATH(object):
   def __init__(self,path):
     self.path=path
     self.rootdir=self.get_root_dir()
-    self.usefuldir=[]
+    self.usefuldir=set()
     self.lstinf = {}
     for path in self.rootdir:
       for o_root, o_dir, o_file in os.walk(os.path.join(path, "OUTPUT"), topdown=True, followlinks=False):
@@ -381,7 +381,7 @@ class PATH(object):
               for LST in l_file:
                 if os.path.splitext(LST)[1] == '.lst':
                   self.lstinf[os.path.join(l_root, LST)] = os.path.join(o_root, INF)
-                  self.usefuldir.append(path)
+                  self.usefuldir.add(path)
 
   def get_root_dir(self):
     rootdir=[]
@@ -410,7 +410,7 @@ class PATH(object):
 
   def header(self,struct):
     header={}
-    head_re = re.compile('typedef.*} %s;[\n]+(.*?)(?:typedef|formset)'%struct,re.M|re.S)
+    head_re = re.compile('typedef.*} %s;[\n]+(.*)(?:typedef|formset)'%struct,re.M|re.S)
     head_re2 = re.compile(r'#line[\s\d]+"(\S+h)"')
     for i in list(self.lstinf.keys()):
       with open(i,'r') as lst:
@@ -421,9 +421,21 @@ class PATH(object):
         if head:
           format = head[0].replace('\\\\','/').replace('\\','/')
           name =format.split('/')[-1]
-          head = self.makefile(name).replace('\\','/')
-          header[struct] = head
+          head = self.headerfileset.get(name)
+          if head:
+            head = head.replace('\\','/')
+            header[struct] = head
     return header
+  @property
+  def headerfileset(self):
+    headerset = dict()
+    for root,dirs,files in os.walk(self.path):
+      for file in files:
+        if os.path.basename(file) == 'deps.txt':
+          with open(os.path.join(root,file),"r") as fr:
+            for line in fr.readlines():
+              headerset[os.path.basename(line).strip()] = line.strip()
+    return headerset
 
   def makefile(self,filename):
     re_format = re.compile(r'DEBUG_DIR.*(?:\S+Pkg)\\(.*\\%s)'%filename)
@@ -433,6 +445,7 @@ class PATH(object):
       dir = re_format.findall(read)
       if dir:
         return dir[0]
+    return None
 
 class mainprocess(object):
 
@@ -479,7 +492,7 @@ class mainprocess(object):
               WARNING.append("Warning: No <HeaderFiles> for struct %s"%struct)
               title2 = '%s%s|{0}|%s|0xFCD00000{\n <HeaderFiles>\n  %s\n <Packages>\n%s\n}\n' % (PCD_NAME, c_name, struct, '', self.LST.package()[self.lst_dict[lstfile]])
             header_list.append(title2)
-          else:
+          elif struct not in lst._ignore:
             struct_dict ={}
             print("ERROR: Struct %s can't found in lst file" %struct)
             ERRORMSG.append("ERROR: Struct %s can't found in lst file" %struct)
-- 
2.27.0.windows.1

Re: [Patch v2 1/1] BaseTools: Collect full Header files for struct finding.

[Bob Feng]

Reviewed-by: Bob Feng <bob.c.feng@intel.com>

-----Original Message-----
From: Chen, Christine <yuwei.chen@intel.com> 
Sent: Tuesday, November 24, 2020 4:40 PM
To: devel@edk2.groups.io
Cc: Feng, Bob C <bob.c.feng@intel.com>; Liming Gao <gaoliming@byosoft.com.cn>
Subject: [Patch v2 1/1] BaseTools: Collect full Header files for struct finding.

Currently, only parts of the Header files can be collected which caused some struct definition can not be found. To solve this issue, Header files full collection has been added in this file to support the struct finding.

Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <gaoliming@byosoft.com.cn>
Signed-off-by: Yuwei Chen <yuwei.chen@intel.com>
---
 BaseTools/Scripts/ConvertFceToStructurePcd.py | 25 ++++++++++++++-----
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/BaseTools/Scripts/ConvertFceToStructurePcd.py b/BaseTools/Scripts/ConvertFceToStructurePcd.py
index 89e6a727a201..aeda3ff26dfe 100644
--- a/BaseTools/Scripts/ConvertFceToStructurePcd.py
+++ b/BaseTools/Scripts/ConvertFceToStructurePcd.py
@@ -370,7 +370,7 @@ class PATH(object):
   def __init__(self,path):
     self.path=path
     self.rootdir=self.get_root_dir()
-    self.usefuldir=[]
+    self.usefuldir=set()
     self.lstinf = {}
     for path in self.rootdir:
       for o_root, o_dir, o_file in os.walk(os.path.join(path, "OUTPUT"), topdown=True, followlinks=False):
@@ -381,7 +381,7 @@ class PATH(object):
               for LST in l_file:
                 if os.path.splitext(LST)[1] == '.lst':
                   self.lstinf[os.path.join(l_root, LST)] = os.path.join(o_root, INF)
-                  self.usefuldir.append(path)
+                  self.usefuldir.add(path)
 
   def get_root_dir(self):
     rootdir=[]
@@ -410,7 +410,7 @@ class PATH(object):
 
   def header(self,struct):
     header={}
-    head_re = re.compile('typedef.*} %s;[\n]+(.*?)(?:typedef|formset)'%struct,re.M|re.S)
+    head_re = re.compile('typedef.*} 
+ %s;[\n]+(.*)(?:typedef|formset)'%struct,re.M|re.S)
     head_re2 = re.compile(r'#line[\s\d]+"(\S+h)"')
     for i in list(self.lstinf.keys()):
       with open(i,'r') as lst:
@@ -421,9 +421,21 @@ class PATH(object):
         if head:
           format = head[0].replace('\\\\','/').replace('\\','/')
           name =format.split('/')[-1]
-          head = self.makefile(name).replace('\\','/')
-          header[struct] = head
+          head = self.headerfileset.get(name)
+          if head:
+            head = head.replace('\\','/')
+            header[struct] = head
     return header
+  @property
+  def headerfileset(self):
+    headerset = dict()
+    for root,dirs,files in os.walk(self.path):
+      for file in files:
+        if os.path.basename(file) == 'deps.txt':
+          with open(os.path.join(root,file),"r") as fr:
+            for line in fr.readlines():
+              headerset[os.path.basename(line).strip()] = line.strip()
+    return headerset
 
   def makefile(self,filename):
     re_format = re.compile(r'DEBUG_DIR.*(?:\S+Pkg)\\(.*\\%s)'%filename)
@@ -433,6 +445,7 @@ class PATH(object):
       dir = re_format.findall(read)
       if dir:
         return dir[0]
+    return None
 
 class mainprocess(object):
 
@@ -479,7 +492,7 @@ class mainprocess(object):
               WARNING.append("Warning: No <HeaderFiles> for struct %s"%struct)
               title2 = '%s%s|{0}|%s|0xFCD00000{\n <HeaderFiles>\n  %s\n <Packages>\n%s\n}\n' % (PCD_NAME, c_name, struct, '', self.LST.package()[self.lst_dict[lstfile]])
             header_list.append(title2)
-          else:
+          elif struct not in lst._ignore:
             struct_dict ={}
             print("ERROR: Struct %s can't found in lst file" %struct)
             ERRORMSG.append("ERROR: Struct %s can't found in lst file" %struct)
--
2.27.0.windows.1

Re: [edk2-devel] [Patch v2 1/1] BaseTools: Collect full Header files for struct finding.

[Bob Feng]

Create a PR for this patch https://github.com/tianocore/edk2/pull/1152 

-----Original Message-----
From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Bob Feng
Sent: Friday, November 27, 2020 7:26 PM
To: Chen, Christine <yuwei.chen@intel.com>; devel@edk2.groups.io
Cc: Liming Gao <gaoliming@byosoft.com.cn>
Subject: Re: [edk2-devel] [Patch v2 1/1] BaseTools: Collect full Header files for struct finding.

Reviewed-by: Bob Feng <bob.c.feng@intel.com>

-----Original Message-----
From: Chen, Christine <yuwei.chen@intel.com> 
Sent: Tuesday, November 24, 2020 4:40 PM
To: devel@edk2.groups.io
Cc: Feng, Bob C <bob.c.feng@intel.com>; Liming Gao <gaoliming@byosoft.com.cn>
Subject: [Patch v2 1/1] BaseTools: Collect full Header files for struct finding.

Currently, only parts of the Header files can be collected which caused some struct definition can not be found. To solve this issue, Header files full collection has been added in this file to support the struct finding.

Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <gaoliming@byosoft.com.cn>
Signed-off-by: Yuwei Chen <yuwei.chen@intel.com>
---
 BaseTools/Scripts/ConvertFceToStructurePcd.py | 25 ++++++++++++++-----
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/BaseTools/Scripts/ConvertFceToStructurePcd.py b/BaseTools/Scripts/ConvertFceToStructurePcd.py
index 89e6a727a201..aeda3ff26dfe 100644
--- a/BaseTools/Scripts/ConvertFceToStructurePcd.py
+++ b/BaseTools/Scripts/ConvertFceToStructurePcd.py
@@ -370,7 +370,7 @@ class PATH(object):
   def __init__(self,path):
     self.path=path
     self.rootdir=self.get_root_dir()
-    self.usefuldir=[]
+    self.usefuldir=set()
     self.lstinf = {}
     for path in self.rootdir:
       for o_root, o_dir, o_file in os.walk(os.path.join(path, "OUTPUT"), topdown=True, followlinks=False):
@@ -381,7 +381,7 @@ class PATH(object):
               for LST in l_file:
                 if os.path.splitext(LST)[1] == '.lst':
                   self.lstinf[os.path.join(l_root, LST)] = os.path.join(o_root, INF)
-                  self.usefuldir.append(path)
+                  self.usefuldir.add(path)
 
   def get_root_dir(self):
     rootdir=[]
@@ -410,7 +410,7 @@ class PATH(object):
 
   def header(self,struct):
     header={}
-    head_re = re.compile('typedef.*} %s;[\n]+(.*?)(?:typedef|formset)'%struct,re.M|re.S)
+    head_re = re.compile('typedef.*} 
+ %s;[\n]+(.*)(?:typedef|formset)'%struct,re.M|re.S)
     head_re2 = re.compile(r'#line[\s\d]+"(\S+h)"')
     for i in list(self.lstinf.keys()):
       with open(i,'r') as lst:
@@ -421,9 +421,21 @@ class PATH(object):
         if head:
           format = head[0].replace('\\\\','/').replace('\\','/')
           name =format.split('/')[-1]
-          head = self.makefile(name).replace('\\','/')
-          header[struct] = head
+          head = self.headerfileset.get(name)
+          if head:
+            head = head.replace('\\','/')
+            header[struct] = head
     return header
+  @property
+  def headerfileset(self):
+    headerset = dict()
+    for root,dirs,files in os.walk(self.path):
+      for file in files:
+        if os.path.basename(file) == 'deps.txt':
+          with open(os.path.join(root,file),"r") as fr:
+            for line in fr.readlines():
+              headerset[os.path.basename(line).strip()] = line.strip()
+    return headerset
 
   def makefile(self,filename):
     re_format = re.compile(r'DEBUG_DIR.*(?:\S+Pkg)\\(.*\\%s)'%filename)
@@ -433,6 +445,7 @@ class PATH(object):
       dir = re_format.findall(read)
       if dir:
         return dir[0]
+    return None
 
 class mainprocess(object):
 
@@ -479,7 +492,7 @@ class mainprocess(object):
               WARNING.append("Warning: No <HeaderFiles> for struct %s"%struct)
               title2 = '%s%s|{0}|%s|0xFCD00000{\n <HeaderFiles>\n  %s\n <Packages>\n%s\n}\n' % (PCD_NAME, c_name, struct, '', self.LST.package()[self.lst_dict[lstfile]])
             header_list.append(title2)
-          else:
+          elif struct not in lst._ignore:
             struct_dict ={}
             print("ERROR: Struct %s can't found in lst file" %struct)
             ERRORMSG.append("ERROR: Struct %s can't found in lst file" %struct)
--
2.27.0.windows.1