[edk2-staging/EdkRepo][PATCH 1/3] EdkRepo: Multiple Command Package Configuration Support

[edk2-staging/EdkRepo][PATCH 1/3] EdkRepo: Multiple Command Package Configuration Support

[Desimone, Ashley E]

Add support for a new required sections, command-packages
and preferred-command-packages,to the GlobalConfig class.
Additionaly update the existing edkrepo.cfg file.

Signed-off-by: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
---
 edkrepo/config/config_factory.py     | 12 +++++++++++-
 edkrepo_installer/Vendor/edkrepo.cfg |  6 ++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/edkrepo/config/config_factory.py b/edkrepo/config/config_factory.py
index 7a39554..38844d7 100644
--- a/edkrepo/config/config_factory.py
+++ b/edkrepo/config/config_factory.py
@@ -123,11 +123,21 @@ class GlobalConfig(BaseConfig):
                 CfgProp('sparsecheckout', 'always_exclude', 'sparsecheckout_always_exclude', None, True),
                 CfgProp('f2f-cherry-pick', 'ignored_folder_substrings', 'f2f_cp_ignored_folder_substrings'),
                 CfgProp('git-ver', 'minimum', 'minimum_req_git_ver', None, True),
-                CfgProp('git-ver', 'recommended', 'rec_git_ver', None, True)]
+                CfgProp('git-ver', 'recommended', 'rec_git_ver', None, True),
+                CfgProp('command-packages', 'packages', 'command_packages', None, True),
+                CfgProp('preferred-command-package', 'preferred-package', 'pref_pkg', None, True)]
         if not os.path.isfile(self.filename):
             raise EdkrepoGlobalConfigNotFoundException("edkrepo global config file {} does not exist".format(self.filename))
         super().__init__(self.filename, True)
 
+    @property
+    def command_packages_list(self):
+        initial_list = self.command_packages.split('|')
+        pkg_list = []
+        for pkg in initial_list:
+            pkg_list.append(pkg.strip())
+        return pkg_list
+
     @property
     def manifest_repo_abs_local_path(self):
         """Provides an absolute path to the manifest repo based on configuration file values."""
diff --git a/edkrepo_installer/Vendor/edkrepo.cfg b/edkrepo_installer/Vendor/edkrepo.cfg
index 08724fd..0312c74 100644
--- a/edkrepo_installer/Vendor/edkrepo.cfg
+++ b/edkrepo_installer/Vendor/edkrepo.cfg
@@ -7,6 +7,12 @@ LocalPath = manifest-master
 minimum = 2.13.0
 recommended = 2.16.2
 
+[command-packages]
+packages = edkrepo.commands
+
+[preferred-command-package]
+preferred-package = edkrepo.commands
+
 [sparsecheckout]
 always_include = BaseTools|Conf|*.*
 always_exclude = 
-- 
2.16.2.windows.1

[edk2-staging/EdkRepo] [PATCH 2/3] EdkRepo: Multiple Command Package Command Factory Update

[Desimone, Ashley E]

Add support for iterating through a list of command packages
from the global config file and generate a list of commands
to be included. Improve the the _is_command() function to
succesfully inspect command classes which do not derive
from the edkrepo base command class

Add support to list a command package as preferred.

Signed-off-by: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
---
 edkrepo/commands/command_factory.py | 51 ++++++++++++++++++++++++++++---------
 1 file changed, 39 insertions(+), 12 deletions(-)

diff --git a/edkrepo/commands/command_factory.py b/edkrepo/commands/command_factory.py
index abdfb1a..9f2e923 100644
--- a/edkrepo/commands/command_factory.py
+++ b/edkrepo/commands/command_factory.py
@@ -13,6 +13,7 @@ import os
 import sys
 from edkrepo.commands.edkrepo_command import EdkrepoCommand
 from edkrepo.commands.composite_command import CompositeCommand
+from edkrepo.config.config_factory import GlobalConfig
 
 def _is_command(CommandClass):
     if CommandClass == EdkrepoCommand:
@@ -20,12 +21,20 @@ def _is_command(CommandClass):
     if CommandClass in EdkrepoCommand.__subclasses__():
         return True
     #Use reflection to see if the class would work as a command anyway
-    funcs = inspect.getmembers(CommandClass, predicate=inspect.ismethod)
+    try:
+        cmd = CommandClass()
+    except:
+        return False
+    funcs = inspect.getmembers(cmd, predicate=inspect.ismethod)
     has_get_metadata = False
     has_run_command = False
     for func in funcs:
         if func[0] == 'get_metadata' and len(inspect.getargspec(func[1])[0]) == 1:
-            has_get_metadata = True
+            try:
+                cmd.get_metadata()
+                has_get_metadata = True
+            except:
+                has_get_metadata = False
         if func[0] == 'run_command':
             arg_spec = inspect.getargspec(func[1])
             if len(arg_spec[0]) == 3 and arg_spec[0][1] == "args" and arg_spec[0][2] == "config":
@@ -36,16 +45,34 @@ def _is_command(CommandClass):
         return False
 
 def get_commands():
-    commands = []
-    for module in os.listdir(os.path.dirname(__file__)):
-        if module == "__init__.py" or os.path.splitext(module)[1] != ".py":
-            continue
-        mod = importlib.import_module("edkrepo.commands.{}".format(os.path.splitext(module)[0]))
-        classes = inspect.getmembers(mod, predicate=inspect.isclass)
-        for cls in classes:
-            if _is_command(cls[1]):
-                commands.append(cls[1])
-    return commands
+    cfg_file = GlobalConfig()
+    cmd_pkg_list = cfg_file.command_packages_list
+    pref_cmd_pkg = cfg_file.pref_pkg
+    commands = {}
+    pref_commands = {}
+    final_cmd_list = []
+    cmd_search_dirs = []
+
+    for cmd_pkg in cmd_pkg_list:
+        mod = importlib.import_module(cmd_pkg)
+        cmd_search_dirs.append((cmd_pkg, os.path.dirname(mod.__file__)))
+    for cmd_dir in cmd_search_dirs:
+        for module in os.listdir(cmd_dir[1]):
+            if module == '__init__.py' or os.path.splitext(module)[1] != '.py':
+                continue
+            mod = importlib.import_module('{}.{}'.format(cmd_dir[0], os.path.splitext(module)[0]))
+            classes = inspect.getmembers(mod, predicate=inspect.isclass)
+            for cls in classes:
+                if _is_command(cls[1]):
+                    if cmd_dir[0] == pref_cmd_pkg:
+                        pref_commands.update([(cls[0], cls[1])])
+                    else:
+                        commands.update([(cls[0], cls[1])])
+    for key in commands.keys():
+        if key not in pref_commands.keys():
+            final_cmd_list.append(commands[key])
+    final_cmd_list.extend(pref_commands.values())
+    return final_cmd_list
 
 def create_composite_command():
     commands = get_commands()
-- 
2.16.2.windows.1

[edk2-staging/EdkRepo] [PATCH 3/3] EdkRepo: Multiple Command Package Build Script Update

[Desimone, Ashley E]

Add support for building extension packages.

Signed-off-by: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
---
 build-scripts/set_version_and_build_wheels.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/build-scripts/set_version_and_build_wheels.py b/build-scripts/set_version_and_build_wheels.py
index e8d467e..6fd5ab5 100755
--- a/build-scripts/set_version_and_build_wheels.py
+++ b/build-scripts/set_version_and_build_wheels.py
@@ -195,16 +195,23 @@ def make_selfextract_version_script(version):
         ver.write('chcp 437\n')
     print("VersionInfo script written to set_version.bat\n")
 
-def build_wheels():
+def build_wheels(extension_pkgs):
     dir_path = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
     file_path = os.path.join(dir_path, "setup.py")
     if ostype == WIN:
         check_call("py {} \"{}\" bdist_wheel".format(PYTHON_VERSION, file_path), shell=True, cwd=dir_path)
     else:
         check_call('python3 "{}" bdist_wheel'.format(file_path), shell=True, cwd=dir_path)
+    for pkg in extension_pkgs:
+        ext_dir_path = os.path.abspath(pkg)
+        ext_file_path = os.path.join(ext_dir_path, 'setup.py')
+        if ostype == WIN:
+            check_call("py {} \"{}\" bdist_wheel".format(PYTHON_VERSION, ext_file_path), shell=True, cwd=ext_dir_path)
+        else:
+            check_call('python3 "{}" bdist_wheel'.format(ext_file_path), shell=True, cwd=ext_dir_path)
     print("Wheels built successfully")
 
-def copy_wheels_and_set_xml(package_version):
+def copy_wheels_and_set_xml(package_version, extension_pkgs):
     dir_path = os.path.join(os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "dist")
     dest_path = os.path.join(dir_path, "self_extract")
     if ostype == LINUX:
@@ -218,6 +225,14 @@ def copy_wheels_and_set_xml(package_version):
             if data:
                 wheels.append((data.group(1), file_name))
                 _copy_file(os.path.join(dir_path, file_name), os.path.join(dest_path, file_name))
+    for pkg in extension_pkgs:
+        ext_dir_path = os.path.join(os.path.abspath(pkg), 'dist')
+        for file_name in os.listdir(ext_dir_path):
+            if fnmatch.fnmatch(file_name, "*.whl"):
+                data = wheel_package.match(file_name)
+                if data:
+                    wheels.append((data.group(1), file_name))
+                    _copy_file(os.path.join(ext_dir_path, file_name), os.path.join(dest_path, file_name))
     _set_version_number_and_wheels_cr_tools_installer_config(package_version, wheels)
 
 def create_final_copy_script(version):
@@ -251,6 +266,7 @@ def make_version_cfg_file(version):
             install_cfg.write(f)
 
 def main():
+    extension_pkgs = []
     print("Building Python packages and generating new version number...")
     (version, package_version) = get_current_version_number()
     set_version_number_setup_py(package_version)
@@ -259,8 +275,8 @@ def main():
         set_version_number_setup_launcher_rc(version)
         make_selfextract_version_script(version)
     print("Version number generated and set successfully")
-    build_wheels()
-    copy_wheels_and_set_xml(package_version)
+    build_wheels(extension_pkgs)
+    copy_wheels_and_set_xml(package_version, extension_pkgs)
     make_version_cfg_file(version)
     create_final_copy_script(version)
 
-- 
2.16.2.windows.1

Re: [edk2-devel] [edk2-staging/EdkRepo][PATCH 1/3] EdkRepo: Multiple Command Package Configuration Support

[Nate DeSimone]

Reviewed-by: Nate DeSimone <nathaniel.l.desimone@intel.com>

-----Original Message-----
From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Desimone, Ashley E
Sent: Monday, October 28, 2019 4:23 PM
To: devel@edk2.groups.io
Cc: Desimone, Nathaniel L <nathaniel.l.desimone@intel.com>; Pandya, Puja <puja.pandya@intel.com>
Subject: [edk2-devel] [edk2-staging/EdkRepo][PATCH 1/3] EdkRepo: Multiple Command Package Configuration Support

Add support for a new required sections, command-packages and preferred-command-packages,to the GlobalConfig class.
Additionaly update the existing edkrepo.cfg file.

Signed-off-by: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
---
 edkrepo/config/config_factory.py     | 12 +++++++++++-
 edkrepo_installer/Vendor/edkrepo.cfg |  6 ++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/edkrepo/config/config_factory.py b/edkrepo/config/config_factory.py
index 7a39554..38844d7 100644
--- a/edkrepo/config/config_factory.py
+++ b/edkrepo/config/config_factory.py
@@ -123,11 +123,21 @@ class GlobalConfig(BaseConfig):
                 CfgProp('sparsecheckout', 'always_exclude', 'sparsecheckout_always_exclude', None, True),
                 CfgProp('f2f-cherry-pick', 'ignored_folder_substrings', 'f2f_cp_ignored_folder_substrings'),
                 CfgProp('git-ver', 'minimum', 'minimum_req_git_ver', None, True),
-                CfgProp('git-ver', 'recommended', 'rec_git_ver', None, True)]
+                CfgProp('git-ver', 'recommended', 'rec_git_ver', None, True),
+                CfgProp('command-packages', 'packages', 'command_packages', None, True),
+                CfgProp('preferred-command-package', 
+ 'preferred-package', 'pref_pkg', None, True)]
         if not os.path.isfile(self.filename):
             raise EdkrepoGlobalConfigNotFoundException("edkrepo global config file {} does not exist".format(self.filename))
         super().__init__(self.filename, True)
 
+    @property
+    def command_packages_list(self):
+        initial_list = self.command_packages.split('|')
+        pkg_list = []
+        for pkg in initial_list:
+            pkg_list.append(pkg.strip())
+        return pkg_list
+
     @property
     def manifest_repo_abs_local_path(self):
         """Provides an absolute path to the manifest repo based on configuration file values."""
diff --git a/edkrepo_installer/Vendor/edkrepo.cfg b/edkrepo_installer/Vendor/edkrepo.cfg
index 08724fd..0312c74 100644
--- a/edkrepo_installer/Vendor/edkrepo.cfg
+++ b/edkrepo_installer/Vendor/edkrepo.cfg
@@ -7,6 +7,12 @@ LocalPath = manifest-master  minimum = 2.13.0  recommended = 2.16.2
 
+[command-packages]
+packages = edkrepo.commands
+
+[preferred-command-package]
+preferred-package = edkrepo.commands
+
 [sparsecheckout]
 always_include = BaseTools|Conf|*.*
 always_exclude =
--
2.16.2.windows.1

Re: [edk2-staging/EdkRepo] [PATCH 2/3] EdkRepo: Multiple Command Package Command Factory Update

[Nate DeSimone]

Reviewed-by: Nate DeSimone <nathaniel.l.desimone@intel.com>

-----Original Message-----
From: Desimone, Ashley E <ashley.e.desimone@intel.com> 
Sent: Monday, October 28, 2019 4:23 PM
To: devel@edk2.groups.io
Cc: Desimone, Nathaniel L <nathaniel.l.desimone@intel.com>; Pandya, Puja <puja.pandya@intel.com>
Subject: [edk2-staging/EdkRepo] [PATCH 2/3] EdkRepo: Multiple Command Package Command Factory Update

Add support for iterating through a list of command packages from the global config file and generate a list of commands to be included. Improve the the _is_command() function to succesfully inspect command classes which do not derive from the edkrepo base command class

Add support to list a command package as preferred.

Signed-off-by: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
---
 edkrepo/commands/command_factory.py | 51 ++++++++++++++++++++++++++++---------
 1 file changed, 39 insertions(+), 12 deletions(-)

diff --git a/edkrepo/commands/command_factory.py b/edkrepo/commands/command_factory.py
index abdfb1a..9f2e923 100644
--- a/edkrepo/commands/command_factory.py
+++ b/edkrepo/commands/command_factory.py
@@ -13,6 +13,7 @@ import os
 import sys
 from edkrepo.commands.edkrepo_command import EdkrepoCommand  from edkrepo.commands.composite_command import CompositeCommand
+from edkrepo.config.config_factory import GlobalConfig
 
 def _is_command(CommandClass):
     if CommandClass == EdkrepoCommand:
@@ -20,12 +21,20 @@ def _is_command(CommandClass):
     if CommandClass in EdkrepoCommand.__subclasses__():
         return True
     #Use reflection to see if the class would work as a command anyway
-    funcs = inspect.getmembers(CommandClass, predicate=inspect.ismethod)
+    try:
+        cmd = CommandClass()
+    except:
+        return False
+    funcs = inspect.getmembers(cmd, predicate=inspect.ismethod)
     has_get_metadata = False
     has_run_command = False
     for func in funcs:
         if func[0] == 'get_metadata' and len(inspect.getargspec(func[1])[0]) == 1:
-            has_get_metadata = True
+            try:
+                cmd.get_metadata()
+                has_get_metadata = True
+            except:
+                has_get_metadata = False
         if func[0] == 'run_command':
             arg_spec = inspect.getargspec(func[1])
             if len(arg_spec[0]) == 3 and arg_spec[0][1] == "args" and arg_spec[0][2] == "config":
@@ -36,16 +45,34 @@ def _is_command(CommandClass):
         return False
 
 def get_commands():
-    commands = []
-    for module in os.listdir(os.path.dirname(__file__)):
-        if module == "__init__.py" or os.path.splitext(module)[1] != ".py":
-            continue
-        mod = importlib.import_module("edkrepo.commands.{}".format(os.path.splitext(module)[0]))
-        classes = inspect.getmembers(mod, predicate=inspect.isclass)
-        for cls in classes:
-            if _is_command(cls[1]):
-                commands.append(cls[1])
-    return commands
+    cfg_file = GlobalConfig()
+    cmd_pkg_list = cfg_file.command_packages_list
+    pref_cmd_pkg = cfg_file.pref_pkg
+    commands = {}
+    pref_commands = {}
+    final_cmd_list = []
+    cmd_search_dirs = []
+
+    for cmd_pkg in cmd_pkg_list:
+        mod = importlib.import_module(cmd_pkg)
+        cmd_search_dirs.append((cmd_pkg, os.path.dirname(mod.__file__)))
+    for cmd_dir in cmd_search_dirs:
+        for module in os.listdir(cmd_dir[1]):
+            if module == '__init__.py' or os.path.splitext(module)[1] != '.py':
+                continue
+            mod = importlib.import_module('{}.{}'.format(cmd_dir[0], os.path.splitext(module)[0]))
+            classes = inspect.getmembers(mod, predicate=inspect.isclass)
+            for cls in classes:
+                if _is_command(cls[1]):
+                    if cmd_dir[0] == pref_cmd_pkg:
+                        pref_commands.update([(cls[0], cls[1])])
+                    else:
+                        commands.update([(cls[0], cls[1])])
+    for key in commands.keys():
+        if key not in pref_commands.keys():
+            final_cmd_list.append(commands[key])
+    final_cmd_list.extend(pref_commands.values())
+    return final_cmd_list
 
 def create_composite_command():
     commands = get_commands()
--
2.16.2.windows.1

Re: [edk2-devel] [edk2-staging/EdkRepo] [PATCH 3/3] EdkRepo: Multiple Command Package Build Script Update

[Nate DeSimone]

Reviewed-by: Nate DeSimone <nathaniel.l.desimone@intel.com>

-----Original Message-----
From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Desimone, Ashley E
Sent: Monday, October 28, 2019 4:23 PM
To: devel@edk2.groups.io
Cc: Desimone, Nathaniel L <nathaniel.l.desimone@intel.com>; Pandya, Puja <puja.pandya@intel.com>
Subject: [edk2-devel] [edk2-staging/EdkRepo] [PATCH 3/3] EdkRepo: Multiple Command Package Build Script Update

Add support for building extension packages.

Signed-off-by: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
---
 build-scripts/set_version_and_build_wheels.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/build-scripts/set_version_and_build_wheels.py b/build-scripts/set_version_and_build_wheels.py
index e8d467e..6fd5ab5 100755
--- a/build-scripts/set_version_and_build_wheels.py
+++ b/build-scripts/set_version_and_build_wheels.py
@@ -195,16 +195,23 @@ def make_selfextract_version_script(version):
         ver.write('chcp 437\n')
     print("VersionInfo script written to set_version.bat\n")
 
-def build_wheels():
+def build_wheels(extension_pkgs):
     dir_path = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
     file_path = os.path.join(dir_path, "setup.py")
     if ostype == WIN:
         check_call("py {} \"{}\" bdist_wheel".format(PYTHON_VERSION, file_path), shell=True, cwd=dir_path)
     else:
         check_call('python3 "{}" bdist_wheel'.format(file_path), shell=True, cwd=dir_path)
+    for pkg in extension_pkgs:
+        ext_dir_path = os.path.abspath(pkg)
+        ext_file_path = os.path.join(ext_dir_path, 'setup.py')
+        if ostype == WIN:
+            check_call("py {} \"{}\" bdist_wheel".format(PYTHON_VERSION, ext_file_path), shell=True, cwd=ext_dir_path)
+        else:
+            check_call('python3 "{}" bdist_wheel'.format(ext_file_path), shell=True, cwd=ext_dir_path)
     print("Wheels built successfully")
 
-def copy_wheels_and_set_xml(package_version):
+def copy_wheels_and_set_xml(package_version, extension_pkgs):
     dir_path = os.path.join(os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "dist")
     dest_path = os.path.join(dir_path, "self_extract")
     if ostype == LINUX:
@@ -218,6 +225,14 @@ def copy_wheels_and_set_xml(package_version):
             if data:
                 wheels.append((data.group(1), file_name))
                 _copy_file(os.path.join(dir_path, file_name), os.path.join(dest_path, file_name))
+    for pkg in extension_pkgs:
+        ext_dir_path = os.path.join(os.path.abspath(pkg), 'dist')
+        for file_name in os.listdir(ext_dir_path):
+            if fnmatch.fnmatch(file_name, "*.whl"):
+                data = wheel_package.match(file_name)
+                if data:
+                    wheels.append((data.group(1), file_name))
+                    _copy_file(os.path.join(ext_dir_path, file_name), os.path.join(dest_path, file_name))
     _set_version_number_and_wheels_cr_tools_installer_config(package_version, wheels)
 
 def create_final_copy_script(version):
@@ -251,6 +266,7 @@ def make_version_cfg_file(version):
             install_cfg.write(f)
 
 def main():
+    extension_pkgs = []
     print("Building Python packages and generating new version number...")
     (version, package_version) = get_current_version_number()
     set_version_number_setup_py(package_version)
@@ -259,8 +275,8 @@ def main():
         set_version_number_setup_launcher_rc(version)
         make_selfextract_version_script(version)
     print("Version number generated and set successfully")
-    build_wheels()
-    copy_wheels_and_set_xml(package_version)
+    build_wheels(extension_pkgs)
+    copy_wheels_and_set_xml(package_version, extension_pkgs)
     make_version_cfg_file(version)
     create_final_copy_script(version)
 
-- 
2.16.2.windows.1