* [edk2-staging/EdkRepo] [PATCH 2/3] EdkRepo: Multiple Command Package Command Factory Update
2019-10-28 23:22 [edk2-staging/EdkRepo][PATCH 1/3] EdkRepo: Multiple Command Package Configuration Support Desimone, Ashley E
@ 2019-10-28 23:22 ` Desimone, Ashley E
2019-10-28 23:47 ` Nate DeSimone
2019-10-28 23:22 ` [edk2-staging/EdkRepo] [PATCH 3/3] EdkRepo: Multiple Command Package Build Script Update Desimone, Ashley E
2019-10-28 23:47 ` [edk2-devel] [edk2-staging/EdkRepo][PATCH 1/3] EdkRepo: Multiple Command Package Configuration Support Nate DeSimone
2 siblings, 1 reply; 6+ messages in thread
From: Desimone, Ashley E @ 2019-10-28 23:22 UTC (permalink / raw)
To: devel; +Cc: Nate DeSimone, Puja Pandya
Add support for iterating through a list of command packages
from the global config file and generate a list of commands
to be included. Improve the the _is_command() function to
succesfully inspect command classes which do not derive
from the edkrepo base command class
Add support to list a command package as preferred.
Signed-off-by: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
---
edkrepo/commands/command_factory.py | 51 ++++++++++++++++++++++++++++---------
1 file changed, 39 insertions(+), 12 deletions(-)
diff --git a/edkrepo/commands/command_factory.py b/edkrepo/commands/command_factory.py
index abdfb1a..9f2e923 100644
--- a/edkrepo/commands/command_factory.py
+++ b/edkrepo/commands/command_factory.py
@@ -13,6 +13,7 @@ import os
import sys
from edkrepo.commands.edkrepo_command import EdkrepoCommand
from edkrepo.commands.composite_command import CompositeCommand
+from edkrepo.config.config_factory import GlobalConfig
def _is_command(CommandClass):
if CommandClass == EdkrepoCommand:
@@ -20,12 +21,20 @@ def _is_command(CommandClass):
if CommandClass in EdkrepoCommand.__subclasses__():
return True
#Use reflection to see if the class would work as a command anyway
- funcs = inspect.getmembers(CommandClass, predicate=inspect.ismethod)
+ try:
+ cmd = CommandClass()
+ except:
+ return False
+ funcs = inspect.getmembers(cmd, predicate=inspect.ismethod)
has_get_metadata = False
has_run_command = False
for func in funcs:
if func[0] == 'get_metadata' and len(inspect.getargspec(func[1])[0]) == 1:
- has_get_metadata = True
+ try:
+ cmd.get_metadata()
+ has_get_metadata = True
+ except:
+ has_get_metadata = False
if func[0] == 'run_command':
arg_spec = inspect.getargspec(func[1])
if len(arg_spec[0]) == 3 and arg_spec[0][1] == "args" and arg_spec[0][2] == "config":
@@ -36,16 +45,34 @@ def _is_command(CommandClass):
return False
def get_commands():
- commands = []
- for module in os.listdir(os.path.dirname(__file__)):
- if module == "__init__.py" or os.path.splitext(module)[1] != ".py":
- continue
- mod = importlib.import_module("edkrepo.commands.{}".format(os.path.splitext(module)[0]))
- classes = inspect.getmembers(mod, predicate=inspect.isclass)
- for cls in classes:
- if _is_command(cls[1]):
- commands.append(cls[1])
- return commands
+ cfg_file = GlobalConfig()
+ cmd_pkg_list = cfg_file.command_packages_list
+ pref_cmd_pkg = cfg_file.pref_pkg
+ commands = {}
+ pref_commands = {}
+ final_cmd_list = []
+ cmd_search_dirs = []
+
+ for cmd_pkg in cmd_pkg_list:
+ mod = importlib.import_module(cmd_pkg)
+ cmd_search_dirs.append((cmd_pkg, os.path.dirname(mod.__file__)))
+ for cmd_dir in cmd_search_dirs:
+ for module in os.listdir(cmd_dir[1]):
+ if module == '__init__.py' or os.path.splitext(module)[1] != '.py':
+ continue
+ mod = importlib.import_module('{}.{}'.format(cmd_dir[0], os.path.splitext(module)[0]))
+ classes = inspect.getmembers(mod, predicate=inspect.isclass)
+ for cls in classes:
+ if _is_command(cls[1]):
+ if cmd_dir[0] == pref_cmd_pkg:
+ pref_commands.update([(cls[0], cls[1])])
+ else:
+ commands.update([(cls[0], cls[1])])
+ for key in commands.keys():
+ if key not in pref_commands.keys():
+ final_cmd_list.append(commands[key])
+ final_cmd_list.extend(pref_commands.values())
+ return final_cmd_list
def create_composite_command():
commands = get_commands()
--
2.16.2.windows.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* Re: [edk2-staging/EdkRepo] [PATCH 2/3] EdkRepo: Multiple Command Package Command Factory Update
2019-10-28 23:22 ` [edk2-staging/EdkRepo] [PATCH 2/3] EdkRepo: Multiple Command Package Command Factory Update Desimone, Ashley E
@ 2019-10-28 23:47 ` Nate DeSimone
0 siblings, 0 replies; 6+ messages in thread
From: Nate DeSimone @ 2019-10-28 23:47 UTC (permalink / raw)
To: Desimone, Ashley E, devel@edk2.groups.io; +Cc: Pandya, Puja
Reviewed-by: Nate DeSimone <nathaniel.l.desimone@intel.com>
-----Original Message-----
From: Desimone, Ashley E <ashley.e.desimone@intel.com>
Sent: Monday, October 28, 2019 4:23 PM
To: devel@edk2.groups.io
Cc: Desimone, Nathaniel L <nathaniel.l.desimone@intel.com>; Pandya, Puja <puja.pandya@intel.com>
Subject: [edk2-staging/EdkRepo] [PATCH 2/3] EdkRepo: Multiple Command Package Command Factory Update
Add support for iterating through a list of command packages from the global config file and generate a list of commands to be included. Improve the the _is_command() function to succesfully inspect command classes which do not derive from the edkrepo base command class
Add support to list a command package as preferred.
Signed-off-by: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
---
edkrepo/commands/command_factory.py | 51 ++++++++++++++++++++++++++++---------
1 file changed, 39 insertions(+), 12 deletions(-)
diff --git a/edkrepo/commands/command_factory.py b/edkrepo/commands/command_factory.py
index abdfb1a..9f2e923 100644
--- a/edkrepo/commands/command_factory.py
+++ b/edkrepo/commands/command_factory.py
@@ -13,6 +13,7 @@ import os
import sys
from edkrepo.commands.edkrepo_command import EdkrepoCommand from edkrepo.commands.composite_command import CompositeCommand
+from edkrepo.config.config_factory import GlobalConfig
def _is_command(CommandClass):
if CommandClass == EdkrepoCommand:
@@ -20,12 +21,20 @@ def _is_command(CommandClass):
if CommandClass in EdkrepoCommand.__subclasses__():
return True
#Use reflection to see if the class would work as a command anyway
- funcs = inspect.getmembers(CommandClass, predicate=inspect.ismethod)
+ try:
+ cmd = CommandClass()
+ except:
+ return False
+ funcs = inspect.getmembers(cmd, predicate=inspect.ismethod)
has_get_metadata = False
has_run_command = False
for func in funcs:
if func[0] == 'get_metadata' and len(inspect.getargspec(func[1])[0]) == 1:
- has_get_metadata = True
+ try:
+ cmd.get_metadata()
+ has_get_metadata = True
+ except:
+ has_get_metadata = False
if func[0] == 'run_command':
arg_spec = inspect.getargspec(func[1])
if len(arg_spec[0]) == 3 and arg_spec[0][1] == "args" and arg_spec[0][2] == "config":
@@ -36,16 +45,34 @@ def _is_command(CommandClass):
return False
def get_commands():
- commands = []
- for module in os.listdir(os.path.dirname(__file__)):
- if module == "__init__.py" or os.path.splitext(module)[1] != ".py":
- continue
- mod = importlib.import_module("edkrepo.commands.{}".format(os.path.splitext(module)[0]))
- classes = inspect.getmembers(mod, predicate=inspect.isclass)
- for cls in classes:
- if _is_command(cls[1]):
- commands.append(cls[1])
- return commands
+ cfg_file = GlobalConfig()
+ cmd_pkg_list = cfg_file.command_packages_list
+ pref_cmd_pkg = cfg_file.pref_pkg
+ commands = {}
+ pref_commands = {}
+ final_cmd_list = []
+ cmd_search_dirs = []
+
+ for cmd_pkg in cmd_pkg_list:
+ mod = importlib.import_module(cmd_pkg)
+ cmd_search_dirs.append((cmd_pkg, os.path.dirname(mod.__file__)))
+ for cmd_dir in cmd_search_dirs:
+ for module in os.listdir(cmd_dir[1]):
+ if module == '__init__.py' or os.path.splitext(module)[1] != '.py':
+ continue
+ mod = importlib.import_module('{}.{}'.format(cmd_dir[0], os.path.splitext(module)[0]))
+ classes = inspect.getmembers(mod, predicate=inspect.isclass)
+ for cls in classes:
+ if _is_command(cls[1]):
+ if cmd_dir[0] == pref_cmd_pkg:
+ pref_commands.update([(cls[0], cls[1])])
+ else:
+ commands.update([(cls[0], cls[1])])
+ for key in commands.keys():
+ if key not in pref_commands.keys():
+ final_cmd_list.append(commands[key])
+ final_cmd_list.extend(pref_commands.values())
+ return final_cmd_list
def create_composite_command():
commands = get_commands()
--
2.16.2.windows.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [edk2-staging/EdkRepo] [PATCH 3/3] EdkRepo: Multiple Command Package Build Script Update
2019-10-28 23:22 [edk2-staging/EdkRepo][PATCH 1/3] EdkRepo: Multiple Command Package Configuration Support Desimone, Ashley E
2019-10-28 23:22 ` [edk2-staging/EdkRepo] [PATCH 2/3] EdkRepo: Multiple Command Package Command Factory Update Desimone, Ashley E
@ 2019-10-28 23:22 ` Desimone, Ashley E
2019-10-28 23:47 ` [edk2-devel] " Nate DeSimone
2019-10-28 23:47 ` [edk2-devel] [edk2-staging/EdkRepo][PATCH 1/3] EdkRepo: Multiple Command Package Configuration Support Nate DeSimone
2 siblings, 1 reply; 6+ messages in thread
From: Desimone, Ashley E @ 2019-10-28 23:22 UTC (permalink / raw)
To: devel; +Cc: Nate DeSimone, Puja Pandya
Add support for building extension packages.
Signed-off-by: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
---
build-scripts/set_version_and_build_wheels.py | 24 ++++++++++++++++++++----
1 file changed, 20 insertions(+), 4 deletions(-)
diff --git a/build-scripts/set_version_and_build_wheels.py b/build-scripts/set_version_and_build_wheels.py
index e8d467e..6fd5ab5 100755
--- a/build-scripts/set_version_and_build_wheels.py
+++ b/build-scripts/set_version_and_build_wheels.py
@@ -195,16 +195,23 @@ def make_selfextract_version_script(version):
ver.write('chcp 437\n')
print("VersionInfo script written to set_version.bat\n")
-def build_wheels():
+def build_wheels(extension_pkgs):
dir_path = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
file_path = os.path.join(dir_path, "setup.py")
if ostype == WIN:
check_call("py {} \"{}\" bdist_wheel".format(PYTHON_VERSION, file_path), shell=True, cwd=dir_path)
else:
check_call('python3 "{}" bdist_wheel'.format(file_path), shell=True, cwd=dir_path)
+ for pkg in extension_pkgs:
+ ext_dir_path = os.path.abspath(pkg)
+ ext_file_path = os.path.join(ext_dir_path, 'setup.py')
+ if ostype == WIN:
+ check_call("py {} \"{}\" bdist_wheel".format(PYTHON_VERSION, ext_file_path), shell=True, cwd=ext_dir_path)
+ else:
+ check_call('python3 "{}" bdist_wheel'.format(ext_file_path), shell=True, cwd=ext_dir_path)
print("Wheels built successfully")
-def copy_wheels_and_set_xml(package_version):
+def copy_wheels_and_set_xml(package_version, extension_pkgs):
dir_path = os.path.join(os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "dist")
dest_path = os.path.join(dir_path, "self_extract")
if ostype == LINUX:
@@ -218,6 +225,14 @@ def copy_wheels_and_set_xml(package_version):
if data:
wheels.append((data.group(1), file_name))
_copy_file(os.path.join(dir_path, file_name), os.path.join(dest_path, file_name))
+ for pkg in extension_pkgs:
+ ext_dir_path = os.path.join(os.path.abspath(pkg), 'dist')
+ for file_name in os.listdir(ext_dir_path):
+ if fnmatch.fnmatch(file_name, "*.whl"):
+ data = wheel_package.match(file_name)
+ if data:
+ wheels.append((data.group(1), file_name))
+ _copy_file(os.path.join(ext_dir_path, file_name), os.path.join(dest_path, file_name))
_set_version_number_and_wheels_cr_tools_installer_config(package_version, wheels)
def create_final_copy_script(version):
@@ -251,6 +266,7 @@ def make_version_cfg_file(version):
install_cfg.write(f)
def main():
+ extension_pkgs = []
print("Building Python packages and generating new version number...")
(version, package_version) = get_current_version_number()
set_version_number_setup_py(package_version)
@@ -259,8 +275,8 @@ def main():
set_version_number_setup_launcher_rc(version)
make_selfextract_version_script(version)
print("Version number generated and set successfully")
- build_wheels()
- copy_wheels_and_set_xml(package_version)
+ build_wheels(extension_pkgs)
+ copy_wheels_and_set_xml(package_version, extension_pkgs)
make_version_cfg_file(version)
create_final_copy_script(version)
--
2.16.2.windows.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* Re: [edk2-devel] [edk2-staging/EdkRepo] [PATCH 3/3] EdkRepo: Multiple Command Package Build Script Update
2019-10-28 23:22 ` [edk2-staging/EdkRepo] [PATCH 3/3] EdkRepo: Multiple Command Package Build Script Update Desimone, Ashley E
@ 2019-10-28 23:47 ` Nate DeSimone
0 siblings, 0 replies; 6+ messages in thread
From: Nate DeSimone @ 2019-10-28 23:47 UTC (permalink / raw)
To: devel@edk2.groups.io, Desimone, Ashley E; +Cc: Pandya, Puja
Reviewed-by: Nate DeSimone <nathaniel.l.desimone@intel.com>
-----Original Message-----
From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Desimone, Ashley E
Sent: Monday, October 28, 2019 4:23 PM
To: devel@edk2.groups.io
Cc: Desimone, Nathaniel L <nathaniel.l.desimone@intel.com>; Pandya, Puja <puja.pandya@intel.com>
Subject: [edk2-devel] [edk2-staging/EdkRepo] [PATCH 3/3] EdkRepo: Multiple Command Package Build Script Update
Add support for building extension packages.
Signed-off-by: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
---
build-scripts/set_version_and_build_wheels.py | 24 ++++++++++++++++++++----
1 file changed, 20 insertions(+), 4 deletions(-)
diff --git a/build-scripts/set_version_and_build_wheels.py b/build-scripts/set_version_and_build_wheels.py
index e8d467e..6fd5ab5 100755
--- a/build-scripts/set_version_and_build_wheels.py
+++ b/build-scripts/set_version_and_build_wheels.py
@@ -195,16 +195,23 @@ def make_selfextract_version_script(version):
ver.write('chcp 437\n')
print("VersionInfo script written to set_version.bat\n")
-def build_wheels():
+def build_wheels(extension_pkgs):
dir_path = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
file_path = os.path.join(dir_path, "setup.py")
if ostype == WIN:
check_call("py {} \"{}\" bdist_wheel".format(PYTHON_VERSION, file_path), shell=True, cwd=dir_path)
else:
check_call('python3 "{}" bdist_wheel'.format(file_path), shell=True, cwd=dir_path)
+ for pkg in extension_pkgs:
+ ext_dir_path = os.path.abspath(pkg)
+ ext_file_path = os.path.join(ext_dir_path, 'setup.py')
+ if ostype == WIN:
+ check_call("py {} \"{}\" bdist_wheel".format(PYTHON_VERSION, ext_file_path), shell=True, cwd=ext_dir_path)
+ else:
+ check_call('python3 "{}" bdist_wheel'.format(ext_file_path), shell=True, cwd=ext_dir_path)
print("Wheels built successfully")
-def copy_wheels_and_set_xml(package_version):
+def copy_wheels_and_set_xml(package_version, extension_pkgs):
dir_path = os.path.join(os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "dist")
dest_path = os.path.join(dir_path, "self_extract")
if ostype == LINUX:
@@ -218,6 +225,14 @@ def copy_wheels_and_set_xml(package_version):
if data:
wheels.append((data.group(1), file_name))
_copy_file(os.path.join(dir_path, file_name), os.path.join(dest_path, file_name))
+ for pkg in extension_pkgs:
+ ext_dir_path = os.path.join(os.path.abspath(pkg), 'dist')
+ for file_name in os.listdir(ext_dir_path):
+ if fnmatch.fnmatch(file_name, "*.whl"):
+ data = wheel_package.match(file_name)
+ if data:
+ wheels.append((data.group(1), file_name))
+ _copy_file(os.path.join(ext_dir_path, file_name), os.path.join(dest_path, file_name))
_set_version_number_and_wheels_cr_tools_installer_config(package_version, wheels)
def create_final_copy_script(version):
@@ -251,6 +266,7 @@ def make_version_cfg_file(version):
install_cfg.write(f)
def main():
+ extension_pkgs = []
print("Building Python packages and generating new version number...")
(version, package_version) = get_current_version_number()
set_version_number_setup_py(package_version)
@@ -259,8 +275,8 @@ def main():
set_version_number_setup_launcher_rc(version)
make_selfextract_version_script(version)
print("Version number generated and set successfully")
- build_wheels()
- copy_wheels_and_set_xml(package_version)
+ build_wheels(extension_pkgs)
+ copy_wheels_and_set_xml(package_version, extension_pkgs)
make_version_cfg_file(version)
create_final_copy_script(version)
--
2.16.2.windows.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [edk2-devel] [edk2-staging/EdkRepo][PATCH 1/3] EdkRepo: Multiple Command Package Configuration Support
2019-10-28 23:22 [edk2-staging/EdkRepo][PATCH 1/3] EdkRepo: Multiple Command Package Configuration Support Desimone, Ashley E
2019-10-28 23:22 ` [edk2-staging/EdkRepo] [PATCH 2/3] EdkRepo: Multiple Command Package Command Factory Update Desimone, Ashley E
2019-10-28 23:22 ` [edk2-staging/EdkRepo] [PATCH 3/3] EdkRepo: Multiple Command Package Build Script Update Desimone, Ashley E
@ 2019-10-28 23:47 ` Nate DeSimone
2 siblings, 0 replies; 6+ messages in thread
From: Nate DeSimone @ 2019-10-28 23:47 UTC (permalink / raw)
To: devel@edk2.groups.io, Desimone, Ashley E; +Cc: Pandya, Puja
Reviewed-by: Nate DeSimone <nathaniel.l.desimone@intel.com>
-----Original Message-----
From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Desimone, Ashley E
Sent: Monday, October 28, 2019 4:23 PM
To: devel@edk2.groups.io
Cc: Desimone, Nathaniel L <nathaniel.l.desimone@intel.com>; Pandya, Puja <puja.pandya@intel.com>
Subject: [edk2-devel] [edk2-staging/EdkRepo][PATCH 1/3] EdkRepo: Multiple Command Package Configuration Support
Add support for a new required sections, command-packages and preferred-command-packages,to the GlobalConfig class.
Additionaly update the existing edkrepo.cfg file.
Signed-off-by: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
---
edkrepo/config/config_factory.py | 12 +++++++++++-
edkrepo_installer/Vendor/edkrepo.cfg | 6 ++++++
2 files changed, 17 insertions(+), 1 deletion(-)
diff --git a/edkrepo/config/config_factory.py b/edkrepo/config/config_factory.py
index 7a39554..38844d7 100644
--- a/edkrepo/config/config_factory.py
+++ b/edkrepo/config/config_factory.py
@@ -123,11 +123,21 @@ class GlobalConfig(BaseConfig):
CfgProp('sparsecheckout', 'always_exclude', 'sparsecheckout_always_exclude', None, True),
CfgProp('f2f-cherry-pick', 'ignored_folder_substrings', 'f2f_cp_ignored_folder_substrings'),
CfgProp('git-ver', 'minimum', 'minimum_req_git_ver', None, True),
- CfgProp('git-ver', 'recommended', 'rec_git_ver', None, True)]
+ CfgProp('git-ver', 'recommended', 'rec_git_ver', None, True),
+ CfgProp('command-packages', 'packages', 'command_packages', None, True),
+ CfgProp('preferred-command-package',
+ 'preferred-package', 'pref_pkg', None, True)]
if not os.path.isfile(self.filename):
raise EdkrepoGlobalConfigNotFoundException("edkrepo global config file {} does not exist".format(self.filename))
super().__init__(self.filename, True)
+ @property
+ def command_packages_list(self):
+ initial_list = self.command_packages.split('|')
+ pkg_list = []
+ for pkg in initial_list:
+ pkg_list.append(pkg.strip())
+ return pkg_list
+
@property
def manifest_repo_abs_local_path(self):
"""Provides an absolute path to the manifest repo based on configuration file values."""
diff --git a/edkrepo_installer/Vendor/edkrepo.cfg b/edkrepo_installer/Vendor/edkrepo.cfg
index 08724fd..0312c74 100644
--- a/edkrepo_installer/Vendor/edkrepo.cfg
+++ b/edkrepo_installer/Vendor/edkrepo.cfg
@@ -7,6 +7,12 @@ LocalPath = manifest-master minimum = 2.13.0 recommended = 2.16.2
+[command-packages]
+packages = edkrepo.commands
+
+[preferred-command-package]
+preferred-package = edkrepo.commands
+
[sparsecheckout]
always_include = BaseTools|Conf|*.*
always_exclude =
--
2.16.2.windows.1
^ permalink raw reply related [flat|nested] 6+ messages in thread