From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mga01.intel.com (mga01.intel.com [192.55.52.88]) by mx.groups.io with SMTP id smtpd.web12.16194.1589678979198516244 for ; Sat, 16 May 2020 18:29:39 -0700 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 192.55.52.88, mailfrom: erik.c.bjorge@intel.com) IronPort-SDR: L68qtA5MZ3EKcuOUwuiOIa4rZskAT/SfXvbVJqQz4a6U6VlLg4wJqBoaO5X1bKbZguvIbEhw6Z shP7Tfc3TaQA== X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga001.jf.intel.com ([10.7.209.18]) by fmsmga101.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 16 May 2020 18:29:38 -0700 IronPort-SDR: sr/dnZAffa+Tw+gyWt0YCUpBilu1vrbgFWnYyD43tsm0a+/6Bm6VCrHS8NKYbHJz7gYo54n/QZ eyC9B9JTEewQ== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.73,401,1583222400"; d="scan'208";a="342454492" Received: from ecbjorge-mobl1.amr.corp.intel.com ([10.209.101.250]) by orsmga001.jf.intel.com with ESMTP; 16 May 2020 18:29:38 -0700 From: "Bjorge, Erik C" To: devel@edk2.groups.io Cc: Ashley E Desimone , Nate DeSimone , Puja Pandya , Bret Barkelew , Prince Agyeman Subject: [edk2-staging/EdkRepo] [PATCH v1 1/2] EdkRepo: Adding selective submodule init script Date: Sat, 16 May 2020 18:29:21 -0700 Message-Id: <0b6d0db81fbb9309a1644ea34eeb4ccf68b19b3b.1589678697.git.erik.c.bjorge@intel.com> X-Mailer: git-send-email 2.21.0.windows.1 In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Adds selective submodule support functions and command line scripting support. The existing support will be ported over to use this functionality. Signed-off-by: Erik Bjorge Cc: Ashley E Desimone Cc: Nate DeSimone Cc: Puja Pandya Cc: Bret Barkelew Cc: Prince Agyeman Cc: Erik Bjorge --- project_utils/arguments/__init__.py | 8 + project_utils/arguments/submodule_args.py | 24 ++ project_utils/project_utils_strings.py | 24 ++ project_utils/submodule.py | 389 ++++++++++++++++++++++ setup.py | 50 +-- 5 files changed, 470 insertions(+), 25 deletions(-) create mode 100644 project_utils/arguments/__init__.py create mode 100644 project_utils/arguments/submodule_args.py create mode 100644 project_utils/project_utils_strings.py create mode 100644 project_utils/submodule.py diff --git a/project_utils/arguments/__init__.py b/project_utils/arguments/__init__.py new file mode 100644 index 0000000..0486261 --- /dev/null +++ b/project_utils/arguments/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +# +## @file +# __init__.py +# +# Copyright (c) 2020, Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +# diff --git a/project_utils/arguments/submodule_args.py b/project_utils/arguments/submodule_args.py new file mode 100644 index 0000000..804ce98 --- /dev/null +++ b/project_utils/arguments/submodule_args.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +## @file +# submodule.py +# +# Copyright (c) 2020, Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +# + +""" +Strings for command line arguments. +""" + +SUBMOD_MANIFEST_HELP = 'The current manifest file.' +SUBMOD_COMBO_HELP = 'The current branch combination in use. If a combo is not specified, the current ' \ + 'combo will be used.' +SUBMOD_NEW_MANIFEST_HELP = 'The new manifest file. Used to test manifest upgrade paths.' +SUBMOD_NEW_COMBO_HELP = 'The new branch combination to use. Used to test switching combos.' +SUBMOD_DEINIT_HELP = 'Performs submodule deinitialization before initializing submodules.' +SUBMOD_INIT_FULL_HELP = 'Initialize all submodules (recursive).' +SUBMOD_DEINIT_FULL_HELP = 'Deinitialize all submodules.' +SUBMOD_WORKSPACE_HELP = 'The project workspace root. If not specified the current working directory will ' \ + 'be used.' +SUBMOD_VERBOSE_HELP = 'Enables verbose messaging.' diff --git a/project_utils/project_utils_strings.py b/project_utils/project_utils_strings.py new file mode 100644 index 0000000..33c22d2 --- /dev/null +++ b/project_utils/project_utils_strings.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +## @file +# humble.py +# +# Copyright (c) 2020, Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +# + +""" +Contains strings used by the project_utils modules. +""" +# Status update messages +SUBMOD_DEINIT = 'Deinitializing submodules' +SUBMOD_DEINIT_FULL = 'Deinitializing all submodules' +SUBMOD_INIT_UPDATE = 'Initializing/Updating submodules' +SUBMOD_INIT_FULL = 'Initializing/Updating all submodules (recursive)' + +# Verbose messages +SUBMOD_INIT_PATH = 'Submodule init: {}' +SUBMOD_DEINIT_PATH = 'Submodule deinit: {}' +SUBMOD_SYNC_PATH = 'Submodule sync: {}' +SUBMOD_UPDATE_PATH = 'Submodule update: {}' +SUBMOD_EXCEPTION = '- Exception: {}' diff --git a/project_utils/submodule.py b/project_utils/submodule.py new file mode 100644 index 0000000..6534f2c --- /dev/null +++ b/project_utils/submodule.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +# +## @file +# submodule.py +# +# Copyright (c) 2020, Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +# +import argparse +import git +import os +import sys +import traceback + +from edkrepo_manifest_parser.edk_manifest import ManifestXml +import project_utils.project_utils_strings as strings +import project_utils.arguments.submodule_args as arguments + + +def _init(repo, submodules=None, verbose=False): + """ + Performs submodule initialization. Note that if no selective submodule initialization + is enabled the init will be handled in _update. + + repo - GitPython Repo object + submodules - The list of selective submodule init objects + verbose - Enable verbose messages + """ + # Only handle selective initialization with this method. For fully recursive init + # it is cleaner to do it as part of the update process. + if submodules is not None: + for sub in submodules: + if verbose: + print(strings.SUBMOD_INIT_PATH.format(sub.path)) + output_data = repo.git.execute(['git', 'submodule', 'init', '--', sub.path], + with_extended_output=True, with_stdout=True) + _display_git_outupt(output_data, verbose) + return + + +def _deinit(repo, submodules=None, verbose=False): + """ + Performs deinitialization of submodules. + + repo - GitPython Repo object + submodules - The list of selective submodule init objects + verbose - Enable verbose messages + """ + if submodules is None: + output_data = repo.git.execute(['git', 'submodule', 'deinit', '--all'], + with_extended_output=True, with_stdout=True) + _display_git_outupt(output_data, verbose) + else: + for sub in submodules: + if verbose: + print(strings.SUBMOD_DEINIT_PATH.format(sub.path)) + output_data = repo.git.execute(['git', 'submodule', 'deinit', '--', sub.path], + with_extended_output=True, with_stdout=True) + _display_git_outupt(output_data, verbose) + return + + +def _update(repo, submodules=None, verbose=False, recursive=False): + """ + Performs the update of submodules. This includes the sync and update operations. + + repo - GitPython Repo object + submodules - The list of selective submodule init objects + verbose - Enable verbose messages + recursive - If submodules is None then use this parameter to determine if initialization + should be recursive + """ + # Now perform the update of the submodules. For a fully recursive submodule init + # this code will update and initialize at the same time. + if submodules is None: + cmd = ['git', 'submodule', 'sync'] + if recursive: + cmd.append('--recursive') + output_data = repo.git.execute(cmd, with_extended_output=True, with_stdout=True) + _display_git_outupt(output_data, verbose) + cmd = ['git', 'submodule', 'update', '--init'] + if recursive: + cmd.append('--recursive') + output_data = repo.git.execute(cmd, with_extended_output=True, with_stdout=True) + _display_git_outupt(output_data, verbose) + else: + for sub in submodules: + if verbose: + print(strings.SUBMOD_SYNC_PATH.format(sub.path)) + cmd = ['git', 'submodule', 'sync'] + if sub.recursive: + cmd.append('--recursive') + cmd.extend(['--', sub.path]) + output_data = repo.git.execute(cmd, with_extended_output=True, with_stdout=True) + _display_git_outupt(output_data, verbose) + if verbose: + print(strings.SUBMOD_UPDATE_PATH.format(sub.path)) + cmd = ['git', 'submodule', 'update', '--init'] + if sub.recursive: + cmd.append('--recursive') + cmd.extend(['--', sub.path]) + output_data = repo.git.execute(cmd, with_extended_output=True, with_stdout=True) + _display_git_outupt(output_data, verbose) + return + + +def _display_git_outupt(output_data, verbose=False): + """ + Displays output from GitPython git commands + + output_data - Output from the git.execute method + verbose - Enable verbose messages + """ + if verbose and output_data[0]: + print(output_data[0]) + if output_data[1]: + print(output_data[1]) + if verbose and output_data[2]: + print(output_data[2]) + + +def _compute_change(current_subs, new_subs): + """ + Determines the list of submodules that have been removed. Also needs to + determine if any submodules need to have recursive init disabled + + current_subs - List of selective submodule init entries for the current combo + new_subs - List of selective submodule init entries for the new combo + """ + # Create data objects for determining what submodules need to be deinitialized + tmp_current = {x.path: x for x in current_subs} + tmp_new = {x.path: x for x in new_subs} + + # Initial deinitialization list + deinit_paths = list(set(tmp_current).difference(set(tmp_new))) + + # Check for change in recursive initialization for the specific submodules + for path in tmp_new: + if path in tmp_new and path in tmp_current: + if tmp_current[path].recursive and not tmp_new[path].recursive: + deinit_paths.append(path) + + # Create the final list of submodules that need to be deinitialized + return [x for x in current_subs if x.path in deinit_paths] + + +def _get_submodule_enable(manifest, remote_name, combo): + """ + Determines if submodules are enabled for the current repo and combo + + manifest - Manifest object + remote_name - The name of the current remote being processed + combo - The current combo name being processed + """ + repo_sources = manifest.get_repo_sources(combo) + for source in repo_sources: + if source.remote_name == remote_name: + return source.enable_submodule + return False + + +def _get_submodule_state(remote_name, start_manifest, start_combo, end_manifest=None, end_combo=None): + """ + Determines the state of submodules across manifest and combo changes. + + remote_name - The name of the current remote being processed. + start_manifest - Initial manifest parser object. + start_combo - Initial combo. + end_manifest - The manifest parser object for the project if the manifest is being updated. + end_combo - The combination name at the end of the operation if being modified. + """ + start_subs = start_manifest.get_submodule_init_paths(remote_name, start_combo) + start_subs_enabled = _get_submodule_enable(start_manifest, remote_name, start_combo) + if end_combo is not None: + if end_manifest is not None: + end_subs = end_manifest.get_submodule_init_paths(remote_name, end_combo) + end_subs_enabled = _get_submodule_enable(end_manifest, remote_name, end_combo) + else: + end_subs = start_manifest.get_submodule_init_paths(remote_name, end_combo) + end_subs_enabled = _get_submodule_enable(start_manifest, remote_name, end_combo) + else: + if end_manifest is not None: + end_subs = end_manifest.get_submodule_init_paths(remote_name, start_combo) + end_subs_enabled = _get_submodule_enable(end_manifest, remote_name, start_combo) + else: + end_subs = start_subs + end_subs_enabled = start_subs_enabled + return start_subs, start_subs_enabled, end_subs, end_subs_enabled + + +def deinit_full(workspace, manifest, verbose=False): + """ + Does full submodule deinit based on the current combo. + + workspace - Path to the current workspace. + manifest - The current manifest parser object. + """ + print(strings.SUBMOD_DEINIT_FULL) + current_combo = manifest.general_config.current_combo + repo_sources = manifest.get_repo_sources(current_combo) + for source in repo_sources: + if _get_submodule_enable(manifest, source.remote_name, current_combo): + # Open the repo and process submodules + try: + repo = git.Repo(os.path.join(workspace, source.root)) + except Exception as repo_error: + if args.verbose: + print(strings.SUBMOD_EXCEPTION.format(repo_error)) + continue + _deinit(repo, None, verbose) + + +def init_full(workspace, manifest, verbose=False): + """ + Does full submodule init based on the current combo. + + workspace - Path to the current workspace. + manifest - The current manifest parser object. + """ + print(strings.SUBMOD_INIT_FULL) + current_combo = manifest.general_config.current_combo + repo_sources = manifest.get_repo_sources(current_combo) + for source in repo_sources: + if _get_submodule_enable(manifest, source.remote_name, current_combo): + # Open the repo and process submodules + try: + repo = git.Repo(os.path.join(workspace, source.root)) + except Exception as repo_error: + if args.verbose: + print(strings.SUBMOD_EXCEPTION.format(repo_error)) + continue + _update(repo, None, verbose, True) + + +def deinit_submodules(workspace, start_manifest, start_combo, + end_manifest=None, end_combo=None, + verbose=False): + """ + Deinitializes the submodules for a project. + + workspace - Path to the current workspace. + start_manifest - The manifest parser object for the project. + start_combo - The combination name at the start of the operation. + end_manifest - The manifest parser object for the project if the manifest is being updated. If the + manifest file is not being updated us the value None. + end_combo - The combination name at the end of the operation. If the combo will not change + use the value None. + verbose - Enable verbose messages. + """ + # Process each repo that may have submodules enabled + print(strings.SUBMOD_DEINIT) + repo_sources = start_manifest.get_repo_sources(start_combo) + for source in repo_sources: + # Open the repo and process submodules + try: + repo = git.Repo(os.path.join(workspace, source.root)) + except Exception as repo_error: + if args.verbose: + print(strings.SUBMOD_EXCEPTION.format(repo_error)) + continue + + # Collect the submodule initialization data from manifest as well as if submodules + # should be processed for the repo. + start_subs, start_subs_enabled, end_subs, end_subs_enabled = _get_submodule_state(source.remote_name, + start_manifest, + start_combo, + end_manifest, + end_combo) + if not start_subs_enabled and not end_subs_enabled: + # At this point submodules are not enabled on this repo + continue + + # Compute the list of submodules that need to be removed and added + deinit_list = _compute_change(start_subs, end_subs) + + # Deinitialize submodules + if (start_subs_enabled and not end_subs_enabled) or (len(start_subs) == 0 and len(end_subs) != 0): + # Submodules are being disabled for the entire repo so do a + # full deinit on the repo. + _deinit(repo, None, verbose) + else: + # Do the deinit based on the list + _deinit(repo, deinit_list, verbose) + + +def init_submodules(workspace, manifest, combo_name, verbose=False): + """ + Updates the submodules for a specific repo. Note that operations like checkout will need to use + the start_combo and end_combo parameter. Operations like clone will just need to use the + start_combo parameter. + + workspace - Path to the current workspace. + manifest - The manifest parser object for the project. + combo_name - The combination name at the start of the operation. + verbose - Enable verbose messages. + """ + # Process each repo that may have submodules enabled + print(strings.SUBMOD_INIT_UPDATE) + repo_sources = manifest.get_repo_sources(combo_name) + for source in repo_sources: + # Open the repo and process submodules + try: + repo = git.Repo(os.path.join(workspace, source.root)) + except Exception as repo_error: + if args.verbose: + print(strings.SUBMOD_EXCEPTION.format(repo_error)) + continue + + # Collect the submodule initialization data from manifest as well as if submodules + # should be processed for the repo. + repo_subs = manifest.get_submodule_init_paths(source.remote_name, combo_name) + repo_subs_enabled = _get_submodule_enable(manifest, source.remote_name, combo_name) + if not repo_subs_enabled: + continue + + # Initialize submodules + if len(repo_subs) > 0: + _init(repo, repo_subs, verbose) + + # Perform sync/update + if len(repo_subs) == 0: + _update(repo, None, verbose) + else: + _update(repo, repo_subs, verbose) + + +if __name__ == '__main__': + def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('manifest_file', metavar='MANIFEST', help=arguments.SUBMOD_MANIFEST_HELP) + parser.add_argument('--combo', default=None, help=arguments.SUBMOD_COMBO_HELP) + parser.add_argument('--new-manifest', default=None, help=arguments.SUBMOD_NEW_MANIFEST_HELP) + parser.add_argument('--new-combo', default=None, help=arguments.SUBMOD_NEW_COMBO_HELP) + parser.add_argument('--deinit', action='store_true', help=arguments.SUBMOD_DEINIT_HELP) + parser.add_argument('--init-full', action='store_true', help=arguments.SUBMOD_INIT_FULL_HELP) + parser.add_argument('--deinit-full', action='store_true', help=arguments.SUBMOD_DEINIT_FULL_HELP) + parser.add_argument('--workspace', default='.', help=arguments.SUBMOD_WORKSPACE_HELP) + parser.add_argument('--verbose', action='store_true', help=arguments.SUBMOD_VERBOSE_HELP) + return parser.parse_args() + + def main(args): + # Optional data init + new_manifest = None + all_new_combos = None + + # Extract basic manifest data + manifest = ManifestXml(args.manifest_file) + init_manifest = manifest + all_combos = manifest.combinations + all_combos.extend(manifest.archived_combinations) + if args.new_manifest is not None: + new_manifest = ManifestXml(args.new_manifest) + all_new_combos = new_manifest.combinations + all_new_combos.extend(new_manifest.archived_combinations) + init_manifest = new_manifest + + # Determine current and new combo information + current_combo = manifest.general_config.current_combo + new_combo = args.new_combo + for combo in all_combos: + if args.combo is not None: + if args.combo.lower() == combo.name.lower(): + current_combo = combo.name + if args.new_combo is not None: + if args.new_combo.lower() == combo.name.lower(): + new_combo = combo.name + init_combo = current_combo + if new_combo is not None: + init_combo = new_combo + + if args.deinit_full: + deinit_full(args.workspace, manifest, args.verbose) + elif args.init_full: + init_full(args.workspace, manifest, args.verbose) + else: + if args.deinit: + deinit_submodules(args.workspace, manifest, current_combo, + new_manifest, new_combo, args.verbose) + init_submodules(args.workspace, init_manifest, init_combo, args.verbose) + + return 0 + + try: + args = parse_args() + sys.exit(main(args)) + except Exception: + if args.verbose: + traceback.print_exc() + sys.exit(1) diff --git a/setup.py b/setup.py index 5d2f55a..a855000 100755 --- a/setup.py +++ b/setup.py @@ -1,27 +1,27 @@ -#!/usr/bin/env python3 -# -## @file setup.py -# -# Copyright (c) 2017 - 2020, Intel Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent -# - -from setuptools import setup - -setup(name='edkrepo', - version='2.0.0', - description='The edkrepo tools', - packages=['edkrepo', 'edkrepo.commands', 'edkrepo.commands.arguments', 'edkrepo.commands.humble', +#!/usr/bin/env python3 +# +## @file setup.py +# +# Copyright (c) 2017 - 2020, Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +# + +from setuptools import setup + +setup(name='edkrepo', + version='2.0.0', + description='The edkrepo tools', + packages=['edkrepo', 'edkrepo.commands', 'edkrepo.commands.arguments', 'edkrepo.commands.humble', 'edkrepo.git_automation', 'edkrepo.common', 'edkrepo.common.workspace_maintenance', 'edkrepo.common.workspace_maintenance.humble', 'edkrepo.config', 'edkrepo.config.humble', - 'edkrepo_manifest_parser', 'project_utils'], - package_data={ - }, - include_package_data=True, - entry_points={ - 'console_scripts': [ - 'edkrepo = edkrepo.edkrepo_entry_point:main', - 'command_completion_edkrepo = edkrepo.command_completion_edkrepo:main' - ] - } - ) + 'edkrepo_manifest_parser', 'project_utils', 'project_utils.arguments'], + package_data={ + }, + include_package_data=True, + entry_points={ + 'console_scripts': [ + 'edkrepo = edkrepo.edkrepo_entry_point:main', + 'command_completion_edkrepo = edkrepo.command_completion_edkrepo:main' + ] + } + ) -- 2.21.0.windows.1