From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mga04.intel.com (mga04.intel.com [192.55.52.120]) by mx.groups.io with SMTP id smtpd.web11.16492.1589678985230694148 for ; Sat, 16 May 2020 18:29:45 -0700 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 192.55.52.120, mailfrom: erik.c.bjorge@intel.com) IronPort-SDR: 1ea2Z6FHrH3rZCTnfQmI+yqIHPYiWjhqphTH+fxLqDZBb/c4HBKQuJgcIGKyliFpKkww0bCRyy 4/6fa2elymGA== X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga001.jf.intel.com ([10.7.209.18]) by fmsmga104.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 16 May 2020 18:29:44 -0700 IronPort-SDR: olRFQ7ziAmMaK9MZqwCmMyL0NRRLCMHqJJmjhExpN+NSY3wBmSOKle8bk09rgdgjQ+BLrTrjWJ SagKV0C/NDCw== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.73,401,1583222400"; d="scan'208";a="342454514" Received: from ecbjorge-mobl1.amr.corp.intel.com ([10.209.101.250]) by orsmga001.jf.intel.com with ESMTP; 16 May 2020 18:29:43 -0700 From: "Bjorge, Erik C" To: devel@edk2.groups.io Cc: Ashley E Desimone , Nate DeSimone , Puja Pandya , Bret Barkelew , Prince Agyeman Subject: [edk2-staging/EdkRepo] [PATCH v1 2/2] EdkRepo: Update commands to use new submodule code Date: Sat, 16 May 2020 18:29:22 -0700 Message-Id: <66a3235a3f4de04e0b8a549901f8321f8d1cf950.1589678697.git.erik.c.bjorge@intel.com> X-Mailer: git-send-email 2.21.0.windows.1 In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Replaced use of maintain_submodules to use new common code. This allows for selective submodule initialization via the manifest file. Cc: Ashley E Desimone Cc: Nate DeSimone Cc: Puja Pandya Cc: Bret Barkelew Cc: Prince Agyeman Cc: Erik Bjorge Signed-off-by: Erik Bjorge --- edkrepo/commands/checkout_pin_command.py | 6 ++++ edkrepo/commands/clone_command.py | 20 +++++++---- edkrepo/commands/sync_command.py | 23 +++++++++---- edkrepo/common/common_repo_functions.py | 43 +++++------------------- 4 files changed, 44 insertions(+), 48 deletions(-) diff --git a/edkrepo/commands/checkout_pin_command.py b/edkrepo/commands/checkout_pin_command.py index 4aaf2b5..c250c49 100644 --- a/edkrepo/commands/checkout_pin_command.py +++ b/edkrepo/commands/checkout_pin_command.py @@ -22,6 +22,8 @@ from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_source_manifest_repo from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest from edkrepo_manifest_parser.edk_manifest import ManifestXml +from project_utils.submodule import deinit_submodules, init_submodules + class CheckoutPinCommand(EdkrepoCommand): def __init__(self): @@ -68,11 +70,15 @@ class CheckoutPinCommand(EdkrepoCommand): if sparse_enabled: print(SPARSE_RESET) reset_sparse_checkout(workspace_path, manifest_sources) + submodule_combo = pin.general_config.current_combo + deinit_submodules(workspace_path, manifest, manifest.general_config.current_combo, + pin, submodule_combo, args.verbose) pin_repo_sources = pin.get_repo_sources(pin.general_config.current_combo) try: checkout_repos(args.verbose, args.override, pin_repo_sources, workspace_path, manifest) manifest.write_current_combo(humble.PIN_COMBO.format(args.pinfile)) finally: + init_submodules(workspace_path, pin, submodule_combo, args.verbose) if sparse_enabled: print(SPARSE_CHECKOUT) sparse_checkout(workspace_path, pin_repo_sources, manifest) diff --git a/edkrepo/commands/clone_command.py b/edkrepo/commands/clone_command.py index 179aaf7..653e10b 100644 --- a/edkrepo/commands/clone_command.py +++ b/edkrepo/commands/clone_command.py @@ -24,6 +24,7 @@ from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import pull from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list_available_manifest_repos from edkrepo.common.workspace_maintenance.humble.manifest_repos_maintenance_humble import PROJ_NOT_IN_REPO, SOURCE_MANIFEST_REPO_NOT_FOUND from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml +from project_utils.submodule import init_submodules class CloneCommand(EdkrepoCommand): @@ -104,7 +105,6 @@ class CloneCommand(EdkrepoCommand): manifest = ManifestXml(local_manifest_path) # Process the combination name and make sure it can be found in the manifest - combo_name = None if args.Combination is not None: try: combo_name = case_insensitive_single_match(args.Combination, combinations_in_manifest(manifest)) @@ -118,13 +118,15 @@ class CloneCommand(EdkrepoCommand): # Since pin files are subset of manifest files they do not have a "default combo" it is set to None. In this # case use the current_combo instead. combo_name = manifest.general_config.current_combo + else: + # If a combo was not specified or a pin file used the default combo should be cloned. Also ensure that the + # current combo is updated to match. + combo_name = manifest.general_config.default_combo + manifest.write_current_combo(combo_name) # Get the list of repos to clone and clone them - manifest_config = manifest.general_config - if combo_name: - repo_sources_to_clone = manifest.get_repo_sources(combo_name) - else: - repo_sources_to_clone = manifest.get_repo_sources(manifest_config.default_combo) + repo_sources_to_clone = manifest.get_repo_sources(combo_name) + #check that the repo sources do not contain duplicated local roots local_roots = [r.root for r in repo_sources_to_clone] for root in local_roots: @@ -137,7 +139,11 @@ class CloneCommand(EdkrepoCommand): # Set up submodule alt url config settings prior to cloning any repos submodule_included_configs = write_included_config(manifest.remotes, manifest.submodule_alternate_remotes, local_manifest_dir) write_conditional_include(workspace_dir, repo_sources_to_clone, submodule_included_configs) - clone_repos(args, workspace_dir, repo_sources_to_clone, project_client_side_hooks, config, args.skip_submodule, manifest) + clone_repos(args, workspace_dir, repo_sources_to_clone, project_client_side_hooks, config, manifest) + + # Init submodules + if not args.skip_submodule: + init_submodules(workspace_dir, manifest, combo_name, args.verbose) # Perform a sparse checkout if requested. use_sparse = args.sparse diff --git a/edkrepo/commands/sync_command.py b/edkrepo/commands/sync_command.py index da99397..d3c724f 100644 --- a/edkrepo/commands/sync_command.py +++ b/edkrepo/commands/sync_command.py @@ -38,7 +38,7 @@ from edkrepo.common.common_repo_functions import checkout_repos, check_dirty_rep from edkrepo.common.common_repo_functions import update_editor_config from edkrepo.common.common_repo_functions import update_repo_commit_template, get_latest_sha from edkrepo.common.common_repo_functions import has_primary_repo_remote, fetch_from_primary_repo, in_sync_with_primary -from edkrepo.common.common_repo_functions import update_hooks, maintain_submodules, combinations_in_manifest +from edkrepo.common.common_repo_functions import update_hooks, combinations_in_manifest from edkrepo.common.common_repo_functions import write_included_config, remove_included_config from edkrepo.common.workspace_maintenance.workspace_maintenance import generate_name_for_obsolete_backup from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import pull_workspace_manifest_repo @@ -48,6 +48,7 @@ from edkrepo.common.ui_functions import init_color_console from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest, get_edkrepo_global_data_directory from edkrepo.config.config_factory import get_workspace_manifest_file from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml +from project_utils.submodule import deinit_submodules, init_submodules class SyncCommand(EdkrepoCommand): @@ -84,6 +85,7 @@ class SyncCommand(EdkrepoCommand): current_combo = initial_manifest.general_config.current_combo initial_sources = initial_manifest.get_repo_sources(current_combo) initial_hooks = initial_manifest.repo_hooks + initial_combo = current_combo source_global_manifest_repo = find_source_manifest_repo(initial_manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo) pull_workspace_manifest_repo(initial_manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo, False) @@ -100,6 +102,7 @@ class SyncCommand(EdkrepoCommand): if not args.update_local_manifest: self.__check_for_new_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory) check_dirty_repos(initial_manifest, workspace_path) + # Determine if sparse checkout needs to be disabled for this operation sparse_settings = initial_manifest.sparse_settings sparse_enabled = sparse_checkout_enabled(workspace_path, initial_sources) @@ -113,7 +116,7 @@ class SyncCommand(EdkrepoCommand): reset_sparse_checkout(workspace_path, initial_sources) # Get the latest manifest if requested - if args.update_local_manifest: #NOTE: hyphens in arg name replaced with underscores due to argparse + if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse self.__update_local_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory) manifest = get_workspace_manifest() if args.update_local_manifest: @@ -127,6 +130,12 @@ class SyncCommand(EdkrepoCommand): repo_sources_to_sync = manifest.get_repo_sources(current_combo) manifest.write_current_combo(current_combo) + # At this point both new and old manifest files are ready so we can deinit any + # submodules that are removed due to a manifest update. + if not args.skip_submodule: + deinit_submodules(workspace_path, initial_manifest, initial_combo, + manifest, current_combo, args.verbose) + sync_error = False # Calculate the hooks which need to be updated, added or removed for the sync if args.update_local_manifest: @@ -195,16 +204,16 @@ class SyncCommand(EdkrepoCommand): elif args.verbose: print(NO_SYNC_DETACHED_HEAD.format(repo_to_sync.root)) - if not args.skip_submodule: - if repo_to_sync.enable_submodule: - # Perform submodule updates and url redirection - maintain_submodules(repo_to_sync, repo) # Update commit message templates update_repo_commit_template(workspace_path, repo, repo_to_sync, config, source_global_manifest_repo) if sync_error: print(SYNC_ERROR) + # Initialize submodules + if not args.skip_submodule: + init_submodules(workspace_path, manifest, current_combo, args.verbose) + # Restore sparse checkout state if sparse_enabled: print(SPARSE_CHECKOUT) @@ -326,7 +335,7 @@ class SyncCommand(EdkrepoCommand): print(path_to_source) if len(sources_to_remove) > 0: print(SYNC_REMOVE_LIST_END_FORMATTING) - clone_repos(args, workspace_path, sources_to_clone, new_manifest_to_check.repo_hooks, config, args.skip_submodule, new_manifest_to_check) + clone_repos(args, workspace_path, sources_to_clone, new_manifest_to_check.repo_hooks, config, new_manifest_to_check) # Make a list of and only checkout repos that were newly cloned. Sync keeps repos on their initial active branches # cloning the entire combo can prevent existing repos from correctly being returned to their proper branch repos_to_checkout = [] diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index 61133f1..18e545b 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -68,12 +68,13 @@ from edkrepo_manifest_parser.edk_manifest_validation import validate_manifestrep from edkrepo_manifest_parser.edk_manifest_validation import get_manifest_validation_status from edkrepo_manifest_parser.edk_manifest_validation import print_manifest_errors from edkrepo_manifest_parser.edk_manifest_validation import validate_manifestfiles +from project_utils.submodule import deinit_submodules, init_submodules CLEAR_LINE = '\x1b[K' DEFAULT_REMOTE_NAME = 'origin' PRIMARY_REMOTE_NAME = 'primary' -def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, skip_submodule, manifest): +def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest): for repo_to_clone in repos_to_clone: local_repo_path = os.path.join(workspace_dir, repo_to_clone.root) local_repo_url = repo_to_clone.remote_url @@ -108,10 +109,6 @@ def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, else: raise EdkrepoManifestInvalidException(MISSING_BRANCH_COMMIT) - if not skip_submodule: - if repo_to_clone.enable_submodule: - maintain_submodules(repo_to_clone, repo, args.verbose) - try: if 'source_manifest_repo' in vars(args).keys(): src_manifest_repo = find_source_manifest_repo(manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo) @@ -184,32 +181,6 @@ def write_conditional_include(workspace_path, repo_sources, included_configs): gitglobalconfig.add_section(section) gitglobalconfig.set(section, 'path', path) -def maintain_submodules(repo_sources, repo, verbose = False): - try: - output_data = repo.git.execute(['git', 'submodule', 'init'], with_extended_output=True, with_stdout=True) - if verbose and output_data[0]: - print(output_data[0]) - if output_data[1]: - print(output_data[1]) - if verbose and output_data[2]: - print(output_data[2]) - output_data = repo.git.execute(['git', 'submodule', 'sync', '--recursive'], with_extended_output=True, with_stdout=True) - if verbose and output_data[0]: - print(output_data[0]) - if output_data[1]: - print(output_data[1]) - if verbose and output_data[2]: - print(output_data[2]) - output_data = repo.git.execute(['git', 'submodule', 'update', '--recursive'], with_extended_output=True, with_stdout=True) - if verbose and output_data[0]: - print(output_data[0]) - if output_data[1]: - print(output_data[1]) - if verbose and output_data[2]: - print(output_data[2]) - except: - raise EdkrepoGitException(SUBMODULE_FAILURE.format(repo_sources.remote_name)) - def install_hooks(hooks, local_repo_path, repo_for_install, config, global_manifest_directory): # Determine the which hooks are for the repo in question and which are from a URL based source or are in a global # manifest repo relative path @@ -399,9 +370,6 @@ def checkout_repos(verbose, override, repos_to_checkout, workspace_path, manifes else: raise EdkrepoManifestInvalidException(MISSING_BRANCH_COMMIT) - if repo_to_checkout.enable_submodule: - maintain_submodules(repo_to_checkout, repo, verbose) - def validate_manifest_repo(manifest_repo, verbose=False, archived=False): print(VERIFY_GLOBAL) if archived: @@ -514,9 +482,11 @@ def checkout(combination_or_sha, verbose=False, override=False, log=None): # Create combo_or_sha so we have original input and do not introduce any # unintended behavior by messing with parameters. combo_or_sha = combination_or_sha + submodule_combo = manifest.general_config.current_combo try: # Try to handle normalize combo name to match the manifest file. combo_or_sha = case_insensitive_single_match(combo_or_sha, combinations_in_manifest(manifest)) + submodule_combo = combo_or_sha except: # No match so leave it alone. It must be a SHA1 or a bad combo name. pass @@ -528,6 +498,10 @@ def checkout(combination_or_sha, verbose=False, override=False, log=None): log=log) initial_repo_sources = manifest.get_repo_sources(manifest.general_config.current_combo) + # Deinit any submodules that have been removed. + deinit_submodules(workspace_path, manifest, manifest.general_config.current_combo, + manifest, submodule_combo, verbose) + # Disable sparse checkout current_repos = initial_repo_sources sparse_enabled = sparse_checkout_enabled(workspace_path, initial_repo_sources) @@ -566,6 +540,7 @@ def checkout(combination_or_sha, verbose=False, override=False, log=None): # Return to the initial combo, since there was an issue with cheking out the selected combo checkout_repos(verbose, override, initial_repo_sources, workspace_path, manifest) finally: + init_submodules(workspace_path, manifest, submodule_combo, verbose) if sparse_enabled or sparse_diff: print(SPARSE_CHECKOUT) sparse_checkout(workspace_path, current_repos, manifest) -- 2.21.0.windows.1