From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mga02.intel.com (mga02.intel.com [134.134.136.20]) by mx.groups.io with SMTP id smtpd.web10.14945.1605136086656442611 for ; Wed, 11 Nov 2020 15:08:06 -0800 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 134.134.136.20, mailfrom: erik.c.bjorge@intel.com) IronPort-SDR: b3w0dCG9D8OtEhfvgwx3XAc7qHOJb84xdabXRJduc/bU349dwowOlk9uPg+5bArWHmA5nR+24/ m5TuxbvzIP0g== X-IronPort-AV: E=McAfee;i="6000,8403,9802"; a="157247468" X-IronPort-AV: E=Sophos;i="5.77,470,1596524400"; d="scan'208";a="157247468" X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga005.jf.intel.com ([10.7.209.41]) by orsmga101.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 11 Nov 2020 15:08:05 -0800 IronPort-SDR: KLaXTiWPfbvIudyW4AYpYoMF2SUyAm9oGGX4DU1XbeNWl4nRcxlghfds3E8tLaMenvFTBslCWG 3yGEiMGQP/Nw== X-IronPort-AV: E=Sophos;i="5.77,470,1596524400"; d="scan'208";a="541992524" Received: from ecbjorge-mobl1.amr.corp.intel.com ([10.212.178.7]) by orsmga005-auth.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 11 Nov 2020 15:08:05 -0800 From: "Bjorge, Erik C" To: devel@edk2.groups.io Cc: Ashley E Desimone , Nate DeSimone , Puja Pandya , Bret Barkelew , Prince Agyeman Subject: [edk2-staging/EdkRepo] [PATCH v1 2/2] EdkRepo: Enable use of repo cache support. Date: Wed, 11 Nov 2020 15:07:29 -0800 Message-Id: X-Mailer: git-send-email 2.21.0.windows.1 In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit This changes enables the local repo cache to be used when cloning and syncing changes. The repo cache applies to submodules as well. Cc: Ashley E Desimone Cc: Nate DeSimone Cc: Puja Pandya Cc: Bret Barkelew Cc: Prince Agyeman Cc: Erik Bjorge Signed-off-by: Erik Bjorge --- edkrepo/commands/checkout_command.py | 3 ++- edkrepo/commands/checkout_pin_command.py | 8 +++++++- edkrepo/commands/clone_command.py | 15 ++++++++++++-- edkrepo/commands/sync_command.py | 12 +++++++++--- edkrepo/common/common_repo_functions.py | 25 ++++++++++++++++++++---- project_utils/submodule.py | 13 ++++++++---- 6 files changed, 61 insertions(+), 15 deletions(-) diff --git a/edkrepo/commands/checkout_command.py b/edkrepo/commands/checkout_command.py index 0169f30..2ce26c0 100644 --- a/edkrepo/commands/checkout_command.py +++ b/edkrepo/commands/checkout_command.py @@ -16,6 +16,7 @@ import os from edkrepo.commands.edkrepo_command import EdkrepoCommand, OverrideArgument import edkrepo.commands.arguments.checkout_args as arguments import edkrepo.commands.humble.checkout_humble as humble +from edkrepo.common.common_cache_functions import get_repo_cache_obj from edkrepo.common.common_repo_functions import checkout, combination_is_in_manifest from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException from edkrepo.config.config_factory import get_workspace_manifest @@ -42,6 +43,6 @@ class CheckoutCommand(EdkrepoCommand): def run_command(self, args, config): if combination_is_in_manifest(args.Combination, get_workspace_manifest()): - checkout(args.Combination, args.verbose, args.override) + checkout(args.Combination, args.verbose, args.override, get_repo_cache_obj(config)) else: raise EdkrepoInvalidParametersException(humble.NO_COMBO.format(args.Combination)) diff --git a/edkrepo/commands/checkout_pin_command.py b/edkrepo/commands/checkout_pin_command.py index 1c58113..0ad1b48 100644 --- a/edkrepo/commands/checkout_pin_command.py +++ b/edkrepo/commands/checkout_pin_command.py @@ -14,6 +14,7 @@ from git import Repo from edkrepo.commands.edkrepo_command import EdkrepoCommand, OverrideArgument, SourceManifestRepoArgument import edkrepo.commands.arguments.checkout_pin_args as arguments import edkrepo.commands.humble.checkout_pin_humble as humble +from edkrepo.common.common_cache_functions import get_repo_cache_obj from edkrepo.common.common_repo_functions import sparse_checkout_enabled, reset_sparse_checkout, sparse_checkout from edkrepo.common.common_repo_functions import check_dirty_repos, checkout_repos, combinations_in_manifest from edkrepo.common.humble import SPARSE_CHECKOUT, SPARSE_RESET, SUBMODULE_DEINIT_FAILED @@ -21,6 +22,7 @@ from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException, from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list_available_manifest_repos from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_source_manifest_repo from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest +from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME from edkrepo_manifest_parser.edk_manifest import ManifestXml from project_utils.submodule import deinit_full, maintain_submodules @@ -82,7 +84,11 @@ class CheckoutPinCommand(EdkrepoCommand): checkout_repos(args.verbose, args.override, pin_repo_sources, workspace_path, manifest) manifest.write_current_combo(humble.PIN_COMBO.format(args.pinfile)) finally: - maintain_submodules(workspace_path, pin, submodule_combo, args.verbose) + cache_path = None + cache_obj = get_repo_cache_obj(config) + if cache_obj is not None: + cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME) + maintain_submodules(workspace_path, pin, submodule_combo, args.verbose, cache_path) if sparse_enabled: print(SPARSE_CHECKOUT) sparse_checkout(workspace_path, pin_repo_sources, manifest) diff --git a/edkrepo/commands/clone_command.py b/edkrepo/commands/clone_command.py index 8769102..56c15c9 100644 --- a/edkrepo/commands/clone_command.py +++ b/edkrepo/commands/clone_command.py @@ -14,6 +14,8 @@ import sys from edkrepo.commands.edkrepo_command import EdkrepoCommand from edkrepo.commands.edkrepo_command import SubmoduleSkipArgument, SourceManifestRepoArgument import edkrepo.commands.arguments.clone_args as arguments +from edkrepo.common.common_cache_functions import get_repo_cache_obj +from edkrepo.common.common_cache_functions import add_missing_cache_repos from edkrepo.common.common_repo_functions import clone_repos, sparse_checkout, verify_single_manifest from edkrepo.common.common_repo_functions import update_editor_config, combinations_in_manifest from edkrepo.common.common_repo_functions import write_included_config, write_conditional_include @@ -28,6 +30,7 @@ from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list from edkrepo.common.workspace_maintenance.humble.manifest_repos_maintenance_humble import PROJ_NOT_IN_REPO, SOURCE_MANIFEST_REPO_NOT_FOUND from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml from project_utils.submodule import maintain_submodules +from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME class CloneCommand(EdkrepoCommand): @@ -151,11 +154,19 @@ class CloneCommand(EdkrepoCommand): # Set up submodule alt url config settings prior to cloning any repos submodule_included_configs = write_included_config(manifest.remotes, manifest.submodule_alternate_remotes, local_manifest_dir) write_conditional_include(workspace_dir, repo_sources_to_clone, submodule_included_configs) - clone_repos(args, workspace_dir, repo_sources_to_clone, project_client_side_hooks, config, manifest) + + # Determine if caching is going to be used and then clone + cache_obj = get_repo_cache_obj(config) + if cache_obj is not None: + add_missing_cache_repos(cache_obj, manifest, args.verbose) + clone_repos(args, workspace_dir, repo_sources_to_clone, project_client_side_hooks, config, manifest, cache_obj) # Init submodules if not args.skip_submodule: - maintain_submodules(workspace_dir, manifest, combo_name, args.verbose) + cache_path = None + if cache_obj is not None: + cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME) + maintain_submodules(workspace_dir, manifest, combo_name, args.verbose, cache_path) # Perform a sparse checkout if requested. use_sparse = args.sparse diff --git a/edkrepo/commands/sync_command.py b/edkrepo/commands/sync_command.py index c4ee330..ff48f50 100644 --- a/edkrepo/commands/sync_command.py +++ b/edkrepo/commands/sync_command.py @@ -32,6 +32,7 @@ from edkrepo.common.humble import MIRROR_BEHIND_PRIMARY_REPO, SYNC_NEEDS_REBASE, from edkrepo.common.humble import SYNC_BRANCH_CHANGE_ON_LOCAL, SYNC_INCOMPATIBLE_COMBO from edkrepo.common.humble import SYNC_REBASE_CALC_FAIL from edkrepo.common.pathfix import get_actual_path, expanduser +from edkrepo.common.common_cache_functions import get_repo_cache_obj from edkrepo.common.common_repo_functions import clone_repos, sparse_checkout_enabled from edkrepo.common.common_repo_functions import reset_sparse_checkout, sparse_checkout, verify_single_manifest from edkrepo.common.common_repo_functions import checkout_repos, check_dirty_repos @@ -47,6 +48,7 @@ from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list from edkrepo.common.ui_functions import init_color_console from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest, get_edkrepo_global_data_directory from edkrepo.config.config_factory import get_workspace_manifest_file +from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml from project_utils.submodule import deinit_submodules, maintain_submodules @@ -102,7 +104,7 @@ class SyncCommand(EdkrepoCommand): if not args.update_local_manifest: self.__check_for_new_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory) check_dirty_repos(initial_manifest, workspace_path) - + # Determine if sparse checkout needs to be disabled for this operation sparse_settings = initial_manifest.sparse_settings sparse_enabled = sparse_checkout_enabled(workspace_path, initial_sources) @@ -116,7 +118,7 @@ class SyncCommand(EdkrepoCommand): reset_sparse_checkout(workspace_path, initial_sources) # Get the latest manifest if requested - if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse + if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse self.__update_local_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory) manifest = get_workspace_manifest() if args.update_local_manifest: @@ -212,7 +214,11 @@ class SyncCommand(EdkrepoCommand): # Initialize submodules if not args.skip_submodule: - maintain_submodules(workspace_path, manifest, current_combo, args.verbose) + cache_path = None + cache_obj = get_repo_cache_obj(config) + if cache_obj is not None: + cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME) + maintain_submodules(workspace_path, manifest, current_combo, args.verbose, cache_path) # Restore sparse checkout state if sparse_enabled: diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index 0d54bbf..0b72715 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -56,6 +56,7 @@ from project_utils.sparse import BuildInfo, process_sparse_checkout from edkrepo.config.config_factory import get_workspace_path from edkrepo.config.config_factory import get_workspace_manifest from edkrepo.config.tool_config import CI_INDEX_FILE_NAME +from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml from edkrepo.common.edkrepo_exception import EdkrepoNotFoundException, EdkrepoGitException, EdkrepoWarningException @@ -75,12 +76,25 @@ CLEAR_LINE = '\x1b[K' DEFAULT_REMOTE_NAME = 'origin' PRIMARY_REMOTE_NAME = 'primary' -def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest): + +def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest, cache_obj=None): for repo_to_clone in repos_to_clone: local_repo_path = os.path.join(workspace_dir, repo_to_clone.root) local_repo_url = repo_to_clone.remote_url + cache_path = None + if cache_obj is not None: + cache_path = cache_obj.get_cache_path(local_repo_url) print("Cloning from: " + str(local_repo_url)) - repo = Repo.clone_from(local_repo_url, local_repo_path, progress=GitProgressHandler(), no_checkout=True) + if cache_path is not None: + print('+ Using cache at {}'.format(cache_path)) + repo = Repo.clone_from(local_repo_url, local_repo_path, + progress=GitProgressHandler(), + reference_if_able=cache_path, + no_checkout=True) + else: + repo = Repo.clone_from(local_repo_url, local_repo_path, + progress=GitProgressHandler(), + no_checkout=True) # Fetch notes repo.remotes.origin.fetch("refs/notes/*:refs/notes/*") @@ -476,7 +490,7 @@ def get_target_sources(combination_or_sha, manifest, workspace_path, log=None): return ManifestXml(pin_filename).get_repo_sources(current_combo) -def checkout(combination_or_sha, verbose=False, override=False, log=None): +def checkout(combination_or_sha, verbose=False, override=False, log=None, cache_obj=None): workspace_path = get_workspace_path() manifest = get_workspace_manifest() @@ -547,7 +561,10 @@ def checkout(combination_or_sha, verbose=False, override=False, log=None): # Return to the initial combo, since there was an issue with cheking out the selected combo checkout_repos(verbose, override, initial_repo_sources, workspace_path, manifest) finally: - maintain_submodules(workspace_path, manifest, submodule_combo, verbose) + cache_path = None + if cache_obj is not None: + cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME) + maintain_submodules(workspace_path, manifest, submodule_combo, verbose, cache_path) if sparse_enabled or sparse_diff: print(SPARSE_CHECKOUT) sparse_checkout(workspace_path, current_repos, manifest) diff --git a/project_utils/submodule.py b/project_utils/submodule.py index 3d1b620..f735125 100644 --- a/project_utils/submodule.py +++ b/project_utils/submodule.py @@ -61,7 +61,7 @@ def _deinit(repo, submodules=None, verbose=False): return -def _update(repo, submodules=None, verbose=False, recursive=False): +def _update(repo, submodules=None, verbose=False, recursive=False, cache_path=None): """ Performs the update of submodules. This includes the sync and update operations. @@ -82,6 +82,8 @@ def _update(repo, submodules=None, verbose=False, recursive=False): cmd = ['git', 'submodule', 'update', '--init'] if recursive: cmd.append('--recursive') + if cache_path is not None: + cmd.extend(['--reference', cache_path]) output_data = repo.git.execute(cmd, with_extended_output=True, with_stdout=True) display_git_output(output_data, verbose) else: @@ -99,6 +101,8 @@ def _update(repo, submodules=None, verbose=False, recursive=False): cmd = ['git', 'submodule', 'update', '--init'] if sub.recursive: cmd.append('--recursive') + if cache_path is not None: + cmd.extend(['--reference', cache_path]) cmd.extend(['--', sub.path]) output_data = repo.git.execute(cmd, with_extended_output=True, with_stdout=True) display_git_output(output_data, verbose) @@ -269,7 +273,7 @@ def deinit_submodules(workspace, start_manifest, start_combo, _deinit(repo, deinit_list, verbose) -def maintain_submodules(workspace, manifest, combo_name, verbose=False): +def maintain_submodules(workspace, manifest, combo_name, verbose=False, cache_path=None): """ Updates the submodules for a specific repo. @@ -277,6 +281,7 @@ def maintain_submodules(workspace, manifest, combo_name, verbose=False): manifest - The manifest parser object for the project. combo_name - The combination name to use for submodule maintenance. verbose - Enable verbose messages. + cache_path - Path to the submodule cache repo. A value of None indicates that no cache repo exists. """ # Process each repo that may have submodules enabled print(strings.SUBMOD_INIT_UPDATE) @@ -303,9 +308,9 @@ def maintain_submodules(workspace, manifest, combo_name, verbose=False): # Perform sync/update if len(repo_subs) == 0: - _update(repo, None, verbose) + _update(repo, None, verbose, cache_path=cache_path) else: - _update(repo, repo_subs, verbose) + _update(repo, repo_subs, verbose, cache_path=cache_path) if __name__ == '__main__': -- 2.21.0.windows.1