* [edk2-staging/EdkRepo] [PATCH v1 1/2] EdkRepo: Add cache command
2020-11-11 23:07 [edk2-staging/EdkRepo] [PATCH v1 0/2] Adding local repo cache support Bjorge, Erik C
@ 2020-11-11 23:07 ` Bjorge, Erik C
2020-11-11 23:07 ` [edk2-staging/EdkRepo] [PATCH v1 2/2] EdkRepo: Enable use of repo cache support Bjorge, Erik C
2020-11-15 3:46 ` [edk2-staging/EdkRepo] [PATCH v1 0/2] Adding local " Nate DeSimone
2 siblings, 0 replies; 5+ messages in thread
From: Bjorge, Erik C @ 2020-11-11 23:07 UTC (permalink / raw)
To: devel
Cc: Ashley E Desimone, Nate DeSimone, Puja Pandya, Bret Barkelew,
Prince Agyeman
Adds a module to add a repo cache and mange it. Also adds a command
to manage the repo cache from EdkRepo. No other commands use the
functionality at this point.
Cc: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Prince Agyeman <prince.agyeman@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
Signed-off-by: Erik Bjorge <erik.c.bjorge@intel.com>
---
edkrepo/commands/arguments/cache_args.py | 19 ++
edkrepo/commands/cache_command.py | 118 ++++++++++++
edkrepo/commands/humble/cache_humble.py | 17 ++
edkrepo/common/common_cache_functions.py | 41 +++++
edkrepo/common/edkrepo_exception.py | 3 +
edkrepo/config/config_factory.py | 14 +-
edkrepo/config/tool_config.py | 5 +-
project_utils/cache.py | 224 +++++++++++++++++++++++
project_utils/project_utils_strings.py | 11 ++
9 files changed, 448 insertions(+), 4 deletions(-)
create mode 100644 edkrepo/commands/arguments/cache_args.py
create mode 100644 edkrepo/commands/cache_command.py
create mode 100644 edkrepo/commands/humble/cache_humble.py
create mode 100644 edkrepo/common/common_cache_functions.py
create mode 100644 project_utils/cache.py
diff --git a/edkrepo/commands/arguments/cache_args.py b/edkrepo/commands/arguments/cache_args.py
new file mode 100644
index 0000000..0080536
--- /dev/null
+++ b/edkrepo/commands/arguments/cache_args.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+#
+## @file
+# cache_args.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+
+''' Contains the help and description strings for arguments in the
+cache command meta data.
+'''
+COMMAND_DESCRIPTION = ('Manages local caching support for project repos. The goal of this feature '
+ 'is to improve clone performance')
+COMMAND_ENABLE_HELP = 'Enables caching support on the system.'
+COMMAND_DISABLE_HELP = 'Disables caching support on the system.'
+COMMAND_UPDATE_HELP = 'Update the repo cache for all cached projects.'
+COMMAND_INFO_HELP = 'Display the current cache information.'
+COMMAND_PROJECT_HELP = 'Project to add to the cache.'
diff --git a/edkrepo/commands/cache_command.py b/edkrepo/commands/cache_command.py
new file mode 100644
index 0000000..9f0d6e9
--- /dev/null
+++ b/edkrepo/commands/cache_command.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+#
+## @file
+# cache_command.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+
+import edkrepo.commands.arguments.cache_args as arguments
+from edkrepo.commands.edkrepo_command import EdkrepoCommand
+from edkrepo.commands.edkrepo_command import SourceManifestRepoArgument
+from edkrepo.commands.humble.cache_humble import CACHE_ENABLED, CACHE_FETCH, CACHE_INFO
+from edkrepo.commands.humble.cache_humble import CACHE_INFO_LINE, PROJECT_NOT_FOUND, NO_INSTANCE
+from edkrepo.commands.humble.cache_humble import UNABLE_TO_LOAD_MANIFEST, UNABLE_TO_PARSE_MANIFEST
+from edkrepo.common.common_cache_functions import add_missing_cache_repos
+from edkrepo.common.common_cache_functions import get_repo_cache_obj
+from edkrepo.common.edkrepo_exception import EdkrepoCacheException
+from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_project_in_all_indices
+from edkrepo.config.config_factory import get_workspace_manifest
+from edkrepo_manifest_parser.edk_manifest import ManifestXml
+
+
+class CacheCommand(EdkrepoCommand):
+ def __init__(self):
+ super().__init__()
+
+ def get_metadata(self):
+ metadata = {}
+ metadata['name'] = 'cache'
+ metadata['help-text'] = arguments.COMMAND_DESCRIPTION
+ args = []
+ metadata['arguments'] = args
+ args.append({'name': 'enable',
+ 'positional': False,
+ 'required': False,
+ 'help-text': arguments.COMMAND_ENABLE_HELP})
+ args.append({'name': 'disable',
+ 'positional': False,
+ 'required': False,
+ 'help-text': arguments.COMMAND_DISABLE_HELP})
+ args.append({'name': 'update',
+ 'positional': False,
+ 'required': False,
+ 'help-text': arguments.COMMAND_UPDATE_HELP})
+ args.append({'name': 'info',
+ 'positional': False,
+ 'required': False,
+ 'help-text': arguments.COMMAND_INFO_HELP})
+ args.append({'name': 'project',
+ 'positional': True,
+ 'required': False,
+ 'help-text': arguments.COMMAND_PROJECT_HELP})
+ args.append(SourceManifestRepoArgument)
+ return metadata
+
+ def run_command(self, args, config):
+ # Process enable disable requests
+ if args.disable:
+ config['user_cfg_file'].set_caching_state(False)
+ elif args.enable:
+ config['user_cfg_file'].set_caching_state(True)
+
+ # Get the current state now that we have processed enable/disable
+ cache_state = config['user_cfg_file'].caching_state
+ print(CACHE_ENABLED.format(cache_state))
+ if not cache_state:
+ return
+
+ # State is enabled so make sure cache directory exists
+ cache_obj = get_repo_cache_obj(config)
+
+ # Check to see if a manifest was provided and add any missing remotes
+ manifest = None
+ if args.project is not None:
+ manifest = _get_manifest(args.project, config, args.source_manifest_repo)
+ else:
+ try:
+ manifest = get_workspace_manifest()
+ except Exception:
+ pass
+
+ # If manifest is provided attempt to add any remotes that do not exist
+ if manifest is not None:
+ add_missing_cache_repos(cache_obj, manifest, True)
+
+ # Display all the cache information
+ if args.info:
+ print(CACHE_INFO)
+ info = cache_obj.get_cache_info(args.verbose)
+ for item in info:
+ print(CACHE_INFO_LINE.format(item.path, item.remote, item.url))
+
+ # Do an update if requested
+ if args.update:
+ print(CACHE_FETCH)
+ cache_obj.update_cache(verbose=True)
+
+ # Close the cache repos
+ cache_obj.close(args.verbose)
+
+
+def _get_manifest(project, config, source_manifest_repo=None):
+ try:
+ manifest_repo, source_cfg, manifest_path = find_project_in_all_indices(
+ project,
+ config['cfg_file'],
+ config['user_cfg_file'],
+ PROJECT_NOT_FOUND.format(project),
+ NO_INSTANCE.format(project),
+ source_manifest_repo)
+ except Exception:
+ raise EdkrepoCacheException(UNABLE_TO_LOAD_MANIFEST)
+ try:
+ manifest = ManifestXml(manifest_path)
+ except Exception:
+ raise EdkrepoCacheException(UNABLE_TO_PARSE_MANIFEST)
+ return manifest
diff --git a/edkrepo/commands/humble/cache_humble.py b/edkrepo/commands/humble/cache_humble.py
new file mode 100644
index 0000000..4f318ac
--- /dev/null
+++ b/edkrepo/commands/humble/cache_humble.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+#
+## @file
+# cache_humble.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+
+CACHE_ENABLED = 'Caching Enabled: {}'
+CACHE_INFO = 'Cache Information:'
+CACHE_INFO_LINE = '+ {}\n {} ({})'
+CACHE_FETCH = 'Fetching all remotes... (this could take a while)'
+PROJECT_NOT_FOUND = 'Project {} does not exist'
+NO_INSTANCE = 'Unable to determine instance to use for {}'
+UNABLE_TO_LOAD_MANIFEST = 'Unable to load manifest file.'
+UNABLE_TO_PARSE_MANIFEST = 'Failed to parse manifest file.'
diff --git a/edkrepo/common/common_cache_functions.py b/edkrepo/common/common_cache_functions.py
new file mode 100644
index 0000000..84bd3ed
--- /dev/null
+++ b/edkrepo/common/common_cache_functions.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+#
+## @file
+# common_cache_functions.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+
+import os
+
+from edkrepo.config.config_factory import get_edkrepo_global_data_directory
+from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
+from project_utils.cache import RepoCache
+
+
+def get_global_cache_directory(config):
+ if config['user_cfg_file'].caching_state:
+ return os.path.join(get_edkrepo_global_data_directory(), '.cache')
+ return None
+
+
+def get_repo_cache_obj(config):
+ cache_obj = None
+ cache_directory = get_global_cache_directory(config)
+ if cache_directory is not None:
+ cache_obj = RepoCache(cache_directory)
+ cache_obj.open()
+ return cache_obj
+
+
+def add_missing_cache_repos(cache_obj, manifest, verbose=False):
+ print('Adding and fetching new remotes... (this could take a while)')
+ for remote in manifest.remotes:
+ cache_obj.add_repo(url=remote.url, verbose=verbose)
+ alt_submodules = manifest.submodule_alternate_remotes
+ if alt_submodules:
+ print('Adding and fetching new submodule remotes... (this could also take a while)')
+ cache_obj.add_repo(name=SUBMODULE_CACHE_REPO_NAME, verbose=verbose)
+ for alt in alt_submodules:
+ cache_obj.add_remote(alt.alternate_url, SUBMODULE_CACHE_REPO_NAME, verbose)
diff --git a/edkrepo/common/edkrepo_exception.py b/edkrepo/common/edkrepo_exception.py
index a56e709..b3f2300 100644
--- a/edkrepo/common/edkrepo_exception.py
+++ b/edkrepo/common/edkrepo_exception.py
@@ -98,3 +98,6 @@ class EdkrepoGitConfigSetupException(EdkrepoException):
def __init__(self, message):
super().__init__(message, 131)
+class EdkrepoCacheException(EdkrepoException):
+ def __init__(self, message):
+ super().__init__(message, 132)
diff --git a/edkrepo/config/config_factory.py b/edkrepo/config/config_factory.py
index fe69460..3680c0b 100644
--- a/edkrepo/config/config_factory.py
+++ b/edkrepo/config/config_factory.py
@@ -225,10 +225,20 @@ class GlobalUserConfig(BaseConfig):
self.filename = os.path.join(get_edkrepo_global_data_directory(), "edkrepo_user.cfg")
self.prop_list = [
CfgProp('scm', 'mirror_geo', 'geo', 'none', False),
- CfgProp('send-review', 'max-patch-set', 'max_patch_set', '10', False)
- ]
+ CfgProp('send-review', 'max-patch-set', 'max_patch_set', '10', False),
+ CfgProp('caching', 'enable-caching', 'enable_caching_text', 'false', False)]
super().__init__(self.filename, get_edkrepo_global_data_directory(), False)
+ @property
+ def caching_state(self):
+ return self.enable_caching_text.lower() == 'true'
+
+ def set_caching_state(self, enable):
+ if enable:
+ self.enable_caching_text = 'true'
+ else:
+ self.enable_caching_text = 'false'
+
@property
def max_patch_set_int(self):
try:
diff --git a/edkrepo/config/tool_config.py b/edkrepo/config/tool_config.py
index eee1326..81f4ddf 100644
--- a/edkrepo/config/tool_config.py
+++ b/edkrepo/config/tool_config.py
@@ -1,10 +1,11 @@
#!/usr/bin/env python3
#
## @file
-# tool)config.py
+# tool_config.py
#
# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
-CI_INDEX_FILE_NAME = 'CiIndex.xml'
\ No newline at end of file
+CI_INDEX_FILE_NAME = 'CiIndex.xml'
+SUBMODULE_CACHE_REPO_NAME = 'submodule-cache'
diff --git a/project_utils/cache.py b/project_utils/cache.py
new file mode 100644
index 0000000..8efd411
--- /dev/null
+++ b/project_utils/cache.py
@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+#
+## @file
+# cache.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+from collections import namedtuple
+import os
+import shutil
+
+from git import Repo
+
+from edkrepo.common.progress_handler import GitProgressHandler
+from project_utils.project_utils_strings import CACHE_ADD_REMOTE, CACHE_ADDING_REPO, CACHE_CHECK_ROOT_DIR
+from project_utils.project_utils_strings import CACHE_FAILED_TO_CLOSE, CACHE_FAILED_TO_OPEN, CACHE_FETCH_REMOTE
+from project_utils.project_utils_strings import CACHE_REMOTE_EXISTS, CACHE_REMOVE_REPO, CACHE_REPO_EXISTS
+
+CacheInfo = namedtuple('CacheInfo', ['path', 'remote', 'url'])
+
+
+class RepoCache(object):
+ """
+ Provides basic management of a cache repo.
+ """
+ def __init__(self, path):
+ self._cache_root_path = path
+ self._repos = {}
+
+ def _create_name(self, url_or_name):
+ """
+ Used to create consistent repo and remote names
+ """
+ dir_name = url_or_name.split('/')[-1]
+ if not dir_name.endswith('.git'):
+ dir_name += '.git'
+ return dir_name
+
+ def _get_repo_path(self, dir_name):
+ return os.path.join(self._cache_root_path, dir_name)
+
+ def _get_repo(self, dir_name):
+ """
+ Returns the git repo object for the cache repo.
+
+ Raises FileNotFoundError if the cache directory does not exist.
+ Raises IOError if the repo cannot be opened
+ """
+ repo_path = self._get_repo_path(dir_name)
+ if not os.path.isdir(repo_path):
+ raise FileNotFoundError
+ try:
+ repo = Repo(repo_path)
+ except Exception:
+ raise IOError
+ return repo
+
+ def _get_cache_dirs(self):
+ if not os.path.isdir(self._cache_root_path):
+ raise FileNotFoundError
+ return [x for x in os.listdir(self._cache_root_path) if os.path.isdir(self._get_repo_path(x))]
+
+ def _add_and_fetch_remote(self, repo, remote_name, url, verbose=False):
+ if verbose:
+ print(CACHE_ADD_REMOTE.format(remote_name, url))
+ repo.create_remote(remote_name, url)
+ if verbose:
+ print(CACHE_FETCH_REMOTE.format(remote_name, url))
+ repo.remotes[remote_name].fetch(progress=GitProgressHandler())
+
+ def open(self, verbose=False):
+ """
+ Opens all cache repos.
+
+ Raises FileNotFoundError if the cache directory does not exist.
+ """
+ if not self._repos:
+ if not os.path.isdir(self._cache_root_path):
+ if verbose:
+ print(CACHE_CHECK_ROOT_DIR.format(self._cache_root_path))
+ os.makedirs(self._cache_root_path)
+
+ for dir_name in self._get_cache_dirs():
+ try:
+ self._repos[dir_name] = self._get_repo(dir_name)
+ except Exception:
+ if verbose:
+ print(CACHE_FAILED_TO_OPEN.format(dir_name))
+
+ def close(self, verbose=False):
+ """
+ Closes all cache repos.
+ """
+ for dir_name in self._repos:
+ try:
+ self._repos[dir_name].close()
+ except Exception:
+ if verbose:
+ print(CACHE_FAILED_TO_CLOSE.format(dir_name))
+ self._repos = {}
+
+ def get_cache_path(self, url_or_name):
+ dir_name = self._create_name(url_or_name)
+ if dir_name not in self._repos:
+ return None
+ return self._get_repo_path(dir_name)
+
+ def get_cache_info(self, verbose=False):
+ """
+ Returns a list of remotes currently configured in the cache.
+
+ Raises FileNotFoundError if the cache repo is not open.
+ """
+ ret_val = []
+ for dir_name in self._repos:
+ for remote in self._repos[dir_name].remotes:
+ ret_val.append(CacheInfo(self._get_repo_path(dir_name), remote.name, remote.url))
+ return ret_val
+
+ def delete_cache_root(self, verbose=False):
+ """
+ Deletes the cache root directory and all caches.
+ """
+ if os.path.isdir(self._cache_root_path):
+ if self._repos:
+ self.close()
+ shutil.rmtree(self._cache_root_path, ignore_errors=True)
+
+ def add_repo(self, url=None, name=None, verbose=False):
+ """
+ Adds a repo to the cache if it does not already exist.
+
+ """
+ remote_name = None
+ if url is None and name is None:
+ raise ValueError
+ elif name is not None:
+ dir_name = self._create_name(name)
+ else:
+ dir_name = self._create_name(url)
+ if url is not None:
+ remote_name = self._create_name(url)
+ repo_path = self._get_repo_path(dir_name)
+
+ if dir_name in self._repos:
+ if verbose:
+ print(CACHE_REPO_EXISTS.format(dir_name))
+ else:
+ if verbose:
+ print(CACHE_ADDING_REPO.format(dir_name))
+ os.makedirs(repo_path)
+ self._repos[dir_name] = Repo.init(repo_path, bare=True)
+
+ if remote_name is not None and remote_name not in self._repos[dir_name].remotes:
+ self._add_and_fetch_remote(self._get_repo(dir_name), remote_name, url)
+ return dir_name
+
+ def remove_repo(self, url=None, name=None, verbose=False):
+ """
+ Removes a remote from the cache repo if it exists
+
+ Raises FileNotFoundError if the cache repo is not open.
+ """
+ if url is None and name is None:
+ raise ValueError
+ elif name is not None:
+ dir_name = self._create_name(name)
+ else:
+ dir_name = self._create_name(url)
+ if dir_name not in self._repos:
+ return
+ if verbose:
+ print(CACHE_REMOVE_REPO.format(dir_name))
+ self._repos.pop(dir_name).close()
+ shutil.rmtree(os.path.join(self._cache_root_path, dir_name), ignore_errors=True)
+
+ def add_remote(self, url, name, verbose=False):
+ remote_name = self._create_name(url)
+ dir_name = self._create_name(name)
+ if dir_name not in self._repos:
+ raise ValueError
+ repo = self._get_repo(dir_name)
+ if remote_name in repo.remotes:
+ if verbose:
+ print(CACHE_REMOTE_EXISTS.format(remote_name))
+ return
+ self._add_and_fetch_remote(repo, remote_name, url, verbose)
+
+ def remove_remote(self, url, name, verbose=False):
+ remote_name = self._create_name(url)
+ dir_name = self._create_name(name)
+ if dir_name not in self._repos:
+ raise ValueError
+ repo = self._get_repo(dir_name)
+ if remote_name not in repo.remotes:
+ raise IndexError
+ repo.remove_remote(repo.remotes[remote_name])
+
+ def update_cache(self, url_or_name=None, verbose=False):
+ if not self._repos:
+ raise FileNotFoundError
+ repo_dirs = self._repos.keys()
+
+ if url_or_name is not None:
+ dir_name = self._create_name(url_or_name)
+ if dir_name in self._repos:
+ repo_dirs = [dir_name]
+ else:
+ return
+
+ for dir_name in repo_dirs:
+ try:
+ repo = self._get_repo(dir_name)
+ except Exception:
+ print(CACHE_FAILED_TO_OPEN.format(dir_name))
+ continue
+ for remote in repo.remotes:
+ if verbose:
+ print(CACHE_FETCH_REMOTE.format(dir_name, remote.url))
+ remote.fetch(progress=GitProgressHandler())
+
+ def clean_cache(self, verbose=False):
+ raise NotImplementedError
diff --git a/project_utils/project_utils_strings.py b/project_utils/project_utils_strings.py
index 33c22d2..1547978 100644
--- a/project_utils/project_utils_strings.py
+++ b/project_utils/project_utils_strings.py
@@ -22,3 +22,14 @@ SUBMOD_DEINIT_PATH = 'Submodule deinit: {}'
SUBMOD_SYNC_PATH = 'Submodule sync: {}'
SUBMOD_UPDATE_PATH = 'Submodule update: {}'
SUBMOD_EXCEPTION = '- Exception: {}'
+
+# Caching support strings
+CACHE_ADD_REMOTE = '+ Adding remote {} ({})'
+CACHE_FETCH_REMOTE = '+ Fetching data for {} ({})'
+CACHE_CHECK_ROOT_DIR = '+ Creating cache root directory: {}'
+CACHE_FAILED_TO_OPEN = '- Failed to open cache: {}'
+CACHE_FAILED_TO_CLOSE = '- Failed to close cache: {}'
+CACHE_REPO_EXISTS = '- Repo {} already exists.'
+CACHE_ADDING_REPO = '+ Adding cache repo {}'
+CACHE_REMOVE_REPO = '- Removing cache repo: {}'
+CACHE_REMOTE_EXISTS = '- Remote {} already exists.'
--
2.21.0.windows.1
^ permalink raw reply related [flat|nested] 5+ messages in thread* [edk2-staging/EdkRepo] [PATCH v1 2/2] EdkRepo: Enable use of repo cache support.
2020-11-11 23:07 [edk2-staging/EdkRepo] [PATCH v1 0/2] Adding local repo cache support Bjorge, Erik C
2020-11-11 23:07 ` [edk2-staging/EdkRepo] [PATCH v1 1/2] EdkRepo: Add cache command Bjorge, Erik C
@ 2020-11-11 23:07 ` Bjorge, Erik C
2020-11-15 3:46 ` [edk2-staging/EdkRepo] [PATCH v1 0/2] Adding local " Nate DeSimone
2 siblings, 0 replies; 5+ messages in thread
From: Bjorge, Erik C @ 2020-11-11 23:07 UTC (permalink / raw)
To: devel
Cc: Ashley E Desimone, Nate DeSimone, Puja Pandya, Bret Barkelew,
Prince Agyeman
This changes enables the local repo cache to be used when cloning
and syncing changes. The repo cache applies to submodules as well.
Cc: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Prince Agyeman <prince.agyeman@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
Signed-off-by: Erik Bjorge <erik.c.bjorge@intel.com>
---
edkrepo/commands/checkout_command.py | 3 ++-
edkrepo/commands/checkout_pin_command.py | 8 +++++++-
edkrepo/commands/clone_command.py | 15 ++++++++++++--
edkrepo/commands/sync_command.py | 12 +++++++++---
edkrepo/common/common_repo_functions.py | 25 ++++++++++++++++++++----
project_utils/submodule.py | 13 ++++++++----
6 files changed, 61 insertions(+), 15 deletions(-)
diff --git a/edkrepo/commands/checkout_command.py b/edkrepo/commands/checkout_command.py
index 0169f30..2ce26c0 100644
--- a/edkrepo/commands/checkout_command.py
+++ b/edkrepo/commands/checkout_command.py
@@ -16,6 +16,7 @@ import os
from edkrepo.commands.edkrepo_command import EdkrepoCommand, OverrideArgument
import edkrepo.commands.arguments.checkout_args as arguments
import edkrepo.commands.humble.checkout_humble as humble
+from edkrepo.common.common_cache_functions import get_repo_cache_obj
from edkrepo.common.common_repo_functions import checkout, combination_is_in_manifest
from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException
from edkrepo.config.config_factory import get_workspace_manifest
@@ -42,6 +43,6 @@ class CheckoutCommand(EdkrepoCommand):
def run_command(self, args, config):
if combination_is_in_manifest(args.Combination, get_workspace_manifest()):
- checkout(args.Combination, args.verbose, args.override)
+ checkout(args.Combination, args.verbose, args.override, get_repo_cache_obj(config))
else:
raise EdkrepoInvalidParametersException(humble.NO_COMBO.format(args.Combination))
diff --git a/edkrepo/commands/checkout_pin_command.py b/edkrepo/commands/checkout_pin_command.py
index 1c58113..0ad1b48 100644
--- a/edkrepo/commands/checkout_pin_command.py
+++ b/edkrepo/commands/checkout_pin_command.py
@@ -14,6 +14,7 @@ from git import Repo
from edkrepo.commands.edkrepo_command import EdkrepoCommand, OverrideArgument, SourceManifestRepoArgument
import edkrepo.commands.arguments.checkout_pin_args as arguments
import edkrepo.commands.humble.checkout_pin_humble as humble
+from edkrepo.common.common_cache_functions import get_repo_cache_obj
from edkrepo.common.common_repo_functions import sparse_checkout_enabled, reset_sparse_checkout, sparse_checkout
from edkrepo.common.common_repo_functions import check_dirty_repos, checkout_repos, combinations_in_manifest
from edkrepo.common.humble import SPARSE_CHECKOUT, SPARSE_RESET, SUBMODULE_DEINIT_FAILED
@@ -21,6 +22,7 @@ from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException,
from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list_available_manifest_repos
from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_source_manifest_repo
from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest
+from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
from edkrepo_manifest_parser.edk_manifest import ManifestXml
from project_utils.submodule import deinit_full, maintain_submodules
@@ -82,7 +84,11 @@ class CheckoutPinCommand(EdkrepoCommand):
checkout_repos(args.verbose, args.override, pin_repo_sources, workspace_path, manifest)
manifest.write_current_combo(humble.PIN_COMBO.format(args.pinfile))
finally:
- maintain_submodules(workspace_path, pin, submodule_combo, args.verbose)
+ cache_path = None
+ cache_obj = get_repo_cache_obj(config)
+ if cache_obj is not None:
+ cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME)
+ maintain_submodules(workspace_path, pin, submodule_combo, args.verbose, cache_path)
if sparse_enabled:
print(SPARSE_CHECKOUT)
sparse_checkout(workspace_path, pin_repo_sources, manifest)
diff --git a/edkrepo/commands/clone_command.py b/edkrepo/commands/clone_command.py
index 8769102..56c15c9 100644
--- a/edkrepo/commands/clone_command.py
+++ b/edkrepo/commands/clone_command.py
@@ -14,6 +14,8 @@ import sys
from edkrepo.commands.edkrepo_command import EdkrepoCommand
from edkrepo.commands.edkrepo_command import SubmoduleSkipArgument, SourceManifestRepoArgument
import edkrepo.commands.arguments.clone_args as arguments
+from edkrepo.common.common_cache_functions import get_repo_cache_obj
+from edkrepo.common.common_cache_functions import add_missing_cache_repos
from edkrepo.common.common_repo_functions import clone_repos, sparse_checkout, verify_single_manifest
from edkrepo.common.common_repo_functions import update_editor_config, combinations_in_manifest
from edkrepo.common.common_repo_functions import write_included_config, write_conditional_include
@@ -28,6 +30,7 @@ from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list
from edkrepo.common.workspace_maintenance.humble.manifest_repos_maintenance_humble import PROJ_NOT_IN_REPO, SOURCE_MANIFEST_REPO_NOT_FOUND
from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml
from project_utils.submodule import maintain_submodules
+from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
class CloneCommand(EdkrepoCommand):
@@ -151,11 +154,19 @@ class CloneCommand(EdkrepoCommand):
# Set up submodule alt url config settings prior to cloning any repos
submodule_included_configs = write_included_config(manifest.remotes, manifest.submodule_alternate_remotes, local_manifest_dir)
write_conditional_include(workspace_dir, repo_sources_to_clone, submodule_included_configs)
- clone_repos(args, workspace_dir, repo_sources_to_clone, project_client_side_hooks, config, manifest)
+
+ # Determine if caching is going to be used and then clone
+ cache_obj = get_repo_cache_obj(config)
+ if cache_obj is not None:
+ add_missing_cache_repos(cache_obj, manifest, args.verbose)
+ clone_repos(args, workspace_dir, repo_sources_to_clone, project_client_side_hooks, config, manifest, cache_obj)
# Init submodules
if not args.skip_submodule:
- maintain_submodules(workspace_dir, manifest, combo_name, args.verbose)
+ cache_path = None
+ if cache_obj is not None:
+ cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME)
+ maintain_submodules(workspace_dir, manifest, combo_name, args.verbose, cache_path)
# Perform a sparse checkout if requested.
use_sparse = args.sparse
diff --git a/edkrepo/commands/sync_command.py b/edkrepo/commands/sync_command.py
index c4ee330..ff48f50 100644
--- a/edkrepo/commands/sync_command.py
+++ b/edkrepo/commands/sync_command.py
@@ -32,6 +32,7 @@ from edkrepo.common.humble import MIRROR_BEHIND_PRIMARY_REPO, SYNC_NEEDS_REBASE,
from edkrepo.common.humble import SYNC_BRANCH_CHANGE_ON_LOCAL, SYNC_INCOMPATIBLE_COMBO
from edkrepo.common.humble import SYNC_REBASE_CALC_FAIL
from edkrepo.common.pathfix import get_actual_path, expanduser
+from edkrepo.common.common_cache_functions import get_repo_cache_obj
from edkrepo.common.common_repo_functions import clone_repos, sparse_checkout_enabled
from edkrepo.common.common_repo_functions import reset_sparse_checkout, sparse_checkout, verify_single_manifest
from edkrepo.common.common_repo_functions import checkout_repos, check_dirty_repos
@@ -47,6 +48,7 @@ from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list
from edkrepo.common.ui_functions import init_color_console
from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest, get_edkrepo_global_data_directory
from edkrepo.config.config_factory import get_workspace_manifest_file
+from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml
from project_utils.submodule import deinit_submodules, maintain_submodules
@@ -102,7 +104,7 @@ class SyncCommand(EdkrepoCommand):
if not args.update_local_manifest:
self.__check_for_new_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory)
check_dirty_repos(initial_manifest, workspace_path)
-
+
# Determine if sparse checkout needs to be disabled for this operation
sparse_settings = initial_manifest.sparse_settings
sparse_enabled = sparse_checkout_enabled(workspace_path, initial_sources)
@@ -116,7 +118,7 @@ class SyncCommand(EdkrepoCommand):
reset_sparse_checkout(workspace_path, initial_sources)
# Get the latest manifest if requested
- if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse
+ if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse
self.__update_local_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory)
manifest = get_workspace_manifest()
if args.update_local_manifest:
@@ -212,7 +214,11 @@ class SyncCommand(EdkrepoCommand):
# Initialize submodules
if not args.skip_submodule:
- maintain_submodules(workspace_path, manifest, current_combo, args.verbose)
+ cache_path = None
+ cache_obj = get_repo_cache_obj(config)
+ if cache_obj is not None:
+ cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME)
+ maintain_submodules(workspace_path, manifest, current_combo, args.verbose, cache_path)
# Restore sparse checkout state
if sparse_enabled:
diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py
index 0d54bbf..0b72715 100644
--- a/edkrepo/common/common_repo_functions.py
+++ b/edkrepo/common/common_repo_functions.py
@@ -56,6 +56,7 @@ from project_utils.sparse import BuildInfo, process_sparse_checkout
from edkrepo.config.config_factory import get_workspace_path
from edkrepo.config.config_factory import get_workspace_manifest
from edkrepo.config.tool_config import CI_INDEX_FILE_NAME
+from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException
from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml
from edkrepo.common.edkrepo_exception import EdkrepoNotFoundException, EdkrepoGitException, EdkrepoWarningException
@@ -75,12 +76,25 @@ CLEAR_LINE = '\x1b[K'
DEFAULT_REMOTE_NAME = 'origin'
PRIMARY_REMOTE_NAME = 'primary'
-def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest):
+
+def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest, cache_obj=None):
for repo_to_clone in repos_to_clone:
local_repo_path = os.path.join(workspace_dir, repo_to_clone.root)
local_repo_url = repo_to_clone.remote_url
+ cache_path = None
+ if cache_obj is not None:
+ cache_path = cache_obj.get_cache_path(local_repo_url)
print("Cloning from: " + str(local_repo_url))
- repo = Repo.clone_from(local_repo_url, local_repo_path, progress=GitProgressHandler(), no_checkout=True)
+ if cache_path is not None:
+ print('+ Using cache at {}'.format(cache_path))
+ repo = Repo.clone_from(local_repo_url, local_repo_path,
+ progress=GitProgressHandler(),
+ reference_if_able=cache_path,
+ no_checkout=True)
+ else:
+ repo = Repo.clone_from(local_repo_url, local_repo_path,
+ progress=GitProgressHandler(),
+ no_checkout=True)
# Fetch notes
repo.remotes.origin.fetch("refs/notes/*:refs/notes/*")
@@ -476,7 +490,7 @@ def get_target_sources(combination_or_sha, manifest, workspace_path, log=None):
return ManifestXml(pin_filename).get_repo_sources(current_combo)
-def checkout(combination_or_sha, verbose=False, override=False, log=None):
+def checkout(combination_or_sha, verbose=False, override=False, log=None, cache_obj=None):
workspace_path = get_workspace_path()
manifest = get_workspace_manifest()
@@ -547,7 +561,10 @@ def checkout(combination_or_sha, verbose=False, override=False, log=None):
# Return to the initial combo, since there was an issue with cheking out the selected combo
checkout_repos(verbose, override, initial_repo_sources, workspace_path, manifest)
finally:
- maintain_submodules(workspace_path, manifest, submodule_combo, verbose)
+ cache_path = None
+ if cache_obj is not None:
+ cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME)
+ maintain_submodules(workspace_path, manifest, submodule_combo, verbose, cache_path)
if sparse_enabled or sparse_diff:
print(SPARSE_CHECKOUT)
sparse_checkout(workspace_path, current_repos, manifest)
diff --git a/project_utils/submodule.py b/project_utils/submodule.py
index 3d1b620..f735125 100644
--- a/project_utils/submodule.py
+++ b/project_utils/submodule.py
@@ -61,7 +61,7 @@ def _deinit(repo, submodules=None, verbose=False):
return
-def _update(repo, submodules=None, verbose=False, recursive=False):
+def _update(repo, submodules=None, verbose=False, recursive=False, cache_path=None):
"""
Performs the update of submodules. This includes the sync and update operations.
@@ -82,6 +82,8 @@ def _update(repo, submodules=None, verbose=False, recursive=False):
cmd = ['git', 'submodule', 'update', '--init']
if recursive:
cmd.append('--recursive')
+ if cache_path is not None:
+ cmd.extend(['--reference', cache_path])
output_data = repo.git.execute(cmd, with_extended_output=True, with_stdout=True)
display_git_output(output_data, verbose)
else:
@@ -99,6 +101,8 @@ def _update(repo, submodules=None, verbose=False, recursive=False):
cmd = ['git', 'submodule', 'update', '--init']
if sub.recursive:
cmd.append('--recursive')
+ if cache_path is not None:
+ cmd.extend(['--reference', cache_path])
cmd.extend(['--', sub.path])
output_data = repo.git.execute(cmd, with_extended_output=True, with_stdout=True)
display_git_output(output_data, verbose)
@@ -269,7 +273,7 @@ def deinit_submodules(workspace, start_manifest, start_combo,
_deinit(repo, deinit_list, verbose)
-def maintain_submodules(workspace, manifest, combo_name, verbose=False):
+def maintain_submodules(workspace, manifest, combo_name, verbose=False, cache_path=None):
"""
Updates the submodules for a specific repo.
@@ -277,6 +281,7 @@ def maintain_submodules(workspace, manifest, combo_name, verbose=False):
manifest - The manifest parser object for the project.
combo_name - The combination name to use for submodule maintenance.
verbose - Enable verbose messages.
+ cache_path - Path to the submodule cache repo. A value of None indicates that no cache repo exists.
"""
# Process each repo that may have submodules enabled
print(strings.SUBMOD_INIT_UPDATE)
@@ -303,9 +308,9 @@ def maintain_submodules(workspace, manifest, combo_name, verbose=False):
# Perform sync/update
if len(repo_subs) == 0:
- _update(repo, None, verbose)
+ _update(repo, None, verbose, cache_path=cache_path)
else:
- _update(repo, repo_subs, verbose)
+ _update(repo, repo_subs, verbose, cache_path=cache_path)
if __name__ == '__main__':
--
2.21.0.windows.1
^ permalink raw reply related [flat|nested] 5+ messages in thread