From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mga07.intel.com (mga07.intel.com [134.134.136.100]) by mx.groups.io with SMTP id smtpd.web12.14752.1605136082055111812 for ; Wed, 11 Nov 2020 15:08:02 -0800 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 134.134.136.100, mailfrom: erik.c.bjorge@intel.com) IronPort-SDR: VdzY8nd1nKDBBgd/fRSMD0tL7LcWYH7ZKUnacnJMOQAGzLDWHmPTKsTl7LxrztNBnqlnBTDxiT hqu2QWs4pj3g== X-IronPort-AV: E=McAfee;i="6000,8403,9802"; a="234396378" X-IronPort-AV: E=Sophos;i="5.77,470,1596524400"; d="scan'208";a="234396378" X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga005.jf.intel.com ([10.7.209.41]) by orsmga105.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 11 Nov 2020 15:07:59 -0800 IronPort-SDR: WONLkZIyAncGG61cUXdg6dJQMl9ZKedZmWpkbo1avOuF21S2q+qKg7/FvpVudWMiiTrintFHAa jlqLNs5qimXw== X-IronPort-AV: E=Sophos;i="5.77,470,1596524400"; d="scan'208";a="541992485" Received: from ecbjorge-mobl1.amr.corp.intel.com ([10.212.178.7]) by orsmga005-auth.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 11 Nov 2020 15:07:59 -0800 From: "Bjorge, Erik C" To: devel@edk2.groups.io Cc: Ashley E Desimone , Nate DeSimone , Puja Pandya , Bret Barkelew , Prince Agyeman Subject: [edk2-staging/EdkRepo] [PATCH v1 1/2] EdkRepo: Add cache command Date: Wed, 11 Nov 2020 15:07:28 -0800 Message-Id: <2b15ebc3afc6e561c98abeb0f14c0ee697cc357d.1605135735.git.erik.c.bjorge@intel.com> X-Mailer: git-send-email 2.21.0.windows.1 In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Adds a module to add a repo cache and mange it. Also adds a command to manage the repo cache from EdkRepo. No other commands use the functionality at this point. Cc: Ashley E Desimone Cc: Nate DeSimone Cc: Puja Pandya Cc: Bret Barkelew Cc: Prince Agyeman Cc: Erik Bjorge Signed-off-by: Erik Bjorge --- edkrepo/commands/arguments/cache_args.py | 19 ++ edkrepo/commands/cache_command.py | 118 ++++++++++++ edkrepo/commands/humble/cache_humble.py | 17 ++ edkrepo/common/common_cache_functions.py | 41 +++++ edkrepo/common/edkrepo_exception.py | 3 + edkrepo/config/config_factory.py | 14 +- edkrepo/config/tool_config.py | 5 +- project_utils/cache.py | 224 +++++++++++++++++++++++ project_utils/project_utils_strings.py | 11 ++ 9 files changed, 448 insertions(+), 4 deletions(-) create mode 100644 edkrepo/commands/arguments/cache_args.py create mode 100644 edkrepo/commands/cache_command.py create mode 100644 edkrepo/commands/humble/cache_humble.py create mode 100644 edkrepo/common/common_cache_functions.py create mode 100644 project_utils/cache.py diff --git a/edkrepo/commands/arguments/cache_args.py b/edkrepo/commands/arguments/cache_args.py new file mode 100644 index 0000000..0080536 --- /dev/null +++ b/edkrepo/commands/arguments/cache_args.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +# +## @file +# cache_args.py +# +# Copyright (c) 2020, Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +# + +''' Contains the help and description strings for arguments in the +cache command meta data. +''' +COMMAND_DESCRIPTION = ('Manages local caching support for project repos. The goal of this feature ' + 'is to improve clone performance') +COMMAND_ENABLE_HELP = 'Enables caching support on the system.' +COMMAND_DISABLE_HELP = 'Disables caching support on the system.' +COMMAND_UPDATE_HELP = 'Update the repo cache for all cached projects.' +COMMAND_INFO_HELP = 'Display the current cache information.' +COMMAND_PROJECT_HELP = 'Project to add to the cache.' diff --git a/edkrepo/commands/cache_command.py b/edkrepo/commands/cache_command.py new file mode 100644 index 0000000..9f0d6e9 --- /dev/null +++ b/edkrepo/commands/cache_command.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# +## @file +# cache_command.py +# +# Copyright (c) 2020, Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +# + +import edkrepo.commands.arguments.cache_args as arguments +from edkrepo.commands.edkrepo_command import EdkrepoCommand +from edkrepo.commands.edkrepo_command import SourceManifestRepoArgument +from edkrepo.commands.humble.cache_humble import CACHE_ENABLED, CACHE_FETCH, CACHE_INFO +from edkrepo.commands.humble.cache_humble import CACHE_INFO_LINE, PROJECT_NOT_FOUND, NO_INSTANCE +from edkrepo.commands.humble.cache_humble import UNABLE_TO_LOAD_MANIFEST, UNABLE_TO_PARSE_MANIFEST +from edkrepo.common.common_cache_functions import add_missing_cache_repos +from edkrepo.common.common_cache_functions import get_repo_cache_obj +from edkrepo.common.edkrepo_exception import EdkrepoCacheException +from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_project_in_all_indices +from edkrepo.config.config_factory import get_workspace_manifest +from edkrepo_manifest_parser.edk_manifest import ManifestXml + + +class CacheCommand(EdkrepoCommand): + def __init__(self): + super().__init__() + + def get_metadata(self): + metadata = {} + metadata['name'] = 'cache' + metadata['help-text'] = arguments.COMMAND_DESCRIPTION + args = [] + metadata['arguments'] = args + args.append({'name': 'enable', + 'positional': False, + 'required': False, + 'help-text': arguments.COMMAND_ENABLE_HELP}) + args.append({'name': 'disable', + 'positional': False, + 'required': False, + 'help-text': arguments.COMMAND_DISABLE_HELP}) + args.append({'name': 'update', + 'positional': False, + 'required': False, + 'help-text': arguments.COMMAND_UPDATE_HELP}) + args.append({'name': 'info', + 'positional': False, + 'required': False, + 'help-text': arguments.COMMAND_INFO_HELP}) + args.append({'name': 'project', + 'positional': True, + 'required': False, + 'help-text': arguments.COMMAND_PROJECT_HELP}) + args.append(SourceManifestRepoArgument) + return metadata + + def run_command(self, args, config): + # Process enable disable requests + if args.disable: + config['user_cfg_file'].set_caching_state(False) + elif args.enable: + config['user_cfg_file'].set_caching_state(True) + + # Get the current state now that we have processed enable/disable + cache_state = config['user_cfg_file'].caching_state + print(CACHE_ENABLED.format(cache_state)) + if not cache_state: + return + + # State is enabled so make sure cache directory exists + cache_obj = get_repo_cache_obj(config) + + # Check to see if a manifest was provided and add any missing remotes + manifest = None + if args.project is not None: + manifest = _get_manifest(args.project, config, args.source_manifest_repo) + else: + try: + manifest = get_workspace_manifest() + except Exception: + pass + + # If manifest is provided attempt to add any remotes that do not exist + if manifest is not None: + add_missing_cache_repos(cache_obj, manifest, True) + + # Display all the cache information + if args.info: + print(CACHE_INFO) + info = cache_obj.get_cache_info(args.verbose) + for item in info: + print(CACHE_INFO_LINE.format(item.path, item.remote, item.url)) + + # Do an update if requested + if args.update: + print(CACHE_FETCH) + cache_obj.update_cache(verbose=True) + + # Close the cache repos + cache_obj.close(args.verbose) + + +def _get_manifest(project, config, source_manifest_repo=None): + try: + manifest_repo, source_cfg, manifest_path = find_project_in_all_indices( + project, + config['cfg_file'], + config['user_cfg_file'], + PROJECT_NOT_FOUND.format(project), + NO_INSTANCE.format(project), + source_manifest_repo) + except Exception: + raise EdkrepoCacheException(UNABLE_TO_LOAD_MANIFEST) + try: + manifest = ManifestXml(manifest_path) + except Exception: + raise EdkrepoCacheException(UNABLE_TO_PARSE_MANIFEST) + return manifest diff --git a/edkrepo/commands/humble/cache_humble.py b/edkrepo/commands/humble/cache_humble.py new file mode 100644 index 0000000..4f318ac --- /dev/null +++ b/edkrepo/commands/humble/cache_humble.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# +## @file +# cache_humble.py +# +# Copyright (c) 2020, Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +# + +CACHE_ENABLED = 'Caching Enabled: {}' +CACHE_INFO = 'Cache Information:' +CACHE_INFO_LINE = '+ {}\n {} ({})' +CACHE_FETCH = 'Fetching all remotes... (this could take a while)' +PROJECT_NOT_FOUND = 'Project {} does not exist' +NO_INSTANCE = 'Unable to determine instance to use for {}' +UNABLE_TO_LOAD_MANIFEST = 'Unable to load manifest file.' +UNABLE_TO_PARSE_MANIFEST = 'Failed to parse manifest file.' diff --git a/edkrepo/common/common_cache_functions.py b/edkrepo/common/common_cache_functions.py new file mode 100644 index 0000000..84bd3ed --- /dev/null +++ b/edkrepo/common/common_cache_functions.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# +## @file +# common_cache_functions.py +# +# Copyright (c) 2020, Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +# + +import os + +from edkrepo.config.config_factory import get_edkrepo_global_data_directory +from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME +from project_utils.cache import RepoCache + + +def get_global_cache_directory(config): + if config['user_cfg_file'].caching_state: + return os.path.join(get_edkrepo_global_data_directory(), '.cache') + return None + + +def get_repo_cache_obj(config): + cache_obj = None + cache_directory = get_global_cache_directory(config) + if cache_directory is not None: + cache_obj = RepoCache(cache_directory) + cache_obj.open() + return cache_obj + + +def add_missing_cache_repos(cache_obj, manifest, verbose=False): + print('Adding and fetching new remotes... (this could take a while)') + for remote in manifest.remotes: + cache_obj.add_repo(url=remote.url, verbose=verbose) + alt_submodules = manifest.submodule_alternate_remotes + if alt_submodules: + print('Adding and fetching new submodule remotes... (this could also take a while)') + cache_obj.add_repo(name=SUBMODULE_CACHE_REPO_NAME, verbose=verbose) + for alt in alt_submodules: + cache_obj.add_remote(alt.alternate_url, SUBMODULE_CACHE_REPO_NAME, verbose) diff --git a/edkrepo/common/edkrepo_exception.py b/edkrepo/common/edkrepo_exception.py index a56e709..b3f2300 100644 --- a/edkrepo/common/edkrepo_exception.py +++ b/edkrepo/common/edkrepo_exception.py @@ -98,3 +98,6 @@ class EdkrepoGitConfigSetupException(EdkrepoException): def __init__(self, message): super().__init__(message, 131) +class EdkrepoCacheException(EdkrepoException): + def __init__(self, message): + super().__init__(message, 132) diff --git a/edkrepo/config/config_factory.py b/edkrepo/config/config_factory.py index fe69460..3680c0b 100644 --- a/edkrepo/config/config_factory.py +++ b/edkrepo/config/config_factory.py @@ -225,10 +225,20 @@ class GlobalUserConfig(BaseConfig): self.filename = os.path.join(get_edkrepo_global_data_directory(), "edkrepo_user.cfg") self.prop_list = [ CfgProp('scm', 'mirror_geo', 'geo', 'none', False), - CfgProp('send-review', 'max-patch-set', 'max_patch_set', '10', False) - ] + CfgProp('send-review', 'max-patch-set', 'max_patch_set', '10', False), + CfgProp('caching', 'enable-caching', 'enable_caching_text', 'false', False)] super().__init__(self.filename, get_edkrepo_global_data_directory(), False) + @property + def caching_state(self): + return self.enable_caching_text.lower() == 'true' + + def set_caching_state(self, enable): + if enable: + self.enable_caching_text = 'true' + else: + self.enable_caching_text = 'false' + @property def max_patch_set_int(self): try: diff --git a/edkrepo/config/tool_config.py b/edkrepo/config/tool_config.py index eee1326..81f4ddf 100644 --- a/edkrepo/config/tool_config.py +++ b/edkrepo/config/tool_config.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 # ## @file -# tool)config.py +# tool_config.py # # Copyright (c) 2020, Intel Corporation. All rights reserved.
# SPDX-License-Identifier: BSD-2-Clause-Patent # -CI_INDEX_FILE_NAME = 'CiIndex.xml' \ No newline at end of file +CI_INDEX_FILE_NAME = 'CiIndex.xml' +SUBMODULE_CACHE_REPO_NAME = 'submodule-cache' diff --git a/project_utils/cache.py b/project_utils/cache.py new file mode 100644 index 0000000..8efd411 --- /dev/null +++ b/project_utils/cache.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +# +## @file +# cache.py +# +# Copyright (c) 2020, Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +# +from collections import namedtuple +import os +import shutil + +from git import Repo + +from edkrepo.common.progress_handler import GitProgressHandler +from project_utils.project_utils_strings import CACHE_ADD_REMOTE, CACHE_ADDING_REPO, CACHE_CHECK_ROOT_DIR +from project_utils.project_utils_strings import CACHE_FAILED_TO_CLOSE, CACHE_FAILED_TO_OPEN, CACHE_FETCH_REMOTE +from project_utils.project_utils_strings import CACHE_REMOTE_EXISTS, CACHE_REMOVE_REPO, CACHE_REPO_EXISTS + +CacheInfo = namedtuple('CacheInfo', ['path', 'remote', 'url']) + + +class RepoCache(object): + """ + Provides basic management of a cache repo. + """ + def __init__(self, path): + self._cache_root_path = path + self._repos = {} + + def _create_name(self, url_or_name): + """ + Used to create consistent repo and remote names + """ + dir_name = url_or_name.split('/')[-1] + if not dir_name.endswith('.git'): + dir_name += '.git' + return dir_name + + def _get_repo_path(self, dir_name): + return os.path.join(self._cache_root_path, dir_name) + + def _get_repo(self, dir_name): + """ + Returns the git repo object for the cache repo. + + Raises FileNotFoundError if the cache directory does not exist. + Raises IOError if the repo cannot be opened + """ + repo_path = self._get_repo_path(dir_name) + if not os.path.isdir(repo_path): + raise FileNotFoundError + try: + repo = Repo(repo_path) + except Exception: + raise IOError + return repo + + def _get_cache_dirs(self): + if not os.path.isdir(self._cache_root_path): + raise FileNotFoundError + return [x for x in os.listdir(self._cache_root_path) if os.path.isdir(self._get_repo_path(x))] + + def _add_and_fetch_remote(self, repo, remote_name, url, verbose=False): + if verbose: + print(CACHE_ADD_REMOTE.format(remote_name, url)) + repo.create_remote(remote_name, url) + if verbose: + print(CACHE_FETCH_REMOTE.format(remote_name, url)) + repo.remotes[remote_name].fetch(progress=GitProgressHandler()) + + def open(self, verbose=False): + """ + Opens all cache repos. + + Raises FileNotFoundError if the cache directory does not exist. + """ + if not self._repos: + if not os.path.isdir(self._cache_root_path): + if verbose: + print(CACHE_CHECK_ROOT_DIR.format(self._cache_root_path)) + os.makedirs(self._cache_root_path) + + for dir_name in self._get_cache_dirs(): + try: + self._repos[dir_name] = self._get_repo(dir_name) + except Exception: + if verbose: + print(CACHE_FAILED_TO_OPEN.format(dir_name)) + + def close(self, verbose=False): + """ + Closes all cache repos. + """ + for dir_name in self._repos: + try: + self._repos[dir_name].close() + except Exception: + if verbose: + print(CACHE_FAILED_TO_CLOSE.format(dir_name)) + self._repos = {} + + def get_cache_path(self, url_or_name): + dir_name = self._create_name(url_or_name) + if dir_name not in self._repos: + return None + return self._get_repo_path(dir_name) + + def get_cache_info(self, verbose=False): + """ + Returns a list of remotes currently configured in the cache. + + Raises FileNotFoundError if the cache repo is not open. + """ + ret_val = [] + for dir_name in self._repos: + for remote in self._repos[dir_name].remotes: + ret_val.append(CacheInfo(self._get_repo_path(dir_name), remote.name, remote.url)) + return ret_val + + def delete_cache_root(self, verbose=False): + """ + Deletes the cache root directory and all caches. + """ + if os.path.isdir(self._cache_root_path): + if self._repos: + self.close() + shutil.rmtree(self._cache_root_path, ignore_errors=True) + + def add_repo(self, url=None, name=None, verbose=False): + """ + Adds a repo to the cache if it does not already exist. + + """ + remote_name = None + if url is None and name is None: + raise ValueError + elif name is not None: + dir_name = self._create_name(name) + else: + dir_name = self._create_name(url) + if url is not None: + remote_name = self._create_name(url) + repo_path = self._get_repo_path(dir_name) + + if dir_name in self._repos: + if verbose: + print(CACHE_REPO_EXISTS.format(dir_name)) + else: + if verbose: + print(CACHE_ADDING_REPO.format(dir_name)) + os.makedirs(repo_path) + self._repos[dir_name] = Repo.init(repo_path, bare=True) + + if remote_name is not None and remote_name not in self._repos[dir_name].remotes: + self._add_and_fetch_remote(self._get_repo(dir_name), remote_name, url) + return dir_name + + def remove_repo(self, url=None, name=None, verbose=False): + """ + Removes a remote from the cache repo if it exists + + Raises FileNotFoundError if the cache repo is not open. + """ + if url is None and name is None: + raise ValueError + elif name is not None: + dir_name = self._create_name(name) + else: + dir_name = self._create_name(url) + if dir_name not in self._repos: + return + if verbose: + print(CACHE_REMOVE_REPO.format(dir_name)) + self._repos.pop(dir_name).close() + shutil.rmtree(os.path.join(self._cache_root_path, dir_name), ignore_errors=True) + + def add_remote(self, url, name, verbose=False): + remote_name = self._create_name(url) + dir_name = self._create_name(name) + if dir_name not in self._repos: + raise ValueError + repo = self._get_repo(dir_name) + if remote_name in repo.remotes: + if verbose: + print(CACHE_REMOTE_EXISTS.format(remote_name)) + return + self._add_and_fetch_remote(repo, remote_name, url, verbose) + + def remove_remote(self, url, name, verbose=False): + remote_name = self._create_name(url) + dir_name = self._create_name(name) + if dir_name not in self._repos: + raise ValueError + repo = self._get_repo(dir_name) + if remote_name not in repo.remotes: + raise IndexError + repo.remove_remote(repo.remotes[remote_name]) + + def update_cache(self, url_or_name=None, verbose=False): + if not self._repos: + raise FileNotFoundError + repo_dirs = self._repos.keys() + + if url_or_name is not None: + dir_name = self._create_name(url_or_name) + if dir_name in self._repos: + repo_dirs = [dir_name] + else: + return + + for dir_name in repo_dirs: + try: + repo = self._get_repo(dir_name) + except Exception: + print(CACHE_FAILED_TO_OPEN.format(dir_name)) + continue + for remote in repo.remotes: + if verbose: + print(CACHE_FETCH_REMOTE.format(dir_name, remote.url)) + remote.fetch(progress=GitProgressHandler()) + + def clean_cache(self, verbose=False): + raise NotImplementedError diff --git a/project_utils/project_utils_strings.py b/project_utils/project_utils_strings.py index 33c22d2..1547978 100644 --- a/project_utils/project_utils_strings.py +++ b/project_utils/project_utils_strings.py @@ -22,3 +22,14 @@ SUBMOD_DEINIT_PATH = 'Submodule deinit: {}' SUBMOD_SYNC_PATH = 'Submodule sync: {}' SUBMOD_UPDATE_PATH = 'Submodule update: {}' SUBMOD_EXCEPTION = '- Exception: {}' + +# Caching support strings +CACHE_ADD_REMOTE = '+ Adding remote {} ({})' +CACHE_FETCH_REMOTE = '+ Fetching data for {} ({})' +CACHE_CHECK_ROOT_DIR = '+ Creating cache root directory: {}' +CACHE_FAILED_TO_OPEN = '- Failed to open cache: {}' +CACHE_FAILED_TO_CLOSE = '- Failed to close cache: {}' +CACHE_REPO_EXISTS = '- Repo {} already exists.' +CACHE_ADDING_REPO = '+ Adding cache repo {}' +CACHE_REMOVE_REPO = '- Removing cache repo: {}' +CACHE_REMOTE_EXISTS = '- Remote {} already exists.' -- 2.21.0.windows.1