Re: [edk2-staging/EdkRepo] [PATCH v2 1/2] EdkRepo: Add cache command

["Ashley E Desimone" <ashley.e.desimone@intel.com>]

Pushed as bbab07f4917f386adc30d65895a82e1a96bb0318

-----Original Message-----
From: Erik Bjorge <erik.c.bjorge@intel.com> 
Sent: Monday, November 16, 2020 8:50 PM
To: devel@edk2.groups.io
Cc: Desimone, Ashley E <ashley.e.desimone@intel.com>; Desimone, Nathaniel L <nathaniel.l.desimone@intel.com>; Pandya, Puja <puja.pandya@intel.com>; Bret Barkelew <Bret.Barkelew@microsoft.com>; Agyeman, Prince <prince.agyeman@intel.com>
Subject: [edk2-staging/EdkRepo] [PATCH v2 1/2] EdkRepo: Add cache command

Adds a module to add a repo cache and mange it.  Also adds a command to manage the repo cache from EdkRepo.  No other commands use the functionality at this point.

Cc: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Prince Agyeman <prince.agyeman@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
Signed-off-by: Erik Bjorge <erik.c.bjorge@intel.com>
---
 edkrepo/commands/arguments/cache_args.py |  19 ++
 edkrepo/commands/cache_command.py        | 118 ++++++++++++
 edkrepo/commands/humble/cache_humble.py  |  17 ++  edkrepo/common/common_cache_functions.py |  41 +++++
 edkrepo/common/edkrepo_exception.py      |   3 +
 edkrepo/config/config_factory.py         |  14 +-
 edkrepo/config/tool_config.py            |   5 +-
 project_utils/cache.py                   | 224 +++++++++++++++++++++++
 project_utils/project_utils_strings.py   |  11 ++
 9 files changed, 448 insertions(+), 4 deletions(-)  create mode 100644 edkrepo/commands/arguments/cache_args.py
 create mode 100644 edkrepo/commands/cache_command.py  create mode 100644 edkrepo/commands/humble/cache_humble.py
 create mode 100644 edkrepo/common/common_cache_functions.py
 create mode 100644 project_utils/cache.py

diff --git a/edkrepo/commands/arguments/cache_args.py b/edkrepo/commands/arguments/cache_args.py
new file mode 100644
index 0000000..0080536
--- /dev/null
+++ b/edkrepo/commands/arguments/cache_args.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+#
+## @file
+# cache_args.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR> # 
+SPDX-License-Identifier: BSD-2-Clause-Patent #
+
+''' Contains the help and description strings for arguments in the 
+cache command meta data.
+'''
+COMMAND_DESCRIPTION = ('Manages local caching support for project repos.  The goal of this feature '
+                       'is to improve clone performance') 
+COMMAND_ENABLE_HELP = 'Enables caching support on the system.'
+COMMAND_DISABLE_HELP = 'Disables caching support on the system.'
+COMMAND_UPDATE_HELP = 'Update the repo cache for all cached projects.'
+COMMAND_INFO_HELP = 'Display the current cache information.'
+COMMAND_PROJECT_HELP = 'Project to add to the cache.'
diff --git a/edkrepo/commands/cache_command.py b/edkrepo/commands/cache_command.py
new file mode 100644
index 0000000..9f0d6e9
--- /dev/null
+++ b/edkrepo/commands/cache_command.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+#
+## @file
+# cache_command.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR> # 
+SPDX-License-Identifier: BSD-2-Clause-Patent #
+
+import edkrepo.commands.arguments.cache_args as arguments from 
+edkrepo.commands.edkrepo_command import EdkrepoCommand from 
+edkrepo.commands.edkrepo_command import SourceManifestRepoArgument from 
+edkrepo.commands.humble.cache_humble import CACHE_ENABLED, CACHE_FETCH, 
+CACHE_INFO from edkrepo.commands.humble.cache_humble import 
+CACHE_INFO_LINE, PROJECT_NOT_FOUND, NO_INSTANCE from 
+edkrepo.commands.humble.cache_humble import UNABLE_TO_LOAD_MANIFEST, 
+UNABLE_TO_PARSE_MANIFEST from edkrepo.common.common_cache_functions 
+import add_missing_cache_repos from 
+edkrepo.common.common_cache_functions import get_repo_cache_obj from 
+edkrepo.common.edkrepo_exception import EdkrepoCacheException from 
+edkrepo.common.workspace_maintenance.manifest_repos_maintenance import 
+find_project_in_all_indices from edkrepo.config.config_factory import 
+get_workspace_manifest from edkrepo_manifest_parser.edk_manifest import 
+ManifestXml
+
+
+class CacheCommand(EdkrepoCommand):
+    def __init__(self):
+        super().__init__()
+
+    def get_metadata(self):
+        metadata = {}
+        metadata['name'] = 'cache'
+        metadata['help-text'] = arguments.COMMAND_DESCRIPTION
+        args = []
+        metadata['arguments'] = args
+        args.append({'name': 'enable',
+                     'positional': False,
+                     'required': False,
+                     'help-text': arguments.COMMAND_ENABLE_HELP})
+        args.append({'name': 'disable',
+                     'positional': False,
+                     'required': False,
+                     'help-text': arguments.COMMAND_DISABLE_HELP})
+        args.append({'name': 'update',
+                     'positional': False,
+                     'required': False,
+                     'help-text': arguments.COMMAND_UPDATE_HELP})
+        args.append({'name': 'info',
+                     'positional': False,
+                     'required': False,
+                     'help-text': arguments.COMMAND_INFO_HELP})
+        args.append({'name': 'project',
+                     'positional': True,
+                     'required': False,
+                     'help-text': arguments.COMMAND_PROJECT_HELP})
+        args.append(SourceManifestRepoArgument)
+        return metadata
+
+    def run_command(self, args, config):
+        # Process enable disable requests
+        if args.disable:
+            config['user_cfg_file'].set_caching_state(False)
+        elif args.enable:
+            config['user_cfg_file'].set_caching_state(True)
+
+        # Get the current state now that we have processed enable/disable
+        cache_state = config['user_cfg_file'].caching_state
+        print(CACHE_ENABLED.format(cache_state))
+        if not cache_state:
+            return
+
+        # State is enabled so make sure cache directory exists
+        cache_obj = get_repo_cache_obj(config)
+
+        # Check to see if a manifest was provided and add any missing remotes
+        manifest = None
+        if args.project is not None:
+            manifest = _get_manifest(args.project, config, args.source_manifest_repo)
+        else:
+            try:
+                manifest = get_workspace_manifest()
+            except Exception:
+                pass
+
+        # If manifest is provided attempt to add any remotes that do not exist
+        if manifest is not None:
+            add_missing_cache_repos(cache_obj, manifest, True)
+
+        # Display all the cache information
+        if args.info:
+            print(CACHE_INFO)
+            info = cache_obj.get_cache_info(args.verbose)
+            for item in info:
+                print(CACHE_INFO_LINE.format(item.path, item.remote, 
+ item.url))
+
+        # Do an update if requested
+        if args.update:
+            print(CACHE_FETCH)
+            cache_obj.update_cache(verbose=True)
+
+        # Close the cache repos
+        cache_obj.close(args.verbose)
+
+
+def _get_manifest(project, config, source_manifest_repo=None):
+    try:
+        manifest_repo, source_cfg, manifest_path = find_project_in_all_indices(
+            project,
+            config['cfg_file'],
+            config['user_cfg_file'],
+            PROJECT_NOT_FOUND.format(project),
+            NO_INSTANCE.format(project),
+            source_manifest_repo)
+    except Exception:
+        raise EdkrepoCacheException(UNABLE_TO_LOAD_MANIFEST)
+    try:
+        manifest = ManifestXml(manifest_path)
+    except Exception:
+        raise EdkrepoCacheException(UNABLE_TO_PARSE_MANIFEST)
+    return manifest
diff --git a/edkrepo/commands/humble/cache_humble.py b/edkrepo/commands/humble/cache_humble.py
new file mode 100644
index 0000000..4f318ac
--- /dev/null
+++ b/edkrepo/commands/humble/cache_humble.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+#
+## @file
+# cache_humble.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR> # 
+SPDX-License-Identifier: BSD-2-Clause-Patent #
+
+CACHE_ENABLED = 'Caching Enabled: {}'
+CACHE_INFO = 'Cache Information:'
+CACHE_INFO_LINE = '+ {}\n    {} ({})'
+CACHE_FETCH = 'Fetching all remotes... (this could take a while)'
+PROJECT_NOT_FOUND = 'Project {} does not exist'
+NO_INSTANCE = 'Unable to determine instance to use for {}'
+UNABLE_TO_LOAD_MANIFEST = 'Unable to load manifest file.'
+UNABLE_TO_PARSE_MANIFEST = 'Failed to parse manifest file.'
diff --git a/edkrepo/common/common_cache_functions.py b/edkrepo/common/common_cache_functions.py
new file mode 100644
index 0000000..84bd3ed
--- /dev/null
+++ b/edkrepo/common/common_cache_functions.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+#
+## @file
+# common_cache_functions.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR> # 
+SPDX-License-Identifier: BSD-2-Clause-Patent #
+
+import os
+
+from edkrepo.config.config_factory import 
+get_edkrepo_global_data_directory from edkrepo.config.tool_config 
+import SUBMODULE_CACHE_REPO_NAME from project_utils.cache import 
+RepoCache
+
+
+def get_global_cache_directory(config):
+    if config['user_cfg_file'].caching_state:
+        return os.path.join(get_edkrepo_global_data_directory(), '.cache')
+    return None
+
+
+def get_repo_cache_obj(config):
+    cache_obj = None
+    cache_directory = get_global_cache_directory(config)
+    if cache_directory is not None:
+        cache_obj = RepoCache(cache_directory)
+        cache_obj.open()
+    return cache_obj
+
+
+def add_missing_cache_repos(cache_obj, manifest, verbose=False):
+    print('Adding and fetching new remotes... (this could take a while)')
+    for remote in manifest.remotes:
+        cache_obj.add_repo(url=remote.url, verbose=verbose)
+    alt_submodules = manifest.submodule_alternate_remotes
+    if alt_submodules:
+        print('Adding and fetching new submodule remotes... (this could also take a while)')
+        cache_obj.add_repo(name=SUBMODULE_CACHE_REPO_NAME, verbose=verbose)
+        for alt in alt_submodules:
+            cache_obj.add_remote(alt.alternate_url, 
+SUBMODULE_CACHE_REPO_NAME, verbose)
diff --git a/edkrepo/common/edkrepo_exception.py b/edkrepo/common/edkrepo_exception.py
index a56e709..b3f2300 100644
--- a/edkrepo/common/edkrepo_exception.py
+++ b/edkrepo/common/edkrepo_exception.py
@@ -98,3 +98,6 @@ class EdkrepoGitConfigSetupException(EdkrepoException):
     def __init__(self, message):
         super().__init__(message, 131)
 
+class EdkrepoCacheException(EdkrepoException):
+    def __init__(self, message):
+        super().__init__(message, 132)
diff --git a/edkrepo/config/config_factory.py b/edkrepo/config/config_factory.py
index fe69460..3680c0b 100644
--- a/edkrepo/config/config_factory.py
+++ b/edkrepo/config/config_factory.py
@@ -225,10 +225,20 @@ class GlobalUserConfig(BaseConfig):
         self.filename = os.path.join(get_edkrepo_global_data_directory(), "edkrepo_user.cfg")
         self.prop_list = [
             CfgProp('scm', 'mirror_geo', 'geo', 'none', False),
-            CfgProp('send-review', 'max-patch-set', 'max_patch_set', '10', False)
-            ]
+            CfgProp('send-review', 'max-patch-set', 'max_patch_set', '10', False),
+            CfgProp('caching', 'enable-caching', 'enable_caching_text', 
+ 'false', False)]
         super().__init__(self.filename, get_edkrepo_global_data_directory(), False)
 
+    @property
+    def caching_state(self):
+        return self.enable_caching_text.lower() == 'true'
+
+    def set_caching_state(self, enable):
+        if enable:
+            self.enable_caching_text = 'true'
+        else:
+            self.enable_caching_text = 'false'
+
     @property
     def max_patch_set_int(self):
         try:
diff --git a/edkrepo/config/tool_config.py b/edkrepo/config/tool_config.py index eee1326..81f4ddf 100644
--- a/edkrepo/config/tool_config.py
+++ b/edkrepo/config/tool_config.py
@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 #
 ## @file
-# tool)config.py
+# tool_config.py
 #
 # Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>  # SPDX-License-Identifier: BSD-2-Clause-Patent  #
 
-CI_INDEX_FILE_NAME = 'CiIndex.xml'
\ No newline at end of file
+CI_INDEX_FILE_NAME = 'CiIndex.xml'
+SUBMODULE_CACHE_REPO_NAME = 'submodule-cache'
diff --git a/project_utils/cache.py b/project_utils/cache.py new file mode 100644 index 0000000..8efd411
--- /dev/null
+++ b/project_utils/cache.py
@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+#
+## @file
+# cache.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR> # 
+SPDX-License-Identifier: BSD-2-Clause-Patent # from collections import 
+namedtuple import os import shutil
+
+from git import Repo
+
+from edkrepo.common.progress_handler import GitProgressHandler from 
+project_utils.project_utils_strings import CACHE_ADD_REMOTE, 
+CACHE_ADDING_REPO, CACHE_CHECK_ROOT_DIR from 
+project_utils.project_utils_strings import CACHE_FAILED_TO_CLOSE, 
+CACHE_FAILED_TO_OPEN, CACHE_FETCH_REMOTE from 
+project_utils.project_utils_strings import CACHE_REMOTE_EXISTS, 
+CACHE_REMOVE_REPO, CACHE_REPO_EXISTS
+
+CacheInfo = namedtuple('CacheInfo', ['path', 'remote', 'url'])
+
+
+class RepoCache(object):
+    """
+    Provides basic management of a cache repo.
+    """
+    def __init__(self, path):
+        self._cache_root_path = path
+        self._repos = {}
+
+    def _create_name(self, url_or_name):
+        """
+        Used to create consistent repo and remote names
+        """
+        dir_name = url_or_name.split('/')[-1]
+        if not dir_name.endswith('.git'):
+            dir_name += '.git'
+        return dir_name
+
+    def _get_repo_path(self, dir_name):
+        return os.path.join(self._cache_root_path, dir_name)
+
+    def _get_repo(self, dir_name):
+        """
+        Returns the git repo object for the cache repo.
+
+        Raises FileNotFoundError if the cache directory does not exist.
+        Raises IOError if the repo cannot be opened
+        """
+        repo_path = self._get_repo_path(dir_name)
+        if not os.path.isdir(repo_path):
+            raise FileNotFoundError
+        try:
+            repo = Repo(repo_path)
+        except Exception:
+            raise IOError
+        return repo
+
+    def _get_cache_dirs(self):
+        if not os.path.isdir(self._cache_root_path):
+            raise FileNotFoundError
+        return [x for x in os.listdir(self._cache_root_path) if 
+ os.path.isdir(self._get_repo_path(x))]
+
+    def _add_and_fetch_remote(self, repo, remote_name, url, verbose=False):
+        if verbose:
+            print(CACHE_ADD_REMOTE.format(remote_name, url))
+        repo.create_remote(remote_name, url)
+        if verbose:
+            print(CACHE_FETCH_REMOTE.format(remote_name, url))
+        repo.remotes[remote_name].fetch(progress=GitProgressHandler())
+
+    def open(self, verbose=False):
+        """
+        Opens all cache repos.
+
+        Raises FileNotFoundError if the cache directory does not exist.
+        """
+        if not self._repos:
+            if not os.path.isdir(self._cache_root_path):
+                if verbose:
+                    print(CACHE_CHECK_ROOT_DIR.format(self._cache_root_path))
+                os.makedirs(self._cache_root_path)
+
+            for dir_name in self._get_cache_dirs():
+                try:
+                    self._repos[dir_name] = self._get_repo(dir_name)
+                except Exception:
+                    if verbose:
+                        print(CACHE_FAILED_TO_OPEN.format(dir_name))
+
+    def close(self, verbose=False):
+        """
+        Closes all cache repos.
+        """
+        for dir_name in self._repos:
+            try:
+                self._repos[dir_name].close()
+            except Exception:
+                if verbose:
+                    print(CACHE_FAILED_TO_CLOSE.format(dir_name))
+        self._repos = {}
+
+    def get_cache_path(self, url_or_name):
+        dir_name = self._create_name(url_or_name)
+        if dir_name not in self._repos:
+            return None
+        return self._get_repo_path(dir_name)
+
+    def get_cache_info(self, verbose=False):
+        """
+        Returns a list of remotes currently configured in the cache.
+
+        Raises FileNotFoundError if the cache repo is not open.
+        """
+        ret_val = []
+        for dir_name in self._repos:
+            for remote in self._repos[dir_name].remotes:
+                ret_val.append(CacheInfo(self._get_repo_path(dir_name), remote.name, remote.url))
+        return ret_val
+
+    def delete_cache_root(self, verbose=False):
+        """
+        Deletes the cache root directory and all caches.
+        """
+        if os.path.isdir(self._cache_root_path):
+            if self._repos:
+                self.close()
+            shutil.rmtree(self._cache_root_path, ignore_errors=True)
+
+    def add_repo(self, url=None, name=None, verbose=False):
+        """
+        Adds a repo to the cache if it does not already exist.
+
+        """
+        remote_name = None
+        if url is None and name is None:
+            raise ValueError
+        elif name is not None:
+            dir_name = self._create_name(name)
+        else:
+            dir_name = self._create_name(url)
+        if url is not None:
+            remote_name = self._create_name(url)
+        repo_path = self._get_repo_path(dir_name)
+
+        if dir_name in self._repos:
+            if verbose:
+                print(CACHE_REPO_EXISTS.format(dir_name))
+        else:
+            if verbose:
+                print(CACHE_ADDING_REPO.format(dir_name))
+            os.makedirs(repo_path)
+            self._repos[dir_name] = Repo.init(repo_path, bare=True)
+
+        if remote_name is not None and remote_name not in self._repos[dir_name].remotes:
+            self._add_and_fetch_remote(self._get_repo(dir_name), remote_name, url)
+        return dir_name
+
+    def remove_repo(self, url=None, name=None, verbose=False):
+        """
+        Removes a remote from the cache repo if it exists
+
+        Raises FileNotFoundError if the cache repo is not open.
+        """
+        if url is None and name is None:
+            raise ValueError
+        elif name is not None:
+            dir_name = self._create_name(name)
+        else:
+            dir_name = self._create_name(url)
+        if dir_name not in self._repos:
+            return
+        if verbose:
+            print(CACHE_REMOVE_REPO.format(dir_name))
+        self._repos.pop(dir_name).close()
+        shutil.rmtree(os.path.join(self._cache_root_path, dir_name), 
+ ignore_errors=True)
+
+    def add_remote(self, url, name, verbose=False):
+        remote_name = self._create_name(url)
+        dir_name = self._create_name(name)
+        if dir_name not in self._repos:
+            raise ValueError
+        repo = self._get_repo(dir_name)
+        if remote_name in repo.remotes:
+            if verbose:
+                print(CACHE_REMOTE_EXISTS.format(remote_name))
+            return
+        self._add_and_fetch_remote(repo, remote_name, url, verbose)
+
+    def remove_remote(self, url, name, verbose=False):
+        remote_name = self._create_name(url)
+        dir_name = self._create_name(name)
+        if dir_name not in self._repos:
+            raise ValueError
+        repo = self._get_repo(dir_name)
+        if remote_name not in repo.remotes:
+            raise IndexError
+        repo.remove_remote(repo.remotes[remote_name])
+
+    def update_cache(self, url_or_name=None, verbose=False):
+        if not self._repos:
+            raise FileNotFoundError
+        repo_dirs = self._repos.keys()
+
+        if url_or_name is not None:
+            dir_name = self._create_name(url_or_name)
+            if dir_name in self._repos:
+                repo_dirs = [dir_name]
+            else:
+                return
+
+        for dir_name in repo_dirs:
+            try:
+                repo = self._get_repo(dir_name)
+            except Exception:
+                print(CACHE_FAILED_TO_OPEN.format(dir_name))
+                continue
+            for remote in repo.remotes:
+                if verbose:
+                    print(CACHE_FETCH_REMOTE.format(dir_name, remote.url))
+                remote.fetch(progress=GitProgressHandler())
+
+    def clean_cache(self, verbose=False):
+        raise NotImplementedError
diff --git a/project_utils/project_utils_strings.py b/project_utils/project_utils_strings.py
index 33c22d2..1547978 100644
--- a/project_utils/project_utils_strings.py
+++ b/project_utils/project_utils_strings.py
@@ -22,3 +22,14 @@ SUBMOD_DEINIT_PATH = 'Submodule deinit: {}'
 SUBMOD_SYNC_PATH = 'Submodule sync: {}'
 SUBMOD_UPDATE_PATH = 'Submodule update: {}'
 SUBMOD_EXCEPTION = '- Exception: {}'
+
+# Caching support strings
+CACHE_ADD_REMOTE = '+ Adding remote {} ({})'
+CACHE_FETCH_REMOTE = '+ Fetching data for {} ({})'
+CACHE_CHECK_ROOT_DIR = '+ Creating cache root directory: {}'
+CACHE_FAILED_TO_OPEN = '- Failed to open cache: {}'
+CACHE_FAILED_TO_CLOSE = '- Failed to close cache: {}'
+CACHE_REPO_EXISTS = '- Repo {} already exists.'
+CACHE_ADDING_REPO = '+ Adding cache repo {}'
+CACHE_REMOVE_REPO = '- Removing cache repo: {}'
+CACHE_REMOTE_EXISTS = '- Remote {} already exists.'
--
2.21.0.windows.1