Re: [edk2-staging/EdkRepo] [PATCH v2 2/2] EdkRepo: Enable use of repo cache support.

["Ashley E Desimone" <ashley.e.desimone@intel.com>]

Pushed as: bc55138c51548d80db4416d91e55b2c35e48ff4e

-----Original Message-----
From: Erik Bjorge <erik.c.bjorge@intel.com> 
Sent: Monday, November 16, 2020 8:50 PM
To: devel@edk2.groups.io
Cc: Desimone, Ashley E <ashley.e.desimone@intel.com>; Desimone, Nathaniel L <nathaniel.l.desimone@intel.com>; Pandya, Puja <puja.pandya@intel.com>; Bret Barkelew <Bret.Barkelew@microsoft.com>; Agyeman, Prince <prince.agyeman@intel.com>
Subject: [edk2-staging/EdkRepo] [PATCH v2 2/2] EdkRepo: Enable use of repo cache support.

This changes enables the local repo cache to be used when cloning and syncing changes.  The repo cache applies to submodules as well.

Cc: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Prince Agyeman <prince.agyeman@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
Signed-off-by: Erik Bjorge <erik.c.bjorge@intel.com>
---
 edkrepo/commands/checkout_command.py     |  3 ++-
 edkrepo/commands/checkout_pin_command.py |  8 +++++++-
 edkrepo/commands/clone_command.py        | 15 ++++++++++++--
 edkrepo/commands/sync_command.py         | 12 +++++++++---
 edkrepo/common/common_repo_functions.py  | 25 ++++++++++++++++++++----
 project_utils/submodule.py               | 13 ++++++++----
 6 files changed, 61 insertions(+), 15 deletions(-)

diff --git a/edkrepo/commands/checkout_command.py b/edkrepo/commands/checkout_command.py
index 0169f30..2ce26c0 100644
--- a/edkrepo/commands/checkout_command.py
+++ b/edkrepo/commands/checkout_command.py
@@ -16,6 +16,7 @@ import os
 from edkrepo.commands.edkrepo_command import EdkrepoCommand, OverrideArgument  import edkrepo.commands.arguments.checkout_args as arguments  import edkrepo.commands.humble.checkout_humble as humble
+from edkrepo.common.common_cache_functions import get_repo_cache_obj
 from edkrepo.common.common_repo_functions import checkout, combination_is_in_manifest  from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException  from edkrepo.config.config_factory import get_workspace_manifest @@ -42,6 +43,6 @@ class CheckoutCommand(EdkrepoCommand):
 
     def run_command(self, args, config):
         if combination_is_in_manifest(args.Combination, get_workspace_manifest()):
-            checkout(args.Combination, args.verbose, args.override)
+            checkout(args.Combination, args.verbose, args.override, 
+ get_repo_cache_obj(config))
         else:
             raise EdkrepoInvalidParametersException(humble.NO_COMBO.format(args.Combination))
diff --git a/edkrepo/commands/checkout_pin_command.py b/edkrepo/commands/checkout_pin_command.py
index 1c58113..0ad1b48 100644
--- a/edkrepo/commands/checkout_pin_command.py
+++ b/edkrepo/commands/checkout_pin_command.py
@@ -14,6 +14,7 @@ from git import Repo
 from edkrepo.commands.edkrepo_command import EdkrepoCommand, OverrideArgument, SourceManifestRepoArgument  import edkrepo.commands.arguments.checkout_pin_args as arguments  import edkrepo.commands.humble.checkout_pin_humble as humble
+from edkrepo.common.common_cache_functions import get_repo_cache_obj
 from edkrepo.common.common_repo_functions import sparse_checkout_enabled, reset_sparse_checkout, sparse_checkout  from edkrepo.common.common_repo_functions import check_dirty_repos, checkout_repos, combinations_in_manifest  from edkrepo.common.humble import SPARSE_CHECKOUT, SPARSE_RESET, SUBMODULE_DEINIT_FAILED @@ -21,6 +22,7 @@ from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException,
 from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list_available_manifest_repos  from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_source_manifest_repo  from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest
+from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
 from edkrepo_manifest_parser.edk_manifest import ManifestXml  from project_utils.submodule import deinit_full, maintain_submodules
 
@@ -82,7 +84,11 @@ class CheckoutPinCommand(EdkrepoCommand):
             checkout_repos(args.verbose, args.override, pin_repo_sources, workspace_path, manifest)
             manifest.write_current_combo(humble.PIN_COMBO.format(args.pinfile))
         finally:
-            maintain_submodules(workspace_path, pin, submodule_combo, args.verbose)
+            cache_path = None
+            cache_obj = get_repo_cache_obj(config)
+            if cache_obj is not None:
+                cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME)
+            maintain_submodules(workspace_path, pin, submodule_combo, 
+ args.verbose, cache_path)
             if sparse_enabled:
                 print(SPARSE_CHECKOUT)
                 sparse_checkout(workspace_path, pin_repo_sources, manifest) diff --git a/edkrepo/commands/clone_command.py b/edkrepo/commands/clone_command.py
index 8769102..56c15c9 100644
--- a/edkrepo/commands/clone_command.py
+++ b/edkrepo/commands/clone_command.py
@@ -14,6 +14,8 @@ import sys
 from edkrepo.commands.edkrepo_command import EdkrepoCommand  from edkrepo.commands.edkrepo_command import SubmoduleSkipArgument, SourceManifestRepoArgument  import edkrepo.commands.arguments.clone_args as arguments
+from edkrepo.common.common_cache_functions import get_repo_cache_obj 
+from edkrepo.common.common_cache_functions import 
+add_missing_cache_repos
 from edkrepo.common.common_repo_functions import clone_repos, sparse_checkout, verify_single_manifest  from edkrepo.common.common_repo_functions import update_editor_config, combinations_in_manifest  from edkrepo.common.common_repo_functions import write_included_config, write_conditional_include @@ -28,6 +30,7 @@ from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list  from edkrepo.common.workspace_maintenance.humble.manifest_repos_maintenance_humble import PROJ_NOT_IN_REPO, SOURCE_MANIFEST_REPO_NOT_FOUND  from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml  from project_utils.submodule import maintain_submodules
+from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
 
 
 class CloneCommand(EdkrepoCommand):
@@ -151,11 +154,19 @@ class CloneCommand(EdkrepoCommand):
         # Set up submodule alt url config settings prior to cloning any repos
         submodule_included_configs = write_included_config(manifest.remotes, manifest.submodule_alternate_remotes, local_manifest_dir)
         write_conditional_include(workspace_dir, repo_sources_to_clone, submodule_included_configs)
-        clone_repos(args, workspace_dir, repo_sources_to_clone, project_client_side_hooks, config, manifest)
+
+        # Determine if caching is going to be used and then clone
+        cache_obj = get_repo_cache_obj(config)
+        if cache_obj is not None:
+            add_missing_cache_repos(cache_obj, manifest, args.verbose)
+        clone_repos(args, workspace_dir, repo_sources_to_clone, 
+ project_client_side_hooks, config, manifest, cache_obj)
 
         # Init submodules
         if not args.skip_submodule:
-            maintain_submodules(workspace_dir, manifest, combo_name, args.verbose)
+            cache_path = None
+            if cache_obj is not None:
+                cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME)
+            maintain_submodules(workspace_dir, manifest, combo_name, 
+ args.verbose, cache_path)
 
         # Perform a sparse checkout if requested.
         use_sparse = args.sparse
diff --git a/edkrepo/commands/sync_command.py b/edkrepo/commands/sync_command.py
index c4ee330..ff48f50 100644
--- a/edkrepo/commands/sync_command.py
+++ b/edkrepo/commands/sync_command.py
@@ -32,6 +32,7 @@ from edkrepo.common.humble import MIRROR_BEHIND_PRIMARY_REPO, SYNC_NEEDS_REBASE,  from edkrepo.common.humble import SYNC_BRANCH_CHANGE_ON_LOCAL, SYNC_INCOMPATIBLE_COMBO  from edkrepo.common.humble import SYNC_REBASE_CALC_FAIL  from edkrepo.common.pathfix import get_actual_path, expanduser
+from edkrepo.common.common_cache_functions import get_repo_cache_obj
 from edkrepo.common.common_repo_functions import clone_repos, sparse_checkout_enabled  from edkrepo.common.common_repo_functions import reset_sparse_checkout, sparse_checkout, verify_single_manifest  from edkrepo.common.common_repo_functions import checkout_repos, check_dirty_repos @@ -47,6 +48,7 @@ from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list  from edkrepo.common.ui_functions import init_color_console  from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest, get_edkrepo_global_data_directory  from edkrepo.config.config_factory import get_workspace_manifest_file
+from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
 from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml  from project_utils.submodule import deinit_submodules, maintain_submodules
 
@@ -102,7 +104,7 @@ class SyncCommand(EdkrepoCommand):
         if not args.update_local_manifest:
             self.__check_for_new_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory)
         check_dirty_repos(initial_manifest, workspace_path)
-
+
         # Determine if sparse checkout needs to be disabled for this operation
         sparse_settings = initial_manifest.sparse_settings
         sparse_enabled = sparse_checkout_enabled(workspace_path, initial_sources) @@ -116,7 +118,7 @@ class SyncCommand(EdkrepoCommand):
             reset_sparse_checkout(workspace_path, initial_sources)
 
         # Get the latest manifest if requested
-        if args.update_local_manifest:  # NOTE: hyphens in arg name replaced with underscores due to argparse
+        if args.update_local_manifest:  # NOTE: hyphens in arg name 
+ replaced with underscores due to argparse
             self.__update_local_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory)
         manifest = get_workspace_manifest()
         if args.update_local_manifest:
@@ -212,7 +214,11 @@ class SyncCommand(EdkrepoCommand):
 
         # Initialize submodules
         if not args.skip_submodule:
-            maintain_submodules(workspace_path, manifest, current_combo, args.verbose)
+            cache_path = None
+            cache_obj = get_repo_cache_obj(config)
+            if cache_obj is not None:
+                cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME)
+            maintain_submodules(workspace_path, manifest, 
+ current_combo, args.verbose, cache_path)
 
         # Restore sparse checkout state
         if sparse_enabled:
diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py
index 2277c1e..336661a 100644
--- a/edkrepo/common/common_repo_functions.py
+++ b/edkrepo/common/common_repo_functions.py
@@ -56,6 +56,7 @@ from project_utils.sparse import BuildInfo, process_sparse_checkout  from edkrepo.config.config_factory import get_workspace_path  from edkrepo.config.config_factory import get_workspace_manifest  from edkrepo.config.tool_config import CI_INDEX_FILE_NAME
+from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
 from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException  from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml  from edkrepo.common.edkrepo_exception import EdkrepoNotFoundException, EdkrepoGitException, EdkrepoWarningException @@ -75,12 +76,25 @@ CLEAR_LINE = '\x1b[K'
 DEFAULT_REMOTE_NAME = 'origin'
 PRIMARY_REMOTE_NAME = 'primary'
 
-def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest):
+
+def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest, cache_obj=None):
     for repo_to_clone in repos_to_clone:
         local_repo_path = os.path.join(workspace_dir, repo_to_clone.root)
         local_repo_url = repo_to_clone.remote_url
+        cache_path = None
+        if cache_obj is not None:
+            cache_path = cache_obj.get_cache_path(local_repo_url)
         print("Cloning from: " + str(local_repo_url))
-        repo = Repo.clone_from(local_repo_url, local_repo_path, progress=GitProgressHandler(), no_checkout=True)
+        if cache_path is not None:
+            print('+ Using cache at {}'.format(cache_path))
+            repo = Repo.clone_from(local_repo_url, local_repo_path,
+                                   progress=GitProgressHandler(),
+                                   reference_if_able=cache_path,
+                                   no_checkout=True)
+        else:
+            repo = Repo.clone_from(local_repo_url, local_repo_path,
+                                   progress=GitProgressHandler(),
+                                   no_checkout=True)
         # Fetch notes
         repo.remotes.origin.fetch("refs/notes/*:refs/notes/*")
 
@@ -427,7 +441,7 @@ def combination_is_in_manifest(combination, manifest):
     return combination in combination_names
 
 
-def checkout(combination, verbose=False, override=False, log=None):
+def checkout(combination, verbose=False, override=False, log=None, cache_obj=None):
     workspace_path = get_workspace_path()
     manifest = get_workspace_manifest()
 
@@ -493,7 +507,10 @@ def checkout(combination, verbose=False, override=False, log=None):
         # Return to the initial combo, since there was an issue with cheking out the selected combo
         checkout_repos(verbose, override, initial_repo_sources, workspace_path, manifest)
     finally:
-        maintain_submodules(workspace_path, manifest, submodule_combo, verbose)
+        cache_path = None
+        if cache_obj is not None:
+            cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME)
+        maintain_submodules(workspace_path, manifest, submodule_combo, 
+ verbose, cache_path)
         if sparse_enabled or sparse_diff:
             print(SPARSE_CHECKOUT)
             sparse_checkout(workspace_path, current_repos, manifest) diff --git a/project_utils/submodule.py b/project_utils/submodule.py index 3d1b620..f735125 100644
--- a/project_utils/submodule.py
+++ b/project_utils/submodule.py
@@ -61,7 +61,7 @@ def _deinit(repo, submodules=None, verbose=False):
     return
 
 
-def _update(repo, submodules=None, verbose=False, recursive=False):
+def _update(repo, submodules=None, verbose=False, recursive=False, cache_path=None):
     """
     Performs the update of submodules.  This includes the sync and update operations.
 
@@ -82,6 +82,8 @@ def _update(repo, submodules=None, verbose=False, recursive=False):
         cmd = ['git', 'submodule', 'update', '--init']
         if recursive:
             cmd.append('--recursive')
+        if cache_path is not None:
+            cmd.extend(['--reference', cache_path])
         output_data = repo.git.execute(cmd, with_extended_output=True, with_stdout=True)
         display_git_output(output_data, verbose)
     else:
@@ -99,6 +101,8 @@ def _update(repo, submodules=None, verbose=False, recursive=False):
             cmd = ['git', 'submodule', 'update', '--init']
             if sub.recursive:
                 cmd.append('--recursive')
+            if cache_path is not None:
+                cmd.extend(['--reference', cache_path])
             cmd.extend(['--', sub.path])
             output_data = repo.git.execute(cmd, with_extended_output=True, with_stdout=True)
             display_git_output(output_data, verbose) @@ -269,7 +273,7 @@ def deinit_submodules(workspace, start_manifest, start_combo,
             _deinit(repo, deinit_list, verbose)
 
 
-def maintain_submodules(workspace, manifest, combo_name, verbose=False):
+def maintain_submodules(workspace, manifest, combo_name, verbose=False, cache_path=None):
     """
     Updates the submodules for a specific repo.
 
@@ -277,6 +281,7 @@ def maintain_submodules(workspace, manifest, combo_name, verbose=False):
         manifest   - The manifest parser object for the project.
         combo_name - The combination name to use for submodule maintenance.
         verbose    - Enable verbose messages.
+        cache_path - Path to the submodule cache repo.  A value of None indicates that no cache repo exists.
     """
     # Process each repo that may have submodules enabled
     print(strings.SUBMOD_INIT_UPDATE)
@@ -303,9 +308,9 @@ def maintain_submodules(workspace, manifest, combo_name, verbose=False):
 
         # Perform sync/update
         if len(repo_subs) == 0:
-            _update(repo, None, verbose)
+            _update(repo, None, verbose, cache_path=cache_path)
         else:
-            _update(repo, repo_subs, verbose)
+            _update(repo, repo_subs, verbose, cache_path=cache_path)
 
 
 if __name__ == '__main__':
--
2.21.0.windows.1