[edk2-staging/EdkRepo] [PATCH] EdkRepo: Initial commit of the Create Pin command

[edk2-staging/EdkRepo] [PATCH] EdkRepo: Initial commit of the Create Pin command

[Desimone, Ashley E]

Add edkrepo create-pin command allowing users to create
a pin file to record their workspace status at a specific
point in time.

Signed-off-by: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
---
 edkrepo/commands/arguments/create_pin_args.py |  21 +++++
 edkrepo/commands/create_pin_command.py        | 127 ++++++++++++++++++++++++++
 edkrepo/common/edkrepo_exception.py           |   6 +-
 edkrepo/common/humble.py                      |  17 +++-
 4 files changed, 169 insertions(+), 2 deletions(-)
 create mode 100644 edkrepo/commands/arguments/create_pin_args.py
 create mode 100644 edkrepo/commands/create_pin_command.py

diff --git a/edkrepo/commands/arguments/create_pin_args.py b/edkrepo/commands/arguments/create_pin_args.py
new file mode 100644
index 0000000..92958ef
--- /dev/null
+++ b/edkrepo/commands/arguments/create_pin_args.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+#
+## @file
+# create_pin_command.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+
+''' Contains the help and description strings for arguments in the create pin
+command meta data.
+'''
+
+COMMAND_DESCRIPTION = 'Creates a PIN file based on the current workspace state'
+NAME_HELP = ('The name of the PIN file. Extension must be .xml. File paths are '
+             'supported only if the --push option is not used.')
+DESCRIPTION_HELP = 'A short summary of the PIN file contents. Must be contained in ""'
+PUSH_HELP = ('Automatically commit and push the PIN file to the global manifest '
+             'repository at the location specified by the Pin-Path field in '
+             'the project manifest file.')
+
diff --git a/edkrepo/commands/create_pin_command.py b/edkrepo/commands/create_pin_command.py
new file mode 100644
index 0000000..df3fccc
--- /dev/null
+++ b/edkrepo/commands/create_pin_command.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+#
+## @file
+# create_pin_command.py
+#
+# Copyright (c) 2017 - 2020, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+
+import os
+from collections import namedtuple
+
+from git import Repo
+
+from edkrepo.commands.edkrepo_command import EdkrepoCommand
+import edkrepo.commands.arguments.create_pin_args as arguments
+from edkrepo.common.common_repo_functions import pull_latest_manifest_repo
+from edkrepo.common.edkrepo_exception import EdkrepoManifestInvalidException, EdkrepoInvalidParametersException
+from edkrepo.common.edkrepo_exception import EdkrepoWorkspaceCorruptException
+from edkrepo.common.humble import WRITING_PIN_FILE, GENERATING_PIN_DATA, GENERATING_REPO_DATA, BRANCH, COMMIT
+from edkrepo.common.humble import COMMIT_MESSAGE, PIN_PATH_NOT_PRESENT, PIN_FILE_ALREADY_EXISTS, PATH_AND_FILEPATH_USED
+from edkrepo.common.humble import MISSING_REPO
+from edkrepo.config.config_factory import get_workspace_manifest, get_workspace_path
+from edkrepo_manifest_parser.edk_manifest import ManifestXml
+
+
+class CreatePinCommand(EdkrepoCommand):
+    def __init__(self):
+        super().__init__()
+
+    def get_metadata(self):
+        metadata = {}
+        metadata['name'] = 'create-pin'
+        metadata['help-text'] = arguments.COMMAND_DESCRIPTION
+        metadata['alias'] = 'crp'
+        args = []
+        metadata['arguments'] = args
+        args.append({'name' : 'PinFileName',
+                     'positional' : True,
+                     'position' : 0,
+                     'required' : True,
+                     'help-text' : arguments.NAME_HELP})
+        args.append({'name' : 'Description',
+                     'positional' : True,
+                     'position' : 1,
+                     'required' : True,
+                     'help-text' : arguments.DESCRIPTION_HELP})
+        args.append({'name': 'push',
+                     'positional': False,
+                     'required': False,
+                     'help-text': arguments.PUSH_HELP})
+        return metadata
+
+    def run_command(self, args, config):
+        # Check if --push and file path provided
+        if args.push and os.path.dirname(args.PinFileName):
+            raise EdkrepoInvalidParametersException(PATH_AND_FILEPATH_USED)
+
+        pull_latest_manifest_repo(args, config)
+        workspace_path = get_workspace_path()
+        manifest = get_workspace_manifest()
+
+        # If the push flag is enabled use general_config.pin_path to determine global manifest relative location to save
+        # pin file to.
+        if args.push and manifest.general_config.pin_path is not None:
+            pin_dir = os.path.join(config['cfg_file'].manifest_repo_abs_local_path, os.path.normpath(manifest.general_config.pin_path))
+            pin_file_name = os.path.join(pin_dir, args.PinFileName)
+        elif args.push and manifest.general_config.pin_path is None:
+            raise EdkrepoManifestInvalidException(PIN_PATH_NOT_PRESENT)
+        # If not using the push flag ignore the local manifest's pin file field and set the pinname/path == to the file
+        # name provided. If a relative paths is provided save the file relative to the current working directory.
+        elif not args.push and os.path.isabs(os.path.normpath(args.PinFileName)):
+            pin_file_name = os.path.normpath(args.PinFileName)
+        elif not args.push:
+            pin_file_name = os.path.abspath(os.path.normpath(args.PinFileName))
+        # If the directory that the pin file is saved in does not exist create it and ensure pin file name uniqueness.
+        if os.path.isfile(pin_file_name):
+            raise EdkrepoInvalidParametersException(PIN_FILE_ALREADY_EXISTS)
+        if not os.path.exists(os.path.dirname(pin_file_name)):
+            os.mkdir(os.path.dirname(pin_file_name))
+
+        repo_sources = manifest.get_repo_sources(manifest.general_config.current_combo)
+
+        # get the repo sources and commit ids for the pin
+        print(GENERATING_PIN_DATA.format(manifest.project_info.codename, manifest.general_config.current_combo))
+        updated_repo_sources = []
+        for repo_source in repo_sources:
+            local_repo_path = os.path.join(workspace_path, repo_source.root)
+            if not os.path.exists(local_repo_path):
+                raise EdkrepoWorkspaceCorruptException(MISSING_REPO.format(repo_source.root))
+            repo = Repo(local_repo_path)
+            commit_id = repo.head.commit.hexsha
+            if args.verbose:
+                print(GENERATING_REPO_DATA.format(repo_source.root))
+                print(BRANCH.format(repo_source.branch))
+                print(COMMIT.format(commit_id))
+            updated_repo_source = repo_source._replace(commit=commit_id)
+            updated_repo_sources.append(updated_repo_source)
+
+        # create the pin
+        print(WRITING_PIN_FILE.format(pin_file_name))
+        manifest.generate_pin_xml(args.Description, manifest.general_config.current_combo, updated_repo_sources,
+                                  filename=pin_file_name)
+
+        # commit and push the pin file
+        if args.push:
+            manifest_repo = Repo(config['cfg_file'].manifest_repo_abs_local_path)
+            # Create a local branch with the same name as the pin file arg and check it out before attempting the push
+            # to master
+            master_branch = manifest_repo.active_branch
+            local_branch = manifest_repo.create_head(args.PinFileName)
+            manifest_repo.heads[local_branch.name].checkout()
+            manifest_repo.git.add(pin_file_name)
+            manifest_repo.git.commit(m=COMMIT_MESSAGE.format(manifest.project_info.codename, args.Description))
+            # Try to push if the push fails re update the manifest repo and rebase from master
+            try:
+                manifest_repo.git.push('origin', 'HEAD:master')
+            except:
+                manifest_repo.heads[master_branch.name].checkout()
+                origin = manifest_repo.remotes.origin
+                origin.pull()
+                manifest_repo.heads[local_branch.name].checkout()
+                manifest_repo.git.rebase('master')
+                manifest_repo.git.push('origin', 'HEAD:master')
+            finally:
+                manifest_repo.heads[master_branch.name].checkout()
+                manifest_repo.delete_head(local_branch, '-D')
\ No newline at end of file
diff --git a/edkrepo/common/edkrepo_exception.py b/edkrepo/common/edkrepo_exception.py
index 9d2b102..b6ea3dd 100644
--- a/edkrepo/common/edkrepo_exception.py
+++ b/edkrepo/common/edkrepo_exception.py
@@ -3,7 +3,7 @@
 ## @file
 # edkrepo_exception.py
 #
-# Copyright (c) 2017- 2019, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2017- 2020, Intel Corporation. All rights reserved.<BR>
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 #
 
@@ -74,6 +74,10 @@ class EdkrepoFoundMultipleException(EdkrepoException):
     def __init__(self, message):
         super().__init__(message, 117)
 
+class EdkrepoWorkspaceCorruptException(EdkrepoException):
+    def __ini__(self, message):
+        super().__init__(message, 118)
+
 class EdkrepoWarningException(EdkrepoException):
     def __init__(self, message):
         super().__init__(message, 123)
diff --git a/edkrepo/common/humble.py b/edkrepo/common/humble.py
index f0fbd29..5326e88 100644
--- a/edkrepo/common/humble.py
+++ b/edkrepo/common/humble.py
@@ -3,7 +3,7 @@
 ## @file
 # humble.py
 #
-# Copyright (c) 2017 - 2019, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2017 - 2020, Intel Corporation. All rights reserved.<BR>
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 #
 
@@ -147,3 +147,18 @@ VERIFY_PROJ_FAIL = 'Unable to verify the global manifest repository entry for pr
 
 # Git Command Error Messages
 GIT_CMD_ERROR = 'The git command: {} failed to complete successfully with the following errors.\n'
+
+# Error messages for create_pin_command.py
+CREATE_PIN_EXIT = 'Exiting without creating pin file ...'
+PIN_PATH_NOT_PRESENT = 'Pin Path not present in Manifest.xml ' + CREATE_PIN_EXIT
+PIN_FILE_ALREADY_EXISTS = 'A pin file with that name already exists for this project. Please rerun the command with a new filename. ' + CREATE_PIN_EXIT
+PATH_AND_FILEPATH_USED = 'Providing a file path for the PinFileName argument is not supported when using the --push flag. ' + CREATE_PIN_EXIT
+MISSING_REPO = 'The {} repository is missing from your workspace. ' + CREATE_PIN_EXIT
+
+# Informational messages for create_pin_command.py
+GENERATING_PIN_DATA = 'Generating pin data for {0} project based on {1} combination ...'
+GENERATING_REPO_DATA = 'Generating pin data for {0} repo:'
+BRANCH = '    Branch : {0}'
+COMMIT = '    Commit Id: {0}'
+WRITING_PIN_FILE = 'Writing pin file to {0} ...'
+COMMIT_MESSAGE = 'Pin file for project: {0} \nPin Description: {1}'
-- 
2.16.2.windows.1

Re: [edk2-staging/EdkRepo] [PATCH] EdkRepo: Initial commit of the Create Pin command

[Nate DeSimone]

Reviewed-by: Nate DeSimone <nathaniel.l.desimone@intel.com>

-----Original Message-----
From: Desimone, Ashley E <ashley.e.desimone@intel.com> 
Sent: Wednesday, March 11, 2020 3:33 PM
To: devel@edk2.groups.io
Cc: Desimone, Nathaniel L <nathaniel.l.desimone@intel.com>; Pandya, Puja <puja.pandya@intel.com>; Bjorge, Erik C <erik.c.bjorge@intel.com>
Subject: [edk2-staging/EdkRepo] [PATCH] EdkRepo: Initial commit of the Create Pin command

Add edkrepo create-pin command allowing users to create a pin file to record their workspace status at a specific point in time.

Signed-off-by: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
---
 edkrepo/commands/arguments/create_pin_args.py |  21 +++++
 edkrepo/commands/create_pin_command.py        | 127 ++++++++++++++++++++++++++
 edkrepo/common/edkrepo_exception.py           |   6 +-
 edkrepo/common/humble.py                      |  17 +++-
 4 files changed, 169 insertions(+), 2 deletions(-)  create mode 100644 edkrepo/commands/arguments/create_pin_args.py
 create mode 100644 edkrepo/commands/create_pin_command.py

diff --git a/edkrepo/commands/arguments/create_pin_args.py b/edkrepo/commands/arguments/create_pin_args.py
new file mode 100644
index 0000000..92958ef
--- /dev/null
+++ b/edkrepo/commands/arguments/create_pin_args.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+#
+## @file
+# create_pin_command.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR> # 
+SPDX-License-Identifier: BSD-2-Clause-Patent #
+
+''' Contains the help and description strings for arguments in the 
+create pin command meta data.
+'''
+
+COMMAND_DESCRIPTION = 'Creates a PIN file based on the current workspace state'
+NAME_HELP = ('The name of the PIN file. Extension must be .xml. File paths are '
+             'supported only if the --push option is not used.') 
+DESCRIPTION_HELP = 'A short summary of the PIN file contents. Must be contained in ""'
+PUSH_HELP = ('Automatically commit and push the PIN file to the global manifest '
+             'repository at the location specified by the Pin-Path field in '
+             'the project manifest file.')
+
diff --git a/edkrepo/commands/create_pin_command.py b/edkrepo/commands/create_pin_command.py
new file mode 100644
index 0000000..df3fccc
--- /dev/null
+++ b/edkrepo/commands/create_pin_command.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+#
+## @file
+# create_pin_command.py
+#
+# Copyright (c) 2017 - 2020, Intel Corporation. All rights 
+reserved.<BR> # SPDX-License-Identifier: BSD-2-Clause-Patent #
+
+import os
+from collections import namedtuple
+
+from git import Repo
+
+from edkrepo.commands.edkrepo_command import EdkrepoCommand import 
+edkrepo.commands.arguments.create_pin_args as arguments from 
+edkrepo.common.common_repo_functions import pull_latest_manifest_repo 
+from edkrepo.common.edkrepo_exception import 
+EdkrepoManifestInvalidException, EdkrepoInvalidParametersException from 
+edkrepo.common.edkrepo_exception import 
+EdkrepoWorkspaceCorruptException from edkrepo.common.humble import 
+WRITING_PIN_FILE, GENERATING_PIN_DATA, GENERATING_REPO_DATA, BRANCH, 
+COMMIT from edkrepo.common.humble import COMMIT_MESSAGE, 
+PIN_PATH_NOT_PRESENT, PIN_FILE_ALREADY_EXISTS, PATH_AND_FILEPATH_USED 
+from edkrepo.common.humble import MISSING_REPO from 
+edkrepo.config.config_factory import get_workspace_manifest, 
+get_workspace_path from edkrepo_manifest_parser.edk_manifest import 
+ManifestXml
+
+
+class CreatePinCommand(EdkrepoCommand):
+    def __init__(self):
+        super().__init__()
+
+    def get_metadata(self):
+        metadata = {}
+        metadata['name'] = 'create-pin'
+        metadata['help-text'] = arguments.COMMAND_DESCRIPTION
+        metadata['alias'] = 'crp'
+        args = []
+        metadata['arguments'] = args
+        args.append({'name' : 'PinFileName',
+                     'positional' : True,
+                     'position' : 0,
+                     'required' : True,
+                     'help-text' : arguments.NAME_HELP})
+        args.append({'name' : 'Description',
+                     'positional' : True,
+                     'position' : 1,
+                     'required' : True,
+                     'help-text' : arguments.DESCRIPTION_HELP})
+        args.append({'name': 'push',
+                     'positional': False,
+                     'required': False,
+                     'help-text': arguments.PUSH_HELP})
+        return metadata
+
+    def run_command(self, args, config):
+        # Check if --push and file path provided
+        if args.push and os.path.dirname(args.PinFileName):
+            raise 
+ EdkrepoInvalidParametersException(PATH_AND_FILEPATH_USED)
+
+        pull_latest_manifest_repo(args, config)
+        workspace_path = get_workspace_path()
+        manifest = get_workspace_manifest()
+
+        # If the push flag is enabled use general_config.pin_path to determine global manifest relative location to save
+        # pin file to.
+        if args.push and manifest.general_config.pin_path is not None:
+            pin_dir = os.path.join(config['cfg_file'].manifest_repo_abs_local_path, os.path.normpath(manifest.general_config.pin_path))
+            pin_file_name = os.path.join(pin_dir, args.PinFileName)
+        elif args.push and manifest.general_config.pin_path is None:
+            raise EdkrepoManifestInvalidException(PIN_PATH_NOT_PRESENT)
+        # If not using the push flag ignore the local manifest's pin file field and set the pinname/path == to the file
+        # name provided. If a relative paths is provided save the file relative to the current working directory.
+        elif not args.push and os.path.isabs(os.path.normpath(args.PinFileName)):
+            pin_file_name = os.path.normpath(args.PinFileName)
+        elif not args.push:
+            pin_file_name = os.path.abspath(os.path.normpath(args.PinFileName))
+        # If the directory that the pin file is saved in does not exist create it and ensure pin file name uniqueness.
+        if os.path.isfile(pin_file_name):
+            raise EdkrepoInvalidParametersException(PIN_FILE_ALREADY_EXISTS)
+        if not os.path.exists(os.path.dirname(pin_file_name)):
+            os.mkdir(os.path.dirname(pin_file_name))
+
+        repo_sources = 
+ manifest.get_repo_sources(manifest.general_config.current_combo)
+
+        # get the repo sources and commit ids for the pin
+        print(GENERATING_PIN_DATA.format(manifest.project_info.codename, manifest.general_config.current_combo))
+        updated_repo_sources = []
+        for repo_source in repo_sources:
+            local_repo_path = os.path.join(workspace_path, repo_source.root)
+            if not os.path.exists(local_repo_path):
+                raise EdkrepoWorkspaceCorruptException(MISSING_REPO.format(repo_source.root))
+            repo = Repo(local_repo_path)
+            commit_id = repo.head.commit.hexsha
+            if args.verbose:
+                print(GENERATING_REPO_DATA.format(repo_source.root))
+                print(BRANCH.format(repo_source.branch))
+                print(COMMIT.format(commit_id))
+            updated_repo_source = repo_source._replace(commit=commit_id)
+            updated_repo_sources.append(updated_repo_source)
+
+        # create the pin
+        print(WRITING_PIN_FILE.format(pin_file_name))
+        manifest.generate_pin_xml(args.Description, manifest.general_config.current_combo, updated_repo_sources,
+                                  filename=pin_file_name)
+
+        # commit and push the pin file
+        if args.push:
+            manifest_repo = Repo(config['cfg_file'].manifest_repo_abs_local_path)
+            # Create a local branch with the same name as the pin file arg and check it out before attempting the push
+            # to master
+            master_branch = manifest_repo.active_branch
+            local_branch = manifest_repo.create_head(args.PinFileName)
+            manifest_repo.heads[local_branch.name].checkout()
+            manifest_repo.git.add(pin_file_name)
+            manifest_repo.git.commit(m=COMMIT_MESSAGE.format(manifest.project_info.codename, args.Description))
+            # Try to push if the push fails re update the manifest repo and rebase from master
+            try:
+                manifest_repo.git.push('origin', 'HEAD:master')
+            except:
+                manifest_repo.heads[master_branch.name].checkout()
+                origin = manifest_repo.remotes.origin
+                origin.pull()
+                manifest_repo.heads[local_branch.name].checkout()
+                manifest_repo.git.rebase('master')
+                manifest_repo.git.push('origin', 'HEAD:master')
+            finally:
+                manifest_repo.heads[master_branch.name].checkout()
+                manifest_repo.delete_head(local_branch, '-D')
\ No newline at end of file
diff --git a/edkrepo/common/edkrepo_exception.py b/edkrepo/common/edkrepo_exception.py
index 9d2b102..b6ea3dd 100644
--- a/edkrepo/common/edkrepo_exception.py
+++ b/edkrepo/common/edkrepo_exception.py
@@ -3,7 +3,7 @@
 ## @file
 # edkrepo_exception.py
 #
-# Copyright (c) 2017- 2019, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2017- 2020, Intel Corporation. All rights reserved.<BR>
 # SPDX-License-Identifier: BSD-2-Clause-Patent  #
 
@@ -74,6 +74,10 @@ class EdkrepoFoundMultipleException(EdkrepoException):
     def __init__(self, message):
         super().__init__(message, 117)
 
+class EdkrepoWorkspaceCorruptException(EdkrepoException):
+    def __ini__(self, message):
+        super().__init__(message, 118)
+
 class EdkrepoWarningException(EdkrepoException):
     def __init__(self, message):
         super().__init__(message, 123)
diff --git a/edkrepo/common/humble.py b/edkrepo/common/humble.py index f0fbd29..5326e88 100644
--- a/edkrepo/common/humble.py
+++ b/edkrepo/common/humble.py
@@ -3,7 +3,7 @@
 ## @file
 # humble.py
 #
-# Copyright (c) 2017 - 2019, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2017 - 2020, Intel Corporation. All rights 
+reserved.<BR>
 # SPDX-License-Identifier: BSD-2-Clause-Patent  #
 
@@ -147,3 +147,18 @@ VERIFY_PROJ_FAIL = 'Unable to verify the global manifest repository entry for pr
 
 # Git Command Error Messages
 GIT_CMD_ERROR = 'The git command: {} failed to complete successfully with the following errors.\n'
+
+# Error messages for create_pin_command.py CREATE_PIN_EXIT = 'Exiting 
+without creating pin file ...'
+PIN_PATH_NOT_PRESENT = 'Pin Path not present in Manifest.xml ' + 
+CREATE_PIN_EXIT PIN_FILE_ALREADY_EXISTS = 'A pin file with that name 
+already exists for this project. Please rerun the command with a new 
+filename. ' + CREATE_PIN_EXIT PATH_AND_FILEPATH_USED = 'Providing a 
+file path for the PinFileName argument is not supported when using the 
+--push flag. ' + CREATE_PIN_EXIT MISSING_REPO = 'The {} repository is 
+missing from your workspace. ' + CREATE_PIN_EXIT
+
+# Informational messages for create_pin_command.py GENERATING_PIN_DATA 
+= 'Generating pin data for {0} project based on {1} combination ...'
+GENERATING_REPO_DATA = 'Generating pin data for {0} repo:'
+BRANCH = '    Branch : {0}'
+COMMIT = '    Commit Id: {0}'
+WRITING_PIN_FILE = 'Writing pin file to {0} ...'
+COMMIT_MESSAGE = 'Pin file for project: {0} \nPin Description: {1}'
--
2.16.2.windows.1