[edk2-staging/EdkRepo] [PATCH v1] EdkRepo: Improve removal of content with AlwaysExclude

[edk2-staging/EdkRepo] [PATCH v1] EdkRepo: Improve removal of content with AlwaysExclude

[Bjorge, Erik C]

The AlwaysExclude XML tag has existed but only removed entries from the
list of sparse objects.  Now items in the AlwaysExclude tag will be
actively removed by prefixing '!' to the entry.

Signed-off-by: Erik Bjorge <erik.c.bjorge@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Ashley DeSimone <ashley.e.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Prince Agyeman <prince.agyeman@intel.com>
---
 project_utils/sparse.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/project_utils/sparse.py b/project_utils/sparse.py
index b17d688..dbe2d80 100644
--- a/project_utils/sparse.py
+++ b/project_utils/sparse.py
@@ -3,7 +3,7 @@
 ## @file
 # sparse.py
 #
-# Copyright (c) 2017- 2019, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2017- 2020, Intel Corporation. All rights reserved.<BR>
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 #
 import os
@@ -552,11 +552,11 @@ class BuildInfo:
 
     def sparse_checkout(self, root=None, always_include=[], always_exclude=[]):
         """Performs a sparse checkout operation on a single repository"""
-        local_prune_data = set()
+        local_prune_data = []
         for item in always_include:
-            local_prune_data.add('/{}'.format(item))
+            local_prune_data.append('/{}'.format(item))
         for item in always_exclude:
-            local_prune_data.discard('/{}'.format(item))
+            local_prune_data.append('!/{}'.format(item))
         try:
             repo = git.Repo(root)
         except:
@@ -578,7 +578,11 @@ def process_sparse_checkout(workspace_root, repo_list, current_combo, manifest):
     workspace_list.extend([os.path.join(workspace_root, os.path.normpath(x.root)) for x in repo_list])
 
     # Filter sparse data entries that apply to the current combo or all combos
-    sparse_data = [x for x in manifest.sparse_data if x.combination is None or x.combination == current_combo]
+    # Build list in three steps (all, repo, combo) to make sure the priority is correct
+    sparse_data = []
+    sparse_data.extend([x for x in manifest.sparse_data if x.remote_name is None and x.combination is None])
+    sparse_data.extend([x for x in manifest.sparse_data if x.remote_name is not None and x.combination is None])
+    sparse_data.extend([x for x in manifest.sparse_data if x.remote_name is not None and x.combination == current_combo])
 
     # Create object that processes build information.
     build_info = BuildInfo(workspace_list)
@@ -602,8 +606,8 @@ if __name__ == "__main__":
     # Program Information
     #
     __title__ = 'Sparse Checkout'
-    __version__ = '0.02.00'
-    __copyright__ = 'Copyright (c) 2017, Intel Corporation. All rights reserved.'
+    __version__ = '0.03.00'
+    __copyright__ = 'Copyright (c) 2017 - 2020, Intel Corporation. All rights reserved.'
 
     #
     # Processes command line arguments
-- 
2.21.0.windows.1

Re: [edk2-staging/EdkRepo] [PATCH v1] EdkRepo: Improve removal of content with AlwaysExclude

[Ashley E Desimone]

Reviewed-by: Ashley DeSimone <ashley.e.desimone@intel.com>

-----Original Message-----
From: Bjorge, Erik C <erik.c.bjorge@intel.com> 
Sent: Tuesday, April 21, 2020 4:27 PM
To: devel@edk2.groups.io
Cc: Desimone, Nathaniel L <nathaniel.l.desimone@intel.com>; Desimone, Ashley E <ashley.e.desimone@intel.com>; Pandya, Puja <puja.pandya@intel.com>; Bret Barkelew <Bret.Barkelew@microsoft.com>; Agyeman, Prince <prince.agyeman@intel.com>
Subject: [edk2-staging/EdkRepo] [PATCH v1] EdkRepo: Improve removal of content with AlwaysExclude

The AlwaysExclude XML tag has existed but only removed entries from the list of sparse objects.  Now items in the AlwaysExclude tag will be actively removed by prefixing '!' to the entry.

Signed-off-by: Erik Bjorge <erik.c.bjorge@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Ashley DeSimone <ashley.e.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Prince Agyeman <prince.agyeman@intel.com>
---
 project_utils/sparse.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/project_utils/sparse.py b/project_utils/sparse.py index b17d688..dbe2d80 100644
--- a/project_utils/sparse.py
+++ b/project_utils/sparse.py
@@ -3,7 +3,7 @@
 ## @file
 # sparse.py
 #
-# Copyright (c) 2017- 2019, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2017- 2020, Intel Corporation. All rights reserved.<BR>
 # SPDX-License-Identifier: BSD-2-Clause-Patent  #  import os @@ -552,11 +552,11 @@ class BuildInfo:
 
     def sparse_checkout(self, root=None, always_include=[], always_exclude=[]):
         """Performs a sparse checkout operation on a single repository"""
-        local_prune_data = set()
+        local_prune_data = []
         for item in always_include:
-            local_prune_data.add('/{}'.format(item))
+            local_prune_data.append('/{}'.format(item))
         for item in always_exclude:
-            local_prune_data.discard('/{}'.format(item))
+            local_prune_data.append('!/{}'.format(item))
         try:
             repo = git.Repo(root)
         except:
@@ -578,7 +578,11 @@ def process_sparse_checkout(workspace_root, repo_list, current_combo, manifest):
     workspace_list.extend([os.path.join(workspace_root, os.path.normpath(x.root)) for x in repo_list])
 
     # Filter sparse data entries that apply to the current combo or all combos
-    sparse_data = [x for x in manifest.sparse_data if x.combination is None or x.combination == current_combo]
+    # Build list in three steps (all, repo, combo) to make sure the priority is correct
+    sparse_data = []
+    sparse_data.extend([x for x in manifest.sparse_data if x.remote_name is None and x.combination is None])
+    sparse_data.extend([x for x in manifest.sparse_data if x.remote_name is not None and x.combination is None])
+    sparse_data.extend([x for x in manifest.sparse_data if 
+ x.remote_name is not None and x.combination == current_combo])
 
     # Create object that processes build information.
     build_info = BuildInfo(workspace_list) @@ -602,8 +606,8 @@ if __name__ == "__main__":
     # Program Information
     #
     __title__ = 'Sparse Checkout'
-    __version__ = '0.02.00'
-    __copyright__ = 'Copyright (c) 2017, Intel Corporation. All rights reserved.'
+    __version__ = '0.03.00'
+    __copyright__ = 'Copyright (c) 2017 - 2020, Intel Corporation. All rights reserved.'
 
     #
     # Processes command line arguments
--
2.21.0.windows.1

Re: [edk2-devel] [edk2-staging/EdkRepo] [PATCH v1] EdkRepo: Improve removal of content with AlwaysExclude

[Nate DeSimone]

Reviewed-by: Nate DeSimone <nathaniel.l.desimone@intel.com>

-----Original Message-----
From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Bjorge, Erik C
Sent: Tuesday, April 21, 2020 4:27 PM
To: devel@edk2.groups.io
Cc: Desimone, Nathaniel L <nathaniel.l.desimone@intel.com>; Desimone, Ashley E <ashley.e.desimone@intel.com>; Pandya, Puja <puja.pandya@intel.com>; Bret Barkelew <Bret.Barkelew@microsoft.com>; Agyeman, Prince <prince.agyeman@intel.com>
Subject: [edk2-devel] [edk2-staging/EdkRepo] [PATCH v1] EdkRepo: Improve removal of content with AlwaysExclude

The AlwaysExclude XML tag has existed but only removed entries from the list of sparse objects.  Now items in the AlwaysExclude tag will be actively removed by prefixing '!' to the entry.

Signed-off-by: Erik Bjorge <erik.c.bjorge@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Ashley DeSimone <ashley.e.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Prince Agyeman <prince.agyeman@intel.com>
---
 project_utils/sparse.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/project_utils/sparse.py b/project_utils/sparse.py index b17d688..dbe2d80 100644
--- a/project_utils/sparse.py
+++ b/project_utils/sparse.py
@@ -3,7 +3,7 @@
 ## @file
 # sparse.py
 #
-# Copyright (c) 2017- 2019, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2017- 2020, Intel Corporation. All rights reserved.<BR>
 # SPDX-License-Identifier: BSD-2-Clause-Patent  #  import os @@ -552,11 +552,11 @@ class BuildInfo:
 
     def sparse_checkout(self, root=None, always_include=[], always_exclude=[]):
         """Performs a sparse checkout operation on a single repository"""
-        local_prune_data = set()
+        local_prune_data = []
         for item in always_include:
-            local_prune_data.add('/{}'.format(item))
+            local_prune_data.append('/{}'.format(item))
         for item in always_exclude:
-            local_prune_data.discard('/{}'.format(item))
+            local_prune_data.append('!/{}'.format(item))
         try:
             repo = git.Repo(root)
         except:
@@ -578,7 +578,11 @@ def process_sparse_checkout(workspace_root, repo_list, current_combo, manifest):
     workspace_list.extend([os.path.join(workspace_root, os.path.normpath(x.root)) for x in repo_list])
 
     # Filter sparse data entries that apply to the current combo or all combos
-    sparse_data = [x for x in manifest.sparse_data if x.combination is None or x.combination == current_combo]
+    # Build list in three steps (all, repo, combo) to make sure the priority is correct
+    sparse_data = []
+    sparse_data.extend([x for x in manifest.sparse_data if x.remote_name is None and x.combination is None])
+    sparse_data.extend([x for x in manifest.sparse_data if x.remote_name is not None and x.combination is None])
+    sparse_data.extend([x for x in manifest.sparse_data if 
+ x.remote_name is not None and x.combination == current_combo])
 
     # Create object that processes build information.
     build_info = BuildInfo(workspace_list) @@ -602,8 +606,8 @@ if __name__ == "__main__":
     # Program Information
     #
     __title__ = 'Sparse Checkout'
-    __version__ = '0.02.00'
-    __copyright__ = 'Copyright (c) 2017, Intel Corporation. All rights reserved.'
+    __version__ = '0.03.00'
+    __copyright__ = 'Copyright (c) 2017 - 2020, Intel Corporation. All rights reserved.'
 
     #
     # Processes command line arguments
--
2.21.0.windows.1

Re: [edk2-devel] [edk2-staging/EdkRepo] [PATCH v1] EdkRepo: Improve removal of content with AlwaysExclude

[Nate DeSimone]

Pushed: https://github.com/tianocore/edk2-staging/commit/8caf3640

-----Original Message-----
From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Bjorge, Erik C
Sent: Tuesday, April 21, 2020 4:27 PM
To: devel@edk2.groups.io
Cc: Desimone, Nathaniel L <nathaniel.l.desimone@intel.com>; Desimone, Ashley E <ashley.e.desimone@intel.com>; Pandya, Puja <puja.pandya@intel.com>; Bret Barkelew <Bret.Barkelew@microsoft.com>; Agyeman, Prince <prince.agyeman@intel.com>
Subject: [edk2-devel] [edk2-staging/EdkRepo] [PATCH v1] EdkRepo: Improve removal of content with AlwaysExclude

The AlwaysExclude XML tag has existed but only removed entries from the list of sparse objects.  Now items in the AlwaysExclude tag will be actively removed by prefixing '!' to the entry.

Signed-off-by: Erik Bjorge <erik.c.bjorge@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Ashley DeSimone <ashley.e.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Prince Agyeman <prince.agyeman@intel.com>
---
 project_utils/sparse.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/project_utils/sparse.py b/project_utils/sparse.py index b17d688..dbe2d80 100644
--- a/project_utils/sparse.py
+++ b/project_utils/sparse.py
@@ -3,7 +3,7 @@
 ## @file
 # sparse.py
 #
-# Copyright (c) 2017- 2019, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2017- 2020, Intel Corporation. All rights reserved.<BR>
 # SPDX-License-Identifier: BSD-2-Clause-Patent  #  import os @@ -552,11 +552,11 @@ class BuildInfo:
 
     def sparse_checkout(self, root=None, always_include=[], always_exclude=[]):
         """Performs a sparse checkout operation on a single repository"""
-        local_prune_data = set()
+        local_prune_data = []
         for item in always_include:
-            local_prune_data.add('/{}'.format(item))
+            local_prune_data.append('/{}'.format(item))
         for item in always_exclude:
-            local_prune_data.discard('/{}'.format(item))
+            local_prune_data.append('!/{}'.format(item))
         try:
             repo = git.Repo(root)
         except:
@@ -578,7 +578,11 @@ def process_sparse_checkout(workspace_root, repo_list, current_combo, manifest):
     workspace_list.extend([os.path.join(workspace_root, os.path.normpath(x.root)) for x in repo_list])
 
     # Filter sparse data entries that apply to the current combo or all combos
-    sparse_data = [x for x in manifest.sparse_data if x.combination is None or x.combination == current_combo]
+    # Build list in three steps (all, repo, combo) to make sure the priority is correct
+    sparse_data = []
+    sparse_data.extend([x for x in manifest.sparse_data if x.remote_name is None and x.combination is None])
+    sparse_data.extend([x for x in manifest.sparse_data if x.remote_name is not None and x.combination is None])
+    sparse_data.extend([x for x in manifest.sparse_data if 
+ x.remote_name is not None and x.combination == current_combo])
 
     # Create object that processes build information.
     build_info = BuildInfo(workspace_list) @@ -602,8 +606,8 @@ if __name__ == "__main__":
     # Program Information
     #
     __title__ = 'Sparse Checkout'
-    __version__ = '0.02.00'
-    __copyright__ = 'Copyright (c) 2017, Intel Corporation. All rights reserved.'
+    __version__ = '0.03.00'
+    __copyright__ = 'Copyright (c) 2017 - 2020, Intel Corporation. All rights reserved.'
 
     #
     # Processes command line arguments
--
2.21.0.windows.1