[edk2-staging/EdkRepo] [PATCH V2 0/2] EdkRepo: Add support for SUBST drives

[edk2-staging/EdkRepo] [PATCH V2 0/2] EdkRepo: Add support for SUBST drives

[Nate DeSimone]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=UTF-8, Size: 1306 bytes --]

Changes in V2:
- Changed get_subst_drive_list() to get_subst_drive_dict()

EdkRepo currently does not handle virtual drives created using the SUBST command.
Specifically, when cloning or syncing a project to a subst drive the includeIf statements
that redirect submodule fetches to mirror servers will be generated with the
subst drive information.  This causes git to not activate the includeif since
it specifies the subst path and not the actual path.

To resolve this, EdkRepo will now enumerate the virtual drives created by SUBST and if
the current workspace is on a SUBST virtual drive EdkRepo will convert the workspace
path to the path on the real volume.

Cc: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Prince Agyeman <prince.agyeman@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
Signed-off-by: Nate DeSimone <nathaniel.l.desimone@intel.com>

Nate DeSimone (2):
  EdkRepo: Add function to enumerate subst drives
  EdkRepo: Add support for subst drives

 edkrepo/commands/clone_command.py |  6 ++++
 edkrepo/common/pathfix.py         | 50 ++++++++++++++++++++++++++++++-
 edkrepo/config/config_factory.py  | 10 ++++++-
 3 files changed, 64 insertions(+), 2 deletions(-)

-- 
2.27.0.windows.1

[edk2-staging/EdkRepo] [PATCH V2 1/2] EdkRepo: Add function to enumerate subst drives

[Nate DeSimone]

Cc: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Prince Agyeman <prince.agyeman@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
Signed-off-by: Nate DeSimone <nathaniel.l.desimone@intel.com>
---
 edkrepo/common/pathfix.py | 50 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 49 insertions(+), 1 deletion(-)

diff --git a/edkrepo/common/pathfix.py b/edkrepo/common/pathfix.py
index 1a9c20f..2775442 100644
--- a/edkrepo/common/pathfix.py
+++ b/edkrepo/common/pathfix.py
@@ -3,7 +3,7 @@
 ## @file
 # checkout_command.py
 #
-# Copyright (c) 2018- 2020, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2018 - 2020, Intel Corporation. All rights reserved.<BR>
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 #
 import os
@@ -11,6 +11,7 @@ import sys
 if sys.platform == "win32":
     from ctypes import windll, POINTER, byref, GetLastError, Structure, WinError
     from ctypes import c_void_p, c_ushort, c_int,  c_ulong, c_wchar, c_wchar_p
+    from ctypes import create_unicode_buffer
 
 def _is_wow64_process():
     kernel32 = windll.kernel32
@@ -211,3 +212,50 @@ def expanduser(path):
         userhome = os.path.join(os.path.dirname(userhome), path[1:i])
 
     return userhome + path[i:]
+
+def get_subst_drive_dict():
+    if sys.platform != "win32":
+        return {}
+    def _query_subst_drive(drive_letter):
+        kernel32 = windll.kernel32
+        QueryDosDevice = kernel32.QueryDosDeviceW
+        QueryDosDevice.argtypes = [c_wchar_p, c_wchar_p, c_ulong]
+        QueryDosDevice.restype = c_ulong
+        MAX_PATH = 260
+
+        if len(drive_letter) > 1 or len(drive_letter) == 0:
+            raise ValueError("Bad drive letter")
+        drive = '{}:'.format(drive_letter.upper())
+        drive_buffer = create_unicode_buffer(drive)
+        target_path_buffer_size = c_ulong(MAX_PATH)
+        target_path_buffer = create_unicode_buffer(target_path_buffer_size.value)
+        while True:
+            count = QueryDosDevice(drive_buffer, target_path_buffer, target_path_buffer_size)
+            if count == 0:
+                last_error = GetLastError()
+                if last_error == 122: #ERROR_INSUFFICIENT_BUFFER
+                    #Increase the buffer size and try again
+                    target_path_buffer_size = c_ulong((target_path_buffer_size.value * 161) / 100)
+                    target_path_buffer = create_unicode_buffer(target_path_buffer_size.value)
+                elif last_error == 2: #ERROR_FILE_NOT_FOUND
+                    #This is an invalid drive, return an empty string
+                    return ''
+                else:
+                    raise WinError(last_error)
+            else:
+                break
+        target_path = target_path_buffer.value
+        if len(target_path) > 4 and target_path[0:4] == '\\??\\':
+            if (ord(target_path[4]) >= ord('A') and ord(target_path[4]) <= ord('Z')) or \
+                (ord(target_path[4]) >= ord('a') and ord(target_path[4]) <= ord('z')):
+                #This is a SUBST'd drive, return the path
+                return target_path[4:].strip()
+        #This is a non-SUBST'd (aka real) drive, return an empty string
+        return ''
+    subst_dict = {}
+    for index in range(26):
+        drive_letter = chr(ord('A') + index)
+        target_path = _query_subst_drive(drive_letter)
+        if target_path != '':
+            subst_dict[drive_letter] = target_path
+    return subst_dict
-- 
2.27.0.windows.1

[edk2-staging/EdkRepo] [PATCH V2 2/2] EdkRepo: Add support for subst drives

[Nate DeSimone]

get_workspace_path() now converts a virtual drive path
to a real path before any git repo operations are done.

Cc: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Prince Agyeman <prince.agyeman@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
Signed-off-by: Nate DeSimone <nathaniel.l.desimone@intel.com>
---
 edkrepo/commands/clone_command.py |  6 ++++++
 edkrepo/config/config_factory.py  | 10 +++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/edkrepo/commands/clone_command.py b/edkrepo/commands/clone_command.py
index f638090..be5ad86 100644
--- a/edkrepo/commands/clone_command.py
+++ b/edkrepo/commands/clone_command.py
@@ -20,6 +20,7 @@ from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException,
 from edkrepo.common.edkrepo_exception import EdkrepoManifestNotFoundException
 from edkrepo.common.humble import CLONE_INVALID_WORKSPACE, CLONE_INVALID_PROJECT_ARG, CLONE_INVALID_COMBO_ARG
 from edkrepo.common.humble import SPARSE_CHECKOUT, CLONE_INVALID_LOCAL_ROOTS
+from edkrepo.common.pathfix import get_subst_drive_dict
 from edkrepo.common.workspace_maintenance.workspace_maintenance import case_insensitive_single_match
 from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import pull_all_manifest_repos, find_project_in_all_indices
 from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list_available_manifest_repos
@@ -77,6 +78,11 @@ class CloneCommand(EdkrepoCommand):
             workspace_dir = os.getcwd()
         else:
             workspace_dir = os.path.abspath(workspace_dir)
+        subst = get_subst_drive_dict()
+        for drive in subst.keys():
+            if '{}:'.format(drive) == os.path.splitdrive(workspace_dir)[0].upper():
+                workspace_dir = os.path.join(subst[drive], os.path.splitdrive(workspace_dir)[1][1:])
+                workspace_dir = os.path.normpath(workspace_dir)
         if os.path.isdir(workspace_dir) and os.listdir(workspace_dir):
             raise EdkrepoInvalidParametersException(CLONE_INVALID_WORKSPACE)
         if not os.path.isdir(workspace_dir):
diff --git a/edkrepo/config/config_factory.py b/edkrepo/config/config_factory.py
index a82a438..1cb6eb7 100644
--- a/edkrepo/config/config_factory.py
+++ b/edkrepo/config/config_factory.py
@@ -11,13 +11,16 @@ import os
 import sys
 import configparser
 import collections
-from ctypes import *
+if sys.platform == "win32":
+    from ctypes import oledll, c_void_p, c_uint32, c_wchar_p
+    from ctypes import create_unicode_buffer
 
 import edkrepo.config.humble.config_factory_humble as humble
 from edkrepo.common.edkrepo_exception import EdkrepoGlobalConfigNotFoundException, EdkrepoConfigFileInvalidException
 from edkrepo.common.edkrepo_exception import EdkrepoWorkspaceInvalidException, EdkrepoGlobalDataDirectoryNotFoundException
 from edkrepo.common.edkrepo_exception import EdkrepoConfigFileReadOnlyException
 from edkrepo.common.humble import MIRROR_PRIMARY_REPOS_MISSING, MIRROR_DECODE_WARNING, MAX_PATCH_SET_INVALID
+from edkrepo.common.pathfix import get_subst_drive_dict
 from edkrepo_manifest_parser import edk_manifest
 from edkrepo.common.pathfix import expanduser
 
@@ -238,6 +241,11 @@ def get_workspace_path():
     while True:
         if os.path.isdir(os.path.join(path, "repo")):
             if os.path.isfile(os.path.join(os.path.join(path, "repo"), "Manifest.xml")):
+                if sys.platform == "win32":
+                    subst = get_subst_drive_dict()
+                    for drive in subst.keys():
+                        if '{}:'.format(drive) == os.path.splitdrive(path)[0].upper():
+                            path = os.path.join(subst[drive], os.path.splitdrive(path)[1][1:])
                 return path
         if os.path.dirname(path) == path:
             break
-- 
2.27.0.windows.1

Re: [edk2-devel] [edk2-staging/EdkRepo] [PATCH V2 2/2] EdkRepo: Add support for subst drives

[Carsey, Jaben]

Instead of looping over the dictionary, could we convert the target drive and see if it's in the dictionary? 

I am thinking like remove the : from os.path.splitdrive(workspace_dir)[0].upper() string and see if that's in the dictionary?


> -----Original Message-----
> From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Nate
> DeSimone
> Sent: Monday, August 31, 2020 12:28 PM
> To: devel@edk2.groups.io
> Cc: Desimone, Ashley E <ashley.e.desimone@intel.com>; Pandya, Puja
> <puja.pandya@intel.com>; Bret Barkelew <Bret.Barkelew@microsoft.com>;
> Agyeman, Prince <prince.agyeman@intel.com>; Bjorge, Erik C
> <erik.c.bjorge@intel.com>
> Subject: [edk2-devel] [edk2-staging/EdkRepo] [PATCH V2 2/2] EdkRepo: Add
> support for subst drives
> 
> get_workspace_path() now converts a virtual drive path to a real path before
> any git repo operations are done.
> 
> Cc: Ashley E Desimone <ashley.e.desimone@intel.com>
> Cc: Puja Pandya <puja.pandya@intel.com>
> Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
> Cc: Prince Agyeman <prince.agyeman@intel.com>
> Cc: Erik Bjorge <erik.c.bjorge@intel.com>
> Signed-off-by: Nate DeSimone <nathaniel.l.desimone@intel.com>
> ---
>  edkrepo/commands/clone_command.py |  6 ++++++
> edkrepo/config/config_factory.py  | 10 +++++++++-
>  2 files changed, 15 insertions(+), 1 deletion(-)
> 
> diff --git a/edkrepo/commands/clone_command.py
> b/edkrepo/commands/clone_command.py
> index f638090..be5ad86 100644
> --- a/edkrepo/commands/clone_command.py
> +++ b/edkrepo/commands/clone_command.py
> @@ -20,6 +20,7 @@ from edkrepo.common.edkrepo_exception import
> EdkrepoInvalidParametersException,
>  from edkrepo.common.edkrepo_exception import
> EdkrepoManifestNotFoundException  from edkrepo.common.humble
> import CLONE_INVALID_WORKSPACE, CLONE_INVALID_PROJECT_ARG,
> CLONE_INVALID_COMBO_ARG  from edkrepo.common.humble import
> SPARSE_CHECKOUT, CLONE_INVALID_LOCAL_ROOTS
> +from edkrepo.common.pathfix import get_subst_drive_dict
>  from edkrepo.common.workspace_maintenance.workspace_maintenance
> import case_insensitive_single_match  from
> edkrepo.common.workspace_maintenance.manifest_repos_maintenance
> import pull_all_manifest_repos, find_project_in_all_indices  from
> edkrepo.common.workspace_maintenance.manifest_repos_maintenance
> import list_available_manifest_repos @@ -77,6 +78,11 @@ class
> CloneCommand(EdkrepoCommand):
>              workspace_dir = os.getcwd()
>          else:
>              workspace_dir = os.path.abspath(workspace_dir)
> +        subst = get_subst_drive_dict()
> +        for drive in subst.keys():
> +            if '{}:'.format(drive) == os.path.splitdrive(workspace_dir)[0].upper():
> +                workspace_dir = os.path.join(subst[drive],
> os.path.splitdrive(workspace_dir)[1][1:])
> +                workspace_dir = os.path.normpath(workspace_dir)
>          if os.path.isdir(workspace_dir) and os.listdir(workspace_dir):
>              raise
> EdkrepoInvalidParametersException(CLONE_INVALID_WORKSPACE)
>          if not os.path.isdir(workspace_dir):
> diff --git a/edkrepo/config/config_factory.py
> b/edkrepo/config/config_factory.py
> index a82a438..1cb6eb7 100644
> --- a/edkrepo/config/config_factory.py
> +++ b/edkrepo/config/config_factory.py
> @@ -11,13 +11,16 @@ import os
>  import sys
>  import configparser
>  import collections
> -from ctypes import *
> +if sys.platform == "win32":
> +    from ctypes import oledll, c_void_p, c_uint32, c_wchar_p
> +    from ctypes import create_unicode_buffer
> 
>  import edkrepo.config.humble.config_factory_humble as humble  from
> edkrepo.common.edkrepo_exception import
> EdkrepoGlobalConfigNotFoundException, EdkrepoConfigFileInvalidException
> from edkrepo.common.edkrepo_exception import
> EdkrepoWorkspaceInvalidException,
> EdkrepoGlobalDataDirectoryNotFoundException
>  from edkrepo.common.edkrepo_exception import
> EdkrepoConfigFileReadOnlyException
>  from edkrepo.common.humble import
> MIRROR_PRIMARY_REPOS_MISSING, MIRROR_DECODE_WARNING,
> MAX_PATCH_SET_INVALID
> +from edkrepo.common.pathfix import get_subst_drive_dict
>  from edkrepo_manifest_parser import edk_manifest  from
> edkrepo.common.pathfix import expanduser
> 
> @@ -238,6 +241,11 @@ def get_workspace_path():
>      while True:
>          if os.path.isdir(os.path.join(path, "repo")):
>              if os.path.isfile(os.path.join(os.path.join(path, "repo"),
> "Manifest.xml")):
> +                if sys.platform == "win32":
> +                    subst = get_subst_drive_dict()
> +                    for drive in subst.keys():
> +                        if '{}:'.format(drive) == os.path.splitdrive(path)[0].upper():
> +                            path = os.path.join(subst[drive],
> + os.path.splitdrive(path)[1][1:])
>                  return path
>          if os.path.dirname(path) == path:
>              break
> --
> 2.27.0.windows.1
> 
> 
>