[edk2-staging/EdkRepo] [PATCH V1 0/2] EdkRepo: Add support for SUBST drives

[edk2-staging/EdkRepo] [PATCH V1 0/2] EdkRepo: Add support for SUBST drives

[Nate DeSimone]

EdkRepo currently does not handle virtual drives created using the SUBST command.
Specifically, when cloning or syncing a project to a subst drive the includeIf statements
that redirect submodule fetches to mirror servers will be generated with the
subst drive information.  This causes git to not activate the includeif since
it specifies the subst path and not the actual path.

To resolve this, EdkRepo will now enumerate the virtual drives created by SUBST and if
the current workspace is on a SUBST virtual drive EdkRepo will convert the workspace
path to the path on the real volume.

Cc: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Prince Agyeman <prince.agyeman@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
Signed-off-by: Nate DeSimone <nathaniel.l.desimone@intel.com>

Nate DeSimone (2):
  EdkRepo: Add function to enumerate subst drives
  EdkRepo: Add support for subst drives

 edkrepo/commands/clone_command.py |  6 ++++
 edkrepo/common/pathfix.py         | 50 ++++++++++++++++++++++++++++++-
 edkrepo/config/config_factory.py  | 10 ++++++-
 3 files changed, 64 insertions(+), 2 deletions(-)

-- 
2.27.0.windows.1

[edk2-staging/EdkRepo] [PATCH V1 1/2] EdkRepo: Add function to enumerate subst drives

[Nate DeSimone]

Cc: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Prince Agyeman <prince.agyeman@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
Signed-off-by: Nate DeSimone <nathaniel.l.desimone@intel.com>
---
 edkrepo/common/pathfix.py | 50 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 49 insertions(+), 1 deletion(-)

diff --git a/edkrepo/common/pathfix.py b/edkrepo/common/pathfix.py
index 1a9c20f..69ed1a4 100644
--- a/edkrepo/common/pathfix.py
+++ b/edkrepo/common/pathfix.py
@@ -3,7 +3,7 @@
 ## @file
 # checkout_command.py
 #
-# Copyright (c) 2018- 2020, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2018 - 2020, Intel Corporation. All rights reserved.<BR>
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 #
 import os
@@ -11,6 +11,7 @@ import sys
 if sys.platform == "win32":
     from ctypes import windll, POINTER, byref, GetLastError, Structure, WinError
     from ctypes import c_void_p, c_ushort, c_int,  c_ulong, c_wchar, c_wchar_p
+    from ctypes import create_unicode_buffer
 
 def _is_wow64_process():
     kernel32 = windll.kernel32
@@ -211,3 +212,50 @@ def expanduser(path):
         userhome = os.path.join(os.path.dirname(userhome), path[1:i])
 
     return userhome + path[i:]
+
+def get_subst_drive_list():
+    if sys.platform != "win32":
+        return {}
+    def _query_subst_drive(drive_letter):
+        kernel32 = windll.kernel32
+        QueryDosDevice = kernel32.QueryDosDeviceW
+        QueryDosDevice.argtypes = [c_wchar_p, c_wchar_p, c_ulong]
+        QueryDosDevice.restype = c_ulong
+        MAX_PATH = 260
+
+        if len(drive_letter) > 1 or len(drive_letter) == 0:
+            raise ValueError("Bad drive letter")
+        drive = '{}:'.format(drive_letter.upper())
+        drive_buffer = create_unicode_buffer(drive)
+        target_path_buffer_size = c_ulong(MAX_PATH)
+        target_path_buffer = create_unicode_buffer(target_path_buffer_size.value)
+        while True:
+            count = QueryDosDevice(drive_buffer, target_path_buffer, target_path_buffer_size)
+            if count == 0:
+                last_error = GetLastError()
+                if last_error == 122: #ERROR_INSUFFICIENT_BUFFER
+                    #Increase the buffer size and try again
+                    target_path_buffer_size = c_ulong((target_path_buffer_size.value * 161) / 100)
+                    target_path_buffer = create_unicode_buffer(target_path_buffer_size.value)
+                elif last_error == 2: #ERROR_FILE_NOT_FOUND
+                    #This is an invalid drive, return an empty string
+                    return ''
+                else:
+                    raise WinError(last_error)
+            else:
+                break
+        target_path = target_path_buffer.value
+        if len(target_path) > 4 and target_path[0:4] == '\\??\\':
+            if (ord(target_path[4]) >= ord('A') and ord(target_path[4]) <= ord('Z')) or \
+                (ord(target_path[4]) >= ord('a') and ord(target_path[4]) <= ord('z')):
+                #This is a SUBST'd drive, return the path
+                return target_path[4:].strip()
+        #This is a non-SUBST'd (aka real) drive, return an empty string
+        return ''
+    subst_dict = {}
+    for index in range(26):
+        drive_letter = chr(ord('A') + index)
+        target_path = _query_subst_drive(drive_letter)
+        if target_path != '':
+            subst_dict[drive_letter] = target_path
+    return subst_dict
-- 
2.27.0.windows.1

Re: [edk2-staging/EdkRepo] [PATCH V1 1/2] EdkRepo: Add function to enumerate subst drives

[Ashley E Desimone]

Hi Nate,

Since get_subst_drive_list() returns a dict I would prefer that we change the name of the function to reflect that.

Thanks,
Ashley 

-----Original Message-----
From: Desimone, Nathaniel L <nathaniel.l.desimone@intel.com> 
Sent: Wednesday, August 5, 2020 10:47 PM
To: devel@edk2.groups.io
Cc: Desimone, Ashley E <ashley.e.desimone@intel.com>; Pandya, Puja <puja.pandya@intel.com>; Bret Barkelew <Bret.Barkelew@microsoft.com>; Agyeman, Prince <prince.agyeman@intel.com>; Bjorge, Erik C <erik.c.bjorge@intel.com>
Subject: [edk2-staging/EdkRepo] [PATCH V1 1/2] EdkRepo: Add function to enumerate subst drives

Cc: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Prince Agyeman <prince.agyeman@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
Signed-off-by: Nate DeSimone <nathaniel.l.desimone@intel.com>
---
 edkrepo/common/pathfix.py | 50 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 49 insertions(+), 1 deletion(-)

diff --git a/edkrepo/common/pathfix.py b/edkrepo/common/pathfix.py index 1a9c20f..69ed1a4 100644
--- a/edkrepo/common/pathfix.py
+++ b/edkrepo/common/pathfix.py
@@ -3,7 +3,7 @@
 ## @file
 # checkout_command.py
 #
-# Copyright (c) 2018- 2020, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2018 - 2020, Intel Corporation. All rights 
+reserved.<BR>
 # SPDX-License-Identifier: BSD-2-Clause-Patent  #  import os @@ -11,6 +11,7 @@ import sys  if sys.platform == "win32":
     from ctypes import windll, POINTER, byref, GetLastError, Structure, WinError
     from ctypes import c_void_p, c_ushort, c_int,  c_ulong, c_wchar, c_wchar_p
+    from ctypes import create_unicode_buffer
 
 def _is_wow64_process():
     kernel32 = windll.kernel32
@@ -211,3 +212,50 @@ def expanduser(path):
         userhome = os.path.join(os.path.dirname(userhome), path[1:i])
 
     return userhome + path[i:]
+
+def get_subst_drive_list():
+    if sys.platform != "win32":
+        return {}
+    def _query_subst_drive(drive_letter):
+        kernel32 = windll.kernel32
+        QueryDosDevice = kernel32.QueryDosDeviceW
+        QueryDosDevice.argtypes = [c_wchar_p, c_wchar_p, c_ulong]
+        QueryDosDevice.restype = c_ulong
+        MAX_PATH = 260
+
+        if len(drive_letter) > 1 or len(drive_letter) == 0:
+            raise ValueError("Bad drive letter")
+        drive = '{}:'.format(drive_letter.upper())
+        drive_buffer = create_unicode_buffer(drive)
+        target_path_buffer_size = c_ulong(MAX_PATH)
+        target_path_buffer = create_unicode_buffer(target_path_buffer_size.value)
+        while True:
+            count = QueryDosDevice(drive_buffer, target_path_buffer, target_path_buffer_size)
+            if count == 0:
+                last_error = GetLastError()
+                if last_error == 122: #ERROR_INSUFFICIENT_BUFFER
+                    #Increase the buffer size and try again
+                    target_path_buffer_size = c_ulong((target_path_buffer_size.value * 161) / 100)
+                    target_path_buffer = create_unicode_buffer(target_path_buffer_size.value)
+                elif last_error == 2: #ERROR_FILE_NOT_FOUND
+                    #This is an invalid drive, return an empty string
+                    return ''
+                else:
+                    raise WinError(last_error)
+            else:
+                break
+        target_path = target_path_buffer.value
+        if len(target_path) > 4 and target_path[0:4] == '\\??\\':
+            if (ord(target_path[4]) >= ord('A') and ord(target_path[4]) <= ord('Z')) or \
+                (ord(target_path[4]) >= ord('a') and ord(target_path[4]) <= ord('z')):
+                #This is a SUBST'd drive, return the path
+                return target_path[4:].strip()
+        #This is a non-SUBST'd (aka real) drive, return an empty string
+        return ''
+    subst_dict = {}
+    for index in range(26):
+        drive_letter = chr(ord('A') + index)
+        target_path = _query_subst_drive(drive_letter)
+        if target_path != '':
+            subst_dict[drive_letter] = target_path
+    return subst_dict
--
2.27.0.windows.1

[edk2-staging/EdkRepo] [PATCH V1 2/2] EdkRepo: Add support for subst drives

[Nate DeSimone]

get_workspace_path() now converts a virtual drive path
to a real path before any git repo operations are done.

Cc: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Prince Agyeman <prince.agyeman@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
Signed-off-by: Nate DeSimone <nathaniel.l.desimone@intel.com>
---
 edkrepo/commands/clone_command.py |  6 ++++++
 edkrepo/config/config_factory.py  | 10 +++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/edkrepo/commands/clone_command.py b/edkrepo/commands/clone_command.py
index f638090..650bc81 100644
--- a/edkrepo/commands/clone_command.py
+++ b/edkrepo/commands/clone_command.py
@@ -20,6 +20,7 @@ from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException,
 from edkrepo.common.edkrepo_exception import EdkrepoManifestNotFoundException
 from edkrepo.common.humble import CLONE_INVALID_WORKSPACE, CLONE_INVALID_PROJECT_ARG, CLONE_INVALID_COMBO_ARG
 from edkrepo.common.humble import SPARSE_CHECKOUT, CLONE_INVALID_LOCAL_ROOTS
+from edkrepo.common.pathfix import get_subst_drive_list
 from edkrepo.common.workspace_maintenance.workspace_maintenance import case_insensitive_single_match
 from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import pull_all_manifest_repos, find_project_in_all_indices
 from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list_available_manifest_repos
@@ -77,6 +78,11 @@ class CloneCommand(EdkrepoCommand):
             workspace_dir = os.getcwd()
         else:
             workspace_dir = os.path.abspath(workspace_dir)
+        subst = get_subst_drive_list()
+        for drive in subst.keys():
+            if '{}:'.format(drive) == os.path.splitdrive(workspace_dir)[0].upper():
+                workspace_dir = os.path.join(subst[drive], os.path.splitdrive(workspace_dir)[1][1:])
+                workspace_dir = os.path.normpath(workspace_dir)
         if os.path.isdir(workspace_dir) and os.listdir(workspace_dir):
             raise EdkrepoInvalidParametersException(CLONE_INVALID_WORKSPACE)
         if not os.path.isdir(workspace_dir):
diff --git a/edkrepo/config/config_factory.py b/edkrepo/config/config_factory.py
index a82a438..45ac16f 100644
--- a/edkrepo/config/config_factory.py
+++ b/edkrepo/config/config_factory.py
@@ -11,13 +11,16 @@ import os
 import sys
 import configparser
 import collections
-from ctypes import *
+if sys.platform == "win32":
+    from ctypes import oledll, c_void_p, c_uint32, c_wchar_p
+    from ctypes import create_unicode_buffer
 
 import edkrepo.config.humble.config_factory_humble as humble
 from edkrepo.common.edkrepo_exception import EdkrepoGlobalConfigNotFoundException, EdkrepoConfigFileInvalidException
 from edkrepo.common.edkrepo_exception import EdkrepoWorkspaceInvalidException, EdkrepoGlobalDataDirectoryNotFoundException
 from edkrepo.common.edkrepo_exception import EdkrepoConfigFileReadOnlyException
 from edkrepo.common.humble import MIRROR_PRIMARY_REPOS_MISSING, MIRROR_DECODE_WARNING, MAX_PATCH_SET_INVALID
+from edkrepo.common.pathfix import get_subst_drive_list
 from edkrepo_manifest_parser import edk_manifest
 from edkrepo.common.pathfix import expanduser
 
@@ -238,6 +241,11 @@ def get_workspace_path():
     while True:
         if os.path.isdir(os.path.join(path, "repo")):
             if os.path.isfile(os.path.join(os.path.join(path, "repo"), "Manifest.xml")):
+                if sys.platform == "win32":
+                    subst = get_subst_drive_list()
+                    for drive in subst.keys():
+                        if '{}:'.format(drive) == os.path.splitdrive(path)[0].upper():
+                            path = os.path.join(subst[drive], os.path.splitdrive(path)[1][1:])
                 return path
         if os.path.dirname(path) == path:
             break
-- 
2.27.0.windows.1

Re: [edk2-staging/EdkRepo] [PATCH V1 2/2] EdkRepo: Add support for subst drives

[Ashley E Desimone]

Reviewed-by: Ashley DeSimone <ashley.e.desimone@intel.com>

-----Original Message-----
From: Desimone, Nathaniel L <nathaniel.l.desimone@intel.com> 
Sent: Wednesday, August 5, 2020 10:47 PM
To: devel@edk2.groups.io
Cc: Desimone, Ashley E <ashley.e.desimone@intel.com>; Pandya, Puja <puja.pandya@intel.com>; Bret Barkelew <Bret.Barkelew@microsoft.com>; Agyeman, Prince <prince.agyeman@intel.com>; Bjorge, Erik C <erik.c.bjorge@intel.com>
Subject: [edk2-staging/EdkRepo] [PATCH V1 2/2] EdkRepo: Add support for subst drives

get_workspace_path() now converts a virtual drive path to a real path before any git repo operations are done.

Cc: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Prince Agyeman <prince.agyeman@intel.com>
Cc: Erik Bjorge <erik.c.bjorge@intel.com>
Signed-off-by: Nate DeSimone <nathaniel.l.desimone@intel.com>
---
 edkrepo/commands/clone_command.py |  6 ++++++  edkrepo/config/config_factory.py  | 10 +++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/edkrepo/commands/clone_command.py b/edkrepo/commands/clone_command.py
index f638090..650bc81 100644
--- a/edkrepo/commands/clone_command.py
+++ b/edkrepo/commands/clone_command.py
@@ -20,6 +20,7 @@ from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException,
 from edkrepo.common.edkrepo_exception import EdkrepoManifestNotFoundException  from edkrepo.common.humble import CLONE_INVALID_WORKSPACE, CLONE_INVALID_PROJECT_ARG, CLONE_INVALID_COMBO_ARG  from edkrepo.common.humble import SPARSE_CHECKOUT, CLONE_INVALID_LOCAL_ROOTS
+from edkrepo.common.pathfix import get_subst_drive_list
 from edkrepo.common.workspace_maintenance.workspace_maintenance import case_insensitive_single_match  from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import pull_all_manifest_repos, find_project_in_all_indices  from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list_available_manifest_repos @@ -77,6 +78,11 @@ class CloneCommand(EdkrepoCommand):
             workspace_dir = os.getcwd()
         else:
             workspace_dir = os.path.abspath(workspace_dir)
+        subst = get_subst_drive_list()
+        for drive in subst.keys():
+            if '{}:'.format(drive) == os.path.splitdrive(workspace_dir)[0].upper():
+                workspace_dir = os.path.join(subst[drive], os.path.splitdrive(workspace_dir)[1][1:])
+                workspace_dir = os.path.normpath(workspace_dir)
         if os.path.isdir(workspace_dir) and os.listdir(workspace_dir):
             raise EdkrepoInvalidParametersException(CLONE_INVALID_WORKSPACE)
         if not os.path.isdir(workspace_dir):
diff --git a/edkrepo/config/config_factory.py b/edkrepo/config/config_factory.py
index a82a438..45ac16f 100644
--- a/edkrepo/config/config_factory.py
+++ b/edkrepo/config/config_factory.py
@@ -11,13 +11,16 @@ import os
 import sys
 import configparser
 import collections
-from ctypes import *
+if sys.platform == "win32":
+    from ctypes import oledll, c_void_p, c_uint32, c_wchar_p
+    from ctypes import create_unicode_buffer
 
 import edkrepo.config.humble.config_factory_humble as humble  from edkrepo.common.edkrepo_exception import EdkrepoGlobalConfigNotFoundException, EdkrepoConfigFileInvalidException  from edkrepo.common.edkrepo_exception import EdkrepoWorkspaceInvalidException, EdkrepoGlobalDataDirectoryNotFoundException
 from edkrepo.common.edkrepo_exception import EdkrepoConfigFileReadOnlyException
 from edkrepo.common.humble import MIRROR_PRIMARY_REPOS_MISSING, MIRROR_DECODE_WARNING, MAX_PATCH_SET_INVALID
+from edkrepo.common.pathfix import get_subst_drive_list
 from edkrepo_manifest_parser import edk_manifest  from edkrepo.common.pathfix import expanduser
 
@@ -238,6 +241,11 @@ def get_workspace_path():
     while True:
         if os.path.isdir(os.path.join(path, "repo")):
             if os.path.isfile(os.path.join(os.path.join(path, "repo"), "Manifest.xml")):
+                if sys.platform == "win32":
+                    subst = get_subst_drive_list()
+                    for drive in subst.keys():
+                        if '{}:'.format(drive) == os.path.splitdrive(path)[0].upper():
+                            path = os.path.join(subst[drive], 
+ os.path.splitdrive(path)[1][1:])
                 return path
         if os.path.dirname(path) == path:
             break
--
2.27.0.windows.1