[edk2-staging/EdkRepo] [PATCH] EdkRepo: Python 2.x deprecation warning in pip breaks EdkRepoInstaller

[edk2-staging/EdkRepo] [PATCH] EdkRepo: Python 2.x deprecation warning in pip breaks EdkRepoInstaller

[Nate DeSimone]

Newer versions of pip add the following output in addition to
the JSON output when running on Python 2.7:

DEPRECATION: Python 2.7 will reach the end of its life on
January 1st, 2020. Please upgrade your Python as Python 2.7 won't
be maintained after that date. A future version of pip will
drop support for Python 2.7.

This breaks machine parsing of the JSON output from pip.
Adding logic to strip the deprecation warning to fix this issue.

Signed-off-by: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
---
 .../EdkRepoInstaller/PythonOperations.cs      | 37 +++++++++++++++++--
 1 file changed, 34 insertions(+), 3 deletions(-)

diff --git a/edkrepo_installer/EdkRepoInstaller/PythonOperations.cs b/edkrepo_installer/EdkRepoInstaller/PythonOperations.cs
index f96502c..73d5bec 100644
--- a/edkrepo_installer/EdkRepoInstaller/PythonOperations.cs
+++ b/edkrepo_installer/EdkRepoInstaller/PythonOperations.cs
@@ -600,17 +600,48 @@ namespace TianoCore.EdkRepoInstaller
             return PythonPackages;
         }
 
+        private static string SanitizePipOutput(string PipOutput)
+        {
+            StringBuilder Sanitized = new StringBuilder();
+            IEnumerable<string> PipLines = PipOutput.SplitLines();
+            foreach(string line in PipLines)
+            {
+                if(line.StartsWith("DEPRECATION:"))
+                {
+                    continue;
+                }
+                if (string.IsNullOrWhiteSpace(line))
+                {
+                    continue;
+                }
+                Sanitized.Append(line.Trim());
+                Sanitized.Append("\r\n");
+            }
+            return Sanitized.ToString().Trim();
+        }
+
         public static List<PythonPackage> GetInstalledPythonPackages(string PythonPath)
         {
             List<PythonPackage> PythonPackages = new List<PythonPackage>();
             SilentProcess.StdoutDataCapture dataCapture = new SilentProcess.StdoutDataCapture();
             SilentProcess process = SilentProcess.StartConsoleProcessSilently(PythonPath, "-m pip list --format=\"json\" --no-index", dataCapture.DataReceivedHandler);
             process.WaitForExit();
+            bool TryLegacy = true;
             if (process.ExitCode == 0)
             {
-                PythonPackages = ParseJsonPipList(dataCapture.GetData());
+                try
+                {
+                    PythonPackages = ParseJsonPipList(SanitizePipOutput(dataCapture.GetData()));
+                    TryLegacy = false;
+                }
+                catch(Exception e)
+                {
+                    InstallLogger.Log("Error occurred while trying to parse pip JSON:");
+                    InstallLogger.Log(e.ToString());
+                    InstallLogger.Log("Falling back to legacy mode");
+                }
             }
-            else
+            if(TryLegacy)
             {
                 //
                 // Older versions of pip don't support the --format flag, parse the legacy format
@@ -620,7 +651,7 @@ namespace TianoCore.EdkRepoInstaller
                 process.WaitForExit();
                 if (process.ExitCode == 0)
                 {
-                    PythonPackages = ParseLegacyPipList(dataCapture.GetData());
+                    PythonPackages = ParseLegacyPipList(SanitizePipOutput(dataCapture.GetData()));
                 }
             }
             return PythonPackages;
-- 
2.23.0.windows.1

Re: [edk2-staging/EdkRepo] [PATCH] EdkRepo: Python 2.x deprecation warning in pip breaks EdkRepoInstaller

[Desimone, Ashley E]

Reviewed-by: Ashley DeSimone <ashley.e.desimone@intel.com>

-----Original Message-----
From: Desimone, Nathaniel L 
Sent: Wednesday, October 30, 2019 4:48 PM
To: devel@edk2.groups.io
Cc: Desimone, Ashley E <ashley.e.desimone@intel.com>; Pandya, Puja <puja.pandya@intel.com>
Subject: [edk2-staging/EdkRepo] [PATCH] EdkRepo: Python 2.x deprecation warning in pip breaks EdkRepoInstaller

Newer versions of pip add the following output in addition to the JSON output when running on Python 2.7:

DEPRECATION: Python 2.7 will reach the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 won't be maintained after that date. A future version of pip will drop support for Python 2.7.

This breaks machine parsing of the JSON output from pip.
Adding logic to strip the deprecation warning to fix this issue.

Signed-off-by: Nate DeSimone <nathaniel.l.desimone@intel.com>
Cc: Ashley E Desimone <ashley.e.desimone@intel.com>
Cc: Puja Pandya <puja.pandya@intel.com>
---
 .../EdkRepoInstaller/PythonOperations.cs      | 37 +++++++++++++++++--
 1 file changed, 34 insertions(+), 3 deletions(-)

diff --git a/edkrepo_installer/EdkRepoInstaller/PythonOperations.cs b/edkrepo_installer/EdkRepoInstaller/PythonOperations.cs
index f96502c..73d5bec 100644
--- a/edkrepo_installer/EdkRepoInstaller/PythonOperations.cs
+++ b/edkrepo_installer/EdkRepoInstaller/PythonOperations.cs
@@ -600,17 +600,48 @@ namespace TianoCore.EdkRepoInstaller
             return PythonPackages;         } +        private static string SanitizePipOutput(string PipOutput)+        {+            StringBuilder Sanitized = new StringBuilder();+            IEnumerable<string> PipLines = PipOutput.SplitLines();+            foreach(string line in PipLines)+            {+                if(line.StartsWith("DEPRECATION:"))+                {+                    continue;+                }+                if (string.IsNullOrWhiteSpace(line))+                {+                    continue;+                }+                Sanitized.Append(line.Trim());+                Sanitized.Append("\r\n");+            }+            return Sanitized.ToString().Trim();+        }+         public static List<PythonPackage> GetInstalledPythonPackages(string PythonPath)         {             List<PythonPackage> PythonPackages = new List<PythonPackage>();             SilentProcess.StdoutDataCapture dataCapture = new SilentProcess.StdoutDataCapture();             SilentProcess process = SilentProcess.StartConsoleProcessSilently(PythonPath, "-m pip list --format=\"json\" --no-index", dataCapture.DataReceivedHandler);             process.WaitForExit();+            bool TryLegacy = true;             if (process.ExitCode == 0)             {-                PythonPackages = ParseJsonPipList(dataCapture.GetData());+                try+                {+                    PythonPackages = ParseJsonPipList(SanitizePipOutput(dataCapture.GetData()));+                    TryLegacy = false;+                }+                catch(Exception e)+                {+                    InstallLogger.Log("Error occurred while trying to parse pip JSON:");+                    InstallLogger.Log(e.ToString());+                    InstallLogger.Log("Falling back to legacy mode");+                }             }-            else+            if(TryLegacy)             {                 //                 // Older versions of pip don't support the --format flag, parse the legacy format@@ -620,7 +651,7 @@ namespace TianoCore.EdkRepoInstaller
                 process.WaitForExit();                 if (process.ExitCode == 0)                 {-                    PythonPackages = ParseLegacyPipList(dataCapture.GetData());+                    PythonPackages = ParseLegacyPipList(SanitizePipOutput(dataCapture.GetData()));                 }             }             return PythonPackages;-- 
2.23.0.windows.1