From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mga06.intel.com (mga06.intel.com [134.134.136.31]) by mx.groups.io with SMTP id smtpd.web11.3711.1584646363990280892 for ; Thu, 19 Mar 2020 12:32:44 -0700 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 134.134.136.31, mailfrom: ashley.e.desimone@intel.com) IronPort-SDR: ksXKbtSQY3SavqC6r4KmgrW8Qbl3W5joatBnYjMrGIwRdCj5cFCZ+62IOTyycqYBvy1QSaZJmb g87N2pttk9zQ== X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga004.fm.intel.com ([10.253.24.48]) by orsmga104.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 19 Mar 2020 12:32:43 -0700 IronPort-SDR: bsA0kMztIhiDgCieSUlPZ4DxZs0d7uhtnQprrb1kax7UeKcX1S4NcNM3BjG87lokSFqAOz1EO5 izWUJvwMzkIA== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.70,572,1574150400"; d="scan'208";a="268844180" Received: from orsmsx102.amr.corp.intel.com ([10.22.225.129]) by fmsmga004.fm.intel.com with ESMTP; 19 Mar 2020 12:32:42 -0700 Received: from orsmsx163.amr.corp.intel.com (10.22.240.88) by ORSMSX102.amr.corp.intel.com (10.22.225.129) with Microsoft SMTP Server (TLS) id 14.3.439.0; Thu, 19 Mar 2020 12:32:42 -0700 Received: from orsmsx116.amr.corp.intel.com ([169.254.7.196]) by ORSMSX163.amr.corp.intel.com ([169.254.9.97]) with mapi id 14.03.0439.000; Thu, 19 Mar 2020 12:32:42 -0700 From: "Desimone, Ashley E" To: "Desimone, Nathaniel L" , "devel@edk2.groups.io" CC: "Pandya, Puja" , "Bjorge, Erik C" , Bret Barkelew Subject: Re: [PATCH 1/4] EdkRepo: Installer should remove obsolete dependencies Thread-Topic: [PATCH 1/4] EdkRepo: Installer should remove obsolete dependencies Thread-Index: AQHV/ZzZ3F32R0TpBUiX4UBpbWbMbqhQTf4Q Date: Thu, 19 Mar 2020 19:32:42 +0000 Message-ID: <4CF3A9EB60ABDA47BE7821A4DA3A0A3353D5B3EA@ORSMSX116.amr.corp.intel.com> References: <20200319031636.2269-1-nathaniel.l.desimone@intel.com> <20200319031636.2269-2-nathaniel.l.desimone@intel.com> In-Reply-To: <20200319031636.2269-2-nathaniel.l.desimone@intel.com> Accept-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: dlp-product: dlpe-windows dlp-version: 11.2.0.6 dlp-reaction: no-action x-originating-ip: [10.22.254.140] MIME-Version: 1.0 Return-Path: ashley.e.desimone@intel.com Content-Language: en-US Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable For the following section why are we singling out gitdb2 and smmap2 when de= leting obsolete dependencies? Is there an alternative to hardcoding these v= alues? + if (DeleteObsoletePackages) + { + // + // Delete obsolete dependencies + // + foreach (string PackageName in new string[] { "smmap2"= , "gitdb2" }) + { + if (InstalledPackages.Where(p =3D> p.Name =3D=3D P= ackageName).FirstOrDefault() !=3D null) + { + InstallLogger.Log(string.Format("Uninstalling = {0}", PackageName)); + PythonOperations.UninstallPythonPackage(Python= Path, PackageName); + } + } + } Thanks, Ashley -----Original Message----- From: Desimone, Nathaniel L =20 Sent: Wednesday, March 18, 2020 8:17 PM To: devel@edk2.groups.io Cc: Desimone, Ashley E ; Pandya, Puja ; Bjorge, Erik C ; Bret Barkelew Subject: [PATCH 1/4] EdkRepo: Installer should remove obsolete dependencies Cc: Ashley DeSimone Cc: Puja Pandya Cc: Erik Bjorge Cc: Bret Barkelew Signed-off-by: Nate DeSimone --- .../EdkRepoInstaller/InstallWorker.cs | 30 +++++++++- edkrepo_installer/linux-scripts/install.py | 58 ++++++++++++------- 2 files changed, 66 insertions(+), 22 deletions(-) diff --git a/edkrepo_installer/EdkRepoInstaller/InstallWorker.cs b/edkrepo_= installer/EdkRepoInstaller/InstallWorker.cs index 8d824f2..b44e8fa 100644 --- a/edkrepo_installer/EdkRepoInstaller/InstallWorker.cs +++ b/edkrepo_installer/EdkRepoInstaller/InstallWorker.cs @@ -710,6 +710,7 @@ namespace TianoCore.EdkRepoInstaller string PythonPath; bool Has32Bit; bool Has64Bit; + bool DeleteObsoletePackages =3D false; PythonOperations.GetPythonBitness(PyInstance.Version.Major= , PyInstance.Version.Minor, out Has32Bit, out Has64Bit); if (Has32Bit && Has64Bit && PyInstance.Architecture =3D=3D= CpuArchitecture.IA32) { @@ -720,14 +721,39 @@ namespace TianoCore.EdkRepoInstaller PythonPath =3D PythonOperations.FindPython(PyInstance.= Version.Major, PyInstance.Version.Minor, false); } List InstalledPackages =3D PythonOperations= .GetInstalledPythonPackages(PythonPath); - foreach(PythonWheel Wheel in PyInstance.Wheels) + foreach (PythonWheel Wheel in PyInstance.Wheels) + { + //If a package is already installed, check if we have = a newer version bundled in the installer + //If yes, the package will be upgraded, make sure obso= lete packages are uninstalled first + PythonPackage InstalledPackage =3D InstalledPackages.W= here(p =3D> p.Name =3D=3D Wheel.Package.Name.Replace('_', '-')).FirstOrDefa= ult(); + if (InstalledPackage !=3D null && InstalledPackage.Ver= sion < Wheel.Package.Version) + { + DeleteObsoletePackages =3D true; + break; + } + } + if (DeleteObsoletePackages) + { + // + // Delete obsolete dependencies + // + foreach (string PackageName in new string[] { "smmap2"= , "gitdb2" }) + { + if (InstalledPackages.Where(p =3D> p.Name =3D=3D P= ackageName).FirstOrDefault() !=3D null) + { + InstallLogger.Log(string.Format("Uninstalling = {0}", PackageName)); + PythonOperations.UninstallPythonPackage(Python= Path, PackageName); + } + } + } + foreach (PythonWheel Wheel in PyInstance.Wheels) { //PythonPackage InstalledPackage =3D (from package in = InstalledPackages // where package.Name= =3D=3D Wheel.Package.Name // select package).Fi= rstOrDefault(); //If the package is already installed, check if we hav= e a newer version bundled in the installer //If so, upgrade the package - PythonPackage InstalledPackage =3D InstalledPackages.W= here(p =3D> p.Name =3D=3D Wheel.Package.Name).FirstOrDefault(); + PythonPackage InstalledPackage =3D=20 + InstalledPackages.Where(p =3D> p.Name =3D=3D Wheel.Package.Name.Replace('= _',=20 + '-')).FirstOrDefault(); if (InstalledPackage !=3D null) { if (InstalledPackage.Version < Wheel.Package.Versi= on) diff --git a/edkrepo_installer/linux-scripts/install.py b/edkrepo_insta= ller/linux-scripts/install.py index 0ea486d..b2cdfed 100755 --- a/edkrepo_installer/linux-scripts/install.py +++ b/edkrepo_installer/linux-scripts/install.py @@ -3,7 +3,7 @@ ## @file # install.py # -# Copyright (c) 2018 - 2019, Intel Corporation. All rights reserved.
+# Copyright (c) 2018 - 2020, Intel Corporation. All rights=20 +reserved.
# SPDX-License-Identifier: BSD-2-Clause-Patent # =20 @@ -53,6 +53,26 @@ def get_args(): parser.add_argument('-v', '--verbose', action=3D'store_true', default= =3DFalse, help=3D'Enables verbose output') return parser.parse_args() =20 +def get_installed_packages(python_command): + pip_cmd =3D [def_python, '-m', 'pip', 'list', '--legacy'] + try: + res =3D default_run(pip_cmd) + except: + pip_cmd.pop(-1) + try: + res =3D default_run(pip_cmd) + except: + return ret_val + installed_packages =3D {} + for pip_mod in res.stdout.split('\n'): + try: + name, version =3D pip_mod.split() + version =3D version.strip().strip('()') + except: + continue + installed_packages[name] =3D version + return installed_packages + def get_required_wheels(): ret_val =3D collections.OrderedDict() if platform.machine() =3D=3D 'x86_64': @@ -79,27 +99,11 @@ def get_required_wheels(): 'version':wheel.attrib['V= ersion'], 'install':True} break - pip_cmd =3D [def_python, '-m', 'pip', 'list', '--legacy'] - try: - res =3D default_run(pip_cmd) - except: - pip_cmd.pop(-1) - try: - res =3D default_run(pip_cmd) - except: - return ret_val - installed_modules =3D {} - for pip_mod in res.stdout.split('\n'): - try: - name, version =3D pip_mod.split() - version =3D version.strip().strip('()') - except: - continue - installed_modules[name] =3D version + installed_packages =3D get_installed_packages(def_python) for name in ret_val: #pip doesn't understand the difference between '_' and '-' - if name.replace('_','-') in installed_modules: - version =3D installed_modules[name.replace('_','-')] + if name.replace('_','-') in installed_packages: + version =3D installed_packages[name.replace('_','-')] if _check_version(version, ret_val[name]['version']) >=3D 0 an= d not ret_val[name]['uninstall']: ret_val[name]['install'] =3D False else: @@ -328,16 +332,30 @@ def do_install(): if not os.path.isdir(whl_src_dir): log.info('- Missing wheel file directory') return 1 + updating_edkrepo =3D False for whl_name in wheels_to_install: uninstall_whl =3D wheels_to_install[whl_name]['uninstall'] whl_name =3D whl_name.replace('_','-') #pip doesn't understan= d the difference between '_' and '-' if uninstall_whl: + updating_edkrepo =3D True try: res =3D default_run([def_python, '-m', 'pip', 'uninsta= ll', '--yes', whl_name]) except: log.info('- Failed to uninstall {}'.format(whl_name)) return 1 log.info('+ Uninstalled {}'.format(whl_name)) + + #Delete obsolete dependencies + if updating_edkrepo: + installed_packages =3D get_installed_packages(def_python) + for whl_name in ['smmap2', 'gitdb2']: + if whl_name in installed_packages: + try: + res =3D default_run([def_python, '-m', 'pip', 'uni= nstall', '--yes', whl_name]) + except: + log.info('- Failed to uninstall {}'.format(whl_nam= e)) + return 1 + log.info('+ Uninstalled {}'.format(whl_name)) for whl_name in wheels_to_install: whl =3D wheels_to_install[whl_name]['wheel'] install_whl =3D wheels_to_install[whl_name]['install'] -- 2.24.0.windows.2