From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mga04.intel.com (mga04.intel.com [192.55.52.120]) by mx.groups.io with SMTP id smtpd.web09.2301.1575491236481080023 for ; Wed, 04 Dec 2019 12:27:16 -0800 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 192.55.52.120, mailfrom: ashley.e.desimone@intel.com) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by fmsmga104.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 04 Dec 2019 12:27:16 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.69,278,1571727600"; d="scan'208";a="411389970" Received: from orsmsx109.amr.corp.intel.com ([10.22.240.7]) by fmsmga005.fm.intel.com with ESMTP; 04 Dec 2019 12:27:16 -0800 Received: from orsmsx124.amr.corp.intel.com (10.22.240.120) by ORSMSX109.amr.corp.intel.com (10.22.240.7) with Microsoft SMTP Server (TLS) id 14.3.439.0; Wed, 4 Dec 2019 12:27:15 -0800 Received: from orsmsx116.amr.corp.intel.com ([169.254.7.11]) by ORSMSX124.amr.corp.intel.com ([169.254.2.83]) with mapi id 14.03.0439.000; Wed, 4 Dec 2019 12:27:15 -0800 From: "Desimone, Ashley E" To: "Desimone, Nathaniel L" , "devel@edk2.groups.io" Subject: Re: [edk2-staging/EdkRepo] [PATCH V3] EdkRepo: Add list-repos command Thread-Topic: [edk2-staging/EdkRepo] [PATCH V3] EdkRepo: Add list-repos command Thread-Index: AQHVqkQsHN7tjlLmFEamvfIXn6E6gaeqbbww Date: Wed, 4 Dec 2019 20:27:14 +0000 Message-ID: <4CF3A9EB60ABDA47BE7821A4DA3A0A3353CB17B8@ORSMSX116.amr.corp.intel.com> References: <20191204014255.2582-1-nathaniel.l.desimone@intel.com> In-Reply-To: <20191204014255.2582-1-nathaniel.l.desimone@intel.com> Accept-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-titus-metadata-40: eyJDYXRlZ29yeUxhYmVscyI6IiIsIk1ldGFkYXRhIjp7Im5zIjoiaHR0cDpcL1wvd3d3LnRpdHVzLmNvbVwvbnNcL0ludGVsMyIsImlkIjoiOGViYjA3MmYtZGQ3Mi00OTZkLTg4OTUtYjBkY2M3YmI2YTkzIiwicHJvcHMiOlt7Im4iOiJDVFBDbGFzc2lmaWNhdGlvbiIsInZhbHMiOlt7InZhbHVlIjoiQ1RQX05UIn1dfV19LCJTdWJqZWN0TGFiZWxzIjpbXSwiVE1DVmVyc2lvbiI6IjE3LjEwLjE4MDQuNDkiLCJUcnVzdGVkTGFiZWxIYXNoIjoianhNYjJJVzZvR3o5dGxTNk1LMDJLT2U4MmpybFZ3WExicnBxWnRcL2VPbnMxQWpmSnV6RDFTUUVLZDlyaDdtMUoifQ== x-ctpclassification: CTP_NT dlp-product: dlpe-windows dlp-version: 11.2.0.6 dlp-reaction: no-action x-originating-ip: [10.22.254.140] MIME-Version: 1.0 Return-Path: ashley.e.desimone@intel.com Content-Language: en-US Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable Reviewed-by: Ashley E Desimone -----Original Message----- From: Desimone, Nathaniel L=20 Sent: Tuesday, December 3, 2019 5:43 PM To: devel@edk2.groups.io Cc: Desimone, Ashley E Subject: [edk2-staging/EdkRepo] [PATCH V3] EdkRepo: Add list-repos command list-repos lists the git repos used by all projects and which branches thos= e projects use. Cc: Ashley E Desimone Signed-off-by: Nate DeSimone --- edkrepo/commands/arguments/list_repos_args.py | 16 ++ edkrepo/commands/humble/__init__.py | 8 + edkrepo/commands/humble/list_repos_humble.py | 26 ++ edkrepo/commands/list_repos_command.py | 248 ++++++++++++++++++ setup.py | 5 +- 5 files changed, 300 insertions(+), 3 deletions(-) create mode 100644 edk= repo/commands/arguments/list_repos_args.py create mode 100644 edkrepo/commands/humble/__init__.py create mode 100644 edkrepo/commands/humble/list_repos_humble.py create mode 100644 edkrepo/commands/list_repos_command.py diff --git a/edkrepo/commands/arguments/list_repos_args.py b/edkrepo/comman= ds/arguments/list_repos_args.py new file mode 100644 index 0000000..7133201 --- /dev/null +++ b/edkrepo/commands/arguments/list_repos_args.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3+#+## @file+# list_repos_args.py+#+# Copyright=20 +(c) 2019, Intel Corporation. All rights reserved.
+#=20 +SPDX-License-Identifier: BSD-2-Clause-Patent+#++''' Contains the help=20 +and description strings for arguments in the+list-repos command meta=20 +data.+'''++COMMAND_DESCRIPTION =3D 'Lists the git repos used by available= =20 +projects and the branches that are used.'+ARCHIVED_HELP =3D 'Include a=20 +listing of archived projects.'+REPOS_HELP =3D 'Only show the given subset= =20 +of repos instead of all repos. The name of a repo is determined by the=20 +name given by the most manifest files.'diff --git=20 +a/edkrepo/commands/humble/__init__.py=20 +b/edkrepo/commands/humble/__init__.py new file mode 100644 index 0000000..dea6eb4 --- /dev/null +++ b/edkrepo/commands/humble/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3+#+## @file+# __init__.py+#+# Copyright (c) 2019,=20 +Intel Corporation. All rights reserved.
+# SPDX-License-Identifier:=20 +BSD-2-Clause-Patent+#diff --git=20 +a/edkrepo/commands/humble/list_repos_humble.py=20 +b/edkrepo/commands/humble/list_repos_humble.py new file mode 100644 index 0000000..9083d8e --- /dev/null +++ b/edkrepo/commands/humble/list_repos_humble.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3+#+## @file+# list_repos_humble.py+#+# Copyright (c)= 2019, Intel Corporation. All rights reserved.
+# SPDX-License-Identifie= r: BSD-2-Clause-Patent+#++'''+Contains user visible strings printed by the = list-repos command.+'''++from colorama import Fore+from colorama import Sty= le++BRANCHES =3D 'Branches:'+BRANCH_FORMAT_STRING =3D ' {}{}{{}}{}'.format= (Fore.BLUE, Style.BRIGHT, Style.RESET_ALL)+COMBO_FORMAT_STRING =3D ' {{}= } {}{}({{}}){}'.format(Fore.CYAN, Style.BRIGHT, Style.RESET_ALL)+DEFAULT_C= OMBO_FORMAT_STRING =3D ' {{}} {}{}*({{}})*{}'.format(Fore.GREEN, Style.B= RIGHT, Style.RESET_ALL)+MANIFEST_DIRECTORY =3D 'Manifest directory:'+PROJEC= T_NAME_FORMAT_STRING =3D '{}{}{{}}{}:'.format(Fore.YELLOW, Style.BRIGHT, St= yle.RESET_ALL)+REPOSITORIES =3D 'Repositories:'+REPO_NAME_AND_URL =3D '{}{}= {{}}{} - [{}{}{{}}{}]'.format(Fore.MAGENTA, Style.BRIGHT, Style.RESET_ALL, = Fore.RED, Style.BRIGHT, Style.RESET_ALL)+REPO_NAME_NOT_FOUND =3D 'repo_name= not found'+REPO_NOT_FOUND_IN_MANIFEST =3D 'Repo(s) {} not found in any man= ifest file'diff --git a/edkrepo/commands/list_repos_command.py b/edkrepo/co= mmands/list_repos_command.py new file mode 100644 index 0000000..caf0373 --- /dev/null +++ b/edkrepo/commands/list_repos_command.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3+#+## @file+# list_repos_command.py+#+# Copyright (c= ) 2019, Intel Corporation. All rights reserved.
+# SPDX-License-Identifi= er: BSD-2-Clause-Patent+#++import collections+import os++#from git import R= epo+from colorama import Fore, Style++# Our modules+from edkrepo.commands.e= dkrepo_command import EdkrepoCommand+from edkrepo.commands.edkrepo_command = import ColorArgument+import edkrepo.commands.arguments.list_repos_args as a= rguments+import edkrepo.commands.humble.list_repos_humble as humble+from ed= krepo.common.common_repo_functions import pull_latest_manifest_repo+from ed= krepo.common.edkrepo_exception import EdkrepoInvalidParametersException, Ed= krepoManifestInvalidException+from edkrepo.common.ui_functions import init_= color_console+from edkrepo_manifest_parser.edk_manifest import CiIndexXml, = ManifestXml++class ListReposCommand(EdkrepoCommand):+ def __init__(self)= :+ super().__init__()+ self.repo_names =3D None++ def get_= metadata(self):+ metadata =3D {}+ metadata['name'] =3D 'list-= repos'+ metadata['help-text'] =3D arguments.COMMAND_DESCRIPTION+ = args =3D []+ metadata['arguments'] =3D args+ args.append(= {'name': 'repos',+ 'positional': False,+ = 'required': False,+ 'action': 'store',+ = 'nargs': '+',+ 'help-text': arguments.REPOS_= HELP})+ args.append({'name': 'archived',+ 'short= -name': 'a',+ 'positional': False,+ = 'required': False,+ 'help-text': arguments.ARCHIVED_HE= LP})+ args.append(ColorArgument)+ return metadata++ def ru= n_command(self, args, config):+ print()+ init_color_console(a= rgs.color)++ # Get path to global manifest file+ global_manif= est_directory =3D config['cfg_file'].manifest_repo_abs_local_path+ i= f args.verbose:+ print(humble.MANIFEST_DIRECTORY)+ pr= int(global_manifest_directory)+ print()+ index_path =3D o= s.path.join(global_manifest_directory, 'CiIndex.xml')++ pull_latest_= manifest_repo(args, config)+ print()++ #Create a dictionary c= ontaining all the manifests listed in the CiIndex.xml file+ ci_index= _xml =3D CiIndexXml(index_path)+ manifests =3D {}+ repo_urls = =3D set()+ project_list =3D list(ci_index_xml.project_list)+ = if args.archived:+ project_list.extend(ci_index_xml.archived_pro= ject_list)+ for project in project_list:+ xml_file =3D ci= _index_xml.get_project_xml(project)+ manifest =3D ManifestXml(os= .path.normpath(os.path.join(global_manifest_directory, xml_file)))+ = manifests[project] =3D manifest+ for combo in [c.name for c = in manifest.combinations]:+ sources =3D manifest.get_repo_so= urces(combo)+ for source in sources:+ rep= o_urls.add(self.get_repo_url(source.remote_url))++ #Sort the manifes= ts so projects will be displayed alphabetically+ manifests =3D colle= ctions.OrderedDict(sorted(manifests.items()))+ project_justify =3D l= en(max(manifests.keys(), key=3Dlen))++ #Determine the names of the r= epositories+ self.generate_repo_names(repo_urls, manifests)+ = print(humble.REPOSITORIES)++ #If the user provided a list of reposit= ories to view, check to make sure+ #at least one repository will be = shown, if not provide an error+ if args.repos and len([x for x in se= lf.repo_names if x in args.repos]) <=3D 0:+ raise EdkrepoInvalid= ParametersException(humble.REPO_NOT_FOUND_IN_MANIFEST.format(','.join(args.= repos)))++ #For each each git repository...+ for repo_name in= self.repo_names:+ if args.repos and repo_name not in args.repos= :+ continue+ repo =3D self.repo_names[repo_name][= 0]+ print(humble.REPO_NAME_AND_URL.format(repo_name, repo))+ = print(humble.BRANCHES)++ #Determine the list of branches= that used by any branch combination in any manifest+ branches = =3D set()+ for project_name in manifests:+ for co= mbo in [c.name for c in manifests[project_name].combinations]:+ = sources =3D manifests[project_name].get_repo_sources(combo)+ = for source in sources:+ if self.get_rep= o_url(source.remote_url) =3D=3D repo:+ branches.= add(source.branch)++ #Sort the branch names so they will be disp= layed alphabetically+ #with the exception that if a branch named= "master" exists, then it+ #will be displayed first+ = branches =3D sorted(branches, key=3Dstr.casefold)+ if 'master' i= n branches:+ branches.remove('master')+ branc= hes.insert(0, 'master')++ #For each interesting branch in the cu= rrent git repository...+ for branch in branches:+ = print(humble.BRANCH_FORMAT_STRING.format(branch))++ #Determ= ine the branch combinations that use that branch+ for projec= t_name in manifests:+ combos =3D []+ = for combo in [c.name for c in manifests[project_name].combinations]:+ = sources =3D manifests[project_name].get_repo_sources(comb= o)+ for source in sources:+ = if self.get_repo_url(source.remote_url) =3D=3D repo and source.branch = =3D=3D branch:+ combos.append(combo)+ = break+ if len(combos) > 0:+ = #Sort the branch combinations so they will be displayed= alphabetically+ #with the exception that the defaul= t branch combination for the manifest+ #file will be= displayed first+ combos =3D sorted(combos, key=3Dst= r.casefold)+ default_combo =3D manifests[project_nam= e].general_config.default_combo+ if default_combo in= combos:+ combos.remove(default_combo)+ = combos.insert(0, default_combo)+ = first_combo =3D True+ for combo in combos:+ = #Print the project name+ if f= irst_combo:+ project_name_print =3D humble.P= ROJECT_NAME_FORMAT_STRING.format(project_name.ljust(project_justify))+ = first_combo =3D False+ = else:+ project_name_print =3D '{} '.format= ((' ' * len(project_name)).ljust(project_justify))+ = #Print the branch combination name, if this is the default branch combi= nation,+ #then print it in green color with *'s = around it+ if default_combo =3D=3D combo:+ = print(humble.DEFAULT_COMBO_FORMAT_STRING.format(p= roject_name_print, combo))+ else:+ = print(humble.COMBO_FORMAT_STRING.format(project_name_prin= t, combo))++ def get_repo_url(self, repo_url):+ if repo_url[-4:].= lower() =3D=3D '.git':+ return repo_url[:-4]+ return repo= _url++ def get_repo_name(self, repo_url, manifests):+ for name in= self.repo_names:+ if self.repo_names[name][0] =3D=3D repo_url:+= return name+ raise EdkrepoInvalidParametersException= (humble.REPO_NAME_NOT_FOUND)++ def generate_repo_names(self, repo_urls, = manifests):+ #Determine the names of the repositories+ self.r= epo_names =3D collections.OrderedDict()+ for repo_url in repo_urls:+= self.__repo_name_worker(repo_url, manifests)++ #Sort the= git repositories so they will be displayed alphabetically+ self.rep= o_names =3D collections.OrderedDict(sorted(self.repo_names.items()))+ = names_to_move =3D []+ for repo_name in self.repo_names:+ = if repo_name.lower().find('edk2') =3D=3D 0:+ names_to_move= .append(repo_name)+ names_to_move =3D sorted(names_to_move, reverse= =3DTrue)+ for name_to_move in names_to_move:+ self.repo_n= ames.move_to_end(name_to_move, False)+ names_to_move =3D []+ = for repo_name in self.repo_names:+ if repo_name.lower().find('in= tel') =3D=3D 0:+ names_to_move.append(repo_name)+ nam= es_to_move =3D sorted(names_to_move, reverse=3DTrue)+ for name_to_mo= ve in names_to_move:+ self.repo_names.move_to_end(name_to_move, = False)++ def __repo_name_worker(self, repo_url, manifests):+ #Thi= s is a heuristic that guesses the "name" of a repository by looking+ = #at the name given to it by the most manifest files.+ names =3D col= lections.defaultdict(int)+ for project_name in manifests:+ = for combo in [c.name for c in manifests[project_name].combinations]:+ = sources =3D manifests[project_name].get_repo_sources(combo)+ = for source in sources:+ if self.get_repo_ur= l(source.remote_url) =3D=3D repo_url:+ names[source.= root] +=3D 1+ found_unique_name =3D False+ original_best_name= =3D None+ original_best_name_frequency =3D 0+ while not foun= d_unique_name:+ best_name =3D None+ best_name_frequen= cy =3D 0+ if len(names) <=3D 0:+ if original_best= _name_frequency =3D=3D 1:+ #If only 1 project uses this = name, then append the project+ #name to the directory na= me to create the repo name+ for project_name in manifest= s:+ for combo in [c.name for c in manifests[project_= name].combinations]:+ sources =3D manifests[proj= ect_name].get_repo_sources(combo)+ for source in= sources:+ if self.get_repo_url(source.remot= e_url) =3D=3D repo_url and source.root =3D=3D original_best_name:+ = best_name =3D "{}-{}".format(original_best_name,= project_name)+ best_name_frequency =3D = original_best_name_frequency+ else:+ best= _name =3D repo_url+ best_name_frequency =3D 0+ = break+ for name in names:+ if names[name] >= best_name_frequency:+ best_name =3D name+ = best_name_frequency =3D names[name]+ if best_name is None:= + raise EdkrepoManifestInvalidException(humble.REPO_NOT_FOUN= D_IN_MANIFEST.format(repo_url))+ if original_best_name is None:+= original_best_name =3D best_name+ original_b= est_name_frequency =3D best_name_frequency+ if best_name in self= .repo_names:+ if self.repo_names[best_name][0] =3D=3D repo_u= rl:+ found_unique_name =3D True+ else:+ = #If there is a name collision, then which repo has the mo= st+ #Usage of the name owns the name+ = if best_name_frequency > self.repo_names[best_name][1]:+ = old_repo_url =3D self.repo_names[name][0]+ del= self.repo_names[best_name]+ found_unique_name =3D T= rue+ self.repo_names[best_name] =3D (repo_url, best_= name_frequency)+ self.__repo_name_worker(old_repo_ur= l, manifests)+ else:+ #Use the na= me given by the second most manifest files+ del name= s[best_name]+ else:+ found_unique_name =3D True+ = self.repo_names[best_name] =3D (repo_url, best_name_frequency)diff -= -git a/setup.py b/setup.py index 73ce9a9..1b9edad 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ ## @file # setup.py # -# Copyright (c) 2017- 2019, Intel Corporation. All rights reserved.
+# Copyright (c) 2017 - 2019, Intel Corporation. All rights=20 +reserved.
# SPDX-License-Identifier: BSD-2-Clause-Patent # =20 @@ -12,7 +12,7 @@ from setuptools import setup setup(name=3D'edkrepo', version=3D'2.0.0', description=3D'The edkrepo tools', - packages=3D['edkrepo', 'edkrepo.commands', 'edkrepo.commands.argumen= ts', 'edkrepo.common', 'edkrepo.config', 'edkrepo_manifest_parser', 'projec= t_utils'], + packages=3D['edkrepo', 'edkrepo.commands',=20 + 'edkrepo.commands.arguments', 'edkrepo.commands.humble',=20 + 'edkrepo.common', 'edkrepo.config', 'edkrepo_manifest_parser',=20 + 'project_utils'], package_data=3D{ }, include_package_data=3DTrue, @@ -22,4 +22,3 @@ setup(name=3D'edkrepo', ] } ) - -- 2.24.0.windows.2