From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mga01.intel.com (mga01.intel.com [192.55.52.88]) by mx.groups.io with SMTP id smtpd.web09.1415.1575422299069380037 for ; Tue, 03 Dec 2019 17:18:19 -0800 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 192.55.52.88, mailfrom: ashley.e.desimone@intel.com) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga004.fm.intel.com ([10.253.24.48]) by fmsmga101.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 03 Dec 2019 17:18:18 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.69,275,1571727600"; d="scan'208";a="236093189" Received: from orsmsx107.amr.corp.intel.com ([10.22.240.5]) by fmsmga004.fm.intel.com with ESMTP; 03 Dec 2019 17:18:18 -0800 Received: from orsmsx116.amr.corp.intel.com ([169.254.7.11]) by ORSMSX107.amr.corp.intel.com ([169.254.1.224]) with mapi id 14.03.0439.000; Tue, 3 Dec 2019 17:18:18 -0800 From: "Desimone, Ashley E" To: "Desimone, Nathaniel L" , "devel@edk2.groups.io" Subject: Re: [edk2-staging/EdkRepo] [PATCH V2] EdkRepo: Add list-repos command Thread-Topic: [edk2-staging/EdkRepo] [PATCH V2] EdkRepo: Add list-repos command Thread-Index: AQHVqi5E8hsaEcxw3UO9MiCBYqxGCqepLIqA Date: Wed, 4 Dec 2019 01:18:18 +0000 Message-ID: <4CF3A9EB60ABDA47BE7821A4DA3A0A3353CB1231@ORSMSX116.amr.corp.intel.com> References: <20191203230552.1311-1-nathaniel.l.desimone@intel.com> In-Reply-To: <20191203230552.1311-1-nathaniel.l.desimone@intel.com> Accept-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-titus-metadata-40: eyJDYXRlZ29yeUxhYmVscyI6IiIsIk1ldGFkYXRhIjp7Im5zIjoiaHR0cDpcL1wvd3d3LnRpdHVzLmNvbVwvbnNcL0ludGVsMyIsImlkIjoiNjZjMTczZGUtNjc2MS00ODYwLWI3OTktYmIzMmI5M2IxOWMwIiwicHJvcHMiOlt7Im4iOiJDVFBDbGFzc2lmaWNhdGlvbiIsInZhbHMiOlt7InZhbHVlIjoiQ1RQX05UIn1dfV19LCJTdWJqZWN0TGFiZWxzIjpbXSwiVE1DVmVyc2lvbiI6IjE3LjEwLjE4MDQuNDkiLCJUcnVzdGVkTGFiZWxIYXNoIjoiRWNncGtyOWlvdiswblY0OEx0V3l5YTI2YWsrOGVyTmVMNjliZG8wT1JNaTMzY256ek9lOFwvS0MyUUdqWkNORUUifQ== x-ctpclassification: CTP_NT dlp-product: dlpe-windows dlp-version: 11.2.0.6 dlp-reaction: no-action x-originating-ip: [10.22.254.138] MIME-Version: 1.0 Content-Language: en-US Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable It looks good for now but in the future I would like to see an error messag= e when the repo name passed with the --repos flag is not found. Reviewed-by: Ashley E DeSimone -----Original Message----- From: Desimone, Nathaniel L=20 Sent: Tuesday, December 3, 2019 3:06 PM To: devel@edk2.groups.io Cc: Desimone, Ashley E Subject: [edk2-staging/EdkRepo] [PATCH V2] EdkRepo: Add list-repos command list-repos lists the git repos used by all projects and which branches thos= e projects use. Cc: Ashley E Desimone Signed-off-by: Nate DeSimone --- edkrepo/commands/arguments/list_repos_args.py | 16 ++ edkrepo/commands/humble/__init__.py | 8 + edkrepo/commands/humble/list_repos_humble.py | 26 ++ edkrepo/commands/list_repos_command.py | 243 ++++++++++++++++++ setup.py | 5 +- 5 files changed, 295 insertions(+), 3 deletions(-) create mode 100644 edk= repo/commands/arguments/list_repos_args.py create mode 100644 edkrepo/commands/humble/__init__.py create mode 100644 edkrepo/commands/humble/list_repos_humble.py create mode 100644 edkrepo/commands/list_repos_command.py diff --git a/edkrepo/commands/arguments/list_repos_args.py b/edkrepo/comman= ds/arguments/list_repos_args.py new file mode 100644 index 0000000..6ff9b9c --- /dev/null +++ b/edkrepo/commands/arguments/list_repos_args.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3+#+## @file+# list_repos_args.py+#+# Copyright=20 +(c) 2019, Intel Corporation. All rights reserved.
+#=20 +SPDX-License-Identifier: BSD-2-Clause-Patent+#++''' Contains the help=20 +and description strings for arguments in the+list-repos command meta=20 +data.+'''++COMMAND_DESCRIPTION =3D 'Lists the git repos used by available= =20 +projects and the branches that are used.'+ARCHIVED_HELP =3D 'Include a=20 +listing of archived projects.'+REPOS_HELP =3D 'Only show the given subset= =20 +of repos instead of all repos.'diff --git=20 +a/edkrepo/commands/humble/__init__.py=20 +b/edkrepo/commands/humble/__init__.py new file mode 100644 index 0000000..dea6eb4 --- /dev/null +++ b/edkrepo/commands/humble/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3+#+## @file+# __init__.py+#+# Copyright (c) 2019,=20 +Intel Corporation. All rights reserved.
+# SPDX-License-Identifier:=20 +BSD-2-Clause-Patent+#diff --git=20 +a/edkrepo/commands/humble/list_repos_humble.py=20 +b/edkrepo/commands/humble/list_repos_humble.py new file mode 100644 index 0000000..bbb05a7 --- /dev/null +++ b/edkrepo/commands/humble/list_repos_humble.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3+#+## @file+# list_repos_humble.py+#+# Copyright (c)= 2019, Intel Corporation. All rights reserved.
+# SPDX-License-Identifie= r: BSD-2-Clause-Patent+#++'''+Contains user visible strings printed by the = list-repos command.+'''++from colorama import Fore+from colorama import Sty= le++BRANCHES =3D 'Branches:'+BRANCH_FORMAT_STRING =3D ' {}{}{{}}{}'.format= (Fore.BLUE, Style.BRIGHT, Style.RESET_ALL)+COMBO_FORMAT_STRING =3D ' {{}= } {}{}({{}}){}'.format(Fore.CYAN, Style.BRIGHT, Style.RESET_ALL)+DEFAULT_C= OMBO_FORMAT_STRING =3D ' {{}} {}{}*({{}})*{}'.format(Fore.GREEN, Style.B= RIGHT, Style.RESET_ALL)+MANIFEST_DIRECTORY =3D 'Manifest directory:'+PROJEC= T_NAME_FORMAT_STRING =3D '{}{}{{}}{}:'.format(Fore.YELLOW, Style.BRIGHT, St= yle.RESET_ALL)+REPOSITORIES =3D 'Repositories:'+REPO_NAME_AND_URL =3D '{}{}= {{}}{} - [{}{}{{}}{}]'.format(Fore.MAGENTA, Style.BRIGHT, Style.RESET_ALL, = Fore.RED, Style.BRIGHT, Style.RESET_ALL)+REPO_NAME_NOT_FOUND =3D 'repo_name= not found'+REPO_NOT_FOUND_IN_MANIFEST =3D 'Repo {} not found in any manife= st file'diff --git a/edkrepo/commands/list_repos_command.py b/edkrepo/comma= nds/list_repos_command.py new file mode 100644 index 0000000..9035993 --- /dev/null +++ b/edkrepo/commands/list_repos_command.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3+#+## @file+# list_repos_command.py+#+# Copyright (c= ) 2019, Intel Corporation. All rights reserved.
+# SPDX-License-Identifi= er: BSD-2-Clause-Patent+#++import collections+import os++#from git import R= epo+from colorama import Fore, Style++# Our modules+from edkrepo.commands.e= dkrepo_command import EdkrepoCommand+from edkrepo.commands.edkrepo_command = import ColorArgument+import edkrepo.commands.arguments.list_repos_args as a= rguments+import edkrepo.commands.humble.list_repos_humble as humble+from ed= krepo.common.common_repo_functions import pull_latest_manifest_repo+from ed= krepo.common.edkrepo_exception import EdkrepoInvalidParametersException, Ed= krepoManifestInvalidException+from edkrepo.common.ui_functions import init_= color_console+from edkrepo_manifest_parser.edk_manifest import CiIndexXml, = ManifestXml++class ListReposCommand(EdkrepoCommand):+ def __init__(self)= :+ super().__init__()+ self.repo_names =3D None++ def get_= metadata(self):+ metadata =3D {}+ metadata['name'] =3D 'list-= repos'+ metadata['help-text'] =3D arguments.COMMAND_DESCRIPTION+ = args =3D []+ metadata['arguments'] =3D args+ args.append(= {'name': 'repos',+ 'positional': False,+ = 'required': False,+ 'action': 'store',+ = 'nargs': '+',+ 'help-text': arguments.REPOS_= HELP})+ args.append({'name': 'archived',+ 'short= -name': 'a',+ 'positional': False,+ = 'required': False,+ 'help-text': arguments.ARCHIVED_HE= LP})+ args.append(ColorArgument)+ return metadata++ def ru= n_command(self, args, config):+ print()+ init_color_console(a= rgs.color)++ # Get path to global manifest file+ global_manif= est_directory =3D config['cfg_file'].manifest_repo_abs_local_path+ i= f args.verbose:+ print(humble.MANIFEST_DIRECTORY)+ pr= int(global_manifest_directory)+ print()+ index_path =3D o= s.path.join(global_manifest_directory, 'CiIndex.xml')++ pull_latest_= manifest_repo(args, config)+ print()++ #Create a dictionary c= ontaining all the manifests listed in the CiIndex.xml file+ ci_index= _xml =3D CiIndexXml(index_path)+ manifests =3D {}+ repo_urls = =3D set()+ project_list =3D list(ci_index_xml.project_list)+ = if args.archived:+ project_list.extend(ci_index_xml.archived_pro= ject_list)+ for project in project_list:+ xml_file =3D ci= _index_xml.get_project_xml(project)+ manifest =3D ManifestXml(os= .path.normpath(os.path.join(global_manifest_directory, xml_file)))+ = manifests[project] =3D manifest+ for combo in [c.name for c = in manifest.combinations]:+ sources =3D manifest.get_repo_so= urces(combo)+ for source in sources:+ rep= o_urls.add(self.get_repo_url(source.remote_url))++ #Sort the manifes= ts so projects will be displayed alphabetically+ manifests =3D colle= ctions.OrderedDict(sorted(manifests.items()))+ project_justify =3D l= en(max(manifests.keys(), key=3Dlen))++ #Determine the names of the r= epositories+ self.generate_repo_names(repo_urls, manifests)+ = print(humble.REPOSITORIES)++ #For each each git repository...+ = for repo_name in self.repo_names:+ if args.repos and repo_name= not in args.repos:+ continue+ repo =3D self.repo= _names[repo_name][0]+ print(humble.REPO_NAME_AND_URL.format(repo= _name, repo))+ print(humble.BRANCHES)++ #Determine th= e list of branches that used by any branch combination in any manifest+ = branches =3D set()+ for project_name in manifests:+ = for combo in [c.name for c in manifests[project_name].combinatio= ns]:+ sources =3D manifests[project_name].get_repo_sourc= es(combo)+ for source in sources:+ = if self.get_repo_url(source.remote_url) =3D=3D repo:+ = branches.add(source.branch)++ #Sort the branch names so = they will be displayed alphabetically+ #with the exception that = if a branch named "master" exists, then it+ #will be displayed f= irst+ branches =3D sorted(branches, key=3Dstr.casefold)+ = if 'master' in branches:+ branches.remove('master')+ = branches.insert(0, 'master')++ #For each interesting= branch in the current git repository...+ for branch in branches= :+ print(humble.BRANCH_FORMAT_STRING.format(branch))++ = #Determine the branch combinations that use that branch+ = for project_name in manifests:+ combos =3D []+ = for combo in [c.name for c in manifests[project_name].comb= inations]:+ sources =3D manifests[project_name].get_= repo_sources(combo)+ for source in sources:+ = if self.get_repo_url(source.remote_url) =3D=3D repo and= source.branch =3D=3D branch:+ combos.append= (combo)+ break+ if len(co= mbos) > 0:+ #Sort the branch combinations so they wi= ll be displayed alphabetically+ #with the exception = that the default branch combination for the manifest+ = #file will be displayed first+ combos =3D sorted(c= ombos, key=3Dstr.casefold)+ default_combo =3D manife= sts[project_name].general_config.default_combo+ if d= efault_combo in combos:+ combos.remove(default_c= ombo)+ combos.insert(0, default_combo)+ = first_combo =3D True+ for combo in co= mbos:+ #Print the project name+ = if first_combo:+ project_name_pri= nt =3D humble.PROJECT_NAME_FORMAT_STRING.format(project_name.ljust(project_= justify))+ first_combo =3D False+ = else:+ project_name_print = =3D '{} '.format((' ' * len(project_name)).ljust(project_justify))+ = #Print the branch combination name, if this is the defa= ult branch combination,+ #then print it in green= color with *'s around it+ if default_combo =3D= =3D combo:+ print(humble.DEFAULT_COMBO_FORMA= T_STRING.format(project_name_print, combo))+ els= e:+ print(humble.COMBO_FORMAT_STRING.format(= project_name_print, combo))++ def get_repo_url(self, repo_url):+ = if repo_url[-4:].lower() =3D=3D '.git':+ return repo_url[:-4]+ = return repo_url++ def get_repo_name(self, repo_url, manifests):+ = for name in self.repo_names:+ if self.repo_names[name][0] = =3D=3D repo_url:+ return name+ raise EdkrepoInvalidPa= rametersException(humble.REPO_NAME_NOT_FOUND)++ def generate_repo_names(= self, repo_urls, manifests):+ #Determine the names of the repositori= es+ self.repo_names =3D collections.OrderedDict()+ for repo_u= rl in repo_urls:+ self.__repo_name_worker(repo_url, manifests)++= #Sort the git repositories so they will be displayed alphabetically= + self.repo_names =3D collections.OrderedDict(sorted(self.repo_names= .items()))+ names_to_move =3D []+ for repo_name in self.repo_= names:+ if repo_name.lower().find('edk2') =3D=3D 0:+ = names_to_move.append(repo_name)+ names_to_move =3D sorted(names_= to_move, reverse=3DTrue)+ for name_to_move in names_to_move:+ = self.repo_names.move_to_end(name_to_move, False)+ names_to_move= =3D []+ for repo_name in self.repo_names:+ if repo_name.= lower().find('intel') =3D=3D 0:+ names_to_move.append(repo_n= ame)+ names_to_move =3D sorted(names_to_move, reverse=3DTrue)+ = for name_to_move in names_to_move:+ self.repo_names.move_to_en= d(name_to_move, False)++ def __repo_name_worker(self, repo_url, manifest= s):+ #This is a heuristic that guesses the "name" of a repository by= looking+ #at the name given to it by the most manifest files.+ = names =3D collections.defaultdict(int)+ for project_name in manif= ests:+ for combo in [c.name for c in manifests[project_name].com= binations]:+ sources =3D manifests[project_name].get_repo_so= urces(combo)+ for source in sources:+ if = self.get_repo_url(source.remote_url) =3D=3D repo_url:+ = names[source.root] +=3D 1+ found_unique_name =3D False+ or= iginal_best_name =3D None+ original_best_name_frequency =3D 0+ = while not found_unique_name:+ best_name =3D None+ b= est_name_frequency =3D 0+ if len(names) <=3D 0:+ = if original_best_name_frequency =3D=3D 1:+ #If only 1 pr= oject uses this name, then append the project+ #name to = the directory name to create the repo name+ for project_= name in manifests:+ for combo in [c.name for c in ma= nifests[project_name].combinations]:+ sources = =3D manifests[project_name].get_repo_sources(combo)+ = for source in sources:+ if self.get_rep= o_url(source.remote_url) =3D=3D repo_url and source.root =3D=3D original_be= st_name:+ best_name =3D "{}-{}".format(o= riginal_best_name, project_name)+ best_n= ame_frequency =3D original_best_name_frequency+ else:+ = best_name =3D repo_url+ best_name_frequenc= y =3D 0+ break+ for name in names:+ = if names[name] > best_name_frequency:+ best_name =3D n= ame+ best_name_frequency =3D names[name]+ if = best_name is None:+ raise EdkrepoManifestInvalidException(hu= mble.REPO_NOT_FOUND_IN_MANIFEST.format(repo_url))+ if original_b= est_name is None:+ original_best_name =3D best_name+ = original_best_name_frequency =3D best_name_frequency+ if= best_name in self.repo_names:+ if self.repo_names[best_name= ][0] =3D=3D repo_url:+ found_unique_name =3D True+ = else:+ #If there is a name collision, then whi= ch repo has the most+ #Usage of the name owns the name+ = if best_name_frequency > self.repo_names[best_name][1]:+= old_repo_url =3D self.repo_names[name][0]+ = del self.repo_names[best_name]+ found= _unique_name =3D True+ self.repo_names[best_name] = =3D (repo_url, best_name_frequency)+ self.__repo_nam= e_worker(old_repo_url, manifests)+ else:+ = #Use the name given by the second most manifest files+ = del names[best_name]+ else:+ found_un= ique_name =3D True+ self.repo_names[best_name] =3D (repo_url, best_n= ame_frequency)diff --git a/setup.py b/setup.py index 73ce9a9..1b9edad 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ ## @file # setup.py # -# Copyright (c) 2017- 2019, Intel Corporation. All rights reserved.
+# Copyright (c) 2017 - 2019, Intel Corporation. All rights=20 +reserved.
# SPDX-License-Identifier: BSD-2-Clause-Patent # =20 @@ -12,7 +12,7 @@ from setuptools import setup setup(name=3D'edkrepo', version=3D'2.0.0', description=3D'The edkrepo tools', - packages=3D['edkrepo', 'edkrepo.commands', 'edkrepo.commands.argumen= ts', 'edkrepo.common', 'edkrepo.config', 'edkrepo_manifest_parser', 'projec= t_utils'], + packages=3D['edkrepo', 'edkrepo.commands',=20 + 'edkrepo.commands.arguments', 'edkrepo.commands.humble',=20 + 'edkrepo.common', 'edkrepo.config', 'edkrepo_manifest_parser',=20 + 'project_utils'], package_data=3D{ }, include_package_data=3DTrue, @@ -22,4 +22,3 @@ setup(name=3D'edkrepo', ] } ) - -- 2.24.0.windows.2