From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mga09.intel.com (mga09.intel.com [134.134.136.24]) by mx.groups.io with SMTP id smtpd.web12.4005.1575414377525699066 for ; Tue, 03 Dec 2019 15:06:17 -0800 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 134.134.136.24, mailfrom: nathaniel.l.desimone@intel.com) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga004.jf.intel.com ([10.7.209.38]) by orsmga102.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 03 Dec 2019 15:06:16 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.69,275,1571727600"; d="scan'208";a="361360484" Received: from nldesimo-desk1.amr.corp.intel.com ([10.7.159.63]) by orsmga004.jf.intel.com with ESMTP; 03 Dec 2019 15:06:16 -0800 From: "Nate DeSimone" To: devel@edk2.groups.io Cc: Ashley E Desimone Subject: [edk2-staging/EdkRepo] [PATCH V2] EdkRepo: Add list-repos command Date: Tue, 3 Dec 2019 15:05:52 -0800 Message-Id: <20191203230552.1311-1-nathaniel.l.desimone@intel.com> X-Mailer: git-send-email 2.24.0.windows.2 MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable list-repos lists the git repos used by all projects and which branches those projects use. Cc: Ashley E Desimone Signed-off-by: Nate DeSimone --- edkrepo/commands/arguments/list_repos_args.py | 16 ++ edkrepo/commands/humble/__init__.py | 8 + edkrepo/commands/humble/list_repos_humble.py | 26 ++ edkrepo/commands/list_repos_command.py | 243 ++++++++++++++++++ setup.py | 5 +- 5 files changed, 295 insertions(+), 3 deletions(-) create mode 100644 edkrepo/commands/arguments/list_repos_args.py create mode 100644 edkrepo/commands/humble/__init__.py create mode 100644 edkrepo/commands/humble/list_repos_humble.py create mode 100644 edkrepo/commands/list_repos_command.py diff --git a/edkrepo/commands/arguments/list_repos_args.py b/edkrepo/comman= ds/arguments/list_repos_args.py new file mode 100644 index 0000000..6ff9b9c --- /dev/null +++ b/edkrepo/commands/arguments/list_repos_args.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3=0D +#=0D +## @file=0D +# list_repos_args.py=0D +#=0D +# Copyright (c) 2019, Intel Corporation. All rights reserved.
=0D +# SPDX-License-Identifier: BSD-2-Clause-Patent=0D +#=0D +=0D +''' Contains the help and description strings for arguments in the=0D +list-repos command meta data.=0D +'''=0D +=0D +COMMAND_DESCRIPTION =3D 'Lists the git repos used by available projects an= d the branches that are used.'=0D +ARCHIVED_HELP =3D 'Include a listing of archived projects.'=0D +REPOS_HELP =3D 'Only show the given subset of repos instead of all repos.'= =0D diff --git a/edkrepo/commands/humble/__init__.py b/edkrepo/commands/humble/= __init__.py new file mode 100644 index 0000000..dea6eb4 --- /dev/null +++ b/edkrepo/commands/humble/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3=0D +#=0D +## @file=0D +# __init__.py=0D +#=0D +# Copyright (c) 2019, Intel Corporation. All rights reserved.
=0D +# SPDX-License-Identifier: BSD-2-Clause-Patent=0D +#=0D diff --git a/edkrepo/commands/humble/list_repos_humble.py b/edkrepo/command= s/humble/list_repos_humble.py new file mode 100644 index 0000000..bbb05a7 --- /dev/null +++ b/edkrepo/commands/humble/list_repos_humble.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3=0D +#=0D +## @file=0D +# list_repos_humble.py=0D +#=0D +# Copyright (c) 2019, Intel Corporation. All rights reserved.
=0D +# SPDX-License-Identifier: BSD-2-Clause-Patent=0D +#=0D +=0D +'''=0D +Contains user visible strings printed by the list-repos command.=0D +'''=0D +=0D +from colorama import Fore=0D +from colorama import Style=0D +=0D +BRANCHES =3D 'Branches:'=0D +BRANCH_FORMAT_STRING =3D ' {}{}{{}}{}'.format(Fore.BLUE, Style.BRIGHT, St= yle.RESET_ALL)=0D +COMBO_FORMAT_STRING =3D ' {{}} {}{}({{}}){}'.format(Fore.CYAN, Style.B= RIGHT, Style.RESET_ALL)=0D +DEFAULT_COMBO_FORMAT_STRING =3D ' {{}} {}{}*({{}})*{}'.format(Fore.GREE= N, Style.BRIGHT, Style.RESET_ALL)=0D +MANIFEST_DIRECTORY =3D 'Manifest directory:'=0D +PROJECT_NAME_FORMAT_STRING =3D '{}{}{{}}{}:'.format(Fore.YELLOW, Style.BRI= GHT, Style.RESET_ALL)=0D +REPOSITORIES =3D 'Repositories:'=0D +REPO_NAME_AND_URL =3D '{}{}{{}}{} - [{}{}{{}}{}]'.format(Fore.MAGENTA, Sty= le.BRIGHT, Style.RESET_ALL, Fore.RED, Style.BRIGHT, Style.RESET_ALL)=0D +REPO_NAME_NOT_FOUND =3D 'repo_name not found'=0D +REPO_NOT_FOUND_IN_MANIFEST =3D 'Repo {} not found in any manifest file'=0D diff --git a/edkrepo/commands/list_repos_command.py b/edkrepo/commands/list= _repos_command.py new file mode 100644 index 0000000..9035993 --- /dev/null +++ b/edkrepo/commands/list_repos_command.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3=0D +#=0D +## @file=0D +# list_repos_command.py=0D +#=0D +# Copyright (c) 2019, Intel Corporation. All rights reserved.
=0D +# SPDX-License-Identifier: BSD-2-Clause-Patent=0D +#=0D +=0D +import collections=0D +import os=0D +=0D +#from git import Repo=0D +from colorama import Fore, Style=0D +=0D +# Our modules=0D +from edkrepo.commands.edkrepo_command import EdkrepoCommand=0D +from edkrepo.commands.edkrepo_command import ColorArgument=0D +import edkrepo.commands.arguments.list_repos_args as arguments=0D +import edkrepo.commands.humble.list_repos_humble as humble=0D +from edkrepo.common.common_repo_functions import pull_latest_manifest_repo= =0D +from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersExcep= tion, EdkrepoManifestInvalidException=0D +from edkrepo.common.ui_functions import init_color_console=0D +from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml=0D +=0D +class ListReposCommand(EdkrepoCommand):=0D + def __init__(self):=0D + super().__init__()=0D + self.repo_names =3D None=0D +=0D + def get_metadata(self):=0D + metadata =3D {}=0D + metadata['name'] =3D 'list-repos'=0D + metadata['help-text'] =3D arguments.COMMAND_DESCRIPTION=0D + args =3D []=0D + metadata['arguments'] =3D args=0D + args.append({'name': 'repos',=0D + 'positional': False,=0D + 'required': False,=0D + 'action': 'store',=0D + 'nargs': '+',=0D + 'help-text': arguments.REPOS_HELP})=0D + args.append({'name': 'archived',=0D + 'short-name': 'a',=0D + 'positional': False,=0D + 'required': False,=0D + 'help-text': arguments.ARCHIVED_HELP})=0D + args.append(ColorArgument)=0D + return metadata=0D +=0D + def run_command(self, args, config):=0D + print()=0D + init_color_console(args.color)=0D +=0D + # Get path to global manifest file=0D + global_manifest_directory =3D config['cfg_file'].manifest_repo_abs= _local_path=0D + if args.verbose:=0D + print(humble.MANIFEST_DIRECTORY)=0D + print(global_manifest_directory)=0D + print()=0D + index_path =3D os.path.join(global_manifest_directory, 'CiIndex.xm= l')=0D +=0D + pull_latest_manifest_repo(args, config)=0D + print()=0D +=0D + #Create a dictionary containing all the manifests listed in the Ci= Index.xml file=0D + ci_index_xml =3D CiIndexXml(index_path)=0D + manifests =3D {}=0D + repo_urls =3D set()=0D + project_list =3D list(ci_index_xml.project_list)=0D + if args.archived:=0D + project_list.extend(ci_index_xml.archived_project_list)=0D + for project in project_list:=0D + xml_file =3D ci_index_xml.get_project_xml(project)=0D + manifest =3D ManifestXml(os.path.normpath(os.path.join(global_= manifest_directory, xml_file)))=0D + manifests[project] =3D manifest=0D + for combo in [c.name for c in manifest.combinations]:=0D + sources =3D manifest.get_repo_sources(combo)=0D + for source in sources:=0D + repo_urls.add(self.get_repo_url(source.remote_url))=0D +=0D + #Sort the manifests so projects will be displayed alphabetically=0D + manifests =3D collections.OrderedDict(sorted(manifests.items()))=0D + project_justify =3D len(max(manifests.keys(), key=3Dlen))=0D +=0D + #Determine the names of the repositories=0D + self.generate_repo_names(repo_urls, manifests)=0D + print(humble.REPOSITORIES)=0D +=0D + #For each each git repository...=0D + for repo_name in self.repo_names:=0D + if args.repos and repo_name not in args.repos:=0D + continue=0D + repo =3D self.repo_names[repo_name][0]=0D + print(humble.REPO_NAME_AND_URL.format(repo_name, repo))=0D + print(humble.BRANCHES)=0D +=0D + #Determine the list of branches that used by any branch combin= ation in any manifest=0D + branches =3D set()=0D + for project_name in manifests:=0D + for combo in [c.name for c in manifests[project_name].comb= inations]:=0D + sources =3D manifests[project_name].get_repo_sources(c= ombo)=0D + for source in sources:=0D + if self.get_repo_url(source.remote_url) =3D=3D rep= o:=0D + branches.add(source.branch)=0D +=0D + #Sort the branch names so they will be displayed alphabeticall= y=0D + #with the exception that if a branch named "master" exists, th= en it=0D + #will be displayed first=0D + branches =3D sorted(branches, key=3Dstr.casefold)=0D + if 'master' in branches:=0D + branches.remove('master')=0D + branches.insert(0, 'master')=0D +=0D + #For each interesting branch in the current git repository...= =0D + for branch in branches:=0D + print(humble.BRANCH_FORMAT_STRING.format(branch))=0D +=0D + #Determine the branch combinations that use that branch=0D + for project_name in manifests:=0D + combos =3D []=0D + for combo in [c.name for c in manifests[project_name].= combinations]:=0D + sources =3D manifests[project_name].get_repo_sourc= es(combo)=0D + for source in sources:=0D + if self.get_repo_url(source.remote_url) =3D=3D= repo and source.branch =3D=3D branch:=0D + combos.append(combo)=0D + break=0D + if len(combos) > 0:=0D + #Sort the branch combinations so they will be disp= layed alphabetically=0D + #with the exception that the default branch combin= ation for the manifest=0D + #file will be displayed first=0D + combos =3D sorted(combos, key=3Dstr.casefold)=0D + default_combo =3D manifests[project_name].general_= config.default_combo=0D + if default_combo in combos:=0D + combos.remove(default_combo)=0D + combos.insert(0, default_combo)=0D + first_combo =3D True=0D + for combo in combos:=0D + #Print the project name=0D + if first_combo:=0D + project_name_print =3D humble.PROJECT_NAME= _FORMAT_STRING.format(project_name.ljust(project_justify))=0D + first_combo =3D False=0D + else:=0D + project_name_print =3D '{} '.format((' ' *= len(project_name)).ljust(project_justify))=0D + #Print the branch combination name, if this is= the default branch combination,=0D + #then print it in green color with *'s around = it=0D + if default_combo =3D=3D combo:=0D + print(humble.DEFAULT_COMBO_FORMAT_STRING.f= ormat(project_name_print, combo))=0D + else:=0D + print(humble.COMBO_FORMAT_STRING.format(pr= oject_name_print, combo))=0D +=0D + def get_repo_url(self, repo_url):=0D + if repo_url[-4:].lower() =3D=3D '.git':=0D + return repo_url[:-4]=0D + return repo_url=0D +=0D + def get_repo_name(self, repo_url, manifests):=0D + for name in self.repo_names:=0D + if self.repo_names[name][0] =3D=3D repo_url:=0D + return name=0D + raise EdkrepoInvalidParametersException(humble.REPO_NAME_NOT_FOUND= )=0D +=0D + def generate_repo_names(self, repo_urls, manifests):=0D + #Determine the names of the repositories=0D + self.repo_names =3D collections.OrderedDict()=0D + for repo_url in repo_urls:=0D + self.__repo_name_worker(repo_url, manifests)=0D +=0D + #Sort the git repositories so they will be displayed alphabeticall= y=0D + self.repo_names =3D collections.OrderedDict(sorted(self.repo_names= .items()))=0D + names_to_move =3D []=0D + for repo_name in self.repo_names:=0D + if repo_name.lower().find('edk2') =3D=3D 0:=0D + names_to_move.append(repo_name)=0D + names_to_move =3D sorted(names_to_move, reverse=3DTrue)=0D + for name_to_move in names_to_move:=0D + self.repo_names.move_to_end(name_to_move, False)=0D + names_to_move =3D []=0D + for repo_name in self.repo_names:=0D + if repo_name.lower().find('intel') =3D=3D 0:=0D + names_to_move.append(repo_name)=0D + names_to_move =3D sorted(names_to_move, reverse=3DTrue)=0D + for name_to_move in names_to_move:=0D + self.repo_names.move_to_end(name_to_move, False)=0D +=0D + def __repo_name_worker(self, repo_url, manifests):=0D + #This is a heuristic that guesses the "name" of a repository by lo= oking=0D + #at the name given to it by the most manifest files.=0D + names =3D collections.defaultdict(int)=0D + for project_name in manifests:=0D + for combo in [c.name for c in manifests[project_name].combinat= ions]:=0D + sources =3D manifests[project_name].get_repo_sources(combo= )=0D + for source in sources:=0D + if self.get_repo_url(source.remote_url) =3D=3D repo_ur= l:=0D + names[source.root] +=3D 1=0D + found_unique_name =3D False=0D + original_best_name =3D None=0D + original_best_name_frequency =3D 0=0D + while not found_unique_name:=0D + best_name =3D None=0D + best_name_frequency =3D 0=0D + if len(names) <=3D 0:=0D + if original_best_name_frequency =3D=3D 1:=0D + #If only 1 project uses this name, then append the pro= ject=0D + #name to the directory name to create the repo name=0D + for project_name in manifests:=0D + for combo in [c.name for c in manifests[project_na= me].combinations]:=0D + sources =3D manifests[project_name].get_repo_s= ources(combo)=0D + for source in sources:=0D + if self.get_repo_url(source.remote_url) = =3D=3D repo_url and source.root =3D=3D original_best_name:=0D + best_name =3D "{}-{}".format(original_= best_name, project_name)=0D + best_name_frequency =3D original_best_= name_frequency=0D + else:=0D + best_name =3D repo_url=0D + best_name_frequency =3D 0=0D + break=0D + for name in names:=0D + if names[name] > best_name_frequency:=0D + best_name =3D name=0D + best_name_frequency =3D names[name]=0D + if best_name is None:=0D + raise EdkrepoManifestInvalidException(humble.REPO_NOT_FOUN= D_IN_MANIFEST.format(repo_url))=0D + if original_best_name is None:=0D + original_best_name =3D best_name=0D + original_best_name_frequency =3D best_name_frequency=0D + if best_name in self.repo_names:=0D + if self.repo_names[best_name][0] =3D=3D repo_url:=0D + found_unique_name =3D True=0D + else:=0D + #If there is a name collision, then which repo has the= most=0D + #Usage of the name owns the name=0D + if best_name_frequency > self.repo_names[best_name][1]= :=0D + old_repo_url =3D self.repo_names[name][0]=0D + del self.repo_names[best_name]=0D + found_unique_name =3D True=0D + self.repo_names[best_name] =3D (repo_url, best_nam= e_frequency)=0D + self.__repo_name_worker(old_repo_url, manifests)=0D + else:=0D + #Use the name given by the second most manifest fi= les=0D + del names[best_name]=0D + else:=0D + found_unique_name =3D True=0D + self.repo_names[best_name] =3D (repo_url, best_name_frequency)=0D diff --git a/setup.py b/setup.py index 73ce9a9..1b9edad 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ ## @file # setup.py # -# Copyright (c) 2017- 2019, Intel Corporation. All rights reserved.
+# Copyright (c) 2017 - 2019, Intel Corporation. All rights reserved.
# SPDX-License-Identifier: BSD-2-Clause-Patent # =20 @@ -12,7 +12,7 @@ from setuptools import setup setup(name=3D'edkrepo', version=3D'2.0.0', description=3D'The edkrepo tools', - packages=3D['edkrepo', 'edkrepo.commands', 'edkrepo.commands.argumen= ts', 'edkrepo.common', 'edkrepo.config', 'edkrepo_manifest_parser', 'projec= t_utils'], + packages=3D['edkrepo', 'edkrepo.commands', 'edkrepo.commands.argumen= ts', 'edkrepo.commands.humble', 'edkrepo.common', 'edkrepo.config', 'edkrep= o_manifest_parser', 'project_utils'], package_data=3D{ }, include_package_data=3DTrue, @@ -22,4 +22,3 @@ setup(name=3D'edkrepo', ] } ) - --=20 2.24.0.windows.2