From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from us-smtp-delivery-124.mimecast.com (us-smtp-delivery-124.mimecast.com [170.10.133.124]) by mx.groups.io with SMTP id smtpd.web11.15096.1678696227185080663 for ; Mon, 13 Mar 2023 01:30:27 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@redhat.com header.s=mimecast20190719 header.b=Xn2xPOyn; spf=pass (domain: redhat.com, ip: 170.10.133.124, mailfrom: kraxel@redhat.com) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=redhat.com; s=mimecast20190719; t=1678696226; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=/vWxzCoOhB7VoEA54x5bvcL72ac7BGvrmSrykO9WOSs=; b=Xn2xPOynygHbLXsweyy9j7az3KqQlo9Sjme+cFOLXe128ozZjR2NGyLzbKNKPVdD48YIIY O9w0Whn/Tve7aL1+CwUYXZmlnXvybnC1bafo+EvUBKSZR7FNJ53kb/5xDK/JHCiOPzZTYi aa4VOa+hOaautw1WetM3cVVfvSLU6P8= Received: from mimecast-mx02.redhat.com (mx3-rdu2.redhat.com [66.187.233.73]) by relay.mimecast.com with ESMTP with STARTTLS (version=TLSv1.2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id us-mta-364-XxnOZ83gPleoRDZ-4SEKAQ-1; Mon, 13 Mar 2023 04:30:22 -0400 X-MC-Unique: XxnOZ83gPleoRDZ-4SEKAQ-1 Received: from smtp.corp.redhat.com (int-mx10.intmail.prod.int.rdu2.redhat.com [10.11.54.10]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mimecast-mx02.redhat.com (Postfix) with ESMTPS id 68F493C10141; Mon, 13 Mar 2023 08:30:22 +0000 (UTC) Received: from sirius.home.kraxel.org (unknown [10.39.192.142]) by smtp.corp.redhat.com (Postfix) with ESMTPS id DD04B400F50; Mon, 13 Mar 2023 08:30:21 +0000 (UTC) Received: by sirius.home.kraxel.org (Postfix, from userid 1000) id 486571801E81; Mon, 13 Mar 2023 09:30:16 +0100 (CET) From: "Gerd Hoffmann" To: devel@edk2.groups.io Cc: Jian J Wang , Pawel Polawski , Xiaoyu Lu , Ard Biesheuvel , Guomin Jiang , Gerd Hoffmann , Jiewen Yao , Oliver Steffen , Jordan Justen Subject: [PATCH 04/22] CryptoPkg/openssl: add openssl3 configure scripts Date: Mon, 13 Mar 2023 09:29:58 +0100 Message-Id: <20230313083016.136448-5-kraxel@redhat.com> In-Reply-To: <20230313083016.136448-1-kraxel@redhat.com> References: <20230313083016.136448-1-kraxel@redhat.com> MIME-Version: 1.0 X-Scanned-By: MIMEDefang 3.1 on 10.11.54.10 X-Mimecast-Spam-Score: 0 X-Mimecast-Originator: redhat.com Content-Transfer-Encoding: 8bit Content-Type: text/plain; charset="US-ASCII"; x-default=true Rewrite the script to configure openssl 3.0 from scratch. It's two scripts now: * Tiny helper script, dumping the perl configdata as json. * Actual configure.py script, written in python, which copies over the generated files to openssl-gen and updates the OpensslLib*.inf file lists and build flags. The configuration workflow has changed a bit: * All generated files are stored in the openssl-gen directory tree. * For ec/no-ec builds two different header files are used. Default is the ec variant, and the new EDK2_OPENSSL_NOEC define is used to select the no-ec build. A five line wrapper include is used to pick the one or the other. * For non-accel builds -DOPENSSL_NO_ASM on the command line is used (same as before). * For configration defines the OPENSSL_FLAGS_$(variant) variable is used, where variant is the architecture for the accelerated builds and 'NOASM' for the non-accelerated builds. Signed-off-by: Gerd Hoffmann --- CryptoPkg/Library/OpensslLib/configure.py | 365 ++++++++++++++++++++++ CryptoPkg/Library/OpensslLib/perl2json.pl | 19 ++ 2 files changed, 384 insertions(+) create mode 100755 CryptoPkg/Library/OpensslLib/configure.py create mode 100755 CryptoPkg/Library/OpensslLib/perl2json.pl diff --git a/CryptoPkg/Library/OpensslLib/configure.py b/CryptoPkg/Library/OpensslLib/configure.py new file mode 100755 index 000000000000..d8aa0cb03edf --- /dev/null +++ b/CryptoPkg/Library/OpensslLib/configure.py @@ -0,0 +1,365 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: BSD-2-Clause-Patent +import os +import sys +import json +import shutil +import pprint +import argparse +import subprocess + +def openssl_configure(openssldir, target, ec = True): + """ Run openssl Configure script. """ + cmdline = [ + 'perl', + 'Configure', + '--config=../UefiAsm.conf', + '--api=1.1.1', + '--with-rand-seed=none', + target, + 'no-afalgeng', + 'no-async', + 'no-autoerrinit', + 'no-autoload-config', + 'no-bf', + 'no-blake2', + 'no-camellia', + 'no-capieng', + 'no-cast', + 'no-chacha', + 'no-cmac', + 'no-cms', + 'no-ct', + 'no-deprecated', + 'no-des', + 'no-dgram', + 'no-dsa', + 'no-dynamic-engine', + 'no-ec2m', + 'no-engine', + 'no-err', + 'no-filenames', + 'no-gost', + 'no-idea', + 'no-md4', + 'no-mdc2', + 'no-pic', + 'no-ocb', + 'no-poly1305', + 'no-posix-io', + 'no-rc2', + 'no-rc4', + 'no-rfc3779', + 'no-rmd160', + 'no-scrypt', + 'no-seed', + 'no-sm4', + 'no-sock', + 'no-srp', + 'no-srtp', + 'no-ssl', + 'no-stdio', + 'no-threads', + 'no-ts', + 'no-ui', + 'no-whirlpool', + ] + if not ec: + cmdline += [ 'no-ec', ] + print('') + print(f'# -*- configure openssl for {target} (ec={ec}) -*-') + rc = subprocess.run(cmdline, cwd = openssldir, + stdout = subprocess.PIPE, + stderr = subprocess.PIPE) + if rc.returncode: + print(rc.stdout) + print(rc.stderr) + sys.exit(rc.returncode) + +def openssl_run_make(openssldir, target): + """ + Run make utility to generate files or cleanup. + Target can be either a string or a list of strings. + """ + cmdline = [ 'make', '--silent' ] + if isinstance(target, list): + cmdline += target + else: + cmdline += [ target, ] + rc = subprocess.run(cmdline, cwd = openssldir) + rc.check_returncode() + +def get_configdata(openssldir): + """ + Slurp openssl config data as JSON, + using a little perl helper script. + """ + cmdline = [ + 'perl', + 'perl2json.pl', + openssldir, + ] + rc = subprocess.run(cmdline, stdout = subprocess.PIPE) + rc.check_returncode() + return json.loads(rc.stdout) + +def is_asm(filename): + """ Check whenevr the passed file is an assembler file """ + if filename.endswith('.s') or filename.endswith('.S'): + return True + return False + +def generate_files(openssldir, opensslgendir, asm, filelist): + """ + Generate files, using make, and copy over the results to the + directory tree for generated openssl files. Creates + subdirectories as needed. + """ + openssl_run_make(openssldir, filelist) + for filename in filelist: + src = os.path.join(openssldir, filename) + if is_asm(filename): + dst = os.path.join(opensslgendir, asm, filename) + else: + dst = os.path.join(opensslgendir, filename) + os.makedirs(os.path.dirname(dst), exist_ok = True) + shutil.copyfile(src, dst) + +def generate_include_files(openssldir, opensslgendir, asm, cfg): + """ Generate openssl include files """ + print('# generate include files') + filelist = cfg['unified_info']['generate'].keys() + filelist = list(filter(lambda f: 'include' in f, filelist)) + generate_files(openssldir, opensslgendir, asm, filelist) + +def generate_library_files(openssldir, opensslgendir, asm, cfg, obj): + """ + Generate openssl source files for a given library. Handles + mostly assembler files, but a few C sources are generated too. + """ + filelist = get_source_list(cfg, obj, True) + if filelist: + print(f'# generate source files for {obj}') + generate_files(openssldir, opensslgendir, asm, filelist) + +def generate_all_files(openssldir, opensslgendir, asm, cfg): + """ Generate all files needed. """ + generate_include_files(openssldir, opensslgendir, asm, cfg) + generate_library_files(openssldir, opensslgendir, asm, cfg, 'libcrypto') + generate_library_files(openssldir, opensslgendir, asm, cfg, 'providers/libcommon.a') + generate_library_files(openssldir, opensslgendir, asm, cfg, 'libssl') + +def get_source_list(cfg, obj, gen): + """ + Gets the list of source files needed to create a specific object. + * If 'gen' is True the function returns the list of generated + files. + * If 'gen' is False the function returns the list of files not + generated (which are used from the submodule directly). + Note: Will call itself recursively to resolve nested dependencies. + """ + sources = cfg['unified_info']['sources'] + generate = cfg['unified_info']['generate'] + srclist = [] + if sources.get(obj): + for item in sources.get(obj): + srclist += get_source_list(cfg, item, gen) + else: + is_generated = generate.get(obj) is not None + if is_generated == gen: + srclist += [ obj, ] + return srclist + +def get_sources(cfg, obj, asm): + """ + Get the list of all sources files. Will fetch both generated + and not generated file lists and update the paths accordingly, so + the openssl submodule or the sub-tree for generated files is + referenced as needed. + """ + srclist = get_source_list(cfg, obj, False) + genlist = get_source_list(cfg, obj, True) + srclist = list(map(lambda x: f'$(OPENSSL_PATH)/{x}', srclist)) + c_list = list(map(lambda x: f'$(OPENSSL_GEN_PATH)/{x}', + filter(lambda x: not is_asm(x), genlist))) + asm_list = list(map(lambda x: f'$(OPENSSL_GEN_PATH)/{asm}/{x}', + filter(is_asm, genlist))) + return srclist + c_list + asm_list + +def sources_filter_fn(filename): + """ + Filter source lists. Drops files we don't want include or + need replace with our own uefi-specific version. + """ + exclude = [ + 'randfile.c', + '/store/', + '/storemgmt/', + ] + for item in exclude: + if item in filename: + return False + return True + +def hash_filter_fn(filename): + """ + Filter source lists. Include source files with hash functions only. + """ + include = [ + '/sha/', + '/sm3/', + 'mem_clr.c', + ] + exclude = [ + 'sha1_one.c', + ] + for item in exclude: + if item in filename: + return False + for item in include: + if item in filename: + return True + return False + +def libcrypto_sources(cfg, asm = None): + """ Get source file list for libcrypto """ + files = get_sources(cfg, 'libcrypto', asm) + files += get_sources(cfg, 'providers/libcommon.a', asm) + files = list(filter(sources_filter_fn, files)) + return files + +def libssl_sources(cfg, asm = None): + """ Get source file list for libssl """ + files = get_sources(cfg, 'libssl', asm) + files = list(filter(sources_filter_fn, files)) + return files + +def hash_sources(cfg, asm = None): + """ Get source file list for hash functions """ + files = get_sources(cfg, 'libcrypto', asm) + files = list(filter(hash_filter_fn, files)) + return files + +def update_inf(filename, sources, arch = None, defines = []): + """ + Update inf file, replace source file list and build flags. + """ + head = '' + tail = '' + state = 0 + + if arch: + section = f'Sources.{arch}' + flags = f'OPENSSL_FLAGS_{arch}' + else: + section = None + flags = f'OPENSSL_FLAGS_NOASM' + state = 1 + + # read and parse file + with open(filename, 'r') as f: + while True: + line = f.readline() + if line == '': + break + if state in [0, 1]: + if flags in line: + (keep, replace) = line.split('=') + args = map(lambda x: f'-D{x}', defines) + head += keep + '= ' + ' '.join(args) + '\r\n' + else: + head += line.rstrip() + '\r\n' + if state == 0 and section in line: + state = 1 + if state == 1 and 'Autogenerated files list starts here' in line: + state = 2 + if state == 2 and 'Autogenerated files list ends here' in line: + state = 3 + if state == 3: + tail += line.rstrip() + '\r\n' + + # write updated file + with open(filename, 'w') as f: + f.write(head) + for src in sources: + f.write(f' {src}\r\n') + f.write(tail) + +def main(): + # prepare + os.chdir(os.path.dirname(__file__)) + openssldir = os.path.join(os.getcwd(), 'openssl') + opensslgendir = os.path.join(os.getcwd(), 'openssl-gen') + + # asm accel configs (see UefiAsm.conf) + for ec in [True, False]: + if ec: + inf = 'OpensslLibFullAccel.inf' + hdr = 'configuration-ec.h' + else: + inf = 'OpensslLibAccel.inf' + hdr = 'configuration-noec.h' + sources = {} + defines = {} + for asm in [ 'UEFI-IA32-MSFT', 'UEFI-IA32-GCC', + 'UEFI-X64-MSFT', 'UEFI-X64-GCC', + 'UEFI-AARCH64-GCC' ]: + (uefi, arch, cc) = asm.split('-') + archcc = f'{arch}-{cc}' + + openssl_configure(openssldir, asm, ec = ec); + cfg = get_configdata(openssldir) + generate_all_files(openssldir, opensslgendir, archcc, cfg) + shutil.move(os.path.join(opensslgendir, 'include', 'openssl', 'configuration.h'), + os.path.join(opensslgendir, 'include', 'openssl', hdr)) + openssl_run_make(openssldir, 'distclean') + + srclist = libcrypto_sources(cfg, archcc) + libssl_sources(cfg, archcc) + sources[archcc] = list(map(lambda x: f'{x} | {cc}', filter(is_asm, srclist))) + sources[arch] = list(filter(lambda x: not is_asm(x), srclist)) + defines[arch] = cfg['unified_info']['defines']['libcrypto'] + + ia32accel = sources['IA32'] + sources['IA32-MSFT'] + sources['IA32-GCC'] + x64accel = sources['X64'] + sources['X64-MSFT'] + sources['X64-GCC'] + aa64accel = sources['AARCH64'] + sources['AARCH64-GCC'] + update_inf(inf, ia32accel, 'IA32', defines['IA32']) + update_inf(inf, x64accel, 'X64', defines['X64']) + update_inf(inf, aa64accel, 'AARCH64', defines['AARCH64']) + + # noaccel - ec enabled + openssl_configure(openssldir, 'UEFI', ec = True); + cfg = get_configdata(openssldir) + generate_all_files(openssldir, opensslgendir, None, cfg) + openssl_run_make(openssldir, 'distclean') + + update_inf('OpensslLibFull.inf', + libcrypto_sources(cfg) + libssl_sources(cfg), + None, cfg['unified_info']['defines']['libcrypto']) + + # noaccel - ec disabled + openssl_configure(openssldir, 'UEFI', ec = False); + cfg = get_configdata(openssldir) + generate_all_files(openssldir, opensslgendir, None, cfg) + openssl_run_make(openssldir, 'distclean') + + update_inf('OpensslLibCrypto.inf', + libcrypto_sources(cfg), + None, cfg['unified_info']['defines']['libcrypto']) + update_inf('OpensslLib.inf', + libcrypto_sources(cfg) + libssl_sources(cfg), + None, cfg['unified_info']['defines']['libcrypto']) + update_inf('OpensslLibHash.inf', + hash_sources(cfg), + None, cfg['unified_info']['defines']['libcrypto']) + + # wrap header file + confighdr = os.path.join(opensslgendir, 'include', 'openssl', 'configuration.h') + with open(confighdr, 'w') as f: + f.write('#ifdef EDK2_OPENSSL_NOEC\n' + '# include "configuration-noec.h"\n' + '#else\n' + '# include "configuration-ec.h"\n' + '#endif\n') + +if __name__ == '__main__': + sys.exit(main()) diff --git a/CryptoPkg/Library/OpensslLib/perl2json.pl b/CryptoPkg/Library/OpensslLib/perl2json.pl new file mode 100755 index 000000000000..f7364596a73d --- /dev/null +++ b/CryptoPkg/Library/OpensslLib/perl2json.pl @@ -0,0 +1,19 @@ +#!/usr/bin/perl +# +# write out configdata.pm as json +# +use strict; +use warnings; +use JSON; + +BEGIN { + my $openssldir = shift; + push @INC, $openssldir; +} +use configdata qw/%config %target %unified_info/; + +my %data; +$data{'config'} = \%config; +$data{'target'} = \%target; +$data{'unified_info'} = \%unified_info; +print encode_json(\%data) -- 2.39.2