* [PATCH 4/4] BaseTools: Add Brotli algorithm tool
@ 2017-03-23 2:16 Song, BinX
0 siblings, 0 replies; only message in thread
From: Song, BinX @ 2017-03-23 2:16 UTC (permalink / raw)
To: edk2-devel@lists.01.org; +Cc: Gao, Liming
- Add Brotli algorithm tool support
Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: Bell Song <binx.song@intel.com>
---
BaseTools/BinWrappers/PosixLike/Brotli | 29 ++++++
BaseTools/BinWrappers/PosixLike/BrotliCompress | 42 ++++++++
BaseTools/Conf/tools_def.template | 6 ++
.../Source/C/BrotliCompress/BrotliCompress.bat | 48 ++++++++++
BaseTools/Source/C/BrotliCompress/GNUmakefile | 43 +++++++++
BaseTools/Source/C/BrotliCompress/Makefile | 60 ++++++++++++
BaseTools/Source/C/BrotliCompress/ReadMe.txt | 2 +
BaseTools/Source/C/BrotliCompress/tools/bro.c | 106 +++++++++++++++++----
BaseTools/Source/C/GNUmakefile | 1 +
BaseTools/Source/C/Makefile | 1 +
10 files changed, 318 insertions(+), 20 deletions(-)
create mode 100644 BaseTools/BinWrappers/PosixLike/Brotli
create mode 100644 BaseTools/BinWrappers/PosixLike/BrotliCompress
create mode 100644 BaseTools/Source/C/BrotliCompress/BrotliCompress.bat
create mode 100644 BaseTools/Source/C/BrotliCompress/GNUmakefile
create mode 100644 BaseTools/Source/C/BrotliCompress/Makefile
create mode 100644 BaseTools/Source/C/BrotliCompress/ReadMe.txt
diff --git a/BaseTools/BinWrappers/PosixLike/Brotli b/BaseTools/BinWrappers/PosixLike/Brotli
new file mode 100644
index 0000000..0945d86
--- /dev/null
+++ b/BaseTools/BinWrappers/PosixLike/Brotli
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+dir=$(dirname "$full_cmd")
+cmd=${full_cmd##*/}
+
+if [ -n "$WORKSPACE" ] && [ -e "$WORKSPACE/Conf/BaseToolsCBinaries" ]
+then
+ exec "$WORKSPACE/Conf/BaseToolsCBinaries/$cmd"
+elif [ -n "$WORKSPACE" ] && [ -e "$EDK_TOOLS_PATH/Source/C" ]
+then
+ if [ ! -e "$EDK_TOOLS_PATH/Source/C/bin/$cmd" ]
+ then
+ echo "BaseTools C Tool binary was not found ($cmd)"
+ echo "You may need to run:"
+ echo " make -C $EDK_TOOLS_PATH/Source/C"
+ else
+ exec "$EDK_TOOLS_PATH/Source/C/bin/$cmd" "$@"
+ fi
+elif [ -e "$dir/../../Source/C/bin/$cmd" ]
+then
+ exec "$dir/../../Source/C/bin/$cmd" "$@"
+else
+ echo "Unable to find the real '$cmd' to run"
+ echo "This message was printed by"
+ echo " $0"
+ exit 127
+fi
+
diff --git a/BaseTools/BinWrappers/PosixLike/BrotliCompress b/BaseTools/BinWrappers/PosixLike/BrotliCompress
new file mode 100644
index 0000000..59c6465
--- /dev/null
+++ b/BaseTools/BinWrappers/PosixLike/BrotliCompress
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# This script will exec Brotli tool.
+#
+# Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
+# This program and the accompanying materials
+# are licensed and made available under the terms and conditions of the BSD License
+# which accompanies this distribution. The full text of the license may be found at
+# http://opensource.org/licenses/bsd-license.php
+#
+# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
+# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+#
+LVL="--quality 9"
+
+while [ $# != 0 ];do
+ case $1 in
+ -d)
+ ARGS+="--decompress "
+ ;;
+ -e)
+ ;;
+ -g)
+ ARGS+="--gap $2 "
+ shift
+ ;;
+ -l)
+ LVL="--quality $2 "
+ shift
+ ;;
+ -o)
+ ARGS+="--output $2 "
+ shift
+ ;;
+ *)
+ ARGS+="--input $1 "
+ esac
+
+shift
+done
+
+exec Brotli $ARGS $LVL
diff --git a/BaseTools/Conf/tools_def.template b/BaseTools/Conf/tools_def.template
index aaae4fc..168e42a 100755
--- a/BaseTools/Conf/tools_def.template
+++ b/BaseTools/Conf/tools_def.template
@@ -7691,6 +7691,12 @@ RELEASE_RVCTCYGWIN_ARM_CC_FLAGS = "$(CCPATH_FLAG)" $(ARCHCC_FLAGS) $(PLATFORM_F
*_*_*_RSA2048SHA256SIGN_GUID = A7717414-C616-4977-9420-844712A735BF
##################
+# BrotliCompress tool definitions
+##################
+*_*_*_BROTLI_PATH = BrotliCompress
+*_*_*_BROTLI_GUID = 3D532050-5CDA-4FD0-879E-0F7F630D5AFB
+
+##################
# LzmaCompress tool definitions
##################
*_*_*_LZMA_PATH = LzmaCompress
diff --git a/BaseTools/Source/C/BrotliCompress/BrotliCompress.bat b/BaseTools/Source/C/BrotliCompress/BrotliCompress.bat
new file mode 100644
index 0000000..62d10f7
--- /dev/null
+++ b/BaseTools/Source/C/BrotliCompress/BrotliCompress.bat
@@ -0,0 +1,48 @@
+@echo off
+@setlocal
+
+set LVL=--quality 9
+
+:Begin
+if "%1"=="" goto End
+
+if "%1"=="-d" (
+ set ARGS=%ARGS% --decompress
+ shift
+ goto Begin
+)
+
+if "%1"=="-e" (
+ shift
+ goto Begin
+)
+
+if "%1"=="-g" (
+ set ARGS=%ARGS% --gap %2
+ shift
+ shift
+ goto Begin
+)
+
+if "%1"=="-l" (
+ set LVL=--quality %2
+ shift
+ shift
+ goto Begin
+)
+
+if "%1"=="-o" (
+ set ARGS=%ARGS% --output %2
+ set INTMP=%2
+ shift
+ shift
+ goto Begin
+)
+
+set ARGS=%ARGS% --input %1
+shift
+goto Begin
+
+:End
+Brotli %ARGS% %LVL%
+@echo on
\ No newline at end of file
diff --git a/BaseTools/Source/C/BrotliCompress/GNUmakefile b/BaseTools/Source/C/BrotliCompress/GNUmakefile
new file mode 100644
index 0000000..89203c8
--- /dev/null
+++ b/BaseTools/Source/C/BrotliCompress/GNUmakefile
@@ -0,0 +1,43 @@
+## @file
+# GNU/Linux makefile for 'Brotli' module build.
+#
+# Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
+# This program and the accompanying materials
+# are licensed and made available under the terms and conditions of the BSD License
+# which accompanies this distribution. The full text of the license may be found at
+# http://opensource.org/licenses/bsd-license.php
+#
+# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
+# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+#
+ARCH ?= IA32
+MAKEROOT ?= ..
+
+APPNAME = Brotli
+
+OBJECTS = \
+ tools/bro.o \
+ common/dictionary.o \
+ dec/bit_reader.o \
+ dec/decode.o \
+ dec/huffman.o \
+ dec/state.o \
+ enc/backward_references.o \
+ enc/bit_cost.o \
+ enc/block_splitter.o \
+ enc/brotli_bit_stream.o \
+ enc/cluster.o \
+ enc/compress_fragment.o \
+ enc/compress_fragment_two_pass.o \
+ enc/encode.o \
+ enc/entropy_encode.o \
+ enc/histogram.o \
+ enc/literal_cost.o \
+ enc/memory.o \
+ enc/metablock.o \
+ enc/static_dict.o \
+ enc/utf8_util.o
+
+include $(MAKEROOT)/Makefiles/app.makefile
+
+LIBS += -lm
diff --git a/BaseTools/Source/C/BrotliCompress/Makefile b/BaseTools/Source/C/BrotliCompress/Makefile
new file mode 100644
index 0000000..9d45ea5
--- /dev/null
+++ b/BaseTools/Source/C/BrotliCompress/Makefile
@@ -0,0 +1,60 @@
+## @file
+# Windows makefile for 'Brotli' module build.
+#
+# Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
+# This program and the accompanying materials
+# are licensed and made available under the terms and conditions of the BSD License
+# which accompanies this distribution. The full text of the license may be found at
+# http://opensource.org/licenses/bsd-license.php
+#
+# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
+# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+#
+!INCLUDE ..\Makefiles\ms.common
+
+CFLAGS = $(CFLAGS) /W2
+
+APPNAME = Brotli
+
+#LIBS = $(LIB_PATH)\Common.lib
+
+COMMON_OBJ = common\dictionary.obj
+DEC_OBJ = \
+ dec\bit_reader.obj \
+ dec\decode.obj \
+ dec\huffman.obj \
+ dec\state.obj
+ENC_OBJ = \
+ enc\backward_references.obj \
+ enc\bit_cost.obj \
+ enc\block_splitter.obj \
+ enc\brotli_bit_stream.obj \
+ enc\cluster.obj \
+ enc\compress_fragment.obj \
+ enc\compress_fragment_two_pass.obj \
+ enc\encode.obj \
+ enc\entropy_encode.obj \
+ enc\histogram.obj \
+ enc\literal_cost.obj \
+ enc\memory.obj \
+ enc\metablock.obj \
+ enc\static_dict.obj \
+ enc\utf8_util.obj
+
+OBJECTS = \
+ tools\bro.obj \
+ $(COMMON_OBJ) \
+ $(DEC_OBJ) \
+ $(ENC_OBJ)
+
+!INCLUDE ..\Makefiles\ms.app
+
+all: $(BIN_PATH)\BrotliCompress.bat
+
+$(BIN_PATH)\BrotliCompress.bat: BrotliCompress.bat
+ copy BrotliCompress.bat $(BIN_PATH)\BrotliCompress.bat /Y
+
+cleanall: localCleanall
+
+localCleanall:
+ del /f /q $(BIN_PATH)\BrotliCompress.bat > nul
diff --git a/BaseTools/Source/C/BrotliCompress/ReadMe.txt b/BaseTools/Source/C/BrotliCompress/ReadMe.txt
new file mode 100644
index 0000000..c19c0a1
--- /dev/null
+++ b/BaseTools/Source/C/BrotliCompress/ReadMe.txt
@@ -0,0 +1,2 @@
+It is based on the Brotli v0.5.2.
+Brotli was released on the website https://github.com/google/brotli.
diff --git a/BaseTools/Source/C/BrotliCompress/tools/bro.c b/BaseTools/Source/C/BrotliCompress/tools/bro.c
index 426c573..f11d192 100644
--- a/BaseTools/Source/C/BrotliCompress/tools/bro.c
+++ b/BaseTools/Source/C/BrotliCompress/tools/bro.c
@@ -73,6 +73,7 @@ static void ParseArgv(int argc, char **argv,
char **dictionary_path,
int *force,
int *quality,
+ int *gapmem,
int *decompress,
int *repeat,
int *verbose,
@@ -160,6 +161,13 @@ static void ParseArgv(int argc, char **argv,
}
++k;
continue;
+ } else if (!strcmp("--gap", argv[k]) ||
+ !strcmp("-g", argv[k])) {
+ if (!ParseQuality(argv[k + 1], gapmem)) {
+ goto error;
+ }
+ ++k;
+ continue;
}
}
goto error;
@@ -167,7 +175,7 @@ static void ParseArgv(int argc, char **argv,
return;
error:
fprintf(stderr,
- "Usage: %s [--force] [--quality n] [--decompress]"
+ "Usage: %s [--force] [--quality n] [--gap n] [--decompress]"
" [--input filename] [--output filename] [--repeat iters]"
" [--verbose] [--window n] [--custom-dictionary filename]\n",
argv[0]);
@@ -225,9 +233,20 @@ static int64_t FileSize(const char *path) {
return retval;
}
+/* Brotli specified memory allocate function */
+static void *BrAlloc (void *memsize, size_t size) {
+ *(int64_t *)memsize += size;
+ return malloc(size);
+}
+
+/* Brotli specified memory free function */
+static void BrFree (void *memsize, void *ptr) {
+ free(ptr);
+}
+
/* Result ownersip is passed to caller.
|*dictionary_size| is set to resulting buffer size. */
-static uint8_t* ReadDictionary(const char* path, size_t* dictionary_size) {
+static uint8_t* ReadDictionary(const char* path, size_t* dictionary_size, void *memsize) {
static const int kMaxDictionarySize = (1 << 24) - 16;
FILE *f = fopen(path, "rb");
int64_t file_size_64;
@@ -252,7 +271,7 @@ static uint8_t* ReadDictionary(const char* path, size_t* dictionary_size) {
}
*dictionary_size = (size_t)file_size_64;
- buffer = (uint8_t*)malloc(*dictionary_size);
+ buffer = (uint8_t *)BrAlloc(memsize, *dictionary_size);
if (!buffer) {
fprintf(stderr, "could not read dictionary: out of memory\n");
exit(1);
@@ -268,7 +287,7 @@ static uint8_t* ReadDictionary(const char* path, size_t* dictionary_size) {
static const size_t kFileBufferSize = 65536;
-static int Decompress(FILE* fin, FILE* fout, const char* dictionary_path) {
+static int Decompress(FILE* fin, FILE* fout, const char* dictionary_path, void *memsize) {
/* Dictionary should be kept during first rounds of decompression. */
uint8_t* dictionary = NULL;
uint8_t* input;
@@ -279,18 +298,18 @@ static int Decompress(FILE* fin, FILE* fout, const char* dictionary_path) {
size_t available_out = kFileBufferSize;
uint8_t* next_out;
BrotliResult result = BROTLI_RESULT_ERROR;
- BrotliState* s = BrotliCreateState(NULL, NULL, NULL);
+ BrotliState* s = BrotliCreateState(BrAlloc, BrFree, memsize);
if (!s) {
fprintf(stderr, "out of memory\n");
return 0;
}
if (dictionary_path != NULL) {
size_t dictionary_size = 0;
- dictionary = ReadDictionary(dictionary_path, &dictionary_size);
+ dictionary = ReadDictionary(dictionary_path, &dictionary_size, memsize);
BrotliSetCustomDictionary(dictionary_size, dictionary, s);
}
- input = (uint8_t*)malloc(kFileBufferSize);
- output = (uint8_t*)malloc(kFileBufferSize);
+ input = (uint8_t*)BrAlloc(memsize, kFileBufferSize);
+ output = (uint8_t*)BrAlloc(memsize, kFileBufferSize);
if (!input || !output) {
fprintf(stderr, "out of memory\n");
goto end;
@@ -331,17 +350,17 @@ static int Decompress(FILE* fin, FILE* fout, const char* dictionary_path) {
}
end:
- free(dictionary);
- free(input);
- free(output);
+ BrFree(memsize, dictionary);
+ BrFree(memsize, input);
+ BrFree(memsize, output);
BrotliDestroyState(s);
return (result == BROTLI_RESULT_SUCCESS) ? 1 : 0;
}
static int Compress(int quality, int lgwin, FILE* fin, FILE* fout,
- const char *dictionary_path) {
- BrotliEncoderState* s = BrotliEncoderCreateInstance(0, 0, 0);
- uint8_t* buffer = (uint8_t*)malloc(kFileBufferSize << 1);
+ const char *dictionary_path, void *memsize) {
+ BrotliEncoderState* s = BrotliEncoderCreateInstance(BrAlloc, BrFree, memsize);
+ uint8_t* buffer = (uint8_t*)BrAlloc(memsize, kFileBufferSize << 1);
uint8_t* input = buffer;
uint8_t* output = buffer + kFileBufferSize;
size_t available_in = 0;
@@ -360,9 +379,9 @@ static int Compress(int quality, int lgwin, FILE* fin, FILE* fout,
BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, (uint32_t)lgwin);
if (dictionary_path != NULL) {
size_t dictionary_size = 0;
- uint8_t* dictionary = ReadDictionary(dictionary_path, &dictionary_size);
+ uint8_t* dictionary = ReadDictionary(dictionary_path, &dictionary_size, memsize);
BrotliEncoderSetCustomDictionary(s, dictionary_size, dictionary);
- free(dictionary);
+ BrFree(memsize, dictionary);
}
while (1) {
@@ -392,7 +411,7 @@ static int Compress(int quality, int lgwin, FILE* fin, FILE* fout,
}
finish:
- free(buffer);
+ BrFree(memsize, buffer);
BrotliEncoderDestroyInstance(s);
if (!is_ok) {
@@ -409,29 +428,41 @@ finish:
return 1;
}
+#define GAP_MEM_BLOCK 4096
+
int main(int argc, char** argv) {
char *input_path = 0;
char *output_path = 0;
char *dictionary_path = 0;
int force = 0;
int quality = 11;
+ int gmem = 1;
int decompress = 0;
int repeat = 1;
int verbose = 0;
int lgwin = 0;
clock_t clock_start;
int i;
+ int64_t originsize = 0;
+ int64_t msize = 0;
ParseArgv(argc, argv, &input_path, &output_path, &dictionary_path, &force,
- &quality, &decompress, &repeat, &verbose, &lgwin);
+ &quality, &gmem, &decompress, &repeat, &verbose, &lgwin);
clock_start = clock();
for (i = 0; i < repeat; ++i) {
FILE* fin = OpenInputFile(input_path);
FILE* fout = OpenOutputFile(output_path, force || repeat);
int is_ok = 0;
if (decompress) {
- is_ok = Decompress(fin, fout, dictionary_path);
+ if (fseek(fin, 16, SEEK_SET) != 0) {
+ fclose(fin);
+ return -1;
+ }
+ is_ok = Decompress(fin, fout, dictionary_path, (void *)&msize);
} else {
- is_ok = Compress(quality, lgwin, fin, fout, dictionary_path);
+ originsize = FileSize(input_path); /* get original file size */
+ fwrite(&originsize, 1, sizeof(int64_t), fout); /* add in original binary file size */
+ fwrite(&msize, 1, sizeof(int64_t), fout); /* add in dummy decompression required memory size */
+ is_ok = Compress(quality, lgwin, fin, fout, dictionary_path, (void *)&msize);
}
if (!is_ok) {
unlink(output_path);
@@ -445,6 +476,41 @@ int main(int argc, char** argv) {
perror("fclose");
exit(1);
}
+ /* after compression operation then execute decompression operation
+ to get decompression required memory size. */
+ if (decompress == 0) {
+ fin = OpenInputFile(output_path);
+ fout = tmpfile ();
+ msize = 0;
+ if (fseek(fin, 16, SEEK_SET) != 0) {
+ fclose(fin);
+ return -1;
+ }
+ is_ok = Decompress(fin, fout, dictionary_path, (void *)&msize);
+ if (!is_ok) {
+ exit(1);
+ }
+ if (fclose(fin) != 0) {
+ perror("fclose");
+ exit(1);
+ }
+ if (fclose(fout) != 0) {
+ perror("fclose");
+ exit(1);
+ }
+ fout = fopen(output_path, "rb+"); /* open output_path file and add in head info */
+ /* seek to the offset of decompression required memory size */
+ if (fseek(fout, 8, SEEK_SET) != 0) {
+ fclose(fout);
+ return -1;
+ }
+ msize += gmem * GAP_MEM_BLOCK; /* there is a memory gap between IA32 and X64 environment*/
+ fwrite(&msize, 1, sizeof(int64_t), fout); /* update final decompression required memory size */
+ if (fclose(fout) != 0) {
+ perror("fclose");
+ exit(1);
+ }
+ }
}
if (verbose) {
clock_t clock_end = clock();
diff --git a/BaseTools/Source/C/GNUmakefile b/BaseTools/Source/C/GNUmakefile
index 66baabb..83e188c 100644
--- a/BaseTools/Source/C/GNUmakefile
+++ b/BaseTools/Source/C/GNUmakefile
@@ -52,6 +52,7 @@ LIBRARIES = Common
APPLICATIONS = \
GnuGenBootSector \
BootSectImage \
+ BrotliCompress \
EfiLdrImage \
EfiRom \
GenFfs \
diff --git a/BaseTools/Source/C/Makefile b/BaseTools/Source/C/Makefile
index 3005a17..77d3b97 100644
--- a/BaseTools/Source/C/Makefile
+++ b/BaseTools/Source/C/Makefile
@@ -17,6 +17,7 @@ ARCH = IA32
LIBRARIES = Common
APPLICATIONS = \
BootSectImage \
+ BrotliCompress \
EfiLdrImage \
EfiRom \
GenBootSector \
--
2.10.2.windows.1
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2017-03-23 2:16 UTC | newest]
Thread overview: (only message) (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2017-03-23 2:16 [PATCH 4/4] BaseTools: Add Brotli algorithm tool Song, BinX
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox