From mboxrd@z Thu Jan 1 00:00:00 1970 Subject: Re: [edk2-devel] [PATCH 1/1] BaseTools/Brotli: Fix compressed data loss issue To: Yuwei Chen ,devel@edk2.groups.io From: "Bob Feng" X-Originating-Location: CN (192.55.46.54) X-Originating-Platform: Windows Chrome 90 User-Agent: GROUPS.IO Web Poster MIME-Version: 1.0 Date: Fri, 14 May 2021 00:58:17 -0700 References: <20210514060444.1224-1-yuwei.chen@intel.com> In-Reply-To: <20210514060444.1224-1-yuwei.chen@intel.com> Message-ID: <13028.1620979097706838875@groups.io> Content-Type: multipart/alternative; boundary="QH9etwr5s18mWJcrQjBX" --QH9etwr5s18mWJcrQjBX Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Reviewed-by: Bob Feng On Fri, May 14, 2021 at 02:04 PM, Yuwei Chen wrote: >=20 > REF: https://bugzilla.tianocore.org/show_bug.cgi?id=3D2946 >=20 > Currenly, when using the Brotli tool to compress data, the output > compressed binary file does not record complete compressed data > when size of input file is too large, which makes the data loss and > will trigger decompress-check issue. >=20 > The Brotli document mentioned: > The brotli tool use BrotliEncoderCompressStream method to compresses > input stream to output stream. Under some circumstances (e.g. lack of > output stream capacity) the BrotliEncoderOperation would require > several calls to BrotliEncoderCompressStream. The method must be > called again until both input stream is depleted and encoder has no > more output after the method is called. >=20 > This patch fixes this issue based on the Brotli document. >=20 > Cc: Bob Feng > Cc: Liming Gao > Signed-off-by: Yuwei Chen > --- > .../Source/C/BrotliCompress/BrotliCompress.c | 61 ++++++++++++------- > 1 file changed, 38 insertions(+), 23 deletions(-) >=20 > diff --git a/BaseTools/Source/C/BrotliCompress/BrotliCompress.c > b/BaseTools/Source/C/BrotliCompress/BrotliCompress.c > index 5a1400fda310..62a6aed3dbd0 100644 > --- a/BaseTools/Source/C/BrotliCompress/BrotliCompress.c > +++ b/BaseTools/Source/C/BrotliCompress/BrotliCompress.c > @@ -156,6 +156,7 @@ int CompressFile(char *InputFile, uint8_t > *InputBuffer, char *OutputFile, uint8_ > uint8_t *NextOut; > uint8_t *Input; > uint8_t *Output; > + size_t TotalOut; > size_t OutSize; > uint32_t SizeHint; > BROTLI_BOOL IsOk; > @@ -214,39 +215,53 @@ int CompressFile(char *InputFile, uint8_t > *InputBuffer, char *OutputFile, uint8_ > IsEof =3D !HasMoreInput(InputFileHandle); > } >=20 > - if (!BrotliEncoderCompressStream(EncodeState, > - IsEof ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS, > - &AvailableIn, &NextIn, &AvailableOut, &NextOut, NULL)) { > - printf("Failed to compress data [%s]\n", InputFile); > - IsOk =3D BROTLI_FALSE; > - goto Finish; > - } > - if (AvailableOut =3D=3D 0) { > - OutSize =3D (size_t)(NextOut - Output); > - if (OutSize > 0) { > - fwrite(Output, 1, OutSize, OutputFileHandle); > - if (ferror(OutputFileHandle)) { > - printf("Failed to write output [%s]\n", OutputFile); > + if (!IsEof){ > + do{ > + if (!BrotliEncoderCompressStream(EncodeState, > + BROTLI_OPERATION_FLUSH, > + &AvailableIn, &NextIn, &AvailableOut, &NextOut, &TotalOut)) { > + printf("Failed to compress data [%s]\n", InputFile); > IsOk =3D BROTLI_FALSE; > goto Finish; > } > + OutSize =3D (size_t)(NextOut - Output); > + if (OutSize > 0) { > + fwrite(Output, 1, OutSize, OutputFileHandle); > + if (ferror(OutputFileHandle)) { > + printf("Failed to write output [%s]\n", OutputFile); > + IsOk =3D BROTLI_FALSE; > + goto Finish; > + } > + } > + NextOut =3D Output; > + AvailableOut =3D kFileBufferSize; > } > - AvailableOut =3D kFileBufferSize; > - NextOut =3D Output; > + while (AvailableIn > 0 || BrotliEncoderHasMoreOutput(EncodeState)); > } > - if (BrotliEncoderIsFinished(EncodeState)) { > - OutSize =3D (size_t)(NextOut - Output); > - if (OutSize > 0) { > - fwrite(Output, 1, OutSize, OutputFileHandle); > - if (ferror(OutputFileHandle)) { > - printf("Failed to write output [%s]\n", OutputFile); > + else{ > + do{ > + if (!BrotliEncoderCompressStream(EncodeState, > + BROTLI_OPERATION_FINISH, > + &AvailableIn, &NextIn, &AvailableOut, &NextOut, &TotalOut)) { > + printf("Failed to compress data [%s]\n", InputFile); > IsOk =3D BROTLI_FALSE; > goto Finish; > } > - AvailableOut =3D 0; > + OutSize =3D (size_t)(NextOut - Output); > + if (OutSize > 0) { > + fwrite(Output, 1, OutSize, OutputFileHandle); > + if (ferror(OutputFileHandle)) { > + printf("Failed to write output [%s]\n", OutputFile); > + IsOk =3D BROTLI_FALSE; > + goto Finish; > + } > + } > + NextOut =3D Output; > + AvailableOut =3D kFileBufferSize; > } > + while (AvailableIn > 0 || BrotliEncoderHasMoreOutput(EncodeState)); > } > - if (IsEof) { > + if (BrotliEncoderIsFinished(EncodeState)){ > break; > } > } > -- > 2.27.0.windows.1 --QH9etwr5s18mWJcrQjBX Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: quoted-printable Reviewed-by: Bob Feng <bob.c.feng@intel.com>

On Fri, May 1= 4, 2021 at 02:04 PM, Yuwei Chen wrote:
REF: https://bugzilla.tianocore.org= /show_bug.cgi?id=3D2946

Currenly, when using the Brotli tool= to compress data, the output
compressed binary file does not record c= omplete compressed data
when size of input file is too large, which ma= kes the data loss and
will trigger decompress-check issue.

= The Brotli document mentioned:
The brotli tool use BrotliEncoderCompre= ssStream method to compresses
input stream to output stream. Under som= e circumstances (e.g. lack of
output stream capacity) the BrotliEncode= rOperation would require
several calls to BrotliEncoderCompressStream.= The method must be
called again until both input stream is depleted a= nd encoder has no
more output after the method is called.

T= his patch fixes this issue based on the Brotli document.

Cc: Bob= Feng <bob.c.feng@intel.com>
Cc: Liming Gao <gaoliming@byosof= t.com.cn>
Signed-off-by: Yuwei Chen <yuwei.chen@intel.com>---
.../Source/C/BrotliCompress/BrotliCompress.c | 61 ++++++++++++-= ------
1 file changed, 38 insertions(+), 23 deletions(-)

di= ff --git a/BaseTools/Source/C/BrotliCompress/BrotliCompress.c b/BaseTools/S= ource/C/BrotliCompress/BrotliCompress.c
index 5a1400fda310..62a6aed3db= d0 100644
--- a/BaseTools/Source/C/BrotliCompress/BrotliCompress.c
+++ b/BaseTools/Source/C/BrotliCompress/BrotliCompress.c
@@ -156,6 += 156,7 @@ int CompressFile(char *InputFile, uint8_t *InputBuffer, char *Outp= utFile, uint8_
uint8_t *NextOut;
uint8_t *Input;
uint8_t *Ou= tput;
+ size_t TotalOut;
size_t OutSize;
uint32_t SizeHint;<= br />BROTLI_BOOL IsOk;
@@ -214,39 +215,53 @@ int CompressFile(char *In= putFile, uint8_t *InputBuffer, char *OutputFile, uint8_
IsEof =3D !Has= MoreInput(InputFileHandle);
}

- if (!BrotliEncoderCompressS= tream(EncodeState,
- IsEof ? BROTLI_OPERATION_FINISH : BROTLI_OPERATIO= N_PROCESS,
- &AvailableIn, &NextIn, &AvailableOut, &Ne= xtOut, NULL)) {
- printf("Failed to compress data [%s]\n", InputFile);=
- IsOk =3D BROTLI_FALSE;
- goto Finish;
- }
- if (Avai= lableOut =3D=3D 0) {
- OutSize =3D (size_t)(NextOut - Output);
- = if (OutSize > 0) {
- fwrite(Output, 1, OutSize, OutputFileHandle);<= br />- if (ferror(OutputFileHandle)) {
- printf("Failed to write outpu= t [%s]\n", OutputFile);
+ if (!IsEof){
+ do{
+ if (!BrotliEn= coderCompressStream(EncodeState,
+ BROTLI_OPERATION_FLUSH,
+ &= ;AvailableIn, &NextIn, &AvailableOut, &NextOut, &TotalOut))= {
+ printf("Failed to compress data [%s]\n", InputFile);
IsOk = =3D BROTLI_FALSE;
goto Finish;
}
+ OutSize =3D (size_t)(Nex= tOut - Output);
+ if (OutSize > 0) {
+ fwrite(Output, 1, OutSi= ze, OutputFileHandle);
+ if (ferror(OutputFileHandle)) {
+ printf= ("Failed to write output [%s]\n", OutputFile);
+ IsOk =3D BROTLI_FALSE= ;
+ goto Finish;
+ }
+ }
+ NextOut =3D Output;
+ A= vailableOut =3D kFileBufferSize;
}
- AvailableOut =3D kFileBuffer= Size;
- NextOut =3D Output;
+ while (AvailableIn > 0 || Brotli= EncoderHasMoreOutput(EncodeState));
}
- if (BrotliEncoderIsFinish= ed(EncodeState)) {
- OutSize =3D (size_t)(NextOut - Output);
- if= (OutSize > 0) {
- fwrite(Output, 1, OutSize, OutputFileHandle);- if (ferror(OutputFileHandle)) {
- printf("Failed to write output = [%s]\n", OutputFile);
+ else{
+ do{
+ if (!BrotliEncoderComp= ressStream(EncodeState,
+ BROTLI_OPERATION_FINISH,
+ &Availab= leIn, &NextIn, &AvailableOut, &NextOut, &TotalOut)) {
= + printf("Failed to compress data [%s]\n", InputFile);
IsOk =3D BROTLI= _FALSE;
goto Finish;
}
- AvailableOut =3D 0;
+ OutSize = = =3D (size_t)(NextOut - Output);
+ if (OutSize > 0) {
+ fwrite= (Output, 1, OutSize, OutputFileHandle);
+ if (ferror(OutputFileHandle)= ) {
+ printf("Failed to write output [%s]\n", OutputFile);
+ IsOk= =3D BROTLI_FALSE;
+ goto Finish;
+ }
+ }
+ NextOut =3D= Output;
+ AvailableOut =3D kFileBufferSize;
}
+ while (Avai= lableIn > 0 || BrotliEncoderHasMoreOutput(EncodeState));
}
- i= f (IsEof) {
+ if (BrotliEncoderIsFinished(EncodeState)){
break;}
}
--
2.27.0.windows.1
--QH9etwr5s18mWJcrQjBX--