2011-12-21 19:00:10 +01:00
|
|
|
/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
|
|
|
|
|
|
|
|
/***
|
|
|
|
This file is part of systemd.
|
|
|
|
|
|
|
|
Copyright 2011 Lennart Poettering
|
|
|
|
|
|
|
|
systemd is free software; you can redistribute it and/or modify it
|
2012-04-12 00:20:58 +02:00
|
|
|
under the terms of the GNU Lesser General Public License as published by
|
|
|
|
the Free Software Foundation; either version 2.1 of the License, or
|
2011-12-21 19:00:10 +01:00
|
|
|
(at your option) any later version.
|
|
|
|
|
|
|
|
systemd is distributed in the hope that it will be useful, but
|
|
|
|
WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
2012-04-12 00:20:58 +02:00
|
|
|
Lesser General Public License for more details.
|
2011-12-21 19:00:10 +01:00
|
|
|
|
2012-04-12 00:20:58 +02:00
|
|
|
You should have received a copy of the GNU Lesser General Public License
|
2011-12-21 19:00:10 +01:00
|
|
|
along with systemd; If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
***/
|
|
|
|
|
|
|
|
#include <assert.h>
|
|
|
|
#include <stdlib.h>
|
|
|
|
#include <string.h>
|
2014-06-25 03:24:46 +02:00
|
|
|
#include <unistd.h>
|
2014-07-04 04:42:22 +02:00
|
|
|
|
|
|
|
#ifdef HAVE_XZ
|
|
|
|
# include <lzma.h>
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef HAVE_LZ4
|
|
|
|
# include <lz4.h>
|
|
|
|
#endif
|
2011-12-21 19:00:10 +01:00
|
|
|
|
|
|
|
#include "compress.h"
|
2014-06-25 03:24:46 +02:00
|
|
|
#include "macro.h"
|
|
|
|
#include "util.h"
|
2014-07-04 04:42:22 +02:00
|
|
|
#include "sparse-endian.h"
|
|
|
|
#include "journal-def.h"
|
|
|
|
|
|
|
|
#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
|
|
|
|
|
|
|
|
static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
|
|
|
|
[OBJECT_COMPRESSED_XZ] = "XZ",
|
|
|
|
[OBJECT_COMPRESSED_LZ4] = "LZ4",
|
|
|
|
};
|
2011-12-21 19:00:10 +01:00
|
|
|
|
2014-07-04 04:42:22 +02:00
|
|
|
DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
|
|
|
|
|
2014-08-04 04:50:00 +02:00
|
|
|
int compress_blob_xz(const void *src, uint64_t src_size, void *dst, size_t *dst_size) {
|
2014-07-04 04:42:22 +02:00
|
|
|
#ifdef HAVE_XZ
|
journal/compress: improve xz compression performance
The new lzma2 compression options at the top of compress_blob_xz are
equivalent to using preset "0", exept for using a 1 MiB dictionary
(the same as preset "1"). This makes the memory usage at most 7.5 MiB
in the compressor, and 1 MiB in the decompressor, instead of the
previous 92 MiB in the compressor and 8 MiB in the decompressor.
According to test-compress-benchmark this commit makes XZ compression
20 times faster, with no increase in compressed data size.
Using more realistic test data (an ELF binary rather than repeating
ASCII letters 'a' through 'z' in order) it only provides a factor 10
speedup, and at a cost if a 10% increase in compressed data size.
But that is still a worthwhile trade-off.
According to test-compress-benchmark XZ compression is still 25 times
slower than LZ4, but the compressed data is one eighth the size.
Using more realistic test data XZ compression is only 18 times slower
than LZ4, and the compressed data is only one quarter the size.
$ ./test-compress-benchmark
XZ: compressed & decompressed 2535300963 bytes in 42.30s (57.15MiB/s), mean compresion 99.95%, skipped 3570 bytes
LZ4: compressed & decompressed 2535303543 bytes in 1.60s (1510.60MiB/s), mean compresion 99.60%, skipped 990 bytes
2014-07-08 18:29:46 +02:00
|
|
|
static const lzma_options_lzma opt = {
|
|
|
|
1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
|
|
|
|
LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4};
|
|
|
|
static const lzma_filter filters[2] = {
|
|
|
|
{LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt},
|
|
|
|
{LZMA_VLI_UNKNOWN, NULL}
|
|
|
|
};
|
2011-12-21 19:00:10 +01:00
|
|
|
lzma_ret ret;
|
2014-06-25 03:24:09 +02:00
|
|
|
size_t out_pos = 0;
|
2011-12-21 19:00:10 +01:00
|
|
|
|
|
|
|
assert(src);
|
|
|
|
assert(src_size > 0);
|
|
|
|
assert(dst);
|
|
|
|
assert(dst_size);
|
|
|
|
|
2014-07-04 04:42:22 +02:00
|
|
|
/* Returns < 0 if we couldn't compress the data or the
|
2011-12-21 19:00:10 +01:00
|
|
|
* compressed result is longer than the original */
|
|
|
|
|
journal/compress: improve xz compression performance
The new lzma2 compression options at the top of compress_blob_xz are
equivalent to using preset "0", exept for using a 1 MiB dictionary
(the same as preset "1"). This makes the memory usage at most 7.5 MiB
in the compressor, and 1 MiB in the decompressor, instead of the
previous 92 MiB in the compressor and 8 MiB in the decompressor.
According to test-compress-benchmark this commit makes XZ compression
20 times faster, with no increase in compressed data size.
Using more realistic test data (an ELF binary rather than repeating
ASCII letters 'a' through 'z' in order) it only provides a factor 10
speedup, and at a cost if a 10% increase in compressed data size.
But that is still a worthwhile trade-off.
According to test-compress-benchmark XZ compression is still 25 times
slower than LZ4, but the compressed data is one eighth the size.
Using more realistic test data XZ compression is only 18 times slower
than LZ4, and the compressed data is only one quarter the size.
$ ./test-compress-benchmark
XZ: compressed & decompressed 2535300963 bytes in 42.30s (57.15MiB/s), mean compresion 99.95%, skipped 3570 bytes
LZ4: compressed & decompressed 2535303543 bytes in 1.60s (1510.60MiB/s), mean compresion 99.60%, skipped 990 bytes
2014-07-08 18:29:46 +02:00
|
|
|
if (src_size < 80)
|
|
|
|
return -ENOBUFS;
|
|
|
|
|
|
|
|
ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL,
|
|
|
|
src, src_size, dst, &out_pos, src_size - 1);
|
2011-12-21 19:00:10 +01:00
|
|
|
if (ret != LZMA_OK)
|
2014-07-04 04:42:22 +02:00
|
|
|
return -ENOBUFS;
|
2011-12-21 19:00:10 +01:00
|
|
|
|
2014-06-25 03:24:09 +02:00
|
|
|
*dst_size = out_pos;
|
2014-07-04 04:42:22 +02:00
|
|
|
return 0;
|
|
|
|
#else
|
|
|
|
return -EPROTONOSUPPORT;
|
|
|
|
#endif
|
2011-12-21 19:00:10 +01:00
|
|
|
}
|
|
|
|
|
2014-08-04 04:50:00 +02:00
|
|
|
int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, size_t *dst_size) {
|
2014-07-04 04:42:22 +02:00
|
|
|
#ifdef HAVE_LZ4
|
|
|
|
int r;
|
|
|
|
|
|
|
|
assert(src);
|
|
|
|
assert(src_size > 0);
|
|
|
|
assert(dst);
|
|
|
|
assert(dst_size);
|
|
|
|
|
|
|
|
/* Returns < 0 if we couldn't compress the data or the
|
|
|
|
* compressed result is longer than the original */
|
|
|
|
|
|
|
|
if (src_size < 9)
|
|
|
|
return -ENOBUFS;
|
2011-12-21 19:00:10 +01:00
|
|
|
|
2014-07-04 04:42:22 +02:00
|
|
|
r = LZ4_compress_limitedOutput(src, dst + 8, src_size, src_size - 8 - 1);
|
|
|
|
if (r <= 0)
|
|
|
|
return -ENOBUFS;
|
|
|
|
|
|
|
|
*(le64_t*) dst = htole64(src_size);
|
|
|
|
*dst_size = r + 8;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
#else
|
|
|
|
return -EPROTONOSUPPORT;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
int decompress_blob_xz(const void *src, uint64_t src_size,
|
2014-08-04 04:50:00 +02:00
|
|
|
void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
|
2014-07-04 04:42:22 +02:00
|
|
|
|
|
|
|
#ifdef HAVE_XZ
|
2014-07-05 01:53:58 +02:00
|
|
|
_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
|
2011-12-21 19:00:10 +01:00
|
|
|
lzma_ret ret;
|
2014-08-04 04:50:00 +02:00
|
|
|
size_t space;
|
2011-12-21 19:00:10 +01:00
|
|
|
|
|
|
|
assert(src);
|
|
|
|
assert(src_size > 0);
|
|
|
|
assert(dst);
|
|
|
|
assert(dst_alloc_size);
|
|
|
|
assert(dst_size);
|
|
|
|
assert(*dst_alloc_size == 0 || *dst);
|
|
|
|
|
|
|
|
ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
|
|
|
|
if (ret != LZMA_OK)
|
2014-07-04 04:42:22 +02:00
|
|
|
return -ENOMEM;
|
2011-12-21 19:00:10 +01:00
|
|
|
|
2014-08-04 04:50:00 +02:00
|
|
|
space = MIN(src_size * 2, dst_max ?: (size_t) -1);
|
2014-07-05 01:53:58 +02:00
|
|
|
if (!greedy_realloc(dst, dst_alloc_size, space, 1))
|
2014-07-19 03:44:36 +02:00
|
|
|
return -ENOMEM;
|
2011-12-21 19:00:10 +01:00
|
|
|
|
|
|
|
s.next_in = src;
|
|
|
|
s.avail_in = src_size;
|
|
|
|
|
|
|
|
s.next_out = *dst;
|
2012-11-21 00:28:00 +01:00
|
|
|
s.avail_out = space;
|
2011-12-21 19:00:10 +01:00
|
|
|
|
|
|
|
for (;;) {
|
2014-08-04 04:50:00 +02:00
|
|
|
size_t used;
|
2011-12-21 19:00:10 +01:00
|
|
|
|
|
|
|
ret = lzma_code(&s, LZMA_FINISH);
|
|
|
|
|
|
|
|
if (ret == LZMA_STREAM_END)
|
|
|
|
break;
|
2014-07-04 04:42:22 +02:00
|
|
|
else if (ret != LZMA_OK)
|
|
|
|
return -ENOMEM;
|
2011-12-21 19:00:10 +01:00
|
|
|
|
2012-11-21 00:28:00 +01:00
|
|
|
if (dst_max > 0 && (space - s.avail_out) >= dst_max)
|
|
|
|
break;
|
2014-07-04 04:42:22 +02:00
|
|
|
else if (dst_max > 0 && space == dst_max)
|
|
|
|
return -ENOBUFS;
|
2012-11-21 00:28:00 +01:00
|
|
|
|
2014-07-05 01:53:58 +02:00
|
|
|
used = space - s.avail_out;
|
2014-08-04 04:50:00 +02:00
|
|
|
space = MIN(2 * space, dst_max ?: (size_t) -1);
|
2014-07-05 01:53:58 +02:00
|
|
|
if (!greedy_realloc(dst, dst_alloc_size, space, 1))
|
2014-07-19 03:44:36 +02:00
|
|
|
return -ENOMEM;
|
2011-12-21 19:00:10 +01:00
|
|
|
|
2014-07-05 01:53:58 +02:00
|
|
|
s.avail_out = space - used;
|
|
|
|
s.next_out = *dst + used;
|
2011-12-21 19:00:10 +01:00
|
|
|
}
|
|
|
|
|
2012-11-21 00:28:00 +01:00
|
|
|
*dst_size = space - s.avail_out;
|
2014-07-04 04:42:22 +02:00
|
|
|
return 0;
|
|
|
|
#else
|
|
|
|
return -EPROTONOSUPPORT;
|
|
|
|
#endif
|
2011-12-21 19:00:10 +01:00
|
|
|
}
|
|
|
|
|
2014-07-04 04:42:22 +02:00
|
|
|
int decompress_blob_lz4(const void *src, uint64_t src_size,
|
2014-08-04 04:50:00 +02:00
|
|
|
void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
|
2014-07-04 04:42:22 +02:00
|
|
|
|
|
|
|
#ifdef HAVE_LZ4
|
|
|
|
char* out;
|
2014-08-04 04:50:00 +02:00
|
|
|
int r, size; /* LZ4 uses int for size */
|
2011-12-21 19:00:10 +01:00
|
|
|
|
2014-07-04 04:42:22 +02:00
|
|
|
assert(src);
|
|
|
|
assert(src_size > 0);
|
|
|
|
assert(dst);
|
|
|
|
assert(dst_alloc_size);
|
|
|
|
assert(dst_size);
|
|
|
|
assert(*dst_alloc_size == 0 || *dst);
|
|
|
|
|
|
|
|
if (src_size <= 8)
|
|
|
|
return -EBADMSG;
|
|
|
|
|
|
|
|
size = le64toh( *(le64_t*)src );
|
2014-08-04 04:50:00 +02:00
|
|
|
if (size < 0 || (le64_t) size != *(le64_t*)src)
|
|
|
|
return -EFBIG;
|
|
|
|
if ((size_t) size > *dst_alloc_size) {
|
2014-07-04 04:42:22 +02:00
|
|
|
out = realloc(*dst, size);
|
|
|
|
if (!out)
|
|
|
|
return -ENOMEM;
|
|
|
|
*dst = out;
|
|
|
|
*dst_alloc_size = size;
|
|
|
|
} else
|
|
|
|
out = *dst;
|
|
|
|
|
|
|
|
r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
|
2014-08-04 04:50:00 +02:00
|
|
|
if (r < 0 || r != size)
|
2014-07-04 04:42:22 +02:00
|
|
|
return -EBADMSG;
|
|
|
|
|
|
|
|
*dst_size = size;
|
|
|
|
return 0;
|
|
|
|
#else
|
|
|
|
return -EPROTONOSUPPORT;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
int decompress_blob(int compression,
|
|
|
|
const void *src, uint64_t src_size,
|
2014-08-04 04:50:00 +02:00
|
|
|
void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
|
2014-07-04 04:42:22 +02:00
|
|
|
if (compression == OBJECT_COMPRESSED_XZ)
|
|
|
|
return decompress_blob_xz(src, src_size,
|
|
|
|
dst, dst_alloc_size, dst_size, dst_max);
|
|
|
|
else if (compression == OBJECT_COMPRESSED_LZ4)
|
|
|
|
return decompress_blob_lz4(src, src_size,
|
|
|
|
dst, dst_alloc_size, dst_size, dst_max);
|
|
|
|
else
|
|
|
|
return -EBADMSG;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
int decompress_startswith_xz(const void *src, uint64_t src_size,
|
2014-08-04 04:50:00 +02:00
|
|
|
void **buffer, size_t *buffer_size,
|
|
|
|
const void *prefix, size_t prefix_len,
|
2014-07-04 04:42:22 +02:00
|
|
|
uint8_t extra) {
|
|
|
|
|
|
|
|
#ifdef HAVE_XZ
|
2014-07-05 01:53:58 +02:00
|
|
|
_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
|
2011-12-21 19:00:10 +01:00
|
|
|
lzma_ret ret;
|
|
|
|
|
2014-07-04 04:42:22 +02:00
|
|
|
/* Checks whether the decompressed blob starts with the
|
2011-12-21 19:00:10 +01:00
|
|
|
* mentioned prefix. The byte extra needs to follow the
|
|
|
|
* prefix */
|
|
|
|
|
|
|
|
assert(src);
|
|
|
|
assert(src_size > 0);
|
|
|
|
assert(buffer);
|
|
|
|
assert(buffer_size);
|
|
|
|
assert(prefix);
|
|
|
|
assert(*buffer_size == 0 || *buffer);
|
|
|
|
|
|
|
|
ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
|
|
|
|
if (ret != LZMA_OK)
|
2014-07-04 04:42:22 +02:00
|
|
|
return -EBADMSG;
|
2011-12-21 19:00:10 +01:00
|
|
|
|
2014-07-04 04:42:22 +02:00
|
|
|
if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
|
|
|
|
return -ENOMEM;
|
2011-12-21 19:00:10 +01:00
|
|
|
|
|
|
|
s.next_in = src;
|
|
|
|
s.avail_in = src_size;
|
|
|
|
|
|
|
|
s.next_out = *buffer;
|
|
|
|
s.avail_out = *buffer_size;
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
ret = lzma_code(&s, LZMA_FINISH);
|
|
|
|
|
|
|
|
if (ret != LZMA_STREAM_END && ret != LZMA_OK)
|
2014-07-04 04:42:22 +02:00
|
|
|
return -EBADMSG;
|
2011-12-21 19:00:10 +01:00
|
|
|
|
2014-07-05 01:53:58 +02:00
|
|
|
if (*buffer_size - s.avail_out >= prefix_len + 1)
|
|
|
|
return memcmp(*buffer, prefix, prefix_len) == 0 &&
|
|
|
|
((const uint8_t*) *buffer)[prefix_len] == extra;
|
2011-12-21 19:00:10 +01:00
|
|
|
|
|
|
|
if (ret == LZMA_STREAM_END)
|
2014-07-04 04:42:22 +02:00
|
|
|
return 0;
|
2011-12-21 19:00:10 +01:00
|
|
|
|
|
|
|
s.avail_out += *buffer_size;
|
|
|
|
|
2014-07-05 01:53:58 +02:00
|
|
|
if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
|
2014-07-04 04:42:22 +02:00
|
|
|
return -ENOMEM;
|
2011-12-21 19:00:10 +01:00
|
|
|
|
2014-07-05 01:53:58 +02:00
|
|
|
s.next_out = *buffer + *buffer_size - s.avail_out;
|
|
|
|
}
|
2014-07-04 04:42:22 +02:00
|
|
|
|
|
|
|
#else
|
|
|
|
return -EPROTONOSUPPORT;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
int decompress_startswith_lz4(const void *src, uint64_t src_size,
|
2014-08-04 04:50:00 +02:00
|
|
|
void **buffer, size_t *buffer_size,
|
|
|
|
const void *prefix, size_t prefix_len,
|
2014-07-04 04:42:22 +02:00
|
|
|
uint8_t extra) {
|
|
|
|
#ifdef HAVE_LZ4
|
|
|
|
/* Checks whether the decompressed blob starts with the
|
|
|
|
* mentioned prefix. The byte extra needs to follow the
|
|
|
|
* prefix */
|
|
|
|
|
|
|
|
int r;
|
|
|
|
|
|
|
|
assert(src);
|
|
|
|
assert(src_size > 0);
|
|
|
|
assert(buffer);
|
|
|
|
assert(buffer_size);
|
|
|
|
assert(prefix);
|
|
|
|
assert(*buffer_size == 0 || *buffer);
|
|
|
|
|
|
|
|
if (src_size <= 8)
|
|
|
|
return -EBADMSG;
|
|
|
|
|
|
|
|
if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8,
|
|
|
|
prefix_len + 1, *buffer_size);
|
|
|
|
|
|
|
|
if (r < 0)
|
|
|
|
return -EBADMSG;
|
|
|
|
if ((unsigned) r >= prefix_len + 1)
|
|
|
|
return memcmp(*buffer, prefix, prefix_len) == 0 &&
|
|
|
|
((const uint8_t*) *buffer)[prefix_len] == extra;
|
|
|
|
else
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
#else
|
|
|
|
return -EPROTONOSUPPORT;
|
|
|
|
#endif
|
2011-12-21 19:00:10 +01:00
|
|
|
}
|
2014-06-25 03:24:46 +02:00
|
|
|
|
2014-07-04 04:42:22 +02:00
|
|
|
int decompress_startswith(int compression,
|
|
|
|
const void *src, uint64_t src_size,
|
2014-08-04 04:50:00 +02:00
|
|
|
void **buffer, size_t *buffer_size,
|
|
|
|
const void *prefix, size_t prefix_len,
|
2014-07-04 04:42:22 +02:00
|
|
|
uint8_t extra) {
|
|
|
|
if (compression == OBJECT_COMPRESSED_XZ)
|
|
|
|
return decompress_startswith_xz(src, src_size,
|
|
|
|
buffer, buffer_size,
|
|
|
|
prefix, prefix_len,
|
|
|
|
extra);
|
|
|
|
else if (compression == OBJECT_COMPRESSED_LZ4)
|
|
|
|
return decompress_startswith_lz4(src, src_size,
|
|
|
|
buffer, buffer_size,
|
|
|
|
prefix, prefix_len,
|
|
|
|
extra);
|
|
|
|
else
|
|
|
|
return -EBADMSG;
|
|
|
|
}
|
|
|
|
|
|
|
|
int compress_stream_xz(int fdf, int fdt, off_t max_bytes) {
|
2014-07-11 16:42:06 +02:00
|
|
|
#ifdef HAVE_XZ
|
2014-06-25 03:24:46 +02:00
|
|
|
_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
|
|
|
|
lzma_ret ret;
|
|
|
|
|
|
|
|
uint8_t buf[BUFSIZ], out[BUFSIZ];
|
|
|
|
lzma_action action = LZMA_RUN;
|
|
|
|
|
|
|
|
assert(fdf >= 0);
|
|
|
|
assert(fdt >= 0);
|
|
|
|
|
2014-07-04 04:42:22 +02:00
|
|
|
ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
|
2014-06-25 03:24:46 +02:00
|
|
|
if (ret != LZMA_OK) {
|
|
|
|
log_error("Failed to initialize XZ encoder: code %d", ret);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
if (s.avail_in == 0 && action == LZMA_RUN) {
|
|
|
|
size_t m = sizeof(buf);
|
|
|
|
ssize_t n;
|
|
|
|
|
|
|
|
if (max_bytes != -1 && m > (size_t) max_bytes)
|
|
|
|
m = max_bytes;
|
|
|
|
|
|
|
|
n = read(fdf, buf, m);
|
|
|
|
if (n < 0)
|
|
|
|
return -errno;
|
|
|
|
if (n == 0)
|
|
|
|
action = LZMA_FINISH;
|
|
|
|
else {
|
|
|
|
s.next_in = buf;
|
|
|
|
s.avail_in = n;
|
|
|
|
|
|
|
|
if (max_bytes != -1) {
|
|
|
|
assert(max_bytes >= n);
|
|
|
|
max_bytes -= n;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (s.avail_out == 0) {
|
|
|
|
s.next_out = out;
|
|
|
|
s.avail_out = sizeof(out);
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = lzma_code(&s, action);
|
|
|
|
if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
|
|
|
|
log_error("Compression failed: code %d", ret);
|
|
|
|
return -EBADMSG;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
|
|
|
|
ssize_t n, k;
|
|
|
|
|
|
|
|
n = sizeof(out) - s.avail_out;
|
|
|
|
|
|
|
|
k = loop_write(fdt, out, n, false);
|
|
|
|
if (k < 0)
|
|
|
|
return k;
|
|
|
|
|
|
|
|
if (ret == LZMA_STREAM_END) {
|
2014-08-04 04:50:00 +02:00
|
|
|
log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
|
2014-06-25 03:24:46 +02:00
|
|
|
s.total_in, s.total_out,
|
|
|
|
(double) s.total_out / s.total_in * 100);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2014-07-11 16:42:06 +02:00
|
|
|
#else
|
|
|
|
return -EPROTONOSUPPORT;
|
|
|
|
#endif
|
2014-06-25 03:24:46 +02:00
|
|
|
}
|
|
|
|
|
2014-07-04 04:42:22 +02:00
|
|
|
#define LZ4_BUFSIZE (512*1024)
|
|
|
|
|
|
|
|
int compress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
|
|
|
|
|
|
|
|
#ifdef HAVE_LZ4
|
|
|
|
|
|
|
|
_cleanup_free_ char *buf1 = NULL, *buf2 = NULL, *out = NULL;
|
|
|
|
char *buf;
|
|
|
|
LZ4_stream_t lz4_data = {};
|
|
|
|
le32_t header;
|
|
|
|
size_t total_in = 0, total_out = sizeof(header);
|
|
|
|
ssize_t n;
|
|
|
|
|
|
|
|
assert(fdf >= 0);
|
|
|
|
assert(fdt >= 0);
|
|
|
|
|
|
|
|
buf1 = malloc(LZ4_BUFSIZE);
|
|
|
|
buf2 = malloc(LZ4_BUFSIZE);
|
|
|
|
out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE));
|
|
|
|
if (!buf1 || !buf2 || !out)
|
|
|
|
return log_oom();
|
|
|
|
|
|
|
|
buf = buf1;
|
|
|
|
for (;;) {
|
|
|
|
size_t m;
|
|
|
|
int r;
|
|
|
|
|
|
|
|
m = LZ4_BUFSIZE;
|
|
|
|
if (max_bytes != -1 && m > (size_t) max_bytes - total_in)
|
|
|
|
m = max_bytes - total_in;
|
|
|
|
|
|
|
|
n = read(fdf, buf, m);
|
|
|
|
if (n < 0)
|
|
|
|
return -errno;
|
|
|
|
if (n == 0)
|
|
|
|
break;
|
|
|
|
|
|
|
|
total_in += n;
|
|
|
|
|
2014-08-30 09:13:43 +02:00
|
|
|
r = LZ4_compress_continue(&lz4_data, buf, out, n);
|
2014-07-04 04:42:22 +02:00
|
|
|
if (r == 0) {
|
2014-08-30 09:13:43 +02:00
|
|
|
log_error("LZ4 compression failed.");
|
|
|
|
return -EBADMSG;
|
2014-07-04 04:42:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
header = htole32(r);
|
|
|
|
errno = 0;
|
|
|
|
|
|
|
|
n = write(fdt, &header, sizeof(header));
|
|
|
|
if (n < 0)
|
|
|
|
return -errno;
|
|
|
|
if (n != sizeof(header))
|
|
|
|
return errno ? -errno : -EIO;
|
|
|
|
|
|
|
|
n = loop_write(fdt, out, r, false);
|
|
|
|
if (n < 0)
|
|
|
|
return n;
|
|
|
|
|
|
|
|
total_out += sizeof(header) + r;
|
|
|
|
|
|
|
|
buf = buf == buf1 ? buf2 : buf1;
|
|
|
|
}
|
|
|
|
|
|
|
|
header = htole32(0);
|
|
|
|
n = write(fdt, &header, sizeof(header));
|
|
|
|
if (n < 0)
|
|
|
|
return -errno;
|
|
|
|
if (n != sizeof(header))
|
|
|
|
return errno ? -errno : -EIO;
|
|
|
|
|
|
|
|
log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
|
|
|
|
total_in, total_out,
|
|
|
|
(double) total_out / total_in * 100);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
#else
|
|
|
|
return -EPROTONOSUPPORT;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
int decompress_stream_xz(int fdf, int fdt, off_t max_bytes) {
|
|
|
|
|
|
|
|
#ifdef HAVE_XZ
|
2014-06-25 03:24:46 +02:00
|
|
|
_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
|
|
|
|
lzma_ret ret;
|
|
|
|
|
|
|
|
uint8_t buf[BUFSIZ], out[BUFSIZ];
|
|
|
|
lzma_action action = LZMA_RUN;
|
|
|
|
|
|
|
|
assert(fdf >= 0);
|
|
|
|
assert(fdt >= 0);
|
|
|
|
|
|
|
|
ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
|
|
|
|
if (ret != LZMA_OK) {
|
|
|
|
log_error("Failed to initialize XZ decoder: code %d", ret);
|
2014-07-04 04:42:22 +02:00
|
|
|
return -ENOMEM;
|
2014-06-25 03:24:46 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
if (s.avail_in == 0 && action == LZMA_RUN) {
|
|
|
|
ssize_t n;
|
|
|
|
|
|
|
|
n = read(fdf, buf, sizeof(buf));
|
|
|
|
if (n < 0)
|
|
|
|
return -errno;
|
|
|
|
if (n == 0)
|
|
|
|
action = LZMA_FINISH;
|
|
|
|
else {
|
|
|
|
s.next_in = buf;
|
|
|
|
s.avail_in = n;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (s.avail_out == 0) {
|
|
|
|
s.next_out = out;
|
|
|
|
s.avail_out = sizeof(out);
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = lzma_code(&s, action);
|
|
|
|
if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
|
|
|
|
log_error("Decompression failed: code %d", ret);
|
|
|
|
return -EBADMSG;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
|
|
|
|
ssize_t n, k;
|
|
|
|
|
|
|
|
n = sizeof(out) - s.avail_out;
|
|
|
|
|
|
|
|
if (max_bytes != -1) {
|
|
|
|
if (max_bytes < n)
|
2014-07-04 04:42:22 +02:00
|
|
|
return -EFBIG;
|
2014-06-25 03:24:46 +02:00
|
|
|
|
|
|
|
max_bytes -= n;
|
|
|
|
}
|
|
|
|
|
|
|
|
k = loop_write(fdt, out, n, false);
|
|
|
|
if (k < 0)
|
|
|
|
return k;
|
|
|
|
|
|
|
|
if (ret == LZMA_STREAM_END) {
|
2014-08-04 04:50:00 +02:00
|
|
|
log_debug("XZ decompression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
|
2014-06-25 03:24:46 +02:00
|
|
|
s.total_in, s.total_out,
|
|
|
|
(double) s.total_out / s.total_in * 100);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2014-07-04 04:42:22 +02:00
|
|
|
#else
|
|
|
|
log_error("Cannot decompress file. Compiled without XZ support.");
|
|
|
|
return -EPROTONOSUPPORT;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
|
|
|
|
|
|
|
|
#ifdef HAVE_LZ4
|
|
|
|
_cleanup_free_ char *buf = NULL, *out = NULL;
|
|
|
|
size_t buf_size = 0;
|
|
|
|
LZ4_streamDecode_t lz4_data = {};
|
|
|
|
le32_t header;
|
|
|
|
size_t total_in = sizeof(header), total_out = 0;
|
|
|
|
|
|
|
|
assert(fdf >= 0);
|
|
|
|
assert(fdt >= 0);
|
|
|
|
|
|
|
|
out = malloc(4*LZ4_BUFSIZE);
|
|
|
|
if (!out)
|
|
|
|
return log_oom();
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
ssize_t n, m;
|
|
|
|
int r;
|
|
|
|
|
|
|
|
n = read(fdf, &header, sizeof(header));
|
|
|
|
if (n < 0)
|
|
|
|
return -errno;
|
|
|
|
if (n != sizeof(header))
|
|
|
|
return errno ? -errno : -EIO;
|
|
|
|
|
|
|
|
m = le32toh(header);
|
|
|
|
if (m == 0)
|
|
|
|
break;
|
|
|
|
|
|
|
|
/* We refuse to use a bigger decompression buffer than
|
|
|
|
* the one used for compression by 4 times. This means
|
|
|
|
* that compression buffer size can be enlarged 4
|
|
|
|
* times. This can be changed, but old binaries might
|
|
|
|
* not accept buffers compressed by newer binaries then.
|
|
|
|
*/
|
|
|
|
if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
|
|
|
|
log_error("Compressed stream block too big: %zd bytes", m);
|
|
|
|
return -EBADMSG;
|
|
|
|
}
|
|
|
|
|
|
|
|
total_in += sizeof(header) + m;
|
|
|
|
|
|
|
|
if (!GREEDY_REALLOC(buf, buf_size, m))
|
|
|
|
return log_oom();
|
|
|
|
|
|
|
|
errno = 0;
|
|
|
|
n = loop_read(fdf, buf, m, false);
|
|
|
|
if (n < 0)
|
|
|
|
return n;
|
|
|
|
if (n != m)
|
|
|
|
return errno ? -errno : -EIO;
|
|
|
|
|
|
|
|
r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
|
|
|
|
if (r <= 0)
|
|
|
|
log_error("LZ4 decompression failed.");
|
|
|
|
|
|
|
|
total_out += r;
|
|
|
|
|
|
|
|
if (max_bytes != -1 && total_out > (size_t) max_bytes) {
|
|
|
|
log_debug("Decompressed stream longer than %zd bytes", max_bytes);
|
|
|
|
return -EFBIG;
|
|
|
|
}
|
|
|
|
|
|
|
|
n = loop_write(fdt, out, r, false);
|
|
|
|
if (n < 0)
|
|
|
|
return n;
|
|
|
|
}
|
|
|
|
|
|
|
|
log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
|
|
|
|
total_in, total_out,
|
|
|
|
(double) total_out / total_in * 100);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
#else
|
|
|
|
log_error("Cannot decompress file. Compiled without LZ4 support.");
|
|
|
|
return -EPROTONOSUPPORT;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes) {
|
|
|
|
|
|
|
|
if (endswith(filename, ".lz4"))
|
|
|
|
return decompress_stream_lz4(fdf, fdt, max_bytes);
|
|
|
|
else if (endswith(filename, ".xz"))
|
|
|
|
return decompress_stream_xz(fdf, fdt, max_bytes);
|
|
|
|
else
|
|
|
|
return -EPROTONOSUPPORT;
|
2014-06-25 03:24:46 +02:00
|
|
|
}
|