picnoir/Systemd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

journal: provide compress_threshold_bytes parameter

Browse source 
Previously the compression threshold was hardcoded to 512, which meant that
smaller values wouldn't be compressed. This left some storage savings on the
table, so instead, we make that number tunable.
This commit is contained in:
Alex Gartrell 2018-02-26 21:56:35 -08:00
parent 3ceae1bc14
commit 57850536d5
11 changed files with 141 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/journal-remote/journal-remote-write.c
View file

@ -22,7 +22,7 @@
#include "journal-remote.h"
static int do_rotate(JournalFile **f, bool compress, bool seal) {
int r = journal_file_rotate(f, compress, seal, NULL);
int r = journal_file_rotate(f, compress, (uint64_t) -1, seal, NULL);
if (r < 0) {
if (*f)
log_error_errno(r, "Failed to rotate %s: %m", (*f)->path);

4
src/journal-remote/journal-remote.c
View file

@ -185,7 +185,7 @@ static int open_output(Writer *w, const char* host) {
r = journal_file_open_reliably(output,
O_RDWR|O_CREAT, 0640,
arg_compress, arg_seal,
arg_compress, (uint64_t) -1, arg_seal,
&w->metrics,
w->mmap, NULL,
NULL, &w->journal);
@ -729,7 +729,7 @@ static int setup_microhttpd_server(RemoteServer *s,
}
r = sd_event_add_time(s->events, &d->timer_event,
CLOCK_MONOTONIC, UINT64_MAX, 0,
CLOCK_MONOTONIC, (uint64_t) -1, 0,
null_timer_event_handler, d);
if (r < 0) {
log_error_errno(r, "Failed to add timer_event: %m");

30
src/journal/journal-file.c
View file

@ -51,7 +51,8 @@
#define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*sizeof(HashItem))
#define DEFAULT_FIELD_HASH_TABLE_SIZE (333ULL*sizeof(HashItem))
#define COMPRESSION_SIZE_THRESHOLD (512ULL)
#define DEFAULT_COMPRESS_THRESHOLD (512ULL)
#define MIN_COMPRESS_THRESHOLD (8ULL)
/* This is the minimum journal file size */
#define JOURNAL_FILE_SIZE_MIN (512ULL*1024ULL) /* 512 KiB */
@ -1552,7 +1553,7 @@ static int journal_file_append_data(
o->data.hash = htole64(hash);
#if HAVE_XZ || HAVE_LZ4
if (JOURNAL_FILE_COMPRESS(f) && size >= COMPRESSION_SIZE_THRESHOLD) {
if (JOURNAL_FILE_COMPRESS(f) && size >= f->compress_threshold_bytes) {
size_t rsize = 0;
compression = compress_blob(data, size, o->data.payload, size - 1, &rsize);
@ -3211,6 +3212,7 @@ int journal_file_open(
int flags,
mode_t mode,
bool compress,
uint64_t compress_threshold_bytes,
bool seal,
JournalMetrics *metrics,
MMapCache *mmap_cache,
@ -3222,6 +3224,7 @@ int journal_file_open(
JournalFile *f;
void *h;
int r;
char bytes[FORMAT_BYTES_MAX];
assert(ret);
assert(fd >= 0 || fname);
@ -3247,10 +3250,20 @@ int journal_file_open(
#elif HAVE_XZ
f->compress_xz = compress;
#endif
if (compress_threshold_bytes == (uint64_t) -1)
f->compress_threshold_bytes = DEFAULT_COMPRESS_THRESHOLD;
else
f->compress_threshold_bytes = MAX(MIN_COMPRESS_THRESHOLD, compress_threshold_bytes);
#if HAVE_GCRYPT
f->seal = seal;
#endif
log_debug("Journal effective settings seal=%s compress=%s compress_threshold_bytes=%s",
yes_no(f->seal), yes_no(JOURNAL_FILE_COMPRESS(f)),
format_bytes(bytes, sizeof(bytes), f->compress_threshold_bytes));
if (mmap_cache)
f->mmap = mmap_cache_ref(mmap_cache);
else {
@ -3435,7 +3448,7 @@ fail:
return r;
}
int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred_closes) {
int journal_file_rotate(JournalFile **f, bool compress, uint64_t compress_threshold_bytes, bool seal, Set *deferred_closes) {
_cleanup_free_ char *p = NULL;
size_t l;
JournalFile *old_file, *new_file = NULL;
@ -3489,7 +3502,9 @@ int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred
* we archive them */
old_file->defrag_on_close = true;
r = journal_file_open(-1, old_file->path, old_file->flags, old_file->mode, compress, seal, NULL, old_file->mmap, deferred_closes, old_file, &new_file);
r = journal_file_open(-1, old_file->path, old_file->flags, old_file->mode, compress,
compress_threshold_bytes, seal, NULL, old_file->mmap, deferred_closes,
old_file, &new_file);
if (deferred_closes &&
set_put(deferred_closes, old_file) >= 0)
@ -3506,6 +3521,7 @@ int journal_file_open_reliably(
int flags,
mode_t mode,
bool compress,
uint64_t compress_threshold_bytes,
bool seal,
JournalMetrics *metrics,
MMapCache *mmap_cache,
@ -3517,7 +3533,8 @@ int journal_file_open_reliably(
size_t l;
_cleanup_free_ char *p = NULL;
r = journal_file_open(-1, fname, flags, mode, compress, seal, metrics, mmap_cache, deferred_closes, template, ret);
r = journal_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, mmap_cache,
deferred_closes, template, ret);
if (!IN_SET(r,
-EBADMSG, /* Corrupted */
-ENODATA, /* Truncated */
@ -3559,7 +3576,8 @@ int journal_file_open_reliably(
log_warning_errno(r, "File %s corrupted or uncleanly shut down, renaming and replacing.", fname);
return journal_file_open(-1, fname, flags, mode, compress, seal, metrics, mmap_cache, deferred_closes, template, ret);
return journal_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, mmap_cache,
deferred_closes, template, ret);
}
int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p, uint64_t *seqnum, Object **ret, uint64_t *offset) {

5
src/journal/journal-file.h
View file

@ -124,6 +124,7 @@ typedef struct JournalFile {
unsigned last_seen_generation;
uint64_t compress_threshold_bytes;
#if HAVE_XZ || HAVE_LZ4
void *compress_buffer;
size_t compress_buffer_size;
@ -153,6 +154,7 @@ int journal_file_open(
int flags,
mode_t mode,
bool compress,
uint64_t compress_threshold_bytes,
bool seal,
JournalMetrics *metrics,
MMapCache *mmap_cache,
@ -169,6 +171,7 @@ int journal_file_open_reliably(
int flags,
mode_t mode,
bool compress,
uint64_t compress_threshold_bytes,
bool seal,
JournalMetrics *metrics,
MMapCache *mmap_cache,
@ -246,7 +249,7 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6
void journal_file_dump(JournalFile *f);
void journal_file_print_header(JournalFile *f);
int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred_closes);
int journal_file_rotate(JournalFile **f, bool compress, uint64_t compress_threshold_bytes, bool seal, Set *deferred_closes);
void journal_file_post_change(JournalFile *f);
int journal_file_enable_post_change_timer(JournalFile *f, sd_event *e, usec_t t);

6
src/journal/journald-server.c
View file

@ -280,9 +280,9 @@ static int open_journal(
assert(ret);
if (reliably)
r = journal_file_open_reliably(fname, flags, 0640, s->compress, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
r = journal_file_open_reliably(fname, flags, 0640, s->compress, (uint64_t) -1, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
else
r = journal_file_open(-1, fname, flags, 0640, s->compress, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
r = journal_file_open(-1, fname, flags, 0640, s->compress, (uint64_t) -1, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
if (r < 0)
return r;
@ -463,7 +463,7 @@ static int do_rotate(
if (!*f)
return -EINVAL;
r = journal_file_rotate(f, s->compress, seal, s->deferred_closes);
r = journal_file_rotate(f, s->compress, (uint64_t) -1, seal, s->deferred_closes);
if (r < 0) {
if (*f)
return log_error_errno(r, "Failed to rotate %s: %m", (*f)->path);

2
src/journal/sd-journal.c
View file

@ -1337,7 +1337,7 @@ static int add_any_file(
goto finish;
}
r = journal_file_open(fd, path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, NULL, &f);
r = journal_file_open(fd, path, O_RDONLY, 0, false, 0, false, NULL, j->mmap, NULL, NULL, &f);
if (r < 0) {
log_debug_errno(r, "Failed to open journal file %s: %m", path);
goto finish;

2
src/journal/test-journal-flush.c
View file

@ -39,7 +39,7 @@ int main(int argc, char *argv[]) {
assert_se(mkdtemp(dn));
fn = strappend(dn, "/test.journal");
r = journal_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, false, NULL, NULL, NULL, NULL, &new_journal);
r = journal_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, 0, false, NULL, NULL, NULL, NULL, &new_journal);
assert_se(r >= 0);
r = sd_journal_open(&j, 0);

8
src/journal/test-journal-interleaving.c
View file

@ -52,7 +52,7 @@ _noreturn_ static void log_assert_errno(const char *text, int error, const char
static JournalFile *test_open(const char *name) {
JournalFile *f;
assert_ret(journal_file_open(-1, name, O_RDWR|O_CREAT, 0644, true, false, NULL, NULL, NULL, NULL, &f));
assert_ret(journal_file_open(-1, name, O_RDWR|O_CREAT, 0644, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f));
return f;
}
@ -217,7 +217,7 @@ static void test_sequence_numbers(void) {
assert_se(chdir(t) >= 0);
assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0644,
true, false, NULL, NULL, NULL, NULL, &one) == 0);
true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &one) == 0);
append_number(one, 1, &seqnum);
printf("seqnum=%"PRIu64"\n", seqnum);
@ -234,7 +234,7 @@ static void test_sequence_numbers(void) {
memcpy(&seqnum_id, &one->header->seqnum_id, sizeof(sd_id128_t));
assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0644,
true, false, NULL, NULL, NULL, one, &two) == 0);
true, (uint64_t) -1, false, NULL, NULL, NULL, one, &two) == 0);
assert_se(two->header->state == STATE_ONLINE);
assert_se(!sd_id128_equal(two->header->file_id, one->header->file_id));
@ -265,7 +265,7 @@ static void test_sequence_numbers(void) {
seqnum = 0;
assert_se(journal_file_open(-1, "two.journal", O_RDWR, 0,
true, false, NULL, NULL, NULL, NULL, &two) == 0);
true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &two) == 0);
assert_se(sd_id128_equal(two->header->seqnum_id, seqnum_id));

6
src/journal/test-journal-stream.c
View file

@ -93,9 +93,9 @@ int main(int argc, char *argv[]) {
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &one) == 0);
assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &two) == 0);
assert_se(journal_file_open(-1, "three.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &three) == 0);
assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &one) == 0);
assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &two) == 0);
assert_se(journal_file_open(-1, "three.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &three) == 0);
for (i = 0; i < N_ENTRIES; i++) {
char *p, *q;

6
src/journal/test-journal-verify.c
View file

@ -56,7 +56,7 @@ static int raw_verify(const char *fn, const char *verification_key) {
JournalFile *f;
int r;
r = journal_file_open(-1, fn, O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f);
r = journal_file_open(-1, fn, O_RDONLY, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f);
if (r < 0)
return r;
@ -89,7 +89,7 @@ int main(int argc, char *argv[]) {
log_info("Generating...");
assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
for (n = 0; n < N_ENTRIES; n++) {
struct iovec iovec;
@ -112,7 +112,7 @@ int main(int argc, char *argv[]) {
log_info("Verifying...");
assert_se(journal_file_open(-1, "test.journal", O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
assert_se(journal_file_open(-1, "test.journal", O_RDONLY, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
/* journal_file_print_header(f); */
journal_file_dump(f);

102
src/journal/test-journal.c
View file

@ -43,7 +43,7 @@ static void test_non_empty(void) {
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f) == 0);
assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f) == 0);
dual_timestamp_get(&ts);
@ -105,8 +105,8 @@ static void test_non_empty(void) {
assert_se(journal_file_move_to_entry_by_seqnum(f, 10, DIRECTION_DOWN, &o, NULL) == 0);
journal_file_rotate(&f, true, true, NULL);
journal_file_rotate(&f, true, true, NULL);
journal_file_rotate(&f, true, (uint64_t) -1, true, NULL);
journal_file_rotate(&f, true, (uint64_t) -1, true, NULL);
(void) journal_file_close(f);
@ -132,13 +132,13 @@ static void test_empty(void) {
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, false, NULL, NULL, NULL, NULL, &f1) == 0);
assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f1) == 0);
assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &f2) == 0);
assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f2) == 0);
assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, true, NULL, NULL, NULL, NULL, &f3) == 0);
assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f3) == 0);
assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f4) == 0);
assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f4) == 0);
journal_file_print_header(f1);
puts("");
@ -165,6 +165,91 @@ static void test_empty(void) {
(void) journal_file_close(f4);
}
#if HAVE_XZ || HAVE_LZ4
static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) {
dual_timestamp ts;
JournalFile *f;
struct iovec iovec;
Object *o;
uint64_t p;
char t[] = "/tmp/journal-XXXXXX";
char data[2048] = {0};
bool is_compressed;
int r;
assert_se(data_size <= sizeof(data));
log_set_max_level(LOG_DEBUG);
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, compress_threshold, true, NULL, NULL, NULL, NULL, &f) == 0);
dual_timestamp_get(&ts);
iovec.iov_base = (void*) data;
iovec.iov_len = data_size;
assert_se(journal_file_append_entry(f, &ts, &iovec, 1, NULL, NULL, NULL) == 0);
#if HAVE_GCRYPT
journal_file_append_tag(f);
#endif
journal_file_dump(f);
/* We have to partially reimplement some of the dump logic, because the normal next_entry does the
* decompression for us. */
p = le64toh(f->header->header_size);
while (true) {
r = journal_file_move_to_object(f, OBJECT_UNUSED, p, &o);
assert_se(r == 0);
if (o->object.type == OBJECT_DATA)
break;
assert_se(p < le64toh(f->header->tail_object_offset));
p = p + ALIGN64(le64toh(o->object.size));
}
is_compressed = (o->object.flags & OBJECT_COMPRESSION_MASK) != 0;
(void) journal_file_close(f);
log_info("Done...");
if (arg_keep)
log_info("Not removing %s", t);
else {
journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
}
puts("------------------------------------------------------------");
return is_compressed;
}
static void test_min_compress_size(void) {
/* Note that XZ will actually fail to compress anything under 80 bytes, so you have to choose the limits
* carefully */
/* DEFAULT_MIN_COMPRESS_SIZE is 512 */
assert_se(!check_compressed((uint64_t) -1, 255));
assert_se(check_compressed((uint64_t) -1, 513));
/* compress everything */
assert_se(check_compressed(0, 96));
assert_se(check_compressed(8, 96));
/* Ensure we don't try to compress less than 8 bytes */
assert_se(!check_compressed(0, 7));
/* check boundary conditions */
assert_se(check_compressed(256, 256));
assert_se(!check_compressed(256, 255));
}
#endif
int main(int argc, char *argv[]) {
arg_keep = argc > 1;
@ -174,6 +259,9 @@ int main(int argc, char *argv[]) {
test_non_empty();
test_empty();
#if HAVE_XZ || HAVE_LZ4
test_min_compress_size();
#endif
return 0;
}