From 1b7cf0e5879ec04cb372766385228e8fb75e5500 Mon Sep 17 00:00:00 2001 From: Alex Gartrell Date: Tue, 27 Feb 2018 09:37:23 -0800 Subject: [PATCH] journal: make the compression threshold tunable Allow a user to set a number of bytes as Compress to use as the compression threshold. --- man/journald.conf.xml | 11 +++-- src/journal/journald-gperf.gperf | 2 +- src/journal/journald-server.c | 46 ++++++++++++++++++-- src/journal/journald-server.h | 8 +++- src/journal/test-journal-config.c | 71 +++++++++++++++++++++++++++++++ src/test/meson.build | 7 +++ 6 files changed, 135 insertions(+), 10 deletions(-) create mode 100644 src/journal/test-journal-config.c diff --git a/man/journald.conf.xml b/man/journald.conf.xml index 844228e324..aaef49efd5 100644 --- a/man/journald.conf.xml +++ b/man/journald.conf.xml @@ -107,10 +107,13 @@ Compress= - Takes a boolean value. If enabled (the - default), data objects that shall be stored in the journal and - are larger than a certain threshold are compressed before they - are written to the file system. + Can take a boolean value. If enabled (the + default), data objects that shall be stored in the journal + and are larger than the default threshold of 512 bytes are + compressed before they are written to the file system. It + can also be set to a number of bytes to specify the + compression threshold directly. Suffixes like K, M, and G + can be used to specify larger units. diff --git a/src/journal/journald-gperf.gperf b/src/journal/journald-gperf.gperf index e8c1f2e2d5..1adcb50b6d 100644 --- a/src/journal/journald-gperf.gperf +++ b/src/journal/journald-gperf.gperf @@ -19,7 +19,7 @@ struct ConfigPerfItem; %includes %% Journal.Storage, config_parse_storage, 0, offsetof(Server, storage) -Journal.Compress, config_parse_bool, 0, offsetof(Server, compress) +Journal.Compress, config_parse_compress, 0, offsetof(Server, compress) Journal.Seal, config_parse_bool, 0, offsetof(Server, seal) Journal.ReadKMsg, config_parse_bool, 0, offsetof(Server, read_kmsg) Journal.SyncIntervalSec, config_parse_sec, 0, offsetof(Server, sync_interval_usec) diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index 570450d321..a5f1315327 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -280,9 +280,12 @@ static int open_journal( assert(ret); if (reliably) - r = journal_file_open_reliably(fname, flags, 0640, s->compress, (uint64_t) -1, seal, metrics, s->mmap, s->deferred_closes, NULL, &f); + r = journal_file_open_reliably(fname, flags, 0640, s->compress.enabled, s->compress.threshold_bytes, + seal, metrics, s->mmap, s->deferred_closes, NULL, &f); else - r = journal_file_open(-1, fname, flags, 0640, s->compress, (uint64_t) -1, seal, metrics, s->mmap, s->deferred_closes, NULL, &f); + r = journal_file_open(-1, fname, flags, 0640, s->compress.enabled, s->compress.threshold_bytes, seal, + metrics, s->mmap, s->deferred_closes, NULL, &f); + if (r < 0) return r; @@ -463,7 +466,7 @@ static int do_rotate( if (!*f) return -EINVAL; - r = journal_file_rotate(f, s->compress, (uint64_t) -1, seal, s->deferred_closes); + r = journal_file_rotate(f, s->compress.enabled, s->compress.threshold_bytes, seal, s->deferred_closes); if (r < 0) { if (*f) return log_error_errno(r, "Failed to rotate %s: %m", (*f)->path); @@ -1695,7 +1698,8 @@ int server_init(Server *s) { zero(*s); s->syslog_fd = s->native_fd = s->stdout_fd = s->dev_kmsg_fd = s->audit_fd = s->hostname_fd = s->notify_fd = -1; - s->compress = true; + s->compress.enabled = true; + s->compress.threshold_bytes = (uint64_t) -1; s->seal = true; s->read_kmsg = true; @@ -2038,3 +2042,37 @@ int config_parse_line_max( return 0; } + +int config_parse_compress(const char* unit, + const char *filename, + unsigned line, + const char *section, + unsigned section_line, + const char *lvalue, + int ltype, + const char *rvalue, + void *data, + void *userdata) { + JournalCompressOptions* compress = data; + int r; + + if (streq(rvalue, "1")) { + log_syntax(unit, LOG_WARNING, filename, line, 0, + "Compress= ambiguously specified as 1, enabling compression with default threshold"); + compress->enabled = true; + } else if (streq(rvalue, "0")) { + log_syntax(unit, LOG_WARNING, filename, line, 0, + "Compress= ambiguously specified as 0, disabling compression"); + compress->enabled = false; + } else if ((r = parse_boolean(rvalue)) >= 0) + compress->enabled = r; + else if (parse_size(rvalue, 1024, &compress->threshold_bytes) == 0) + compress->enabled = true; + else if (isempty(rvalue)) { + compress->enabled = true; + compress->threshold_bytes = (uint64_t) -1; + } else + log_syntax(unit, LOG_ERR, filename, line, r, "Failed to parse Compress= value, ignoring: %s", rvalue); + + return 0; +} diff --git a/src/journal/journald-server.h b/src/journal/journald-server.h index bf4ba6897a..505652723d 100644 --- a/src/journal/journald-server.h +++ b/src/journal/journald-server.h @@ -52,6 +52,11 @@ typedef enum SplitMode { _SPLIT_INVALID = -1 } SplitMode; +typedef struct JournalCompressOptions { + bool enabled; + uint64_t threshold_bytes; +} JournalCompressOptions; + typedef struct JournalStorageSpace { usec_t timestamp; @@ -113,7 +118,7 @@ struct Server { JournalStorage runtime_storage; JournalStorage system_storage; - bool compress; + JournalCompressOptions compress; bool seal; bool read_kmsg; @@ -205,6 +210,7 @@ const struct ConfigPerfItem* journald_gperf_lookup(const char *key, GPERF_LEN_TY int config_parse_storage(const char *unit, const char *filename, unsigned line, const char *section, unsigned section_line, const char *lvalue, int ltype, const char *rvalue, void *data, void *userdata); int config_parse_line_max(const char *unit, const char *filename, unsigned line, const char *section, unsigned section_line, const char *lvalue, int ltype, const char *rvalue, void *data, void *userdata); +int config_parse_compress(const char *unit, const char *filename, unsigned line, const char *section, unsigned section_line, const char *lvalue, int ltype, const char *rvalue, void *data, void *userdata); const char *storage_to_string(Storage s) _const_; Storage storage_from_string(const char *s) _pure_; diff --git a/src/journal/test-journal-config.c b/src/journal/test-journal-config.c new file mode 100644 index 0000000000..948739486e --- /dev/null +++ b/src/journal/test-journal-config.c @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +/*** + This file is part of systemd. + + Copyright 2011 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +#include + +#include "journald-server.h" + +#define _COMPRESS_PARSE_CHECK(str, enab, thresh, varname) \ + do { \ + JournalCompressOptions varname = {true, 111}; \ + config_parse_compress("", "", 0, "", 0, "", 0, str, \ + &varname, NULL); \ + assert_se((enab) == varname.enabled); \ + if (varname.enabled) \ + assert_se((thresh) == varname.threshold_bytes); \ + } while (0) + +#define COMPRESS_PARSE_CHECK(str, enabled, threshold) \ + _COMPRESS_PARSE_CHECK(str, enabled, threshold, conf##__COUNTER__) + +static void test_config_compress(void) { + COMPRESS_PARSE_CHECK("yes", true, 111); + COMPRESS_PARSE_CHECK("no", false, 111); + COMPRESS_PARSE_CHECK("y", true, 111); + COMPRESS_PARSE_CHECK("n", false, 111); + COMPRESS_PARSE_CHECK("true", true, 111); + COMPRESS_PARSE_CHECK("false", false, 111); + COMPRESS_PARSE_CHECK("t", true, 111); + COMPRESS_PARSE_CHECK("f", false, 111); + COMPRESS_PARSE_CHECK("on", true, 111); + COMPRESS_PARSE_CHECK("off", false, 111); + + /* Weird size/bool overlapping case. We preserve backward compatibility instead of assuming these are byte + * counts. */ + COMPRESS_PARSE_CHECK("1", true, 111); + COMPRESS_PARSE_CHECK("0", false, 111); + + /* IEC sizing */ + COMPRESS_PARSE_CHECK("1B", true, 1); + COMPRESS_PARSE_CHECK("1K", true, 1024); + COMPRESS_PARSE_CHECK("1M", true, 1024 * 1024); + COMPRESS_PARSE_CHECK("1G", true, 1024 * 1024 * 1024); + + /* Invalid Case */ + COMPRESS_PARSE_CHECK("-1", true, 111); + COMPRESS_PARSE_CHECK("blah blah", true, 111); + COMPRESS_PARSE_CHECK("", true, (uint64_t)-1); +} + +int main(int argc, char *argv[]) { + test_config_compress(); + + return 0; +} diff --git a/src/test/meson.build b/src/test/meson.build index e45618ef88..205a09d62a 100644 --- a/src/test/meson.build +++ b/src/test/meson.build @@ -725,6 +725,13 @@ tests += [ libxz, liblz4]], + [['src/journal/test-journal-config.c'], + [libjournal_core, + libshared], + [libxz, + liblz4, + libselinux]], + [['src/journal/test-journal-verify.c'], [libjournal_core, libshared],