picnoir/Systemd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

journal: size journal data hash table based on maximum file size metrics

Browse source 
The default of 2047 hash table entries turned out to result in way too
many collisions for bigger files, hence scale the hash table size by the
estimated maximum file size.
This commit is contained in:
Lennart Poettering 2012-07-17 00:36:15 +02:00
parent 71fa6f006f
commit 4a92baf3fa
6 changed files with 49 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
src/journal/journal-file.c
View file

@ -32,8 +32,8 @@
#include "lookup3.h"
#include "compress.h"
#define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*16ULL)
#define DEFAULT_FIELD_HASH_TABLE_SIZE (2047ULL*16ULL)
#define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*sizeof(HashItem))
#define DEFAULT_FIELD_HASH_TABLE_SIZE (333ULL*sizeof(HashItem))
#define DEFAULT_WINDOW_SIZE (8ULL*1024ULL*1024ULL)
@ -509,7 +509,17 @@ static int journal_file_setup_data_hash_table(JournalFile *f) {
assert(f);
s = DEFAULT_DATA_HASH_TABLE_SIZE;
/* We estimate that we need 1 hash table entry per 2K of
journal file and we want to make sure we never get beyond
75% fill level. Calculate the hash table size for the
maximum file size based on these metrics. */
s = (f->metrics.max_size * 4 / 2048 / 3) * sizeof(HashItem);
if (s < DEFAULT_DATA_HASH_TABLE_SIZE)
s = DEFAULT_DATA_HASH_TABLE_SIZE;
log_info("Reserving %llu entries in hash table.", (unsigned long long) s);
r = journal_file_append_object(f,
OBJECT_DATA_HASH_TABLE,
offsetof(Object, hash_table.items) + s,
@ -1939,6 +1949,7 @@ int journal_file_open(
const char *fname,
int flags,
mode_t mode,
JournalMetrics *metrics,
JournalFile *template,
JournalFile **ret) {
@ -1965,10 +1976,8 @@ int journal_file_open(
f->writable = (flags & O_ACCMODE) != O_RDONLY;
f->prot = prot_from_flags(flags);
if (template) {
f->metrics = template->metrics;
if (template)
f->compress = template->compress;
}
f->path = strdup(fname);
if (!f->path) {
@ -2019,6 +2028,12 @@ int journal_file_open(
}
if (f->writable) {
if (metrics) {
journal_default_metrics(metrics, f->fd);
f->metrics = *metrics;
} else if (template)
f->metrics = template->metrics;
r = journal_file_refresh_header(f);
if (r < 0)
goto fail;
@ -2093,7 +2108,7 @@ int journal_file_rotate(JournalFile **f) {
old_file->header->state = STATE_ARCHIVED;
r = journal_file_open(old_file->path, old_file->flags, old_file->mode, old_file, &new_file);
r = journal_file_open(old_file->path, old_file->flags, old_file->mode, NULL, old_file, &new_file);
journal_file_close(old_file);
*f = new_file;
@ -2104,6 +2119,7 @@ int journal_file_open_reliably(
const char *fname,
int flags,
mode_t mode,
JournalMetrics *metrics,
JournalFile *template,
JournalFile **ret) {
@ -2111,7 +2127,7 @@ int journal_file_open_reliably(
size_t l;
char *p;
r = journal_file_open(fname, flags, mode, template, ret);
r = journal_file_open(fname, flags, mode, metrics, template, ret);
if (r != -EBADMSG && /* corrupted */
r != -ENODATA && /* truncated */
r != -EHOSTDOWN && /* other machine */
@ -2140,7 +2156,7 @@ int journal_file_open_reliably(
log_warning("File %s corrupted, renaming and replacing.", fname);
return journal_file_open(fname, flags, mode, template, ret);
return journal_file_open(fname, flags, mode, metrics, template, ret);
}
struct vacuum_info {

17
src/journal/journal-file.h
View file

@ -88,10 +88,23 @@ typedef enum direction {
DIRECTION_DOWN
} direction_t;
int journal_file_open(const char *fname, int flags, mode_t mode, JournalFile *template, JournalFile **ret);
int journal_file_open(
const char *fname,
int flags,
mode_t mode,
JournalMetrics *metrics,
JournalFile *template,
JournalFile **ret);
void journal_file_close(JournalFile *j);
int journal_file_open_reliably(const char *fname, int flags, mode_t mode, JournalFile *template, JournalFile **ret);
int journal_file_open_reliably(
const char *fname,
int flags,
mode_t mode,
JournalMetrics *metrics,
JournalFile *template,
JournalFile **ret);
int journal_file_move_to_object(JournalFile *f, int type, uint64_t offset, Object **ret);

14
src/journal/journald.c
View file

@ -313,7 +313,7 @@ static JournalFile* find_journal(Server *s, uid_t uid) {
journal_file_close(f);
}
r = journal_file_open_reliably(p, O_RDWR|O_CREAT, 0640, s->system_journal, &f);
r = journal_file_open_reliably(p, O_RDWR|O_CREAT, 0640, &s->system_metrics, s->system_journal, &f);
free(p);
if (r < 0)
@ -2005,13 +2005,10 @@ static int system_journal_open(Server *s) {
if (!fn)
return -ENOMEM;
r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, NULL, &s->system_journal);
r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, &s->system_metrics, NULL, &s->system_journal);
free(fn);
if (r >= 0) {
journal_default_metrics(&s->system_metrics, s->system_journal->fd);
s->system_journal->metrics = s->system_metrics;
s->system_journal->compress = s->compress;
server_fix_perms(s, s->system_journal, 0);
@ -2037,7 +2034,7 @@ static int system_journal_open(Server *s) {
* if it already exists, so that we can flush
* it into the system journal */
r = journal_file_open(fn, O_RDWR, 0640, NULL, &s->runtime_journal);
r = journal_file_open(fn, O_RDWR, 0640, &s->runtime_metrics, NULL, &s->runtime_journal);
free(fn);
if (r < 0) {
@ -2053,7 +2050,7 @@ static int system_journal_open(Server *s) {
* it if necessary. */
(void) mkdir_parents(fn, 0755);
r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, NULL, &s->runtime_journal);
r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, &s->runtime_metrics, NULL, &s->runtime_journal);
free(fn);
if (r < 0) {
@ -2063,9 +2060,6 @@ static int system_journal_open(Server *s) {
}
if (s->runtime_journal) {
journal_default_metrics(&s->runtime_metrics, s->runtime_journal->fd);
s->runtime_journal->metrics = s->runtime_metrics;
s->runtime_journal->compress = s->compress;
server_fix_perms(s, s->runtime_journal, 0);

2
src/journal/sd-journal.c
View file

@ -1116,7 +1116,7 @@ static int add_file(sd_journal *j, const char *prefix, const char *filename) {
return 0;
}
r = journal_file_open(path, O_RDONLY, 0, NULL, &f);
r = journal_file_open(path, O_RDONLY, 0, NULL, NULL, &f);
free(path);
if (r < 0) {

6
src/journal/test-journal-stream.c
View file

@ -79,9 +79,9 @@ int main(int argc, char *argv[]) {
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
assert_se(journal_file_open("one.journal", O_RDWR|O_CREAT, 0666, NULL, &one) == 0);
assert_se(journal_file_open("two.journal", O_RDWR|O_CREAT, 0666, NULL, &two) == 0);
assert_se(journal_file_open("three.journal", O_RDWR|O_CREAT, 0666, NULL, &three) == 0);
assert_se(journal_file_open("one.journal", O_RDWR|O_CREAT, 0666, NULL, NULL, &one) == 0);
assert_se(journal_file_open("two.journal", O_RDWR|O_CREAT, 0666, NULL, NULL, &two) == 0);
assert_se(journal_file_open("three.journal", O_RDWR|O_CREAT, 0666, NULL, NULL, &three) == 0);
for (i = 0; i < N_ENTRIES; i++) {
char *p, *q;

2
src/journal/test-journal.c
View file

@ -41,7 +41,7 @@ int main(int argc, char *argv[]) {
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
assert_se(journal_file_open("test.journal", O_RDWR|O_CREAT, 0666, NULL, &f) == 0);
assert_se(journal_file_open("test.journal", O_RDWR|O_CREAT, 0666, NULL, NULL, &f) == 0);
dual_timestamp_get(&ts);