2020-11-09 05:23:58 +01:00
|
|
|
/* SPDX-License-Identifier: LGPL-2.1-or-later */
|
2012-07-18 19:07:51 +02:00
|
|
|
#pragma once
|
2011-10-07 21:06:39 +02:00
|
|
|
|
|
|
|
#include <inttypes.h>
|
2019-03-28 15:55:09 +01:00
|
|
|
#include <sys/uio.h>
|
2011-10-07 21:06:39 +02:00
|
|
|
|
2017-10-03 10:41:51 +02:00
|
|
|
#if HAVE_GCRYPT
|
2018-11-19 13:29:16 +01:00
|
|
|
# include <gcrypt.h>
|
2012-08-13 20:31:10 +02:00
|
|
|
#endif
|
|
|
|
|
2018-11-19 13:29:16 +01:00
|
|
|
#include "sd-event.h"
|
2014-12-30 20:57:53 +01:00
|
|
|
#include "sd-id128.h"
|
2012-01-05 16:01:58 +01:00
|
|
|
|
2015-11-18 22:46:33 +01:00
|
|
|
#include "hashmap.h"
|
2011-10-07 21:06:39 +02:00
|
|
|
#include "journal-def.h"
|
2012-08-14 22:02:24 +02:00
|
|
|
#include "mmap-cache.h"
|
2015-11-18 22:46:33 +01:00
|
|
|
#include "sparse-endian.h"
|
2019-03-27 11:32:41 +01:00
|
|
|
#include "time-util.h"
|
2011-11-08 18:20:03 +01:00
|
|
|
|
2011-12-20 02:38:36 +01:00
|
|
|
typedef struct JournalMetrics {
|
2015-10-02 23:21:59 +02:00
|
|
|
/* For all these: -1 means "pick automatically", and 0 means "no limit enforced" */
|
|
|
|
uint64_t max_size; /* how large journal files grow at max */
|
|
|
|
uint64_t min_size; /* how large journal files grow at least */
|
|
|
|
uint64_t max_use; /* how much disk space to use in total at max, keep_free permitting */
|
|
|
|
uint64_t min_use; /* how much disk space to use in total at least, even if keep_free says not to */
|
|
|
|
uint64_t keep_free; /* how much to keep free on disk */
|
|
|
|
uint64_t n_max_files; /* how many files to keep around at max */
|
2011-12-20 02:38:36 +01:00
|
|
|
} JournalMetrics;
|
|
|
|
|
2013-06-07 04:28:05 +02:00
|
|
|
typedef enum direction {
|
|
|
|
DIRECTION_UP,
|
|
|
|
DIRECTION_DOWN
|
|
|
|
} direction_t;
|
|
|
|
|
2014-12-16 20:51:58 +01:00
|
|
|
typedef enum LocationType {
|
|
|
|
/* The first and last entries, resp. */
|
|
|
|
LOCATION_HEAD,
|
|
|
|
LOCATION_TAIL,
|
|
|
|
|
|
|
|
/* We already read the entry we currently point to, and the
|
|
|
|
* next one to read should probably not be this one again. */
|
|
|
|
LOCATION_DISCRETE,
|
|
|
|
|
|
|
|
/* We should seek to the precise location specified, and
|
|
|
|
* return it, as we haven't read it yet. */
|
|
|
|
LOCATION_SEEK
|
|
|
|
} LocationType;
|
|
|
|
|
2016-02-12 13:59:57 +01:00
|
|
|
typedef enum OfflineState {
|
|
|
|
OFFLINE_JOINED,
|
|
|
|
OFFLINE_SYNCING,
|
|
|
|
OFFLINE_OFFLINING,
|
|
|
|
OFFLINE_CANCEL,
|
|
|
|
OFFLINE_AGAIN_FROM_SYNCING,
|
|
|
|
OFFLINE_AGAIN_FROM_OFFLINING,
|
|
|
|
OFFLINE_DONE
|
|
|
|
} OfflineState;
|
|
|
|
|
2011-10-07 23:03:07 +02:00
|
|
|
typedef struct JournalFile {
|
|
|
|
int fd;
|
2017-07-11 01:24:56 +02:00
|
|
|
MMapFileDescriptor *cache_fd;
|
2013-06-07 04:28:05 +02:00
|
|
|
|
2011-10-13 05:19:35 +02:00
|
|
|
mode_t mode;
|
2012-08-13 20:31:10 +02:00
|
|
|
|
2011-10-13 05:19:35 +02:00
|
|
|
int flags;
|
2013-11-26 18:40:23 +01:00
|
|
|
bool writable:1;
|
2014-07-04 04:42:22 +02:00
|
|
|
bool compress_xz:1;
|
|
|
|
bool compress_lz4:1;
|
2020-06-01 23:26:55 +02:00
|
|
|
bool compress_zstd:1;
|
2013-11-26 18:40:23 +01:00
|
|
|
bool seal:1;
|
2015-01-06 19:51:03 +01:00
|
|
|
bool defrag_on_close:1;
|
2016-04-25 00:31:24 +02:00
|
|
|
bool close_fd:1;
|
2016-04-27 08:29:43 +02:00
|
|
|
bool archive:1;
|
2020-05-30 00:00:50 +02:00
|
|
|
bool keyed_hash:1;
|
2012-08-13 20:31:10 +02:00
|
|
|
|
2013-06-07 04:28:05 +02:00
|
|
|
direction_t last_direction;
|
2014-12-16 21:03:36 +01:00
|
|
|
LocationType location_type;
|
2014-12-18 14:21:55 +01:00
|
|
|
uint64_t last_n_entries;
|
2013-06-07 04:28:05 +02:00
|
|
|
|
|
|
|
char *path;
|
|
|
|
struct stat last_stat;
|
2015-01-05 02:09:01 +01:00
|
|
|
usec_t last_stat_usec;
|
2013-06-07 04:28:05 +02:00
|
|
|
|
2011-10-07 23:03:07 +02:00
|
|
|
Header *header;
|
2011-11-08 18:20:03 +01:00
|
|
|
HashItem *data_hash_table;
|
|
|
|
HashItem *field_hash_table;
|
2011-10-07 23:03:07 +02:00
|
|
|
|
|
|
|
uint64_t current_offset;
|
2014-12-16 21:03:36 +01:00
|
|
|
uint64_t current_seqnum;
|
|
|
|
uint64_t current_realtime;
|
|
|
|
uint64_t current_monotonic;
|
|
|
|
sd_id128_t current_boot_id;
|
|
|
|
uint64_t current_xor_hash;
|
2011-12-20 02:38:36 +01:00
|
|
|
|
|
|
|
JournalMetrics metrics;
|
2012-08-14 22:02:24 +02:00
|
|
|
MMapCache *mmap;
|
2011-12-21 02:40:59 +01:00
|
|
|
|
2015-12-11 07:42:22 +01:00
|
|
|
sd_event_source *post_change_timer;
|
|
|
|
usec_t post_change_timer_period;
|
|
|
|
|
2014-10-14 17:58:13 +02:00
|
|
|
OrderedHashmap *chain_cache;
|
2012-10-26 03:24:03 +02:00
|
|
|
|
2016-02-12 13:59:57 +01:00
|
|
|
pthread_t offline_thread;
|
|
|
|
volatile OfflineState offline_state;
|
|
|
|
|
2018-02-08 20:11:43 +01:00
|
|
|
unsigned last_seen_generation;
|
|
|
|
|
2018-02-27 06:56:35 +01:00
|
|
|
uint64_t compress_threshold_bytes;
|
2020-06-24 16:33:41 +02:00
|
|
|
#if HAVE_COMPRESSION
|
2011-12-21 02:40:59 +01:00
|
|
|
void *compress_buffer;
|
2014-08-04 04:50:00 +02:00
|
|
|
size_t compress_buffer_size;
|
2011-12-21 02:40:59 +01:00
|
|
|
#endif
|
2012-08-13 20:31:10 +02:00
|
|
|
|
2017-10-03 10:41:51 +02:00
|
|
|
#if HAVE_GCRYPT
|
2012-08-13 20:31:10 +02:00
|
|
|
gcry_md_hd_t hmac;
|
|
|
|
bool hmac_running;
|
|
|
|
|
2012-08-17 00:45:18 +02:00
|
|
|
FSSHeader *fss_file;
|
|
|
|
size_t fss_file_size;
|
|
|
|
|
|
|
|
uint64_t fss_start_usec;
|
|
|
|
uint64_t fss_interval_usec;
|
2012-08-16 20:51:43 +02:00
|
|
|
|
|
|
|
void *fsprg_state;
|
|
|
|
size_t fsprg_state_size;
|
|
|
|
|
|
|
|
void *fsprg_seed;
|
|
|
|
size_t fsprg_seed_size;
|
2012-08-13 20:31:10 +02:00
|
|
|
#endif
|
2011-10-07 23:03:07 +02:00
|
|
|
} JournalFile;
|
|
|
|
|
2012-07-17 00:36:15 +02:00
|
|
|
int journal_file_open(
|
2016-04-25 00:31:24 +02:00
|
|
|
int fd,
|
2012-07-17 00:36:15 +02:00
|
|
|
const char *fname,
|
|
|
|
int flags,
|
|
|
|
mode_t mode,
|
2012-08-13 20:31:10 +02:00
|
|
|
bool compress,
|
2018-02-27 06:56:35 +01:00
|
|
|
uint64_t compress_threshold_bytes,
|
2012-08-17 00:45:18 +02:00
|
|
|
bool seal,
|
2012-07-17 00:36:15 +02:00
|
|
|
JournalMetrics *metrics,
|
2012-08-16 19:30:36 +02:00
|
|
|
MMapCache *mmap_cache,
|
2016-02-18 02:37:10 +01:00
|
|
|
Set *deferred_closes,
|
2012-07-17 00:36:15 +02:00
|
|
|
JournalFile *template,
|
|
|
|
JournalFile **ret);
|
|
|
|
|
2016-02-12 13:59:57 +01:00
|
|
|
int journal_file_set_offline(JournalFile *f, bool wait);
|
2016-02-18 02:37:10 +01:00
|
|
|
bool journal_file_is_offlining(JournalFile *f);
|
2015-10-02 22:36:33 +02:00
|
|
|
JournalFile* journal_file_close(JournalFile *j);
|
2020-02-04 14:23:14 +01:00
|
|
|
int journal_file_fstat(JournalFile *f);
|
2019-05-28 11:07:01 +02:00
|
|
|
DEFINE_TRIVIAL_CLEANUP_FUNC(JournalFile*, journal_file_close);
|
2011-10-07 21:06:39 +02:00
|
|
|
|
2012-07-17 00:36:15 +02:00
|
|
|
int journal_file_open_reliably(
|
|
|
|
const char *fname,
|
|
|
|
int flags,
|
|
|
|
mode_t mode,
|
2012-08-13 20:31:10 +02:00
|
|
|
bool compress,
|
2018-02-27 06:56:35 +01:00
|
|
|
uint64_t compress_threshold_bytes,
|
2012-08-17 00:45:18 +02:00
|
|
|
bool seal,
|
2012-07-17 00:36:15 +02:00
|
|
|
JournalMetrics *metrics,
|
2012-08-16 19:30:36 +02:00
|
|
|
MMapCache *mmap_cache,
|
2016-02-18 02:37:10 +01:00
|
|
|
Set *deferred_closes,
|
2012-07-17 00:36:15 +02:00
|
|
|
JournalFile *template,
|
|
|
|
JournalFile **ret);
|
2012-03-15 01:13:01 +01:00
|
|
|
|
2012-08-16 01:51:54 +02:00
|
|
|
#define ALIGN64(x) (((x) + 7ULL) & ~7ULL)
|
2012-08-18 00:37:21 +02:00
|
|
|
#define VALID64(x) (((x) & 7ULL) == 0ULL)
|
2012-08-16 01:51:54 +02:00
|
|
|
|
2013-12-29 01:33:23 +01:00
|
|
|
/* Use six characters to cover the offsets common in smallish journal
|
|
|
|
* files without adding too many zeros. */
|
|
|
|
#define OFSfmt "%06"PRIx64
|
|
|
|
|
2012-08-19 15:15:59 +02:00
|
|
|
static inline bool VALID_REALTIME(uint64_t u) {
|
|
|
|
/* This considers timestamps until the year 3112 valid. That should be plenty room... */
|
|
|
|
return u > 0 && u < (1ULL << 55);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool VALID_MONOTONIC(uint64_t u) {
|
|
|
|
/* This considers timestamps until 1142 years of runtime valid. */
|
|
|
|
return u < (1ULL << 55);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool VALID_EPOCH(uint64_t u) {
|
|
|
|
/* This allows changing the key for 1142 years, every usec. */
|
|
|
|
return u < (1ULL << 55);
|
|
|
|
}
|
|
|
|
|
2012-08-16 01:51:54 +02:00
|
|
|
#define JOURNAL_HEADER_CONTAINS(h, field) \
|
|
|
|
(le64toh((h)->header_size) >= offsetof(Header, field) + sizeof((h)->field))
|
|
|
|
|
2012-08-20 16:11:42 +02:00
|
|
|
#define JOURNAL_HEADER_SEALED(h) \
|
2020-05-29 23:35:25 +02:00
|
|
|
FLAGS_SET(le32toh((h)->compatible_flags), HEADER_COMPATIBLE_SEALED)
|
2012-08-20 16:11:42 +02:00
|
|
|
|
2014-07-04 04:42:22 +02:00
|
|
|
#define JOURNAL_HEADER_COMPRESSED_XZ(h) \
|
2020-05-29 23:35:25 +02:00
|
|
|
FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_COMPRESSED_XZ)
|
2014-07-04 04:42:22 +02:00
|
|
|
|
|
|
|
#define JOURNAL_HEADER_COMPRESSED_LZ4(h) \
|
2020-05-29 23:35:25 +02:00
|
|
|
FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_COMPRESSED_LZ4)
|
2012-08-20 16:11:42 +02:00
|
|
|
|
2020-06-01 23:26:55 +02:00
|
|
|
#define JOURNAL_HEADER_COMPRESSED_ZSTD(h) \
|
|
|
|
FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_COMPRESSED_ZSTD)
|
|
|
|
|
2020-05-30 00:00:50 +02:00
|
|
|
#define JOURNAL_HEADER_KEYED_HASH(h) \
|
|
|
|
FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_KEYED_HASH)
|
|
|
|
|
2014-12-10 15:18:49 +01:00
|
|
|
int journal_file_move_to_object(JournalFile *f, ObjectType type, uint64_t offset, Object **ret);
|
2011-10-07 21:06:39 +02:00
|
|
|
|
2013-05-03 04:51:50 +02:00
|
|
|
uint64_t journal_file_entry_n_items(Object *o) _pure_;
|
|
|
|
uint64_t journal_file_entry_array_n_items(Object *o) _pure_;
|
|
|
|
uint64_t journal_file_hash_table_n_items(Object *o) _pure_;
|
2011-10-07 21:06:39 +02:00
|
|
|
|
2014-12-10 15:18:49 +01:00
|
|
|
int journal_file_append_object(JournalFile *f, ObjectType type, uint64_t size, Object **ret, uint64_t *offset);
|
2018-05-27 11:27:58 +02:00
|
|
|
int journal_file_append_entry(
|
|
|
|
JournalFile *f,
|
|
|
|
const dual_timestamp *ts,
|
|
|
|
const sd_id128_t *boot_id,
|
|
|
|
const struct iovec iovec[], unsigned n_iovec,
|
|
|
|
uint64_t *seqno,
|
|
|
|
Object **ret,
|
|
|
|
uint64_t *offset);
|
2011-10-07 21:06:39 +02:00
|
|
|
|
2011-11-08 18:20:03 +01:00
|
|
|
int journal_file_find_data_object(JournalFile *f, const void *data, uint64_t size, Object **ret, uint64_t *offset);
|
|
|
|
int journal_file_find_data_object_with_hash(JournalFile *f, const void *data, uint64_t size, uint64_t hash, Object **ret, uint64_t *offset);
|
2011-10-07 21:06:39 +02:00
|
|
|
|
2012-10-18 03:29:19 +02:00
|
|
|
int journal_file_find_field_object(JournalFile *f, const void *field, uint64_t size, Object **ret, uint64_t *offset);
|
|
|
|
int journal_file_find_field_object_with_hash(JournalFile *f, const void *field, uint64_t size, uint64_t hash, Object **ret, uint64_t *offset);
|
|
|
|
|
2014-12-16 20:54:56 +01:00
|
|
|
void journal_file_reset_location(JournalFile *f);
|
2015-02-24 19:45:17 +01:00
|
|
|
void journal_file_save_location(JournalFile *f, Object *o, uint64_t offset);
|
2014-12-17 14:06:28 +01:00
|
|
|
int journal_file_compare_locations(JournalFile *af, JournalFile *bf);
|
2014-12-17 15:45:10 +01:00
|
|
|
int journal_file_next_entry(JournalFile *f, uint64_t p, direction_t direction, Object **ret, uint64_t *offset);
|
2011-10-07 21:06:39 +02:00
|
|
|
|
2011-11-08 18:20:03 +01:00
|
|
|
int journal_file_next_entry_for_data(JournalFile *f, Object *o, uint64_t p, uint64_t data_offset, direction_t direction, Object **ret, uint64_t *offset);
|
|
|
|
|
|
|
|
int journal_file_move_to_entry_by_seqnum(JournalFile *f, uint64_t seqnum, direction_t direction, Object **ret, uint64_t *offset);
|
|
|
|
int journal_file_move_to_entry_by_realtime(JournalFile *f, uint64_t realtime, direction_t direction, Object **ret, uint64_t *offset);
|
|
|
|
int journal_file_move_to_entry_by_monotonic(JournalFile *f, sd_id128_t boot_id, uint64_t monotonic, direction_t direction, Object **ret, uint64_t *offset);
|
|
|
|
|
2012-07-13 00:29:26 +02:00
|
|
|
int journal_file_move_to_entry_by_offset_for_data(JournalFile *f, uint64_t data_offset, uint64_t p, direction_t direction, Object **ret, uint64_t *offset);
|
2011-11-08 18:20:03 +01:00
|
|
|
int journal_file_move_to_entry_by_seqnum_for_data(JournalFile *f, uint64_t data_offset, uint64_t seqnum, direction_t direction, Object **ret, uint64_t *offset);
|
|
|
|
int journal_file_move_to_entry_by_realtime_for_data(JournalFile *f, uint64_t data_offset, uint64_t realtime, direction_t direction, Object **ret, uint64_t *offset);
|
2012-07-13 00:29:26 +02:00
|
|
|
int journal_file_move_to_entry_by_monotonic_for_data(JournalFile *f, uint64_t data_offset, sd_id128_t boot_id, uint64_t monotonic, direction_t direction, Object **ret, uint64_t *offset);
|
2011-10-07 21:06:39 +02:00
|
|
|
|
2018-05-27 11:08:44 +02:00
|
|
|
int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p);
|
2011-12-29 15:00:57 +01:00
|
|
|
|
2011-10-07 21:06:39 +02:00
|
|
|
void journal_file_dump(JournalFile *f);
|
2012-07-16 22:24:02 +02:00
|
|
|
void journal_file_print_header(JournalFile *f);
|
2011-10-07 21:06:39 +02:00
|
|
|
|
2018-10-25 18:33:11 +02:00
|
|
|
int journal_file_archive(JournalFile *f);
|
|
|
|
JournalFile* journal_initiate_close(JournalFile *f, Set *deferred_closes);
|
2018-02-27 06:56:35 +01:00
|
|
|
int journal_file_rotate(JournalFile **f, bool compress, uint64_t compress_threshold_bytes, bool seal, Set *deferred_closes);
|
2011-10-13 05:19:35 +02:00
|
|
|
|
2018-10-25 19:23:23 +02:00
|
|
|
int journal_file_dispose(int dir_fd, const char *fname);
|
|
|
|
|
2011-12-29 15:00:57 +01:00
|
|
|
void journal_file_post_change(JournalFile *f);
|
2015-12-11 07:42:22 +01:00
|
|
|
int journal_file_enable_post_change_timer(JournalFile *f, sd_event *e, usec_t t);
|
2011-12-29 15:00:57 +01:00
|
|
|
|
2015-10-02 23:21:59 +02:00
|
|
|
void journal_reset_metrics(JournalMetrics *m);
|
2011-12-31 02:31:54 +01:00
|
|
|
void journal_default_metrics(JournalMetrics *m, int fd);
|
|
|
|
|
2012-06-09 10:32:38 +02:00
|
|
|
int journal_file_get_cutoff_realtime_usec(JournalFile *f, usec_t *from, usec_t *to);
|
|
|
|
int journal_file_get_cutoff_monotonic_usec(JournalFile *f, sd_id128_t boot, usec_t *from, usec_t *to);
|
|
|
|
|
2012-10-16 22:58:07 +02:00
|
|
|
bool journal_file_rotate_suggested(JournalFile *f, usec_t max_file_usec);
|
2015-07-24 01:55:45 +02:00
|
|
|
|
|
|
|
int journal_file_map_data_hash_table(JournalFile *f);
|
|
|
|
int journal_file_map_field_hash_table(JournalFile *f);
|
2015-10-24 13:17:54 +02:00
|
|
|
|
|
|
|
static inline bool JOURNAL_FILE_COMPRESS(JournalFile *f) {
|
|
|
|
assert(f);
|
2020-06-01 23:26:55 +02:00
|
|
|
return f->compress_xz || f->compress_lz4 || f->compress_zstd;
|
2015-10-24 13:17:54 +02:00
|
|
|
}
|
2020-05-30 00:00:50 +02:00
|
|
|
|
|
|
|
uint64_t journal_file_hash_data(JournalFile *f, const void *data, size_t sz);
|
2020-12-15 20:36:14 +01:00
|
|
|
|
|
|
|
bool journal_field_valid(const char *p, size_t l, bool allow_protected);
|