diff --git a/man/pstore.conf.xml b/man/pstore.conf.xml new file mode 100644 index 0000000000..2b9c8b1a71 --- /dev/null +++ b/man/pstore.conf.xml @@ -0,0 +1,89 @@ + + + + + + + pstore.conf + systemd + + + + pstore.conf + 5 + + + + pstore.conf + pstore.conf.d + PStore configuration file + + + + + /etc/systemd/pstore.conf + /etc/systemd/pstore.conf.d/* + + + + + Description + + This file configures the behavior of + systemd-pstore8, + a tool for archiving the contents of the persistent storage filesystem, + pstore. + + + + + + + Options + + All options are configured in the + [PStore] section: + + + + + Storage= + + Controls where to archive (i.e. copy) files from the pstore filesystem. One of none, + external, and journal. When + none, the tool exits without processing files in the pstore filesystem. + When external (the default), files are archived into /var/lib/systemd/pstore/, + and logged into the journal. + When journal, pstore file contents are logged only in the journal. + + + + + + Unlink= + + Controls whether or not files are removed from pstore after processing. + Takes a boolean value. When true, a pstore file is removed from the pstore once it has been + archived (either to disk or into the journal). When false, processing of pstore files occurs + normally, but the files remain in the pstore. + The default is true in order to maintain the pstore in a nearly empty state, so that the pstore + has storage available for the next kernel error event. + + + + + The defaults for all values are listed as comments in the + template /etc/systemd/pstore.conf file that + is installed by default. + + + + See Also + + systemd-journald.service8, + + + + diff --git a/man/rules/meson.build b/man/rules/meson.build index e459697221..7e32e732c1 100644 --- a/man/rules/meson.build +++ b/man/rules/meson.build @@ -46,6 +46,7 @@ manpages = [ ['os-release', '5', [], ''], ['pam_systemd', '8', [], 'HAVE_PAM'], ['portablectl', '1', [], 'ENABLE_PORTABLED'], + ['pstore.conf', '5', ['pstore.conf.d'], 'ENABLE_PSTORE'], ['resolvectl', '1', ['resolvconf'], 'ENABLE_RESOLVE'], ['resolved.conf', '5', ['resolved.conf.d'], 'ENABLE_RESOLVE'], ['runlevel', '8', [], 'ENABLE_UTMP'], @@ -746,6 +747,7 @@ manpages = [ ['systemd-nspawn', '1', [], ''], ['systemd-path', '1', [], ''], ['systemd-portabled.service', '8', ['systemd-portabled'], 'ENABLE_PORTABLED'], + ['systemd-pstore', '8', ['systemd-pstore.service'], 'ENABLE_PSTORE'], ['systemd-quotacheck.service', '8', ['systemd-quotacheck'], diff --git a/man/systemd-pstore.xml b/man/systemd-pstore.xml new file mode 100644 index 0000000000..dd1aa5e83b --- /dev/null +++ b/man/systemd-pstore.xml @@ -0,0 +1,99 @@ + + + + + + + + systemd-pstore + systemd + + + + systemd-pstore + 8 + + + + systemd-pstore + systemd-pstore.service + Tool to archive contents of the persistent storage filesytem + + + + /usr/lib/systemd/systemd-pstore + systemd-pstore.service + + + + Description + systemd-pstore.service is a system service that archives the + contents of the Linux persistent storage filesystem, pstore, to other storage, + thus preserving the existing information contained in the pstore, and clearing + pstore storage for future error events. + + Linux provides a persistent storage file system, pstore, that can store + error records when the kernel dies (or reboots or powers-off). These records in + turn can be referenced to debug kernel problems (currently the kernel stuffs + the tail of the dmesg, which also contains a stack backtrace, into pstore). + + The pstore file system supports a variety of backends that map onto persistent + storage, such as the ACPI ERST and UEFI variables. The pstore backends + typically offer a relatively small amount of persistent storage, e.g. 64KiB, + which can quickly fill up and thus prevent subsequent kernel crashes from + recording errors. Thus there is a need to monitor and extract the pstore + contents so that future kernel problems can also record information in the + pstore. + + The pstore service is independent of the kdump service. In cloud environments + specifically, host and guest filesystems are on remote filesystems (eg. iSCSI + or NFS), thus kdump relies [implicitly and/or explicitly] upon proper operation + of networking software *and* hardware *and* infrastructure. Thus it may not be + possible to capture a kernel coredump to a file since writes over the network + may not be possible. + + The pstore backend, on the other hand, is completely local and provides a path + to store error records which will survive a reboot and aid in post-mortem + debugging. + + The systemd-pstore executable does the actual work. Upon starting, + the pstore.conf is read to obtain options, then the /sys/fs/pstore + directory contents are processed according to the options. Pstore files are written to the + journal, and optionally saved into /var/lib/systemd/pstore. + + + + Configuration + + The behavior of systemd-pstore is configured through the configuration file + /etc/systemd/pstore.conf and corresponding snippets + /etc/systemd/pstore.conf.d/*.conf, see + pstore.conf5. + + + + Disabling pstore processing + + To disable pstore processing by systemd-pstore, + set Storage=none in + pstore.conf5. + + + + + + Usage + Data stored in the journal can be viewed with + journalctl1 + as usual. + + + + See Also + + pstore.conf5 + + + diff --git a/meson.build b/meson.build index 323221bb43..3593eac710 100644 --- a/meson.build +++ b/meson.build @@ -1271,6 +1271,7 @@ foreach term : ['utmp', 'environment-d', 'binfmt', 'coredump', + 'pstore', 'resolve', 'logind', 'hostnamed', @@ -1483,6 +1484,7 @@ subdir('src/network') subdir('src/analyze') subdir('src/journal-remote') subdir('src/coredump') +subdir('src/pstore') subdir('src/hostname') subdir('src/import') subdir('src/kernel-install') @@ -2254,6 +2256,23 @@ if conf.get('ENABLE_COREDUMP') == 1 public_programs += exe endif +if conf.get('ENABLE_PSTORE') == 1 + executable('systemd-pstore', + systemd_pstore_sources, + include_directories : includes, + link_with : [libshared], + dependencies : [threads, + libacl, + libdw, + libxz, + liblz4], + install_rpath : rootlibexecdir, + install : true, + install_dir : rootlibexecdir) + + public_programs += exe +endif + if conf.get('ENABLE_BINFMT') == 1 exe = executable('systemd-binfmt', 'src/binfmt/binfmt.c', @@ -3182,6 +3201,7 @@ foreach tuple : [ ['DNS-over-TLS(gnutls)', conf.get('DNS_OVER_TLS_USE_GNUTLS') == 1], ['DNS-over-TLS(openssl)', conf.get('DNS_OVER_TLS_USE_OPENSSL') == 1], ['coredump'], + ['pstore'], ['polkit'], ['legacy pkla', install_polkit_pkla], ['efi'], diff --git a/meson_options.txt b/meson_options.txt index 17dd2949b9..67aaa89919 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -78,6 +78,8 @@ option('binfmt', type : 'boolean', description : 'support for custom binary formats') option('coredump', type : 'boolean', description : 'install the coredump handler') +option('pstore', type : 'boolean', + description : 'install the pstore archival tool') option('logind', type : 'boolean', description : 'install the systemd-logind stack') option('hostnamed', type : 'boolean', diff --git a/src/pstore/meson.build b/src/pstore/meson.build new file mode 100644 index 0000000000..adbac24b54 --- /dev/null +++ b/src/pstore/meson.build @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: LGPL-2.1+ + +systemd_pstore_sources = files(''' + pstore.c +'''.split()) + +if conf.get('ENABLE_PSTORE') == 1 + install_data('pstore.conf', + install_dir : pkgsysconfdir) +endif diff --git a/src/pstore/pstore.c b/src/pstore/pstore.c new file mode 100644 index 0000000000..0c4e2f08a3 --- /dev/null +++ b/src/pstore/pstore.c @@ -0,0 +1,393 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +/* Copyright © 2019 Oracle and/or its affiliates. */ + +/* Generally speaking, the pstore contains a small number of files + * that in turn contain a small amount of data. */ +#include +#include +#include +#include +#include +#include + +#include "sd-daemon.h" +#include "sd-journal.h" +#include "sd-login.h" +#include "sd-messages.h" + +#include "acl-util.h" +#include "alloc-util.h" +#include "capability-util.h" +#include "cgroup-util.h" +#include "compress.h" +#include "conf-parser.h" +#include "copy.h" +#include "dirent-util.h" +#include "escape.h" +#include "fd-util.h" +#include "fileio.h" +#include "fs-util.h" +#include "io-util.h" +#include "journal-importer.h" +#include "log.h" +#include "macro.h" +#include "main-func.h" +#include "missing.h" +#include "mkdir.h" +#include "parse-util.h" +#include "process-util.h" +#include "signal-util.h" +#include "socket-util.h" +#include "special.h" +#include "sort-util.h" +#include "string-table.h" +#include "string-util.h" +#include "strv.h" +#include "tmpfile-util.h" +#include "user-util.h" +#include "util.h" + +/* Command line argument handling */ +typedef enum PStoreStorage { + PSTORE_STORAGE_NONE, + PSTORE_STORAGE_EXTERNAL, + PSTORE_STORAGE_JOURNAL, + _PSTORE_STORAGE_MAX, + _PSTORE_STORAGE_INVALID = -1 +} PStoreStorage; + +static const char* const pstore_storage_table[_PSTORE_STORAGE_MAX] = { + [PSTORE_STORAGE_NONE] = "none", + [PSTORE_STORAGE_EXTERNAL] = "external", + [PSTORE_STORAGE_JOURNAL] = "journal", +}; + +DEFINE_PRIVATE_STRING_TABLE_LOOKUP(pstore_storage, PStoreStorage); +static DEFINE_CONFIG_PARSE_ENUM(config_parse_pstore_storage, pstore_storage, PStoreStorage, "Failed to parse storage setting"); + +static PStoreStorage arg_storage = PSTORE_STORAGE_EXTERNAL; + +static bool arg_unlink = true; +static const char *arg_sourcedir = "/sys/fs/pstore"; +static const char *arg_archivedir = "/var/lib/systemd/pstore"; + +static int parse_config(void) { + static const ConfigTableItem items[] = { + { "PStore", "Unlink", config_parse_bool, 0, &arg_unlink }, + { "PStore", "Storage", config_parse_pstore_storage, 0, &arg_storage }, + {} + }; + + return config_parse_many_nulstr(PKGSYSCONFDIR "/pstore.conf", + CONF_PATHS_NULSTR("systemd/pstore.conf.d"), + "PStore\0", + config_item_table_lookup, items, + CONFIG_PARSE_WARN, NULL); +} + +/* File list handling - PStoreEntry is the struct and + * and PStoreEntry is the type that contains all info + * about a pstore entry. */ +typedef struct PStoreEntry { + struct dirent dirent; + bool is_binary; + bool handled; + char *content; + size_t content_size; +} PStoreEntry; + +typedef struct PStoreList { + PStoreEntry *entries; + size_t n_entries; + size_t n_entries_allocated; +} PStoreList; + +static void pstore_entries_reset(PStoreList *list) { + for (size_t i = 0; i < list->n_entries; i++) + free(list->entries[i].content); + free(list->entries); + list->n_entries = 0; +} + +static int compare_pstore_entries(const void *_a, const void *_b) { + PStoreEntry *a = (PStoreEntry *)_a, *b = (PStoreEntry *)_b; + return strcmp(a->dirent.d_name, b->dirent.d_name); +} + +static int move_file(PStoreEntry *pe, const char *subdir) { + _cleanup_free_ char *ifd_path = NULL; + _cleanup_free_ char *ofd_path = NULL; + int r = 0; + struct iovec iovec[2] = {}; + int n_iovec = 0; + _cleanup_free_ void *field = NULL; + const char *suffix = NULL; + size_t field_size; + + if (pe->handled) + return 0; + + ifd_path = path_join(arg_sourcedir, pe->dirent.d_name); + if (!ifd_path) + return log_oom(); + + ofd_path = path_join(arg_archivedir, subdir, pe->dirent.d_name); + if (!ofd_path) + return log_oom(); + + /* Always log to the journal */ + suffix = arg_storage == PSTORE_STORAGE_EXTERNAL ? strjoina(" moved to ", ofd_path) : (char *)"."; + field = strjoina("MESSAGE=PStore ", pe->dirent.d_name, suffix); + iovec[n_iovec++] = IOVEC_MAKE_STRING(field); + + field_size = strlen("FILE=") + pe->content_size; + field = malloc(field_size); + if (!field) + return log_oom(); + memcpy(stpcpy(field, "FILE="), pe->content, pe->content_size); + iovec[n_iovec++] = IOVEC_MAKE(field, field_size); + + r = sd_journal_sendv(iovec, n_iovec); + if (r < 0) + return log_error_errno(r, "Failed to log pstore entry: %m"); + + if (arg_storage == PSTORE_STORAGE_EXTERNAL) { + /* Move file from pstore to external storage */ + r = mkdir_parents(ofd_path, 0755); + if (r < 0) + return log_error_errno(r, "Failed to create directoy %s: %m", ofd_path); + r = copy_file_atomic(ifd_path, ofd_path, 0600, 0, 0, COPY_REPLACE); + if (r < 0) + return log_error_errno(r, "Failed to copy_file_atomic: %s to %s", ifd_path, ofd_path); + } + + /* If file copied properly, remove it from pstore */ + if (arg_unlink) + (void) unlink(ifd_path); + + pe->handled = true; + + return 0; +} + +static int write_dmesg(const char *dmesg, size_t size, const char *id) { + _cleanup_(unlink_and_freep) char *ofd_path = NULL; + _cleanup_free_ char *tmp_path = NULL; + _cleanup_close_ int ofd = -1; + ssize_t wr; + int r; + + if (isempty(dmesg) || size == 0) + return 0; + + /* log_info("Record ID %s", id); */ + + ofd_path = path_join(arg_archivedir, id, "dmesg.txt"); + if (!ofd_path) + return log_oom(); + + ofd = open_tmpfile_linkable(ofd_path, O_CLOEXEC|O_CREAT|O_TRUNC|O_WRONLY, &tmp_path); + if (ofd < 0) + return log_error_errno(ofd, "Failed to open temporary file %s: %m", ofd_path); + wr = write(ofd, dmesg, size); + if (wr < 0) + return log_error_errno(errno, "Failed to store dmesg to %s: %m", ofd_path); + if (wr != (ssize_t)size) + return log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to store dmesg to %s. %zu bytes are lost.", ofd_path, size - wr); + r = link_tmpfile(ofd, tmp_path, ofd_path); + if (r < 0) + return log_error_errno(r, "Failed to write temporary file %s: %m", ofd_path); + ofd_path = mfree(ofd_path); + + return 0; +} + +static void process_dmesg_files(PStoreList *list) { + /* Move files, reconstruct dmesg.txt */ + PStoreEntry *pe; + _cleanup_free_ char *dmesg = NULL; + size_t dmesg_size = 0; + _cleanup_free_ char *dmesg_id = NULL; + + /* Handle each dmesg file: files processed in reverse + * order so as to properly reconstruct original dmesg */ + for (size_t n = list->n_entries; n > 0; n--) { + bool move_file_and_continue = false; + _cleanup_free_ char *pe_id = NULL; + char *p; + size_t plen; + + pe = &list->entries[n-1]; + + if (pe->handled) + continue; + if (!startswith(pe->dirent.d_name, "dmesg-")) + continue; + + if (endswith(pe->dirent.d_name, ".enc.z")) /* indicates a problem */ + move_file_and_continue = true; + p = strrchr(pe->dirent.d_name, '-'); + if (!p) + move_file_and_continue = true; + + if (move_file_and_continue) { + /* A dmesg file on which we do NO additional processing */ + (void) move_file(pe, NULL); + continue; + } + + /* See if this file is one of a related group of files + * in order to reconstruct dmesg */ + + /* When dmesg is written into pstore, it is done so in + * small chunks, whatever the exchange buffer size is + * with the underlying pstore backend (ie. EFI may be + * ~2KiB), which means an example pstore with approximately + * 64KB of storage may have up to roughly 32 dmesg files + * that could be related, depending upon the size of the + * original dmesg. + * + * Here we look at the dmesg filename and try to discern + * if files are part of a related group, meaning the same + * original dmesg. + * + * The two known pstore backends are EFI and ERST. These + * backends store data in the Common Platform Error + * Record, CPER, format. The dmesg- filename contains the + * CPER record id, a 64bit number (in decimal notation). + * In Linux, the record id is encoded with two digits for + * the dmesg part (chunk) number and 3 digits for the + * count number. So allowing an additional digit to + * compensate for advancing time, this code ignores the + * last six digits of the filename in determining the + * record id. + * + * For the EFI backend, the record id encodes an id in the + * upper 32 bits, and a timestamp in the lower 32-bits. + * So ignoring the least significant 6 digits has proven + * to generally identify related dmesg entries. */ +#define PSTORE_FILENAME_IGNORE 6 + + /* determine common portion of record id */ + ++p; /* move beyond dmesg- */ + plen = strlen(p); + if (plen > PSTORE_FILENAME_IGNORE) { + pe_id = memdup_suffix0(p, plen - PSTORE_FILENAME_IGNORE); + if (!pe_id) { + log_oom(); + return; + } + } else + pe_id = mfree(pe_id); + + /* Now move file from pstore to archive storage */ + move_file(pe, pe_id); + + /* If the current record id is NOT the same as the + * previous record id, then start a new dmesg.txt file */ + if (!pe_id || !dmesg_id || !streq(pe_id, dmesg_id)) { + /* Encountered a new dmesg group, close out old one, open new one */ + if (dmesg) { + (void) write_dmesg(dmesg, dmesg_size, dmesg_id); + dmesg = mfree(dmesg); + dmesg_size = 0; + } + + /* now point dmesg_id to storage of pe_id */ + free_and_replace(dmesg_id, pe_id); + } + + /* Reconstruction of dmesg is done as a useful courtesy, do not log errors */ + dmesg = realloc(dmesg, dmesg_size + strlen(pe->dirent.d_name) + strlen(":\n") + pe->content_size + 1); + if (dmesg) { + dmesg_size += sprintf(&dmesg[dmesg_size], "%s:\n", pe->dirent.d_name); + if (pe->content) { + memcpy(&dmesg[dmesg_size], pe->content, pe->content_size); + dmesg_size += pe->content_size; + } + } + + pe_id = mfree(pe_id); + } + if (dmesg) + (void) write_dmesg(dmesg, dmesg_size, dmesg_id); +} + +static int list_files(PStoreList *list, const char *sourcepath) { + _cleanup_(closedirp) DIR *dirp = NULL; + struct dirent *de; + int r = 0; + + dirp = opendir(sourcepath); + if (!dirp) + return log_error_errno(errno, "Failed to opendir %s: %m", sourcepath); + + FOREACH_DIRENT(de, dirp, return log_error_errno(errno, "Failed to iterate through %s: %m", sourcepath)) { + _cleanup_free_ char *ifd_path = NULL; + + ifd_path = path_join(sourcepath, de->d_name); + if (!ifd_path) + return log_oom(); + + _cleanup_free_ char *buf = NULL; + size_t buf_size; + + /* Now read contents of pstore file */ + r = read_full_file(ifd_path, &buf, &buf_size); + if (r < 0) { + log_warning_errno(r, "Failed to read file %s: %m", ifd_path); + continue; + } + + if (!GREEDY_REALLOC(list->entries, list->n_entries_allocated, list->n_entries + 1)) + return log_oom(); + + list->entries[list->n_entries++] = (PStoreEntry) { + .dirent = *de, + .content = TAKE_PTR(buf), + .content_size = buf_size, + .is_binary = true, + .handled = false, + }; + } + + return r; +} + +static int run(int argc, char *argv[]) { + _cleanup_(pstore_entries_reset) PStoreList list = {}; + int r; + + log_open(); + + /* Ignore all parse errors */ + (void) parse_config(); + + log_debug("Selected storage '%s'.", pstore_storage_to_string(arg_storage)); + log_debug("Selected Unlink '%d'.", arg_unlink); + + if (arg_storage == PSTORE_STORAGE_NONE) + /* Do nothing, intentionally, leaving pstore untouched */ + return 0; + + /* Obtain list of files in pstore */ + r = list_files(&list, arg_sourcedir); + if (r < 0) + return r; + + /* Handle each pstore file */ + /* Sort files lexigraphically ascending, generally needed by all */ + qsort_safe(list.entries, list.n_entries, sizeof(PStoreEntry), compare_pstore_entries); + + /* Process known file types */ + process_dmesg_files(&list); + + /* Move left over files out of pstore */ + for (size_t n = 0; n < list.n_entries; n++) + move_file(&list.entries[n], NULL); + + return 0; +} + +DEFINE_MAIN_FUNCTION(run); diff --git a/src/pstore/pstore.conf b/src/pstore/pstore.conf new file mode 100644 index 0000000000..93a8b6707c --- /dev/null +++ b/src/pstore/pstore.conf @@ -0,0 +1,16 @@ +# This file is part of systemd. +# +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# Entries in this file show the compile time defaults. +# You can change settings by editing this file. +# Defaults can be restored by simply deleting this file. +# +# See pstore.conf(5) for details. + +[PStore] +#Storage=external +#Unlink=yes diff --git a/units/meson.build b/units/meson.build index 52f8b7712f..10137691cf 100644 --- a/units/meson.build +++ b/units/meson.build @@ -135,6 +135,7 @@ in_units = [ ['systemd-bless-boot.service', 'ENABLE_EFI HAVE_BLKID'], ['systemd-boot-check-no-failures.service', ''], ['systemd-coredump@.service', 'ENABLE_COREDUMP'], + ['systemd-pstore.service', 'ENABLE_PSTORE'], ['systemd-firstboot.service', 'ENABLE_FIRSTBOOT', 'sysinit.target.wants/'], ['systemd-fsck-root.service', ''], diff --git a/units/systemd-pstore.service.in b/units/systemd-pstore.service.in new file mode 100644 index 0000000000..fec2b1aebf --- /dev/null +++ b/units/systemd-pstore.service.in @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: LGPL-2.1+ +# +# This file is part of systemd. +# +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=Platform Persistent Storage Archival +Documentation=man:systemd-pstore(8) +DefaultDependencies=no +Wants=systemd-remount-fs.service +After=systemd-remount-fs.service + +[Service] +Type=oneshot +ExecStart=@rootlibexecdir@/systemd-pstore +RemainAfterExit=yes +StateDirectory=systemd/pstore + +[Install] +WantedBy=systemd-remount-fs.service