Systemd/src/coredump/coredump.c

1428 lines
50 KiB
C
Raw Normal View History

/* SPDX-License-Identifier: LGPL-2.1+ */
/***
This file is part of systemd.
Copyright 2012 Lennart Poettering
systemd is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
systemd is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/
#include <errno.h>
#include <stdio.h>
#include <stdio_ext.h>
#include <sys/prctl.h>
#include <sys/xattr.h>
#include <unistd.h>
#if HAVE_ELFUTILS
#include <dwarf.h>
#include <elfutils/libdwfl.h>
#endif
#include "sd-daemon.h"
#include "sd-journal.h"
#include "sd-login.h"
#include "sd-messages.h"
#include "acl-util.h"
#include "alloc-util.h"
#include "capability-util.h"
#include "cgroup-util.h"
#include "compress.h"
#include "conf-parser.h"
#include "copy.h"
#include "coredump-vacuum.h"
#include "dirent-util.h"
#include "escape.h"
#include "fd-util.h"
#include "fileio.h"
#include "fs-util.h"
#include "io-util.h"
#include "journal-importer.h"
#include "log.h"
#include "macro.h"
#include "missing.h"
#include "mkdir.h"
#include "parse-util.h"
2015-04-10 19:10:00 +02:00
#include "process-util.h"
#include "signal-util.h"
#include "socket-util.h"
#include "special.h"
#include "stacktrace.h"
#include "string-table.h"
#include "string-util.h"
#include "strv.h"
#include "user-util.h"
#include "util.h"
/* The maximum size up to which we process coredumps */
#define PROCESS_SIZE_MAX ((uint64_t) (2LLU*1024LLU*1024LLU*1024LLU))
/* The maximum size up to which we leave the coredump around on disk */
#define EXTERNAL_SIZE_MAX PROCESS_SIZE_MAX
/* The maximum size up to which we store the coredump in the journal */
#define JOURNAL_SIZE_MAX ((size_t) (767LU*1024LU*1024LU))
/* Make sure to not make this larger than the maximum journal entry
* size. See DATA_SIZE_MAX in journald-native.c. */
assert_cc(JOURNAL_SIZE_MAX <= DATA_SIZE_MAX);
enum {
/* We use this as array indexes for a couple of special fields we use for
* naming coredump files, and attaching xattrs, and for indexing argv[].
* Our pattern for man:systectl(1) kernel.core_pattern is such that the
* kernel passes fields until CONTEXT_RLIMIT as arguments in argv[]. After
* that it gets complicated: the kernel passes "comm" as one or more fields
* starting at index CONTEXT_COMM (in other words, full "comm" is under index
* CONTEXT_COMM when it does not contain spaces, which is the common
* case). This mapping is not reversible, so we prefer to retrieve "comm"
* from /proc. We only fall back to argv[CONTEXT_COMM...] when that fails.
*
* In the internal context[] array, fields before CONTEXT_COMM are the
* strings from argv[], so they should not be freed. The strings at indices
* CONTEXT_COMM and higher are allocated by us and should be freed at the
* end.
*/
CONTEXT_PID,
CONTEXT_UID,
CONTEXT_GID,
CONTEXT_SIGNAL,
CONTEXT_TIMESTAMP,
CONTEXT_RLIMIT,
CONTEXT_HOSTNAME,
CONTEXT_COMM,
CONTEXT_EXE,
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
CONTEXT_UNIT,
_CONTEXT_MAX
};
typedef enum CoredumpStorage {
COREDUMP_STORAGE_NONE,
COREDUMP_STORAGE_EXTERNAL,
COREDUMP_STORAGE_JOURNAL,
_COREDUMP_STORAGE_MAX,
_COREDUMP_STORAGE_INVALID = -1
} CoredumpStorage;
static const char* const coredump_storage_table[_COREDUMP_STORAGE_MAX] = {
[COREDUMP_STORAGE_NONE] = "none",
[COREDUMP_STORAGE_EXTERNAL] = "external",
[COREDUMP_STORAGE_JOURNAL] = "journal",
};
DEFINE_PRIVATE_STRING_TABLE_LOOKUP(coredump_storage, CoredumpStorage);
static DEFINE_CONFIG_PARSE_ENUM(config_parse_coredump_storage, coredump_storage, CoredumpStorage, "Failed to parse storage setting");
static CoredumpStorage arg_storage = COREDUMP_STORAGE_EXTERNAL;
static bool arg_compress = true;
static uint64_t arg_process_size_max = PROCESS_SIZE_MAX;
static uint64_t arg_external_size_max = EXTERNAL_SIZE_MAX;
static uint64_t arg_journal_size_max = JOURNAL_SIZE_MAX;
static uint64_t arg_keep_free = (uint64_t) -1;
static uint64_t arg_max_use = (uint64_t) -1;
static int parse_config(void) {
static const ConfigTableItem items[] = {
{ "Coredump", "Storage", config_parse_coredump_storage, 0, &arg_storage },
{ "Coredump", "Compress", config_parse_bool, 0, &arg_compress },
{ "Coredump", "ProcessSizeMax", config_parse_iec_uint64, 0, &arg_process_size_max },
{ "Coredump", "ExternalSizeMax", config_parse_iec_uint64, 0, &arg_external_size_max },
{ "Coredump", "JournalSizeMax", config_parse_iec_size, 0, &arg_journal_size_max },
{ "Coredump", "KeepFree", config_parse_iec_uint64, 0, &arg_keep_free },
{ "Coredump", "MaxUse", config_parse_iec_uint64, 0, &arg_max_use },
{}
};
return config_parse_many_nulstr(PKGSYSCONFDIR "/coredump.conf",
CONF_PATHS_NULSTR("systemd/coredump.conf.d"),
"Coredump\0",
config_item_table_lookup, items,
CONFIG_PARSE_WARN, NULL);
}
static inline uint64_t storage_size_max(void) {
return arg_storage == COREDUMP_STORAGE_EXTERNAL ? arg_external_size_max : arg_journal_size_max;
}
static int fix_acl(int fd, uid_t uid) {
#if HAVE_ACL
_cleanup_(acl_freep) acl_t acl = NULL;
acl_entry_t entry;
acl_permset_t permset;
int r;
assert(fd >= 0);
if (uid_is_system(uid) || uid_is_dynamic(uid) || uid == UID_NOBODY)
return 0;
/* Make sure normal users can read (but not write or delete)
* their own coredumps */
acl = acl_get_fd(fd);
if (!acl)
return log_error_errno(errno, "Failed to get ACL: %m");
if (acl_create_entry(&acl, &entry) < 0 ||
acl_set_tag_type(entry, ACL_USER) < 0 ||
acl_set_qualifier(entry, &uid) < 0)
return log_error_errno(errno, "Failed to patch ACL: %m");
if (acl_get_permset(entry, &permset) < 0 ||
acl_add_perm(permset, ACL_READ) < 0)
return log_warning_errno(errno, "Failed to patch ACL: %m");
r = calc_acl_mask_if_needed(&acl);
if (r < 0)
return log_warning_errno(r, "Failed to patch ACL: %m");
if (acl_set_fd(fd, acl) < 0)
return log_error_errno(errno, "Failed to apply ACL: %m");
#endif
return 0;
}
static int fix_xattr(int fd, const char *context[_CONTEXT_MAX]) {
static const char * const xattrs[_CONTEXT_MAX] = {
[CONTEXT_PID] = "user.coredump.pid",
[CONTEXT_UID] = "user.coredump.uid",
[CONTEXT_GID] = "user.coredump.gid",
[CONTEXT_SIGNAL] = "user.coredump.signal",
[CONTEXT_TIMESTAMP] = "user.coredump.timestamp",
[CONTEXT_RLIMIT] = "user.coredump.rlimit",
[CONTEXT_HOSTNAME] = "user.coredump.hostname",
[CONTEXT_COMM] = "user.coredump.comm",
[CONTEXT_EXE] = "user.coredump.exe",
};
int r = 0;
unsigned i;
assert(fd >= 0);
/* Attach some metadata to coredumps via extended
* attributes. Just because we can. */
for (i = 0; i < _CONTEXT_MAX; i++) {
int k;
if (isempty(context[i]) || !xattrs[i])
continue;
k = fsetxattr(fd, xattrs[i], context[i], strlen(context[i]), XATTR_CREATE);
if (k < 0 && r == 0)
r = -errno;
}
return r;
}
#define filename_escape(s) xescape((s), "./ ")
static inline const char *coredump_tmpfile_name(const char *s) {
return s ? s : "(unnamed temporary file)";
}
static int fix_permissions(
int fd,
const char *filename,
const char *target,
const char *context[_CONTEXT_MAX],
uid_t uid) {
int r;
assert(fd >= 0);
assert(target);
assert(context);
/* Ignore errors on these */
(void) fchmod(fd, 0640);
(void) fix_acl(fd, uid);
(void) fix_xattr(fd, context);
if (fsync(fd) < 0)
return log_error_errno(errno, "Failed to sync coredump %s: %m", coredump_tmpfile_name(filename));
(void) fsync_directory_of_file(fd);
r = link_tmpfile(fd, filename, target);
if (r < 0)
return log_error_errno(r, "Failed to move coredump %s into place: %m", target);
return 0;
}
static int maybe_remove_external_coredump(const char *filename, uint64_t size) {
/* Returns 1 if might remove, 0 if will not remove, < 0 on error. */
if (arg_storage == COREDUMP_STORAGE_EXTERNAL &&
size <= arg_external_size_max)
return 0;
if (!filename)
return 1;
if (unlink(filename) < 0 && errno != ENOENT)
return log_error_errno(errno, "Failed to unlink %s: %m", filename);
return 1;
}
static int make_filename(const char *context[_CONTEXT_MAX], char **ret) {
_cleanup_free_ char *c = NULL, *u = NULL, *p = NULL, *t = NULL;
sd_id128_t boot = {};
int r;
assert(context);
c = filename_escape(context[CONTEXT_COMM]);
if (!c)
return -ENOMEM;
u = filename_escape(context[CONTEXT_UID]);
if (!u)
return -ENOMEM;
r = sd_id128_get_boot(&boot);
if (r < 0)
return r;
p = filename_escape(context[CONTEXT_PID]);
if (!p)
return -ENOMEM;
t = filename_escape(context[CONTEXT_TIMESTAMP]);
if (!t)
return -ENOMEM;
if (asprintf(ret,
"/var/lib/systemd/coredump/core.%s.%s." SD_ID128_FORMAT_STR ".%s.%s000000",
c,
u,
SD_ID128_FORMAT_VAL(boot),
p,
t) < 0)
return -ENOMEM;
return 0;
}
static int save_external_coredump(
const char *context[_CONTEXT_MAX],
int input_fd,
char **ret_filename,
coredump: fix bug that loses core dump files when core dumps are compressed and disk space is low. Previously the save_external_coredump function returned a file descriptor corresponding to the dumped file. This descriptor was used for two different purposes by calling code: a) access to the raw core dump data; b) testing candidate files (via inode comparisons) while vacuuming to protect the current core dump from vacuuming. The descriptor returned always corresponded to a file containing the raw core dump data. However if compresson was used and the core dump was compressed then the descriptor returned did not correspond to the file that would eventually be left on disk (ie the compressed file). Thus the file was never protected by vacuuming. When disk space was low all core dumps including the current one would be vacuumed and the corresponding log message referred to a file that no longer existed. This resulted in the following error message from coredumpctl if the missing core dump was requested: Cannot retrieve coredump from journal nor disk. Failed to retrieve core: No such file or directory save_external_coredump now returns two descriptors, one to be used for inode comparisons to prevent overzealous vacuuming and one to be used for raw data access. When compression is not used the returned inode comparison descriptor will be invalid, indicating that the raw data access descriptor should be used for inode comparisons as well. Corresponding use of save_external_coredump and the returned descriptors also updated.
2015-12-23 19:59:31 +01:00
int *ret_node_fd,
int *ret_data_fd,
uint64_t *ret_size,
bool *ret_truncated) {
_cleanup_free_ char *fn = NULL, *tmp = NULL;
_cleanup_close_ int fd = -1;
uint64_t rlimit, max_size;
struct stat st;
uid_t uid;
int r;
assert(context);
assert(ret_filename);
coredump: fix bug that loses core dump files when core dumps are compressed and disk space is low. Previously the save_external_coredump function returned a file descriptor corresponding to the dumped file. This descriptor was used for two different purposes by calling code: a) access to the raw core dump data; b) testing candidate files (via inode comparisons) while vacuuming to protect the current core dump from vacuuming. The descriptor returned always corresponded to a file containing the raw core dump data. However if compresson was used and the core dump was compressed then the descriptor returned did not correspond to the file that would eventually be left on disk (ie the compressed file). Thus the file was never protected by vacuuming. When disk space was low all core dumps including the current one would be vacuumed and the corresponding log message referred to a file that no longer existed. This resulted in the following error message from coredumpctl if the missing core dump was requested: Cannot retrieve coredump from journal nor disk. Failed to retrieve core: No such file or directory save_external_coredump now returns two descriptors, one to be used for inode comparisons to prevent overzealous vacuuming and one to be used for raw data access. When compression is not used the returned inode comparison descriptor will be invalid, indicating that the raw data access descriptor should be used for inode comparisons as well. Corresponding use of save_external_coredump and the returned descriptors also updated.
2015-12-23 19:59:31 +01:00
assert(ret_node_fd);
assert(ret_data_fd);
assert(ret_size);
r = parse_uid(context[CONTEXT_UID], &uid);
if (r < 0)
return log_error_errno(r, "Failed to parse UID: %m");
r = safe_atou64(context[CONTEXT_RLIMIT], &rlimit);
if (r < 0)
return log_error_errno(r, "Failed to parse resource limit: %s", context[CONTEXT_RLIMIT]);
if (rlimit < page_size()) {
/* Is coredumping disabled? Then don't bother saving/processing the coredump.
* Anything below PAGE_SIZE cannot give a readable coredump (the kernel uses
* ELF_EXEC_PAGESIZE which is not easily accessible, but is usually the same as PAGE_SIZE. */
log_info("Resource limits disable core dumping for process %s (%s).",
context[CONTEXT_PID], context[CONTEXT_COMM]);
return -EBADSLT;
}
/* Never store more than the process configured, or than we actually shall keep or process */
max_size = MIN(rlimit, MAX(arg_process_size_max, storage_size_max()));
r = make_filename(context, &fn);
if (r < 0)
return log_error_errno(r, "Failed to determine coredump file name: %m");
mkdir_p_label("/var/lib/systemd/coredump", 0755);
fd = open_tmpfile_linkable(fn, O_RDWR|O_CLOEXEC, &tmp);
if (fd < 0)
return log_error_errno(fd, "Failed to create temporary file for coredump %s: %m", fn);
r = copy_bytes(input_fd, fd, max_size, 0);
if (r < 0) {
log_error_errno(r, "Cannot store coredump of %s (%s): %m", context[CONTEXT_PID], context[CONTEXT_COMM]);
goto fail;
}
*ret_truncated = r == 1;
if (*ret_truncated)
log_struct(LOG_INFO,
LOG_MESSAGE("Core file was truncated to %zu bytes.", max_size),
"SIZE_LIMIT=%zu", max_size,
tree-wide: add SD_ID128_MAKE_STR, remove LOG_MESSAGE_ID Embedding sd_id128_t's in constant strings was rather cumbersome. We had SD_ID128_CONST_STR which returned a const char[], but it had two problems: - it wasn't possible to statically concatanate this array with a normal string - gcc wasn't really able to optimize this, and generated code to perform the "conversion" at runtime. Because of this, even our own code in coredumpctl wasn't using SD_ID128_CONST_STR. Add a new macro to generate a constant string: SD_ID128_MAKE_STR. It is not as elegant as SD_ID128_CONST_STR, because it requires a repetition of the numbers, but in practice it is more convenient to use, and allows gcc to generate smarter code: $ size .libs/systemd{,-logind,-journald}{.old,} text data bss dec hex filename 1265204 149564 4808 1419576 15a938 .libs/systemd.old 1260268 149564 4808 1414640 1595f0 .libs/systemd 246805 13852 209 260866 3fb02 .libs/systemd-logind.old 240973 13852 209 255034 3e43a .libs/systemd-logind 146839 4984 34 151857 25131 .libs/systemd-journald.old 146391 4984 34 151409 24f71 .libs/systemd-journald It is also much easier to check if a certain binary uses a certain MESSAGE_ID: $ strings .libs/systemd.old|grep MESSAGE_ID MESSAGE_ID=%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x MESSAGE_ID=%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x MESSAGE_ID=%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x MESSAGE_ID=%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x $ strings .libs/systemd|grep MESSAGE_ID MESSAGE_ID=c7a787079b354eaaa9e77b371893cd27 MESSAGE_ID=b07a249cd024414a82dd00cd181378ff MESSAGE_ID=641257651c1b4ec9a8624d7a40a9e1e7 MESSAGE_ID=de5b426a63be47a7b6ac3eaac82e2f6f MESSAGE_ID=d34d037fff1847e6ae669a370e694725 MESSAGE_ID=7d4958e842da4a758f6c1cdc7b36dcc5 MESSAGE_ID=1dee0369c7fc4736b7099b38ecb46ee7 MESSAGE_ID=39f53479d3a045ac8e11786248231fbf MESSAGE_ID=be02cf6855d2428ba40df7e9d022f03d MESSAGE_ID=7b05ebc668384222baa8881179cfda54 MESSAGE_ID=9d1aaa27d60140bd96365438aad20286
2016-11-06 18:48:23 +01:00
"MESSAGE_ID=" SD_MESSAGE_TRUNCATED_CORE_STR,
NULL);
if (fstat(fd, &st) < 0) {
log_error_errno(errno, "Failed to fstat core file %s: %m", coredump_tmpfile_name(tmp));
goto fail;
}
if (lseek(fd, 0, SEEK_SET) == (off_t) -1) {
log_error_errno(errno, "Failed to seek on %s: %m", coredump_tmpfile_name(tmp));
goto fail;
}
#if HAVE_XZ || HAVE_LZ4
/* If we will remove the coredump anyway, do not compress. */
if (arg_compress && !maybe_remove_external_coredump(NULL, st.st_size)) {
_cleanup_free_ char *fn_compressed = NULL, *tmp_compressed = NULL;
_cleanup_close_ int fd_compressed = -1;
fn_compressed = strappend(fn, COMPRESSED_EXT);
if (!fn_compressed) {
log_oom();
goto uncompressed;
}
fd_compressed = open_tmpfile_linkable(fn_compressed, O_RDWR|O_CLOEXEC, &tmp_compressed);
if (fd_compressed < 0) {
log_error_errno(fd_compressed, "Failed to create temporary file for coredump %s: %m", fn_compressed);
goto uncompressed;
}
r = compress_stream(fd, fd_compressed, -1);
if (r < 0) {
log_error_errno(r, "Failed to compress %s: %m", coredump_tmpfile_name(tmp_compressed));
goto fail_compressed;
}
r = fix_permissions(fd_compressed, tmp_compressed, fn_compressed, context, uid);
if (r < 0)
goto fail_compressed;
/* OK, this worked, we can get rid of the uncompressed version now */
if (tmp)
unlink_noerrno(tmp);
*ret_filename = fn_compressed; /* compressed */
coredump: fix bug that loses core dump files when core dumps are compressed and disk space is low. Previously the save_external_coredump function returned a file descriptor corresponding to the dumped file. This descriptor was used for two different purposes by calling code: a) access to the raw core dump data; b) testing candidate files (via inode comparisons) while vacuuming to protect the current core dump from vacuuming. The descriptor returned always corresponded to a file containing the raw core dump data. However if compresson was used and the core dump was compressed then the descriptor returned did not correspond to the file that would eventually be left on disk (ie the compressed file). Thus the file was never protected by vacuuming. When disk space was low all core dumps including the current one would be vacuumed and the corresponding log message referred to a file that no longer existed. This resulted in the following error message from coredumpctl if the missing core dump was requested: Cannot retrieve coredump from journal nor disk. Failed to retrieve core: No such file or directory save_external_coredump now returns two descriptors, one to be used for inode comparisons to prevent overzealous vacuuming and one to be used for raw data access. When compression is not used the returned inode comparison descriptor will be invalid, indicating that the raw data access descriptor should be used for inode comparisons as well. Corresponding use of save_external_coredump and the returned descriptors also updated.
2015-12-23 19:59:31 +01:00
*ret_node_fd = fd_compressed; /* compressed */
*ret_data_fd = fd; /* uncompressed */
*ret_size = (uint64_t) st.st_size; /* uncompressed */
fn_compressed = NULL;
coredump: fix bug that loses core dump files when core dumps are compressed and disk space is low. Previously the save_external_coredump function returned a file descriptor corresponding to the dumped file. This descriptor was used for two different purposes by calling code: a) access to the raw core dump data; b) testing candidate files (via inode comparisons) while vacuuming to protect the current core dump from vacuuming. The descriptor returned always corresponded to a file containing the raw core dump data. However if compresson was used and the core dump was compressed then the descriptor returned did not correspond to the file that would eventually be left on disk (ie the compressed file). Thus the file was never protected by vacuuming. When disk space was low all core dumps including the current one would be vacuumed and the corresponding log message referred to a file that no longer existed. This resulted in the following error message from coredumpctl if the missing core dump was requested: Cannot retrieve coredump from journal nor disk. Failed to retrieve core: No such file or directory save_external_coredump now returns two descriptors, one to be used for inode comparisons to prevent overzealous vacuuming and one to be used for raw data access. When compression is not used the returned inode comparison descriptor will be invalid, indicating that the raw data access descriptor should be used for inode comparisons as well. Corresponding use of save_external_coredump and the returned descriptors also updated.
2015-12-23 19:59:31 +01:00
fd = fd_compressed = -1;
return 0;
fail_compressed:
if (tmp_compressed)
(void) unlink(tmp_compressed);
}
uncompressed:
#endif
coredump: fix bug that loses core dump files when core dumps are compressed and disk space is low. Previously the save_external_coredump function returned a file descriptor corresponding to the dumped file. This descriptor was used for two different purposes by calling code: a) access to the raw core dump data; b) testing candidate files (via inode comparisons) while vacuuming to protect the current core dump from vacuuming. The descriptor returned always corresponded to a file containing the raw core dump data. However if compresson was used and the core dump was compressed then the descriptor returned did not correspond to the file that would eventually be left on disk (ie the compressed file). Thus the file was never protected by vacuuming. When disk space was low all core dumps including the current one would be vacuumed and the corresponding log message referred to a file that no longer existed. This resulted in the following error message from coredumpctl if the missing core dump was requested: Cannot retrieve coredump from journal nor disk. Failed to retrieve core: No such file or directory save_external_coredump now returns two descriptors, one to be used for inode comparisons to prevent overzealous vacuuming and one to be used for raw data access. When compression is not used the returned inode comparison descriptor will be invalid, indicating that the raw data access descriptor should be used for inode comparisons as well. Corresponding use of save_external_coredump and the returned descriptors also updated.
2015-12-23 19:59:31 +01:00
r = fix_permissions(fd, tmp, fn, context, uid);
if (r < 0)
goto fail;
*ret_filename = fn;
coredump: fix bug that loses core dump files when core dumps are compressed and disk space is low. Previously the save_external_coredump function returned a file descriptor corresponding to the dumped file. This descriptor was used for two different purposes by calling code: a) access to the raw core dump data; b) testing candidate files (via inode comparisons) while vacuuming to protect the current core dump from vacuuming. The descriptor returned always corresponded to a file containing the raw core dump data. However if compresson was used and the core dump was compressed then the descriptor returned did not correspond to the file that would eventually be left on disk (ie the compressed file). Thus the file was never protected by vacuuming. When disk space was low all core dumps including the current one would be vacuumed and the corresponding log message referred to a file that no longer existed. This resulted in the following error message from coredumpctl if the missing core dump was requested: Cannot retrieve coredump from journal nor disk. Failed to retrieve core: No such file or directory save_external_coredump now returns two descriptors, one to be used for inode comparisons to prevent overzealous vacuuming and one to be used for raw data access. When compression is not used the returned inode comparison descriptor will be invalid, indicating that the raw data access descriptor should be used for inode comparisons as well. Corresponding use of save_external_coredump and the returned descriptors also updated.
2015-12-23 19:59:31 +01:00
*ret_data_fd = fd;
*ret_node_fd = -1;
*ret_size = (uint64_t) st.st_size;
fn = NULL;
fd = -1;
return 0;
fail:
if (tmp)
(void) unlink(tmp);
return r;
}
static int allocate_journal_field(int fd, size_t size, char **ret, size_t *ret_size) {
_cleanup_free_ char *field = NULL;
ssize_t n;
assert(fd >= 0);
assert(ret);
assert(ret_size);
if (lseek(fd, 0, SEEK_SET) == (off_t) -1)
return log_warning_errno(errno, "Failed to seek: %m");
field = malloc(9 + size);
if (!field) {
log_warning("Failed to allocate memory for coredump, coredump will not be stored.");
return -ENOMEM;
}
memcpy(field, "COREDUMP=", 9);
n = read(fd, field + 9, size);
if (n < 0)
return log_error_errno((int) n, "Failed to read core data: %m");
if ((size_t) n < size) {
log_error("Core data too short.");
return -EIO;
}
*ret = field;
*ret_size = size + 9;
field = NULL;
return 0;
}
/* Joins /proc/[pid]/fd/ and /proc/[pid]/fdinfo/ into the following lines:
* 0:/dev/pts/23
* pos: 0
* flags: 0100002
*
* 1:/dev/pts/23
* pos: 0
* flags: 0100002
*
* 2:/dev/pts/23
* pos: 0
* flags: 0100002
* EOF
*/
static int compose_open_fds(pid_t pid, char **open_fds) {
_cleanup_closedir_ DIR *proc_fd_dir = NULL;
_cleanup_close_ int proc_fdinfo_fd = -1;
_cleanup_free_ char *buffer = NULL;
_cleanup_fclose_ FILE *stream = NULL;
2014-11-27 05:31:35 +01:00
const char *fddelim = "", *path;
struct dirent *dent = NULL;
size_t size = 0;
int r = 0;
assert(pid >= 0);
assert(open_fds != NULL);
2014-11-27 05:31:35 +01:00
path = procfs_file_alloca(pid, "fd");
proc_fd_dir = opendir(path);
2014-11-27 05:31:35 +01:00
if (!proc_fd_dir)
return -errno;
proc_fdinfo_fd = openat(dirfd(proc_fd_dir), "../fdinfo", O_DIRECTORY|O_NOFOLLOW|O_CLOEXEC|O_PATH);
2014-11-27 05:31:35 +01:00
if (proc_fdinfo_fd < 0)
return -errno;
stream = open_memstream(&buffer, &size);
if (!stream)
return -ENOMEM;
(void) __fsetlocking(stream, FSETLOCKING_BYCALLER);
FOREACH_DIRENT(dent, proc_fd_dir, return -errno) {
_cleanup_fclose_ FILE *fdinfo = NULL;
_cleanup_free_ char *fdname = NULL;
2014-11-27 05:31:35 +01:00
char line[LINE_MAX];
int fd;
2014-11-27 05:31:35 +01:00
r = readlinkat_malloc(dirfd(proc_fd_dir), dent->d_name, &fdname);
if (r < 0)
return r;
fprintf(stream, "%s%s:%s\n", fddelim, dent->d_name, fdname);
fddelim = "\n";
/* Use the directory entry from /proc/[pid]/fd with /proc/[pid]/fdinfo */
2014-11-27 05:31:35 +01:00
fd = openat(proc_fdinfo_fd, dent->d_name, O_NOFOLLOW|O_CLOEXEC|O_RDONLY);
if (fd < 0)
continue;
2014-11-27 05:31:35 +01:00
fdinfo = fdopen(fd, "re");
if (!fdinfo) {
safe_close(fd);
continue;
2014-11-27 05:31:35 +01:00
}
FOREACH_LINE(line, fdinfo, break) {
fputs(line, stream);
if (!endswith(line, "\n"))
fputc('\n', stream);
}
}
errno = 0;
stream = safe_fclose(stream);
if (errno > 0)
return -errno;
*open_fds = TAKE_PTR(buffer);
return 0;
}
static int get_process_ns(pid_t pid, const char *namespace, ino_t *ns) {
const char *p;
struct stat stbuf;
_cleanup_close_ int proc_ns_dir_fd;
p = procfs_file_alloca(pid, "ns");
proc_ns_dir_fd = open(p, O_DIRECTORY | O_CLOEXEC | O_RDONLY);
if (proc_ns_dir_fd < 0)
return -errno;
if (fstatat(proc_ns_dir_fd, namespace, &stbuf, /* flags */0) < 0)
return -errno;
*ns = stbuf.st_ino;
return 0;
}
static int get_mount_namespace_leader(pid_t pid, pid_t *container_pid) {
pid_t cpid = pid, ppid = 0;
ino_t proc_mntns;
int r = 0;
r = get_process_ns(pid, "mnt", &proc_mntns);
if (r < 0)
return r;
for (;;) {
ino_t parent_mntns;
r = get_process_ppid(cpid, &ppid);
if (r < 0)
return r;
r = get_process_ns(ppid, "mnt", &parent_mntns);
if (r < 0)
return r;
if (proc_mntns != parent_mntns)
break;
if (ppid == 1)
return -ENOENT;
cpid = ppid;
}
*container_pid = ppid;
return 0;
}
/* Returns 1 if the parent was found.
* Returns 0 if there is not a process we can call the pid's
* container parent (the pid's process isn't 'containerized').
* Returns a negative number on errors.
*/
static int get_process_container_parent_cmdline(pid_t pid, char** cmdline) {
int r = 0;
pid_t container_pid;
const char *proc_root_path;
struct stat root_stat, proc_root_stat;
/* To compare inodes of / and /proc/[pid]/root */
if (stat("/", &root_stat) < 0)
return -errno;
proc_root_path = procfs_file_alloca(pid, "root");
if (stat(proc_root_path, &proc_root_stat) < 0)
return -errno;
/* The process uses system root. */
if (proc_root_stat.st_ino == root_stat.st_ino) {
*cmdline = NULL;
return 0;
}
r = get_mount_namespace_leader(pid, &container_pid);
if (r < 0)
return r;
r = get_process_cmdline(container_pid, 0, false, cmdline);
if (r < 0)
return r;
return 1;
}
static int change_uid_gid(const char *context[]) {
uid_t uid;
gid_t gid;
int r;
r = parse_uid(context[CONTEXT_UID], &uid);
if (r < 0)
return r;
if (uid <= SYSTEM_UID_MAX) {
const char *user = "systemd-coredump";
r = get_user_creds(&user, &uid, &gid, NULL, NULL);
if (r < 0) {
log_warning_errno(r, "Cannot resolve %s user. Proceeding to dump core as root: %m", user);
uid = gid = 0;
}
} else {
r = parse_gid(context[CONTEXT_GID], &gid);
if (r < 0)
return r;
}
return drop_privileges(uid, gid, 0);
}
static bool is_journald_crash(const char *context[_CONTEXT_MAX]) {
assert(context);
return streq_ptr(context[CONTEXT_UNIT], SPECIAL_JOURNALD_SERVICE);
}
static bool is_pid1_crash(const char *context[_CONTEXT_MAX]) {
assert(context);
return streq_ptr(context[CONTEXT_UNIT], SPECIAL_INIT_SCOPE) ||
streq_ptr(context[CONTEXT_PID], "1");
}
#define SUBMIT_COREDUMP_FIELDS 4
static int submit_coredump(
const char *context[_CONTEXT_MAX],
struct iovec *iovec,
size_t n_iovec_allocated,
size_t n_iovec,
int input_fd) {
coredump: fix bug that loses core dump files when core dumps are compressed and disk space is low. Previously the save_external_coredump function returned a file descriptor corresponding to the dumped file. This descriptor was used for two different purposes by calling code: a) access to the raw core dump data; b) testing candidate files (via inode comparisons) while vacuuming to protect the current core dump from vacuuming. The descriptor returned always corresponded to a file containing the raw core dump data. However if compresson was used and the core dump was compressed then the descriptor returned did not correspond to the file that would eventually be left on disk (ie the compressed file). Thus the file was never protected by vacuuming. When disk space was low all core dumps including the current one would be vacuumed and the corresponding log message referred to a file that no longer existed. This resulted in the following error message from coredumpctl if the missing core dump was requested: Cannot retrieve coredump from journal nor disk. Failed to retrieve core: No such file or directory save_external_coredump now returns two descriptors, one to be used for inode comparisons to prevent overzealous vacuuming and one to be used for raw data access. When compression is not used the returned inode comparison descriptor will be invalid, indicating that the raw data access descriptor should be used for inode comparisons as well. Corresponding use of save_external_coredump and the returned descriptors also updated.
2015-12-23 19:59:31 +01:00
_cleanup_close_ int coredump_fd = -1, coredump_node_fd = -1;
_cleanup_free_ char *core_message = NULL, *filename = NULL, *coredump_data = NULL;
uint64_t coredump_size = UINT64_MAX;
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
bool truncated = false, journald_crash;
int r;
assert(context);
assert(iovec);
assert(n_iovec_allocated >= n_iovec + SUBMIT_COREDUMP_FIELDS);
assert(input_fd >= 0);
journald_crash = is_journald_crash(context);
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
/* Vacuum before we write anything again */
(void) coredump_vacuum(-1, arg_keep_free, arg_max_use);
/* Always stream the coredump to disk, if that's possible */
r = save_external_coredump(context, input_fd,
&filename, &coredump_node_fd, &coredump_fd, &coredump_size, &truncated);
if (r < 0)
/* Skip whole core dumping part */
goto log;
/* If we don't want to keep the coredump on disk, remove it now, as later on we will lack the privileges for
* it. However, we keep the fd to it, so that we can still process it and log it. */
r = maybe_remove_external_coredump(filename, coredump_size);
if (r < 0)
return r;
if (r == 0) {
const char *coredump_filename;
coredump_filename = strjoina("COREDUMP_FILENAME=", filename);
iovec[n_iovec++] = IOVEC_MAKE_STRING(coredump_filename);
} else if (arg_storage == COREDUMP_STORAGE_EXTERNAL)
coredump: fix format string on 32 bits In file included from ./src/basic/macro.h:415:0, from ./src/shared/acl-util.h:28, from src/coredump/coredump.c:36: src/coredump/coredump.c: In function ‘submit_coredump’: src/coredump/coredump.c:711:26: warning: format ‘%zu’ expects argument of type ‘size_t’, but argument 7 has type ‘uint64_t {aka long long unsigned int}’ [-Wformat=] log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^ ./src/basic/log.h:175:82: note: in definition of macro ‘log_full_errno’ ? log_internal(_level, _e, __FILE__, __LINE__, __func__, __VA_ARGS__) \ ^~~~~~~~~~~ ./src/basic/log.h:183:28: note: in expansion of macro ‘log_full’ #define log_info(...) log_full(LOG_INFO, __VA_ARGS__) ^~~~~~~~ src/coredump/coredump.c:711:17: note: in expansion of macro ‘log_info’ log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^~~~~~~~ src/coredump/coredump.c:711:26: warning: format ‘%zu’ expects argument of type ‘size_t’, but argument 8 has type ‘uint64_t {aka long long unsigned int}’ [-Wformat=] log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^ ./src/basic/log.h:175:82: note: in definition of macro ‘log_full_errno’ ? log_internal(_level, _e, __FILE__, __LINE__, __func__, __VA_ARGS__) \ ^~~~~~~~~~~ ./src/basic/log.h:183:28: note: in expansion of macro ‘log_full’ #define log_info(...) log_full(LOG_INFO, __VA_ARGS__) ^~~~~~~~ src/coredump/coredump.c:711:17: note: in expansion of macro ‘log_info’ log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^~~~~~~~ src/coredump/coredump.c:741:27: warning: format ‘%zu’ expects argument of type ‘size_t’, but argument 7 has type ‘uint64_t {aka long long unsigned int}’ [-Wformat=] log_debug("Not generating stack trace: core size %zu is greater than %zu (the configured maximum)", ^ ./src/basic/log.h:175:82: note: in definition of macro ‘log_full_errno’ ? log_internal(_level, _e, __FILE__, __LINE__, __func__, __VA_ARGS__) \ ^~~~~~~~~~~ ./src/basic/log.h:182:28: note: in expansion of macro ‘log_full’ #define log_debug(...) log_full(LOG_DEBUG, __VA_ARGS__) ^~~~~~~~ src/coredump/coredump.c:741:17: note: in expansion of macro ‘log_debug’ log_debug("Not generating stack trace: core size %zu is greater than %zu (the configured maximum)", ^~~~~~~~~ src/coredump/coredump.c:741:27: warning: format ‘%zu’ expects argument of type ‘size_t’, but argument 8 has type ‘uint64_t {aka long long unsigned int}’ [-Wformat=] log_debug("Not generating stack trace: core size %zu is greater than %zu (the configured maximum)", ^ ./src/basic/log.h:175:82: note: in definition of macro ‘log_full_errno’ ? log_internal(_level, _e, __FILE__, __LINE__, __func__, __VA_ARGS__) \ ^~~~~~~~~~~ ./src/basic/log.h:182:28: note: in expansion of macro ‘log_full’ #define log_debug(...) log_full(LOG_DEBUG, __VA_ARGS__) ^~~~~~~~ src/coredump/coredump.c:741:17: note: in expansion of macro ‘log_debug’ log_debug("Not generating stack trace: core size %zu is greater than %zu (the configured maximum)", ^~~~~~~~~ src/coredump/coredump.c:768:34: warning: format ‘%zu’ expects argument of type ‘size_t’, but argument 7 has type ‘uint64_t {aka long long unsigned int}’ [-Wformat=] log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^ ./src/basic/log.h:175:82: note: in definition of macro ‘log_full_errno’ ? log_internal(_level, _e, __FILE__, __LINE__, __func__, __VA_ARGS__) \ ^~~~~~~~~~~ ./src/basic/log.h:183:28: note: in expansion of macro ‘log_full’ #define log_info(...) log_full(LOG_INFO, __VA_ARGS__) ^~~~~~~~ src/coredump/coredump.c:768:25: note: in expansion of macro ‘log_info’ log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^~~~~~~~
2016-11-07 17:46:38 +01:00
log_info("The core will not be stored: size %"PRIu64" is greater than %"PRIu64" (the configured maximum)",
coredump_size, arg_external_size_max);
/* Vacuum again, but exclude the coredump we just created */
(void) coredump_vacuum(coredump_node_fd >= 0 ? coredump_node_fd : coredump_fd, arg_keep_free, arg_max_use);
/* Now, let's drop privileges to become the user who owns the segfaulted process and allocate the coredump
* memory under the user's uid. This also ensures that the credentials journald will see are the ones of the
* coredumping user, thus making sure the user gets access to the core dump. Let's also get rid of all
* capabilities, if we run as root, we won't need them anymore. */
r = change_uid_gid(context);
if (r < 0)
return log_error_errno(r, "Failed to drop privileges: %m");
#if HAVE_ELFUTILS
/* Try to get a strack trace if we can */
if (coredump_size <= arg_process_size_max) {
_cleanup_free_ char *stacktrace = NULL;
r = coredump_make_stack_trace(coredump_fd, context[CONTEXT_EXE], &stacktrace);
if (r >= 0)
core_message = strjoin("MESSAGE=Process ", context[CONTEXT_PID],
" (", context[CONTEXT_COMM], ") of user ",
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
context[CONTEXT_UID], " dumped core.",
journald_crash ? "\nCoredump diverted to " : "",
journald_crash ? filename : "",
"\n\n", stacktrace);
else if (r == -EINVAL)
log_warning("Failed to generate stack trace: %s", dwfl_errmsg(dwfl_errno()));
else
log_warning_errno(r, "Failed to generate stack trace: %m");
} else
coredump: fix format string on 32 bits In file included from ./src/basic/macro.h:415:0, from ./src/shared/acl-util.h:28, from src/coredump/coredump.c:36: src/coredump/coredump.c: In function ‘submit_coredump’: src/coredump/coredump.c:711:26: warning: format ‘%zu’ expects argument of type ‘size_t’, but argument 7 has type ‘uint64_t {aka long long unsigned int}’ [-Wformat=] log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^ ./src/basic/log.h:175:82: note: in definition of macro ‘log_full_errno’ ? log_internal(_level, _e, __FILE__, __LINE__, __func__, __VA_ARGS__) \ ^~~~~~~~~~~ ./src/basic/log.h:183:28: note: in expansion of macro ‘log_full’ #define log_info(...) log_full(LOG_INFO, __VA_ARGS__) ^~~~~~~~ src/coredump/coredump.c:711:17: note: in expansion of macro ‘log_info’ log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^~~~~~~~ src/coredump/coredump.c:711:26: warning: format ‘%zu’ expects argument of type ‘size_t’, but argument 8 has type ‘uint64_t {aka long long unsigned int}’ [-Wformat=] log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^ ./src/basic/log.h:175:82: note: in definition of macro ‘log_full_errno’ ? log_internal(_level, _e, __FILE__, __LINE__, __func__, __VA_ARGS__) \ ^~~~~~~~~~~ ./src/basic/log.h:183:28: note: in expansion of macro ‘log_full’ #define log_info(...) log_full(LOG_INFO, __VA_ARGS__) ^~~~~~~~ src/coredump/coredump.c:711:17: note: in expansion of macro ‘log_info’ log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^~~~~~~~ src/coredump/coredump.c:741:27: warning: format ‘%zu’ expects argument of type ‘size_t’, but argument 7 has type ‘uint64_t {aka long long unsigned int}’ [-Wformat=] log_debug("Not generating stack trace: core size %zu is greater than %zu (the configured maximum)", ^ ./src/basic/log.h:175:82: note: in definition of macro ‘log_full_errno’ ? log_internal(_level, _e, __FILE__, __LINE__, __func__, __VA_ARGS__) \ ^~~~~~~~~~~ ./src/basic/log.h:182:28: note: in expansion of macro ‘log_full’ #define log_debug(...) log_full(LOG_DEBUG, __VA_ARGS__) ^~~~~~~~ src/coredump/coredump.c:741:17: note: in expansion of macro ‘log_debug’ log_debug("Not generating stack trace: core size %zu is greater than %zu (the configured maximum)", ^~~~~~~~~ src/coredump/coredump.c:741:27: warning: format ‘%zu’ expects argument of type ‘size_t’, but argument 8 has type ‘uint64_t {aka long long unsigned int}’ [-Wformat=] log_debug("Not generating stack trace: core size %zu is greater than %zu (the configured maximum)", ^ ./src/basic/log.h:175:82: note: in definition of macro ‘log_full_errno’ ? log_internal(_level, _e, __FILE__, __LINE__, __func__, __VA_ARGS__) \ ^~~~~~~~~~~ ./src/basic/log.h:182:28: note: in expansion of macro ‘log_full’ #define log_debug(...) log_full(LOG_DEBUG, __VA_ARGS__) ^~~~~~~~ src/coredump/coredump.c:741:17: note: in expansion of macro ‘log_debug’ log_debug("Not generating stack trace: core size %zu is greater than %zu (the configured maximum)", ^~~~~~~~~ src/coredump/coredump.c:768:34: warning: format ‘%zu’ expects argument of type ‘size_t’, but argument 7 has type ‘uint64_t {aka long long unsigned int}’ [-Wformat=] log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^ ./src/basic/log.h:175:82: note: in definition of macro ‘log_full_errno’ ? log_internal(_level, _e, __FILE__, __LINE__, __func__, __VA_ARGS__) \ ^~~~~~~~~~~ ./src/basic/log.h:183:28: note: in expansion of macro ‘log_full’ #define log_info(...) log_full(LOG_INFO, __VA_ARGS__) ^~~~~~~~ src/coredump/coredump.c:768:25: note: in expansion of macro ‘log_info’ log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^~~~~~~~
2016-11-07 17:46:38 +01:00
log_debug("Not generating stack trace: core size %"PRIu64" is greater than %"PRIu64" (the configured maximum)",
coredump_size, arg_process_size_max);
if (!core_message)
#endif
log:
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
core_message = strjoin("MESSAGE=Process ", context[CONTEXT_PID],
" (", context[CONTEXT_COMM], ") of user ",
context[CONTEXT_UID], " dumped core.",
journald_crash ? "\nCoredump diverted to " : NULL,
journald_crash ? filename : NULL);
if (!core_message)
return log_oom();
if (journald_crash) {
/* We cannot log to the journal, so just print the MESSAGE.
* The target was set previously to something safe. */
log_dispatch(LOG_ERR, 0, core_message);
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
return 0;
}
iovec[n_iovec++] = IOVEC_MAKE_STRING(core_message);
if (truncated)
iovec[n_iovec++] = IOVEC_MAKE_STRING("COREDUMP_TRUNCATED=1");
/* Optionally store the entire coredump in the journal */
if (arg_storage == COREDUMP_STORAGE_JOURNAL) {
if (coredump_size <= arg_journal_size_max) {
size_t sz = 0;
/* Store the coredump itself in the journal */
r = allocate_journal_field(coredump_fd, (size_t) coredump_size, &coredump_data, &sz);
if (r >= 0)
iovec[n_iovec++] = IOVEC_MAKE(coredump_data, sz);
else
log_warning_errno(r, "Failed to attach the core to the journal entry: %m");
} else
coredump: fix format string on 32 bits In file included from ./src/basic/macro.h:415:0, from ./src/shared/acl-util.h:28, from src/coredump/coredump.c:36: src/coredump/coredump.c: In function ‘submit_coredump’: src/coredump/coredump.c:711:26: warning: format ‘%zu’ expects argument of type ‘size_t’, but argument 7 has type ‘uint64_t {aka long long unsigned int}’ [-Wformat=] log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^ ./src/basic/log.h:175:82: note: in definition of macro ‘log_full_errno’ ? log_internal(_level, _e, __FILE__, __LINE__, __func__, __VA_ARGS__) \ ^~~~~~~~~~~ ./src/basic/log.h:183:28: note: in expansion of macro ‘log_full’ #define log_info(...) log_full(LOG_INFO, __VA_ARGS__) ^~~~~~~~ src/coredump/coredump.c:711:17: note: in expansion of macro ‘log_info’ log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^~~~~~~~ src/coredump/coredump.c:711:26: warning: format ‘%zu’ expects argument of type ‘size_t’, but argument 8 has type ‘uint64_t {aka long long unsigned int}’ [-Wformat=] log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^ ./src/basic/log.h:175:82: note: in definition of macro ‘log_full_errno’ ? log_internal(_level, _e, __FILE__, __LINE__, __func__, __VA_ARGS__) \ ^~~~~~~~~~~ ./src/basic/log.h:183:28: note: in expansion of macro ‘log_full’ #define log_info(...) log_full(LOG_INFO, __VA_ARGS__) ^~~~~~~~ src/coredump/coredump.c:711:17: note: in expansion of macro ‘log_info’ log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^~~~~~~~ src/coredump/coredump.c:741:27: warning: format ‘%zu’ expects argument of type ‘size_t’, but argument 7 has type ‘uint64_t {aka long long unsigned int}’ [-Wformat=] log_debug("Not generating stack trace: core size %zu is greater than %zu (the configured maximum)", ^ ./src/basic/log.h:175:82: note: in definition of macro ‘log_full_errno’ ? log_internal(_level, _e, __FILE__, __LINE__, __func__, __VA_ARGS__) \ ^~~~~~~~~~~ ./src/basic/log.h:182:28: note: in expansion of macro ‘log_full’ #define log_debug(...) log_full(LOG_DEBUG, __VA_ARGS__) ^~~~~~~~ src/coredump/coredump.c:741:17: note: in expansion of macro ‘log_debug’ log_debug("Not generating stack trace: core size %zu is greater than %zu (the configured maximum)", ^~~~~~~~~ src/coredump/coredump.c:741:27: warning: format ‘%zu’ expects argument of type ‘size_t’, but argument 8 has type ‘uint64_t {aka long long unsigned int}’ [-Wformat=] log_debug("Not generating stack trace: core size %zu is greater than %zu (the configured maximum)", ^ ./src/basic/log.h:175:82: note: in definition of macro ‘log_full_errno’ ? log_internal(_level, _e, __FILE__, __LINE__, __func__, __VA_ARGS__) \ ^~~~~~~~~~~ ./src/basic/log.h:182:28: note: in expansion of macro ‘log_full’ #define log_debug(...) log_full(LOG_DEBUG, __VA_ARGS__) ^~~~~~~~ src/coredump/coredump.c:741:17: note: in expansion of macro ‘log_debug’ log_debug("Not generating stack trace: core size %zu is greater than %zu (the configured maximum)", ^~~~~~~~~ src/coredump/coredump.c:768:34: warning: format ‘%zu’ expects argument of type ‘size_t’, but argument 7 has type ‘uint64_t {aka long long unsigned int}’ [-Wformat=] log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^ ./src/basic/log.h:175:82: note: in definition of macro ‘log_full_errno’ ? log_internal(_level, _e, __FILE__, __LINE__, __func__, __VA_ARGS__) \ ^~~~~~~~~~~ ./src/basic/log.h:183:28: note: in expansion of macro ‘log_full’ #define log_info(...) log_full(LOG_INFO, __VA_ARGS__) ^~~~~~~~ src/coredump/coredump.c:768:25: note: in expansion of macro ‘log_info’ log_info("The core will not be stored: size %zu is greater than %zu (the configured maximum)", ^~~~~~~~
2016-11-07 17:46:38 +01:00
log_info("The core will not be stored: size %"PRIu64" is greater than %"PRIu64" (the configured maximum)",
coredump_size, arg_journal_size_max);
}
assert(n_iovec <= n_iovec_allocated);
r = sd_journal_sendv(iovec, n_iovec);
if (r < 0)
return log_error_errno(r, "Failed to log coredump: %m");
return 0;
}
static void map_context_fields(const struct iovec *iovec, const char* context[]) {
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
static const char * const context_field_names[] = {
[CONTEXT_PID] = "COREDUMP_PID=",
[CONTEXT_UID] = "COREDUMP_UID=",
[CONTEXT_GID] = "COREDUMP_GID=",
[CONTEXT_SIGNAL] = "COREDUMP_SIGNAL=",
[CONTEXT_TIMESTAMP] = "COREDUMP_TIMESTAMP=",
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
[CONTEXT_RLIMIT] = "COREDUMP_RLIMIT=",
[CONTEXT_HOSTNAME] = "COREDUMP_HOSTNAME=",
[CONTEXT_COMM] = "COREDUMP_COMM=",
[CONTEXT_EXE] = "COREDUMP_EXE=",
};
unsigned i;
assert(iovec);
assert(context);
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
for (i = 0; i < ELEMENTSOF(context_field_names); i++) {
size_t l;
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
if (!context_field_names[i])
continue;
l = strlen(context_field_names[i]);
if (iovec->iov_len < l)
continue;
if (memcmp(iovec->iov_base, context_field_names[i], l) != 0)
continue;
/* Note that these strings are NUL terminated, because we made sure that a trailing NUL byte is in the
* buffer, though not included in the iov_len count. (see below) */
context[i] = (char*) iovec->iov_base + l;
break;
}
}
static int process_socket(int fd) {
_cleanup_close_ int coredump_fd = -1;
struct iovec *iovec = NULL;
size_t n_iovec = 0, n_allocated = 0, i, k;
const char *context[_CONTEXT_MAX] = {};
int r;
assert(fd >= 0);
log_set_target(LOG_TARGET_AUTO);
log_parse_environment();
log_open();
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
log_debug("Processing coredump received on stdin...");
for (;;) {
union {
struct cmsghdr cmsghdr;
uint8_t buf[CMSG_SPACE(sizeof(int))];
} control = {};
struct msghdr mh = {
.msg_control = &control,
.msg_controllen = sizeof(control),
.msg_iovlen = 1,
};
ssize_t n;
ssize_t l;
if (!GREEDY_REALLOC(iovec, n_allocated, n_iovec + SUBMIT_COREDUMP_FIELDS)) {
r = log_oom();
goto finish;
}
l = next_datagram_size_fd(fd);
if (l < 0) {
r = log_error_errno(l, "Failed to determine datagram size to read: %m");
goto finish;
}
assert(l >= 0);
iovec[n_iovec].iov_len = l;
iovec[n_iovec].iov_base = malloc(l + 1);
if (!iovec[n_iovec].iov_base) {
r = log_oom();
goto finish;
}
mh.msg_iov = iovec + n_iovec;
n = recvmsg(fd, &mh, MSG_NOSIGNAL|MSG_CMSG_CLOEXEC);
if (n < 0) {
free(iovec[n_iovec].iov_base);
r = log_error_errno(errno, "Failed to receive datagram: %m");
goto finish;
}
if (n == 0) {
struct cmsghdr *cmsg, *found = NULL;
/* The final zero-length datagram carries the file descriptor and tells us that we're done. */
free(iovec[n_iovec].iov_base);
CMSG_FOREACH(cmsg, &mh) {
if (cmsg->cmsg_level == SOL_SOCKET &&
cmsg->cmsg_type == SCM_RIGHTS &&
cmsg->cmsg_len == CMSG_LEN(sizeof(int))) {
assert(!found);
found = cmsg;
}
}
if (!found) {
log_error("Coredump file descriptor missing.");
r = -EBADMSG;
goto finish;
}
assert(coredump_fd < 0);
coredump_fd = *(int*) CMSG_DATA(found);
break;
}
/* Add trailing NUL byte, in case these are strings */
((char*) iovec[n_iovec].iov_base)[n] = 0;
iovec[n_iovec].iov_len = (size_t) n;
cmsg_close_all(&mh);
map_context_fields(iovec + n_iovec, context);
n_iovec++;
}
if (!GREEDY_REALLOC(iovec, n_allocated, n_iovec + SUBMIT_COREDUMP_FIELDS)) {
r = log_oom();
goto finish;
}
/* Make sure we got all data we really need */
assert(context[CONTEXT_PID]);
assert(context[CONTEXT_UID]);
assert(context[CONTEXT_GID]);
assert(context[CONTEXT_SIGNAL]);
assert(context[CONTEXT_TIMESTAMP]);
assert(context[CONTEXT_RLIMIT]);
assert(context[CONTEXT_HOSTNAME]);
assert(context[CONTEXT_COMM]);
assert(coredump_fd >= 0);
/* Small quirk: the journal fields contain the timestamp padded with six zeroes, so that the kernel-supplied 1s
* granularity timestamps becomes 1µs granularity, i.e. the granularity systemd usually operates in. Since we
* are reconstructing the original kernel context, we chop this off again, here. */
k = strlen(context[CONTEXT_TIMESTAMP]);
if (k > 6)
context[CONTEXT_TIMESTAMP] = strndupa(context[CONTEXT_TIMESTAMP], k - 6);
coredump: with --backtrace accept a journal entry on stdin The entry must be a single entry in the journal export format, including the terminating double newline. The MESSAGE field is now generated on the sender side. The advantage is that the reporter can easily pass additional metadata. Continuing with the example of the python excepthook: COREDUMP_PYTHON_EXECUTABLE=/usr/bin/python3 COREDUMP_PYTHON_VERSION=3.5.2 (default, Sep 14 2016, 11:28:32) [GCC 6.2.1 20160901 (Red Hat 6.2.1-1)] COREDUMP_PYTHON_THREAD_INFO=sys.thread_info(name='pthread', lock='semaphore', version='NPTL 2.24') COREDUMP_PYTHON_EXCEPTION_TYPE=ZeroDivisionError COREDUMP_PYTHON_EXCEPTION_VALUE=division by zero MESSAGE=Process 29514 (systemd_coredump_exception_handler.py) of user zbyszek failed with ZeroDivisionError: division by zero Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 134, in <module> g() File "systemd_coredump_exception_handler.py", line 133, in g f() File "systemd_coredump_exception_handler.py", line 131, in f div0 = 1 / 0 ZeroDivisionError: division by zero Local variables in innermost frame: a=3 h=<function f at 0x7efdc14b6ea0> One consideration is whether to use the Journal Export Format, or send packets over a UNIX socket instead. The advantage of current solution is that although parsing is more complicated on the receiver side, it is much easier to use on the sender side. I hope this can be used by various languages for which writing binary structures to a UNIX socket is harder and more likely to be done wrong than piping of a simple textyish format.
2016-11-06 16:06:32 +01:00
r = submit_coredump(context, iovec, n_allocated, n_iovec, coredump_fd);
finish:
for (i = 0; i < n_iovec; i++)
free(iovec[i].iov_base);
free(iovec);
return r;
}
static int send_iovec(const struct iovec iovec[], size_t n_iovec, int input_fd) {
static const union sockaddr_union sa = {
.un.sun_family = AF_UNIX,
.un.sun_path = "/run/systemd/coredump",
};
_cleanup_close_ int fd = -1;
size_t i;
int r;
assert(iovec || n_iovec <= 0);
assert(input_fd >= 0);
fd = socket(AF_UNIX, SOCK_SEQPACKET|SOCK_CLOEXEC, 0);
if (fd < 0)
return log_error_errno(errno, "Failed to create coredump socket: %m");
if (connect(fd, &sa.sa, SOCKADDR_UN_LEN(sa.un)) < 0)
return log_error_errno(errno, "Failed to connect to coredump service: %m");
for (i = 0; i < n_iovec; i++) {
struct msghdr mh = {
.msg_iov = (struct iovec*) iovec + i,
.msg_iovlen = 1,
};
struct iovec copy[2];
for (;;) {
if (sendmsg(fd, &mh, MSG_NOSIGNAL) >= 0)
break;
if (errno == EMSGSIZE && mh.msg_iov[0].iov_len > 0) {
/* This field didn't fit? That's a pity. Given that this is just metadata,
* let's truncate the field at half, and try again. We append three dots, in
* order to show that this is truncated. */
if (mh.msg_iov != copy) {
/* We don't want to modify the caller's iovec, hence let's create our
* own array, consisting of two new iovecs, where the first is a
* (truncated) copy of what we want to send, and the second one
* contains the trailing dots. */
copy[0] = iovec[i];
copy[1] = (struct iovec) {
.iov_base = (char[]) { '.', '.', '.' },
.iov_len = 3,
};
mh.msg_iov = copy;
mh.msg_iovlen = 2;
}
copy[0].iov_len /= 2; /* halve it, and try again */
continue;
}
return log_error_errno(errno, "Failed to send coredump datagram: %m");
}
}
r = send_one_fd(fd, input_fd, 0);
if (r < 0)
return log_error_errno(r, "Failed to send coredump fd: %m");
return 0;
}
static char* set_iovec_field(struct iovec *iovec, size_t *n_iovec, const char *field, const char *value) {
char *x;
x = strappend(field, value);
if (x)
iovec[(*n_iovec)++] = IOVEC_MAKE_STRING(x);
return x;
}
static char* set_iovec_field_free(struct iovec *iovec, size_t *n_iovec, const char *field, char *value) {
char *x;
x = set_iovec_field(iovec, n_iovec, field, value);
free(value);
return x;
}
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
static int gather_pid_metadata(
char* context[_CONTEXT_MAX],
char **comm_fallback,
coredump: with --backtrace accept a journal entry on stdin The entry must be a single entry in the journal export format, including the terminating double newline. The MESSAGE field is now generated on the sender side. The advantage is that the reporter can easily pass additional metadata. Continuing with the example of the python excepthook: COREDUMP_PYTHON_EXECUTABLE=/usr/bin/python3 COREDUMP_PYTHON_VERSION=3.5.2 (default, Sep 14 2016, 11:28:32) [GCC 6.2.1 20160901 (Red Hat 6.2.1-1)] COREDUMP_PYTHON_THREAD_INFO=sys.thread_info(name='pthread', lock='semaphore', version='NPTL 2.24') COREDUMP_PYTHON_EXCEPTION_TYPE=ZeroDivisionError COREDUMP_PYTHON_EXCEPTION_VALUE=division by zero MESSAGE=Process 29514 (systemd_coredump_exception_handler.py) of user zbyszek failed with ZeroDivisionError: division by zero Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 134, in <module> g() File "systemd_coredump_exception_handler.py", line 133, in g f() File "systemd_coredump_exception_handler.py", line 131, in f div0 = 1 / 0 ZeroDivisionError: division by zero Local variables in innermost frame: a=3 h=<function f at 0x7efdc14b6ea0> One consideration is whether to use the Journal Export Format, or send packets over a UNIX socket instead. The advantage of current solution is that although parsing is more complicated on the receiver side, it is much easier to use on the sender side. I hope this can be used by various languages for which writing binary structures to a UNIX socket is harder and more likely to be done wrong than piping of a simple textyish format.
2016-11-06 16:06:32 +01:00
struct iovec *iovec, size_t *n_iovec) {
/* We need 27 empty slots in iovec!
*
* Note that if we fail on oom later on, we do not roll-back changes to the iovec structure. (It remains valid,
* with the first n_iovec fields initialized.) */
uid_t owner_uid;
pid_t pid;
char *t;
const char *p;
int r, signo;
r = parse_pid(context[CONTEXT_PID], &pid);
if (r < 0)
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
return log_error_errno(r, "Failed to parse PID \"%s\": %m", context[CONTEXT_PID]);
r = get_process_comm(pid, &context[CONTEXT_COMM]);
if (r < 0) {
log_warning_errno(r, "Failed to get COMM, falling back to the command line: %m");
context[CONTEXT_COMM] = strv_join(comm_fallback, " ");
if (!context[CONTEXT_COMM])
return log_oom();
}
r = get_process_exe(pid, &context[CONTEXT_EXE]);
if (r < 0)
log_warning_errno(r, "Failed to get EXE, ignoring: %m");
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
if (cg_pid_get_unit(pid, &context[CONTEXT_UNIT]) >= 0) {
if (!is_journald_crash((const char**) context)) {
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
/* OK, now we know it's not the journal, hence we can make use of it now. */
log_set_target(LOG_TARGET_JOURNAL_OR_KMSG);
log_open();
}
/* If this is PID 1 disable coredump collection, we'll unlikely be able to process it later on. */
if (is_pid1_crash((const char**) context)) {
log_notice("Due to PID 1 having crashed coredump collection will now be turned off.");
disable_coredumps();
}
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
set_iovec_field(iovec, n_iovec, "COREDUMP_UNIT=", context[CONTEXT_UNIT]);
}
if (cg_pid_get_user_unit(pid, &t) >= 0)
set_iovec_field_free(iovec, n_iovec, "COREDUMP_USER_UNIT=", t);
/* The next few are mandatory */
if (!set_iovec_field(iovec, n_iovec, "COREDUMP_PID=", context[CONTEXT_PID]))
return log_oom();
if (!set_iovec_field(iovec, n_iovec, "COREDUMP_UID=", context[CONTEXT_UID]))
return log_oom();
if (!set_iovec_field(iovec, n_iovec, "COREDUMP_GID=", context[CONTEXT_GID]))
return log_oom();
if (!set_iovec_field(iovec, n_iovec, "COREDUMP_SIGNAL=", context[CONTEXT_SIGNAL]))
return log_oom();
if (!set_iovec_field(iovec, n_iovec, "COREDUMP_RLIMIT=", context[CONTEXT_RLIMIT]))
return log_oom();
if (!set_iovec_field(iovec, n_iovec, "COREDUMP_HOSTNAME=", context[CONTEXT_HOSTNAME]))
return log_oom();
if (!set_iovec_field(iovec, n_iovec, "COREDUMP_COMM=", context[CONTEXT_COMM]))
return log_oom();
if (context[CONTEXT_EXE] &&
!set_iovec_field(iovec, n_iovec, "COREDUMP_EXE=", context[CONTEXT_EXE]))
return log_oom();
if (sd_pid_get_session(pid, &t) >= 0)
set_iovec_field_free(iovec, n_iovec, "COREDUMP_SESSION=", t);
if (sd_pid_get_owner_uid(pid, &owner_uid) >= 0) {
r = asprintf(&t, "COREDUMP_OWNER_UID=" UID_FMT, owner_uid);
if (r > 0)
iovec[(*n_iovec)++] = IOVEC_MAKE_STRING(t);
}
if (sd_pid_get_slice(pid, &t) >= 0)
set_iovec_field_free(iovec, n_iovec, "COREDUMP_SLICE=", t);
if (get_process_cmdline(pid, 0, false, &t) >= 0)
set_iovec_field_free(iovec, n_iovec, "COREDUMP_CMDLINE=", t);
if (cg_pid_get_path_shifted(pid, NULL, &t) >= 0)
set_iovec_field_free(iovec, n_iovec, "COREDUMP_CGROUP=", t);
if (compose_open_fds(pid, &t) >= 0)
set_iovec_field_free(iovec, n_iovec, "COREDUMP_OPEN_FDS=", t);
p = procfs_file_alloca(pid, "status");
if (read_full_file(p, &t, NULL) >= 0)
set_iovec_field_free(iovec, n_iovec, "COREDUMP_PROC_STATUS=", t);
p = procfs_file_alloca(pid, "maps");
if (read_full_file(p, &t, NULL) >= 0)
set_iovec_field_free(iovec, n_iovec, "COREDUMP_PROC_MAPS=", t);
p = procfs_file_alloca(pid, "limits");
if (read_full_file(p, &t, NULL) >= 0)
set_iovec_field_free(iovec, n_iovec, "COREDUMP_PROC_LIMITS=", t);
p = procfs_file_alloca(pid, "cgroup");
if (read_full_file(p, &t, NULL) >=0)
set_iovec_field_free(iovec, n_iovec, "COREDUMP_PROC_CGROUP=", t);
p = procfs_file_alloca(pid, "mountinfo");
if (read_full_file(p, &t, NULL) >=0)
set_iovec_field_free(iovec, n_iovec, "COREDUMP_PROC_MOUNTINFO=", t);
if (get_process_cwd(pid, &t) >= 0)
set_iovec_field_free(iovec, n_iovec, "COREDUMP_CWD=", t);
if (get_process_root(pid, &t) >= 0) {
bool proc_self_root_is_slash;
proc_self_root_is_slash = strcmp(t, "/") == 0;
set_iovec_field_free(iovec, n_iovec, "COREDUMP_ROOT=", t);
/* If the process' root is "/", then there is a chance it has
* mounted own root and hence being containerized. */
if (proc_self_root_is_slash && get_process_container_parent_cmdline(pid, &t) > 0)
set_iovec_field_free(iovec, n_iovec, "COREDUMP_CONTAINER_CMDLINE=", t);
}
if (get_process_environ(pid, &t) >= 0)
set_iovec_field_free(iovec, n_iovec, "COREDUMP_ENVIRON=", t);
t = strjoin("COREDUMP_TIMESTAMP=", context[CONTEXT_TIMESTAMP], "000000");
if (t)
iovec[(*n_iovec)++] = IOVEC_MAKE_STRING(t);
if (safe_atoi(context[CONTEXT_SIGNAL], &signo) >= 0 && SIGNAL_VALID(signo))
set_iovec_field(iovec, n_iovec, "COREDUMP_SIGNAL_NAME=SIG", signal_to_string(signo));
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
return 0; /* we successfully acquired all metadata */
}
static int process_kernel(int argc, char* argv[]) {
char* context[_CONTEXT_MAX] = {};
struct iovec iovec[29 + SUBMIT_COREDUMP_FIELDS];
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
size_t i, n_iovec, n_to_free = 0;
int r;
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
log_debug("Processing coredump received from the kernel...");
if (argc < CONTEXT_COMM + 1) {
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
log_error("Not enough arguments passed by the kernel (%i, expected %i).", argc - 1, CONTEXT_COMM + 1 - 1);
return -EINVAL;
}
context[CONTEXT_PID] = argv[1 + CONTEXT_PID];
context[CONTEXT_UID] = argv[1 + CONTEXT_UID];
context[CONTEXT_GID] = argv[1 + CONTEXT_GID];
context[CONTEXT_SIGNAL] = argv[1 + CONTEXT_SIGNAL];
context[CONTEXT_TIMESTAMP] = argv[1 + CONTEXT_TIMESTAMP];
context[CONTEXT_RLIMIT] = argv[1 + CONTEXT_RLIMIT];
context[CONTEXT_HOSTNAME] = argv[1 + CONTEXT_HOSTNAME];
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
r = gather_pid_metadata(context, argv + 1 + CONTEXT_COMM, iovec, &n_to_free);
if (r < 0)
goto finish;
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
n_iovec = n_to_free;
iovec[n_iovec++] = IOVEC_MAKE_STRING("MESSAGE_ID=" SD_MESSAGE_COREDUMP_STR);
assert_cc(2 == LOG_CRIT);
iovec[n_iovec++] = IOVEC_MAKE_STRING("PRIORITY=2");
assert(n_iovec <= ELEMENTSOF(iovec));
if (is_journald_crash((const char**) context) || is_pid1_crash((const char**) context))
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
r = submit_coredump((const char**) context,
iovec, ELEMENTSOF(iovec), n_iovec,
STDIN_FILENO);
else
r = send_iovec(iovec, n_iovec, STDIN_FILENO);
finish:
for (i = 0; i < n_to_free; i++)
free(iovec[i].iov_base);
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
/* Those fields are allocated by gather_pid_metadata */
free(context[CONTEXT_COMM]);
free(context[CONTEXT_EXE]);
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
free(context[CONTEXT_UNIT]);
return r;
}
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
static int process_backtrace(int argc, char *argv[]) {
char *context[_CONTEXT_MAX] = {};
_cleanup_free_ char *message = NULL;
coredump: with --backtrace accept a journal entry on stdin The entry must be a single entry in the journal export format, including the terminating double newline. The MESSAGE field is now generated on the sender side. The advantage is that the reporter can easily pass additional metadata. Continuing with the example of the python excepthook: COREDUMP_PYTHON_EXECUTABLE=/usr/bin/python3 COREDUMP_PYTHON_VERSION=3.5.2 (default, Sep 14 2016, 11:28:32) [GCC 6.2.1 20160901 (Red Hat 6.2.1-1)] COREDUMP_PYTHON_THREAD_INFO=sys.thread_info(name='pthread', lock='semaphore', version='NPTL 2.24') COREDUMP_PYTHON_EXCEPTION_TYPE=ZeroDivisionError COREDUMP_PYTHON_EXCEPTION_VALUE=division by zero MESSAGE=Process 29514 (systemd_coredump_exception_handler.py) of user zbyszek failed with ZeroDivisionError: division by zero Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 134, in <module> g() File "systemd_coredump_exception_handler.py", line 133, in g f() File "systemd_coredump_exception_handler.py", line 131, in f div0 = 1 / 0 ZeroDivisionError: division by zero Local variables in innermost frame: a=3 h=<function f at 0x7efdc14b6ea0> One consideration is whether to use the Journal Export Format, or send packets over a UNIX socket instead. The advantage of current solution is that although parsing is more complicated on the receiver side, it is much easier to use on the sender side. I hope this can be used by various languages for which writing binary structures to a UNIX socket is harder and more likely to be done wrong than piping of a simple textyish format.
2016-11-06 16:06:32 +01:00
_cleanup_free_ struct iovec *iovec = NULL;
size_t n_iovec, n_allocated, n_to_free = 0, i;
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
int r;
coredump: with --backtrace accept a journal entry on stdin The entry must be a single entry in the journal export format, including the terminating double newline. The MESSAGE field is now generated on the sender side. The advantage is that the reporter can easily pass additional metadata. Continuing with the example of the python excepthook: COREDUMP_PYTHON_EXECUTABLE=/usr/bin/python3 COREDUMP_PYTHON_VERSION=3.5.2 (default, Sep 14 2016, 11:28:32) [GCC 6.2.1 20160901 (Red Hat 6.2.1-1)] COREDUMP_PYTHON_THREAD_INFO=sys.thread_info(name='pthread', lock='semaphore', version='NPTL 2.24') COREDUMP_PYTHON_EXCEPTION_TYPE=ZeroDivisionError COREDUMP_PYTHON_EXCEPTION_VALUE=division by zero MESSAGE=Process 29514 (systemd_coredump_exception_handler.py) of user zbyszek failed with ZeroDivisionError: division by zero Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 134, in <module> g() File "systemd_coredump_exception_handler.py", line 133, in g f() File "systemd_coredump_exception_handler.py", line 131, in f div0 = 1 / 0 ZeroDivisionError: division by zero Local variables in innermost frame: a=3 h=<function f at 0x7efdc14b6ea0> One consideration is whether to use the Journal Export Format, or send packets over a UNIX socket instead. The advantage of current solution is that although parsing is more complicated on the receiver side, it is much easier to use on the sender side. I hope this can be used by various languages for which writing binary structures to a UNIX socket is harder and more likely to be done wrong than piping of a simple textyish format.
2016-11-06 16:06:32 +01:00
JournalImporter importer = {
.fd = STDIN_FILENO,
};
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
log_debug("Processing backtrace on stdin...");
if (argc < CONTEXT_COMM + 1) {
log_error("Not enough arguments passed (%i, expected %i).", argc - 1, CONTEXT_COMM + 1 - 1);
return -EINVAL;
}
context[CONTEXT_PID] = argv[2 + CONTEXT_PID];
context[CONTEXT_UID] = argv[2 + CONTEXT_UID];
context[CONTEXT_GID] = argv[2 + CONTEXT_GID];
context[CONTEXT_SIGNAL] = argv[2 + CONTEXT_SIGNAL];
context[CONTEXT_TIMESTAMP] = argv[2 + CONTEXT_TIMESTAMP];
context[CONTEXT_RLIMIT] = argv[2 + CONTEXT_RLIMIT];
context[CONTEXT_HOSTNAME] = argv[2 + CONTEXT_HOSTNAME];
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
n_allocated = 34 + COREDUMP_STORAGE_EXTERNAL;
/* 26 metadata, 2 static, +unknown input, 4 storage, rounded up */
coredump: with --backtrace accept a journal entry on stdin The entry must be a single entry in the journal export format, including the terminating double newline. The MESSAGE field is now generated on the sender side. The advantage is that the reporter can easily pass additional metadata. Continuing with the example of the python excepthook: COREDUMP_PYTHON_EXECUTABLE=/usr/bin/python3 COREDUMP_PYTHON_VERSION=3.5.2 (default, Sep 14 2016, 11:28:32) [GCC 6.2.1 20160901 (Red Hat 6.2.1-1)] COREDUMP_PYTHON_THREAD_INFO=sys.thread_info(name='pthread', lock='semaphore', version='NPTL 2.24') COREDUMP_PYTHON_EXCEPTION_TYPE=ZeroDivisionError COREDUMP_PYTHON_EXCEPTION_VALUE=division by zero MESSAGE=Process 29514 (systemd_coredump_exception_handler.py) of user zbyszek failed with ZeroDivisionError: division by zero Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 134, in <module> g() File "systemd_coredump_exception_handler.py", line 133, in g f() File "systemd_coredump_exception_handler.py", line 131, in f div0 = 1 / 0 ZeroDivisionError: division by zero Local variables in innermost frame: a=3 h=<function f at 0x7efdc14b6ea0> One consideration is whether to use the Journal Export Format, or send packets over a UNIX socket instead. The advantage of current solution is that although parsing is more complicated on the receiver side, it is much easier to use on the sender side. I hope this can be used by various languages for which writing binary structures to a UNIX socket is harder and more likely to be done wrong than piping of a simple textyish format.
2016-11-06 16:06:32 +01:00
iovec = new(struct iovec, n_allocated);
if (!iovec)
return log_oom();
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
r = gather_pid_metadata(context, argv + 2 + CONTEXT_COMM, iovec, &n_to_free);
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
if (r < 0)
goto finish;
if (r > 0) {
/* This was a special crash, and has already been processed. */
r = 0;
goto finish;
}
coredump: with --backtrace accept a journal entry on stdin The entry must be a single entry in the journal export format, including the terminating double newline. The MESSAGE field is now generated on the sender side. The advantage is that the reporter can easily pass additional metadata. Continuing with the example of the python excepthook: COREDUMP_PYTHON_EXECUTABLE=/usr/bin/python3 COREDUMP_PYTHON_VERSION=3.5.2 (default, Sep 14 2016, 11:28:32) [GCC 6.2.1 20160901 (Red Hat 6.2.1-1)] COREDUMP_PYTHON_THREAD_INFO=sys.thread_info(name='pthread', lock='semaphore', version='NPTL 2.24') COREDUMP_PYTHON_EXCEPTION_TYPE=ZeroDivisionError COREDUMP_PYTHON_EXCEPTION_VALUE=division by zero MESSAGE=Process 29514 (systemd_coredump_exception_handler.py) of user zbyszek failed with ZeroDivisionError: division by zero Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 134, in <module> g() File "systemd_coredump_exception_handler.py", line 133, in g f() File "systemd_coredump_exception_handler.py", line 131, in f div0 = 1 / 0 ZeroDivisionError: division by zero Local variables in innermost frame: a=3 h=<function f at 0x7efdc14b6ea0> One consideration is whether to use the Journal Export Format, or send packets over a UNIX socket instead. The advantage of current solution is that although parsing is more complicated on the receiver side, it is much easier to use on the sender side. I hope this can be used by various languages for which writing binary structures to a UNIX socket is harder and more likely to be done wrong than piping of a simple textyish format.
2016-11-06 16:06:32 +01:00
n_iovec = n_to_free;
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
for (;;) {
coredump: with --backtrace accept a journal entry on stdin The entry must be a single entry in the journal export format, including the terminating double newline. The MESSAGE field is now generated on the sender side. The advantage is that the reporter can easily pass additional metadata. Continuing with the example of the python excepthook: COREDUMP_PYTHON_EXECUTABLE=/usr/bin/python3 COREDUMP_PYTHON_VERSION=3.5.2 (default, Sep 14 2016, 11:28:32) [GCC 6.2.1 20160901 (Red Hat 6.2.1-1)] COREDUMP_PYTHON_THREAD_INFO=sys.thread_info(name='pthread', lock='semaphore', version='NPTL 2.24') COREDUMP_PYTHON_EXCEPTION_TYPE=ZeroDivisionError COREDUMP_PYTHON_EXCEPTION_VALUE=division by zero MESSAGE=Process 29514 (systemd_coredump_exception_handler.py) of user zbyszek failed with ZeroDivisionError: division by zero Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 134, in <module> g() File "systemd_coredump_exception_handler.py", line 133, in g f() File "systemd_coredump_exception_handler.py", line 131, in f div0 = 1 / 0 ZeroDivisionError: division by zero Local variables in innermost frame: a=3 h=<function f at 0x7efdc14b6ea0> One consideration is whether to use the Journal Export Format, or send packets over a UNIX socket instead. The advantage of current solution is that although parsing is more complicated on the receiver side, it is much easier to use on the sender side. I hope this can be used by various languages for which writing binary structures to a UNIX socket is harder and more likely to be done wrong than piping of a simple textyish format.
2016-11-06 16:06:32 +01:00
r = journal_importer_process_data(&importer);
if (r < 0) {
log_error_errno(r, "Failed to parse journal entry on stdin: %m");
goto finish;
}
if (r == 1 || /* complete entry */
journal_importer_eof(&importer)) /* end of data */
coredump: with --backtrace accept a journal entry on stdin The entry must be a single entry in the journal export format, including the terminating double newline. The MESSAGE field is now generated on the sender side. The advantage is that the reporter can easily pass additional metadata. Continuing with the example of the python excepthook: COREDUMP_PYTHON_EXECUTABLE=/usr/bin/python3 COREDUMP_PYTHON_VERSION=3.5.2 (default, Sep 14 2016, 11:28:32) [GCC 6.2.1 20160901 (Red Hat 6.2.1-1)] COREDUMP_PYTHON_THREAD_INFO=sys.thread_info(name='pthread', lock='semaphore', version='NPTL 2.24') COREDUMP_PYTHON_EXCEPTION_TYPE=ZeroDivisionError COREDUMP_PYTHON_EXCEPTION_VALUE=division by zero MESSAGE=Process 29514 (systemd_coredump_exception_handler.py) of user zbyszek failed with ZeroDivisionError: division by zero Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 134, in <module> g() File "systemd_coredump_exception_handler.py", line 133, in g f() File "systemd_coredump_exception_handler.py", line 131, in f div0 = 1 / 0 ZeroDivisionError: division by zero Local variables in innermost frame: a=3 h=<function f at 0x7efdc14b6ea0> One consideration is whether to use the Journal Export Format, or send packets over a UNIX socket instead. The advantage of current solution is that although parsing is more complicated on the receiver side, it is much easier to use on the sender side. I hope this can be used by various languages for which writing binary structures to a UNIX socket is harder and more likely to be done wrong than piping of a simple textyish format.
2016-11-06 16:06:32 +01:00
break;
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
}
coredump: with --backtrace accept a journal entry on stdin The entry must be a single entry in the journal export format, including the terminating double newline. The MESSAGE field is now generated on the sender side. The advantage is that the reporter can easily pass additional metadata. Continuing with the example of the python excepthook: COREDUMP_PYTHON_EXECUTABLE=/usr/bin/python3 COREDUMP_PYTHON_VERSION=3.5.2 (default, Sep 14 2016, 11:28:32) [GCC 6.2.1 20160901 (Red Hat 6.2.1-1)] COREDUMP_PYTHON_THREAD_INFO=sys.thread_info(name='pthread', lock='semaphore', version='NPTL 2.24') COREDUMP_PYTHON_EXCEPTION_TYPE=ZeroDivisionError COREDUMP_PYTHON_EXCEPTION_VALUE=division by zero MESSAGE=Process 29514 (systemd_coredump_exception_handler.py) of user zbyszek failed with ZeroDivisionError: division by zero Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 134, in <module> g() File "systemd_coredump_exception_handler.py", line 133, in g f() File "systemd_coredump_exception_handler.py", line 131, in f div0 = 1 / 0 ZeroDivisionError: division by zero Local variables in innermost frame: a=3 h=<function f at 0x7efdc14b6ea0> One consideration is whether to use the Journal Export Format, or send packets over a UNIX socket instead. The advantage of current solution is that although parsing is more complicated on the receiver side, it is much easier to use on the sender side. I hope this can be used by various languages for which writing binary structures to a UNIX socket is harder and more likely to be done wrong than piping of a simple textyish format.
2016-11-06 16:06:32 +01:00
if (!GREEDY_REALLOC(iovec, n_allocated, n_iovec + importer.iovw.count + 2))
return log_oom();
if (journal_importer_eof(&importer)) {
log_warning("Did not receive a full journal entry on stdin, ignoring message sent by reporter");
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
message = strjoin("MESSAGE=Process ", context[CONTEXT_PID],
" (", context[CONTEXT_COMM], ")"
coredump: with --backtrace accept a journal entry on stdin The entry must be a single entry in the journal export format, including the terminating double newline. The MESSAGE field is now generated on the sender side. The advantage is that the reporter can easily pass additional metadata. Continuing with the example of the python excepthook: COREDUMP_PYTHON_EXECUTABLE=/usr/bin/python3 COREDUMP_PYTHON_VERSION=3.5.2 (default, Sep 14 2016, 11:28:32) [GCC 6.2.1 20160901 (Red Hat 6.2.1-1)] COREDUMP_PYTHON_THREAD_INFO=sys.thread_info(name='pthread', lock='semaphore', version='NPTL 2.24') COREDUMP_PYTHON_EXCEPTION_TYPE=ZeroDivisionError COREDUMP_PYTHON_EXCEPTION_VALUE=division by zero MESSAGE=Process 29514 (systemd_coredump_exception_handler.py) of user zbyszek failed with ZeroDivisionError: division by zero Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 134, in <module> g() File "systemd_coredump_exception_handler.py", line 133, in g f() File "systemd_coredump_exception_handler.py", line 131, in f div0 = 1 / 0 ZeroDivisionError: division by zero Local variables in innermost frame: a=3 h=<function f at 0x7efdc14b6ea0> One consideration is whether to use the Journal Export Format, or send packets over a UNIX socket instead. The advantage of current solution is that although parsing is more complicated on the receiver side, it is much easier to use on the sender side. I hope this can be used by various languages for which writing binary structures to a UNIX socket is harder and more likely to be done wrong than piping of a simple textyish format.
2016-11-06 16:06:32 +01:00
" of user ", context[CONTEXT_UID],
" failed with ", context[CONTEXT_SIGNAL]);
if (!message) {
r = log_oom();
goto finish;
}
iovec[n_iovec++] = IOVEC_MAKE_STRING(message);
coredump: with --backtrace accept a journal entry on stdin The entry must be a single entry in the journal export format, including the terminating double newline. The MESSAGE field is now generated on the sender side. The advantage is that the reporter can easily pass additional metadata. Continuing with the example of the python excepthook: COREDUMP_PYTHON_EXECUTABLE=/usr/bin/python3 COREDUMP_PYTHON_VERSION=3.5.2 (default, Sep 14 2016, 11:28:32) [GCC 6.2.1 20160901 (Red Hat 6.2.1-1)] COREDUMP_PYTHON_THREAD_INFO=sys.thread_info(name='pthread', lock='semaphore', version='NPTL 2.24') COREDUMP_PYTHON_EXCEPTION_TYPE=ZeroDivisionError COREDUMP_PYTHON_EXCEPTION_VALUE=division by zero MESSAGE=Process 29514 (systemd_coredump_exception_handler.py) of user zbyszek failed with ZeroDivisionError: division by zero Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 134, in <module> g() File "systemd_coredump_exception_handler.py", line 133, in g f() File "systemd_coredump_exception_handler.py", line 131, in f div0 = 1 / 0 ZeroDivisionError: division by zero Local variables in innermost frame: a=3 h=<function f at 0x7efdc14b6ea0> One consideration is whether to use the Journal Export Format, or send packets over a UNIX socket instead. The advantage of current solution is that although parsing is more complicated on the receiver side, it is much easier to use on the sender side. I hope this can be used by various languages for which writing binary structures to a UNIX socket is harder and more likely to be done wrong than piping of a simple textyish format.
2016-11-06 16:06:32 +01:00
} else {
for (i = 0; i < importer.iovw.count; i++)
iovec[n_iovec++] = importer.iovw.iovec[i];
}
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
iovec[n_iovec++] = IOVEC_MAKE_STRING("MESSAGE_ID=" SD_MESSAGE_BACKTRACE_STR);
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
assert_cc(2 == LOG_CRIT);
iovec[n_iovec++] = IOVEC_MAKE_STRING("PRIORITY=2");
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
coredump: with --backtrace accept a journal entry on stdin The entry must be a single entry in the journal export format, including the terminating double newline. The MESSAGE field is now generated on the sender side. The advantage is that the reporter can easily pass additional metadata. Continuing with the example of the python excepthook: COREDUMP_PYTHON_EXECUTABLE=/usr/bin/python3 COREDUMP_PYTHON_VERSION=3.5.2 (default, Sep 14 2016, 11:28:32) [GCC 6.2.1 20160901 (Red Hat 6.2.1-1)] COREDUMP_PYTHON_THREAD_INFO=sys.thread_info(name='pthread', lock='semaphore', version='NPTL 2.24') COREDUMP_PYTHON_EXCEPTION_TYPE=ZeroDivisionError COREDUMP_PYTHON_EXCEPTION_VALUE=division by zero MESSAGE=Process 29514 (systemd_coredump_exception_handler.py) of user zbyszek failed with ZeroDivisionError: division by zero Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 134, in <module> g() File "systemd_coredump_exception_handler.py", line 133, in g f() File "systemd_coredump_exception_handler.py", line 131, in f div0 = 1 / 0 ZeroDivisionError: division by zero Local variables in innermost frame: a=3 h=<function f at 0x7efdc14b6ea0> One consideration is whether to use the Journal Export Format, or send packets over a UNIX socket instead. The advantage of current solution is that although parsing is more complicated on the receiver side, it is much easier to use on the sender side. I hope this can be used by various languages for which writing binary structures to a UNIX socket is harder and more likely to be done wrong than piping of a simple textyish format.
2016-11-06 16:06:32 +01:00
assert(n_iovec <= n_allocated);
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
r = sd_journal_sendv(iovec, n_iovec);
if (r < 0)
log_error_errno(r, "Failed to log backtrace: %m");
finish:
for (i = 0; i < n_to_free; i++)
free(iovec[i].iov_base);
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
/* Those fields are allocated by gather_pid_metadata */
free(context[CONTEXT_COMM]);
free(context[CONTEXT_EXE]);
coredump: process special crashes in an (almost) normal way We would only log a terse message when pid1 or systemd-journald crashed. It seems better to reuse the normal code paths as much as possible, with the following differences: - if pid1 crashes, we cannot launch the helper, so we don't analyze the coredump, just write it to file directly from the helper invoked by the kernel; - if journald crashes, we can produce the backtrace, but we don't log full structured messages. With comparison to previous code, advantages are: - we go through most of the steps, so for example vacuuming is performed, - we gather and log more data. In particular for journald and pid1 crashes we generate a backtrace, and for pid1 crashes we record the metadata (fdinfo, maps, etc.), - coredumpctl shows pid1 crashes. A disavantage (inefficiency) is that we gather metadata for journald crashes which is then ignored because _TRANSPORT=kernel does not support structued messages. Messages for the systemd-journald "crash" have _TRANSPORT=kernel, and _TRANSPORT=journal for the pid1 "crash". Feb 26 16:27:55 systemd[1]: systemd-journald.service: Main process exited, code=dumped, status=11/SEGV Feb 26 16:27:55 systemd[1]: systemd-journald.service: Unit entered failed state. Feb 26 16:37:54 systemd-coredump[18801]: Process 18729 (systemd-journal) of user 0 dumped core. Feb 26 16:37:54 systemd-coredump[18801]: Coredump diverted to /var/lib/systemd/coredump/core.systemd-journal.0.36c14bf3c6ce4c38914f441038990979.18729.1488145074000000.lz4 Feb 26 16:37:54 systemd-coredump[18801]: Stack trace of thread 18729: Feb 26 16:37:54 systemd-coredump[18801]: #0 0x00007f46d6a06b8d fsync (libpthread.so.0) Feb 26 16:37:54 systemd-coredump[18801]: #1 0x00007f46d71bfc47 journal_file_set_online (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #2 0x00007f46d71c1c31 journal_file_append_object (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #3 0x00007f46d71c3405 journal_file_append_data (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #4 0x00007f46d71c4b7c journal_file_append_entry (libsystemd-shared-233.so) Feb 26 16:37:54 systemd-coredump[18801]: #5 0x00005577688cf056 write_to_journal (systemd-journald) Feb 26 16:37:54 systemd-coredump[18801]: #6 0x00005577688d2e98 dispatch_message_real (systemd-journald) Feb 26 16:37:54 kernel: systemd-coredum: 9 output lines suppressed due to ratelimiting Feb 26 16:37:54 systemd-journald[18810]: Journal started Feb 26 16:50:59 systemd-coredump[19229]: Due to PID 1 having crashed coredump collection will now be turned off. Feb 26 16:51:00 systemd[1]: Caught <SEGV>, dumped core as pid 19228. Feb 26 16:51:00 systemd[1]: Freezing execution. Feb 26 16:51:00 systemd-coredump[19229]: Process 19228 (systemd) of user 0 dumped core. Stack trace of thread 19228: #0 0x00007fab82075c47 kill (libc.so.6) #1 0x000055fdf7c38b6b crash (systemd) #2 0x00007fab824175c0 __restore_rt (libpthread.so.0) #3 0x00007fab82148573 epoll_wait (libc.so.6) #4 0x00007fab8366f84a sd_event_wait (libsystemd-shared-233.so) #5 0x00007fab836701de sd_event_run (libsystemd-shared-233.so) #6 0x000055fdf7c4a380 manager_loop (systemd) #7 0x000055fdf7c402c2 main (systemd) #8 0x00007fab82060401 __libc_start_main (libc.so.6) #9 0x000055fdf7c3818a _start (systemd) Poor machine ;)
2017-02-26 22:46:23 +01:00
free(context[CONTEXT_UNIT]);
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
return r;
}
int main(int argc, char *argv[]) {
int r;
/* First, log to a safe place, since we don't know what crashed and it might
* be journald which we'd rather not log to then. */
log_set_target(LOG_TARGET_KMSG);
log_open();
/* Make sure we never enter a loop */
(void) prctl(PR_SET_DUMPABLE, 0);
/* Ignore all parse errors */
(void) parse_config();
log_debug("Selected storage '%s'.", coredump_storage_to_string(arg_storage));
log_debug("Selected compression %s.", yes_no(arg_compress));
r = sd_listen_fds(false);
if (r < 0) {
log_error_errno(r, "Failed to determine number of file descriptor: %m");
goto finish;
}
/* If we got an fd passed, we are running in coredumpd mode. Otherwise we
* are invoked from the kernel as coredump handler. */
coredump: implement logging of external backtraces with --backtrace This is useful for example for Python progams. By installing a python sys.execepthook we can store the backtrace in the journal. We gather the backtrace in the python process, and call systemd-coredump to attach additional fields (COREDUMP_COMM, COREDUMP_EXE, COREDUMP_UNIT, COREDUMP_USER_UNIT, COREDUMP_OWNER_UID, COREDUMP_SLICE, COREDUMP_CMDLINE, COREDUMP_CGROUP, COREDUMP_OPEN_FDS, COREDUMP_PROC_STATUS, COREDUMP_PROC_MAPS, COREDUMP_PROC_LIMITS, COREDUMP_PROC_MOUNTINFO, COREDUMP_CWD, COREDUMP_ROOT, COREDUMP_ENVIRON, COREDUMP_CONTAINER_CMDLINE). This could also be done in the python process, but doing this in systemd-coredump saves quite a bit of duplicate work and unifies the handling of various tricky fields like COREDUMP_CONTAINER_CMDLINE in one place. (Of course this applies to any other language which does not dump cores but wants to log a traceback, e.g. ruby.) journal entry: _TRANSPORT=journal _UID=1002 _GID=1002 _CAP_EFFECTIVE=0 _AUDIT_LOGINUID=1002 _SYSTEMD_OWNER_UID=1002 _SYSTEMD_SLICE=user-1002.slice _SYSTEMD_USER_SLICE=-.slice _SELINUX_CONTEXT=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 _BOOT_ID=1531fd22ec84429e85ae888b12fadb91 _MACHINE_ID=519a16632fbd4c71966ce9305b360c9c _HOSTNAME=laptop _AUDIT_SESSION=1 _SYSTEMD_UNIT=user@1002.service _SYSTEMD_INVOCATION_ID=3c4238d790a44aca9576ecdb2c7576d3 COREDUMP_UNIT=user@1002.service COREDUMP_USER_UNIT=gnome-terminal-server.service COREDUMP_UID=1002 COREDUMP_GID=1002 COREDUMP_OWNER_UID=1002 COREDUMP_SLICE=user-1002.slice COREDUMP_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_LIMITS=Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 15413 15413 processes Max open files 4096 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 15413 15413 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us COREDUMP_PROC_CGROUP=1:name=systemd:/ 0::/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service COREDUMP_PROC_MOUNTINFO=17 39 0:17 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel 18 39 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw 19 39 0:6 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1972980k,nr_inodes=493245,mode=755 20 17 0:18 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw 21 19 0:19 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel 22 19 0:20 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 23 39 0:21 / /run rw,nosuid,nodev shared:12 - tmpfs tmpfs rw,seclabel,mode=755 24 17 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup rw 25 17 0:23 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:9 - pstore pstore rw,seclabel 36 17 0:24 / /sys/kernel/config rw,relatime shared:10 - configfs configfs rw 39 0 0:26 /root / rw,relatime shared:1 - btrfs /dev/mapper/fedora-root2 rw,seclabel,ssd,space_cache,subvolid=257,subvol=/root 26 17 0:16 / /sys/fs/selinux rw,relatime shared:11 - selinuxfs selinuxfs rw 27 19 0:15 / /dev/mqueue rw,relatime shared:13 - mqueue mqueue rw,seclabel 28 18 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:14 - autofs systemd-1 rw,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13663 29 17 0:7 / /sys/kernel/debug rw,relatime shared:15 - debugfs debugfs rw,seclabel 30 19 0:31 / /dev/hugepages rw,relatime shared:16 - hugetlbfs hugetlbfs rw,seclabel 31 18 0:32 / /proc/fs/nfsd rw,relatime shared:17 - nfsd nfsd rw 32 28 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:18 - binfmt_misc binfmt_misc rw 57 39 0:34 / /tmp rw,relatime shared:19 - tmpfs none rw,seclabel 61 57 0:35 / /tmp/test rw,relatime shared:20 - autofs systemd-1 rw,fd=48,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18251 59 39 8:1 / /boot rw,relatime shared:21 - ext4 /dev/sda1 rw,seclabel,data=ordered 60 39 253:2 / /home rw,relatime shared:22 - ext4 /dev/mapper/fedora-home rw,seclabel,data=ordered 65 39 0:37 / /var/lib/nfs/rpc_pipefs rw,relatime shared:23 - rpc_pipefs sunrpc rw 136 23 0:39 / /run/user/1002 rw,nosuid,nodev,relatime shared:91 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1002,gid=1002 211 23 0:41 / /run/user/42 rw,nosuid,nodev,relatime shared:163 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=42,gid=42 329 136 0:44 / /run/user/1002/gvfs rw,nosuid,nodev,relatime shared:277 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1002,group_id=1002 287 61 253:3 / /tmp/test rw,relatime shared:236 - ext4 /dev/mapper/fedora-test rw,seclabel,data=ordered 217 23 0:42 / /run/user/1000 rw,nosuid,nodev,relatime shared:168 - tmpfs tmpfs rw,seclabel,size=397432k,mode=700,uid=1000,gid=1000 225 217 0:43 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:175 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 COREDUMP_ROOT=/ PRIORITY=2 CODE_FILE=src/coredump/coredump.c SYSLOG_IDENTIFIER=lt-systemd-coredump _COMM=lt-systemd-core _SYSTEMD_CGROUP=/user.slice/user-1002.slice/user@1002.service/gnome-terminal-server.service _SYSTEMD_USER_UNIT=gnome-terminal-server.service MESSAGE_ID=1f4e0a44a88649939aaea34fc6da8c95 CODE_FUNC=process_traceback COREDUMP_COMM=python3 COREDUMP_EXE=/usr/bin/python3.5 COREDUMP_CMDLINE=python3 systemd_coredump_exception_handler.py COREDUMP_CWD=/home/zbyszek/src/systemd-coredump-python COREDUMP_RLIMIT=-1 COREDUMP_OPEN_FDS=0:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 1:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 2:/dev/pts/1 pos: 0 flags: 0102002 mnt_id: 22 CODE_LINE=1284 COREDUMP_SIGNAL=ZeroDivisionError: division by zero COREDUMP_ENVIRON=LANG=en_US.utf8 DISPLAY=:0 ... MANWIDTH=90 LC_MESSAGES=en_US.utf8 PYTHONPATH=. _=/usr/bin/python3 COREDUMP_PID=14498 COREDUMP_PROC_STATUS=Name: python3 Umask: 0002 State: S (sleeping) Tgid: 14498 Ngid: 0 Pid: 14498 PPid: 16245 TracerPid: 0 Uid: 1002 1002 1002 1002 Gid: 1002 1002 1002 1002 FDSize: 64 Groups: NStgid: 14498 NSpid: 14498 NSpgid: 14498 NSsid: 16245 VmPeak: 34840 kB VmSize: 34792 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 9332 kB VmRSS: 9332 kB RssAnon: 4872 kB RssFile: 4460 kB RssShmem: 0 kB VmData: 5012 kB VmStk: 136 kB VmExe: 4 kB VmLib: 5452 kB VmPTE: 84 kB VmPMD: 12 kB VmSwap: 0 kB HugetlbPages: 0 kB Threads: 1 SigQ: 0/15413 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000000 SigIgn: 0000000001001000 SigCgt: 0000000180000002 CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000003fffffffff CapAmb: 0000000000000000 Seccomp: 0 Cpus_allowed: f Cpus_allowed_list: 0-3 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 2 nonvoluntary_ctxt_switches: 47 COREDUMP_PROC_MAPS=55cb7b7fe000-55cb7b7ff000 r-xp 00000000 00:1a 5289186 /usr/bin/python3.5 55cb7b9ff000-55cb7ba00000 r--p 00001000 00:1a 5289186 /usr/bin/python3.5 55cb7ba00000-55cb7ba01000 rw-p 00002000 00:1a 5289186 /usr/bin/python3.5 55cb7c007000-55cb7c189000 rw-p 00000000 00:00 0 [heap] 7f4da2d51000-7f4da2d54000 r-xp 00000000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2d54000-7f4da2f53000 ---p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f53000-7f4da2f54000 r--p 00002000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f54000-7f4da2f55000 rw-p 00003000 00:1a 5279150 /usr/lib64/python3.5/lib-dynload/resource.cpython-35m-x86_64-linux-gnu.so 7f4da2f55000-7f4da2f5d000 r-xp 00000000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da2f5d000-7f4da315c000 ---p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315c000-7f4da315d000 r--p 00007000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315d000-7f4da315f000 rw-p 00008000 00:1a 5279143 /usr/lib64/python3.5/lib-dynload/math.cpython-35m-x86_64-linux-gnu.so 7f4da315f000-7f4da319f000 rw-p 00000000 00:00 0 7f4da319f000-7f4da31a4000 r-xp 00000000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da31a4000-7f4da33a3000 ---p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a3000-7f4da33a4000 r--p 00004000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a4000-7f4da33a6000 rw-p 00005000 00:1a 5279151 /usr/lib64/python3.5/lib-dynload/select.cpython-35m-x86_64-linux-gnu.so 7f4da33a6000-7f4da33a9000 r-xp 00000000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da33a9000-7f4da35a8000 ---p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a8000-7f4da35a9000 r--p 00002000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35a9000-7f4da35aa000 rw-p 00003000 00:1a 5279130 /usr/lib64/python3.5/lib-dynload/_posixsubprocess.cpython-35m-x86_64-linux-gnu.so 7f4da35aa000-7f4da362a000 rw-p 00000000 00:00 0 7f4da362a000-7f4da362c000 r-xp 00000000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da362c000-7f4da382b000 ---p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382b000-7f4da382c000 r--p 00001000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382c000-7f4da382e000 rw-p 00002000 00:1a 5279122 /usr/lib64/python3.5/lib-dynload/_heapq.cpython-35m-x86_64-linux-gnu.so 7f4da382e000-7f4da39ee000 rw-p 00000000 00:00 0 7f4da39ee000-7f4da3bab000 r-xp 00000000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3bab000-7f4da3daa000 ---p 001bd000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3daa000-7f4da3dae000 r--p 001bc000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3dae000-7f4da3db0000 rw-p 001c0000 00:1a 4844904 /usr/lib64/libc-2.24.so 7f4da3db0000-7f4da3db4000 rw-p 00000000 00:00 0 7f4da3db4000-7f4da3ebc000 r-xp 00000000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da3ebc000-7f4da40bb000 ---p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bb000-7f4da40bc000 r--p 00107000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bc000-7f4da40bd000 rw-p 00108000 00:1a 4844910 /usr/lib64/libm-2.24.so 7f4da40bd000-7f4da40bf000 r-xp 00000000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da40bf000-7f4da42be000 ---p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42be000-7f4da42bf000 r--p 00001000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42bf000-7f4da42c0000 rw-p 00002000 00:1a 4844928 /usr/lib64/libutil-2.24.so 7f4da42c0000-7f4da42c3000 r-xp 00000000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da42c3000-7f4da44c2000 ---p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c2000-7f4da44c3000 r--p 00002000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c3000-7f4da44c4000 rw-p 00003000 00:1a 4844908 /usr/lib64/libdl-2.24.so 7f4da44c4000-7f4da44dc000 r-xp 00000000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da44dc000-7f4da46dc000 ---p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dc000-7f4da46dd000 r--p 00018000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46dd000-7f4da46de000 rw-p 00019000 00:1a 4844920 /usr/lib64/libpthread-2.24.so 7f4da46de000-7f4da46e2000 rw-p 00000000 00:00 0 7f4da46e2000-7f4da4917000 r-xp 00000000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4917000-7f4da4b17000 ---p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b17000-7f4da4b1c000 r--p 00235000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b1c000-7f4da4b7f000 rw-p 0023a000 00:1a 5277535 /usr/lib64/libpython3.5m.so.1.0 7f4da4b7f000-7f4da4baf000 rw-p 00000000 00:00 0 7f4da4baf000-7f4da4bd4000 r-xp 00000000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4bdf000-7f4da4c10000 rw-p 00000000 00:00 0 7f4da4c10000-7f4da4c61000 r--p 00000000 00:1a 5225117 /usr/lib/locale/pl_PL.utf8/LC_CTYPE 7f4da4c61000-7f4da4d91000 r--p 00000000 00:1a 4844827 /usr/lib/locale/en_US.utf8/LC_COLLATE 7f4da4d91000-7f4da4d95000 rw-p 00000000 00:00 0 7f4da4dc1000-7f4da4dc2000 r--p 00000000 00:1a 4844832 /usr/lib/locale/en_US.utf8/LC_NUMERIC 7f4da4dc2000-7f4da4dc3000 r--p 00000000 00:1a 4844795 /usr/lib/locale/en_US.utf8/LC_TIME 7f4da4dc3000-7f4da4dc4000 r--p 00000000 00:1a 4844793 /usr/lib/locale/en_US.utf8/LC_MONETARY 7f4da4dc4000-7f4da4dc5000 r--p 00000000 00:1a 4844830 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES 7f4da4dc5000-7f4da4dc6000 r--p 00000000 00:1a 4844847 /usr/lib/locale/en_US.utf8/LC_PAPER 7f4da4dc6000-7f4da4dc7000 r--p 00000000 00:1a 4844831 /usr/lib/locale/en_US.utf8/LC_NAME 7f4da4dc7000-7f4da4dc8000 r--p 00000000 00:1a 4844790 /usr/lib/locale/en_US.utf8/LC_ADDRESS 7f4da4dc8000-7f4da4dc9000 r--p 00000000 00:1a 4844794 /usr/lib/locale/en_US.utf8/LC_TELEPHONE 7f4da4dc9000-7f4da4dca000 r--p 00000000 00:1a 4844792 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT 7f4da4dca000-7f4da4dd1000 r--s 00000000 00:1a 4845203 /usr/lib64/gconv/gconv-modules.cache 7f4da4dd1000-7f4da4dd2000 r--p 00000000 00:1a 4844791 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION 7f4da4dd2000-7f4da4dd4000 rw-p 00000000 00:00 0 7f4da4dd4000-7f4da4dd5000 r--p 00025000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd5000-7f4da4dd6000 rw-p 00026000 00:1a 4844897 /usr/lib64/ld-2.24.so 7f4da4dd6000-7f4da4dd7000 rw-p 00000000 00:00 0 7ffd24da1000-7ffd24dc2000 rw-p 00000000 00:00 0 [stack] 7ffd24de8000-7ffd24dea000 r--p 00000000 00:00 0 [vvar] 7ffd24dea000-7ffd24dec000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] COREDUMP_TIMESTAMP=1477877460000000 MESSAGE=Process 14498 (python3) of user 1002 failed with ZeroDivisionError: division by zero: Traceback (most recent call last): File "systemd_coredump_exception_handler.py", line 89, in <module> g() File "systemd_coredump_exception_handler.py", line 88, in g f() File "systemd_coredump_exception_handler.py", line 86, in f div0 = 1 / 0 # pylint: disable=W0612 ZeroDivisionError: division by zero Local variables in innermost frame: h=<function f at 0x7f4da3606e18> a=3 _PID=14499 _SOURCE_REALTIME_TIMESTAMP=1477877460025975
2016-10-31 03:42:44 +01:00
if (r == 0) {
if (streq_ptr(argv[1], "--backtrace"))
r = process_backtrace(argc, argv);
else
r = process_kernel(argc, argv);
} else if (r == 1)
r = process_socket(SD_LISTEN_FDS_START);
else {
log_error("Received unexpected number of file descriptors.");
r = -EINVAL;
}
finish:
return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
}