picnoir
/
Systemd
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

systemd-oomd: manager/daemon

This commit is contained in:
Anita Zhang 2020-03-23 17:32:27 -07:00
parent b41dcc51eb
commit 9de5e32136
9 changed files with 700 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
meson.build
View File

@ -2662,6 +2662,17 @@ if conf.get('ENABLE_PSTORE') == 1
install_dir : rootlibexecdir)
endif
if conf.get('ENABLE_OOMD') == 1
executable('systemd-oomd',
systemd_oomd_sources,
include_directories : includes,
link_with : [libshared],
dependencies : [],
install_rpath : rootlibexecdir,
install : true,
install_dir : rootlibexecdir)
endif
if conf.get('ENABLE_BINFMT') == 1
public_programs += executable(
'systemd-binfmt',

6
src/oom/meson.build
View File

@ -1,8 +1,11 @@
# SPDX-License-Identifier: LGPL-2.1+
systemd_oomd_sources = files('''
oomd-manager.c
oomd-manager.h
oomd-util.c
oomd-util.h
oomd.c
'''.split())
if conf.get('ENABLE_OOMD') == 1
@ -13,4 +16,7 @@ if conf.get('ENABLE_OOMD') == 1
[],
[]]
]
install_data('oomd.conf',
install_dir : pkgsysconfdir)
endif

466
src/oom/oomd-manager.c Normal file
View File

@ -0,0 +1,466 @@
/* SPDX-License-Identifier: LGPL-2.1+ */
#include "cgroup-util.h"
#include "fd-util.h"
#include "fileio.h"
#include "oomd-manager.h"
#include "path-util.h"
typedef struct ManagedOOMReply {
ManagedOOMMode mode;
char *path;
char *property;
unsigned limit;
} ManagedOOMReply;
static void managed_oom_reply_destroy(ManagedOOMReply *reply) {
assert(reply);
free(reply->path);
free(reply->property);
}
static int managed_oom_mode(const char *name, JsonVariant *v, JsonDispatchFlags flags, void *userdata) {
ManagedOOMMode *mode = userdata, m;
const char *s;
assert(mode);
assert_se(s = json_variant_string(v));
m = managed_oom_mode_from_string(s);
if (m < 0)
return json_log(v, flags, SYNTHETIC_ERRNO(EINVAL), "%s is not a valid ManagedOOMMode", s);
*mode = m;
return 0;
}
static int process_managed_oom_reply(
Varlink *link,
JsonVariant *parameters,
const char *error_id,
VarlinkReplyFlags flags,
void *userdata) {
JsonVariant *c, *cgroups;
Manager *m = userdata;
int r = 0;
assert(m);
static const JsonDispatch dispatch_table[] = {
{ "mode", JSON_VARIANT_STRING, managed_oom_mode, offsetof(ManagedOOMReply, mode), JSON_MANDATORY },
{ "path", JSON_VARIANT_STRING, json_dispatch_string, offsetof(ManagedOOMReply, path), JSON_MANDATORY },
{ "property", JSON_VARIANT_STRING, json_dispatch_string, offsetof(ManagedOOMReply, property), JSON_MANDATORY },
{ "limit", JSON_VARIANT_UNSIGNED, json_dispatch_unsigned, offsetof(ManagedOOMReply, limit), 0 },
{},
};
if (error_id) {
r = -EIO;
log_debug("Error getting ManagedOOM cgroups: %s", error_id);
goto finish;
}
cgroups = json_variant_by_key(parameters, "cgroups");
if (!cgroups) {
r = -EINVAL;
goto finish;
}
/* Skip malformed elements and keep processing in case the others are good */
JSON_VARIANT_ARRAY_FOREACH(c, cgroups) {
_cleanup_(managed_oom_reply_destroy) ManagedOOMReply reply = {};
OomdCGroupContext *ctx;
Hashmap *monitor_hm;
loadavg_t limit;
int ret;
if (!json_variant_is_object(c))
continue;
ret = json_dispatch(c, dispatch_table, NULL, 0, &reply);
if (ret == -ENOMEM) {
r = ret;
goto finish;
} else if (ret < 0)
continue;
monitor_hm = streq(reply.property, "ManagedOOMSwap") ?
m->monitored_swap_cgroup_contexts : m->monitored_mem_pressure_cgroup_contexts;
if (reply.mode == MANAGED_OOM_AUTO) {
(void) oomd_cgroup_context_free(hashmap_remove(monitor_hm, reply.path));
continue;
}
limit = m->default_mem_pressure_limit;
if (streq(reply.property, "ManagedOOMMemoryPressure")) {
if (reply.limit > 100)
continue;
else if (reply.limit != 0) {
ret = store_loadavg_fixed_point((unsigned long) reply.limit, 0, &limit);
if (ret < 0)
continue;
}
}
ret = oomd_insert_cgroup_context(NULL, monitor_hm, reply.path);
if (ret == -ENOMEM) {
r = ret;
goto finish;
}
/* Always update the limit in case it was changed. For non-memory pressure detection the value is
* ignored so always updating it here is not a problem. */
ctx = hashmap_get(monitor_hm, reply.path);
if (ctx)
ctx->mem_pressure_limit = limit;
}
finish:
if (!FLAGS_SET(flags, VARLINK_REPLY_CONTINUES))
m->varlink = varlink_close_unref(link);
return r;
}
/* Fill `new_h` with `path`'s descendent OomdCGroupContexts. Only include descendent cgroups that are possible
* candidates for action. That is, only leaf cgroups or cgroups with memory.oom.group set to "1".
*
* This function ignores most errors in order to handle cgroups that may have been cleaned up while populating
* the hashmap.
*
* `new_h` is of the form { key: cgroup paths -> value: OomdCGroupContext } */
static int recursively_get_cgroup_context(Hashmap *new_h, const char *path) {
_cleanup_free_ char *subpath = NULL;
_cleanup_closedir_ DIR *d = NULL;
int r;
assert(new_h);
assert(path);
r = cg_enumerate_subgroups(SYSTEMD_CGROUP_CONTROLLER, path, &d);
if (r < 0)
return r;
r = cg_read_subgroup(d, &subpath);
if (r < 0)
return r;
else if (r == 0) { /* No subgroups? We're a leaf node */
r = oomd_insert_cgroup_context(NULL, new_h, path);
return (r == -ENOMEM) ? r : 0;
}
do {
_cleanup_free_ char *cg_path = NULL;
bool oom_group;
cg_path = path_join(empty_to_root(path), subpath);
if (!cg_path)
return -ENOMEM;
subpath = mfree(subpath);
r = cg_get_attribute_as_bool("memory", cg_path, "memory.oom.group", &oom_group);
/* The cgroup might be gone. Skip it as a candidate since we can't get information on it. */
if (r < 0)
return (r == -ENOMEM) ? r : 0;
if (oom_group) {
r = oomd_insert_cgroup_context(NULL, new_h, cg_path);
if (r == -ENOMEM)
return r;
} else {
r = recursively_get_cgroup_context(new_h, cg_path);
if (r == -ENOMEM)
return r;
}
} while ((r = cg_read_subgroup(d, &subpath)) > 0);
return 0;
}
static int update_monitored_cgroup_contexts(Hashmap **monitored_cgroups) {
_cleanup_hashmap_free_ Hashmap *new_base = NULL;
OomdCGroupContext *ctx;
int r;
assert(monitored_cgroups);
new_base = hashmap_new(&oomd_cgroup_ctx_hash_ops);
if (!new_base)
return -ENOMEM;
HASHMAP_FOREACH(ctx, *monitored_cgroups) {
/* Skip most errors since the cgroup we're trying to update might not exist anymore. */
r = oomd_insert_cgroup_context(*monitored_cgroups, new_base, ctx->path);
if (r == -ENOMEM)
return r;
}
hashmap_free(*monitored_cgroups);
*monitored_cgroups = TAKE_PTR(new_base);
return 0;
}
static int get_monitored_cgroup_contexts_candidates(Hashmap *monitored_cgroups, Hashmap **ret_candidates) {
_cleanup_hashmap_free_ Hashmap *candidates = NULL;
OomdCGroupContext *ctx;
int r;
assert(monitored_cgroups);
assert(ret_candidates);
candidates = hashmap_new(&oomd_cgroup_ctx_hash_ops);
if (!candidates)
return -ENOMEM;
HASHMAP_FOREACH(ctx, monitored_cgroups) {
r = recursively_get_cgroup_context(candidates, ctx->path);
if (r == -ENOMEM)
return r;
}
*ret_candidates = TAKE_PTR(candidates);
return 0;
}
static int acquire_managed_oom_connect(Manager *m) {
_cleanup_(varlink_close_unrefp) Varlink *link = NULL;
int r;
assert(m);
assert(m->event);
r = varlink_connect_address(&link, VARLINK_ADDR_PATH_MANAGED_OOM);
if (r < 0)
return log_error_errno(r, "Failed to connect to %s: %m", VARLINK_ADDR_PATH_MANAGED_OOM);
(void) varlink_set_userdata(link, m);
(void) varlink_set_description(link, "oomd");
(void) varlink_set_relative_timeout(link, USEC_INFINITY);
r = varlink_attach_event(link, m->event, SD_EVENT_PRIORITY_NORMAL);
if (r < 0)
return log_error_errno(r, "Failed to attach varlink connection to event loop: %m");
r = varlink_bind_reply(link, process_managed_oom_reply);
if (r < 0)
return log_error_errno(r, "Failed to bind reply callback: %m");
r = varlink_observe(link, "io.systemd.ManagedOOM.SubscribeManagedOOMCGroups", NULL);
if (r < 0)
return log_error_errno(r, "Failed to observe varlink call: %m");
m->varlink = TAKE_PTR(link);
return 0;
}
static int monitor_cgroup_contexts_handler(sd_event_source *s, uint64_t usec, void *userdata) {
_cleanup_set_free_ Set *targets = NULL;
Manager *m = userdata;
usec_t usec_now;
int r;
assert(s);
assert(userdata);
/* Reset timer */
r = sd_event_now(sd_event_source_get_event(s), CLOCK_MONOTONIC, &usec_now);
if (r < 0)
return log_error_errno(r, "Failed to reset event timer");
r = sd_event_source_set_time_relative(s, INTERVAL_USEC);
if (r < 0)
return log_error_errno(r, "Failed to set relative time for timer");
/* Reconnect if our connection dropped */
if (!m->varlink) {
r = acquire_managed_oom_connect(m);
if (r < 0)
return log_error_errno(r, "Failed to acquire varlink connection");
}
/* Update the cgroups used for detection/action */
r = update_monitored_cgroup_contexts(&m->monitored_swap_cgroup_contexts);
if (r == -ENOMEM)
return log_error_errno(r, "Failed to update monitored swap cgroup contexts");
r = update_monitored_cgroup_contexts(&m->monitored_mem_pressure_cgroup_contexts);
if (r == -ENOMEM)
return log_error_errno(r, "Failed to update monitored memory pressure cgroup contexts");
r = oomd_system_context_acquire("/proc/swaps", &m->system_context);
/* If there aren't units depending on swap actions, the only error we exit on is ENOMEM */
if (r == -ENOMEM || (r < 0 && !hashmap_isempty(m->monitored_swap_cgroup_contexts)))
return log_error_errno(r, "Failed to acquire system context");
/* If we're still recovering from a kill, don't try to kill again yet */
if (m->post_action_delay_start > 0) {
if (m->post_action_delay_start + POST_ACTION_DELAY_USEC > usec_now)
return 0;
else
m->post_action_delay_start = 0;
}
r = oomd_pressure_above(m->monitored_mem_pressure_cgroup_contexts, PRESSURE_DURATION_USEC, &targets);
if (r == -ENOMEM)
return log_error_errno(r, "Failed to check if memory pressure exceeded limits");
else if (r == 1) {
/* Check if there was reclaim activity in the last interval. The concern is the following case:
* Pressure climbed, a lot of high-frequency pages were reclaimed, and we killed the offending
* cgroup. Even after this, well-behaved processes will fault in recently resident pages and
* this will cause pressure to remain high. Thus if there isn't any reclaim pressure, no need
* to kill something (it won't help anyways). */
if (oomd_memory_reclaim(m->monitored_mem_pressure_cgroup_contexts)) {
_cleanup_hashmap_free_ Hashmap *candidates = NULL;
OomdCGroupContext *t;
r = get_monitored_cgroup_contexts_candidates(m->monitored_mem_pressure_cgroup_contexts, &candidates);
if (r == -ENOMEM)
return log_error_errno(r, "Failed to get monitored memory pressure cgroup candidates");
SET_FOREACH(t, targets) {
log_notice("Memory pressure for %s is greater than %lu for more than %"PRIu64" seconds and there was reclaim activity",
t->path, LOAD_INT(t->mem_pressure_limit), PRESSURE_DURATION_USEC / USEC_PER_SEC);
r = oomd_kill_by_pgscan(candidates, t->path, m->dry_run);
if (r == -ENOMEM)
return log_error_errno(r, "Failed to kill cgroup processes by pgscan");
if (r < 0)
log_info("Failed to kill any cgroup(s) under %s based on pressure", t->path);
else {
/* Don't act on all the high pressure cgroups at once; return as soon as we kill one */
m->post_action_delay_start = usec_now;
return 0;
}
}
}
}
if (oomd_swap_free_below(&m->system_context, (100 - m->swap_used_limit))) {
_cleanup_hashmap_free_ Hashmap *candidates = NULL;
log_notice("Swap used (%"PRIu64") / total (%"PRIu64") is more than %u%%",
m->system_context.swap_used, m->system_context.swap_total, m->swap_used_limit);
r = get_monitored_cgroup_contexts_candidates(m->monitored_swap_cgroup_contexts, &candidates);
if (r == -ENOMEM)
return log_error_errno(r, "Failed to get monitored swap cgroup candidates");
r = oomd_kill_by_swap_usage(candidates, m->dry_run);
if (r == -ENOMEM)
return log_error_errno(r, "Failed to kill cgroup processes by swap usage");
if (r < 0)
log_info("Failed to kill any cgroup(s) based on swap");
else {
m->post_action_delay_start = usec_now;
return 0;
}
}
return 0;
}
static int monitor_cgroup_contexts(Manager *m) {
_cleanup_(sd_event_source_unrefp) sd_event_source *s = NULL;
int r;
assert(m);
assert(m->event);
r = sd_event_add_time(m->event, &s, CLOCK_MONOTONIC, 0, 0, monitor_cgroup_contexts_handler, m);
if (r < 0)
return r;
r = sd_event_source_set_exit_on_failure(s, true);
if (r < 0)
return r;
r = sd_event_source_set_enabled(s, SD_EVENT_ON);
if (r < 0)
return r;
(void) sd_event_source_set_description(s, "oomd-timer");
m->cgroup_context_event_source = TAKE_PTR(s);
return 0;
}
void manager_free(Manager *m) {
assert(m);
varlink_close_unref(m->varlink);
sd_event_source_unref(m->cgroup_context_event_source);
sd_event_unref(m->event);
hashmap_free(m->monitored_swap_cgroup_contexts);
hashmap_free(m->monitored_mem_pressure_cgroup_contexts);
free(m);
}
int manager_new(Manager **ret) {
_cleanup_(manager_freep) Manager *m = NULL;
int r;
assert(ret);
m = new0(Manager, 1);
if (!m)
return -ENOMEM;
r = sd_event_default(&m->event);
if (r < 0)
return r;
(void) sd_event_set_watchdog(m->event, true);
r = sd_event_add_signal(m->event, NULL, SIGINT, NULL, NULL);
if (r < 0)
return r;
r = sd_event_add_signal(m->event, NULL, SIGTERM, NULL, NULL);
if (r < 0)
return r;
m->monitored_swap_cgroup_contexts = hashmap_new(&oomd_cgroup_ctx_hash_ops);
if (!m->monitored_swap_cgroup_contexts)
return -ENOMEM;
m->monitored_mem_pressure_cgroup_contexts = hashmap_new(&oomd_cgroup_ctx_hash_ops);
if (!m->monitored_mem_pressure_cgroup_contexts)
return -ENOMEM;
*ret = TAKE_PTR(m);
return 0;
}
int manager_start(Manager *m, bool dry_run, int swap_used_limit, int mem_pressure_limit) {
unsigned long l;
int r;
assert(m);
m->dry_run = dry_run;
m->swap_used_limit = swap_used_limit != -1 ? swap_used_limit : DEFAULT_SWAP_USED_LIMIT;
assert(m->swap_used_limit <= 100);
l = mem_pressure_limit != -1 ? mem_pressure_limit : DEFAULT_MEM_PRESSURE_LIMIT;
r = store_loadavg_fixed_point(l, 0, &m->default_mem_pressure_limit);
if (r < 0)
return r;
r = acquire_managed_oom_connect(m);
if (r < 0)
return r;
r = monitor_cgroup_contexts(m);
if (r < 0)
return r;
return 0;
}

53
src/oom/oomd-manager.h Normal file
View File

@ -0,0 +1,53 @@
/* SPDX-License-Identifier: LGPL-2.1+ */
#pragma once
#include "conf-parser.h"
#include "oomd-util.h"
#include "sd-event.h"
#include "varlink.h"
/* Polling interval for monitoring stats */
#define INTERVAL_USEC (1 * USEC_PER_SEC)
/* Used to weight the averages */
#define AVERAGE_SIZE_DECAY 4
/* Take action if 10s of memory pressure > 60 for more than 30s. We use the "full" value from PSI so this is the
* percentage of time all tasks were delayed (i.e. unproductive).
* Generally 60 or higher might be acceptable for something like system.slice with no memory.high set; processes in
* system.slice are assumed to be less latency sensitive. */
#define PRESSURE_DURATION_USEC (30 * USEC_PER_SEC)
#define DEFAULT_MEM_PRESSURE_LIMIT 60
#define DEFAULT_SWAP_USED_LIMIT 90
#define POST_ACTION_DELAY_USEC (15 * USEC_PER_SEC)
typedef struct Manager Manager;
struct Manager {
sd_event *event;
bool dry_run;
unsigned swap_used_limit;
loadavg_t default_mem_pressure_limit;
/* k: cgroup paths -> v: OomdCGroupContext
* Used to detect when to take action. */
Hashmap *monitored_swap_cgroup_contexts;
Hashmap *monitored_mem_pressure_cgroup_contexts;
OomdSystemContext system_context;
usec_t post_action_delay_start;
sd_event_source *cgroup_context_event_source;
Varlink *varlink;
};
void manager_free(Manager *m);
DEFINE_TRIVIAL_CLEANUP_FUNC(Manager*, manager_free);
int manager_new(Manager **ret);
int manager_start(Manager *m, bool dry_run, int swap_used_limit, int mem_pressure_limit);

1
src/oom/oomd-util.c
View File

@ -3,6 +3,7 @@
#include <sys/xattr.h>
#include <unistd.h>
#include "cgroup-util.h"
#include "fd-util.h"
#include "format-util.h"
#include "oomd-util.h"

144
src/oom/oomd.c Normal file
View File

@ -0,0 +1,144 @@
/* SPDX-License-Identifier: LGPL-2.1+ */
#include <getopt.h>
#include "cgroup-util.h"
#include "conf-parser.h"
#include "daemon-util.h"
#include "log.h"
#include "main-func.h"
#include "oomd-manager.h"
#include "parse-util.h"
#include "pretty-print.c"
#include "psi-util.h"
#include "signal-util.h"
static bool arg_dry_run = false;
static int arg_swap_used_limit = -1;
static int arg_mem_pressure_limit = -1;
static int parse_config(void) {
static const ConfigTableItem items[] = {
{ "OOM", "SwapUsedLimitPercent", config_parse_percent, 0, &arg_swap_used_limit },
{ "OOM", "DefaultMemoryPressureLimitPercent", config_parse_percent, 0, &arg_mem_pressure_limit },
{}
};
return config_parse_many_nulstr(PKGSYSCONFDIR "/oomd.conf",
CONF_PATHS_NULSTR("systemd/oomd.conf.d"),
"OOM\0",
config_item_table_lookup,
items,
CONFIG_PARSE_WARN,
NULL,
NULL);
}
static int help(void) {
_cleanup_free_ char *link = NULL;
int r;
r = terminal_urlify_man("systemd-oomd", "1", &link);
if (r < 0)
return log_oom();
printf("%s [OPTIONS...]\n\n"
"Run the userspace out-of-memory (OOM) killer.\n\n"
" -h --help Show this help\n"
" --dry-run Log write/destructive actions instead of doing them\n"
"\nSee the %s for details.\n"
, program_invocation_short_name
, link
);
return 0;
}
static int parse_argv(int argc, char *argv[]) {
enum {
ARG_DRY_RUN,
};
static const struct option options[] = {
{ "help", no_argument, NULL, 'h' },
{ "dry-run", no_argument, NULL, ARG_DRY_RUN },
{}
};
int c;
assert(argc >= 0);
assert(argv);
while ((c = getopt_long(argc, argv, "h", options, NULL)) >= 0)
switch (c) {
case 'h':
return help();
case ARG_DRY_RUN:
arg_dry_run = true;
break;
case '?':
return -EINVAL;
default:
assert_not_reached("Invalid option passed.");
}
return 1;
}
static int run(int argc, char *argv[]) {
_cleanup_(notify_on_cleanup) const char *notify_msg = NULL;
_cleanup_(manager_freep) Manager *m = NULL;
int r;
log_setup_service();
r = parse_argv(argc, argv);
if (r <= 0)
return r;
r = parse_config();
if (r < 0)
return r;
/* Do some basic requirement checks for running systemd-oomd. It's not exhaustive as some of the other
* requirements do not have a reliable means to check for in code. */
if (access("/proc/swaps", F_OK) < 0)
return log_error_errno(errno, "Swap not enabled: %m");
if (!is_pressure_supported())
return log_error_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), "Pressure Stall Information (PSI) is not supported");
r = cg_all_unified();
if (r < 0)
return log_error_errno(r, "Failed to determine whether the unified cgroups hierarchy is used: %m");
if (r == 0)
return log_error_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), "Requires the unified cgroups hierarchy");
assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0);
r = manager_new(&m);
if (r < 0)
return log_error_errno(r, "Failed to create manager: %m");
r = manager_start(m, arg_dry_run, arg_swap_used_limit, arg_mem_pressure_limit);
if (r < 0)
return log_error_errno(r, "Failed to start up daemon: %m");
notify_msg = notify_start(NOTIFY_READY, NOTIFY_STOPPING);
log_info("systemd-oomd starting%s!", arg_dry_run ? " in dry run mode" : "");
r = sd_event_loop(m->event);
if (r < 0)
return log_error_errno(r, "Event loop failed: %m");
return 0;
}
DEFINE_MAIN_FUNCTION(run);

16
src/oom/oomd.conf Normal file
View File

@ -0,0 +1,16 @@
# This file is part of systemd.
#
# systemd is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# Entries in this file show the compile time defaults.
# You can change settings by editing this file.
# Defaults can be restored by simply deleting this file.
#
# See oomd.conf(5) for details
[OOM]
#SwapUsedLimitPercent=90%
#DefaultMemoryPressureLimitPercent=60%

2
src/shared/conf-parser.c
View File

@ -1243,3 +1243,5 @@ int config_parse_vlanprotocol(const char* unit,
return 0;
}
DEFINE_CONFIG_PARSE(config_parse_percent, parse_percent, "Failed to parse percent value");

1
src/shared/conf-parser.h
View File

@ -147,6 +147,7 @@ CONFIG_PARSER_PROTOTYPE(config_parse_ip_port);
CONFIG_PARSER_PROTOTYPE(config_parse_mtu);
CONFIG_PARSER_PROTOTYPE(config_parse_rlimit);
CONFIG_PARSER_PROTOTYPE(config_parse_vlanprotocol);
CONFIG_PARSER_PROTOTYPE(config_parse_percent);
typedef enum Disabled {
DISABLED_CONFIGURATION,