Systemd/src/import/import-dkr.c
Lennart Poettering 3d7415f43f import: introduce new mini-daemon systemd-importd, and make machinectl a client to it
The old "systemd-import" binary is now an internal tool. We still use it
as asynchronous backend for systemd-importd. Since the import tool might
require some IO and CPU resources (due to qcow2 explosion, and
decompression), and because we might want to run it with more minimal
priviliges we still keep it around as the worker binary to execute as
child process of importd.

machinectl now has verbs for pulling down images, cancelling them and
listing them.
2015-01-22 04:02:07 +01:00

858 lines
25 KiB
C

/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
/***
This file is part of systemd.
Copyright 2014 Lennart Poettering
systemd is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
systemd is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/
#include <curl/curl.h>
#include <sys/prctl.h>
#include "set.h"
#include "json.h"
#include "strv.h"
#include "btrfs-util.h"
#include "utf8.h"
#include "mkdir.h"
#include "import-util.h"
#include "curl-util.h"
#include "aufs-util.h"
#include "import-job.h"
#include "import-common.h"
#include "import-dkr.h"
struct DkrImport {
sd_event *event;
CurlGlue *glue;
char *index_url;
char *image_root;
ImportJob *images_job;
ImportJob *tags_job;
ImportJob *ancestry_job;
ImportJob *json_job;
ImportJob *layer_job;
char *name;
char *tag;
char *id;
char *response_token;
char **response_registries;
char **ancestry;
unsigned current_ancestry;
DkrImportFinished on_finished;
void *userdata;
char *local;
bool force_local;
char *temp_path;
char *final_path;
pid_t tar_pid;
};
#define PROTOCOL_PREFIX "https://"
#define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:"
#define HEADER_REGISTRY "X-Do" /*the HTTP header for the registry */ "cker-Endpoints:"
#define LAYERS_MAX 2048
static void dkr_import_job_on_finished(ImportJob *j);
DkrImport* dkr_import_unref(DkrImport *i) {
if (!i)
return NULL;
if (i->tar_pid > 1) {
(void) kill_and_sigcont(i->tar_pid, SIGKILL);
(void) wait_for_terminate(i->tar_pid, NULL);
}
import_job_unref(i->images_job);
import_job_unref(i->tags_job);
import_job_unref(i->ancestry_job);
import_job_unref(i->json_job);
import_job_unref(i->layer_job);
curl_glue_unref(i->glue);
sd_event_unref(i->event);
if (i->temp_path) {
(void) btrfs_subvol_remove(i->temp_path);
(void) rm_rf_dangerous(i->temp_path, false, true, false);
free(i->temp_path);
}
free(i->name);
free(i->tag);
free(i->id);
free(i->response_token);
free(i->response_registries);
strv_free(i->ancestry);
free(i->final_path);
free(i->index_url);
free(i->image_root);
free(i->local);
free(i);
return NULL;
}
int dkr_import_new(
DkrImport **ret,
sd_event *event,
const char *index_url,
const char *image_root,
DkrImportFinished on_finished,
void *userdata) {
_cleanup_(dkr_import_unrefp) DkrImport *i = NULL;
char *e;
int r;
assert(ret);
assert(index_url);
if (!http_url_is_valid(index_url))
return -EINVAL;
i = new0(DkrImport, 1);
if (!i)
return -ENOMEM;
i->on_finished = on_finished;
i->userdata = userdata;
i->image_root = strdup(image_root ?: "/var/lib/machines");
if (!i->image_root)
return -ENOMEM;
i->index_url = strdup(index_url);
if (!i->index_url)
return -ENOMEM;
e = endswith(i->index_url, "/");
if (e)
*e = 0;
if (event)
i->event = sd_event_ref(event);
else {
r = sd_event_default(&i->event);
if (r < 0)
return r;
}
r = curl_glue_new(&i->glue, i->event);
if (r < 0)
return r;
i->glue->on_finished = import_job_curl_on_finished;
i->glue->userdata = i;
*ret = i;
i = NULL;
return 0;
}
static int parse_id(const void *payload, size_t size, char **ret) {
_cleanup_free_ char *buf = NULL, *id = NULL, *other = NULL;
union json_value v = {};
void *json_state = NULL;
const char *p;
int t;
assert(payload);
assert(ret);
if (size <= 0)
return -EBADMSG;
if (memchr(payload, 0, size))
return -EBADMSG;
buf = strndup(payload, size);
if (!buf)
return -ENOMEM;
p = buf;
t = json_tokenize(&p, &id, &v, &json_state, NULL);
if (t < 0)
return t;
if (t != JSON_STRING)
return -EBADMSG;
t = json_tokenize(&p, &other, &v, &json_state, NULL);
if (t < 0)
return t;
if (t != JSON_END)
return -EBADMSG;
if (!dkr_id_is_valid(id))
return -EBADMSG;
*ret = id;
id = NULL;
return 0;
}
static int parse_ancestry(const void *payload, size_t size, char ***ret) {
_cleanup_free_ char *buf = NULL;
void *json_state = NULL;
const char *p;
enum {
STATE_BEGIN,
STATE_ITEM,
STATE_COMMA,
STATE_END,
} state = STATE_BEGIN;
_cleanup_strv_free_ char **l = NULL;
size_t n = 0, allocated = 0;
if (size <= 0)
return -EBADMSG;
if (memchr(payload, 0, size))
return -EBADMSG;
buf = strndup(payload, size);
if (!buf)
return -ENOMEM;
p = buf;
for (;;) {
_cleanup_free_ char *str;
union json_value v = {};
int t;
t = json_tokenize(&p, &str, &v, &json_state, NULL);
if (t < 0)
return t;
switch (state) {
case STATE_BEGIN:
if (t == JSON_ARRAY_OPEN)
state = STATE_ITEM;
else
return -EBADMSG;
break;
case STATE_ITEM:
if (t == JSON_STRING) {
if (!dkr_id_is_valid(str))
return -EBADMSG;
if (n+1 > LAYERS_MAX)
return -EFBIG;
if (!GREEDY_REALLOC(l, allocated, n + 2))
return -ENOMEM;
l[n++] = str;
str = NULL;
l[n] = NULL;
state = STATE_COMMA;
} else if (t == JSON_ARRAY_CLOSE)
state = STATE_END;
else
return -EBADMSG;
break;
case STATE_COMMA:
if (t == JSON_COMMA)
state = STATE_ITEM;
else if (t == JSON_ARRAY_CLOSE)
state = STATE_END;
else
return -EBADMSG;
break;
case STATE_END:
if (t == JSON_END) {
if (strv_isempty(l))
return -EBADMSG;
if (!strv_is_uniq(l))
return -EBADMSG;
l = strv_reverse(l);
*ret = l;
l = NULL;
return 0;
} else
return -EBADMSG;
}
}
}
static const char *dkr_import_current_layer(DkrImport *i) {
assert(i);
if (strv_isempty(i->ancestry))
return NULL;
return i->ancestry[i->current_ancestry];
}
static const char *dkr_import_current_base_layer(DkrImport *i) {
assert(i);
if (strv_isempty(i->ancestry))
return NULL;
if (i->current_ancestry <= 0)
return NULL;
return i->ancestry[i->current_ancestry-1];
}
static int dkr_import_add_token(DkrImport *i, ImportJob *j) {
const char *t;
assert(i);
assert(j);
if (i->response_token)
t = strappenda("Authorization: Token ", i->response_token);
else
t = HEADER_TOKEN " true";
j->request_header = curl_slist_new("Accept: application/json", t, NULL);
if (!j->request_header)
return -ENOMEM;
return 0;
}
static bool dkr_import_is_done(DkrImport *i) {
assert(i);
assert(i->images_job);
if (i->images_job->state != IMPORT_JOB_DONE)
return false;
if (!i->tags_job || i->tags_job->state != IMPORT_JOB_DONE)
return false;
if (!i->ancestry_job || i->ancestry_job->state != IMPORT_JOB_DONE)
return false;
if (!i->json_job || i->json_job->state != IMPORT_JOB_DONE)
return false;
if (i->layer_job && i->layer_job->state != IMPORT_JOB_DONE)
return false;
if (dkr_import_current_layer(i))
return false;
return true;
}
static int dkr_import_make_local_copy(DkrImport *i) {
int r;
assert(i);
if (!i->local)
return 0;
if (!i->final_path) {
i->final_path = strjoin(i->image_root, "/.dkr-", i->id, NULL);
if (!i->final_path)
return log_oom();
}
r = import_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
if (r < 0)
return r;
return 0;
}
static int dkr_import_job_on_open_disk(ImportJob *j) {
_cleanup_close_pair_ int pipefd[2] = { -1, -1 };
const char *base;
DkrImport *i;
int r;
assert(j);
assert(j->userdata);
i = j->userdata;
assert(i->layer_job == j);
assert(i->final_path);
assert(!i->temp_path);
assert(i->tar_pid <= 0);
r = tempfn_random(i->final_path, &i->temp_path);
if (r < 0)
return log_oom();
mkdir_parents_label(i->temp_path, 0700);
base = dkr_import_current_base_layer(i);
if (base) {
const char *base_path;
base_path = strappenda(i->image_root, "/.dkr-", base);
r = btrfs_subvol_snapshot(base_path, i->temp_path, false, true);
} else
r = btrfs_subvol_make(i->temp_path);
if (r < 0)
return log_error_errno(r, "Failed to make btrfs subvolume %s: %m", i->temp_path);
if (pipe2(pipefd, O_CLOEXEC) < 0)
return log_error_errno(errno, "Failed to create pipe for tar: %m");
i->tar_pid = fork();
if (i->tar_pid < 0)
return log_error_errno(errno, "Failed to fork off tar: %m");
if (i->tar_pid == 0) {
int null_fd;
/* Child */
reset_all_signal_handlers();
reset_signal_mask();
assert_se(prctl(PR_SET_PDEATHSIG, SIGTERM) == 0);
pipefd[1] = safe_close(pipefd[1]);
if (dup2(pipefd[0], STDIN_FILENO) != STDIN_FILENO) {
log_error_errno(errno, "Failed to dup2() fd: %m");
_exit(EXIT_FAILURE);
}
if (pipefd[0] != STDIN_FILENO)
pipefd[0] = safe_close(pipefd[0]);
null_fd = open("/dev/null", O_WRONLY|O_NOCTTY);
if (null_fd < 0) {
log_error_errno(errno, "Failed to open /dev/null: %m");
_exit(EXIT_FAILURE);
}
if (dup2(null_fd, STDOUT_FILENO) != STDOUT_FILENO) {
log_error_errno(errno, "Failed to dup2() fd: %m");
_exit(EXIT_FAILURE);
}
if (null_fd != STDOUT_FILENO)
null_fd = safe_close(null_fd);
fd_cloexec(STDIN_FILENO, false);
fd_cloexec(STDOUT_FILENO, false);
fd_cloexec(STDERR_FILENO, false);
execlp("tar", "tar", "--numeric-owner", "-C", i->temp_path, "-px", NULL);
log_error_errno(errno, "Failed to execute tar: %m");
_exit(EXIT_FAILURE);
}
pipefd[0] = safe_close(pipefd[0]);
j->disk_fd = pipefd[1];
pipefd[1] = -1;
return 0;
}
static int dkr_import_pull_layer(DkrImport *i) {
_cleanup_free_ char *path = NULL;
const char *url, *layer = NULL;
int r;
assert(i);
assert(!i->layer_job);
assert(!i->temp_path);
assert(!i->final_path);
for (;;) {
layer = dkr_import_current_layer(i);
if (!layer)
return 0; /* no more layers */
path = strjoin(i->image_root, "/.dkr-", layer, NULL);
if (!path)
return log_oom();
if (laccess(path, F_OK) < 0) {
if (errno == ENOENT)
break;
return log_error_errno(errno, "Failed to check for container: %m");
}
log_info("Layer %s already exists, skipping.", layer);
i->current_ancestry++;
free(path);
path = NULL;
}
log_info("Pulling layer %s...", layer);
i->final_path = path;
path = NULL;
url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", layer, "/layer");
r = import_job_new(&i->layer_job, url, i->glue, i);
if (r < 0)
return log_error_errno(r, "Failed to allocate layer job: %m");
r = dkr_import_add_token(i, i->layer_job);
if (r < 0)
return log_oom();
i->layer_job->on_finished = dkr_import_job_on_finished;
i->layer_job->on_open_disk = dkr_import_job_on_open_disk;
r = import_job_begin(i->layer_job);
if (r < 0)
return log_error_errno(r, "Failed to start layer job: %m");
return 0;
}
static void dkr_import_job_on_finished(ImportJob *j) {
DkrImport *i;
int r;
assert(j);
assert(j->userdata);
i = j->userdata;
if (j->error != 0) {
if (j == i->images_job)
log_error_errno(j->error, "Failed to retrieve images list. (Wrong index URL?)");
else if (j == i->tags_job)
log_error_errno(j->error, "Failed to retrieve tags list.");
else if (j == i->ancestry_job)
log_error_errno(j->error, "Failed to retrieve ancestry list.");
else if (j == i->json_job)
log_error_errno(j->error, "Failed to retrieve json data.");
else
log_error_errno(j->error, "Failed to retrieve layer data.");
r = j->error;
goto finish;
}
if (i->images_job == j) {
const char *url;
assert(!i->tags_job);
assert(!i->ancestry_job);
assert(!i->json_job);
assert(!i->layer_job);
if (strv_isempty(i->response_registries)) {
r = -EBADMSG;
log_error("Didn't get registry information.");
goto finish;
}
log_info("Index lookup succeeded, directed to registry %s.", i->response_registries[0]);
url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/repositories/", i->name, "/tags/", i->tag);
r = import_job_new(&i->tags_job, url, i->glue, i);
if (r < 0) {
log_error_errno(r, "Failed to allocate tags job: %m");
goto finish;
}
r = dkr_import_add_token(i, i->tags_job);
if (r < 0) {
log_oom();
goto finish;
}
i->tags_job->on_finished = dkr_import_job_on_finished;
r = import_job_begin(i->tags_job);
if (r < 0) {
log_error_errno(r, "Failed to start tags job: %m");
goto finish;
}
} else if (i->tags_job == j) {
const char *url;
char *id = NULL;
assert(!i->ancestry_job);
assert(!i->json_job);
assert(!i->layer_job);
r = parse_id(j->payload, j->payload_size, &id);
if (r < 0) {
log_error_errno(r, "Failed to parse JSON id.");
goto finish;
}
free(i->id);
i->id = id;
log_info("Tag lookup succeeded, resolved to layer %s.", i->id);
url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/ancestry");
r = import_job_new(&i->ancestry_job, url, i->glue, i);
if (r < 0) {
log_error_errno(r, "Failed to allocate ancestry job: %m");
goto finish;
}
r = dkr_import_add_token(i, i->ancestry_job);
if (r < 0) {
log_oom();
goto finish;
}
i->ancestry_job->on_finished = dkr_import_job_on_finished;
url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/json");
r = import_job_new(&i->json_job, url, i->glue, i);
if (r < 0) {
log_error_errno(r, "Failed to allocate json job: %m");
goto finish;
}
r = dkr_import_add_token(i, i->json_job);
if (r < 0) {
log_oom();
goto finish;
}
i->json_job->on_finished = dkr_import_job_on_finished;
r = import_job_begin(i->ancestry_job);
if (r < 0) {
log_error_errno(r, "Failed to start ancestry job: %m");
goto finish;
}
r = import_job_begin(i->json_job);
if (r < 0) {
log_error_errno(r, "Failed to start json job: %m");
goto finish;
}
} else if (i->ancestry_job == j) {
char **ancestry = NULL, **k;
unsigned n;
assert(!i->layer_job);
r = parse_ancestry(j->payload, j->payload_size, &ancestry);
if (r < 0) {
log_error_errno(r, "Failed to parse JSON id.");
goto finish;
}
n = strv_length(ancestry);
if (n <= 0 || !streq(ancestry[n-1], i->id)) {
log_error("Ancestry doesn't end in main layer.");
strv_free(ancestry);
r = -EBADMSG;
goto finish;
}
log_info("Ancestor lookup succeeded, requires layers:\n");
STRV_FOREACH(k, ancestry)
log_info("\t%s", *k);
strv_free(i->ancestry);
i->ancestry = ancestry;
i->current_ancestry = 0;
r = dkr_import_pull_layer(i);
if (r < 0)
goto finish;
} else if (i->layer_job == j) {
assert(i->temp_path);
assert(i->final_path);
j->disk_fd = safe_close(j->disk_fd);
if (i->tar_pid > 0) {
r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
i->tar_pid = 0;
if (r < 0)
goto finish;
}
r = aufs_resolve(i->temp_path);
if (r < 0) {
log_error_errno(r, "Failed to resolve aufs whiteouts: %m");
goto finish;
}
r = btrfs_subvol_set_read_only(i->temp_path, true);
if (r < 0) {
log_error_errno(r, "Failed to mark snapshort read-only: %m");
goto finish;
}
if (rename(i->temp_path, i->final_path) < 0) {
log_error_errno(errno, "Failed to rename snaphsot: %m");
goto finish;
}
log_info("Completed writing to layer %s.", i->final_path);
i->layer_job = import_job_unref(i->layer_job);
free(i->temp_path);
i->temp_path = NULL;
free(i->final_path);
i->final_path = NULL;
i->current_ancestry ++;
r = dkr_import_pull_layer(i);
if (r < 0)
goto finish;
} else if (i->json_job != j)
assert_not_reached("Got finished event for unknown curl object");
if (!dkr_import_is_done(i))
return;
r = dkr_import_make_local_copy(i);
if (r < 0)
goto finish;
r = 0;
finish:
if (i->on_finished)
i->on_finished(i, r, i->userdata);
else
sd_event_exit(i->event, r);
}
static int dkr_import_job_on_header(ImportJob *j, const char *header, size_t sz) {
_cleanup_free_ char *registry = NULL;
char *token;
DkrImport *i;
int r;
assert(j);
assert(j->userdata);
i = j->userdata;
r = curl_header_strdup(header, sz, HEADER_TOKEN, &token);
if (r < 0)
return log_oom();
if (r > 0) {
free(i->response_token);
i->response_token = token;
return 0;
}
r = curl_header_strdup(header, sz, HEADER_REGISTRY, &registry);
if (r < 0)
return log_oom();
if (r > 0) {
char **l, **k;
l = strv_split(registry, ",");
if (!l)
return log_oom();
STRV_FOREACH(k, l) {
if (!hostname_is_valid(*k)) {
log_error("Registry hostname is not valid.");
strv_free(l);
return -EBADMSG;
}
}
strv_free(i->response_registries);
i->response_registries = l;
}
return 0;
}
int dkr_import_pull(DkrImport *i, const char *name, const char *tag, const char *local, bool force_local) {
const char *url;
int r;
assert(i);
if (!dkr_name_is_valid(name))
return -EINVAL;
if (tag && !dkr_tag_is_valid(tag))
return -EINVAL;
if (local && !machine_name_is_valid(local))
return -EINVAL;
if (i->images_job)
return -EBUSY;
if (!tag)
tag = "latest";
r = free_and_strdup(&i->local, local);
if (r < 0)
return r;
i->force_local = force_local;
r = free_and_strdup(&i->name, name);
if (r < 0)
return r;
r = free_and_strdup(&i->tag, tag);
if (r < 0)
return r;
url = strappenda(i->index_url, "/v1/repositories/", name, "/images");
r = import_job_new(&i->images_job, url, i->glue, i);
if (r < 0)
return r;
r = dkr_import_add_token(i, i->images_job);
if (r < 0)
return r;
i->images_job->on_finished = dkr_import_job_on_finished;
i->images_job->on_header = dkr_import_job_on_header;
return import_job_begin(i->images_job);
}