machine: add support for importing containers from plain directories

Fixes: #2728

This is also supposed to be preparation for doing #10234 eventually,
where a very similar operation is requested: instead of importing a tree
to /var/lib/machines it would need to be imported into
/var/lib/portables/.
This commit is contained in:
Lennart Poettering 2018-10-10 21:20:08 +02:00
parent b3cade0c27
commit 1d7579c473
8 changed files with 561 additions and 18 deletions

View File

@ -815,6 +815,15 @@
<command>cancel-transfer</command>.</para></listitem>
</varlistentry>
<varlistentry>
<term><command>import-fs</command> <replaceable>DIRECTORY</replaceable> [<replaceable>NAME</replaceable>]</term>
<listitem><para>Imports a container image stored in a local directory into
<filename>/var/lib/machines/</filename>, operates similar to <command>import-tar</command> or
<command>import-raw</command>, but the first argument is the source directory. If supported, this command will
create btrfs snapshot or subvolume for the new image.</para></listitem>
</varlistentry>
<varlistentry>
<term><command>export-tar</command> <replaceable>NAME</replaceable> [<replaceable>FILE</replaceable>]</term>
<term><command>export-raw</command> <replaceable>NAME</replaceable> [<replaceable>FILE</replaceable>]</term>

View File

@ -227,6 +227,7 @@ conf.set_quoted('ROOTLIBEXECDIR', rootlibexecdir)
conf.set_quoted('BOOTLIBDIR', bootlibdir)
conf.set_quoted('SYSTEMD_PULL_PATH', join_paths(rootlibexecdir, 'systemd-pull'))
conf.set_quoted('SYSTEMD_IMPORT_PATH', join_paths(rootlibexecdir, 'systemd-import'))
conf.set_quoted('SYSTEMD_IMPORT_FS_PATH', join_paths(rootlibexecdir, 'systemd-import-fs'))
conf.set_quoted('SYSTEMD_EXPORT_PATH', join_paths(rootlibexecdir, 'systemd-export'))
conf.set_quoted('VENDOR_KEYRING_PATH', join_paths(rootlibexecdir, 'import-pubring.gpg'))
conf.set_quoted('USER_KEYRING_PATH', join_paths(pkgsysconfdir, 'import-pubring.gpg'))
@ -2126,6 +2127,14 @@ if conf.get('ENABLE_IMPORTD') == 1
install : true,
install_dir : rootlibexecdir)
systemd_import_fs = executable('systemd-import-fs',
systemd_import_fs_sources,
include_directories : includes,
link_with : [libshared],
install_rpath : rootlibexecdir,
install : true,
install_dir : rootlibexecdir)
systemd_export = executable('systemd-export',
systemd_export_sources,
include_directories : includes,
@ -2137,7 +2146,8 @@ if conf.get('ENABLE_IMPORTD') == 1
install_rpath : rootlibexecdir,
install : true,
install_dir : rootlibexecdir)
public_programs += [systemd_pull, systemd_import, systemd_export]
public_programs += [systemd_pull, systemd_import, systemd_import_fs, systemd_export]
endif
if conf.get('ENABLE_REMOTE') == 1 and conf.get('HAVE_LIBCURL') == 1

View File

@ -22,3 +22,11 @@ static inline void rm_rf_physical_and_free(char *p) {
free(p);
}
DEFINE_TRIVIAL_CLEANUP_FUNC(char*, rm_rf_physical_and_free);
/* Similar as above, but also has magic btrfs subvolume powers */
static inline void rm_rf_subvolume_and_free(char *p) {
PROTECT_ERRNO;
(void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
free(p);
}
DEFINE_TRIVIAL_CLEANUP_FUNC(char*, rm_rf_subvolume_and_free);

323
src/import/import-fs.c Normal file
View File

@ -0,0 +1,323 @@
/* SPDX-License-Identifier: LGPL-2.1+ */
#include <getopt.h>
#include "alloc-util.h"
#include "btrfs-util.h"
#include "fd-util.h"
#include "fileio.h"
#include "fs-util.h"
#include "hostname-util.h"
#include "import-common.h"
#include "import-util.h"
#include "machine-image.h"
#include "mkdir.h"
#include "ratelimit.h"
#include "rm-rf.h"
#include "string-util.h"
#include "verbs.h"
#include "parse-util.h"
static bool arg_force = false;
static bool arg_read_only = false;
static const char *arg_image_root = "/var/lib/machines";
typedef struct ProgressInfo {
RateLimit limit;
char *path;
uint64_t size;
bool started;
bool logged_incomplete;
} ProgressInfo;
static volatile sig_atomic_t cancelled = false;
static void sigterm_sigint(int sig) {
cancelled = true;
}
static void progress_info_free(ProgressInfo *p) {
free(p->path);
}
static void progress_show(ProgressInfo *p) {
assert(p);
/* Show progress only every now and then. */
if (!ratelimit_below(&p->limit))
return;
/* Suppress the first message, start with the second one */
if (!p->started) {
p->started = true;
return;
}
/* Mention the list is incomplete before showing first output. */
if (!p->logged_incomplete) {
log_notice("(Note, file list shown below is incomplete, and is intended as sporadic progress report only.)");
p->logged_incomplete = true;
}
if (p->size == 0)
log_info("Copying tree, currently at '%s'...", p->path);
else {
char buffer[FORMAT_BYTES_MAX];
log_info("Copying tree, currently at '%s' (@%s)...", p->path, format_bytes(buffer, sizeof(buffer), p->size));
}
}
static int progress_path(const char *path, const struct stat *st, void *userdata) {
ProgressInfo *p = userdata;
int r;
assert(p);
if (cancelled)
return -EOWNERDEAD;
r = free_and_strdup(&p->path, path);
if (r < 0)
return r;
p->size = 0;
progress_show(p);
return 0;
}
static int progress_bytes(uint64_t nbytes, void *userdata) {
ProgressInfo *p = userdata;
assert(p);
assert(p->size != UINT64_MAX);
if (cancelled)
return -EOWNERDEAD;
p->size += nbytes;
progress_show(p);
return 0;
}
static int import_fs(int argc, char *argv[], void *userdata) {
_cleanup_(rm_rf_subvolume_and_freep) char *temp_path = NULL;
_cleanup_(progress_info_free) ProgressInfo progress = {};
const char *path = NULL, *local = NULL, *final_path;
_cleanup_close_ int open_fd = -1;
struct sigaction old_sigint_sa, old_sigterm_sa;
static const struct sigaction sa = {
.sa_handler = sigterm_sigint,
.sa_flags = SA_RESTART,
};
int r, fd;
if (argc >= 2)
path = argv[1];
if (isempty(path) || streq(path, "-"))
path = NULL;
if (argc >= 3)
local = argv[2];
else if (path)
local = basename(path);
if (isempty(local) || streq(local, "-"))
local = NULL;
if (local) {
if (!machine_name_is_valid(local)) {
log_error("Local image name '%s' is not valid.", local);
return -EINVAL;
}
if (!arg_force) {
r = image_find(IMAGE_MACHINE, local, NULL);
if (r < 0) {
if (r != -ENOENT)
return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
} else {
log_error("Image '%s' already exists.", local);
return -EEXIST;
}
}
} else
local = "imported";
if (path) {
open_fd = open(path, O_DIRECTORY|O_RDONLY|O_CLOEXEC);
if (open_fd < 0)
return log_error_errno(errno, "Failed to open directory to import: %m");
fd = open_fd;
log_info("Importing '%s', saving as '%s'.", path, local);
} else {
_cleanup_free_ char *pretty = NULL;
fd = STDIN_FILENO;
(void) fd_get_path(fd, &pretty);
log_info("Importing '%s', saving as '%s'.", strempty(pretty), local);
}
final_path = strjoina(arg_image_root, "/", local);
r = tempfn_random(final_path, NULL, &temp_path);
if (r < 0)
return log_oom();
(void) mkdir_parents_label(temp_path, 0700);
RATELIMIT_INIT(progress.limit, 200*USEC_PER_MSEC, 1);
/* Hook into SIGINT/SIGTERM, so that we can cancel things then */
assert(sigaction(SIGINT, &sa, &old_sigint_sa) >= 0);
assert(sigaction(SIGTERM, &sa, &old_sigterm_sa) >= 0);
r = btrfs_subvol_snapshot_fd_full(
fd,
temp_path,
BTRFS_SNAPSHOT_FALLBACK_COPY|BTRFS_SNAPSHOT_RECURSIVE|BTRFS_SNAPSHOT_FALLBACK_DIRECTORY|BTRFS_SNAPSHOT_QUOTA,
progress_path,
progress_bytes,
&progress);
if (r == -EOWNERDEAD) { /* SIGINT + SIGTERM cause this, see signal handler above */
log_error("Copy cancelled.");
goto finish;
}
if (r < 0) {
log_error_errno(r, "Failed to copy directory: %m");
goto finish;
}
(void) import_assign_pool_quota_and_warn(temp_path);
if (arg_read_only) {
r = import_make_read_only(temp_path);
if (r < 0) {
log_error_errno(r, "Failed to make directory read-only: %m");
goto finish;
}
}
if (arg_force)
(void) rm_rf(final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
r = rename_noreplace(AT_FDCWD, temp_path, AT_FDCWD, final_path);
if (r < 0) {
log_error_errno(r, "Failed to move image into place: %m");
goto finish;
}
temp_path = mfree(temp_path);
log_info("Exiting.");
finish:
/* Put old signal handlers into place */
assert(sigaction(SIGINT, &old_sigint_sa, NULL) >= 0);
assert(sigaction(SIGTERM, &old_sigterm_sa, NULL) >= 0);
return 0;
}
static int help(int argc, char *argv[], void *userdata) {
printf("%s [OPTIONS...] {COMMAND} ...\n\n"
"Import container images from a file system.\n\n"
" -h --help Show this help\n"
" --version Show package version\n"
" --force Force creation of image\n"
" --image-root=PATH Image root directory\n"
" --read-only Create a read-only image\n\n"
"Commands:\n"
" run DIRECTORY [NAME] Import a directory\n",
program_invocation_short_name);
return 0;
}
static int parse_argv(int argc, char *argv[]) {
enum {
ARG_VERSION = 0x100,
ARG_FORCE,
ARG_IMAGE_ROOT,
ARG_READ_ONLY,
};
static const struct option options[] = {
{ "help", no_argument, NULL, 'h' },
{ "version", no_argument, NULL, ARG_VERSION },
{ "force", no_argument, NULL, ARG_FORCE },
{ "image-root", required_argument, NULL, ARG_IMAGE_ROOT },
{ "read-only", no_argument, NULL, ARG_READ_ONLY },
{}
};
int c;
assert(argc >= 0);
assert(argv);
while ((c = getopt_long(argc, argv, "h", options, NULL)) >= 0)
switch (c) {
case 'h':
return help(0, NULL, NULL);
case ARG_VERSION:
return version();
case ARG_FORCE:
arg_force = true;
break;
case ARG_IMAGE_ROOT:
arg_image_root = optarg;
break;
case ARG_READ_ONLY:
arg_read_only = true;
break;
case '?':
return -EINVAL;
default:
assert_not_reached("Unhandled option");
}
return 1;
}
static int import_fs_main(int argc, char *argv[]) {
static const Verb verbs[] = {
{ "help", VERB_ANY, VERB_ANY, 0, help },
{ "run", 2, 3, 0, import_fs },
{}
};
return dispatch_verb(argc, argv, verbs, NULL);
}
int main(int argc, char *argv[]) {
int r;
setlocale(LC_ALL, "");
log_parse_environment();
log_open();
r = parse_argv(argc, argv);
if (r <= 0)
goto finish;
r = import_fs_main(argc, argv);
finish:
return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
}

View File

@ -10,6 +10,7 @@
#include "bus-util.h"
#include "def.h"
#include "fd-util.h"
#include "float.h"
#include "hostname-util.h"
#include "import-util.h"
#include "machine-pool.h"
@ -34,6 +35,7 @@ typedef struct Manager Manager;
typedef enum TransferType {
TRANSFER_IMPORT_TAR,
TRANSFER_IMPORT_RAW,
TRANSFER_IMPORT_FS,
TRANSFER_EXPORT_TAR,
TRANSFER_EXPORT_RAW,
TRANSFER_PULL_TAR,
@ -94,6 +96,7 @@ struct Manager {
static const char* const transfer_type_table[_TRANSFER_TYPE_MAX] = {
[TRANSFER_IMPORT_TAR] = "import-tar",
[TRANSFER_IMPORT_RAW] = "import-raw",
[TRANSFER_IMPORT_FS] = "import-fs",
[TRANSFER_EXPORT_TAR] = "export-tar",
[TRANSFER_EXPORT_RAW] = "export-raw",
[TRANSFER_PULL_TAR] = "pull-tar",
@ -156,6 +159,7 @@ static int transfer_new(Manager *m, Transfer **ret) {
.stdin_fd = -1,
.stdout_fd = -1,
.verify = _IMPORT_VERIFY_INVALID,
.progress_percent= (unsigned) -1,
};
id = m->current_transfer_id + 1;
@ -177,6 +181,15 @@ static int transfer_new(Manager *m, Transfer **ret) {
return 0;
}
static double transfer_percent_as_double(Transfer *t) {
assert(t);
if (t->progress_percent == (unsigned) -1)
return -DBL_MAX;
return (double) t->progress_percent / 100.0;
}
static void transfer_send_log_line(Transfer *t, const char *line) {
int r, priority = LOG_INFO;
@ -357,7 +370,7 @@ static int transfer_start(Transfer *t) {
return r;
if (r == 0) {
const char *cmd[] = {
NULL, /* systemd-import, systemd-export or systemd-pull */
NULL, /* systemd-import, systemd-import-fs, systemd-export or systemd-pull */
NULL, /* tar, raw */
NULL, /* --verify= */
NULL, /* verify argument */
@ -390,17 +403,52 @@ static int transfer_start(Transfer *t) {
_exit(EXIT_FAILURE);
}
if (IN_SET(t->type, TRANSFER_IMPORT_TAR, TRANSFER_IMPORT_RAW))
cmd[k++] = SYSTEMD_IMPORT_PATH;
else if (IN_SET(t->type, TRANSFER_EXPORT_TAR, TRANSFER_EXPORT_RAW))
cmd[k++] = SYSTEMD_EXPORT_PATH;
else
cmd[k++] = SYSTEMD_PULL_PATH;
switch (t->type) {
if (IN_SET(t->type, TRANSFER_IMPORT_TAR, TRANSFER_EXPORT_TAR, TRANSFER_PULL_TAR))
case TRANSFER_IMPORT_TAR:
case TRANSFER_IMPORT_RAW:
cmd[k++] = SYSTEMD_IMPORT_PATH;
break;
case TRANSFER_IMPORT_FS:
cmd[k++] = SYSTEMD_IMPORT_FS_PATH;
break;
case TRANSFER_EXPORT_TAR:
case TRANSFER_EXPORT_RAW:
cmd[k++] = SYSTEMD_EXPORT_PATH;
break;
case TRANSFER_PULL_TAR:
case TRANSFER_PULL_RAW:
cmd[k++] = SYSTEMD_PULL_PATH;
break;
default:
assert_not_reached("Unexpected transfer type");
}
switch (t->type) {
case TRANSFER_IMPORT_TAR:
case TRANSFER_EXPORT_TAR:
case TRANSFER_PULL_TAR:
cmd[k++] = "tar";
else
break;
case TRANSFER_IMPORT_RAW:
case TRANSFER_EXPORT_RAW:
case TRANSFER_PULL_RAW:
cmd[k++] = "raw";
break;
case TRANSFER_IMPORT_FS:
cmd[k++] = "run";
break;
default:
break;
}
if (t->verify != _IMPORT_VERIFY_INVALID) {
cmd[k++] = "--verify";
@ -704,6 +752,68 @@ static int method_import_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_
return sd_bus_reply_method_return(msg, "uo", id, object);
}
static int method_import_fs(sd_bus_message *msg, void *userdata, sd_bus_error *error) {
_cleanup_(transfer_unrefp) Transfer *t = NULL;
int fd, force, read_only, r;
const char *local, *object;
Manager *m = userdata;
uint32_t id;
assert(msg);
assert(m);
r = bus_verify_polkit_async(
msg,
CAP_SYS_ADMIN,
"org.freedesktop.import1.import",
NULL,
false,
UID_INVALID,
&m->polkit_registry,
error);
if (r < 0)
return r;
if (r == 0)
return 1; /* Will call us back */
r = sd_bus_message_read(msg, "hsbb", &fd, &local, &force, &read_only);
if (r < 0)
return r;
if (!machine_name_is_valid(local))
return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Local name %s is invalid", local);
r = setup_machine_directory((uint64_t) -1, error);
if (r < 0)
return r;
r = transfer_new(m, &t);
if (r < 0)
return r;
t->type = TRANSFER_IMPORT_FS;
t->force_local = force;
t->read_only = read_only;
t->local = strdup(local);
if (!t->local)
return -ENOMEM;
t->stdin_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3);
if (t->stdin_fd < 0)
return -errno;
r = transfer_start(t);
if (r < 0)
return r;
object = t->object_path;
id = t->id;
t = NULL;
return sd_bus_reply_method_return(msg, "uo", id, object);
}
static int method_export_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_error *error) {
_cleanup_(transfer_unrefp) Transfer *t = NULL;
int fd, r;
@ -879,7 +989,7 @@ static int method_list_transfers(sd_bus_message *msg, void *userdata, sd_bus_err
transfer_type_to_string(t->type),
t->remote,
t->local,
(double) t->progress_percent / 100.0,
transfer_percent_as_double(t),
t->object_path);
if (r < 0)
return r;
@ -975,7 +1085,7 @@ static int property_get_progress(
assert(reply);
assert(t);
return sd_bus_message_append(reply, "d", (double) t->progress_percent / 100.0);
return sd_bus_message_append(reply, "d", transfer_percent_as_double(t));
}
static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_type, transfer_type, TransferType);
@ -998,6 +1108,7 @@ static const sd_bus_vtable manager_vtable[] = {
SD_BUS_VTABLE_START(0),
SD_BUS_METHOD("ImportTar", "hsbb", "uo", method_import_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED),
SD_BUS_METHOD("ImportRaw", "hsbb", "uo", method_import_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED),
SD_BUS_METHOD("ImportFileSystem", "hsbb", "uo", method_import_fs, SD_BUS_VTABLE_UNPRIVILEGED),
SD_BUS_METHOD("ExportTar", "shs", "uo", method_export_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED),
SD_BUS_METHOD("ExportRaw", "shs", "uo", method_export_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED),
SD_BUS_METHOD("PullTar", "sssb", "uo", method_pull_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED),

View File

@ -38,6 +38,12 @@ systemd_import_sources = files('''
qcow2-util.h
'''.split())
systemd_import_fs_sources = files('''
import-fs.c
import-common.c
import-common.h
'''.split())
systemd_export_sources = files('''
export.c
export-tar.c

View File

@ -54,6 +54,10 @@
send_interface="org.freedesktop.import1.Manager"
send_member="ImportRaw"/>
<allow send_destination="org.freedesktop.import1"
send_interface="org.freedesktop.import1.Manager"
send_member="ImportFileSystem"/>
<allow send_destination="org.freedesktop.import1"
send_interface="org.freedesktop.import1.Manager"
send_member="ExportTar"/>

View File

@ -5,6 +5,7 @@
#include <fcntl.h>
#include <getopt.h>
#include <locale.h>
#include <math.h>
#include <net/if.h>
#include <netinet/in.h>
#include <string.h>
@ -2133,6 +2134,66 @@ static int import_raw(int argc, char *argv[], void *userdata) {
return transfer_image_common(bus, m);
}
static int import_fs(int argc, char *argv[], void *userdata) {
_cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL;
_cleanup_close_ int fd = -1;
const char *local = NULL, *path = NULL;
sd_bus *bus = userdata;
int r;
assert(bus);
if (argc >= 2)
path = argv[1];
if (isempty(path) || streq(path, "-"))
path = NULL;
if (argc >= 3)
local = argv[2];
else if (path)
local = basename(path);
if (isempty(local) || streq(local, "-"))
local = NULL;
if (!local) {
log_error("Need either path or local name.");
return -EINVAL;
}
if (!machine_name_is_valid(local)) {
log_error("Local name %s is not a suitable machine name.", local);
return -EINVAL;
}
if (path) {
fd = open(path, O_DIRECTORY|O_RDONLY|O_CLOEXEC);
if (fd < 0)
return log_error_errno(errno, "Failed to open directory '%s': %m", path);
}
r = sd_bus_message_new_method_call(
bus,
&m,
"org.freedesktop.import1",
"/org/freedesktop/import1",
"org.freedesktop.import1.Manager",
"ImportFileSystem");
if (r < 0)
return bus_log_create_error(r);
r = sd_bus_message_append(
m,
"hsbb",
fd >= 0 ? fd : STDIN_FILENO,
local,
arg_force,
arg_read_only);
if (r < 0)
return bus_log_create_error(r);
return transfer_image_common(bus, m);
}
static void determine_compression_from_filename(const char *p) {
if (arg_format)
return;
@ -2464,12 +2525,21 @@ static int list_transfers(int argc, char *argv[], void *userdata) {
(int) max_remote, "REMOTE");
for (j = 0; j < n_transfers; j++)
printf("%*" PRIu32 " %*u%% %-*s %-*s %-*s\n",
(int) MAX(2U, DECIMAL_STR_WIDTH(max_id)), transfers[j].id,
(int) 6, (unsigned) (transfers[j].progress * 100),
(int) max_type, transfers[j].type,
(int) max_local, transfers[j].local,
(int) max_remote, transfers[j].remote);
if (transfers[j].progress < 0)
printf("%*" PRIu32 " %*s %-*s %-*s %-*s\n",
(int) MAX(2U, DECIMAL_STR_WIDTH(max_id)), transfers[j].id,
(int) 7, "n/a",
(int) max_type, transfers[j].type,
(int) max_local, transfers[j].local,
(int) max_remote, transfers[j].remote);
else
printf("%*" PRIu32 " %*u%% %-*s %-*s %-*s\n",
(int) MAX(2U, DECIMAL_STR_WIDTH(max_id)), transfers[j].id,
(int) 6, (unsigned) (transfers[j].progress * 100),
(int) max_type, transfers[j].type,
(int) max_local, transfers[j].local,
(int) max_remote, transfers[j].remote);
if (arg_legend) {
if (n_transfers > 0)
@ -2687,6 +2757,7 @@ static int help(int argc, char *argv[], void *userdata) {
" pull-raw URL [NAME] Download a RAW container or VM image\n"
" import-tar FILE [NAME] Import a local TAR container image\n"
" import-raw FILE [NAME] Import a local RAW container or VM image\n"
" import-fs DIRECTORY [NAME] Import a local directory container image\n"
" export-tar NAME [FILE] Export a TAR container image locally\n"
" export-raw NAME [FILE] Export a RAW container or VM image locally\n"
" list-transfers Show list of downloads in progress\n"
@ -3008,6 +3079,7 @@ static int machinectl_main(int argc, char *argv[], sd_bus *bus) {
{ "disable", 2, VERB_ANY, 0, enable_machine },
{ "import-tar", 2, 3, 0, import_tar },
{ "import-raw", 2, 3, 0, import_raw },
{ "import-fs", 2, 3, 0, import_fs },
{ "export-tar", 2, 3, 0, export_tar },
{ "export-raw", 2, 3, 0, export_raw },
{ "pull-tar", 2, 3, 0, pull_tar },