picnoir
/
Systemd
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

import: support importing qcow2 images 

With this change the import tool will now unpack qcow2 images into
normal raw disk images, suitable for usage with nspawn.

This allows has the benefit of also allowing importing Ubuntu Cloud
images for usage with nspawn.
This commit is contained in:
Lennart Poettering 2015-01-16 20:07:25 +01:00
parent 49bb233bb7
commit edce2aed3a
8 changed files with 560 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -227,6 +227,7 @@
/test-pppoe
/test-prioq
/test-pty
/test-qcow2
/test-ratelimit
/test-replace-var
/test-resolve

30
Makefile.am
View File

@ -5233,19 +5233,43 @@ systemd_import_SOURCES = \
src/import/curl-util.c \
src/import/curl-util.h \
src/import/aufs-util.c \
src/import/aufs-util.h
src/import/aufs-util.h \
src/import/qcow2-util.c \
src/import/qcow2-util.h
systemd_import_CFLAGS = \
$(AM_CFLAGS) \
$(LIBCURL_CFLAGS) \
$(XZ_CFLAGS)
$(XZ_CFLAGS) \
$(ZLIB_CFLAGS)
systemd_import_LDADD = \
libsystemd-internal.la \
libsystemd-label.la \
libsystemd-shared.la \
$(LIBCURL_LIBS) \
$(XZ_LIBS)
$(XZ_LIBS) \
$(ZLIB_LIBS)
endif
if HAVE_ZLIB
manual_tests += \
test-qcow2
test_qcow2_SOURCES = \
src/import/test-qcow2.c \
src/import/qcow2-util.c \
src/import/qcow2-util.h
test_qcow2_CFLAGS = \
$(AM_CFLAGS) \
$(ZLIB_CFLAGS)
test_qcow2_LDADD = \
libsystemd-internal.la \
libsystemd-shared.la \
$(ZLIB_LIBS)
endif
endif

13
configure.ac
View File

@ -563,6 +563,18 @@ if test "x$enable_xz" != "xno"; then
fi
AM_CONDITIONAL(HAVE_XZ, [test "$have_xz" = "yes"])
# ------------------------------------------------------------------------------
have_zlib=no
AC_ARG_ENABLE(zlib, AS_HELP_STRING([--disable-zlib], [Disable optional ZLIB support]))
if test "x$enable_zlib" != "xno"; then
PKG_CHECK_MODULES(ZLIB, [ zlib ],
[AC_DEFINE(HAVE_ZLIB, 1, [Define if ZLIB is available]) have_zlib=yes])
if test "x$have_zlib" = xno -a "x$enable_zlib" = xyes; then
AC_MSG_ERROR([*** ZLIB support requested but libraries not found])
fi
fi
AM_CONDITIONAL(HAVE_ZLIB, [test "$have_zlib" = "yes"])
# ------------------------------------------------------------------------------
have_lz4=no
AC_ARG_ENABLE(lz4, AS_HELP_STRING([--enable-lz4], [Enable optional LZ4 support]))
@ -1410,6 +1422,7 @@ AC_MSG_RESULT([
SELinux: ${have_selinux}
SECCOMP: ${have_seccomp}
SMACK: ${have_smack}
ZLIB: ${have_zlib}
XZ: ${have_xz}
LZ4: ${have_lz4}
ACL: ${have_acl}

48
src/import/import-raw.c
View File

@ -27,6 +27,7 @@
#include "hashmap.h"
#include "utf8.h"
#include "curl-util.h"
#include "qcow2-util.h"
#include "import-raw.h"
#include "strv.h"
#include "copy.h"
@ -235,6 +236,49 @@ finish:
raw_import_finish(f->import, r);
}
static int raw_import_maybe_convert_qcow2(RawImportFile *f) {
_cleanup_close_ int converted_fd = -1;
_cleanup_free_ char *t = NULL;
int r;
assert(f);
assert(f->disk_fd);
assert(f->temp_path);
r = qcow2_detect(f->disk_fd);
if (r < 0)
return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
if (r == 0)
return 0;
/* This is a QCOW2 image, let's convert it */
r = tempfn_random(f->final_path, &t);
if (r < 0)
return log_oom();
converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
if (converted_fd < 0)
return log_error_errno(errno, "Failed to create %s: %m", t);
r = qcow2_convert(f->disk_fd, converted_fd);
if (r < 0) {
unlink(t);
return log_error_errno(r, "Failed to convert qcow2 image: %m");
}
unlink(f->temp_path);
free(f->temp_path);
f->temp_path = t;
t = NULL;
safe_close(f->disk_fd);
f->disk_fd = converted_fd;
converted_fd = -1;
return 1;
}
static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
RawImportFile *f = NULL;
struct stat st;
@ -288,6 +332,10 @@ static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result
goto fail;
}
r = raw_import_maybe_convert_qcow2(f);
if (r < 0)
goto fail;
if (f->etag)
(void) fsetxattr(f->disk_fd, "user.source_etag", f->etag, strlen(f->etag), 0);
if (f->url)

52
src/import/import.c
View File

@ -45,10 +45,48 @@ static void on_raw_finished(RawImport *import, int error, void *userdata) {
sd_event_exit(event, error);
}
static int strip_raw_suffixes(const char *p, char **ret) {
static const char suffixes[] =
".xz\0"
".raw\0"
".qcow2\0"
".img\0";
_cleanup_free_ char *q = NULL;
q = strdup(p);
if (!q)
return -ENOMEM;
for (;;) {
const char *sfx;
bool changed = false;
NULSTR_FOREACH(sfx, suffixes) {
char *e;
e = endswith(q, sfx);
if (e) {
*e = 0;
changed = true;
}
}
if (!changed)
break;
}
*ret = q;
q = NULL;
return 0;
}
static int pull_raw(int argc, char *argv[], void *userdata) {
_cleanup_(raw_import_unrefp) RawImport *import = NULL;
_cleanup_event_unref_ sd_event *event = NULL;
const char *url, *local, *suffix;
const char *url, *local;
_cleanup_free_ char *l = NULL;
int r;
url = argv[1];
@ -79,13 +117,11 @@ static int pull_raw(int argc, char *argv[], void *userdata) {
if (local) {
const char *p;
suffix = endswith(local, ".raw.xz");
if (!suffix)
suffix = endswith(local, ".raw");
if (!suffix)
suffix = endswith(local, ".xz");
if (suffix)
local = strndupa(local, suffix - local);
r = strip_raw_suffixes(local, &l);
if (r < 0)
return log_oom();
local = l;
if (!machine_name_is_valid(local)) {
log_error("Local image name '%s' is not valid.", local);

347
src/import/qcow2-util.c Normal file
View File

@ -0,0 +1,347 @@
/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
/***
This file is part of systemd.
Copyright 2015 Lennart Poettering
systemd is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
systemd is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/
#include <zlib.h>
#include "util.h"
#include "sparse-endian.h"
#include "qcow2-util.h"
#define QCOW2_MAGIC 0x514649fb
#define QCOW2_COPIED (1ULL << 63)
#define QCOW2_COMPRESSED (1ULL << 62)
#define QCOW2_ZERO (1ULL << 0)
typedef struct _packed_ Header {
be32_t magic;
be32_t version;
be64_t backing_file_offset;
be32_t backing_file_size;
be32_t cluster_bits;
be64_t size;
be32_t crypt_method;
be32_t l1_size;
be64_t l1_table_offset;
be64_t refcount_table_offset;
be32_t refcount_table_clusters;
be32_t nb_snapshots;
be64_t snapshots_offset;
/* The remainder is only present on QCOW3 */
be64_t incompatible_features;
be64_t compatible_features;
be64_t autoclear_features;
be32_t refcount_order;
be32_t header_length;
} Header;
#define HEADER_MAGIC(header) be32toh((header)->magic)
#define HEADER_VERSION(header) be32toh((header)->version)
#define HEADER_CLUSTER_BITS(header) be32toh((header)->cluster_bits)
#define HEADER_CLUSTER_SIZE(header) (1ULL << HEADER_CLUSTER_BITS(header))
#define HEADER_L2_BITS(header) (HEADER_CLUSTER_BITS(header) - 3)
#define HEADER_SIZE(header) be64toh((header)->size)
#define HEADER_CRYPT_METHOD(header) be32toh((header)->crypt_method)
#define HEADER_L1_SIZE(header) be32toh((header)->l1_size)
#define HEADER_L2_SIZE(header) (HEADER_CLUSTER_SIZE(header)/sizeof(uint64_t))
#define HEADER_L1_TABLE_OFFSET(header) be64toh((header)->l1_table_offset)
static uint32_t HEADER_HEADER_LENGTH(const Header *h) {
if (HEADER_VERSION(h) < 3)
return offsetof(Header, incompatible_features);
return be32toh(h->header_length);
}
static int copy_cluster(
int sfd, uint64_t soffset,
int dfd, uint64_t doffset,
uint64_t cluster_size,
void *buffer) {
ssize_t l;
l = pread(sfd, buffer, cluster_size, soffset);
if (l < 0)
return -errno;
if ((uint64_t) l != cluster_size)
return -EIO;
l = pwrite(dfd, buffer, cluster_size, doffset);
if (l < 0)
return -errno;
if ((uint64_t) l != cluster_size)
return -EIO;
return 0;
}
static int decompress_cluster(
int sfd, uint64_t soffset,
int dfd, uint64_t doffset,
uint64_t compressed_size,
uint64_t cluster_size,
void *buffer1,
void *buffer2) {
_cleanup_free_ void *large_buffer = NULL;
z_stream s = {};
uint64_t sz;
ssize_t l;
int r;
if (compressed_size > cluster_size) {
/* The usual cluster buffer doesn't suffice, let's
* allocate a larger one, temporarily */
large_buffer = malloc(compressed_size);
if (!large_buffer)
return -ENOMEM;
buffer1 = large_buffer;
}
l = pread(sfd, buffer1, compressed_size, soffset);
if (l < 0)
return -errno;
if ((uint64_t) l != compressed_size)
return -EIO;
s.next_in = buffer1;
s.avail_in = compressed_size;
s.next_out = buffer2;
s.avail_out = cluster_size;
r = inflateInit2(&s, -12);
if (r != Z_OK)
return -EIO;
r = inflate(&s, Z_FINISH);
sz = (uint8_t*) s.next_out - (uint8_t*) buffer2;
inflateEnd(&s);
if (r != Z_STREAM_END || sz != cluster_size)
return -EIO;
l = pwrite(dfd, buffer2, cluster_size, doffset);
if (l < 0)
return -errno;
if ((uint64_t) l != cluster_size)
return -EIO;
return 0;
}
static int normalize_offset(
const Header *header,
uint64_t p,
uint64_t *ret,
bool *compressed,
uint64_t *compressed_size) {
uint64_t q;
q = be64toh(p);
if (q & QCOW2_COMPRESSED) {
uint64_t sz, csize_shift, csize_mask;
if (!compressed)
return -ENOTSUP;
csize_shift = 64 - 2 - (HEADER_CLUSTER_BITS(header) - 8);
csize_mask = (1ULL << (HEADER_CLUSTER_BITS(header) - 8)) - 1;
sz = (((q >> csize_shift) & csize_mask) + 1) * 512 - (q & 511);
q &= ((1ULL << csize_shift) - 1);
if (compressed_size)
*compressed_size = sz;
*compressed = true;
} else {
if (compressed) {
*compressed = false;
*compressed_size = 0;
}
if (q & QCOW2_ZERO) {
/* We make no distinction between zero blocks and holes */
*ret = 0;
return 0;
}
q &= ~QCOW2_COPIED;
}
*ret = q;
return q > 0; /* returns positive if not a hole */
}
static int verify_header(const Header *header) {
assert(header);
if (HEADER_MAGIC(header) != QCOW2_MAGIC)
return -EBADMSG;
if (HEADER_VERSION(header) != 2 &&
HEADER_VERSION(header) != 3)
return -ENOTSUP;
if (HEADER_CRYPT_METHOD(header) != 0)
return -ENOTSUP;
if (HEADER_CLUSTER_BITS(header) < 9) /* 512K */
return -EBADMSG;
if (HEADER_CLUSTER_BITS(header) > 21) /* 2MB */
return -EBADMSG;
if (HEADER_SIZE(header) % HEADER_CLUSTER_SIZE(header) != 0)
return -EBADMSG;
if (HEADER_L1_SIZE(header) > 32*1024*1024) /* 32MB */
return -EBADMSG;
if (HEADER_VERSION(header) == 3) {
if (header->incompatible_features != 0)
return -ENOTSUP;
if (HEADER_HEADER_LENGTH(header) < sizeof(Header))
return -EBADMSG;
}
return 0;
}
int qcow2_convert(int qcow2_fd, int raw_fd) {
_cleanup_free_ void *buffer1 = NULL, *buffer2 = NULL;
_cleanup_free_ be64_t *l1_table = NULL, *l2_table = NULL;
uint64_t sz, i;
Header header;
ssize_t l;
int r;
l = pread(qcow2_fd, &header, sizeof(header), 0);
if (l < 0)
return -errno;
if (l != sizeof(header))
return -EIO;
r = verify_header(&header);
if (r < 0)
return r;
l1_table = new(be64_t, HEADER_L1_SIZE(&header));
if (!l1_table)
return -ENOMEM;
l2_table = malloc(HEADER_CLUSTER_SIZE(&header));
if (!l2_table)
return -ENOMEM;
buffer1 = malloc(HEADER_CLUSTER_SIZE(&header));
if (!buffer1)
return -ENOMEM;
buffer2 = malloc(HEADER_CLUSTER_SIZE(&header));
if (!buffer2)
return -ENOMEM;
/* Empty the file if it exists, we rely on zero bits */
if (ftruncate(raw_fd, 0) < 0)
return -errno;
if (ftruncate(raw_fd, HEADER_SIZE(&header)) < 0)
return -errno;
sz = sizeof(uint64_t) * HEADER_L1_SIZE(&header);
l = pread(qcow2_fd, l1_table, sz, HEADER_L1_TABLE_OFFSET(&header));
if (l < 0)
return -errno;
if ((uint64_t) l != sz)
return -EIO;
for (i = 0; i < HEADER_L1_SIZE(&header); i ++) {
uint64_t l2_begin, j;
r = normalize_offset(&header, l1_table[i], &l2_begin, NULL, NULL);
if (r < 0)
return r;
if (r == 0)
continue;
l = pread(qcow2_fd, l2_table, HEADER_CLUSTER_SIZE(&header), l2_begin);
if (l < 0)
return -errno;
if ((uint64_t) l != HEADER_CLUSTER_SIZE(&header))
return -EIO;
for (j = 0; j < HEADER_L2_SIZE(&header); j++) {
uint64_t data_begin, p, compressed_size;
bool compressed;
p = ((i << HEADER_L2_BITS(&header)) + j) << HEADER_CLUSTER_BITS(&header);
r = normalize_offset(&header, l2_table[j], &data_begin, &compressed, &compressed_size);
if (r < 0)
return r;
if (r == 0)
continue;
if (compressed)
r = decompress_cluster(
qcow2_fd, data_begin,
raw_fd, p,
compressed_size, HEADER_CLUSTER_SIZE(&header),
buffer1, buffer2);
else
r = copy_cluster(
qcow2_fd, data_begin,
raw_fd, p,
HEADER_CLUSTER_SIZE(&header), buffer1);
if (r < 0)
return r;
}
}
return 0;
}
int qcow2_detect(int fd) {
be32_t id;
ssize_t l;
l = pread(fd, &id, sizeof(id), 0);
if (l < 0)
return -errno;
if (l != sizeof(id))
return -EIO;
return htobe32(QCOW2_MAGIC) == id;
}

25
src/import/qcow2-util.h Normal file
View File

@ -0,0 +1,25 @@
/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
#pragma once
/***
This file is part of systemd.
Copyright 2015 Lennart Poettering
systemd is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
systemd is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/
int qcow2_detect(int fd);
int qcow2_convert(int qcow2_fd, int raw_fd);

55
src/import/test-qcow2.c Normal file
View File

@ -0,0 +1,55 @@
/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
/***
This file is part of systemd.
Copyright 2015 Lennart Poettering
systemd is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
systemd is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/
#include "log.h"
#include "util.h"
#include "qcow2-util.h"
int main(int argc, char *argv[]) {
_cleanup_close_ int sfd = -1, dfd = -1;
int r;
if (argc != 3) {
log_error("Needs two arguments.");
return EXIT_FAILURE;
}
sfd = open(argv[1], O_RDONLY|O_CLOEXEC|O_NOCTTY);
if (sfd < 0) {
log_error_errno(errno, "Can't open source file: %m");
return EXIT_FAILURE;
}
dfd = open(argv[2], O_WRONLY|O_CREAT|O_CLOEXEC|O_NOCTTY, 0666);
if (dfd < 0) {
log_error_errno(errno, "Can't open destination file: %m");
return EXIT_FAILURE;
}
r = qcow2_convert(sfd, dfd);
if (r < 0) {
log_error_errno(r, "Failed to unpack: %m");
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}