copy: optionally, reproduce hardlinks from source in destination

This is useful for duplicating trees that contain hardlinks: we keep
track of potential hardlinks and try to reproduce them within the
destination tree. (We do not hardlink between source and destination!).

This is useful for trees like ostree images which heavily use hardlinks
and which are otherwise exploded into separate copies of all files when
we duplicate the trees.

src/basic/copy.c

@@ -22,8 +22,10 @@
 #include "missing_syscall.h"
 #include "mountpoint-util.h"
 #include "nulstr-util.h"
+#include "rm-rf.h"
 #include "selinux-util.h"
 #include "stat-util.h"
+#include "stdio-util.h"
 #include "string-util.h"
 #include "strv.h"
 #include "time-util.h"
@@ -394,6 +396,188 @@ static int fd_copy_symlink(
         return 0;
 }
 
+/* Encapsulates the database we store potential hardlink targets in */
+typedef struct HardlinkContext {
+        int dir_fd;    /* An fd to the directory we use as lookup table. Never AT_FDCWD. Lazily created, when
+                        * we add the first entry. */
+
+        /* These two fields are used to create the hardlink repository directory above — via
+         * mkdirat(parent_fd, subdir) — and are kept so that we can automatically remove the directory again
+         * when we are done. */
+        int parent_fd; /* Possibly AT_FDCWD */
+        char *subdir;
+} HardlinkContext;
+
+static int hardlink_context_setup(
+                HardlinkContext *c,
+                int dt,
+                const char *to,
+                CopyFlags copy_flags) {
+
+        _cleanup_close_ int dt_copy = -1;
+        int r;
+
+        assert(c);
+        assert(c->dir_fd < 0 && c->dir_fd != AT_FDCWD);
+        assert(c->parent_fd < 0);
+        assert(!c->subdir);
+
+        /* If hardlink recreation is requested we have to maintain a database of inodes that are potential
+         * hardlink sources. Given that generally disk sizes have to be assumed to be larger than what fits
+         * into physical RAM we cannot maintain that database in dynamic memory alone. Here we opt to
+         * maintain it on disk, to simplify things: inside the destination directory we'll maintain a
+         * temporary directory consisting of hardlinks of every inode we copied that might be subject of
+         * hardlinks. We can then use that as hardlink source later on. Yes, this means additional disk IO
+         * but thankfully Linux is optimized for this kind of thing. If this ever becomes a performance
+         * bottleneck we can certainly place an in-memory hash table in front of this, but for the beginning,
+         * let's keep things simple, and just use the disk as lookup table for inodes.
+         *
+         * Note that this should have zero performace impact as long as .n_link of all files copied remains
+         * <= 0, because in that case we will not actually allocate the hardlink inode lookup table directory
+         * on disk (we do so lazily, when the first candidate with .n_link > 1 is seen). This means, in the
+         * common case where hardlinks are not used at all or only for few files the fact that we store the
+         * table on disk shouldn't matter perfomance-wise. */
+
+        if (!FLAGS_SET(copy_flags, COPY_HARDLINKS))
+                return 0;
+
+        if (dt == AT_FDCWD)
+                dt_copy = AT_FDCWD;
+        else if (dt < 0)
+                return -EBADF;
+        else {
+                dt_copy = fcntl(dt, F_DUPFD_CLOEXEC, 3);
+                if (dt_copy < 0)
+                        return -errno;
+        }
+
+        r = tempfn_random_child(to, "hardlink", &c->subdir);
+        if (r < 0)
+                return r;
+
+        c->parent_fd = TAKE_FD(dt_copy);
+
+        /* We don't actually create the directory we keep the table in here, that's done on-demand when the
+         * first entry is added, using hardlink_context_realize() below. */
+        return 1;
+}
+
+static int hardlink_context_realize(HardlinkContext *c) {
+        int r;
+
+        if (!c)
+                return 0;
+
+        if (c->dir_fd >= 0) /* Already realized */
+                return 1;
+
+        if (c->parent_fd < 0 && c->parent_fd != AT_FDCWD) /* Not configured */
+                return 0;
+
+        assert(c->subdir);
+
+        if (mkdirat(c->parent_fd, c->subdir, 0700) < 0)
+                return -errno;
+
+        c->dir_fd = openat(c->parent_fd, c->subdir, O_RDONLY|O_DIRECTORY|O_CLOEXEC);
+        if (c->dir_fd < 0) {
+                r = -errno;
+                unlinkat(c->parent_fd, c->subdir, AT_REMOVEDIR);
+                return r;
+        }
+
+        return 1;
+}
+
+static void hardlink_context_destroy(HardlinkContext *c) {
+        int r;
+
+        assert(c);
+
+        /* Automatically remove the hardlink lookup table directory again after we are done. This is used via
+         * _cleanup_() so that we really delete this, even on failure. */
+
+        if (c->dir_fd >= 0) {
+                r = rm_rf_children(TAKE_FD(c->dir_fd), REMOVE_PHYSICAL, NULL); /* consumes dir_fd in all cases, even on failure */
+                if (r < 0)
+                        log_debug_errno(r, "Failed to remove hardlink store (%s) contents, ignoring: %m", c->subdir);
+
+                assert(c->parent_fd >= 0 || c->parent_fd == AT_FDCWD);
+                assert(c->subdir);
+
+                if (unlinkat(c->parent_fd, c->subdir, AT_REMOVEDIR) < 0)
+                        log_debug_errno(errno, "Failed to remove hardlink store (%s) directory, ignoring: %m", c->subdir);
+        }
+
+        assert_cc(AT_FDCWD < 0);
+        c->parent_fd = safe_close(c->parent_fd);
+
+        c->subdir = mfree(c->subdir);
+}
+
+static int try_hardlink(
+                HardlinkContext *c,
+                const struct stat *st,
+                int dt,
+                const char *to) {
+
+        char dev_ino[DECIMAL_STR_MAX(dev_t)*2 + DECIMAL_STR_MAX(uint64_t) + 4];
+
+        assert(st);
+        assert(dt >= 0 || dt == AT_FDCWD);
+        assert(to);
+
+        if (!c) /* No temporary hardlink directory, don't bother */
+                return 0;
+
+        if (st->st_nlink <= 1) /* Source not hardlinked, don't bother */
+                return 0;
+
+        if (c->dir_fd < 0) /* not yet realized, hence empty */
+                return 0;
+
+        xsprintf(dev_ino, "%u:%u:%" PRIu64, major(st->st_dev), minor(st->st_dev), (uint64_t) st->st_ino);
+        if (linkat(c->dir_fd, dev_ino, dt, to, 0) < 0)  {
+                if (errno != ENOENT) /* doesn't exist in store yet */
+                        log_debug_errno(errno, "Failed to hardlink %s to %s, ignoring: %m", dev_ino, to);
+                return 0;
+        }
+
+        return 1;
+}
+
+static int memorize_hardlink(
+                HardlinkContext *c,
+                const struct stat *st,
+                int dt,
+                const char *to) {
+
+        char dev_ino[DECIMAL_STR_MAX(dev_t)*2 + DECIMAL_STR_MAX(uint64_t) + 4];
+        int r;
+
+        assert(st);
+        assert(dt >= 0 || dt == AT_FDCWD);
+        assert(to);
+
+        if (!c) /* No temporary hardlink directory, don't bother */
+                return 0;
+
+        if (st->st_nlink <= 1) /* Source not hardlinked, don't bother */
+                return 0;
+
+        r = hardlink_context_realize(c); /* Create the hardlink store lazily */
+        if (r < 0)
+                return r;
+
+        xsprintf(dev_ino, "%u:%u:%" PRIu64, major(st->st_dev), minor(st->st_dev), (uint64_t) st->st_ino);
+        if (linkat(dt, to, c->dir_fd, dev_ino, 0) < 0) {
+                log_debug_errno(errno, "Failed to hardlink %s to %s, ignoring: %m", to, dev_ino);
+                return 0;
+        }
+
+        return 1;
+}
+
 static int fd_copy_regular(
                 int df,
                 const char *from,
@@ -403,6 +587,7 @@ static int fd_copy_regular(
                 uid_t override_uid,
                 gid_t override_gid,
                 CopyFlags copy_flags,
+                HardlinkContext *hardlink_context,
                 copy_progress_bytes_t progress,
                 void *userdata) {
 
@@ -414,6 +599,12 @@ static int fd_copy_regular(
         assert(st);
         assert(to);
 
+        r = try_hardlink(hardlink_context, st, dt, to);
+        if (r < 0)
+                return r;
+        if (r > 0) /* worked! */
+                return 0;
+
         fdf = openat(df, from, O_RDONLY|O_CLOEXEC|O_NOCTTY|O_NOFOLLOW);
         if (fdf < 0)
                 return -errno;
@@ -456,6 +647,7 @@ static int fd_copy_regular(
                 (void) unlinkat(dt, to, 0);
         }
 
+        (void) memorize_hardlink(hardlink_context, st, dt, to);
         return r;
 }
 
@@ -467,13 +659,20 @@ static int fd_copy_fifo(
                 const char *to,
                 uid_t override_uid,
                 gid_t override_gid,
-                CopyFlags copy_flags) {
+                CopyFlags copy_flags,
+                HardlinkContext *hardlink_context) {
         int r;
 
         assert(from);
         assert(st);
         assert(to);
 
+        r = try_hardlink(hardlink_context, st, dt, to);
+        if (r < 0)
+                return r;
+        if (r > 0) /* worked! */
+                return 0;
+
         if (copy_flags & COPY_MAC_CREATE) {
                 r = mac_selinux_create_file_prepare_at(dt, to, S_IFIFO);
                 if (r < 0)
@@ -494,6 +693,7 @@ static int fd_copy_fifo(
         if (fchmodat(dt, to, st->st_mode & 07777, 0) < 0)
                 r = -errno;
 
+        (void) memorize_hardlink(hardlink_context, st, dt, to);
         return r;
 }
 
@@ -505,13 +705,20 @@ static int fd_copy_node(
                 const char *to,
                 uid_t override_uid,
                 gid_t override_gid,
-                CopyFlags copy_flags) {
+                CopyFlags copy_flags,
+                HardlinkContext *hardlink_context) {
         int r;
 
         assert(from);
         assert(st);
         assert(to);
 
+        r = try_hardlink(hardlink_context, st, dt, to);
+        if (r < 0)
+                return r;
+        if (r > 0) /* worked! */
+                return 0;
+
         if (copy_flags & COPY_MAC_CREATE) {
                 r = mac_selinux_create_file_prepare_at(dt, to, st->st_mode & S_IFMT);
                 if (r < 0)
@@ -532,6 +739,7 @@ static int fd_copy_node(
         if (fchmodat(dt, to, st->st_mode & 07777, 0) < 0)
                 r = -errno;
 
+        (void) memorize_hardlink(hardlink_context, st, dt, to);
         return r;
 }
 
@@ -546,11 +754,17 @@ static int fd_copy_directory(
                 uid_t override_uid,
                 gid_t override_gid,
                 CopyFlags copy_flags,
+                HardlinkContext *hardlink_context,
                 const char *display_path,
                 copy_progress_path_t progress_path,
                 copy_progress_bytes_t progress_bytes,
                 void *userdata) {
 
+        _cleanup_(hardlink_context_destroy) HardlinkContext our_hardlink_context = {
+                .dir_fd = -1,
+                .parent_fd = -1,
+        };
+
         _cleanup_close_ int fdf = -1, fdt = -1;
         _cleanup_closedir_ DIR *d = NULL;
         struct dirent *de;
@@ -570,6 +784,16 @@ static int fd_copy_directory(
         if (fdf < 0)
                 return -errno;
 
+        if (!hardlink_context) {
+                /* If recreating hardlinks is requested let's set up a context for that now. */
+                r = hardlink_context_setup(&our_hardlink_context, dt, to, copy_flags);
+                if (r < 0)
+                        return r;
+                if (r > 0) /* It's enabled and allocated, let's now use the same context for all recursive
+                            * invocations from here down */
+                        hardlink_context = &our_hardlink_context;
+        }
+
         d = take_fdopendir(&fdf);
         if (!d)
                 return -errno;
@@ -668,15 +892,15 @@ static int fd_copy_directory(
                                         continue;
                         }
 
-                        q = fd_copy_directory(dirfd(d), de->d_name, &buf, fdt, de->d_name, original_device, depth_left-1, override_uid, override_gid, copy_flags, child_display_path, progress_path, progress_bytes, userdata);
+                        q = fd_copy_directory(dirfd(d), de->d_name, &buf, fdt, de->d_name, original_device, depth_left-1, override_uid, override_gid, copy_flags, hardlink_context, child_display_path, progress_path, progress_bytes, userdata);
                 } else if (S_ISREG(buf.st_mode))
-                        q = fd_copy_regular(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags, progress_bytes, userdata);
+                        q = fd_copy_regular(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags, hardlink_context, progress_bytes, userdata);
                 else if (S_ISLNK(buf.st_mode))
                         q = fd_copy_symlink(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags);
                 else if (S_ISFIFO(buf.st_mode))
-                        q = fd_copy_fifo(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags);
+                        q = fd_copy_fifo(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags, hardlink_context);
                 else if (S_ISBLK(buf.st_mode) || S_ISCHR(buf.st_mode) || S_ISSOCK(buf.st_mode))
-                        q = fd_copy_node(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags);
+                        q = fd_copy_node(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags, hardlink_context);
                 else
                         q = -EOPNOTSUPP;
 
@@ -730,15 +954,15 @@ int copy_tree_at_full(
                 return -errno;
 
         if (S_ISREG(st.st_mode))
-                return fd_copy_regular(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags, progress_bytes, userdata);
+                return fd_copy_regular(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags, NULL, progress_bytes, userdata);
         else if (S_ISDIR(st.st_mode))
-                return fd_copy_directory(fdf, from, &st, fdt, to, st.st_dev, COPY_DEPTH_MAX, override_uid, override_gid, copy_flags, NULL, progress_path, progress_bytes, userdata);
+                return fd_copy_directory(fdf, from, &st, fdt, to, st.st_dev, COPY_DEPTH_MAX, override_uid, override_gid, copy_flags, NULL, NULL, progress_path, progress_bytes, userdata);
         else if (S_ISLNK(st.st_mode))
                 return fd_copy_symlink(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags);
         else if (S_ISFIFO(st.st_mode))
-                return fd_copy_fifo(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags);
+                return fd_copy_fifo(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags, NULL);
         else if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode) || S_ISSOCK(st.st_mode))
-                return fd_copy_node(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags);
+                return fd_copy_node(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags, NULL);
         else
                 return -EOPNOTSUPP;
 }
@@ -762,7 +986,7 @@ int copy_directory_fd_full(
         if (!S_ISDIR(st.st_mode))
                 return -ENOTDIR;
 
-        return fd_copy_directory(dirfd, NULL, &st, AT_FDCWD, to, st.st_dev, COPY_DEPTH_MAX, UID_INVALID, GID_INVALID, copy_flags, NULL, progress_path, progress_bytes, userdata);
+        return fd_copy_directory(dirfd, NULL, &st, AT_FDCWD, to, st.st_dev, COPY_DEPTH_MAX, UID_INVALID, GID_INVALID, copy_flags, NULL, NULL, progress_path, progress_bytes, userdata);
 }
 
 int copy_directory_full(
@@ -784,7 +1008,7 @@ int copy_directory_full(
         if (!S_ISDIR(st.st_mode))
                 return -ENOTDIR;
 
-        return fd_copy_directory(AT_FDCWD, from, &st, AT_FDCWD, to, st.st_dev, COPY_DEPTH_MAX, UID_INVALID, GID_INVALID, copy_flags, NULL, progress_path, progress_bytes, userdata);
+        return fd_copy_directory(AT_FDCWD, from, &st, AT_FDCWD, to, st.st_dev, COPY_DEPTH_MAX, UID_INVALID, GID_INVALID, copy_flags, NULL, NULL, progress_path, progress_bytes, userdata);
 }
 
 int copy_file_fd_full(

src/basic/copy.h

@@ -17,6 +17,7 @@ typedef enum CopyFlags {
         COPY_CRTIME      = 1 << 5, /* Generate a user.crtime_usec xattr off the source crtime if there is one, on copying */
         COPY_SIGINT      = 1 << 6, /* Check for SIGINT regularly and return EINTR if seen (caller needs to block SIGINT) */
         COPY_MAC_CREATE  = 1 << 7, /* Create files with the correct MAC label (currently SELinux only) */
+        COPY_HARDLINKS   = 1 << 8, /* Try to reproduce hard links */
 } CopyFlags;
 
 typedef int (*copy_progress_bytes_t)(uint64_t n_bytes, void *userdata);

src/dissect/dissect.c

@@ -627,7 +627,7 @@ static int action_copy(DissectedImage *m, LoopDevice *d) {
                 }
 
                 /* Try to copy as directory? */
-                r = copy_directory_fd(source_fd, arg_target, COPY_REFLINK|COPY_MERGE_EMPTY|COPY_SIGINT);
+                r = copy_directory_fd(source_fd, arg_target, COPY_REFLINK|COPY_MERGE_EMPTY|COPY_SIGINT|COPY_HARDLINKS);
                 if (r >= 0)
                         return 0;
                 if (r != -ENOTDIR)
@@ -699,9 +699,9 @@ static int action_copy(DissectedImage *m, LoopDevice *d) {
                                 if (errno != ENOENT)
                                         return log_error_errno(errno, "Failed to open destination '%s': %m", arg_target);
 
-                                r = copy_tree_at(source_fd, ".", dfd, basename(arg_target), UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_REPLACE|COPY_SIGINT);
+                                r = copy_tree_at(source_fd, ".", dfd, basename(arg_target), UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_REPLACE|COPY_SIGINT|COPY_HARDLINKS);
                         } else
-                                r = copy_tree_at(source_fd, ".", target_fd, ".", UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_REPLACE|COPY_SIGINT);
+                                r = copy_tree_at(source_fd, ".", target_fd, ".", UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_REPLACE|COPY_SIGINT|COPY_HARDLINKS);
                         if (r < 0)
                                 return log_error_errno(r, "Failed to copy '%s' to '%s' in image '%s': %m", arg_source, arg_target, arg_image);
 

src/test/test-copy.c

@@ -81,10 +81,12 @@ static void test_copy_tree(void) {
         char original_dir[] = "/tmp/test-copy_tree/";
         char copy_dir[] = "/tmp/test-copy_tree-copy/";
         char **files = STRV_MAKE("file", "dir1/file", "dir1/dir2/file", "dir1/dir2/dir3/dir4/dir5/file");
-        char **links = STRV_MAKE("link", "file",
-                                 "link2", "dir1/file");
+        char **symlinks = STRV_MAKE("link", "file",
+                                    "link2", "dir1/file");
+        char **hardlinks = STRV_MAKE("hlink", "file",
+                                     "hlink2", "dir1/file");
         const char *unixsockp;
-        char **p, **link;
+        char **p, **ll;
         struct stat st;
         int xattr_worked = -1; /* xattr support is optional in temporary directories, hence use it if we can,
                                 * but don't fail if we can't */
@@ -110,20 +112,30 @@ static void test_copy_tree(void) {
                 xattr_worked = k >= 0;
         }
 
-        STRV_FOREACH_PAIR(link, p, links) {
+        STRV_FOREACH_PAIR(ll, p, symlinks) {
                 _cleanup_free_ char *f, *l;
 
                 assert_se(f = path_join(original_dir, *p));
-                assert_se(l = path_join(original_dir, *link));
+                assert_se(l = path_join(original_dir, *ll));
 
                 assert_se(mkdir_parents(l, 0755) >= 0);
                 assert_se(symlink(f, l) == 0);
         }
 
+        STRV_FOREACH_PAIR(ll, p, hardlinks) {
+                _cleanup_free_ char *f, *l;
+
+                assert_se(f = path_join(original_dir, *p));
+                assert_se(l = path_join(original_dir, *ll));
+
+                assert_se(mkdir_parents(l, 0755) >= 0);
+                assert_se(link(f, l) == 0);
+        }
+
         unixsockp = strjoina(original_dir, "unixsock");
         assert_se(mknod(unixsockp, S_IFSOCK|0644, 0) >= 0);
 
-        assert_se(copy_tree(original_dir, copy_dir, UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_MERGE) == 0);
+        assert_se(copy_tree(original_dir, copy_dir, UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_MERGE|COPY_HARDLINKS) == 0);
 
         STRV_FOREACH(p, files) {
                 _cleanup_free_ char *buf, *f, *c = NULL;
@@ -147,16 +159,30 @@ static void test_copy_tree(void) {
                 }
         }
 
-        STRV_FOREACH_PAIR(link, p, links) {
+        STRV_FOREACH_PAIR(ll, p, symlinks) {
                 _cleanup_free_ char *target, *f, *l;
 
                 assert_se(f = strjoin(original_dir, *p));
-                assert_se(l = strjoin(copy_dir, *link));
+                assert_se(l = strjoin(copy_dir, *ll));
 
                 assert_se(chase_symlinks(l, NULL, 0, &target, NULL) == 1);
                 assert_se(path_equal(f, target));
         }
 
+        STRV_FOREACH_PAIR(ll, p, hardlinks) {
+                _cleanup_free_ char *f, *l;
+                struct stat a, b;
+
+                assert_se(f = strjoin(copy_dir, *p));
+                assert_se(l = strjoin(copy_dir, *ll));
+
+                assert_se(lstat(f, &a) >= 0);
+                assert_se(lstat(l, &b) >= 0);
+
+                assert_se(a.st_ino == b.st_ino);
+                assert_se(a.st_dev == b.st_dev);
+        }
+
         unixsockp = strjoina(copy_dir, "unixsock");
         assert_se(stat(unixsockp, &st) >= 0);
         assert_se(S_ISSOCK(st.st_mode));