copy: support getting progress feedback from the various copy functions

This adds two optional functions that may be passed to the various copy
functions. One is invoked whenever we start copying a new file object,
the other while we copy file payload in each loop iteration.

When the caller passes one or both they can get notifications about copy
progress, for example to log where things are.

src/basic/btrfs-util.c

@@ -1503,7 +1503,12 @@ static int copy_subtree_quota_limits(int fd, uint64_t old_subvol, uint64_t new_s
         return changed;
 }
 
-static int subvol_snapshot_children(int old_fd, int new_fd, const char *subvolume, uint64_t old_subvol_id, BtrfsSnapshotFlags flags) {
+static int subvol_snapshot_children(
+                int old_fd,
+                int new_fd,
+                const char *subvolume,
+                uint64_t old_subvol_id,
+                BtrfsSnapshotFlags flags) {
 
         struct btrfs_ioctl_search_args args = {
                 .key.tree_id = BTRFS_ROOT_TREE_OBJECTID,
@@ -1683,7 +1688,14 @@ static int subvol_snapshot_children(int old_fd, int new_fd, const char *subvolum
         return 0;
 }
 
-int btrfs_subvol_snapshot_fd(int old_fd, const char *new_path, BtrfsSnapshotFlags flags) {
+int btrfs_subvol_snapshot_fd_full(
+                int old_fd,
+                const char *new_path,
+                BtrfsSnapshotFlags flags,
+                copy_progress_path_t progress_path,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
+
         _cleanup_close_ int new_fd = -1;
         const char *subvolume;
         int r;
@@ -1711,7 +1723,7 @@ int btrfs_subvol_snapshot_fd(int old_fd, const char *new_path, BtrfsSnapshotFlag
                 } else if (r < 0)
                         return r;
 
-                r = copy_directory_fd(old_fd, new_path, COPY_MERGE|COPY_REFLINK);
+                r = copy_directory_fd_full(old_fd, new_path, COPY_MERGE|COPY_REFLINK, progress_path, progress_bytes, userdata);
                 if (r < 0)
                         goto fallback_fail;
 
@@ -1748,7 +1760,14 @@ int btrfs_subvol_snapshot_fd(int old_fd, const char *new_path, BtrfsSnapshotFlag
         return subvol_snapshot_children(old_fd, new_fd, subvolume, 0, flags);
 }
 
-int btrfs_subvol_snapshot(const char *old_path, const char *new_path, BtrfsSnapshotFlags flags) {
+int btrfs_subvol_snapshot_full(
+                const char *old_path,
+                const char *new_path,
+                BtrfsSnapshotFlags flags,
+                copy_progress_path_t progress_path,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
+
         _cleanup_close_ int old_fd = -1;
 
         assert(old_path);
@@ -1758,7 +1777,7 @@ int btrfs_subvol_snapshot(const char *old_path, const char *new_path, BtrfsSnaps
         if (old_fd < 0)
                 return -errno;
 
-        return btrfs_subvol_snapshot_fd(old_fd, new_path, flags);
+        return btrfs_subvol_snapshot_fd_full(old_fd, new_path, flags, progress_path, progress_bytes, userdata);
 }
 
 int btrfs_qgroup_find_parents(int fd, uint64_t qgroupid, uint64_t **ret) {

src/basic/btrfs-util.h

@@ -7,6 +7,7 @@
 
 #include "sd-id128.h"
 
+#include "copy.h"
 #include "time-util.h"
 
 typedef struct BtrfsSubvolInfo {
@@ -67,8 +68,15 @@ int btrfs_resize_loopback(const char *path, uint64_t size, bool grow_only);
 int btrfs_subvol_make(const char *path);
 int btrfs_subvol_make_fd(int fd, const char *subvolume);
 
-int btrfs_subvol_snapshot_fd(int old_fd, const char *new_path, BtrfsSnapshotFlags flags);
-int btrfs_subvol_snapshot(const char *old_path, const char *new_path, BtrfsSnapshotFlags flags);
+int btrfs_subvol_snapshot_fd_full(int old_fd, const char *new_path, BtrfsSnapshotFlags flags, copy_progress_path_t progress_path, copy_progress_bytes_t progress_bytes, void *userdata);
+static inline int btrfs_subvol_snapshot_fd(int old_fd, const char *new_path, BtrfsSnapshotFlags flags) {
+        return btrfs_subvol_snapshot_fd_full(old_fd, new_path, flags, NULL, NULL, NULL);
+}
+
+int btrfs_subvol_snapshot_full(const char *old_path, const char *new_path, BtrfsSnapshotFlags flags, copy_progress_path_t progress_path, copy_progress_bytes_t progress_bytes, void *userdata);
+static inline int btrfs_subvol_snapshot(const char *old_path, const char *new_path, BtrfsSnapshotFlags flags) {
+        return btrfs_subvol_snapshot_full(old_path, new_path, flags, NULL, NULL, NULL);
+}
 
 int btrfs_subvol_remove(const char *path, BtrfsRemoveFlags flags);
 int btrfs_subvol_remove_fd(int fd, const char *subvolume, BtrfsRemoveFlags flags);

src/basic/copy.c

@@ -90,7 +90,9 @@ int copy_bytes_full(
                 uint64_t max_bytes,
                 CopyFlags copy_flags,
                 void **ret_remains,
-                size_t *ret_remains_size) {
+                size_t *ret_remains_size,
+                copy_progress_bytes_t progress,
+                void *userdata) {
 
         bool try_cfr = true, try_sendfile = true, try_splice = true;
         int r, nonblock_pipe = -1;
@@ -308,10 +310,17 @@ int copy_bytes_full(
                 }
 
         next:
+                if (progress) {
+                        r = progress(n, userdata);
+                        if (r < 0)
+                                return r;
+                }
+
                 if (max_bytes != (uint64_t) -1) {
                         assert(max_bytes >= (uint64_t) n);
                         max_bytes -= n;
                 }
+
                 /* sendfile accepts at most SSIZE_MAX-offset bytes to copy,
                  * so reduce our maximum by the amount we already copied,
                  * but don't go below our copy buffer size, unless we are
@@ -363,7 +372,9 @@ static int fd_copy_regular(
                 const char *to,
                 uid_t override_uid,
                 gid_t override_gid,
-                CopyFlags copy_flags) {
+                CopyFlags copy_flags,
+                copy_progress_bytes_t progress,
+                void *userdata) {
 
         _cleanup_close_ int fdf = -1, fdt = -1;
         struct timespec ts[2];
@@ -381,7 +392,7 @@ static int fd_copy_regular(
         if (fdt < 0)
                 return -errno;
 
-        r = copy_bytes(fdf, fdt, (uint64_t) -1, copy_flags);
+        r = copy_bytes_full(fdf, fdt, (uint64_t) -1, copy_flags, NULL, NULL, progress, userdata);
         if (r < 0) {
                 (void) unlinkat(dt, to, 0);
                 return r;
@@ -483,7 +494,11 @@ static int fd_copy_directory(
                 unsigned depth_left,
                 uid_t override_uid,
                 gid_t override_gid,
-                CopyFlags copy_flags) {
+                CopyFlags copy_flags,
+                const char *display_path,
+                copy_progress_path_t progress_path,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
 
         _cleanup_close_ int fdf = -1, fdt = -1;
         _cleanup_closedir_ DIR *d = NULL;
@@ -524,6 +539,8 @@ static int fd_copy_directory(
         r = 0;
 
         FOREACH_DIRENT_ALL(de, d, return -errno) {
+                const char *child_display_path = NULL;
+                _cleanup_free_ char *dp = NULL;
                 struct stat buf;
                 int q;
 
@@ -535,6 +552,17 @@ static int fd_copy_directory(
                         continue;
                 }
 
+                if (progress_path) {
+                        if (display_path)
+                                child_display_path = dp = strjoin(display_path, "/", de->d_name);
+                        else
+                                child_display_path = de->d_name;
+
+                        r = progress_path(child_display_path, &buf, userdata);
+                        if (r < 0)
+                                return r;
+                }
+
                 if (S_ISDIR(buf.st_mode)) {
                         /*
                          * Don't descend into directories on other file systems, if this is requested. We do a simple
@@ -566,9 +594,9 @@ static int fd_copy_directory(
                                         continue;
                         }
 
-                        q = fd_copy_directory(dirfd(d), de->d_name, &buf, fdt, de->d_name, original_device, depth_left-1, override_uid, override_gid, copy_flags);
+                        q = fd_copy_directory(dirfd(d), de->d_name, &buf, fdt, de->d_name, original_device, depth_left-1, override_uid, override_gid, copy_flags, child_display_path, progress_path, progress_bytes, userdata);
                 } else if (S_ISREG(buf.st_mode))
-                        q = fd_copy_regular(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags);
+                        q = fd_copy_regular(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags, progress_bytes, userdata);
                 else if (S_ISLNK(buf.st_mode))
                         q = fd_copy_symlink(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags);
                 else if (S_ISFIFO(buf.st_mode))
@@ -606,7 +634,18 @@ static int fd_copy_directory(
         return r;
 }
 
-int copy_tree_at(int fdf, const char *from, int fdt, const char *to, uid_t override_uid, gid_t override_gid, CopyFlags copy_flags) {
+int copy_tree_at_full(
+                int fdf,
+                const char *from,
+                int fdt,
+                const char *to,
+                uid_t override_uid,
+                gid_t override_gid,
+                CopyFlags copy_flags,
+                copy_progress_path_t progress_path,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
+
         struct stat st;
 
         assert(from);
@@ -616,9 +655,9 @@ int copy_tree_at(int fdf, const char *from, int fdt, const char *to, uid_t overr
                 return -errno;
 
         if (S_ISREG(st.st_mode))
-                return fd_copy_regular(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags);
+                return fd_copy_regular(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags, progress_bytes, userdata);
         else if (S_ISDIR(st.st_mode))
-                return fd_copy_directory(fdf, from, &st, fdt, to, st.st_dev, COPY_DEPTH_MAX, override_uid, override_gid, copy_flags);
+                return fd_copy_directory(fdf, from, &st, fdt, to, st.st_dev, COPY_DEPTH_MAX, override_uid, override_gid, copy_flags, NULL, progress_path, progress_bytes, userdata);
         else if (S_ISLNK(st.st_mode))
                 return fd_copy_symlink(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags);
         else if (S_ISFIFO(st.st_mode))
@@ -629,11 +668,14 @@ int copy_tree_at(int fdf, const char *from, int fdt, const char *to, uid_t overr
                 return -EOPNOTSUPP;
 }
 
-int copy_tree(const char *from, const char *to, uid_t override_uid, gid_t override_gid, CopyFlags copy_flags) {
-        return copy_tree_at(AT_FDCWD, from, AT_FDCWD, to, override_uid, override_gid, copy_flags);
-}
+int copy_directory_fd_full(
+                int dirfd,
+                const char *to,
+                CopyFlags copy_flags,
+                copy_progress_path_t progress_path,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
 
-int copy_directory_fd(int dirfd, const char *to, CopyFlags copy_flags) {
         struct stat st;
 
         assert(dirfd >= 0);
@@ -645,10 +687,17 @@ int copy_directory_fd(int dirfd, const char *to, CopyFlags copy_flags) {
         if (!S_ISDIR(st.st_mode))
                 return -ENOTDIR;
 
-        return fd_copy_directory(dirfd, NULL, &st, AT_FDCWD, to, st.st_dev, COPY_DEPTH_MAX, UID_INVALID, GID_INVALID, copy_flags);
+        return fd_copy_directory(dirfd, NULL, &st, AT_FDCWD, to, st.st_dev, COPY_DEPTH_MAX, UID_INVALID, GID_INVALID, copy_flags, NULL, progress_path, progress_bytes, userdata);
 }
 
-int copy_directory(const char *from, const char *to, CopyFlags copy_flags) {
+int copy_directory_full(
+                const char *from,
+                const char *to,
+                CopyFlags copy_flags,
+                copy_progress_path_t progress_path,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
+
         struct stat st;
 
         assert(from);
@@ -660,10 +709,16 @@ int copy_directory(const char *from, const char *to, CopyFlags copy_flags) {
         if (!S_ISDIR(st.st_mode))
                 return -ENOTDIR;
 
-        return fd_copy_directory(AT_FDCWD, from, &st, AT_FDCWD, to, st.st_dev, COPY_DEPTH_MAX, UID_INVALID, GID_INVALID, copy_flags);
+        return fd_copy_directory(AT_FDCWD, from, &st, AT_FDCWD, to, st.st_dev, COPY_DEPTH_MAX, UID_INVALID, GID_INVALID, copy_flags, NULL, progress_path, progress_bytes, userdata);
 }
 
-int copy_file_fd(const char *from, int fdt, CopyFlags copy_flags) {
+int copy_file_fd_full(
+                const char *from,
+                int fdt,
+                CopyFlags copy_flags,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
+
         _cleanup_close_ int fdf = -1;
         int r;
 
@@ -674,7 +729,7 @@ int copy_file_fd(const char *from, int fdt, CopyFlags copy_flags) {
         if (fdf < 0)
                 return -errno;
 
-        r = copy_bytes(fdf, fdt, (uint64_t) -1, copy_flags);
+        r = copy_bytes_full(fdf, fdt, (uint64_t) -1, copy_flags, NULL, NULL, progress_bytes, userdata);
 
         (void) copy_times(fdf, fdt);
         (void) copy_xattr(fdf, fdt);
@@ -682,7 +737,16 @@ int copy_file_fd(const char *from, int fdt, CopyFlags copy_flags) {
         return r;
 }
 
-int copy_file(const char *from, const char *to, int flags, mode_t mode, unsigned chattr_flags, CopyFlags copy_flags) {
+int copy_file_full(
+                const char *from,
+                const char *to,
+                int flags,
+                mode_t mode,
+                unsigned chattr_flags,
+                CopyFlags copy_flags,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
+
         int fdt = -1, r;
 
         assert(from);
@@ -697,7 +761,7 @@ int copy_file(const char *from, const char *to, int flags, mode_t mode, unsigned
         if (chattr_flags != 0)
                 (void) chattr_fd(fdt, chattr_flags, (unsigned) -1, NULL);
 
-        r = copy_file_fd(from, fdt, copy_flags);
+        r = copy_file_fd_full(from, fdt, copy_flags, progress_bytes, userdata);
         if (r < 0) {
                 close(fdt);
                 (void) unlink(to);
@@ -712,7 +776,15 @@ int copy_file(const char *from, const char *to, int flags, mode_t mode, unsigned
         return 0;
 }
 
-int copy_file_atomic(const char *from, const char *to, mode_t mode, unsigned chattr_flags, CopyFlags copy_flags) {
+int copy_file_atomic_full(
+                const char *from,
+                const char *to,
+                mode_t mode,
+                unsigned chattr_flags,
+                CopyFlags copy_flags,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
+
         _cleanup_(unlink_and_freep) char *t = NULL;
         _cleanup_close_ int fdt = -1;
         int r;
@@ -745,7 +817,7 @@ int copy_file_atomic(const char *from, const char *to, mode_t mode, unsigned cha
         if (chattr_flags != 0)
                 (void) chattr_fd(fdt, chattr_flags, (unsigned) -1, NULL);
 
-        r = copy_file_fd(from, fdt, copy_flags);
+        r = copy_file_fd_full(from, fdt, copy_flags, progress_bytes, userdata);
         if (r < 0)
                 return r;
 

src/basic/copy.h

@@ -1,9 +1,11 @@
 /* SPDX-License-Identifier: LGPL-2.1+ */
 #pragma once
 
+#include <fcntl.h>
 #include <inttypes.h>
 #include <stdbool.h>
 #include <stdint.h>
+#include <sys/stat.h>
 #include <sys/types.h>
 
 typedef enum CopyFlags {
@@ -13,16 +15,46 @@ typedef enum CopyFlags {
         COPY_SAME_MOUNT = 1 << 3, /* Don't descend recursively into other file systems, across mount point boundaries */
 } CopyFlags;
 
-int copy_file_fd(const char *from, int to, CopyFlags copy_flags);
-int copy_file(const char *from, const char *to, int open_flags, mode_t mode, unsigned chattr_flags, CopyFlags copy_flags);
-int copy_file_atomic(const char *from, const char *to, mode_t mode, unsigned chattr_flags, CopyFlags copy_flags);
-int copy_tree(const char *from, const char *to, uid_t override_uid, gid_t override_gid, CopyFlags copy_flags);
-int copy_tree_at(int fdf, const char *from, int fdt, const char *to, uid_t override_uid, gid_t override_gid, CopyFlags copy_flags);
-int copy_directory_fd(int dirfd, const char *to, CopyFlags copy_flags);
-int copy_directory(const char *from, const char *to, CopyFlags copy_flags);
-int copy_bytes_full(int fdf, int fdt, uint64_t max_bytes, CopyFlags copy_flags, void **ret_remains, size_t *ret_remains_size);
-static inline int copy_bytes(int fdf, int fdt, uint64_t max_bytes, CopyFlags copy_flags) {
-        return copy_bytes_full(fdf, fdt, max_bytes, copy_flags, NULL, NULL);
+typedef int (*copy_progress_bytes_t)(uint64_t n_bytes, void *userdata);
+typedef int (*copy_progress_path_t)(const char *path, const struct stat *st, void *userdata);
+
+int copy_file_fd_full(const char *from, int to, CopyFlags copy_flags, copy_progress_bytes_t progress, void *userdata);
+static inline int copy_file_fd(const char *from, int to, CopyFlags copy_flags) {
+        return copy_file_fd_full(from, to, copy_flags, NULL, NULL);
 }
+
+int copy_file_full(const char *from, const char *to, int open_flags, mode_t mode, unsigned chattr_flags, CopyFlags copy_flags, copy_progress_bytes_t progress, void *userdata);
+static inline int copy_file(const char *from, const char *to, int open_flags, mode_t mode, unsigned chattr_flags, CopyFlags copy_flags) {
+        return copy_file_full(from, to, open_flags, mode, chattr_flags, copy_flags, NULL, NULL);
+}
+
+int copy_file_atomic_full(const char *from, const char *to, mode_t mode, unsigned chattr_flags, CopyFlags copy_flags, copy_progress_bytes_t progress, void *userdata);
+static inline int copy_file_atomic(const char *from, const char *to, mode_t mode, unsigned chattr_flags, CopyFlags copy_flags) {
+        return copy_file_atomic_full(from, to, mode, chattr_flags, copy_flags, NULL, NULL);
+}
+
+int copy_tree_at_full(int fdf, const char *from, int fdt, const char *to, uid_t override_uid, gid_t override_gid, CopyFlags copy_flags, copy_progress_path_t progress_path, copy_progress_bytes_t progress_bytes, void *userdata);
+static inline int copy_tree_at(int fdf, const char *from, int fdt, const char *to, uid_t override_uid, gid_t override_gid, CopyFlags copy_flags) {
+        return copy_tree_at_full(fdf, from, fdt, to, override_uid, override_gid, copy_flags, NULL, NULL, NULL);
+}
+static inline int copy_tree(const char *from, const char *to, uid_t override_uid, gid_t override_gid, CopyFlags copy_flags) {
+        return copy_tree_at_full(AT_FDCWD, from, AT_FDCWD, to, override_uid, override_gid, copy_flags, NULL, NULL, NULL);
+}
+
+int copy_directory_fd_full(int dirfd, const char *to, CopyFlags copy_flags, copy_progress_path_t progress_path, copy_progress_bytes_t progress_bytes, void *userdata);
+static inline int copy_directory_fd(int dirfd, const char *to, CopyFlags copy_flags) {
+        return copy_directory_fd_full(dirfd, to, copy_flags, NULL, NULL, NULL);
+}
+
+int copy_directory_full(const char *from, const char *to, CopyFlags copy_flags, copy_progress_path_t progress_path, copy_progress_bytes_t progress_bytes, void *userdata);
+static inline int copy_directory(const char *from, const char *to, CopyFlags copy_flags) {
+        return copy_directory_full(from, to, copy_flags, NULL, NULL, NULL);
+}
+
+int copy_bytes_full(int fdf, int fdt, uint64_t max_bytes, CopyFlags copy_flags, void **ret_remains, size_t *ret_remains_size, copy_progress_bytes_t progress, void *userdata);
+static inline int copy_bytes(int fdf, int fdt, uint64_t max_bytes, CopyFlags copy_flags) {
+        return copy_bytes_full(fdf, fdt, max_bytes, copy_flags, NULL, NULL, NULL, NULL);
+}
+
 int copy_times(int fdf, int fdt);
 int copy_xattr(int fdf, int fdt);

src/basic/fd-util.c

@@ -648,7 +648,7 @@ int fd_duplicate_data_fd(int fd) {
 
                         if ((size_t) isz >= DATA_FD_MEMORY_LIMIT) {
 
-                                r = copy_bytes_full(fd, pipefds[1], DATA_FD_MEMORY_LIMIT, 0, &remains, &remains_size);
+                                r = copy_bytes_full(fd, pipefds[1], DATA_FD_MEMORY_LIMIT, 0, &remains, &remains_size, NULL, NULL);
                                 if (r < 0 && r != -EAGAIN)
                                         return r; /* If we get EAGAIN it could be because of the source or because of
                                                    * the destination fd, we can't know, as sendfile() and friends won't