From 14de16aabef531227b183b17853b5661ae82256f Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Thu, 7 Apr 2016 15:33:32 +0200 Subject: [PATCH 1/9] btrbk: add action "clone": recursively send/receive all backups to an archive directory --- btrbk | 256 +++++++++++++++++++++++++++++++++++++++++++++++++++- doc/btrbk.1 | 22 +++++ 2 files changed, 276 insertions(+), 2 deletions(-) diff --git a/btrbk b/btrbk index 9b82ae3..e4d9a74 100755 --- a/btrbk +++ b/btrbk @@ -86,6 +86,8 @@ my %config_options = ( snapshot_preserve_min => { default => "1d", accept => [ "all", "latest" ], accept_regexp => qr/^[1-9][0-9]*[hdwmy]$/, context => [ "root", "volume", "subvolume" ], }, target_preserve => { default => undef, accept => [ "no" ], accept_preserve_matrix => 1 }, target_preserve_min => { default => undef, accept => [ "all", "latest", "no" ], accept_regexp => qr/^[0-9]+[hdwmy]$/ }, + archive_preserve => { default => undef, accept => [ "no" ], accept_preserve_matrix => 1 }, + archive_preserve_min => { default => "all", accept => [ "all", "latest", "no" ], accept_regexp => qr/^[0-9]+[hdwmy]$/ }, btrfs_commit_delete => { default => undef, accept => [ "after", "each", "no" ] }, ssh_identity => { default => undef, accept_file => { absolute => 1 } }, ssh_user => { default => "root", accept_regexp => qr/^[a-z_][a-z0-9_-]*$/ }, @@ -255,6 +257,7 @@ sub HELP_MESSAGE print STDERR " volume configured volume sections\n"; print STDERR " target configured targets\n"; print STDERR " clean delete incomplete (garbled) backups\n"; + print STDERR " clone recursively copy all subvolumes (experimental)\n"; print STDERR " usage print filesystem usage\n"; print STDERR " origin print origin information for subvolume\n"; print STDERR " diff shows new files since subvolume for subvolume \n"; @@ -2479,7 +2482,7 @@ sub macro_send_receive(@) { # create backup from latest common if($parent) { - INFO "Incremental from parent subvolume: $parent->{PRINT}"; + INFO "Incremental from parent: $parent->{PRINT}"; } elsif($incremental ne "strict") { INFO "No common parent subvolume present, creating full backup"; @@ -2614,6 +2617,90 @@ sub macro_delete($$$$$;@) } +sub macro_clone_target($$$;$) +{ + my $sroot = shift || die; + my $droot = shift || die; + my $snapshot_name = shift // die; + my $schedule_options = shift // {}; + my @schedule; + + foreach my $dvol (@{vinfo_subvol_list($droot, sort => 'path')}) + { + next unless($dvol->{btrbk_direct_leaf} && ($dvol->{BTRBK_BASENAME} eq $snapshot_name)); + next unless($dvol->{node}{readonly}); + + # add all present archives to schedule, with no value. + # these are needed for correct results of schedule() + push @schedule, { value => undef, + btrbk_date => $dvol->{BTRBK_DATE}, + preserve => $dvol->{FORCE_PRESERVE}, + }; + } + + # NOTE: this is pretty much the same as "resume missing" + my $abort_unexpected_location = 0; + foreach my $svol (@{vinfo_subvol_list($sroot, sort => 'path')}) + { + next unless($svol->{node}{readonly}); + next unless($svol->{btrbk_direct_leaf} && ($svol->{BTRBK_BASENAME} eq $snapshot_name)); + + my $unexpected_count = 0; + my @receive_targets = get_receive_targets($droot, $svol, exact_match => 1, warn_unexpected => 1, ret_unexpected => \$unexpected_count); + # don't abort right here, we want to warn about all unexpected targets + $abort_unexpected_location += $unexpected_count; + next if($abort_unexpected_location || scalar(@receive_targets)); + + DEBUG "Adding target candidate: $svol->{PRINT}"; + push @schedule, { value => $svol, + name => $svol->{PRINT}, # only for logging + btrbk_date => $svol->{BTRBK_DATE}, + preserve => $svol->{FORCE_PRESERVE}, + }; + } + if($abort_unexpected_location) { + ABORTED($droot, "Receive targets of archive candidates exist at unexpected location"); + WARN "Skipping archiving of \"$sroot->{PRINT}/${snapshot_name}.*\": $abrt"; + return undef; + } + + my ($preserve, undef) = schedule( + schedule => \@schedule, + preserve => config_preserve_hash($droot, "archive"), + result_preserve_action_text => 'archive', + result_delete_action_text => '', + %$schedule_options + ); + my @archive = grep defined, @$preserve; # remove entries with no value from list (archive subvolumes) + my $archive_total = scalar @archive; + my $archive_success = 0; + foreach my $svol (@archive) + { + my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot, ""); + if(macro_send_receive(source => $svol, + target => $droot, + parent => $latest_common_src, + latest_common_target => $latest_common_target, + )) + { + $archive_success++; + } + else { + ERROR("Error while cloning, aborting"); + last; + } + } + + if($archive_total) { + INFO "Archived $archive_success/$archive_total subvolumes"; + } else { + INFO "No missing archived subvolume found"; + } + + return $archive_success; +} + + sub cmp_date($$) { my ($a,$b) = @_; @@ -3074,7 +3161,7 @@ MAIN: WARN 'Found option "--progress", but "pv" is not present: (please install "pv")'; $show_progress = 0; } - my ($action_run, $action_usage, $action_resolve, $action_diff, $action_origin, $action_config_print, $action_list, $action_clean); + my ($action_run, $action_usage, $action_resolve, $action_diff, $action_origin, $action_config_print, $action_list, $action_clean, $action_clone); my @filter_args; my $args_allow_group = 1; my $args_expected_min = 0; @@ -3089,6 +3176,12 @@ MAIN: $action_clean = 1; @filter_args = @ARGV; } + elsif ($command eq "clone") { + $action_clone = 1; + $args_expected_min = $args_expected_max = 2; + $args_allow_group = 0; + @filter_args = @ARGV; + } elsif ($command eq "usage") { $action_usage = 1; @filter_args = @ARGV; @@ -3299,6 +3392,165 @@ MAIN: } + if($action_clone) + { + # + # clone (archive) tree + # + # NOTE: This is intended to work without a config file! The only + # thing used from the configuration is the SSH and transaction log + # stuff. + # + init_transaction_log(config_key($config, "transaction_log")); + + my $src_url = $filter_args[0] || die; + my $target_url = $filter_args[1] || die; + + # FIXME: add command line options for preserve logic + $config->{SUBSECTION} = []; # clear configured subsections, we build them dynamically + + my $clone_src_root = vinfo($src_url, $config); + unless(vinfo_init_root($clone_src_root, resolve_subdir => 1)) { + ERROR "Failed to fetch subvolume detail for '$clone_src_root->{PRINT}'" . ($err ? ": $err" : ""); + exit 1; + } + my $target_root = vinfo($target_url, $config); + unless(vinfo_init_root($target_root, resolve_subdir => 1)) { + ERROR "Failed to fetch subvolume detail for '$target_root->{PRINT}'" . ($err ? ": $err" : ""); + exit 1; + } + + my %name_uniq; + my @subvol_list = @{vinfo_subvol_list($clone_src_root)}; + my @sorted = sort { ($a->{subtree_depth} <=> $b->{subtree_depth}) || ($a->{SUBVOL_DIR} cmp $b->{SUBVOL_DIR}) } @subvol_list; + foreach my $vol (@sorted) { + next unless($vol->{node}{readonly}); + my $snapshot_name = $vol->{BTRBK_BASENAME}; + unless(defined($snapshot_name)) { + WARN "Skipping subvolume (not a btrbk subvolume): $vol->{PRINT}"; + next; + } + my $subvol_dir = $vol->{SUBVOL_DIR}; + next if($name_uniq{"$subvol_dir/$snapshot_name"}); + $name_uniq{"$subvol_dir/$snapshot_name"} = 1; + $subvol_dir = '/' . $subvol_dir if($subvol_dir ne ""); + my $droot_url = $target_url . $subvol_dir; + my $sroot_url = $src_url . $subvol_dir; + my $config_clone_src = { CONTEXT => "clone_source", + PARENT => $config, + url => $sroot_url, # ABORTED() needs this + snapshot_name => $snapshot_name, + }; + my $config_target = { CONTEXT => "target", + PARENT => $config_clone_src, + target_type => "send-receive", # macro_send_receive checks this + url => $droot_url, # ABORTED() needs this + }; + $config_clone_src->{SUBSECTION} = [ $config_target ]; + push(@{$config->{SUBSECTION}}, $config_clone_src); + + my $sroot = vinfo($sroot_url, $config_clone_src); + vinfo_assign_config($sroot, $config_clone_src); + unless(vinfo_init_root($sroot, resolve_subdir => 1)) { + ABORTED($sroot, "Failed to fetch subvolume detail" . ($err ? ": $err" : "")); + WARN "Skipping clone source \"$sroot->{PRINT}\": $abrt"; + next; + } + + my $droot = vinfo($droot_url, $config_target); + vinfo_assign_config($droot, $config_target); + unless(vinfo_init_root($droot, resolve_subdir => 1)) { + ABORTED($droot, "Failed to fetch subvolume detail" . ($err ? ": $err" : "")); + WARN "Skipping clone target \"$droot->{PRINT}\": $abrt"; + next; + } + if(_is_child_of($droot->{node}->{TREE_ROOT}, $vol->{node}{uuid})) { + ERROR "Source and target subvolumes are on the same btrfs filesystem!"; + exit 1; + } + } + + foreach my $sroot (vinfo_subsection($config, 'clone_source')) { + foreach my $droot (vinfo_subsection($sroot, 'target')) { + my $snapshot_name = config_key($droot, "snapshot_name") // die; + INFO "Archiving subvolumes: $sroot->{PRINT}/${snapshot_name}.*"; + macro_clone_target($sroot, $droot, $snapshot_name); + } + } + + my $exit_status = exit_status($config); + my $time_elapsed = time - $start_time; + INFO "Completed within: ${time_elapsed}s (" . localtime(time) . ")"; + action("finished", + status => $exit_status ? "partial" : "success", + duration => $time_elapsed, + message => $exit_status ? "At least one backup task aborted" : undef, + ); + close_transaction_log(); + + unless($quiet) + { + # + # print summary + # + $output_format ||= "custom"; + if($output_format eq "custom") + { + my @unrecoverable; + my @out; + foreach my $sroot (vinfo_subsection($config, 'clone_source')) { + foreach my $droot (vinfo_subsection($sroot, 'target', 1)) { + my @subvol_out; + foreach(@{$droot->{SUBVOL_RECEIVED} // []}) { + my $create_mode = "***"; + $create_mode = ">>>" if($_->{parent}); + $create_mode = "!!!" if($_->{ERROR}); + push @subvol_out, "$create_mode $_->{received_subvolume}->{PRINT}"; + } + if(ABORTED($droot) && (ABORTED($droot) ne "USER_SKIP")) { + push @subvol_out, "!!! Target \"$droot->{PRINT}\" aborted: " . ABORTED($droot);; + } + if($droot->{CONFIG}->{UNRECOVERABLE}) { + push(@unrecoverable, $droot->{CONFIG}->{UNRECOVERABLE}); + } + if(@subvol_out) { + push @out, "$sroot->{PRINT}/$sroot->{CONFIG}->{snapshot_name}.*", @subvol_out, ""; + } + } + } + + print_header(title => "Clone Summary", + time => $start_time, + legend => [ + "--- deleted subvolume", + "*** received subvolume (non-incremental)", + ">>> received subvolume (incremental)", + ], + ); + + print join("\n", @out); + + if($exit_status) { + print "\nNOTE: Some errors occurred, which may result in missing backups!\n"; + print "Please check warning and error messages above.\n"; + print join("\n", @unrecoverable) . "\n" if(@unrecoverable); + } + if($dryrun) { + print "\nNOTE: Dryrun was active, none of the operations above were actually executed!\n"; + } + } + else + { + # print action log (without transaction start messages) + my @data = grep { $_->{status} ne "starting" } @transaction_log; + print_formatted("transaction", \@data, title => "TRANSACTION LOG"); + } + } + + exit $exit_status; + } + + # # expand subvolume globs (wildcards) # diff --git a/doc/btrbk.1 b/doc/btrbk.1 index 725dd3c..261fc35 100644 --- a/doc/btrbk.1 +++ b/doc/btrbk.1 @@ -167,6 +167,28 @@ by the \fBrun\fR command. Use in conjunction with \fI\-l debug\fR to see the btrfs commands that would be executed. .RE .PP +.B clone + +.I *experimental* +.RS 4 +Recursively copy all subvolumes created by btrbk from to + directory, optionally rescheduled using +\fIarchive_preserve_*\fR configuration options. Useful for creating +extra archive copies (clones) from your backup disks. Note that you +can continue using btrbk after swapping your backup disk with the +cloned disk. +.PP +Note that this feature needs a \fBlinux kernel >=4.4\fR to work +correctly! Kernels >=4.1 and <4.4 have a bug when re-sending +subvolumes (the cloned subvolumes will have incorrect received_uuid, +see ), so +make sure you run a recent kernel. +.PP +Known bugs: On the target filesystem, you have to create all +directories by hand, or "btrbk clone" will complain: "WARNING: +Skipping clone target <...>: Failed to fetch subvolume detail". +.RE +.PP .B stats [filter...] .RS 4 From a5c7c53a86aff2737009b6061148dd1d659bef73 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Thu, 14 Apr 2016 15:39:50 +0200 Subject: [PATCH 2/9] btrbk: action "clone": print scheduler results if --print-schedule option is set --- btrbk | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/btrbk b/btrbk index e4d9a74..a6fbdcc 100755 --- a/btrbk +++ b/btrbk @@ -2720,6 +2720,8 @@ sub schedule(@) my $preserve = $args{preserve} || die; my $results_list = $args{results}; my $result_hints = $args{result_hints} // {}; + my $result_preserve_action_text = $args{result_preserve_action_text}; + my $result_delete_action_text = $args{result_delete_action_text} // 'delete'; my $preserve_day_of_week = $preserve->{dow} || die; my $preserve_min_n = $preserve->{min_n}; @@ -2848,7 +2850,7 @@ sub schedule(@) push(@preserve, $href->{value}); DEBUG "Schedule: $href->{name}: $href->{preserve}" if($href->{name}); push @$results_list, { %result_base, - # action => "preserve", + action => $result_preserve_action_text, reason => $href->{preserve}, value => $href->{value}, } if($results_list); @@ -2858,7 +2860,7 @@ sub schedule(@) push(@delete, $href->{value}); DEBUG "Schedule: $href->{name}: delete" if($href->{name}); push @$results_list, { %result_base, - action => "delete", + action => $result_delete_action_text, value => $href->{value}, } if($results_list); } @@ -3470,11 +3472,12 @@ MAIN: } } + my $schedule_results = []; foreach my $sroot (vinfo_subsection($config, 'clone_source')) { foreach my $droot (vinfo_subsection($sroot, 'target')) { my $snapshot_name = config_key($droot, "snapshot_name") // die; INFO "Archiving subvolumes: $sroot->{PRINT}/${snapshot_name}.*"; - macro_clone_target($sroot, $droot, $snapshot_name); + macro_clone_target($sroot, $droot, $snapshot_name, { results => $schedule_results }); } } @@ -3490,9 +3493,14 @@ MAIN: unless($quiet) { - # + # print scheduling results + if($print_schedule) { + my @data = map { { %$_, vinfo_prefixed_keys("", $_->{value}) }; } @$schedule_results; + print_formatted("schedule", \@data, title => "CLONE SCHEDULE"); + print "\n"; + } + # print summary - # $output_format ||= "custom"; if($output_format eq "custom") { From d0cfba7914dbdefddbd4d7fab29ea8f25e3bd201 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Thu, 14 Apr 2016 18:46:35 +0200 Subject: [PATCH 3/9] btrbk: action "clone": create missing directories --- btrbk | 44 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/btrbk b/btrbk index a6fbdcc..996f2e3 100755 --- a/btrbk +++ b/btrbk @@ -1154,6 +1154,23 @@ sub system_realpath($) } +sub system_mkdir($) +{ + my $vol = shift // die; + my $path = $vol->{PATH} // die;; + INFO "Creating directory: $vol->{PRINT}/"; + my $ret = run_cmd(cmd => [ qw(mkdir), '-p', $path ], + rsh => $vol->{RSH}, + ); + action("mkdir", + vinfo_prefixed_keys("target", $vol), + status => ($dryrun ? "DRYRUN" : (defined($ret) ? "success" : "ERROR")), + ); + return undef unless(defined($ret)); + return 1; +} + + sub btrfs_mountpoint($) { my $vol = shift // die; @@ -3435,9 +3452,8 @@ MAIN: my $subvol_dir = $vol->{SUBVOL_DIR}; next if($name_uniq{"$subvol_dir/$snapshot_name"}); $name_uniq{"$subvol_dir/$snapshot_name"} = 1; - $subvol_dir = '/' . $subvol_dir if($subvol_dir ne ""); - my $droot_url = $target_url . $subvol_dir; - my $sroot_url = $src_url . $subvol_dir; + my $droot_url = $target_url . ($subvol_dir eq "" ? "" : "/$subvol_dir"); + my $sroot_url = $src_url . ($subvol_dir eq "" ? "" : "/$subvol_dir"); my $config_clone_src = { CONTEXT => "clone_source", PARENT => $config, url => $sroot_url, # ABORTED() needs this @@ -3462,9 +3478,25 @@ MAIN: my $droot = vinfo($droot_url, $config_target); vinfo_assign_config($droot, $config_target); unless(vinfo_init_root($droot, resolve_subdir => 1)) { - ABORTED($droot, "Failed to fetch subvolume detail" . ($err ? ": $err" : "")); - WARN "Skipping clone target \"$droot->{PRINT}\": $abrt"; - next; + DEBUG("Failed to fetch subvolume detail" . ($err ? ": $err" : "")); + unless(system_mkdir($droot)) { + ABORTED($droot, "Failed to create directory: $droot->{PRINT}/"); + WARN "Skipping clone target \"$droot->{PRINT}\": $abrt"; + next; + } + if($dryrun) { + # we need to fake this directory on dryrun + $droot->{node} = $target_root->{node}; + $droot->{NODE_SUBDIR} = $subvol_dir; + } + else { + # after directory is created, try to init again + unless(vinfo_init_root($droot, resolve_subdir => 1)) { + ABORTED($droot, "Failed to fetch subvolume detail" . ($err ? ": $err" : "")); + WARN "Skipping clone target \"$droot->{PRINT}\": $abrt"; + next; + } + } } if(_is_child_of($droot->{node}->{TREE_ROOT}, $vol->{node}{uuid})) { ERROR "Source and target subvolumes are on the same btrfs filesystem!"; From cc20dfb8c3eda3294a1d906e4a4f1ebbba1b0258 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Fri, 15 Apr 2016 22:33:19 +0200 Subject: [PATCH 4/9] btrbk: schedule(): use "informative_only" option instead of checking for empty value; make sure real values are always in front --- btrbk | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/btrbk b/btrbk index 996f2e3..cc7508e 100755 --- a/btrbk +++ b/btrbk @@ -2642,19 +2642,6 @@ sub macro_clone_target($$$;$) my $schedule_options = shift // {}; my @schedule; - foreach my $dvol (@{vinfo_subvol_list($droot, sort => 'path')}) - { - next unless($dvol->{btrbk_direct_leaf} && ($dvol->{BTRBK_BASENAME} eq $snapshot_name)); - next unless($dvol->{node}{readonly}); - - # add all present archives to schedule, with no value. - # these are needed for correct results of schedule() - push @schedule, { value => undef, - btrbk_date => $dvol->{BTRBK_DATE}, - preserve => $dvol->{FORCE_PRESERVE}, - }; - } - # NOTE: this is pretty much the same as "resume missing" my $abort_unexpected_location = 0; foreach my $svol (@{vinfo_subvol_list($sroot, sort => 'path')}) @@ -2668,11 +2655,11 @@ sub macro_clone_target($$$;$) $abort_unexpected_location += $unexpected_count; next if($abort_unexpected_location || scalar(@receive_targets)); - DEBUG "Adding target candidate: $svol->{PRINT}"; + DEBUG "Adding archive candidate: $svol->{PRINT}"; + push @schedule, { value => $svol, - name => $svol->{PRINT}, # only for logging btrbk_date => $svol->{BTRBK_DATE}, - preserve => $svol->{FORCE_PRESERVE}, + preserve => $svol->{node}{FORCE_PRESERVE}, }; } if($abort_unexpected_location) { @@ -2681,6 +2668,18 @@ sub macro_clone_target($$$;$) return undef; } + foreach my $dvol (@{vinfo_subvol_list($droot, sort => 'path')}) + { + next unless($dvol->{btrbk_direct_leaf} && ($dvol->{BTRBK_BASENAME} eq $snapshot_name)); + next unless($dvol->{node}{readonly}); + # add all present archives to schedule, with no value. + # these are needed for correct results of schedule() + push @schedule, { informative_only => 1, + value => $dvol, + btrbk_date => $dvol->{BTRBK_DATE}, + }; + } + my ($preserve, undef) = schedule( schedule => \@schedule, preserve => config_preserve_hash($droot, "archive"), @@ -2755,7 +2754,10 @@ sub schedule(@) DEBUG "Schedule: " . format_preserve_matrix($preserve, format => "debug_text"); # sort the schedule, ascending by date - my @sorted_schedule = sort { cmp_date($a->{btrbk_date}, $b->{btrbk_date} ) } @$schedule; + # regular entries come in front of informative_only + my @sorted_schedule = sort { cmp_date($a->{btrbk_date}, $b->{btrbk_date} ) || + (($a->{informative_only} ? ($b->{informative_only} ? 0 : 1) : ($b->{informative_only} ? -1 : 0))) + } @$schedule; # first, do our calendar calculations # note: our week starts on $preserve_day_of_week @@ -2861,20 +2863,19 @@ sub schedule(@) my $count_defined = 0; foreach my $href (@sorted_schedule) { - next unless(defined($href->{value})); - $count_defined++; + $count_defined++ unless($href->{informative_only}); if($href->{preserve}) { - push(@preserve, $href->{value}); + push(@preserve, $href->{value}) unless($href->{informative_only}); DEBUG "Schedule: $href->{name}: $href->{preserve}" if($href->{name}); push @$results_list, { %result_base, - action => $result_preserve_action_text, + action => $href->{informative_only} ? 'seen' : $result_preserve_action_text, reason => $href->{preserve}, value => $href->{value}, } if($results_list); } else { - push(@delete, $href->{value}); + push(@delete, $href->{value}) unless($href->{informative_only}); DEBUG "Schedule: $href->{name}: delete" if($href->{name}); push @$results_list, { %result_base, action => $result_delete_action_text, @@ -4513,10 +4514,9 @@ MAIN: TRACE "Receive target does not match btrbk filename scheme, skipping: $vol->{PRINT}"; next; } - push(@schedule, { value => undef, + push(@schedule, { informative_only => 1, + value => $vol, btrbk_date => $vol->{BTRBK_DATE}, - # not enforcing resuming of latest snapshot anymore (since v0.23.0) - # preserve => $vol->{node}{FORCE_PRESERVE}, }); } my ($preserve, undef) = schedule( From 4bd68a2e35f7bb64ddaabe3904526db384c997e6 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Sat, 16 Apr 2016 00:45:16 +0200 Subject: [PATCH 5/9] btrbk: action "clone": abort all as soon as one target aborts --- btrbk | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/btrbk b/btrbk index cc7508e..4bca40a 100755 --- a/btrbk +++ b/btrbk @@ -3511,7 +3511,14 @@ MAIN: my $snapshot_name = config_key($droot, "snapshot_name") // die; INFO "Archiving subvolumes: $sroot->{PRINT}/${snapshot_name}.*"; macro_clone_target($sroot, $droot, $snapshot_name, { results => $schedule_results }); + if(ABORTED($droot)) { + # also abort $sroot + ABORTED($sroot, "At least one target aborted"); + WARN "Skipping archiving of \"$sroot->{PRINT}/\": $abrt"; + last; + } } + last if(ABORTED($sroot)); } my $exit_status = exit_status($config); From de05b997572cee11349db6bdb98e44d67f8cd318 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Sat, 16 Apr 2016 01:09:17 +0200 Subject: [PATCH 6/9] btrbk: rename action "clone" to "archive" (should have been like this from the beginning) --- btrbk | 100 ++++++++++++++++++++++++++-------------------------- doc/FAQ.md | 47 +++++++++--------------- doc/btrbk.1 | 19 +++++----- 3 files changed, 77 insertions(+), 89 deletions(-) diff --git a/btrbk b/btrbk index 4bca40a..92b8947 100755 --- a/btrbk +++ b/btrbk @@ -245,22 +245,22 @@ sub HELP_MESSAGE print STDERR " --progress show progress bar on send-receive operation\n"; print STDERR "\n"; print STDERR "commands:\n"; - print STDERR " run perform backup operations as defined in the config file\n"; - print STDERR " dryrun don't run btrfs commands; show what would be executed\n"; - print STDERR " stats print snapshot/backup statistics\n"; - print STDERR " list available subcommands are:\n"; - print STDERR " backups all backups and corresponding snapshots\n"; - print STDERR " snapshots all snapshots and corresponding backups\n"; - print STDERR " latest most recent snapshots and backups\n"; - print STDERR " config configured source/snapshot/target relations\n"; - print STDERR " source configured source/snapshot relations\n"; - print STDERR " volume configured volume sections\n"; - print STDERR " target configured targets\n"; - print STDERR " clean delete incomplete (garbled) backups\n"; - print STDERR " clone recursively copy all subvolumes (experimental)\n"; - print STDERR " usage print filesystem usage\n"; - print STDERR " origin print origin information for subvolume\n"; - print STDERR " diff shows new files since subvolume for subvolume \n"; + print STDERR " run perform backup operations as defined in the config\n"; + print STDERR " dryrun don't run btrfs commands; show what would be executed\n"; + print STDERR " stats print snapshot/backup statistics\n"; + print STDERR " list available subcommands are:\n"; + print STDERR " backups all backups and corresponding snapshots\n"; + print STDERR " snapshots all snapshots and corresponding backups\n"; + print STDERR " latest most recent snapshots and backups\n"; + print STDERR " config configured source/snapshot/target relations\n"; + print STDERR " source configured source/snapshot relations\n"; + print STDERR " volume configured volume sections\n"; + print STDERR " target configured targets\n"; + print STDERR " clean delete incomplete (garbled) backups\n"; + print STDERR " archive recursively copy all subvolumes (experimental)\n"; + print STDERR " usage print filesystem usage\n"; + print STDERR " origin print origin information for subvolume\n"; + print STDERR " diff shows new files between related subvolumes\n"; print STDERR "\n"; print STDERR "For additional information, see $PROJECT_HOME\n"; } @@ -2634,7 +2634,7 @@ sub macro_delete($$$$$;@) } -sub macro_clone_target($$$;$) +sub macro_archive_target($$$;$) { my $sroot = shift || die; my $droot = shift || die; @@ -3181,7 +3181,7 @@ MAIN: WARN 'Found option "--progress", but "pv" is not present: (please install "pv")'; $show_progress = 0; } - my ($action_run, $action_usage, $action_resolve, $action_diff, $action_origin, $action_config_print, $action_list, $action_clean, $action_clone); + my ($action_run, $action_usage, $action_resolve, $action_diff, $action_origin, $action_config_print, $action_list, $action_clean, $action_archive); my @filter_args; my $args_allow_group = 1; my $args_expected_min = 0; @@ -3196,8 +3196,8 @@ MAIN: $action_clean = 1; @filter_args = @ARGV; } - elsif ($command eq "clone") { - $action_clone = 1; + elsif ($command eq "archive") { + $action_archive = 1; $args_expected_min = $args_expected_max = 2; $args_allow_group = 0; @filter_args = @ARGV; @@ -3412,10 +3412,10 @@ MAIN: } - if($action_clone) + if($action_archive) { # - # clone (archive) tree + # archive (clone) tree # # NOTE: This is intended to work without a config file! The only # thing used from the configuration is the SSH and transaction log @@ -3429,9 +3429,9 @@ MAIN: # FIXME: add command line options for preserve logic $config->{SUBSECTION} = []; # clear configured subsections, we build them dynamically - my $clone_src_root = vinfo($src_url, $config); - unless(vinfo_init_root($clone_src_root, resolve_subdir => 1)) { - ERROR "Failed to fetch subvolume detail for '$clone_src_root->{PRINT}'" . ($err ? ": $err" : ""); + my $src_root = vinfo($src_url, $config); + unless(vinfo_init_root($src_root, resolve_subdir => 1)) { + ERROR "Failed to fetch subvolume detail for '$src_root->{PRINT}'" . ($err ? ": $err" : ""); exit 1; } my $target_root = vinfo($target_url, $config); @@ -3441,7 +3441,7 @@ MAIN: } my %name_uniq; - my @subvol_list = @{vinfo_subvol_list($clone_src_root)}; + my @subvol_list = @{vinfo_subvol_list($src_root)}; my @sorted = sort { ($a->{subtree_depth} <=> $b->{subtree_depth}) || ($a->{SUBVOL_DIR} cmp $b->{SUBVOL_DIR}) } @subvol_list; foreach my $vol (@sorted) { next unless($vol->{node}{readonly}); @@ -3455,34 +3455,34 @@ MAIN: $name_uniq{"$subvol_dir/$snapshot_name"} = 1; my $droot_url = $target_url . ($subvol_dir eq "" ? "" : "/$subvol_dir"); my $sroot_url = $src_url . ($subvol_dir eq "" ? "" : "/$subvol_dir"); - my $config_clone_src = { CONTEXT => "clone_source", - PARENT => $config, - url => $sroot_url, # ABORTED() needs this - snapshot_name => $snapshot_name, - }; - my $config_target = { CONTEXT => "target", - PARENT => $config_clone_src, - target_type => "send-receive", # macro_send_receive checks this - url => $droot_url, # ABORTED() needs this - }; - $config_clone_src->{SUBSECTION} = [ $config_target ]; - push(@{$config->{SUBSECTION}}, $config_clone_src); + my $config_sroot = { CONTEXT => "archive_source", + PARENT => $config, + url => $sroot_url, # ABORTED() needs this + snapshot_name => $snapshot_name, + }; + my $config_droot = { CONTEXT => "target", + PARENT => $config_sroot, + target_type => "send-receive", # macro_send_receive checks this + url => $droot_url, # ABORTED() needs this + }; + $config_sroot->{SUBSECTION} = [ $config_droot ]; + push(@{$config->{SUBSECTION}}, $config_sroot); - my $sroot = vinfo($sroot_url, $config_clone_src); - vinfo_assign_config($sroot, $config_clone_src); + my $sroot = vinfo($sroot_url, $config_sroot); + vinfo_assign_config($sroot, $config_sroot); unless(vinfo_init_root($sroot, resolve_subdir => 1)) { ABORTED($sroot, "Failed to fetch subvolume detail" . ($err ? ": $err" : "")); - WARN "Skipping clone source \"$sroot->{PRINT}\": $abrt"; + WARN "Skipping archive source \"$sroot->{PRINT}\": $abrt"; next; } - my $droot = vinfo($droot_url, $config_target); - vinfo_assign_config($droot, $config_target); + my $droot = vinfo($droot_url, $config_droot); + vinfo_assign_config($droot, $config_droot); unless(vinfo_init_root($droot, resolve_subdir => 1)) { DEBUG("Failed to fetch subvolume detail" . ($err ? ": $err" : "")); unless(system_mkdir($droot)) { ABORTED($droot, "Failed to create directory: $droot->{PRINT}/"); - WARN "Skipping clone target \"$droot->{PRINT}\": $abrt"; + WARN "Skipping archive target \"$droot->{PRINT}\": $abrt"; next; } if($dryrun) { @@ -3494,7 +3494,7 @@ MAIN: # after directory is created, try to init again unless(vinfo_init_root($droot, resolve_subdir => 1)) { ABORTED($droot, "Failed to fetch subvolume detail" . ($err ? ": $err" : "")); - WARN "Skipping clone target \"$droot->{PRINT}\": $abrt"; + WARN "Skipping archive target \"$droot->{PRINT}\": $abrt"; next; } } @@ -3506,11 +3506,11 @@ MAIN: } my $schedule_results = []; - foreach my $sroot (vinfo_subsection($config, 'clone_source')) { + foreach my $sroot (vinfo_subsection($config, 'archive_source')) { foreach my $droot (vinfo_subsection($sroot, 'target')) { my $snapshot_name = config_key($droot, "snapshot_name") // die; INFO "Archiving subvolumes: $sroot->{PRINT}/${snapshot_name}.*"; - macro_clone_target($sroot, $droot, $snapshot_name, { results => $schedule_results }); + macro_archive_target($sroot, $droot, $snapshot_name, { results => $schedule_results }); if(ABORTED($droot)) { # also abort $sroot ABORTED($sroot, "At least one target aborted"); @@ -3536,7 +3536,7 @@ MAIN: # print scheduling results if($print_schedule) { my @data = map { { %$_, vinfo_prefixed_keys("", $_->{value}) }; } @$schedule_results; - print_formatted("schedule", \@data, title => "CLONE SCHEDULE"); + print_formatted("schedule", \@data, title => "ARCHIVE SCHEDULE"); print "\n"; } @@ -3546,7 +3546,7 @@ MAIN: { my @unrecoverable; my @out; - foreach my $sroot (vinfo_subsection($config, 'clone_source')) { + foreach my $sroot (vinfo_subsection($config, 'archive_source')) { foreach my $droot (vinfo_subsection($sroot, 'target', 1)) { my @subvol_out; foreach(@{$droot->{SUBVOL_RECEIVED} // []}) { @@ -3567,7 +3567,7 @@ MAIN: } } - print_header(title => "Clone Summary", + print_header(title => "Archive Summary", time => $start_time, legend => [ "--- deleted subvolume", diff --git a/doc/FAQ.md b/doc/FAQ.md index 97576c1..7a71fb6 100644 --- a/doc/FAQ.md +++ b/doc/FAQ.md @@ -199,7 +199,23 @@ data physically, either to the datacenter or to your safe in the basement. -### Answer 1: Use external storage as "stream-fifo" +### Answer 1: Use "btrbk archive" + +A robust approach is to use external disks as archives (secondary +backups), and regularly run "btrbk archive" on them. As a nice side +effect, this also detects possible read-errors on your backup targets +(Note that a "btrfs scrub" is still more effective for that purpose). + +See **btrbk archive** command in [btrbk(1)] for more details. + +**Note that kernels >=4.1 and <4.4 have a bug when re-sending +subvolumes**, make sure you run a recent/patched kernel or step 3 will +fail. Read +[this thread on gmane](http://thread.gmane.org/gmane.comp.file-systems.btrfs/48798) +(the patch provided is confirmed working on kernels 4.2.x and 4.3.x). + + +### Answer 2: Use external storage as "stream-fifo" This example uses a USB disk as "stream-fifo" for transferring (cloning) of btrfs subvolumes: @@ -217,32 +233,3 @@ This approach has the advantage that you don't need to reformat your USB disk. This works fine, but be aware that you may run into trouble if a single stream gets corrupted, making all subsequent streams unusable. - - -### Answer 2: Clone btrfs subvolumes - -A more robust approach is to use the USB disk as secondary backup. -This has the advantage that possible errors can already be detected by -btrfs on the source side: - -1. Initialize USB disk: - - `mkfs.btrfs /dev/usbX` - -2. For all source subvolumes (in order of generation): - - `btrfs send /source/subvolX -p PARENT | btrfs receive /usbdisk/` - -3. At the target location (in order of generation): - - `btrfs send /usbdisk/subvolX -p PARENT | btrfs receive /target` - -If you simply want to have a clone of the source disk, skip step 3 and -store your USB disk in a safe. You will be able to use it for -restoring backups later, or *as a replacement for your backup disks*. - -**Note that kernels >=4.1 and <4.4 have a bug when re-sending -subvolumes**, make sure you run a recent/patched kernel or step 3 will -fail. Read -[this thread on gmane](http://thread.gmane.org/gmane.comp.file-systems.btrfs/48798) -(the patch provided is confirmed working on kernels 4.2.x and 4.3.x). diff --git a/doc/btrbk.1 b/doc/btrbk.1 index 261fc35..8769369 100644 --- a/doc/btrbk.1 +++ b/doc/btrbk.1 @@ -167,26 +167,27 @@ by the \fBrun\fR command. Use in conjunction with \fI\-l debug\fR to see the btrfs commands that would be executed. .RE .PP -.B clone +.B archive .I *experimental* .RS 4 Recursively copy all subvolumes created by btrbk from to directory, optionally rescheduled using -\fIarchive_preserve_*\fR configuration options. Useful for creating -extra archive copies (clones) from your backup disks. Note that you -can continue using btrbk after swapping your backup disk with the -cloned disk. +\fIarchive_preserve_*\fR configuration options. Also creates directory +tree on (see bugs below). Useful for creating extra archive +copies (clones) from your backup disks. Note that you can continue +using btrbk after swapping your backup disk with the archive disk. .PP Note that this feature needs a \fBlinux kernel >=4.4\fR to work correctly! Kernels >=4.1 and <4.4 have a bug when re-sending -subvolumes (the cloned subvolumes will have incorrect received_uuid, +subvolumes (the archived subvolumes will have incorrect received_uuid, see ), so make sure you run a recent kernel. .PP -Known bugs: On the target filesystem, you have to create all -directories by hand, or "btrbk clone" will complain: "WARNING: -Skipping clone target <...>: Failed to fetch subvolume detail". +Known bugs: If you want to use nested subvolumes on the target +filesystem, you need to create them by hand (e.g. by running "btrfs +subvolume create /dir"). Check the output of --dry-run if +unsure. .RE .PP .B stats From 774e6ef842473772a1611e8d14ee6b00de21846e Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Sat, 16 Apr 2016 01:32:25 +0200 Subject: [PATCH 7/9] btrbk: macro_archive_target(): only warn on unexpected targets, not abort --- btrbk | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/btrbk b/btrbk index 92b8947..3be4f86 100755 --- a/btrbk +++ b/btrbk @@ -2653,7 +2653,7 @@ sub macro_archive_target($$$;$) my @receive_targets = get_receive_targets($droot, $svol, exact_match => 1, warn_unexpected => 1, ret_unexpected => \$unexpected_count); # don't abort right here, we want to warn about all unexpected targets $abort_unexpected_location += $unexpected_count; - next if($abort_unexpected_location || scalar(@receive_targets)); + next if(scalar(@receive_targets)); DEBUG "Adding archive candidate: $svol->{PRINT}"; @@ -2662,11 +2662,13 @@ sub macro_archive_target($$$;$) preserve => $svol->{node}{FORCE_PRESERVE}, }; } - if($abort_unexpected_location) { - ABORTED($droot, "Receive targets of archive candidates exist at unexpected location"); - WARN "Skipping archiving of \"$sroot->{PRINT}/${snapshot_name}.*\": $abrt"; - return undef; - } + + # this is a bit harsh, disabled for now + # if($abort_unexpected_location) { + # ABORTED($droot, "Receive targets of archive candidates exist at unexpected location"); + # WARN "Skipping archiving of \"$sroot->{PRINT}/${snapshot_name}.*\": $abrt"; + # return undef; + # } foreach my $dvol (@{vinfo_subvol_list($droot, sort => 'path')}) { From 5a06a85619808e40a9df5afabf005723e4852730 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Sat, 16 Apr 2016 16:05:57 +0200 Subject: [PATCH 8/9] btrbk: moved get_receive_targets_fsroot() out of get_receive_targets(), while cleaning up --- btrbk | 121 +++++++++++++++++++++++++++++++++------------------------- 1 file changed, 70 insertions(+), 51 deletions(-) diff --git a/btrbk b/btrbk index 3be4f86..fbb342b 100755 --- a/btrbk +++ b/btrbk @@ -1835,28 +1835,24 @@ sub get_receive_targets($$;@) my $src_vol = shift || die; my %opts = @_; my $droot_subvols = vinfo_subvol_list($droot); - my @ret_receive_targets; - my @all_receive_targets; + my @ret; my $unexpected_count = 0; if($src_vol->{node}{is_root}) { DEBUG "Skip search for targets: source subvolume is btrfs root: $src_vol->{PRINT}"; - return @ret_receive_targets; + return @ret; } unless($src_vol->{node}{readonly}) { DEBUG "Skip search for targets: source subvolume is not read-only: $src_vol->{PRINT}"; - return @ret_receive_targets; + return @ret; } # find matches by comparing uuid / received_uuid my $uuid = $src_vol->{node}{uuid}; - my $received_uuid; - if($src_vol->{node}{received_uuid} ne '-') { - TRACE "get_receive_targets: source subvolume has received_uuid"; - $received_uuid = $src_vol->{node}{received_uuid}; - } + my $received_uuid = $src_vol->{node}{received_uuid}; + $received_uuid = undef if($received_uuid eq '-'); + TRACE "get_receive_targets: src_vol=\"$src_vol->{PRINT}\", droot=\"$droot->{PRINT}\""; - die("subvolume info not present: $uuid") unless($uuid_cache{$uuid}); foreach (@$droot_subvols) { next unless($_->{node}{readonly}); my $matched = undef; @@ -1866,43 +1862,64 @@ sub get_receive_targets($$;@) elsif(defined($received_uuid) && ($_->{node}{received_uuid} eq $received_uuid)) { $matched = 'by-received_uuid'; } - if($matched) { - push(@all_receive_targets, $_); - if($opts{exact_match} && (not exists($_->{BTRBK_RAW}))) { - unless($_->{direct_leaf} && ($_->{NAME} eq $src_vol->{NAME})) { - TRACE "get_receive_targets: $matched: skip non-exact match: $_->{PRINT}"; - WARN "Receive target of \"$src_vol->{PRINT}\" exists at unexpected location: $_->{PRINT}" if($opts{warn_unexpected}); - $unexpected_count++; - next; - } - } - TRACE "get_receive_targets: $matched: Found receive target: $_->{SUBVOL_PATH}"; - push(@ret_receive_targets, $_); - } + next unless($matched); - if($opts{warn_unexpected}) { - # search in filesystem for matching received_uuid - my @fs_match = grep({ (not $_->{is_root}) && - (($_->{received_uuid} eq $uuid) || - (defined($received_uuid) && ($_->{received_uuid} eq $received_uuid))) - } values(%{$droot->{node}{TREE_ROOT}{ID_HASH}}) ); - foreach my $node (@fs_match) { - next if(scalar grep( { $_->{node}{id} == $node->{id} } @all_receive_targets)); - my $text; - my @url = get_cached_url_by_uuid($node->{uuid}); - if(scalar(@url)) { - $text = vinfo($url[0])->{PRINT}; - } else { - $text = '"' . _fs_path($node) . "\" (in filesystem at \"$droot->{PRINT}\")"; - } - WARN "Receive target of \"$src_vol->{PRINT}\" exists at unexpected location: $text"; - $unexpected_count++; + TRACE "get_receive_targets: $matched: Found receive target: $_->{SUBVOL_PATH}"; + push(@{$opts{seen}}, $_) if($opts{seen}); + if($opts{exact_match} && !exists($_->{BTRBK_RAW})) { + if($_->{direct_leaf} && ($_->{NAME} eq $src_vol->{NAME})) { + TRACE "get_receive_targets: exact_match: $_->{SUBVOL_PATH}"; + } + else { + TRACE "get_receive_targets: $matched: skip non-exact match: $_->{PRINT}"; + WARN "Receive target of \"$src_vol->{PRINT}\" exists at unexpected location: $_->{PRINT}" if($opts{warn}); + next; } } - ${$opts{ret_unexpected}} = $unexpected_count if(ref $opts{ret_unexpected}); + push(@ret, $_); } - DEBUG "Found " . scalar(@ret_receive_targets) . " receive targets in \"$droot->{PRINT}/\" for: $src_vol->{PRINT}"; - return @ret_receive_targets; + DEBUG "Found " . scalar(@ret) . " receive targets in \"$droot->{PRINT}/\" for: $src_vol->{PRINT}"; + return @ret; +} + + +sub get_receive_targets_fsroot($$@) +{ + my $droot = shift // die; + my $src_vol = shift // die; + my %opts = @_; + my $id = $src_vol->{node}{id}; + my $uuid = $src_vol->{node}{uuid}; + my $received_uuid = $src_vol->{node}{received_uuid}; + $received_uuid = undef if(defined($received_uuid) && ($received_uuid eq '-')); + + my @unexpected; + my @exclude; + @exclude = map { $_->{node}{id} } @{$opts{exclude}} if($opts{exclude}); + + TRACE "get_receive_target_fsroot: uuid=$uuid, received_uuid=" . ($received_uuid // '-') . " exclude id={ " . join(', ', @exclude) . " }"; + + # search in filesystem for matching received_uuid + foreach my $node ( + grep({ (not $_->{is_root}) && + (($_->{received_uuid} eq $uuid) || + (defined($received_uuid) && ($_->{received_uuid} eq $received_uuid))) + } values(%{$droot->{node}{TREE_ROOT}{ID_HASH}}) ) ) + { + next if(scalar grep($_ == $node->{id}, @exclude)); + push @unexpected, $node; + if($opts{warn}) { + my $text; + my @url = get_cached_url_by_uuid($node->{uuid}); + if(scalar(@url)) { + $text = vinfo($url[0])->{PRINT}; + } else { + $text = '"' . _fs_path($node) . "\" (in filesystem at \"$droot->{PRINT}\")"; + } + WARN "Receive target of \"$src_vol->{PRINT}\" exists at unexpected location: $text"; + } + } + return \@unexpected; } @@ -2643,18 +2660,17 @@ sub macro_archive_target($$$;$) my @schedule; # NOTE: this is pretty much the same as "resume missing" - my $abort_unexpected_location = 0; + my @unexpected_location; foreach my $svol (@{vinfo_subvol_list($sroot, sort => 'path')}) { next unless($svol->{node}{readonly}); next unless($svol->{btrbk_direct_leaf} && ($svol->{BTRBK_BASENAME} eq $snapshot_name)); - my $unexpected_count = 0; - my @receive_targets = get_receive_targets($droot, $svol, exact_match => 1, warn_unexpected => 1, ret_unexpected => \$unexpected_count); - # don't abort right here, we want to warn about all unexpected targets - $abort_unexpected_location += $unexpected_count; - next if(scalar(@receive_targets)); + my $warning_seen = []; + my @receive_targets = get_receive_targets($droot, $svol, exact_match => 1, warn => 1, seen => $warning_seen ); + push @unexpected_location, get_receive_targets_fsroot($droot, $svol, exclude => $warning_seen, warn => 1); # warn if unexpected on fs + next if(scalar(@receive_targets)); DEBUG "Adding archive candidate: $svol->{PRINT}"; push @schedule, { value => $svol, @@ -2664,7 +2680,7 @@ sub macro_archive_target($$$;$) } # this is a bit harsh, disabled for now - # if($abort_unexpected_location) { + # if(scalar(@unexpected_location) { # ABORTED($droot, "Receive targets of archive candidates exist at unexpected location"); # WARN "Skipping archiving of \"$sroot->{PRINT}/${snapshot_name}.*\": $abrt"; # return undef; @@ -4498,7 +4514,10 @@ MAIN: foreach my $child (@snapshot_children) { - next if(scalar(get_receive_targets($droot, $child, exact_match => 1, warn_unexpected => 1))); + my $warning_seen = []; + my @receive_targets = get_receive_targets($droot, $child, exact_match => 1, warn => 1, seen => $warning_seen ); + get_receive_targets_fsroot($droot, $child, exclude => $warning_seen, warn => 1); # warn on unexpected on fs + next if(scalar(@receive_targets)); if(my $err_vol = vinfo_subvol($droot, $child->{NAME})) { WARN "Target subvolume \"$err_vol->{PRINT}\" exists, but is not a receive target of \"$child->{PRINT}\""; From 689d3d123371b353e1011ae99de9494ee31de3f0 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Sat, 16 Apr 2016 17:13:19 +0200 Subject: [PATCH 9/9] btrbk: action "archive": print created subdirectories on summary; cleanup --- ChangeLog | 1 + btrbk | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index ed74d54..080fdda 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,7 @@ btrbk-current * Allow wildcards in subvolume section (close: #71). * Propagate targets defined in "volume" or "root" context to all "subvolume" sections (close: #78). + * Added "archive" command (close: #79). * Added configuration option "rate_limit" (close: #72). * Added "--print-schedule" command line option. * Detect interrupted transfers of raw targets (close: #75). diff --git a/btrbk b/btrbk index fbb342b..459597b 100755 --- a/btrbk +++ b/btrbk @@ -3442,7 +3442,7 @@ MAIN: init_transaction_log(config_key($config, "transaction_log")); my $src_url = $filter_args[0] || die; - my $target_url = $filter_args[1] || die; + my $archive_url = $filter_args[1] || die; # FIXME: add command line options for preserve logic $config->{SUBSECTION} = []; # clear configured subsections, we build them dynamically @@ -3452,9 +3452,9 @@ MAIN: ERROR "Failed to fetch subvolume detail for '$src_root->{PRINT}'" . ($err ? ": $err" : ""); exit 1; } - my $target_root = vinfo($target_url, $config); - unless(vinfo_init_root($target_root, resolve_subdir => 1)) { - ERROR "Failed to fetch subvolume detail for '$target_root->{PRINT}'" . ($err ? ": $err" : ""); + my $archive_root = vinfo($archive_url, $config); + unless(vinfo_init_root($archive_root, resolve_subdir => 1)) { + ERROR "Failed to fetch subvolume detail for '$archive_root->{PRINT}'" . ($err ? ": $err" : ""); exit 1; } @@ -3471,7 +3471,7 @@ MAIN: my $subvol_dir = $vol->{SUBVOL_DIR}; next if($name_uniq{"$subvol_dir/$snapshot_name"}); $name_uniq{"$subvol_dir/$snapshot_name"} = 1; - my $droot_url = $target_url . ($subvol_dir eq "" ? "" : "/$subvol_dir"); + my $droot_url = $archive_url . ($subvol_dir eq "" ? "" : "/$subvol_dir"); my $sroot_url = $src_url . ($subvol_dir eq "" ? "" : "/$subvol_dir"); my $config_sroot = { CONTEXT => "archive_source", PARENT => $config, @@ -3503,9 +3503,10 @@ MAIN: WARN "Skipping archive target \"$droot->{PRINT}\": $abrt"; next; } + $droot->{SUBDIR_CREATED} = 1; if($dryrun) { # we need to fake this directory on dryrun - $droot->{node} = $target_root->{node}; + $droot->{node} = $archive_root->{node}; $droot->{NODE_SUBDIR} = $subvol_dir; } else { @@ -3567,6 +3568,9 @@ MAIN: foreach my $sroot (vinfo_subsection($config, 'archive_source')) { foreach my $droot (vinfo_subsection($sroot, 'target', 1)) { my @subvol_out; + if($droot->{SUBDIR_CREATED}) { + push @subvol_out, "++. $droot->{PRINT}/"; + } foreach(@{$droot->{SUBVOL_RECEIVED} // []}) { my $create_mode = "***"; $create_mode = ">>>" if($_->{parent}); @@ -3574,7 +3578,7 @@ MAIN: push @subvol_out, "$create_mode $_->{received_subvolume}->{PRINT}"; } if(ABORTED($droot) && (ABORTED($droot) ne "USER_SKIP")) { - push @subvol_out, "!!! Target \"$droot->{PRINT}\" aborted: " . ABORTED($droot);; + push @subvol_out, "!!! Target \"$droot->{PRINT}\" aborted: " . ABORTED($droot); } if($droot->{CONFIG}->{UNRECOVERABLE}) { push(@unrecoverable, $droot->{CONFIG}->{UNRECOVERABLE}); @@ -3588,7 +3592,8 @@ MAIN: print_header(title => "Archive Summary", time => $start_time, legend => [ - "--- deleted subvolume", + # "--- deleted subvolume", + "++. created directory", "*** received subvolume (non-incremental)", ">>> received subvolume (incremental)", ],