diff --git a/btrbk b/btrbk index 9b82ae3..e4d9a74 100755 --- a/btrbk +++ b/btrbk @@ -86,6 +86,8 @@ my %config_options = ( snapshot_preserve_min => { default => "1d", accept => [ "all", "latest" ], accept_regexp => qr/^[1-9][0-9]*[hdwmy]$/, context => [ "root", "volume", "subvolume" ], }, target_preserve => { default => undef, accept => [ "no" ], accept_preserve_matrix => 1 }, target_preserve_min => { default => undef, accept => [ "all", "latest", "no" ], accept_regexp => qr/^[0-9]+[hdwmy]$/ }, + archive_preserve => { default => undef, accept => [ "no" ], accept_preserve_matrix => 1 }, + archive_preserve_min => { default => "all", accept => [ "all", "latest", "no" ], accept_regexp => qr/^[0-9]+[hdwmy]$/ }, btrfs_commit_delete => { default => undef, accept => [ "after", "each", "no" ] }, ssh_identity => { default => undef, accept_file => { absolute => 1 } }, ssh_user => { default => "root", accept_regexp => qr/^[a-z_][a-z0-9_-]*$/ }, @@ -255,6 +257,7 @@ sub HELP_MESSAGE print STDERR " volume configured volume sections\n"; print STDERR " target configured targets\n"; print STDERR " clean delete incomplete (garbled) backups\n"; + print STDERR " clone recursively copy all subvolumes (experimental)\n"; print STDERR " usage print filesystem usage\n"; print STDERR " origin print origin information for subvolume\n"; print STDERR " diff shows new files since subvolume for subvolume \n"; @@ -2479,7 +2482,7 @@ sub macro_send_receive(@) { # create backup from latest common if($parent) { - INFO "Incremental from parent subvolume: $parent->{PRINT}"; + INFO "Incremental from parent: $parent->{PRINT}"; } elsif($incremental ne "strict") { INFO "No common parent subvolume present, creating full backup"; @@ -2614,6 +2617,90 @@ sub macro_delete($$$$$;@) } +sub macro_clone_target($$$;$) +{ + my $sroot = shift || die; + my $droot = shift || die; + my $snapshot_name = shift // die; + my $schedule_options = shift // {}; + my @schedule; + + foreach my $dvol (@{vinfo_subvol_list($droot, sort => 'path')}) + { + next unless($dvol->{btrbk_direct_leaf} && ($dvol->{BTRBK_BASENAME} eq $snapshot_name)); + next unless($dvol->{node}{readonly}); + + # add all present archives to schedule, with no value. + # these are needed for correct results of schedule() + push @schedule, { value => undef, + btrbk_date => $dvol->{BTRBK_DATE}, + preserve => $dvol->{FORCE_PRESERVE}, + }; + } + + # NOTE: this is pretty much the same as "resume missing" + my $abort_unexpected_location = 0; + foreach my $svol (@{vinfo_subvol_list($sroot, sort => 'path')}) + { + next unless($svol->{node}{readonly}); + next unless($svol->{btrbk_direct_leaf} && ($svol->{BTRBK_BASENAME} eq $snapshot_name)); + + my $unexpected_count = 0; + my @receive_targets = get_receive_targets($droot, $svol, exact_match => 1, warn_unexpected => 1, ret_unexpected => \$unexpected_count); + # don't abort right here, we want to warn about all unexpected targets + $abort_unexpected_location += $unexpected_count; + next if($abort_unexpected_location || scalar(@receive_targets)); + + DEBUG "Adding target candidate: $svol->{PRINT}"; + push @schedule, { value => $svol, + name => $svol->{PRINT}, # only for logging + btrbk_date => $svol->{BTRBK_DATE}, + preserve => $svol->{FORCE_PRESERVE}, + }; + } + if($abort_unexpected_location) { + ABORTED($droot, "Receive targets of archive candidates exist at unexpected location"); + WARN "Skipping archiving of \"$sroot->{PRINT}/${snapshot_name}.*\": $abrt"; + return undef; + } + + my ($preserve, undef) = schedule( + schedule => \@schedule, + preserve => config_preserve_hash($droot, "archive"), + result_preserve_action_text => 'archive', + result_delete_action_text => '', + %$schedule_options + ); + my @archive = grep defined, @$preserve; # remove entries with no value from list (archive subvolumes) + my $archive_total = scalar @archive; + my $archive_success = 0; + foreach my $svol (@archive) + { + my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot, ""); + if(macro_send_receive(source => $svol, + target => $droot, + parent => $latest_common_src, + latest_common_target => $latest_common_target, + )) + { + $archive_success++; + } + else { + ERROR("Error while cloning, aborting"); + last; + } + } + + if($archive_total) { + INFO "Archived $archive_success/$archive_total subvolumes"; + } else { + INFO "No missing archived subvolume found"; + } + + return $archive_success; +} + + sub cmp_date($$) { my ($a,$b) = @_; @@ -3074,7 +3161,7 @@ MAIN: WARN 'Found option "--progress", but "pv" is not present: (please install "pv")'; $show_progress = 0; } - my ($action_run, $action_usage, $action_resolve, $action_diff, $action_origin, $action_config_print, $action_list, $action_clean); + my ($action_run, $action_usage, $action_resolve, $action_diff, $action_origin, $action_config_print, $action_list, $action_clean, $action_clone); my @filter_args; my $args_allow_group = 1; my $args_expected_min = 0; @@ -3089,6 +3176,12 @@ MAIN: $action_clean = 1; @filter_args = @ARGV; } + elsif ($command eq "clone") { + $action_clone = 1; + $args_expected_min = $args_expected_max = 2; + $args_allow_group = 0; + @filter_args = @ARGV; + } elsif ($command eq "usage") { $action_usage = 1; @filter_args = @ARGV; @@ -3299,6 +3392,165 @@ MAIN: } + if($action_clone) + { + # + # clone (archive) tree + # + # NOTE: This is intended to work without a config file! The only + # thing used from the configuration is the SSH and transaction log + # stuff. + # + init_transaction_log(config_key($config, "transaction_log")); + + my $src_url = $filter_args[0] || die; + my $target_url = $filter_args[1] || die; + + # FIXME: add command line options for preserve logic + $config->{SUBSECTION} = []; # clear configured subsections, we build them dynamically + + my $clone_src_root = vinfo($src_url, $config); + unless(vinfo_init_root($clone_src_root, resolve_subdir => 1)) { + ERROR "Failed to fetch subvolume detail for '$clone_src_root->{PRINT}'" . ($err ? ": $err" : ""); + exit 1; + } + my $target_root = vinfo($target_url, $config); + unless(vinfo_init_root($target_root, resolve_subdir => 1)) { + ERROR "Failed to fetch subvolume detail for '$target_root->{PRINT}'" . ($err ? ": $err" : ""); + exit 1; + } + + my %name_uniq; + my @subvol_list = @{vinfo_subvol_list($clone_src_root)}; + my @sorted = sort { ($a->{subtree_depth} <=> $b->{subtree_depth}) || ($a->{SUBVOL_DIR} cmp $b->{SUBVOL_DIR}) } @subvol_list; + foreach my $vol (@sorted) { + next unless($vol->{node}{readonly}); + my $snapshot_name = $vol->{BTRBK_BASENAME}; + unless(defined($snapshot_name)) { + WARN "Skipping subvolume (not a btrbk subvolume): $vol->{PRINT}"; + next; + } + my $subvol_dir = $vol->{SUBVOL_DIR}; + next if($name_uniq{"$subvol_dir/$snapshot_name"}); + $name_uniq{"$subvol_dir/$snapshot_name"} = 1; + $subvol_dir = '/' . $subvol_dir if($subvol_dir ne ""); + my $droot_url = $target_url . $subvol_dir; + my $sroot_url = $src_url . $subvol_dir; + my $config_clone_src = { CONTEXT => "clone_source", + PARENT => $config, + url => $sroot_url, # ABORTED() needs this + snapshot_name => $snapshot_name, + }; + my $config_target = { CONTEXT => "target", + PARENT => $config_clone_src, + target_type => "send-receive", # macro_send_receive checks this + url => $droot_url, # ABORTED() needs this + }; + $config_clone_src->{SUBSECTION} = [ $config_target ]; + push(@{$config->{SUBSECTION}}, $config_clone_src); + + my $sroot = vinfo($sroot_url, $config_clone_src); + vinfo_assign_config($sroot, $config_clone_src); + unless(vinfo_init_root($sroot, resolve_subdir => 1)) { + ABORTED($sroot, "Failed to fetch subvolume detail" . ($err ? ": $err" : "")); + WARN "Skipping clone source \"$sroot->{PRINT}\": $abrt"; + next; + } + + my $droot = vinfo($droot_url, $config_target); + vinfo_assign_config($droot, $config_target); + unless(vinfo_init_root($droot, resolve_subdir => 1)) { + ABORTED($droot, "Failed to fetch subvolume detail" . ($err ? ": $err" : "")); + WARN "Skipping clone target \"$droot->{PRINT}\": $abrt"; + next; + } + if(_is_child_of($droot->{node}->{TREE_ROOT}, $vol->{node}{uuid})) { + ERROR "Source and target subvolumes are on the same btrfs filesystem!"; + exit 1; + } + } + + foreach my $sroot (vinfo_subsection($config, 'clone_source')) { + foreach my $droot (vinfo_subsection($sroot, 'target')) { + my $snapshot_name = config_key($droot, "snapshot_name") // die; + INFO "Archiving subvolumes: $sroot->{PRINT}/${snapshot_name}.*"; + macro_clone_target($sroot, $droot, $snapshot_name); + } + } + + my $exit_status = exit_status($config); + my $time_elapsed = time - $start_time; + INFO "Completed within: ${time_elapsed}s (" . localtime(time) . ")"; + action("finished", + status => $exit_status ? "partial" : "success", + duration => $time_elapsed, + message => $exit_status ? "At least one backup task aborted" : undef, + ); + close_transaction_log(); + + unless($quiet) + { + # + # print summary + # + $output_format ||= "custom"; + if($output_format eq "custom") + { + my @unrecoverable; + my @out; + foreach my $sroot (vinfo_subsection($config, 'clone_source')) { + foreach my $droot (vinfo_subsection($sroot, 'target', 1)) { + my @subvol_out; + foreach(@{$droot->{SUBVOL_RECEIVED} // []}) { + my $create_mode = "***"; + $create_mode = ">>>" if($_->{parent}); + $create_mode = "!!!" if($_->{ERROR}); + push @subvol_out, "$create_mode $_->{received_subvolume}->{PRINT}"; + } + if(ABORTED($droot) && (ABORTED($droot) ne "USER_SKIP")) { + push @subvol_out, "!!! Target \"$droot->{PRINT}\" aborted: " . ABORTED($droot);; + } + if($droot->{CONFIG}->{UNRECOVERABLE}) { + push(@unrecoverable, $droot->{CONFIG}->{UNRECOVERABLE}); + } + if(@subvol_out) { + push @out, "$sroot->{PRINT}/$sroot->{CONFIG}->{snapshot_name}.*", @subvol_out, ""; + } + } + } + + print_header(title => "Clone Summary", + time => $start_time, + legend => [ + "--- deleted subvolume", + "*** received subvolume (non-incremental)", + ">>> received subvolume (incremental)", + ], + ); + + print join("\n", @out); + + if($exit_status) { + print "\nNOTE: Some errors occurred, which may result in missing backups!\n"; + print "Please check warning and error messages above.\n"; + print join("\n", @unrecoverable) . "\n" if(@unrecoverable); + } + if($dryrun) { + print "\nNOTE: Dryrun was active, none of the operations above were actually executed!\n"; + } + } + else + { + # print action log (without transaction start messages) + my @data = grep { $_->{status} ne "starting" } @transaction_log; + print_formatted("transaction", \@data, title => "TRANSACTION LOG"); + } + } + + exit $exit_status; + } + + # # expand subvolume globs (wildcards) # diff --git a/doc/btrbk.1 b/doc/btrbk.1 index 725dd3c..261fc35 100644 --- a/doc/btrbk.1 +++ b/doc/btrbk.1 @@ -167,6 +167,28 @@ by the \fBrun\fR command. Use in conjunction with \fI\-l debug\fR to see the btrfs commands that would be executed. .RE .PP +.B clone + +.I *experimental* +.RS 4 +Recursively copy all subvolumes created by btrbk from to + directory, optionally rescheduled using +\fIarchive_preserve_*\fR configuration options. Useful for creating +extra archive copies (clones) from your backup disks. Note that you +can continue using btrbk after swapping your backup disk with the +cloned disk. +.PP +Note that this feature needs a \fBlinux kernel >=4.4\fR to work +correctly! Kernels >=4.1 and <4.4 have a bug when re-sending +subvolumes (the cloned subvolumes will have incorrect received_uuid, +see ), so +make sure you run a recent kernel. +.PP +Known bugs: On the target filesystem, you have to create all +directories by hand, or "btrbk clone" will complain: "WARNING: +Skipping clone target <...>: Failed to fetch subvolume detail". +.RE +.PP .B stats [filter...] .RS 4