ambiser
/
btrbk
mirror of https://github.com/digint/btrbk
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

btrbk: add action "clone": recursively send/receive all backups to an archive directory

pull/88/head
Axel Burri 2016-04-07 15:33:32 +02:00
parent 3c9aff4c40
commit 14de16aabe
2 changed files with 276 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

256
btrbk
View File

@ -86,6 +86,8 @@ my %config_options = (
snapshot_preserve_min => { default => "1d", accept => [ "all", "latest" ], accept_regexp => qr/^[1-9][0-9]*[hdwmy]$/, context => [ "root", "volume", "subvolume" ], },
target_preserve => { default => undef, accept => [ "no" ], accept_preserve_matrix => 1 },
target_preserve_min => { default => undef, accept => [ "all", "latest", "no" ], accept_regexp => qr/^[0-9]+[hdwmy]$/ },
archive_preserve => { default => undef, accept => [ "no" ], accept_preserve_matrix => 1 },
archive_preserve_min => { default => "all", accept => [ "all", "latest", "no" ], accept_regexp => qr/^[0-9]+[hdwmy]$/ },
btrfs_commit_delete => { default => undef, accept => [ "after", "each", "no" ] },
ssh_identity => { default => undef, accept_file => { absolute => 1 } },
ssh_user => { default => "root", accept_regexp => qr/^[a-z_][a-z0-9_-]*$/ },
@ -255,6 +257,7 @@ sub HELP_MESSAGE
print STDERR " volume configured volume sections\n";
print STDERR " target configured targets\n";
print STDERR " clean delete incomplete (garbled) backups\n";
print STDERR " clone <src> <dst> recursively copy all subvolumes (experimental)\n";
print STDERR " usage print filesystem usage\n";
print STDERR " origin <subvol> print origin information for subvolume\n";
print STDERR " diff <from> <to> shows new files since subvolume <from> for subvolume <to>\n";
@ -2479,7 +2482,7 @@ sub macro_send_receive(@)
{
# create backup from latest common
if($parent) {
INFO "Incremental from parent subvolume: $parent->{PRINT}";
INFO "Incremental from parent: $parent->{PRINT}";
}
elsif($incremental ne "strict") {
INFO "No common parent subvolume present, creating full backup";
@ -2614,6 +2617,90 @@ sub macro_delete($$$$$;@)
}
sub macro_clone_target($$$;$)
{
my $sroot = shift || die;
my $droot = shift || die;
my $snapshot_name = shift // die;
my $schedule_options = shift // {};
my @schedule;
foreach my $dvol (@{vinfo_subvol_list($droot, sort => 'path')})
{
next unless($dvol->{btrbk_direct_leaf} && ($dvol->{BTRBK_BASENAME} eq $snapshot_name));
next unless($dvol->{node}{readonly});
# add all present archives to schedule, with no value.
# these are needed for correct results of schedule()
push @schedule, { value => undef,
btrbk_date => $dvol->{BTRBK_DATE},
preserve => $dvol->{FORCE_PRESERVE},
};
}
# NOTE: this is pretty much the same as "resume missing"
my $abort_unexpected_location = 0;
foreach my $svol (@{vinfo_subvol_list($sroot, sort => 'path')})
{
next unless($svol->{node}{readonly});
next unless($svol->{btrbk_direct_leaf} && ($svol->{BTRBK_BASENAME} eq $snapshot_name));
my $unexpected_count = 0;
my @receive_targets = get_receive_targets($droot, $svol, exact_match => 1, warn_unexpected => 1, ret_unexpected => \$unexpected_count);
# don't abort right here, we want to warn about all unexpected targets
$abort_unexpected_location += $unexpected_count;
next if($abort_unexpected_location || scalar(@receive_targets));
DEBUG "Adding target candidate: $svol->{PRINT}";
push @schedule, { value => $svol,
name => $svol->{PRINT}, # only for logging
btrbk_date => $svol->{BTRBK_DATE},
preserve => $svol->{FORCE_PRESERVE},
};
}
if($abort_unexpected_location) {
ABORTED($droot, "Receive targets of archive candidates exist at unexpected location");
WARN "Skipping archiving of \"$sroot->{PRINT}/${snapshot_name}.*\": $abrt";
return undef;
}
my ($preserve, undef) = schedule(
schedule => \@schedule,
preserve => config_preserve_hash($droot, "archive"),
result_preserve_action_text => 'archive',
result_delete_action_text => '',
%$schedule_options
);
my @archive = grep defined, @$preserve; # remove entries with no value from list (archive subvolumes)
my $archive_total = scalar @archive;
my $archive_success = 0;
foreach my $svol (@archive)
{
my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot, "");
if(macro_send_receive(source => $svol,
target => $droot,
parent => $latest_common_src,
latest_common_target => $latest_common_target,
))
{
$archive_success++;
}
else {
ERROR("Error while cloning, aborting");
last;
}
}
if($archive_total) {
INFO "Archived $archive_success/$archive_total subvolumes";
} else {
INFO "No missing archived subvolume found";
}
return $archive_success;
}
sub cmp_date($$)
{
my ($a,$b) = @_;
@ -3074,7 +3161,7 @@ MAIN:
WARN 'Found option "--progress", but "pv" is not present: (please install "pv")';
$show_progress = 0;
}
my ($action_run, $action_usage, $action_resolve, $action_diff, $action_origin, $action_config_print, $action_list, $action_clean);
my ($action_run, $action_usage, $action_resolve, $action_diff, $action_origin, $action_config_print, $action_list, $action_clean, $action_clone);
my @filter_args;
my $args_allow_group = 1;
my $args_expected_min = 0;
@ -3089,6 +3176,12 @@ MAIN:
$action_clean = 1;
@filter_args = @ARGV;
}
elsif ($command eq "clone") {
$action_clone = 1;
$args_expected_min = $args_expected_max = 2;
$args_allow_group = 0;
@filter_args = @ARGV;
}
elsif ($command eq "usage") {
$action_usage = 1;
@filter_args = @ARGV;
@ -3299,6 +3392,165 @@ MAIN:
}
if($action_clone)
{
#
# clone (archive) tree
#
# NOTE: This is intended to work without a config file! The only
# thing used from the configuration is the SSH and transaction log
# stuff.
#
init_transaction_log(config_key($config, "transaction_log"));
my $src_url = $filter_args[0] || die;
my $target_url = $filter_args[1] || die;
# FIXME: add command line options for preserve logic
$config->{SUBSECTION} = []; # clear configured subsections, we build them dynamically
my $clone_src_root = vinfo($src_url, $config);
unless(vinfo_init_root($clone_src_root, resolve_subdir => 1)) {
ERROR "Failed to fetch subvolume detail for '$clone_src_root->{PRINT}'" . ($err ? ": $err" : "");
exit 1;
}
my $target_root = vinfo($target_url, $config);
unless(vinfo_init_root($target_root, resolve_subdir => 1)) {
ERROR "Failed to fetch subvolume detail for '$target_root->{PRINT}'" . ($err ? ": $err" : "");
exit 1;
}
my %name_uniq;
my @subvol_list = @{vinfo_subvol_list($clone_src_root)};
my @sorted = sort { ($a->{subtree_depth} <=> $b->{subtree_depth}) || ($a->{SUBVOL_DIR} cmp $b->{SUBVOL_DIR}) } @subvol_list;
foreach my $vol (@sorted) {
next unless($vol->{node}{readonly});
my $snapshot_name = $vol->{BTRBK_BASENAME};
unless(defined($snapshot_name)) {
WARN "Skipping subvolume (not a btrbk subvolume): $vol->{PRINT}";
next;
}
my $subvol_dir = $vol->{SUBVOL_DIR};
next if($name_uniq{"$subvol_dir/$snapshot_name"});
$name_uniq{"$subvol_dir/$snapshot_name"} = 1;
$subvol_dir = '/' . $subvol_dir if($subvol_dir ne "");
my $droot_url = $target_url . $subvol_dir;
my $sroot_url = $src_url . $subvol_dir;
my $config_clone_src = { CONTEXT => "clone_source",
PARENT => $config,
url => $sroot_url, # ABORTED() needs this
snapshot_name => $snapshot_name,
};
my $config_target = { CONTEXT => "target",
PARENT => $config_clone_src,
target_type => "send-receive", # macro_send_receive checks this
url => $droot_url, # ABORTED() needs this
};
$config_clone_src->{SUBSECTION} = [ $config_target ];
push(@{$config->{SUBSECTION}}, $config_clone_src);
my $sroot = vinfo($sroot_url, $config_clone_src);
vinfo_assign_config($sroot, $config_clone_src);
unless(vinfo_init_root($sroot, resolve_subdir => 1)) {
ABORTED($sroot, "Failed to fetch subvolume detail" . ($err ? ": $err" : ""));
WARN "Skipping clone source \"$sroot->{PRINT}\": $abrt";
next;
}
my $droot = vinfo($droot_url, $config_target);
vinfo_assign_config($droot, $config_target);
unless(vinfo_init_root($droot, resolve_subdir => 1)) {
ABORTED($droot, "Failed to fetch subvolume detail" . ($err ? ": $err" : ""));
WARN "Skipping clone target \"$droot->{PRINT}\": $abrt";
next;
}
if(_is_child_of($droot->{node}->{TREE_ROOT}, $vol->{node}{uuid})) {
ERROR "Source and target subvolumes are on the same btrfs filesystem!";
exit 1;
}
}
foreach my $sroot (vinfo_subsection($config, 'clone_source')) {
foreach my $droot (vinfo_subsection($sroot, 'target')) {
my $snapshot_name = config_key($droot, "snapshot_name") // die;
INFO "Archiving subvolumes: $sroot->{PRINT}/${snapshot_name}.*";
macro_clone_target($sroot, $droot, $snapshot_name);
}
}
my $exit_status = exit_status($config);
my $time_elapsed = time - $start_time;
INFO "Completed within: ${time_elapsed}s (" . localtime(time) . ")";
action("finished",
status => $exit_status ? "partial" : "success",
duration => $time_elapsed,
message => $exit_status ? "At least one backup task aborted" : undef,
);
close_transaction_log();
unless($quiet)
{
#
# print summary
#
$output_format ||= "custom";
if($output_format eq "custom")
{
my @unrecoverable;
my @out;
foreach my $sroot (vinfo_subsection($config, 'clone_source')) {
foreach my $droot (vinfo_subsection($sroot, 'target', 1)) {
my @subvol_out;
foreach(@{$droot->{SUBVOL_RECEIVED} // []}) {
my $create_mode = "***";
$create_mode = ">>>" if($_->{parent});
$create_mode = "!!!" if($_->{ERROR});
push @subvol_out, "$create_mode $_->{received_subvolume}->{PRINT}";
}
if(ABORTED($droot) && (ABORTED($droot) ne "USER_SKIP")) {
push @subvol_out, "!!! Target \"$droot->{PRINT}\" aborted: " . ABORTED($droot);;
}
if($droot->{CONFIG}->{UNRECOVERABLE}) {
push(@unrecoverable, $droot->{CONFIG}->{UNRECOVERABLE});
}
if(@subvol_out) {
push @out, "$sroot->{PRINT}/$sroot->{CONFIG}->{snapshot_name}.*", @subvol_out, "";
}
}
}
print_header(title => "Clone Summary",
time => $start_time,
legend => [
"--- deleted subvolume",
"*** received subvolume (non-incremental)",
">>> received subvolume (incremental)",
],
);
print join("\n", @out);
if($exit_status) {
print "\nNOTE: Some errors occurred, which may result in missing backups!\n";
print "Please check warning and error messages above.\n";
print join("\n", @unrecoverable) . "\n" if(@unrecoverable);
}
if($dryrun) {
print "\nNOTE: Dryrun was active, none of the operations above were actually executed!\n";
}
}
else
{
# print action log (without transaction start messages)
my @data = grep { $_->{status} ne "starting" } @transaction_log;
print_formatted("transaction", \@data, title => "TRANSACTION LOG");
}
}
exit $exit_status;
}
#
# expand subvolume globs (wildcards)
#

22
doc/btrbk.1
View File

@ -167,6 +167,28 @@ by the \fBrun\fR command. Use in conjunction with \fI\-l debug\fR to
see the btrfs commands that would be executed.
.RE
.PP
.B clone
<source> <target>
.I *experimental*
.RS 4
Recursively copy all subvolumes created by btrbk from <source> to
<target> directory, optionally rescheduled using
\fIarchive_preserve_*\fR configuration options. Useful for creating
extra archive copies (clones) from your backup disks. Note that you
can continue using btrbk after swapping your backup disk with the
cloned disk.
.PP
Note that this feature needs a \fBlinux kernel >=4.4\fR to work
correctly! Kernels >=4.1 and <4.4 have a bug when re-sending
subvolumes (the cloned subvolumes will have incorrect received_uuid,
see <http://thread.gmane.org/gmane.comp.file-systems.btrfs/48798>), so
make sure you run a recent kernel.
.PP
Known bugs: On the target filesystem, you have to create all
directories by hand, or "btrbk clone" will complain: "WARNING:
Skipping clone target <...>: Failed to fetch subvolume detail".
.RE
.PP
.B stats
[filter...]
.RS 4