ambiser
/
btrbk
mirror of https://github.com/digint/btrbk
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

btrbk: detect interrupted transfers of raw targets; delete incomplete raw targets on action "clean"

pull/88/head
Axel Burri 2016-03-22 19:05:12 +01:00
parent d47cc25f60
commit 7bb18050f8
2 changed files with 44 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ChangeLog
View File

@ -4,6 +4,7 @@ btrbk-current
* Added "{snapshot,target}_preserve NNd NNw NNm NNy" shortcut. * Added "{snapshot,target}_preserve NNd NNw NNm NNy" shortcut.
* Added yearly retention policies (close: #69). * Added yearly retention policies (close: #69).
* Always read "readonly" flag (additional call to btrfs-progs). * Always read "readonly" flag (additional call to btrfs-progs).
* Detect interrupted transfers of raw targets (close: #75).
* Improvements of internal data structures. * Improvements of internal data structures.
btrbk-0.22.2 btrbk-0.22.2

65
btrbk
View File

@ -1008,7 +1008,7 @@ sub btrfs_send_to_file($$$$;@)
}; };
} }
push @cmd_pipe, { push @cmd_pipe, {
cmd => [ 'dd', 'status=none', "of=$target_path/$target_filename" ], cmd => [ 'dd', 'status=none', "of=${target_path}/${target_filename}.part" ],
rsh => $target->{RSH}, rsh => $target->{RSH},
name => 'dd', name => 'dd',
}; };
@ -1022,7 +1022,7 @@ sub btrfs_send_to_file($$$$;@)
DEBUG "[btrfs] send-to-file" . ($parent ? " (incremental)" : " (complete)") . ":"; DEBUG "[btrfs] send-to-file" . ($parent ? " (incremental)" : " (complete)") . ":";
DEBUG "[btrfs] source: $source->{PRINT}"; DEBUG "[btrfs] source: $source->{PRINT}";
DEBUG "[btrfs] parent: $parent->{PRINT}" if($parent); DEBUG "[btrfs] parent: $parent->{PRINT}" if($parent);
DEBUG "[btrfs] target: $target->{PRINT}"; DEBUG "[btrfs] target: $target->{PRINT}/";
start_transaction("send-to-raw", start_transaction("send-to-raw",
vinfo_prefixed_keys("target", $vol_received), vinfo_prefixed_keys("target", $vol_received),
@ -1033,12 +1033,20 @@ sub btrfs_send_to_file($$$$;@)
if(defined($ret)) { if(defined($ret)) {
# Test target file for "exists and size > 0" after writing, # Test target file for "exists and size > 0" after writing,
# as we can not rely on the exit status of 'dd' # as we can not rely on the exit status of 'dd'
DEBUG "Testing target file (non-zero size): $target->{PRINT}"; DEBUG "Testing target file (non-zero size): $target->{PRINT}.part";
$ret = run_cmd({ $ret = run_cmd({
cmd => ['test', '-s', "$target_path/$target_filename"], cmd => ['test', '-s', "${target_path}/${target_filename}.part"],
rsh => $target->{RSH}, rsh => $target->{RSH},
name => "test", name => "test",
}); });
if(defined($ret)) {
DEBUG "Renaming target file (remove postfix '.part'): $target->{PRINT}";
$ret = run_cmd({
cmd => ['mv', "${target_path}/${target_filename}.part", "${target_path}/${target_filename}"],
rsh => $target->{RSH},
name => "mv",
});
}
} }
end_transaction("send-to-raw", ($dryrun ? "DRYRUN" : (defined($ret) ? "success" : "ERROR"))); end_transaction("send-to-raw", ($dryrun ? "DRYRUN" : (defined($ret) ? "success" : "ERROR")));
unless(defined($ret)) { unless(defined($ret)) {
@ -1583,21 +1591,23 @@ sub check_file($$;$$)
# returns { btrbk_date => [ yyyy, mm, dd, hh, mm, <date_ext> ] } or undef # returns { btrbk_date => [ yyyy, mm, dd, hh, mm, <date_ext> ] } or undef
# fixed array length of 6, all individually defaulting to 0 # fixed array length of 6, all individually defaulting to 0
sub parse_filename($$;$) sub parse_filename($$;@)
{ {
my $file = shift; my $file = shift;
my $name_match = shift; my $name_match = shift;
my $raw_format = shift || 0; my %opts = @_;
my %raw_info; my $raw_target = $opts{target_type} ? ($opts{target_type} eq "raw") : 0; # assume normal target if not set
if($raw_format) if($raw_target)
{ {
return undef unless($file =~ /^\Q$name_match\E$timestamp_postfix_match$raw_postfix_match$/); my $incomplete_match = $opts{incomplete_raw} ? qr/(\.(?<incomplete>part))?/ : "";
return undef unless($file =~ /^\Q$name_match\E$timestamp_postfix_match$raw_postfix_match$incomplete_match$/);
die unless($+{YYYY} && $+{MM} && $+{DD}); die unless($+{YYYY} && $+{MM} && $+{DD});
return { btrbk_date => [ $+{YYYY}, $+{MM}, $+{DD}, ($+{hh} // 0), ($+{mm} // 0), ($+{NN} // 0) ], return { btrbk_date => [ $+{YYYY}, $+{MM}, $+{DD}, ($+{hh} // 0), ($+{mm} // 0), ($+{NN} // 0) ],
received_uuid => $+{received_uuid} // die, received_uuid => $+{received_uuid} // die,
REMOTE_PARENT_UUID => $+{parent_uuid} // '-', REMOTE_PARENT_UUID => $+{parent_uuid} // '-',
ENCRYPT => $+{encrypt} // "", ENCRYPT => $+{encrypt} // "",
COMPRESS => $+{compress} // "", COMPRESS => $+{compress} // "",
INCOMPLETE => $+{incomplete} ? 1 : 0,
}; };
} }
else else
@ -2066,11 +2076,11 @@ sub macro_delete($$$$;@)
my $result_vinfo = shift || die; my $result_vinfo = shift || die;
my $schedule_options = shift || die; my $schedule_options = shift || die;
my %delete_options = @_; my %delete_options = @_;
my $raw_format = ($root_subvol->{CONFIG}->{CONTEXT} eq "target") ? ($root_subvol->{CONFIG}->{target_type} eq "raw") : undef; my $target_type = ($root_subvol->{CONFIG}->{CONTEXT} eq "target") ? $root_subvol->{CONFIG}->{target_type} : undef;
my @schedule; my @schedule;
foreach my $vol (@{vinfo_subvol_list($root_subvol)}) { foreach my $vol (@{vinfo_subvol_list($root_subvol)}) {
my $filename_info = parse_filename($vol->{SUBVOL_PATH}, $subvol_basename, $raw_format); my $filename_info = parse_filename($vol->{SUBVOL_PATH}, $subvol_basename, target_type => $target_type);
unless($filename_info) { unless($filename_info) {
TRACE "Target subvolume does not match btrbk filename scheme, skipping: $vol->{PRINT}"; TRACE "Target subvolume does not match btrbk filename scheme, skipping: $vol->{PRINT}";
next; next;
@ -3088,9 +3098,7 @@ MAIN:
{ {
DEBUG "Creating raw subvolume list: $droot->{PRINT}"; DEBUG "Creating raw subvolume list: $droot->{PRINT}";
my $ret = run_cmd( my $ret = run_cmd(
# NOTE: check for file size >0, which causes bad (zero-sized) images to be resumed cmd => [ 'find', $droot->{PATH} . '/', '-maxdepth', '1', '-type', 'f' ],
# TODO: fix btrfs_send_to_file() to never create bad images
cmd => [ 'find', $droot->{PATH} . '/', '-maxdepth', '1', '-type', 'f', '-size', '+0' ],
rsh => $droot->{RSH}, rsh => $droot->{RSH},
# note: use something like this to get the real (link resolved) path # note: use something like this to get the real (link resolved) path
# cmd => [ "find", $droot->{PATH} . '/', "-maxdepth", "1", "-name", "$snapshot_basename.\*.raw\*", '-printf', '%f\0', '-exec', 'realpath', '-z', '{}', ';' ], # cmd => [ "find", $droot->{PATH} . '/', "-maxdepth", "1", "-name", "$snapshot_basename.\*.raw\*", '-printf', '%f\0', '-exec', 'realpath', '-z', '{}', ';' ],
@ -3115,7 +3123,7 @@ MAIN:
last; last;
} }
my $snapshot_basename = config_key($svol, "snapshot_name") // die; my $snapshot_basename = config_key($svol, "snapshot_name") // die;
my $filename_info = parse_filename($file, $snapshot_basename, 1); my $filename_info = parse_filename($file, $snapshot_basename, target_type => "raw", incomplete_raw => 1);
unless($filename_info) { unless($filename_info) {
DEBUG "Skipping file (filename scheme mismatch): \"$file\""; DEBUG "Skipping file (filename scheme mismatch): \"$file\"";
next; next;
@ -3127,7 +3135,7 @@ MAIN:
# "parent of the received subvolume". # "parent of the received subvolume".
my $subvol = vinfo_child($droot, $file); my $subvol = vinfo_child($droot, $file);
$subvol->{node} = { uuid => "FAKE_UUID:" . $subvol->{URL}, $subvol->{node} = { uuid => "FAKE_UUID:" . $subvol->{URL},
received_uuid => $filename_info->{received_uuid}, received_uuid => ($filename_info->{INCOMPLETE} ? '-' : $filename_info->{received_uuid}), # empty received_uuid is detected as incomplete backup
# parent_uuid => '-', # correct value gets inserted below # parent_uuid => '-', # correct value gets inserted below
readonly => 1, # fake subvolume readonly flag readonly => 1, # fake subvolume readonly flag
}; };
@ -3348,7 +3356,7 @@ MAIN:
} }
else { else {
# don't display all subvolumes in $droot, only the ones matching snapshot_name # don't display all subvolumes in $droot, only the ones matching snapshot_name
if(parse_filename($target_vol->{SUBVOL_PATH}, $snapshot_name, ($droot->{CONFIG}->{target_type} eq "raw"))) { if(parse_filename($target_vol->{SUBVOL_PATH}, $snapshot_name, target_type => $droot->{CONFIG}->{target_type}, incomplete_raw => 1)) {
if($incomplete_backup) { $stats_incomplete++; } else { $stats_orphaned++; } if($incomplete_backup) { $stats_incomplete++; } else { $stats_orphaned++; }
push @data, { type => "received", push @data, { type => "received",
status => ($incomplete_backup ? "incomplete" : "orphaned"), status => ($incomplete_backup ? "incomplete" : "orphaned"),
@ -3455,6 +3463,7 @@ MAIN:
WARN "btrfs_progs_compat is set, skipping cleanup of target: $droot->{PRINT}"; WARN "btrfs_progs_compat is set, skipping cleanup of target: $droot->{PRINT}";
next; next;
} }
my $target_type = $droot->{CONFIG}->{target_type} || die;
INFO "Cleaning incomplete backups in: $droot->{PRINT}/$snapshot_name.*"; INFO "Cleaning incomplete backups in: $droot->{PRINT}/$snapshot_name.*";
push @out, "$droot->{PRINT}/$snapshot_name.*"; push @out, "$droot->{PRINT}/$snapshot_name.*";
@ -3462,13 +3471,25 @@ MAIN:
foreach my $target_vol (sort { $a->{SUBVOL_PATH} cmp $b->{SUBVOL_PATH} } @{vinfo_subvol_list($droot)}) { foreach my $target_vol (sort { $a->{SUBVOL_PATH} cmp $b->{SUBVOL_PATH} } @{vinfo_subvol_list($droot)}) {
# incomplete received (garbled) subvolumes have no received_uuid (as of btrfs-progs v4.3.1). # incomplete received (garbled) subvolumes have no received_uuid (as of btrfs-progs v4.3.1).
# a subvolume in droot matching our naming is considered incomplete if received_uuid is not set! # a subvolume in droot matching our naming is considered incomplete if received_uuid is not set!
if(($target_vol->{node}{received_uuid} eq '-') && parse_filename($target_vol->{SUBVOL_PATH}, $snapshot_name)) { next unless(parse_filename($target_vol->{SUBVOL_PATH}, $snapshot_name, target_type => $target_type, incomplete_raw => 1));
if($target_vol->{node}{received_uuid} eq '-') {
DEBUG "Found incomplete target subvolume: $target_vol->{PRINT}"; DEBUG "Found incomplete target subvolume: $target_vol->{PRINT}";
push(@delete, $target_vol); push(@delete, $target_vol);
push @out, "--- $target_vol->{PRINT}"; push @out, "--- $target_vol->{PRINT}";
} }
} }
my $ret = btrfs_subvolume_delete(\@delete, commit => config_key($droot, "btrfs_commit_delete"), type => "delete_garbled"); my $ret;
if($target_type eq "raw") {
DEBUG "[raw] delete:";
DEBUG "[raw] file: $_->{PRINT}" foreach(@delete);
$ret = run_cmd({
cmd => ['rm', (map { $_->{PATH} } @delete) ],
rsh => $droot->{RSH},
});
}
else {
$ret = btrfs_subvolume_delete(\@delete, commit => config_key($droot, "btrfs_commit_delete"), type => "delete_garbled");
}
if(defined($ret)) { if(defined($ret)) {
INFO "Deleted $ret incomplete backups in: $droot->{PRINT}/$snapshot_name.*"; INFO "Deleted $ret incomplete backups in: $droot->{PRINT}/$snapshot_name.*";
$droot->{SUBVOL_DELETED} //= []; $droot->{SUBVOL_DELETED} //= [];
@ -3649,7 +3670,7 @@ MAIN:
my @receive_targets = get_receive_targets($droot, $child); my @receive_targets = get_receive_targets($droot, $child);
foreach(@receive_targets) { foreach(@receive_targets) {
unless(parse_filename($_->{SUBVOL_PATH}, $snapshot_basename, ($droot->{CONFIG}->{target_type} eq "raw"))) { unless(parse_filename($_->{SUBVOL_PATH}, $snapshot_basename, target_type => $droot->{CONFIG}->{target_type})) {
WARN "Receive target of resume candidate \"$child->{PRINT}\" exists at unexpected location \"$_->{PRINT}\", skipping"; WARN "Receive target of resume candidate \"$child->{PRINT}\" exists at unexpected location \"$_->{PRINT}\", skipping";
} }
} }
@ -3674,7 +3695,7 @@ MAIN:
# add all present backups to schedule, with no value # add all present backups to schedule, with no value
# these are needed for correct results of schedule() # these are needed for correct results of schedule()
foreach my $vol (@{vinfo_subvol_list($droot)}) { foreach my $vol (@{vinfo_subvol_list($droot)}) {
my $filename_info = parse_filename($vol->{SUBVOL_PATH}, $snapshot_basename, ($droot->{CONFIG}->{target_type} eq "raw")); my $filename_info = parse_filename($vol->{SUBVOL_PATH}, $snapshot_basename, target_type => $droot->{CONFIG}->{target_type});
unless($filename_info) { unless($filename_info) {
TRACE "Receive target does not match btrbk filename scheme, skipping: $vol->{PRINT}"; TRACE "Receive target does not match btrbk filename scheme, skipping: $vol->{PRINT}";
next; next;