ambiser
/
btrbk
mirror of https://github.com/digint/btrbk
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'resume_missing'

pull/30/head
Axel Burri 2015-04-01 15:00:37 +02:00
parent be6547c1bd 10b8983296
commit 2a65895d18
6 changed files with 229 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ChangeLog
View File

@ -24,5 +24,9 @@
* btrbk-current
- added configuration option "btrfs_progs_compat", to be enabled if
using btrfs-progs < 3.17. Fixes issue #6
- added configuration option "resume_missing", for automatic resume
of missing backups
- removed configuration option "receive_log" in favor of printing
errors from "btrfs receive"
- bugfix: show correct exit code on external command failure
- bugfix: no crash if "commit_delete" option is set to "no"

3
README.md
View File

@ -14,9 +14,10 @@ Key Features:
- Atomic snapshots
- Incremental backups
- Configurable retention policy
- Backups to multiple destinations
- Transfer via ssh
- Configurable retention policy
- Resume of backups (if backup disk was not attached for a while)
- Display file changes between two backups
btrbk is intended to be run as a cron job.

256
btrbk
View File

@ -64,6 +64,7 @@ my %config_options = (
receive_log => { default => undef, accept => [ "sidecar", "no" ], accept_file => { absolute => 1 }, deprecated => "removed" },
incremental => { default => "yes", accept => [ "yes", "no", "strict" ] },
snapshot_create_always => { default => undef, accept => [ "yes", "no" ] },
resume_missing => { default => undef, accept => [ "yes", "no" ] },
preserve_day_of_week => { default => "sunday", accept => [ (keys %day_of_week_map) ] },
snapshot_preserve_daily => { default => "all", accept => [ "all" ], accept_numeric => 1 },
snapshot_preserve_weekly => { default => 0, accept => [ "all" ], accept_numeric => 1 },
@ -801,7 +802,7 @@ sub btrfs_subvolume_delete($@)
}
sub btrfs_send_receive($$$$;$)
sub btrfs_send_receive($$$;$)
{
my $src = shift || die;
my $target = shift || die;
@ -835,19 +836,80 @@ sub btrfs_send_receive($$$$;$)
}
# sets $config->{ABORTED} on failure
# sets $config->{subvol_received}
sub macro_send_receive($@)
{
my $config = shift || die;
my %info = @_;
my $incremental = config_key($config, "incremental");
INFO "Receiving from snapshot: $info{src}";
# add info to $config->{subvol_received}
my $src_name = $info{src};
$src_name =~ s/^.*\///;
$info{received_name} = "$info{target}/$src_name";
$config->{subvol_received} //= [];
push(@{$config->{subvol_received}}, \%info);
if($incremental)
{
# create backup from latest common
if($info{parent}) {
INFO "Incremental from parent snapshot: $info{parent}";
}
elsif($incremental ne "strict") {
INFO "No common parent subvolume present, creating full backup";
}
else {
WARN "Backup to $info{target} failed: no common parent subvolume found, and option \"incremental\" is set to \"strict\"";
$info{ERROR} = 1;
$config->{ABORTED} = "No common parent subvolume found, and option \"incremental\" is set to \"strict\"";
return undef;
}
}
else {
INFO "Option \"incremental\" is not set, creating full backup";
delete $info{parent};
}
if(btrfs_send_receive($info{src}, $info{target}, $info{parent}, $config)) {
return 1;
} else {
$info{ERROR} = 1;
$config->{ABORTED} = "btrfs send/receive command failed";
return undef;
}
}
sub get_date_tag($)
{
my $name = shift;
$name =~ s/_([0-9]+)$//;
my $postfix_counter = $1;
my $date = undef;
if($name =~ /\.([0-9]{4})([0-9]{2})([0-9]{2})$/) {
$date = [ $1, $2, $3 ];
}
return ($date, $postfix_counter);
}
sub get_snapshot_children($$)
{
my $sroot = shift || die;
my $svol = shift || die;
my $svol_node = subvol($sroot, $svol);
die("subvolume info not present: $sroot/$svol") unless($svol_node);
DEBUG "Getting snapshot children of: $sroot/$svol";
my @ret;
foreach (values %{$vol_info{$sroot}}) {
next unless($_->{node}->{parent_uuid} eq $svol_node->{uuid});
DEBUG "Found snapshot child: $_->{SUBVOL_PATH}";
TRACE "get_snapshot_children: Found snapshot child: $_->{SUBVOL_PATH}";
push(@ret, $_);
}
DEBUG "Found " . scalar(@ret) . " snapshot children of: $sroot/$svol";
return @ret;
}
@ -858,29 +920,44 @@ sub get_receive_targets_by_uuid($$)
my $uuid = shift || die;
die("root subvolume info not present: $droot") unless($vol_info{$droot});
die("subvolume info not present: $uuid") unless($uuid_info{$uuid});
DEBUG "Getting receive targets in \"$droot/\" for: $uuid_info{$uuid}->{path}";
my @ret;
foreach (values %{$vol_info{$droot}}) {
next unless($_->{node}->{received_uuid} eq $uuid);
DEBUG "Found receive target: $_->{SUBVOL_PATH}";
TRACE "get_receive_targets_by_uuid: Found receive target: $_->{SUBVOL_PATH}";
push(@ret, $_);
}
DEBUG "Found " . scalar(@ret) . " receive targets in \"$droot/\" for: $uuid_info{$uuid}->{path}";
return @ret;
}
sub get_latest_common($$$)
sub get_latest_common($$$;$)
{
my $sroot = shift || die;
my $svol = shift || die;
my $droot = shift || die;
my $threshold_gen = shift; # skip all snapshot children with generation >= $threshold_gen
die("source subvolume info not present: $sroot") unless($vol_info{$sroot});
die("target subvolume info not present: $droot") unless($vol_info{$droot});
my $debug_src = "$sroot/$svol";
$debug_src .= "@" . $threshold_gen if($threshold_gen);
# sort children of svol descending by generation
foreach my $child (sort { $b->{node}->{gen} <=> $a->{node}->{gen} } get_snapshot_children($sroot, $svol)) {
TRACE "get_latest_common: checking source snapshot: $child->{SUBVOL_PATH}";
if($threshold_gen && ($child->{node}->{gen} >= $threshold_gen)) {
TRACE "get_latest_common: skipped gen=$child->{node}->{gen} >= $threshold_gen: $child->{SUBVOL_PATH}";
next;
}
if($child->{RECEIVE_TARGET_PRESENT} && ($child->{RECEIVE_TARGET_PRESENT} eq $droot)) {
# little hack to keep track of previously received subvolumes
DEBUG("Latest common snapshots for: $debug_src: src=$child->{FS_PATH} target=<previously received>");
return ($child, undef);
}
if($vol_btrfs_progs_compat{$droot}) {
# guess matches by subvolume name (node->received_uuid is not available if BTRFS_PROGS_COMPAT is set)
my $child_name = $child->{node}->{REL_PATH};
@ -889,7 +966,7 @@ sub get_latest_common($$$)
my $backup_name = $backup->{node}->{REL_PATH};
$backup_name =~ s/^.*\///; # strip path
if($backup_name eq $child_name) {
DEBUG("Latest common snapshots for: $sroot/$svol: src=$child->{FS_PATH} target=$backup->{FS_PATH} (NOTE: guessed by subvolume name)");
DEBUG("Latest common snapshots for: $debug_src: src=$child->{FS_PATH} target=$backup->{FS_PATH} (NOTE: guessed by subvolume name)");
return ($child, $backup);
}
}
@ -897,13 +974,13 @@ sub get_latest_common($$$)
else {
foreach (get_receive_targets_by_uuid($droot, $child->{node}->{uuid})) {
TRACE "get_latest_common: found receive target: $_->{FS_PATH}";
DEBUG("Latest common snapshots for: $sroot/$svol: src=$child->{FS_PATH} target=$_->{FS_PATH}");
DEBUG("Latest common snapshots for: $debug_src: src=$child->{FS_PATH} target=$_->{FS_PATH}");
return ($child, $_);
}
}
TRACE "get_latest_common: no matching targets found for: $child->{FS_PATH}";
}
DEBUG("No common snapshots for \"$sroot/$svol\" found in src=$sroot/ target=$droot/");
DEBUG("No common snapshots for \"$debug_src\" found in src=$sroot/ target=$droot/");
return (undef, undef);
}
@ -949,16 +1026,19 @@ sub schedule_deletion(@)
my $preserve_daily = $args{preserve_daily} // die;
my $preserve_weekly = $args{preserve_weekly} // die;
my $preserve_monthly = $args{preserve_monthly} // die;
my $log_verbose = $args{log_verbose};
INFO "Filter scheme: preserving all within $preserve_daily days";
INFO "Filter scheme: preserving first in week (starting on $preserve_day_of_week), for $preserve_weekly weeks";
INFO "Filter scheme: preserving last weekly of month, for $preserve_monthly months";
if($log_verbose) {
INFO "Filter scheme: preserving all within $preserve_daily days";
INFO "Filter scheme: preserving first in week (starting on $preserve_day_of_week), for $preserve_weekly weeks";
INFO "Filter scheme: preserving last weekly of month, for $preserve_monthly months";
}
# first, do our calendar calculations
# note: our week starts on $preserve_day_of_week
my $delta_days_to_eow_from_today = $day_of_week_map{$preserve_day_of_week} - Day_of_Week(@today) - 1;
$delta_days_to_eow_from_today = $delta_days_to_eow_from_today + 7 if($delta_days_to_eow_from_today < 0);
DEBUG "last day before next $preserve_day_of_week is in $delta_days_to_eow_from_today days";
TRACE "last day before next $preserve_day_of_week is in $delta_days_to_eow_from_today days";
foreach my $href (@$schedule)
{
my @date = @{$href->{date}};
@ -1002,13 +1082,14 @@ sub schedule_deletion(@)
foreach my $href (sort { $a->{sort} cmp $b->{sort} } @$schedule)
{
if($href->{preserve}) {
INFO "=== $href->{sort}: $href->{preserve}";
INFO "=== $href->{sort}: $href->{preserve}" if($log_verbose);
}
else {
INFO "<<< $href->{sort}";
INFO "<<< $href->{sort}" if($log_verbose);
push(@delete, $href->{name});
}
}
DEBUG "Preserving " . (@$schedule - @delete) . "/" . @$schedule . " items" unless($log_verbose);
return @delete;
}
@ -1511,48 +1592,95 @@ MAIN:
my $droot = $config_target->{droot} || die;
my $target_type = $config_target->{target_type} || die;
my $success = 0;
if($target_type eq "send-receive")
{
INFO "Creating subvolume backup ($target_type) for: $sroot/$svol";
INFO "Using previously created snapshot: $snapshot";
if(config_key($config_target, "receive_log")) {
WARN "Ignoring deprecated option \"receive_log\" for target: $droot"
}
my $incremental = config_key($config_target, "incremental");
if($incremental)
my $parent_snap = "";
# resume missing backups (resume_missing)
if(config_key($config_target, "resume_missing"))
{
my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot);
if($latest_common_src && $latest_common_target) {
my $parent_snap = $latest_common_src->{FS_PATH};
INFO "Incremental from parent snapshot: $parent_snap";
$success = btrfs_send_receive($snapshot, $droot, $parent_snap, $config_target);
INFO "Checking for missing backups of subvolume \"$sroot/$svol\" in: $droot/";
my $found_missing = 0;
# sort children of svol ascending by generation
foreach my $child (sort { $a->{node}->{gen} <=> $b->{node}->{gen} } get_snapshot_children($sroot, $svol))
{
last if($config_target->{ABORTED});
DEBUG "Checking for missing receive targets for \"$child->{FS_PATH}\" in: $droot/";
# TODO: fix for btrfs_progs_compat
if(scalar get_receive_targets_by_uuid($droot, $child->{node}->{uuid})) {
DEBUG "Found matching receive target, skipping: $child->{FS_PATH}";
}
else {
DEBUG "No matching receive targets found, checking schedule for: $child->{FS_PATH}";
# check if the target would be preserved
my ($date, undef) = get_date_tag($child->{SUBVOL_PATH});
next unless($date);
if(scalar schedule_deletion(
schedule => [ { name => $child->{FS_PATH}, sort => $child->{SUBVOL_PATH}, date => $date } ],
today => \@today,
preserve_day_of_week => config_key($config_target, "preserve_day_of_week"),
preserve_daily => config_key($config_target, "target_preserve_daily"),
preserve_weekly => config_key($config_target, "target_preserve_weekly"),
preserve_monthly => config_key($config_target, "target_preserve_monthly"),
))
{
DEBUG "Target would have been deleted by target_perserve rules, skipping resume of: $child->{FS_PATH}";
}
else
{
$found_missing++;
my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot, $child->{node}->{gen});
$parent_snap = $latest_common_src->{FS_PATH} if($latest_common_src);
INFO "Resuming subvolume backup (send-receive) for: $child->{FS_PATH}";
if(macro_send_receive($config_target,
src => $child->{FS_PATH},
target => $droot,
parent => $parent_snap,
resume => 1, # propagated to $config_target->{subvol_received}
))
{
# tag the source snapshot, so that get_latest_common() above can make use of the newly received subvolume
$child->{RECEIVE_TARGET_PRESENT} = $droot;
}
else {
# note: ABORTED flag is already set by macro_send_receive()
ERROR("Error while resuming backups, aborting");
}
}
}
}
elsif($incremental ne "strict") {
INFO "No common parent subvolume present, creating full backup";
$config_target->{subvol_non_incremental} = 1;
$success = btrfs_send_receive($snapshot, $droot, undef, $config_target);
}
else {
WARN "Backup to $droot failed: no common parent subvolume found, and option \"incremental\" is set to \"strict\"";
if($found_missing) {
INFO "Resumed $found_missing backups";
} else {
INFO "No missing backups found";
}
}
else {
INFO "Creating full backup (option \"incremental\" is not set)";
$config_target->{subvol_non_incremental} = 1;
$success = btrfs_send_receive($snapshot, $droot, undef, $config_target);
}
# skip creation if resume_missing failed
next if($config_target->{ABORTED});
# finally receive the previously created snapshot
INFO "Creating subvolume backup (send-receive) for: $sroot/$svol";
my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot);
$parent_snap = $latest_common_src ? $latest_common_src->{FS_PATH} : undef;
macro_send_receive($config_target,
src => $snapshot,
target => $droot,
parent => $parent_snap
);
}
else {
ERROR "Unknown target type \"$target_type\", skipping: $sroot/$svol";
}
if($success) {
$config_target->{subvol_created} = "$droot/$snapshot_name";
}
else {
$config_target->{ABORTED} = "btrfs send/receive command failed";
$config_target->{ABORTED} = "Unknown target type \"$target_type\"";
}
}
}
@ -1591,9 +1719,9 @@ MAIN:
INFO "Cleaning backups of subvolume \"$sroot/$svol\": $droot/$svol.*";
my @schedule;
foreach my $vol (keys %{$vol_info{$droot}}) {
if($vol =~ /^$svol\.([0-9]{4})([0-9]{2})([0-9]{2})/) {
push(@schedule, { name => "$droot/$vol", sort => $vol, date => [ $1, $2, $3 ] });
}
my ($date, undef) = get_date_tag($vol);
next unless($date && ($vol =~ /^svol\./));
push(@schedule, { name => "$droot/$vol", sort => $vol, date => $date });
}
my @delete = schedule_deletion(
schedule => \@schedule,
@ -1602,6 +1730,7 @@ MAIN:
preserve_daily => config_key($config_target, "target_preserve_daily"),
preserve_weekly => config_key($config_target, "target_preserve_weekly"),
preserve_monthly => config_key($config_target, "target_preserve_monthly"),
log_verbose => 1,
);
my $ret = btrfs_subvolume_delete($config_target, @delete);
if(defined($ret)) {
@ -1625,9 +1754,9 @@ MAIN:
INFO "Cleaning snapshots: $sroot/$snapdir$svol.*";
my @schedule;
foreach my $vol (keys %{$vol_info{$sroot}}) {
if($vol =~ /^$snapdir$svol\.([0-9]{4})([0-9]{2})([0-9]{2})/) {
push(@schedule, { name => "$sroot/$vol", sort => $vol, date => [ $1, $2, $3 ] });
}
my ($date, undef) = get_date_tag($vol);
next unless($date && ($vol =~ /^$snapdir$svol\./));
push(@schedule, { name => "$sroot/$vol", sort => $vol, date => $date });
}
my @delete = schedule_deletion(
schedule => \@schedule,
@ -1636,6 +1765,7 @@ MAIN:
preserve_daily => config_key($config_subvol, "snapshot_preserve_daily"),
preserve_weekly => config_key($config_subvol, "snapshot_preserve_weekly"),
preserve_monthly => config_key($config_subvol, "snapshot_preserve_monthly"),
log_verbose => 1,
);
my $ret = btrfs_subvolume_delete($config_subvol, @delete);
if(defined($ret)) {
@ -1663,6 +1793,12 @@ MAIN:
print "Backup Summary ($version_info)\n\n";
print " Date: " . localtime($start_time) . "\n";
print " Config: $config->{SRC_FILE}\n";
print "\nLegend:\n";
print " +++ created subvolume (source snapshot)\n";
print " --- deleted subvolume (source snapshot)\n";
print " *** received subvolume (non-incremental)\n";
print " >>> received subvolume (incremental)\n";
# print " %>> received subvolume (incremental, resume_missing)\n";
print "--------------------------------------------------------------------------------";
foreach my $config_vol (@{$config->{VOLUME}})
{
@ -1688,20 +1824,28 @@ MAIN:
}
foreach my $config_target (@{$config_subvol->{TARGET}})
{
if($config_target->{ABORTED}) {
print "!!! Target \"$config_target->{droot}\" aborted: $config_target->{ABORTED}\n";
$err_count++ unless($config_target->{ABORTED_NOERR});
}
# if($config_target->{schedule}) {
# foreach (sort { $a->{sort} cmp $b->{sort} } @{$config_target->{schedule}}) {
# print(($_->{preserve} ? "===" : "---") . " $_->{name}\n");
# }
# }
my $create_mode = ($config_target->{subvol_non_incremental} ? "***" : ">>>");
print "$create_mode $config_target->{subvol_created}\n" if($config_target->{subvol_created});
foreach(@{$config_target->{subvol_received} // []}) {
my $create_mode = "***";
$create_mode = ">>>" if($_->{parent});
# substr($create_mode, 0, 1, '%') if($_->{resume});
$create_mode = "!!!" if($_->{ERROR});
print "$create_mode $_->{received_name}\n";
}
if($config_target->{subvol_deleted}) {
print "--- $_\n" foreach(sort { $b cmp $a} @{$config_target->{subvol_deleted}});
}
if($config_target->{ABORTED}) {
print "!!! Target \"$config_target->{droot}\" aborted: $config_target->{ABORTED}\n";
$err_count++ unless($config_target->{ABORTED_NOERR});
}
}
}
}

10
btrbk.conf.example
View File

@ -16,13 +16,17 @@
#
snapshot_dir _btrbk_snap
# Always create snapshots, even if the target volume is unreachable
snapshot_create_always yes
# Perform incremental backups (set to "strict" if you want to prevent
# creation of initial backups if no parent is found)
incremental yes
# Always create snapshots, even if the target volume is unreachable
snapshot_create_always yes
# Resume missing backups if the target volume is reachable again.
# Useful in conjunction with "snapshot_create_always".
resume_missing yes
# ssh key for ssh volumes/targets
ssh_identity /etc/btrbk/ssh/id_ed25519
ssh_user root

3
doc/btrbk.1
View File

@ -62,7 +62,8 @@ subvolumes specified in the configuration file.
Then, for each specified target, btrbk creates a new backup subvolume,
incremental from the latest common snapshot / backup subvolume
found. If no common parent subvolume is found, a full backup is
created.
created. This is also done for all missing backups if the
\fIresume_missing\fR option is set.
.PP
In a last step, previous snapshots and backup subvolumes that are not
preserved by their configured retention policy will be deleted. This

20
doc/btrbk.conf.5
View File

@ -45,16 +45,24 @@ Directory in which the btrfs snapshots are created. Relative to
does not autmatically create this directory, and the snapshot creation
will fail if it is not present.
.TP
\fBsnapshot_create_always\fR yes|no
If set, the snapshots are always created, even if the backup subvolume
cannot be created (e.g. if the target subvolume cannot be
reached). Defaults to \(lqno\(rq. Useful for subvolumes on laptops to
make sure the snapshots are created even if you are on the road.
.TP
\fBincremental\fR yes|no|strict
Perform incremental backups. Defaults to \(lqyes\(rq. If set to
\(lqstrict\(rq, non-incremental (initial) backups are never created.
.TP
\fBsnapshot_create_always\fR yes|no
If set, the snapshots are always created, even if the backup subvolume
cannot be created (e.g. if the target subvolume cannot be
reached). Use in conjunction with the \fIresume_missing\fR option to
make sure that the backups are created as soon as the target subvolume
is reachable again. Useful for laptop filesystems in order to make
sure the snapshots are created even if you are on the road. Defaults
to \(lqno\(rq.
.TP
\fBresume_missing\fR yes|no
If set, the backups in the target directory are compared to the source
snapshots, and missing backups are created if needed (complying to the
target preserve matrix). Defaults to \(lqyes\(rq.
.TP
\fBtarget_preserve_daily\fR all|<number>
How many days of backups should be preserved. Defaults to \(lqall\(rq.
.TP