diff --git a/ChangeLog b/ChangeLog index f445572..d0b7b15 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,8 +6,12 @@ btrbk-current (machine-readable) output for "(dry)run" and "tree" commands. * Added "config dump" command (experimental). * Added configuration option "ssh_cipher_spec" (close: #47). + * Added "target raw", with GnuPG and compression support + (experimental). + * Added configuration option "timestamp_format short|long". + * Bugfix: correctly handle "incremental no" option. * Hardened ssh_filter_btrbk.sh script: fine-grained access control, - restrict-path option, sudo option (close: #45) + restrict-path option, sudo option (close: #45). btrbk-0.20.0 diff --git a/README.md b/README.md index fd17a82..ff6435a 100644 --- a/README.md +++ b/README.md @@ -78,10 +78,10 @@ btrbk is in AUR: https://aur.archlinux.org/packages/btrbk/ Synopsis ======== -Please consult the [btrbk(1) man-page] provided with this package for a +Please consult the [btrbk(1)] man-page provided with this package for a full description of the command line options. - [btrbk(1) man-page]: http://www.digint.ch/btrbk/doc/btrbk.html + [btrbk(1)]: http://www.digint.ch/btrbk/doc/btrbk.html Configuration File @@ -90,7 +90,7 @@ Configuration File Before running `btrbk`, you will need to create a configuration file. You might want to take a look at `btrbk.conf.example` provided with this package. For a detailed description, please consult the -[btrbk.conf(5) man-page]. +[btrbk.conf(5)] man-page. When playing around with config-files, it is highly recommended to check the output using the `dryrun` command before executing the @@ -102,7 +102,7 @@ This will read all btrfs information on the source/target filesystems and show what actions would be performed (without writing anything to the disks). - [btrbk.conf(5) man-page]: http://www.digint.ch/btrbk/doc/btrbk.conf.html + [btrbk.conf(5)]: http://www.digint.ch/btrbk/doc/btrbk.conf.html Example: laptop with usb-disk for backups @@ -262,6 +262,35 @@ from 192.168.0.42. The source filesystem is never altered because of `snapshot_preserve_daily all`. +Example: encrypted backup to non-btrfs target +--------------------------------------------- + +If your backup server does not support btrfs, you can send your +subvolumes to a raw file. + +Note: this is an _experimental_ feature! + +/etc/btrbk/btrbk.conf: + + raw_target_compress xz + raw_target_encrypt gpg + gpg_keyring /etc/btrbk/gpg/pubring.gpg + gpg_recipient btrbk@mydomain.com + + volume /mnt/btr_pool + subvolume home + target raw ssh://myserver.mydomain.com/backup + ssh_user btrbk + incremental no + +This will create a GnuPG encrypted, compressed file +`/backup/home.YYYYMMDD.btrfs_.xz.gpg` on the target +host. + +While incremental backups are also supported for raw targets, this is +not recommended (see [btrbk.conf(5)] for details). + + Setting up SSH ============== diff --git a/btrbk b/btrbk index 7de840b..c944565 100755 --- a/btrbk +++ b/btrbk @@ -43,7 +43,7 @@ use strict; use warnings FATAL => qw( all ); use Carp qw(confess); -use Date::Calc qw(Today Delta_Days Day_of_Week); +use Date::Calc qw(Today_and_Now Delta_Days Day_of_Week); use Getopt::Long qw(GetOptions); use Data::Dumper; @@ -59,7 +59,9 @@ my $ip_addr_match = qr/(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([ my $host_name_match = qr/(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])/; my $file_match = qr/[0-9a-zA-Z_@\+\-\.\/]+/; # note: ubuntu uses '@' in the subvolume layout: my $ssh_prefix_match = qr/ssh:\/\/($ip_addr_match|$host_name_match)/; -my $snapshot_postfix_match = qr/\.[0-9]{8}(_[0-9]+)?/; +my $uuid_match = qr/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/; +my $timestamp_postfix_match = qr/\.(?[0-9]{4})(?[0-9]{2})(?
[0-9]{2})(T(?[0-9]{2})(?[0-9]{2}))?(_(?[0-9]+))?/; # matches "YYYYMMDD[Thhmm][_NN]" +my $raw_postfix_match = qr/--(?$uuid_match)(\@(?$uuid_match))\.btrfs?(\.(?(gz|bz2|xz)))?(\.(?gpg))?/; # matches ".btrfs_[@][.gz|bz2|xz][.gpg]" my $group_match = qr/[a-zA-Z0-9_:-]+/; my $ssh_cipher_match = qr/[a-z0-9][a-z0-9@.-]+/; @@ -68,6 +70,7 @@ my %day_of_week_map = ( monday => 1, tuesday => 2, wednesday => 3, thursday => 4 my %config_options = ( # NOTE: the parser always maps "no" to undef # NOTE: keys "volume", "subvolume" and "target" are hardcoded + timestamp_format => { default => "short", accept => [ "short", "long" ], context => [ "root", "volume", "subvolume" ] }, snapshot_dir => { default => undef, accept_file => { relative => 1 } }, snapshot_name => { default => undef, accept_file => { name_only => 1 }, context => [ "subvolume" ] }, # NOTE: defaults to the subvolume name (hardcoded) snapshot_create => { default => "always", accept => [ "no", "always", "ondemand", "onchange" ] }, @@ -86,6 +89,12 @@ my %config_options = ( ssh_port => { default => "default", accept => [ "default" ], accept_numeric => 1 }, ssh_compression => { default => undef, accept => [ "yes", "no" ] }, ssh_cipher_spec => { default => "default", accept_regexp => qr/^$ssh_cipher_match(,$ssh_cipher_match)*$/ }, + + raw_target_compress => { default => undef, accept => [ "no", "gzip", "bzip2", "xz" ] }, + raw_target_encrypt => { default => undef, accept => [ "no", "gpg" ] }, + gpg_keyring => { default => undef, accept_file => { absolute => 1 } }, + gpg_recipient => { default => undef, accept_regexp => qr/^[0-9a-zA-Z_@\+\-\.]+$/ }, + btrfs_progs_compat => { default => undef, accept => [ "yes", "no" ] }, group => { default => undef, accept_regexp => qr/^$group_match(\s*,\s*$group_match)*$/, split => qr/\s*,\s*/ }, @@ -106,7 +115,7 @@ my %config_options = ( } ); -my @config_target_types = qw(send-receive); +my @config_target_types = qw(send-receive raw); my %root_tree_cache; # map URL to SUBTREE (needed since "btrfs subvolume list" does not provide us with the uuid of the btrfs root node) my %vinfo_cache; # map URL to vinfo @@ -232,7 +241,7 @@ sub vinfo($$) my $name = $url; $name =~ s/^.*\///; my %info = ( - URL => $url, + URL => $url, NAME => $name, ); @@ -927,21 +936,21 @@ sub btrfs_subvolume_delete($@) } -sub btrfs_send_receive($$$) +sub btrfs_send_receive($$$$) { my $snapshot = shift || die; my $target = shift || die; my $parent = shift; + my $ret_vol_received = shift; my $snapshot_path = $snapshot->{PATH} // die; - my $snapshot_rsh = $snapshot->{RSH}; my $target_path = $target->{PATH} // die; - my $target_rsh = $target->{RSH}; my $parent_path = $parent ? $parent->{PATH} : undef; - my $snapshot_name = $snapshot_path; - $snapshot_name =~ s/^.*\///; - INFO ">>> $target->{PRINT}/$snapshot_name"; - print STDOUT "Receiving subvol: $target->{PRINT}/$snapshot_name\n" if($show_progress && (not $dryrun)); + my $vol_received = vinfo_child($target, $snapshot->{NAME}); + $$ret_vol_received = $vol_received if(ref $ret_vol_received); + + INFO ">>> $vol_received->{PRINT}"; + print STDOUT "Receiving subvol: $vol_received->{PRINT}\n" if($show_progress && (not $dryrun)); DEBUG "[btrfs] send/receive" . ($parent ? " (incremental)" : " (complete)") . ":"; DEBUG "[btrfs] source: $snapshot->{PRINT}"; @@ -957,7 +966,7 @@ sub btrfs_send_receive($$$) my @cmd_pipe; push @cmd_pipe, { cmd => [ qw(btrfs send), @send_options, $snapshot_path ], - rsh => $snapshot_rsh, + rsh => $snapshot->{RSH}, name => "btrfs send", }; push @cmd_pipe, { @@ -965,12 +974,116 @@ sub btrfs_send_receive($$$) } if($show_progress); push @cmd_pipe, { cmd => [ qw(btrfs receive), @receive_options, $target_path . '/' ], - rsh => $target_rsh, + rsh => $target->{RSH}, name => "btrfs receive", }; my $ret = run_cmd(@cmd_pipe); unless(defined($ret)) { ERROR "Failed to send/receive btrfs subvolume: $snapshot->{PRINT} " . ($parent_path ? "[$parent_path]" : "") . " -> $target->{PRINT}"; + + # NOTE: btrfs-progs v3.19.1 does not delete garbled received subvolume, + # we need to do this by hand. + # TODO: remove this as soon as btrfs-progs handle receive errors correctly. + DEBUG "send/received failed, deleting (possibly present and garbled) received subvolume: $vol_received->{PRINT}"; + my $ret = btrfs_subvolume_delete($vol_received, commit => "after"); + if(defined($ret)) { + WARN "Deleted partially received (garbled) subvolume: $vol_received->{PRINT}"; + } + else { + WARN "Deletion of partially received (garbled) subvolume failed, assuming clean environment: $vol_received->{PRINT}"; + } + + return undef; + } + return 1; +} + + +sub btrfs_send_to_file($$$$;@) +{ + my $snapshot = shift || die; + my $target = shift || die; + my $parent = shift; + my $ret_vol_received = shift; + my %opts = @_; + my $snapshot_path = $snapshot->{PATH} // die; + my $target_path = $target->{PATH} // die; + my $parent_path = $parent ? $parent->{PATH} : undef; + my $parent_uuid = $parent ? $parent->{uuid} : undef ; + my $received_uuid = $snapshot->{uuid}; + $received_uuid = "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX" if((not $received_uuid) && $dryrun); + die unless($received_uuid); + die if($parent && !$parent_uuid); + + my $target_filename = $snapshot->{NAME} || die; + $target_filename .= "--$received_uuid"; + $target_filename .= '@' . $parent_uuid if($parent_uuid); + $target_filename .= ".btrfs"; + + my %compress = ( gzip => { pipe => { cmd => [ 'gzip' ], name => 'gzip' }, postfix => '.gz' }, + bzip2 => { pipe => { cmd => [ 'bzip2' ], name => 'bzip2' }, postfix => '.bz2' }, + xz => { pipe => { cmd => [ 'xz' ], name => 'xz' }, postfix => '.xz' }, + ); + + my @send_options; + push(@send_options, '-v') if($loglevel >= 3); + push(@send_options, '-p', $parent_path) if($parent_path); + + my @cmd_pipe; + push @cmd_pipe, { + cmd => [ qw(btrfs send), @send_options, $snapshot_path ], + rsh => $snapshot->{RSH}, + name => "btrfs send", + }; + push @cmd_pipe, { + cmd => [ '/usr/bin/pv' ], + } if($show_progress); + if($opts{compress}) { + die unless($compress{$opts{compress}}); + $target_filename .= $compress{$opts{compress}}->{postfix}; + push @cmd_pipe, $compress{$opts{compress}}->{pipe}; + } + if($opts{encrypt}) { + die unless($opts{encrypt}->{type} eq "gpg"); + $target_filename .= '.gpg'; + my @gpg_options = ( '--batch', '--no-tty', '--trust-model', 'always' ); + push(@gpg_options, ( '--no-default-keyring', '--keyring', $opts{encrypt}->{keyring} )) if($opts{encrypt}->{keyring}); + push(@gpg_options, ( '--default-recipient', $opts{encrypt}->{recipient} )) if($opts{encrypt}->{recipient}); + push @cmd_pipe, { + cmd => [ 'gpg', @gpg_options, '--encrypt' ], + name => 'gpg', + }; + } + push @cmd_pipe, { + cmd => [ 'dd', 'status=none', "of=$target_path/$target_filename" ], + rsh => $target->{RSH}, + name => 'dd', + }; + + my $vol_received = vinfo_child($target, $target_filename); + $$ret_vol_received = $vol_received if(ref $ret_vol_received); + + INFO ">>> $vol_received->{PRINT}"; + print STDOUT "Receiving subvol (raw): $vol_received->{PRINT}\n" if($show_progress && (not $dryrun)); + + DEBUG "[btrfs] send-to-file" . ($parent ? " (incremental)" : " (complete)") . ":"; + DEBUG "[btrfs] source: $snapshot->{PRINT}"; + DEBUG "[btrfs] parent: $parent->{PRINT}" if($parent); + DEBUG "[btrfs] target: $target->{PRINT}"; + + my $ret = run_cmd(@cmd_pipe); + if(defined($ret)) { + # Test target file for "exists and size > 0" after writing, + # as we can not rely on the exit status of 'dd' + DEBUG "Testing target file (non-zero size): $target->{PRINT}"; + $ret = run_cmd({ + cmd => ['test', '-s', "$target_path/$target_filename"], + rsh => $target->{RSH}, + name => "test", + }); + } + unless(defined($ret)) { + ERROR "Failed to send btrfs subvolume to raw file: $snapshot->{PRINT} " . ($parent_path ? "[$parent_path]" : "") . " -> $vol_received->{PRINT}"; return undef; } return 1; @@ -1122,6 +1235,7 @@ sub macro_send_receive($@) my $snapshot = $info{snapshot} || die; my $target = $info{target} || die; my $parent = $info{parent}; + my $target_type = $config_target->{target_type} || die; my $incremental = config_key($config_target, "incremental"); INFO "Receiving from snapshot: $snapshot->{PRINT}"; @@ -1136,12 +1250,6 @@ sub macro_send_receive($@) return undef; } - # add info to $config->{SUBVOL_RECEIVED} - my $vol_received = vinfo_child($target, $snapshot->{NAME}); - $info{received_subvolume} = $vol_received; - $config_target->{SUBVOL_RECEIVED} //= []; - push(@{$config_target->{SUBVOL_RECEIVED}}, \%info); - if($incremental) { # create backup from latest common @@ -1160,42 +1268,86 @@ sub macro_send_receive($@) } else { INFO "Option \"incremental\" is not set, creating full backup"; + $parent = undef; delete $info{parent}; } - if(btrfs_send_receive($snapshot, $target, $parent)) { - return 1; - } else { + my $ret; + my $vol_received; + if($target_type eq "send-receive") + { + $ret = btrfs_send_receive($snapshot, $target, $parent, \$vol_received); + $config_target->{ABORTED} = "Failed to send/receive subvolume" unless($ret); + } + elsif($target_type eq "raw") + { + unless($dryrun) { + # make sure we know the snapshot uuid + unless($snapshot->{uuid}) { + DEBUG "Fetching uuid of new snapshot: $snapshot->{PRINT}"; + my $detail = btrfs_subvolume_detail($snapshot); + die unless($detail->{uuid}); + vinfo_set_detail($snapshot, { uuid => $detail->{uuid} }); + } + } + + my $compress = config_key($config_target, "raw_target_compress"); + my $encrypt = undef; + my $encrypt_type = config_key($config_target, "raw_target_encrypt"); + if($encrypt_type) { + die unless($encrypt_type eq "gpg"); + $encrypt = { type => $encrypt_type, + keyring => config_key($config_target, "gpg_keyring"), + recipient => config_key($config_target, "gpg_recipient"), + } + } + $ret = btrfs_send_to_file($snapshot, $target, $parent, \$vol_received, compress => $compress, encrypt => $encrypt); + $config_target->{ABORTED} = "Failed to send subvolume to raw file" unless($ret); + } + else + { + die "Illegal target type \"$target_type\""; + } + + # add info to $config->{SUBVOL_RECEIVED} + $info{received_type} = $target_type || die; + $info{received_subvolume} = $vol_received || die; + $config_target->{SUBVOL_RECEIVED} //= []; + push(@{$config_target->{SUBVOL_RECEIVED}}, \%info); + + unless($ret) { $info{ERROR} = 1; - $config_target->{ABORTED} = "Failed to send/receive subvolume"; - - # NOTE: btrfs-progs v3.19.1 does not delete garbled received subvolume, - # we need to do this by hand. - # TODO: remove this as soon as btrfs-progs handle receive errors correctly. - DEBUG "send/received failed, deleting (possibly present and garbled) received subvolume: $vol_received->{PRINT}"; - my $ret = btrfs_subvolume_delete($vol_received, commit => "after"); - if(defined($ret)) { - WARN "Deleted partially received (garbled) subvolume: $vol_received->{PRINT}"; - } - else { - WARN "Deletion of partially received (garbled) subvolume failed, assuming clean environment: $vol_received->{PRINT}"; - } - return undef; } + return 1; } -sub get_date_tag($) +# returns { btrbk_date => [ yyyy, mm, dd, hh, mm, ] } or undef +# fixed array length of 6, all individually defaulting to 0 +sub parse_filename($$;$) { - my $name = shift; - $name =~ s/_([0-9]+)$//; - my $postfix_counter = $1 // 0; - my $date = undef; - if($name =~ /\.([0-9]{4})([0-9]{2})([0-9]{2})$/) { - $date = [ $1, $2, $3 ]; + my $file = shift; + my $name_match = shift; + my $raw_format = shift || 0; + my %raw_info; + if($raw_format) + { + return undef unless($file =~ /^\Q$name_match\E$timestamp_postfix_match$raw_postfix_match$/); + die unless($+{YYYY} && $+{MM} && $+{DD}); + return { btrbk_date => [ $+{YYYY}, $+{MM}, $+{DD}, ($+{hh} // 0), ($+{mm} // 0), ($+{NN} // 0) ], + received_uuid => $+{received_uuid} // die, + parent_uuid => $+{parent_uuid} // '-', + ENCRYPT => $+{encrypt} // "", + COMPRESS => $+{compress} // "", + }; + } + else + { + return undef unless($file =~ /^\Q$name_match\E$timestamp_postfix_match$/); + die unless($+{YYYY} && $+{MM} && $+{DD}); + return { btrbk_date => [ $+{YYYY}, $+{MM}, $+{DD}, ($+{hh} // 0), ($+{mm} // 0), ($+{NN} // 0) ] }; } - return ($date, $postfix_counter); } @@ -1360,10 +1512,12 @@ sub schedule(@) } # sort the schedule, ascending by date - my @sorted_schedule = sort { ($a->{date}->[0] <=> $b->{date}->[0]) || - ($a->{date}->[1] <=> $b->{date}->[1]) || - ($a->{date}->[2] <=> $b->{date}->[2]) || - ($a->{date_ext} <=> $b->{date_ext}) + my @sorted_schedule = sort { ($a->{btrbk_date}->[0] <=> $b->{btrbk_date}->[0]) || + ($a->{btrbk_date}->[1] <=> $b->{btrbk_date}->[1]) || + ($a->{btrbk_date}->[2] <=> $b->{btrbk_date}->[2]) || + ($a->{btrbk_date}->[3] <=> $b->{btrbk_date}->[3]) || + ($a->{btrbk_date}->[4] <=> $b->{btrbk_date}->[4]) || + ($a->{btrbk_date}->[5] <=> $b->{btrbk_date}->[5]) } @$schedule; # first, do our calendar calculations @@ -1373,7 +1527,7 @@ sub schedule(@) TRACE "last day before next $preserve_day_of_week is in $delta_days_to_eow_from_today days"; foreach my $href (@sorted_schedule) { - my @date = @{$href->{date}}; + my @date = @{$href->{btrbk_date}}[0..2]; # Date::Calc takes: @date = ( yy, mm, dd ) my $delta_days = Delta_Days(@date, @today); my $delta_days_to_eow = $delta_days + $delta_days_to_eow_from_today; { @@ -1388,7 +1542,7 @@ sub schedule(@) if($preserve_latest && (scalar @sorted_schedule)) { my $href = $sorted_schedule[-1]; - $href->{preserve} ||= "preserve forced: latest in list"; + $href->{preserve} ||= $preserve_latest; } # filter daily, weekly, monthly @@ -1481,7 +1635,8 @@ MAIN: Getopt::Long::Configure qw(gnu_getopt); $Data::Dumper::Sortkeys = 1; my $start_time = time; - my @today = Today(); + my @today_and_now = Today_and_Now(); + my @today = @today_and_now[0..2]; my ($config_cmdline, $quiet, $verbose, $preserve_backups, $resume_only); @@ -1969,10 +2124,91 @@ MAIN: { next if($config_target->{ABORTED}); my $droot = vinfo($config_target->{url}, $config_target); - unless(vinfo_root($droot)) { - $config_target->{ABORTED} = "Failed to fetch subvolume detail" . ($err ? ": $err" : ""); - WARN "Skipping target \"$droot->{PRINT}\": $config_target->{ABORTED}"; - next; + + my $target_type = $config_target->{target_type} || die; + if($target_type eq "send-receive") + { + unless(vinfo_root($droot)) { + $config_target->{ABORTED} = "Failed to fetch subvolume detail" . ($err ? ": $err" : ""); + WARN "Skipping target \"$droot->{PRINT}\": $config_target->{ABORTED}"; + next; + } + } + elsif($target_type eq "raw") + { + DEBUG "Creating raw subvolume list: $droot->{PRINT}"; + my $ret = run_cmd( + # NOTE: check for file size >0, which causes bad (zero-sized) images to be resumed + # TODO: fix btrfs_send_to_file() to never create bad images + cmd => [ 'find', $droot->{PATH} . '/', '-maxdepth', '1', '-type', 'f', '-size', '+0' ], + rsh => $droot->{RSH}, + # note: use something like this to get the real (link resolved) path + # cmd => [ "find", $droot->{PATH} . '/', "-maxdepth", "1", "-name", "$snapshot_basename.\*.raw\*", '-printf', '%f\0', '-exec', 'realpath', '-z', '{}', ';' ], + non_destructive => 1, + ); + unless(defined($ret)) { + $config_target->{ABORTED} = "Failed to list files from: $droot->{PATH}"; + WARN "Skipping target \"$droot->{PRINT}\": $config_target->{ABORTED}"; + next; + } + + my %subvol_list; + my %parent_uuid_list; + foreach my $file (split("\n", $ret)) + { + unless($file =~ /^$file_match$/) { + DEBUG "Skipping non-parseable file: \"$file\""; + next; + } + unless($file =~ s/^\Q$droot->{PATH}\E\///) { + $config_target->{ABORTED} = "Unexpected result from 'find': file \"$file\" is not under \"$droot->{PATH}\""; + last; + } + my $filename_info = parse_filename($file, $snapshot_basename, 1); + unless($filename_info) { + DEBUG "Skipping file (not btrbk raw): \"$file\""; + next; + } + + # Fake btrfs subvolume information (received_uuid, parent_uuid) from filename info. + # + # NOTE: parent_uuid in $filename_info is the "parent of the source subvolume", NOT the + # "parent of the received subvolume". We fake the real parent_uuid with the one from + # the filename here. + my $subvol = vinfo_child($droot, $file); + vinfo_set_detail($subvol, $filename_info); + + $subvol_list{$file} = $subvol; + $parent_uuid_list{$filename_info->{parent_uuid}} = $subvol if($filename_info->{parent_uuid} ne '-'); + } + if($config_target->{ABORTED}) { + WARN "Skipping target \"$droot->{PRINT}\": $config_target->{ABORTED}"; + next; + } + DEBUG "Found " . scalar(keys %subvol_list) . " raw subvolume backups of: $svol->{PRINT}"; + $droot->{SUBVOL_LIST} = \%subvol_list; + $droot->{REAL_URL} = $droot->{URL}; # ignore links here + + # Make sure that incremental backup chains are never broken: + foreach my $subvol (values %subvol_list) + { + # If restoring a backup from raw btrfs images (using "incremental yes|strict"): + # "btrfs send -p parent source > svol.btrfs", the backups + # on the target will get corrupted (unusable!) as soon as + # an any files in the chain gets deleted. + # + # We need to make sure btrbk will NEVER delete those: + # - svol.--.btrfs : root (full) image + # - svol.--[@].btrfs : incremental image + + if(my $child = $parent_uuid_list{$subvol->{received_uuid}}) { + DEBUG "Found parent/child partners, forcing preserve of: \"$subvol->{PRINT}\", \"$child->{PRINT}\""; + $subvol->{FORCE_PRESERVE} = "preserve forced: parent of another raw target"; + $child->{FORCE_PRESERVE} ||= "preserve forced: child of another raw target"; + } + } + + # TRACE(Data::Dumper->Dump([\%subvol_list], ["vinfo_raw_subvol_list{$droot}"])); } $config_target->{droot} = $droot; @@ -2113,7 +2349,6 @@ MAIN: # # create snapshots # - my $timestamp = sprintf("%04d%02d%02d", @today); foreach my $config_vol (@{$config->{VOLUME}}) { next if($config_vol->{ABORTED}); @@ -2152,10 +2387,10 @@ MAIN: elsif($snapshot_create eq "ondemand") { # check if at least one target is present if(scalar grep { not $_->{ABORTED} } @{$config_subvol->{TARGET}}) { - DEBUG "Snapshot creation enabled (snapshot_create=ondemand): at least one send-receive target is present"; + DEBUG "Snapshot creation enabled (snapshot_create=ondemand): at least one target is present"; } else { - INFO "Snapshot creation skipped: snapshot_create=ondemand, and no send-receive target is present for: $svol->{PRINT}"; + INFO "Snapshot creation skipped: snapshot_create=ondemand, and no target is present for: $svol->{PRINT}"; next; } } @@ -2164,6 +2399,9 @@ MAIN: } # find unique snapshot name + my $timestamp = ((config_key($config_subvol, "timestamp_format") eq "short") ? + sprintf("%04d%02d%02d", @today) : + sprintf("%04d%02d%02dT%02d%02d", @today_and_now)); my @unconfirmed_target_name; my @lookup = keys %{vinfo_subvol_list($sroot)}; @lookup = grep s/^\Q$snapdir\E// , @lookup; @@ -2219,110 +2457,109 @@ MAIN: { next if($config_target->{ABORTED}); my $droot = $config_target->{droot} || die; - my $target_type = $config_target->{target_type} || die; - if($target_type eq "send-receive") + # + # resume missing backups (resume_missing) + # + if(config_key($config_target, "resume_missing")) { - # - # resume missing backups (resume_missing) - # - if(config_key($config_target, "resume_missing")) - { - INFO "Checking for missing backups of subvolume \"$svol->{PRINT}\" in: $droot->{PRINT}/"; - my @schedule; - my $resume_total = 0; - my $resume_success = 0; + INFO "Checking for missing backups of subvolume \"$svol->{PRINT}\" in: $droot->{PRINT}/"; + my @schedule; + my $resume_total = 0; + my $resume_success = 0; - foreach my $child (sort { $a->{cgen} <=> $b->{cgen} } get_snapshot_children($sroot, $svol)) + foreach my $child (sort { $a->{cgen} <=> $b->{cgen} } get_snapshot_children($sroot, $svol)) + { + my $filename_info = parse_filename($child->{SUBVOL_PATH}, $snapdir . $snapshot_basename); + next unless($filename_info); # ignore non-btrbk files + + if(scalar get_receive_targets($droot, $child)) { + DEBUG "Found matching receive target, skipping: $child->{PRINT}"; + } + else { + DEBUG "No matching receive targets found, adding resume candidate: $child->{PRINT}"; + + if(my $err_vol = vinfo_subvol($droot, $child->{NAME})) { + WARN "Target subvolume \"$err_vol->{PRINT}\" exists, but is not a receive target of \"$child->{PRINT}\""; + } + + # check if the target would be preserved + push(@schedule, { value => $child, + btrbk_date => $filename_info->{btrbk_date}, + preserve => $child->{FORCE_PRESERVE}, + }), + } + } + + if(scalar @schedule) + { + DEBUG "Checking schedule for resume candidates"; + # add all present backups to schedule, with no value + # these are needed for correct results of schedule() + foreach my $vol (values %{vinfo_subvol_list($droot)}) { + my $filename_info = parse_filename($vol->{SUBVOL_PATH}, $snapshot_basename, ($config_target->{target_type} eq "raw")); + next unless($filename_info); # ignore non-btrbk files + push(@schedule, { value => undef, + btrbk_date => $filename_info->{btrbk_date}, + preserve => $vol->{FORCE_PRESERVE}, + }); + } + my ($preserve, undef) = schedule( + schedule => \@schedule, + today => \@today, + preserve_day_of_week => config_key($config_target, "preserve_day_of_week"), + preserve_daily => config_key($config_target, "target_preserve_daily"), + preserve_weekly => config_key($config_target, "target_preserve_weekly"), + preserve_monthly => config_key($config_target, "target_preserve_monthly"), + preserve_latest => $preserve_latest, + ); + my @resume = grep defined, @$preserve; # remove entries with no value from list (target subvolumes) + $resume_total = scalar @resume; + + foreach my $child (sort { $a->{cgen} <=> $b->{cgen} } @resume) { - if(scalar get_receive_targets($droot, $child)) { - DEBUG "Found matching receive target, skipping: $child->{PRINT}"; + INFO "Resuming subvolume backup (send-receive) for: $child->{PRINT}"; + my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot, $child->{cgen}); + if(macro_send_receive($config_target, + snapshot => $child, + target => $droot, + parent => $latest_common_src, # this is if no common found + resume => 1, # propagated to $config_target->{SUBVOL_RECEIVED} + )) + { + # tag the source snapshot, so that get_latest_common() above can make use of the newly received subvolume + $child->{RECEIVE_TARGET_PRESENT} = $droot->{URL}; + $resume_success++; } else { - DEBUG "No matching receive targets found, adding resume candidate: $child->{PRINT}"; - - if(my $err_vol = vinfo_subvol($droot, $child->{NAME})) { - WARN "Target subvolume \"$err_vol->{PRINT}\" exists, but is not a receive target of \"$child->{PRINT}\""; - } - - # check if the target would be preserved - my ($date, $date_ext) = get_date_tag($child->{SUBVOL_PATH}); - next unless($date && ($child->{SUBVOL_PATH} =~ /^\Q$snapdir$snapshot_basename\E$snapshot_postfix_match$/)); - push(@schedule, { value => $child, date => $date, date_ext => $date_ext }), + # note: ABORTED flag is already set by macro_send_receive() + ERROR("Error while resuming backups, aborting"); + last; } } - - if(scalar @schedule) - { - DEBUG "Checking schedule for resume candidates"; - # add all present backups to schedule, with no value - # these are needed for correct results of schedule() - foreach my $vol (values %{vinfo_subvol_list($droot)}) { - next unless($vol->{SUBVOL_PATH} =~ /^\Q$snapshot_basename\E$snapshot_postfix_match$/); - my ($date, $date_ext) = get_date_tag($vol->{NAME}); - next unless($date); - push(@schedule, { value => undef, date => $date, date_ext => $date_ext }); - } - my ($preserve, undef) = schedule( - schedule => \@schedule, - today => \@today, - preserve_day_of_week => config_key($config_target, "preserve_day_of_week"), - preserve_daily => config_key($config_target, "target_preserve_daily"), - preserve_weekly => config_key($config_target, "target_preserve_weekly"), - preserve_monthly => config_key($config_target, "target_preserve_monthly"), - preserve_latest => $preserve_latest, - ); - my @resume = grep defined, @$preserve; # remove entries with no value from list (target subvolumes) - $resume_total = scalar @resume; - - foreach my $child (sort { $a->{cgen} <=> $b->{cgen} } @resume) { - INFO "Resuming subvolume backup (send-receive) for: $child->{PRINT}"; - my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot, $child->{cgen}); - if(macro_send_receive($config_target, - snapshot => $child, - target => $droot, - parent => $latest_common_src, # this is if no common found - resume => 1, # propagated to $config_target->{SUBVOL_RECEIVED} - )) - { - # tag the source snapshot, so that get_latest_common() above can make use of the newly received subvolume - $child->{RECEIVE_TARGET_PRESENT} = $droot->{URL}; - $resume_success++; - } - else { - # note: ABORTED flag is already set by macro_send_receive() - ERROR("Error while resuming backups, aborting"); - last; - } - } - } - - if($resume_total) { - INFO "Resumed $resume_success/$resume_total missing backups"; - } else { - INFO "No missing backups found"; - } - } # /resume_missing - - unless($resume_only) - { - # skip creation if resume_missing failed - next if($config_target->{ABORTED}); - next unless($config_subvol->{SNAPSHOT}); - - # finally receive the previously created snapshot - INFO "Creating subvolume backup (send-receive) for: $svol->{PRINT}"; - my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot); - macro_send_receive($config_target, - snapshot => $config_subvol->{SNAPSHOT}, - target => $droot, - parent => $latest_common_src, # this is if no common found - ); } - } - else { - ERROR "Unknown target type \"$target_type\", skipping: $svol->{PRINT}"; - $config_target->{ABORTED} = "Unknown target type \"$target_type\""; + + if($resume_total) { + INFO "Resumed $resume_success/$resume_total missing backups"; + } else { + INFO "No missing backups found"; + } + } # /resume_missing + + unless($resume_only) + { + # skip creation if resume_missing failed + next if($config_target->{ABORTED}); + next unless($config_subvol->{SNAPSHOT}); + + # finally receive the previously created snapshot + INFO "Creating subvolume backup (send-receive) for: $svol->{PRINT}"; + my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot); + macro_send_receive($config_target, + snapshot => $config_subvol->{SNAPSHOT}, + target => $droot, + parent => $latest_common_src, # this is if no common found + ); } } } @@ -2347,7 +2584,8 @@ MAIN: my $svol = $config_subvol->{svol} || die; my $snapdir = config_key($config_subvol, "snapshot_dir", postfix => '/') // ""; my $snapshot_basename = config_key($config_subvol, "snapshot_name") // die; - my $preserve_latest = $config_subvol->{SNAPSHOT} ? 0 : 1; + my $preserve_latest_snapshot = $config_subvol->{SNAPSHOT} ? 0 : "preserve forced: latest in list"; + my $preserve_latest_backup = $preserve_latest_snapshot; my $target_aborted = 0; foreach my $config_target (@{$config_subvol->{TARGET}}) @@ -2361,6 +2599,18 @@ MAIN: next; } my $droot = $config_target->{droot} || die; + if($config_target->{target_type} eq "raw") { + if(config_key($config_target, "incremental")) { + # In incremental mode, the latest backup is most certainly our parent. + # (see note on FORCE_PRESERVE above) + $preserve_latest_backup ||= "preserve forced: possibly parent of latest backup"; + # Note that we could check against $config_subvol->{SNAPSHOT}->{parent_uuid} to be certain, + # but this information is not available in $dryrun: + # foreach my $vol (values %{vinfo_subvol_list($droot)}) { + # $vol->{FORCE_PRESERVE} = 1 if($vol->{received_uuid} eq $config_subvol->{SNAPSHOT}->{parent_uuid}); + # } + } + } # # delete backups @@ -2368,15 +2618,19 @@ MAIN: INFO "Cleaning backups of subvolume \"$svol->{PRINT}\": $droot->{PRINT}/$snapshot_basename.*"; my @schedule; foreach my $vol (values %{vinfo_subvol_list($droot)}) { - next unless($vol->{SUBVOL_PATH} =~ /^\Q$snapshot_basename\E$snapshot_postfix_match$/); + my $filename_info = parse_filename($vol->{SUBVOL_PATH}, $snapshot_basename, ($config_target->{target_type} eq "raw")); + next unless($filename_info); # ignore non-btrbk files + # NOTE: checking received_uuid does not make much sense, as this received_uuid is propagated to snapshots # if($vol->{received_uuid} && ($vol->{received_uuid} eq '-')) { # INFO "Target subvolume is not a received backup, skipping deletion of: $vol->{PRINT}"; # next; # } - my ($date, $date_ext) = get_date_tag($vol->{NAME}); - next unless($date); - push(@schedule, { value => $vol, name => $vol->{PRINT}, date => $date, date_ext => $date_ext }); + push(@schedule, { value => $vol, + name => $vol->{PRINT}, + btrbk_date => $filename_info->{btrbk_date}, + preserve => $vol->{FORCE_PRESERVE} + }); } my (undef, $delete) = schedule( schedule => \@schedule, @@ -2385,7 +2639,7 @@ MAIN: preserve_daily => config_key($config_target, "target_preserve_daily"), preserve_weekly => config_key($config_target, "target_preserve_weekly"), preserve_monthly => config_key($config_target, "target_preserve_monthly"), - preserve_latest => $preserve_latest, + preserve_latest => $preserve_latest_backup, log_verbose => 1, ); my $ret = btrfs_subvolume_delete($delete, commit => config_key($config_target, "btrfs_commit_delete")); @@ -2413,10 +2667,12 @@ MAIN: INFO "Cleaning snapshots: $sroot->{PRINT}/$snapdir$snapshot_basename.*"; my @schedule; foreach my $vol (values %{vinfo_subvol_list($sroot)}) { - next unless($vol->{SUBVOL_PATH} =~ /^\Q$snapdir$snapshot_basename\E$snapshot_postfix_match$/); - my ($date, $date_ext) = get_date_tag($vol->{NAME}); - next unless($date); - push(@schedule, { value => $vol, name => $vol->{PRINT}, date => $date, date_ext => $date_ext }); + my $filename_info = parse_filename($vol->{SUBVOL_PATH}, $snapdir . $snapshot_basename); + next unless($filename_info); # ignore non-btrbk files + push(@schedule, { value => $vol, + name => $vol->{PRINT}, + btrbk_date => $filename_info->{btrbk_date} + }); } my (undef, $delete) = schedule( schedule => \@schedule, @@ -2425,7 +2681,7 @@ MAIN: preserve_daily => config_key($config_subvol, "snapshot_preserve_daily"), preserve_weekly => config_key($config_subvol, "snapshot_preserve_weekly"), preserve_monthly => config_key($config_subvol, "snapshot_preserve_monthly"), - preserve_latest => $preserve_latest, + preserve_latest => $preserve_latest_snapshot, log_verbose => 1, ); my $ret = btrfs_subvolume_delete($delete, commit => config_key($config_subvol, "btrfs_commit_delete")); diff --git a/doc/btrbk.1 b/doc/btrbk.1 index 77d15d4..aa2dc37 100644 --- a/doc/btrbk.1 +++ b/doc/btrbk.1 @@ -1,4 +1,4 @@ -.TH "btrbk" "1" "2015-09-02" "btrbk v0.20.0" "" +.TH "btrbk" "1" "2015-09-29" "btrbk v0.21.0-dev" "" .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) @@ -21,12 +21,17 @@ from one source to multiple destinations. Snapshots as well as backup subvolume names are created in form: .PP .RS 4 -.YYYYMMDD[_N] +.[_N] .RE .PP -Where YYYY is the year, MM is the month, and DD is the day of -creation, and, if multiple backups are created on the same day, N will -be incremented on each backup, starting at 1. +Where is identical to the source subvolume name, +unless the configuration option \fIsnapshot_name\fR is set. The + is either "YYYYMMDD" or "YYYYMMDDThhmm" (dependent of the +\fItimestamp_format\fR configuration option), where "YYYY" is the +year, "MM" is the month, "DD" is the day, "hh" is the hour and "mm" is +the minute of the creation time (local time of the host running +btrbk). If multiple snapshots/backups are created on the same +date/time, N will be incremented on each backup, starting at 1. .SH OPTIONS .PP \-h, \-\-help diff --git a/doc/btrbk.conf.5 b/doc/btrbk.conf.5 index f3a8b48..11f96a7 100644 --- a/doc/btrbk.conf.5 +++ b/doc/btrbk.conf.5 @@ -1,4 +1,4 @@ -.TH "btrbk.conf" "5" "2015-09-02" "btrbk v0.20.0" "" +.TH "btrbk.conf" "5" "2015-09-29" "btrbk v0.21.0-dev" "" .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) @@ -19,8 +19,7 @@ defined either globally or within a section. The options specified always apply to the last section encountered, superseding the values set in upper-level sections. This means that global options must be set before any sections are defined. -.PP -The sections are: +.SH SECTIONS .PP \fBvolume\fR | .RS 4 @@ -33,17 +32,19 @@ filesystem mounted with the \fIsubvolid=0\fR option. \fBsubvolume\fR .RS 4 Subvolume to be backed up, relative to the \fI\fR -specified in the \fIvolume\fR section. +specified in the \fIvolume\fR section. Multiple \fIsubvolume\fR +sections are allowed within \fIvolume\fR sections. .RE .PP -\fBtarget\fR | +\fBtarget\fR | .RS 4 Target type and directory where the backup subvolumes are to be -created. \fI\fR must be an absolute path and point -to a btrfs volume (or subvolume). Currently the the only valid -\fI\fR is \[lq]send\-receive\[rq]. +created. See the TARGET TYPES section for supported +\fI\fR. Multiple \fItarget\fR sections are allowed within +\fIsubvolume\fR sections. +.RE .PP -For the \fIvolume\fR and \fItarget\fR sections, you can also specify a +For the \fIvolume\fR and \fItarget\fR sections, you can specify a ssh-url instead of a local directory. The syntax for \fI\fR is: .PP .RS 4 @@ -56,8 +57,67 @@ Note that btrfs is very picky on file names (mainly for security reasons), only the characters [0-9] [a-z] [A-Z] and "._+-@" are allowed. .RE +.SH TARGET TYPES .PP -The configuration options are: +\fBsend-receive\fR +.RS 4 +Backup to a btrfs filesystem, using "btrfs send/receive". The +\fI\fR must be an absolute path and point to a btrfs +volume (or subvolume). See btrfs-send(8), btrfs-receive(8). +.RE +.PP +\fBraw\fR \fI*experimental*\fR +.RS 4 +Backup to a raw (filesystem independent) file from the output of +btrfs-send(8), with optional compression and encryption. +.PP +Additional options for raw target: +.PP +.RS 4 +raw_target_compress gzip|bzip2|xz|no +.PD 0 +.PP +raw_target_encrypt gpg|no +.PP +gpg_keyring +.PP +gpg_recipient +.RE +.PD +.PP +The target file name is: +.PP +.RS 4 +--[@].btrfs[.gz|.bz2|.xz][.gpg] +.RE +.PP +It is recommended to set "incremental no" for raw backups, for the +following reasons: +.IP \[bu] 2 +The target preserve mechanism is disabled for raw \fIincremental\fR +backups, since deleting of any intermediate backup would destroy the +consistency of all subsequent backups (btrbk will never delete any +backup in a raw incremental chain). +.IP \[bu] +No rotation of incremental backups: if \fIincremental\fR is set, a +full backup must be triggered manually from time to time in order to +be able to delete old backups. +.RE +.SH OPTIONS +.PP +\fBtimestamp_format\fR short|long +.RS 4 +Timestamp format used as postfix for new snapshot subvolume names. +defaults to \[lq]short\[rq]. +.IP \[bu] 2 +\fIshort\fR: YYYYMMDD[_N] (e.g. "20150825", "20150825_1") +.IP \[bu] +\fIlong\fR: YYYYMMDDhhmm[_N] (e.g. "20150825T1531") +.PP +Note that a postfix "_N" is only appended to the timestamp if a +snapshot/backup already exists with the timestamp of current +date/time. +.RE .PP \fBsnapshot_dir\fR .RS 4