From 3ebb816290beb0c6fce0b1aa598c2619018e0c52 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 14 Apr 2015 02:17:17 +0200 Subject: [PATCH 01/31] btrbk: added vinfo hash: keep global subvolume detail info; new three-level versioning scheme --- btrbk | 74 +++++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 15 deletions(-) diff --git a/btrbk b/btrbk index 5a31631..8a9a335 100755 --- a/btrbk +++ b/btrbk @@ -47,7 +47,7 @@ use Date::Calc qw(Today Delta_Days Day_of_Week); use Getopt::Std; use Data::Dumper; -our $VERSION = "0.17-dev"; +our $VERSION = "0.17.0-dev"; our $AUTHOR = 'Axel Burri '; our $PROJECT_HOME = ''; @@ -80,7 +80,8 @@ my %config_options = ( my @config_target_types = qw(send-receive); -my %vol_info; +my %vol_detail; +my %vol_info; # !!! TODO: rename my %uuid_info; my %uuid_fs_map; my %vol_btrfs_progs_compat; # hacky, maps all subvolumes without received_uuid information @@ -177,6 +178,30 @@ sub subvol($$) } +sub vinfo($;$) +{ + my $url = shift // die; + my $config = shift; + if($vol_detail{$url}) { + DEBUG "vinfo cache hit: $url"; + return $vol_detail{$url}; + } + + my $detail = btr_subvolume_detail($url, $config); + + unless($detail) { + $vol_detail{$url} = { ABORTED => "Failed to fetch subvolume detail for: $url" }; + return undef; + } + + $vol_detail{$url} = $detail; + DEBUG "vinfo updated for: $url"; + TRACE(Data::Dumper->Dump([$detail], ["vinfo{$url}"])); + + return $detail; +} + + sub get_rsh($$) { my $url = shift // die; @@ -213,10 +238,10 @@ sub config_key($$) } -sub check_file($$$$) +sub check_file($$;$$) { - my $file = shift; - my $accept = shift; + my $file = shift // die; + my $accept = shift || die; my $key = shift; # only for error text my $config_file = shift; # only for error text @@ -479,11 +504,27 @@ sub btr_subvolume_detail($;$) my $ret = run_cmd("$rsh /sbin/btrfs subvolume show $real_vol 2>/dev/null", 1); if($ret) { - if($ret eq "$real_vol is btrfs root") { - DEBUG "found btrfs root: $vol"; - return { id => 5, is_root => 1 }; + my $fs_path; + if($ret =~ /^($file_match)/) { + $fs_path = $1; + DEBUG "Real path for subvolume \"$vol\" is: $fs_path" if($fs_path ne $real_vol); + return undef unless(check_file($fs_path, { absolute => 1 })); } - elsif($ret =~ /^$real_vol/) { + else { + $fs_path = $real_vol; + WARN "No real path provided by \"btrfs subvolume show\" for subvolume \"$vol\", using: $real_vol"; + } + my %detail = ( FS_PATH => $fs_path, + RSH => $rsh, # !!! TODO: use this everywhere + # FS_PATH_ORIGINAL => $real_vol, + ); + + if($ret eq "$fs_path is btrfs root") { + DEBUG "found btrfs root: $vol"; + $detail{id} = 5; + $detail{is_root} = 1; + } + elsif($ret =~ /^$fs_path/) { TRACE "btr_detail: found btrfs subvolume: $vol"; my %trans = ( name => "Name", @@ -497,7 +538,6 @@ sub btr_subvolume_detail($;$) top_level => "Top Level", flags => "Flags", ); - my %detail; foreach (keys %trans) { if($ret =~ /^\s+$trans{$_}:\s+(.*)$/m) { $detail{$_} = $1; @@ -507,8 +547,8 @@ sub btr_subvolume_detail($;$) } DEBUG "parsed " . scalar(keys %detail) . " subvolume detail items: $vol"; TRACE "btr_detail for $vol: " . Dumper \%detail; - return \%detail; } + return \%detail; } WARN "Failed to fetch subvolume detail for: $vol"; return undef; @@ -722,7 +762,7 @@ sub btr_fs_info($;$) { my $fs_path = shift || die; my $config = shift; - my $detail = btr_subvolume_detail($fs_path, $config); + my $detail = vinfo($fs_path, $config); return undef unless($detail); my $tree = btr_tree($fs_path, $config); @@ -1206,13 +1246,13 @@ MAIN: my $target_vol = $subvol_args[1] || die; # FIXME: allow ssh:// src/dest (does not work since the configuration is not yet read). - my $src_detail = btr_subvolume_detail($src_vol); + my $src_detail = vinfo($src_vol); unless($src_detail) { exit 1; } if($src_detail->{is_root}) { ERROR "subvolume at \"$src_vol\" is btrfs root!"; exit 1; } unless($src_detail->{cgen}) { ERROR "subvolume at \"$src_vol\" does not provide cgen"; exit 1; } # if($src_detail->{parent_uuid} eq "-") { ERROR "subvolume at \"$src_vol\" has no parent, aborting."; exit 1; } - my $target_detail = btr_subvolume_detail($target_vol); + my $target_detail = vinfo($target_vol); unless($target_detail) { exit 1; } unless($src_detail->{cgen}) { ERROR "subvolume at \"$src_vol\" does not provide cgen"; exit 1; } # if($src_detail->{parent_uuid} eq "-") { ERROR "subvolume at \"$src_vol\" has no parent, aborting."; exit 1; } @@ -1369,6 +1409,8 @@ MAIN: foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { my $svol = $config_subvol->{svol} // die; + vinfo($sroot, $config_vol); + vinfo("$sroot/$svol", $config_vol); # filter subvolumes matching command line arguments if($action_run && scalar(@subvol_args)) { @@ -1393,6 +1435,7 @@ MAIN: foreach my $config_target (@{$config_subvol->{TARGET}}) { my $droot = $config_target->{droot} || die; + vinfo($droot, $config_target); $vol_info{$droot} //= btr_fs_info($droot, $config_target); unless($vol_info{$droot}) { $config_target->{ABORTED} = "Failed to read btrfs subvolume list for \"$droot\""; @@ -1417,7 +1460,7 @@ MAIN: my $subvol = $subvol_args[0] || die; my $dump_uuid = 0; - my $detail = btr_subvolume_detail($subvol); + my $detail = vinfo($subvol); exit 1 unless($detail); if($detail->{is_root}) { @@ -1429,6 +1472,7 @@ MAIN: unless($node) { DEBUG "Subvolume not parsed yet, fetching info: $subvol"; + vinfo($subvol); $vol_info{$subvol} //= btr_fs_info($subvol); $node = $uuid_info{$uuid} || die; } From e7e28c2418254dd89d942cf79c9ff0cc30f73dbc Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 14 Apr 2015 02:40:25 +0200 Subject: [PATCH 02/31] btrbk: renamed variables: use $url and $path instead of $vol --- btrbk | 69 +++++++++++++++++++++++++++++------------------------------ 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/btrbk b/btrbk index 8a9a335..e9c73af 100755 --- a/btrbk +++ b/btrbk @@ -207,7 +207,7 @@ sub get_rsh($$) my $url = shift // die; my $config = shift; if($config && ($url =~ /^ssh:\/\/(\S+?)(\/\S+)$/)) { - my ($ssh_host, $file) = ($1, $2); + my ($ssh_host, $path) = ($1, $2); my $ssh_user = config_key($config, "ssh_user"); my $ssh_identity = config_key($config, "ssh_identity"); my $ssh_options = ""; @@ -218,7 +218,7 @@ sub get_rsh($$) WARN "No SSH identity provided (option ssh_identity is not set) for: $url"; } my $rsh = "/usr/bin/ssh $ssh_options " . $ssh_user . '@' . $ssh_host; - return ($rsh, $file); + return ($rsh, $path); } return ("", $url); } @@ -468,64 +468,63 @@ sub btr_filesystem_show_all_local() sub btr_filesystem_show($;$) { - my $vol = shift || die; + my $url = shift || die; my $config = shift; - my ($rsh, $real_vol) = get_rsh($vol, $config); - my $ret = run_cmd("$rsh /sbin/btrfs filesystem show $real_vol", 1); + my ($rsh, $path) = get_rsh($url, $config); + my $ret = run_cmd("$rsh /sbin/btrfs filesystem show $path", 1); return $ret; } sub btr_filesystem_df($;$) { - my $vol = shift || die; + my $url = shift || die; my $config = shift; - my ($rsh, $real_vol) = get_rsh($vol, $config); - my $ret = run_cmd("$rsh /sbin/btrfs filesystem df $real_vol", 1); + my ($rsh, $path) = get_rsh($url, $config); + my $ret = run_cmd("$rsh /sbin/btrfs filesystem df $path", 1); return $ret; } sub btr_filesystem_usage($;$) { - my $vol = shift || die; + my $url = shift || die; my $config = shift; - my ($rsh, $real_vol) = get_rsh($vol, $config); - my $ret = run_cmd("$rsh /sbin/btrfs filesystem usage $real_vol", 1); + my ($rsh, $path) = get_rsh($url, $config); + my $ret = run_cmd("$rsh /sbin/btrfs filesystem usage $path", 1); return $ret; } sub btr_subvolume_detail($;$) { - my $vol = shift || die; + my $url = shift || die; my $config = shift; - my ($rsh, $real_vol) = get_rsh($vol, $config); - my $ret = run_cmd("$rsh /sbin/btrfs subvolume show $real_vol 2>/dev/null", 1); + my ($rsh, $path) = get_rsh($url, $config); + my $ret = run_cmd("$rsh /sbin/btrfs subvolume show $path 2>/dev/null", 1); if($ret) { my $fs_path; if($ret =~ /^($file_match)/) { $fs_path = $1; - DEBUG "Real path for subvolume \"$vol\" is: $fs_path" if($fs_path ne $real_vol); + DEBUG "Real path for subvolume \"$url\" is: $fs_path" if($fs_path ne $path); return undef unless(check_file($fs_path, { absolute => 1 })); } else { - $fs_path = $real_vol; - WARN "No real path provided by \"btrfs subvolume show\" for subvolume \"$vol\", using: $real_vol"; + $fs_path = $path; + WARN "No real path provided by \"btrfs subvolume show\" for subvolume \"$url\", using: $path"; } - my %detail = ( FS_PATH => $fs_path, - RSH => $rsh, # !!! TODO: use this everywhere - # FS_PATH_ORIGINAL => $real_vol, + my %detail = ( FS_PATH => $fs_path, + URL => $url, ); if($ret eq "$fs_path is btrfs root") { - DEBUG "found btrfs root: $vol"; + DEBUG "found btrfs root: $url"; $detail{id} = 5; $detail{is_root} = 1; } elsif($ret =~ /^$fs_path/) { - TRACE "btr_detail: found btrfs subvolume: $vol"; + TRACE "btr_detail: found btrfs subvolume: $url"; my %trans = ( name => "Name", uuid => "uuid", @@ -545,19 +544,19 @@ sub btr_subvolume_detail($;$) WARN "Failed to parse subvolume detail \"$trans{$_}\": $ret"; } } - DEBUG "parsed " . scalar(keys %detail) . " subvolume detail items: $vol"; - TRACE "btr_detail for $vol: " . Dumper \%detail; + DEBUG "parsed " . scalar(keys %detail) . " subvolume detail items: $url"; + TRACE "btr_detail for $url: " . Dumper \%detail; } return \%detail; } - WARN "Failed to fetch subvolume detail for: $vol"; + WARN "Failed to fetch subvolume detail for: $url"; return undef; } sub btr_subvolume_list($;$@) { - my $vol = shift || die; + my $url = shift || die; my $config = shift; my %opts = @_; my $btrfs_progs_compat = config_key($config, "btrfs_progs_compat"); @@ -565,10 +564,10 @@ sub btr_subvolume_list($;$@) $filter_option = "-o" if($opts{subvol_only}); my $display_options = "-c -u -q"; $display_options .= " -R" unless($btrfs_progs_compat); - my ($rsh, $real_vol) = get_rsh($vol, $config); + my ($rsh, $real_vol) = get_rsh($url, $config); my $ret = run_cmd("$rsh /sbin/btrfs subvolume list $filter_option $display_options $real_vol", 1); unless(defined($ret)) { - WARN "Failed to fetch btrfs subvolume list for: $vol"; + WARN "Failed to fetch btrfs subvolume list for: $url"; return undef; } my @nodes; @@ -618,20 +617,20 @@ sub btr_subvolume_list($;$@) push @nodes, \%node; # $node{parent_uuid} = undef if($node{parent_uuid} eq '-'); } - DEBUG "parsed " . scalar(@nodes) . " total subvolumes for filesystem at: $vol"; + DEBUG "parsed " . scalar(@nodes) . " total subvolumes for filesystem at: $url"; return \@nodes; } sub btr_subvolume_find_new($$;$) { - my $vol = shift || die; + my $url = shift || die; my $lastgen = shift // die; my $config = shift; - my ($rsh, $real_vol) = get_rsh($vol, $config); + my ($rsh, $real_vol) = get_rsh($url, $config); my $ret = run_cmd("$rsh /sbin/btrfs subvolume find-new $real_vol $lastgen"); unless(defined($ret)) { - ERROR "Failed to fetch modified files for: $vol"; + ERROR "Failed to fetch modified files for: $url"; return undef; } @@ -682,14 +681,14 @@ sub btr_subvolume_find_new($$;$) sub btr_tree($;$) { - my $vol = shift || die; + my $url = shift || die; my $config = shift; my %tree; my %id; - my $subvol_list = btr_subvolume_list($vol, $config, subvol_only => 0); + my $subvol_list = btr_subvolume_list($url, $config, subvol_only => 0); return undef unless(ref($subvol_list) eq "ARRAY"); - TRACE "btr_tree: processing subvolume list of: $vol"; + TRACE "btr_tree: processing subvolume list of: $url"; foreach my $node (@$subvol_list) { From 55358b5b5bf5e98f3ddca223c49113c7aae47ca1 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 14 Apr 2015 03:24:32 +0200 Subject: [PATCH 03/31] btrbk: renamed FS_PATH with URL where applicable --- btrbk | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/btrbk b/btrbk index e9c73af..041ecb5 100755 --- a/btrbk +++ b/btrbk @@ -751,35 +751,35 @@ sub _subtree_list # returns hash of: -# SUBVOL_PATH relative path to $fs_path -# FS_PATH absolute path +# SUBVOL_PATH relative path to URL +# URL absolute path # node href to tree node # -# returns an empty hash if the subvolume at $fs_path exists, but contains no subvolumes -# returns undef if the subvolume at $fs_path does not exists +# returns an empty hash if the subvolume at $url exists, but contains no subvolumes +# returns undef if the subvolume at $url does not exists sub btr_fs_info($;$) { - my $fs_path = shift || die; + my $url = shift || die; my $config = shift; - my $detail = vinfo($fs_path, $config); + my $detail = vinfo($url, $config); return undef unless($detail); - my $tree = btr_tree($fs_path, $config); + my $tree = btr_tree($url, $config); my $tree_root; if($detail->{is_root}) { $tree_root = $tree; } else { die unless $uuid_info{$detail->{uuid}}; - $uuid_fs_map{$detail->{uuid}}->{$fs_path} = 1; + $uuid_fs_map{$detail->{uuid}}->{$url} = 1; $tree_root = $uuid_info{$detail->{uuid}}->{SUBTREE}; unless($tree_root) { - DEBUG "No subvolumes found in: $fs_path"; + DEBUG "No subvolumes found in: $url"; return {}; } } - # recurse into $tree_root, returns list of href: { FS_PATH, node } + # recurse into $tree_root, returns list of href: { URL, node } my $list = _subtree_list($tree_root, [], ""); # return a hash of relative subvolume path @@ -787,11 +787,11 @@ sub btr_fs_info($;$) foreach(@$list) { my $subvol_path = $_->{SUBVOL_PATH}; die if exists $ret{$subvol_path}; - $_->{FS_PATH} = $fs_path . '/' . $subvol_path; - $uuid_fs_map{$_->{node}->{uuid}}->{$fs_path . '/' . $subvol_path} = 1; + $_->{URL} = $url . '/' . $subvol_path; + $uuid_fs_map{$_->{node}->{uuid}}->{$url . '/' . $subvol_path} = 1; $ret{$subvol_path} = $_; } - $vol_btrfs_progs_compat{$fs_path} = config_key($config, "btrfs_progs_compat"); # missing received_uuid in node{} + $vol_btrfs_progs_compat{$url} = config_key($config, "btrfs_progs_compat"); # missing received_uuid in node{} return \%ret; } @@ -945,7 +945,7 @@ sub get_snapshot_children($$) my @ret; foreach (values %{$vol_info{$sroot}}) { next unless($_->{node}->{parent_uuid} eq $svol_node->{uuid}); - TRACE "get_snapshot_children: found: $_->{FS_PATH}"; + TRACE "get_snapshot_children: found: $_->{URL}"; push(@ret, $_); } DEBUG "Found " . scalar(@ret) . " snapshot children of: $sroot/$svol"; @@ -986,7 +986,7 @@ sub get_receive_targets($$) push(@ret, $_); } } - DEBUG "Found " . scalar(@ret) . " receive targets in \"$droot/\" for: $src_href->{FS_PATH}"; + DEBUG "Found " . scalar(@ret) . " receive targets in \"$droot/\" for: $src_href->{URL}"; return @ret; } @@ -1014,16 +1014,16 @@ sub get_latest_common($$$;$) if($child->{RECEIVE_TARGET_PRESENT} && ($child->{RECEIVE_TARGET_PRESENT} eq $droot)) { # little hack to keep track of previously received subvolumes - DEBUG("Latest common snapshots for: $debug_src: src=$child->{FS_PATH} target="); + DEBUG("Latest common snapshots for: $debug_src: src=$child->{URL} target="); return ($child, undef); } foreach (get_receive_targets($droot, $child)) { - TRACE "get_latest_common: found receive target: $_->{FS_PATH}"; - DEBUG("Latest common snapshots for: $debug_src: src=$child->{FS_PATH} target=$_->{FS_PATH}"); + TRACE "get_latest_common: found receive target: $_->{URL}"; + DEBUG("Latest common snapshots for: $debug_src: src=$child->{URL} target=$_->{URL}"); return ($child, $_); } - TRACE "get_latest_common: no matching targets found for: $child->{FS_PATH}"; + TRACE "get_latest_common: no matching targets found for: $child->{URL}"; } DEBUG("No common snapshots for \"$debug_src\" found in src=$sroot/ target=$droot/"); return (undef, undef); @@ -1532,7 +1532,7 @@ MAIN: next unless $vol_info{$droot}; $droot_compat{$droot} = 1 if($vol_btrfs_progs_compat{$droot}); foreach (sort { $a->{SUBVOL_PATH} cmp $b->{SUBVOL_PATH} } get_receive_targets($droot, $snapshot)) { - print "| | ^== $_->{FS_PATH}\n"; + print "| | ^== $_->{URL}\n"; } } } @@ -1665,10 +1665,10 @@ MAIN: foreach my $child (get_snapshot_children($sroot, $svol)) { if(scalar get_receive_targets($droot, $child)) { - DEBUG "Found matching receive target, skipping: $child->{FS_PATH}"; + DEBUG "Found matching receive target, skipping: $child->{URL}"; } else { - DEBUG "No matching receive targets found, adding resume candidate: $child->{FS_PATH}"; + DEBUG "No matching receive targets found, adding resume candidate: $child->{URL}"; # check if the target would be preserved my ($date, $date_ext) = get_date_tag($child->{SUBVOL_PATH}); @@ -1699,13 +1699,13 @@ MAIN: my @resume = grep defined, @$preserve; # remove entries with no value from list (target subvolumes) foreach my $child (sort { $a->{node}->{gen} <=> $b->{node}->{gen} } @resume) { - INFO "Resuming subvolume backup (send-receive) for: $child->{FS_PATH}"; + INFO "Resuming subvolume backup (send-receive) for: $child->{URL}"; $found_missing++; my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot, $child->{node}->{gen}); if(macro_send_receive($config_target, - src => $child->{FS_PATH}, + src => $child->{URL}, target => $droot, - parent => $latest_common_src ? $latest_common_src->{FS_PATH} : undef, + parent => $latest_common_src ? $latest_common_src->{URL} : undef, resume => 1, # propagated to $config_target->{subvol_received} )) { @@ -1736,7 +1736,7 @@ MAIN: macro_send_receive($config_target, src => $snapshot, target => $droot, - parent => $latest_common_src ? $latest_common_src->{FS_PATH} : undef, + parent => $latest_common_src ? $latest_common_src->{URL} : undef, ); } else { From 0a9c193d13ee2efba91f5d166764957da907921e Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 14 Apr 2015 13:52:16 +0200 Subject: [PATCH 04/31] btrbk: filter subvolumes matching command line arguments before checking the configuration --- btrbk | 59 ++++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/btrbk b/btrbk index 041ecb5..d888679 100755 --- a/btrbk +++ b/btrbk @@ -1398,22 +1398,20 @@ MAIN: exit 0; } - # - # fill vol_info hash, basic checks on configuration - # - my $subvol_filter_count = undef; - foreach my $config_vol (@{$config->{VOLUME}}) - { - my $sroot = $config_vol->{sroot} || die; - foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) - { - my $svol = $config_subvol->{svol} // die; - vinfo($sroot, $config_vol); - vinfo("$sroot/$svol", $config_vol); - # filter subvolumes matching command line arguments - if($action_run && scalar(@subvol_args)) { - $subvol_filter_count //= 0; + # + # filter subvolumes matching command line arguments + # + if($action_run && scalar(@subvol_args)) + { + my $filter_count = undef; + foreach my $config_vol (@{$config->{VOLUME}}) + { + my $subvol_filter_count = 0; + my $sroot = $config_vol->{sroot} || die; + foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) + { + my $svol = $config_subvol->{svol} // die; if(grep(/^$sroot\/$svol$/, @subvol_args)) { $subvol_filter_count++; } @@ -1424,6 +1422,33 @@ MAIN: next; } } + if($subvol_filter_count == 0) { + $config_vol->{ABORTED} = "No match on subvolume command line arguments"; + $config_vol->{ABORTED_NOERR} = 1; + } + $filter_count += $subvol_filter_count; + } + if($filter_count == 0) { + ERROR "Subvolume command line arguments do not match any volume/subvolume declaration from configuration file, aborting."; + exit 1; + } + } + + + # + # fill vol_info hash, basic checks on configuration + # + foreach my $config_vol (@{$config->{VOLUME}}) + { + next if($config_vol->{ABORTED}); + my $sroot = $config_vol->{sroot} || die; + vinfo($sroot, $config_vol); + + foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) + { + next if($config_subvol->{ABORTED}); + my $svol = $config_subvol->{svol} // die; + vinfo("$sroot/$svol", $config_vol); $vol_info{$sroot} //= btr_fs_info($sroot, $config_vol); unless(subvol($sroot, $svol)) { @@ -1444,10 +1469,6 @@ MAIN: } } } - if(defined($subvol_filter_count) && ($subvol_filter_count == 0)) { - ERROR "Subvolume command line arguments do not match any volume/subvolume declaration from configuration file, aborting."; - exit 1; - } TRACE(Data::Dumper->Dump([\%vol_info], ["vol_info"])); From 72cbca13d747f0b84db3c9aeed6539abfb91bd57 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 14 Apr 2015 16:03:31 +0200 Subject: [PATCH 05/31] btrbk: add rsh information to vinfo; btr_subvolume_detail() now takes real options instead of a config hash --- btrbk | 65 ++++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/btrbk b/btrbk index d888679..fe44888 100755 --- a/btrbk +++ b/btrbk @@ -187,18 +187,42 @@ sub vinfo($;$) return $vol_detail{$url}; } - my $detail = btr_subvolume_detail($url, $config); + my %info = ( URL => $url, + PATH => $url, + ); - unless($detail) { - $vol_detail{$url} = { ABORTED => "Failed to fetch subvolume detail for: $url" }; - return undef; + if($config && ($url =~ /^ssh:\/\/(\S+?)(\/\S+)$/)) { + my %remote = ( + URL => $url, + HOST => $1, + PATH => $2, + RSH_TYPE => "ssh", + SSH_USER => config_key($config, "ssh_user"), + SSH_IDENTITY => config_key($config, "ssh_identity"), + ); + my $ssh_options = ""; + if($remote{SSH_IDENTITY}) { + $ssh_options .= "-i $remote{SSH_IDENTITY} "; + } + else { + WARN "No SSH identity provided (option ssh_identity is not set) for: $url"; + } + $remote{RSH} = "/usr/bin/ssh $ssh_options" . $remote{SSH_USER} . '@' . $remote{HOST}; + + $info{PATH} = $remote{PATH}; + $info{REMOTE} = \%remote; } - $vol_detail{$url} = $detail; - DEBUG "vinfo updated for: $url"; - TRACE(Data::Dumper->Dump([$detail], ["vinfo{$url}"])); + my $detail = btr_subvolume_detail($info{PATH}, $info{REMOTE}); + $detail ||= { ERROR => "Failed to fetch subvolume detail for: $url" }; - return $detail; + %info = ( %$detail, %info ); + + $vol_detail{$url} = \%info; + DEBUG "vinfo updated for: $url"; + TRACE(Data::Dumper->Dump([\%info], ["vinfo{$url}"])); + + return \%info; } @@ -498,32 +522,31 @@ sub btr_filesystem_usage($;$) sub btr_subvolume_detail($;$) { - my $url = shift || die; - my $config = shift; - my ($rsh, $path) = get_rsh($url, $config); + my $path = shift // die; + my $opts = shift || {}; + my $rsh = $opts->{RSH} || ""; + my $url = $opts->{URL} || $path; # used only for logging my $ret = run_cmd("$rsh /sbin/btrfs subvolume show $path 2>/dev/null", 1); if($ret) { - my $fs_path; + my $real_path; if($ret =~ /^($file_match)/) { - $fs_path = $1; - DEBUG "Real path for subvolume \"$url\" is: $fs_path" if($fs_path ne $path); - return undef unless(check_file($fs_path, { absolute => 1 })); + $real_path = $1; + DEBUG "Real path for subvolume \"$url\" is: $real_path" if($real_path ne $path); + return undef unless(check_file($real_path, { absolute => 1 })); } else { - $fs_path = $path; + $real_path = $path; WARN "No real path provided by \"btrfs subvolume show\" for subvolume \"$url\", using: $path"; } - my %detail = ( FS_PATH => $fs_path, - URL => $url, - ); + my %detail = ( REAL_PATH => $real_path ); - if($ret eq "$fs_path is btrfs root") { + if($ret eq "$real_path is btrfs root") { DEBUG "found btrfs root: $url"; $detail{id} = 5; $detail{is_root} = 1; } - elsif($ret =~ /^$fs_path/) { + elsif($ret =~ /^$real_path/) { TRACE "btr_detail: found btrfs subvolume: $url"; my %trans = ( name => "Name", From 0068e078f23d341b3c2e52f340dfd0018a0064c1 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Thu, 16 Apr 2015 12:00:04 +0200 Subject: [PATCH 06/31] btrbk: globally replaced %vol_info by vinfo->{VOL_INFO}, use vinfo() where applicable; changed btr_* function arguments; adapted snapshotting and send-receive --- btrbk | 582 ++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 320 insertions(+), 262 deletions(-) diff --git a/btrbk b/btrbk index fe44888..23a511f 100755 --- a/btrbk +++ b/btrbk @@ -80,11 +80,9 @@ my %config_options = ( my @config_target_types = qw(send-receive); -my %vol_detail; -my %vol_info; # !!! TODO: rename +my %vol_info; my %uuid_info; my %uuid_fs_map; -my %vol_btrfs_progs_compat; # hacky, maps all subvolumes without received_uuid information my $dryrun; my $loglevel = 1; @@ -171,8 +169,8 @@ sub subvol($$) { my $root = shift || die; my $vol = shift // die; - if($vol_info{$root} && $vol_info{$root}->{$vol}) { - return $vol_info{$root}->{$vol}->{node}; + if($root->{SUBVOL_INFO} && $root->{SUBVOL_INFO}->{$vol}) { + return $root->{SUBVOL_INFO}->{$vol}->{node}; } return undef; } @@ -182,50 +180,89 @@ sub vinfo($;$) { my $url = shift // die; my $config = shift; - if($vol_detail{$url}) { - DEBUG "vinfo cache hit: $url"; - return $vol_detail{$url}; + if($vol_info{$url}) { + TRACE "vinfo cache hit: $url"; + return $vol_info{$url}; } - my %info = ( URL => $url, - PATH => $url, - ); + die unless($config); - if($config && ($url =~ /^ssh:\/\/(\S+?)(\/\S+)$/)) { - my %remote = ( - URL => $url, - HOST => $1, - PATH => $2, - RSH_TYPE => "ssh", - SSH_USER => config_key($config, "ssh_user"), - SSH_IDENTITY => config_key($config, "ssh_identity"), - ); - my $ssh_options = ""; - if($remote{SSH_IDENTITY}) { - $ssh_options .= "-i $remote{SSH_IDENTITY} "; + my %info = ( URL => $url ); + + if($url =~ /^ssh:\/\/(\S+?)(\/\S+)$/) { + my ($host, $path) = ($1, $2); + my $ssh_user = config_key($config, "ssh_user"); + my $ssh_identity = config_key($config, "ssh_identity"); + my $ssh_options = ""; + if($ssh_identity) { + $ssh_options .= "-i $ssh_identity "; } else { WARN "No SSH identity provided (option ssh_identity is not set) for: $url"; } - $remote{RSH} = "/usr/bin/ssh $ssh_options" . $remote{SSH_USER} . '@' . $remote{HOST}; - - $info{PATH} = $remote{PATH}; - $info{REMOTE} = \%remote; + %info = ( + %info, + HOST => $host, + PATH => $path, + PRINT => "$host:$path", + RSH_TYPE => "ssh", + SSH_USER => $ssh_user, + SSH_IDENTITY => $ssh_identity, + RSH => "/usr/bin/ssh $ssh_options" . $ssh_user . '@' . $host, + ); + } + elsif(($url =~ /^\//) && ($url =~ /^$file_match$/)) { + %info = ( + %info, + PATH => $url, + PRINT => $url, + ); + } + else { + die "Ambiguous vinfo url: $url"; } - my $detail = btr_subvolume_detail($info{PATH}, $info{REMOTE}); - $detail ||= { ERROR => "Failed to fetch subvolume detail for: $url" }; + my $btrfs_progs_compat = config_key($config, "btrfs_progs_compat"); + $info{BTRFS_PROGS_COMPAT} = $btrfs_progs_compat if($btrfs_progs_compat); - %info = ( %$detail, %info ); - - $vol_detail{$url} = \%info; - DEBUG "vinfo updated for: $url"; + DEBUG "vinfo created for: $url"; TRACE(Data::Dumper->Dump([\%info], ["vinfo{$url}"])); + $vol_info{$url} = \%info; return \%info; } +sub vinfo_read_detail($) +{ + my $vol = shift || die; + + if($vol->{id}) { + TRACE "vinfo detail cache hit: $vol->{URL}"; + return $vol; + } + + my $detail = btr_subvolume_detail($vol); + unless($detail) { + WARN "Failed to fetch subvolume detail for: $vol->{PRINT}"; + return undef; + } + + # add detail data to vinfo hash + foreach(keys %$detail) { + if((defined $vol->{$_}) && ($vol->{$_} ne $detail->{$_})) { + WARN "Subvolume detail key \"$_\" is already present, with a different value: old=\"$vol->{$_}\", new=\"$detail->{$_}\""; + WARN "Using new value for \"$_\": $detail->{$_}"; + } + $vol->{$_} = $detail->{$_}; + } + DEBUG "vinfo updated for: $vol->{URL}"; + TRACE(Data::Dumper->Dump([$vol], ["vinfo{$vol->{URL}}"])); + + return $vol; +} + + sub get_rsh($$) { my $url = shift // die; @@ -348,7 +385,7 @@ sub parse_config(@) DEBUG "config: adding volume \"$value\" to root context"; my $volume = { CONTEXT => "volume", PARENT => $cur, - sroot => $value, + url => $value, }; $cur->{VOLUME} //= []; push(@{$cur->{VOLUME}}, $volume); @@ -373,10 +410,11 @@ sub parse_config(@) return undef; } - DEBUG "config: adding subvolume \"$value\" to volume context: $cur->{sroot}"; + DEBUG "config: adding subvolume \"$value\" to volume context: $cur->{url}"; my $subvolume = { CONTEXT => "subvolume", PARENT => $cur, - svol => $value, + rel_path => $value, + url => $cur->{url} . '/' . $value, }; $cur->{SUBVOLUME} //= []; push(@{$cur->{SUBVOLUME}}, $subvolume); @@ -404,11 +442,11 @@ sub parse_config(@) $droot =~ s/\/+$//; # remove trailing slash $droot =~ s/^\/+/\//; # sanitize leading slash - DEBUG "config: adding target \"$droot\" (type=$target_type) to subvolume context: $cur->{PARENT}->{sroot}/$cur->{svol}"; + DEBUG "config: adding target \"$droot\" (type=$target_type) to subvolume context: $cur->{url}"; my $target = { CONTEXT => "target", PARENT => $cur, target_type => $target_type, - droot => $droot, + url => $droot, }; $cur->{TARGET} //= []; push(@{$cur->{TARGET}}, $target); @@ -490,64 +528,64 @@ sub btr_filesystem_show_all_local() } -sub btr_filesystem_show($;$) +sub btr_filesystem_show($) { - my $url = shift || die; - my $config = shift; - my ($rsh, $path) = get_rsh($url, $config); + my $vol = shift || die; + my $path = $vol->{PATH} // die; + my $rsh = $vol->{RSH} || ""; my $ret = run_cmd("$rsh /sbin/btrfs filesystem show $path", 1); return $ret; } -sub btr_filesystem_df($;$) +sub btr_filesystem_df($) { - my $url = shift || die; - my $config = shift; - my ($rsh, $path) = get_rsh($url, $config); + my $vol = shift || die; + my $path = $vol->{PATH} // die; + my $rsh = $vol->{RSH} || ""; my $ret = run_cmd("$rsh /sbin/btrfs filesystem df $path", 1); return $ret; } -sub btr_filesystem_usage($;$) +sub btr_filesystem_usage($) { - my $url = shift || die; - my $config = shift; - my ($rsh, $path) = get_rsh($url, $config); + my $vol = shift || die; + my $path = $vol->{PATH} // die; + my $rsh = $vol->{RSH} || ""; my $ret = run_cmd("$rsh /sbin/btrfs filesystem usage $path", 1); return $ret; } -sub btr_subvolume_detail($;$) +sub btr_subvolume_detail($) { - my $path = shift // die; - my $opts = shift || {}; - my $rsh = $opts->{RSH} || ""; - my $url = $opts->{URL} || $path; # used only for logging + my $vol = shift || die; + my $path = $vol->{PATH} // die; + my $rsh = $vol->{RSH} || ""; + my $vol_print = $vol->{PRINT} || $path; # used only for logging my $ret = run_cmd("$rsh /sbin/btrfs subvolume show $path 2>/dev/null", 1); if($ret) { my $real_path; if($ret =~ /^($file_match)/) { $real_path = $1; - DEBUG "Real path for subvolume \"$url\" is: $real_path" if($real_path ne $path); + DEBUG "Real path for subvolume \"$vol_print\" is: $real_path" if($real_path ne $path); return undef unless(check_file($real_path, { absolute => 1 })); } else { $real_path = $path; - WARN "No real path provided by \"btrfs subvolume show\" for subvolume \"$url\", using: $path"; + WARN "No real path provided by \"btrfs subvolume show\" for subvolume \"$vol_print\", using: $path"; } my %detail = ( REAL_PATH => $real_path ); if($ret eq "$real_path is btrfs root") { - DEBUG "found btrfs root: $url"; + DEBUG "found btrfs root: $vol_print"; $detail{id} = 5; $detail{is_root} = 1; } elsif($ret =~ /^$real_path/) { - TRACE "btr_detail: found btrfs subvolume: $url"; + TRACE "btr_detail: found btrfs subvolume: $vol_print"; my %trans = ( name => "Name", uuid => "uuid", @@ -567,30 +605,30 @@ sub btr_subvolume_detail($;$) WARN "Failed to parse subvolume detail \"$trans{$_}\": $ret"; } } - DEBUG "parsed " . scalar(keys %detail) . " subvolume detail items: $url"; - TRACE "btr_detail for $url: " . Dumper \%detail; + DEBUG "parsed " . scalar(keys %detail) . " subvolume detail items: $vol_print"; + TRACE "btr_detail for $vol_print: " . Dumper \%detail; } return \%detail; } - WARN "Failed to fetch subvolume detail for: $url"; return undef; } -sub btr_subvolume_list($;$@) +sub btr_subvolume_list($;@) { - my $url = shift || die; - my $config = shift; + my $vol = shift || die; my %opts = @_; - my $btrfs_progs_compat = config_key($config, "btrfs_progs_compat"); + my $path = $vol->{PATH} // die; + my $rsh = $vol->{RSH} || ""; + my $vol_print = $vol->{PRINT} || $path; # used only for logging + my $btrfs_progs_compat = $vol->{BTRFS_PROGS_COMPAT} || $opts{btrfs_progs_compat}; my $filter_option = "-a"; $filter_option = "-o" if($opts{subvol_only}); my $display_options = "-c -u -q"; $display_options .= " -R" unless($btrfs_progs_compat); - my ($rsh, $real_vol) = get_rsh($url, $config); - my $ret = run_cmd("$rsh /sbin/btrfs subvolume list $filter_option $display_options $real_vol", 1); + my $ret = run_cmd("$rsh /sbin/btrfs subvolume list $filter_option $display_options $path", 1); unless(defined($ret)) { - WARN "Failed to fetch btrfs subvolume list for: $url"; + WARN "Failed to fetch btrfs subvolume list for: $vol_print"; return undef; } my @nodes; @@ -640,7 +678,7 @@ sub btr_subvolume_list($;$@) push @nodes, \%node; # $node{parent_uuid} = undef if($node{parent_uuid} eq '-'); } - DEBUG "parsed " . scalar(@nodes) . " total subvolumes for filesystem at: $url"; + DEBUG "parsed " . scalar(@nodes) . " total subvolumes for filesystem at: $vol_print"; return \@nodes; } @@ -702,16 +740,15 @@ sub btr_subvolume_find_new($$;$) } -sub btr_tree($;$) +sub btr_tree($) { - my $url = shift || die; - my $config = shift; + my $vol = shift; my %tree; my %id; - my $subvol_list = btr_subvolume_list($url, $config, subvol_only => 0); + my $subvol_list = btr_subvolume_list($vol, subvol_only => 0); return undef unless(ref($subvol_list) eq "ARRAY"); - TRACE "btr_tree: processing subvolume list of: $url"; + TRACE "btr_tree: processing subvolume list of: $vol->{URL}"; foreach my $node (@$subvol_list) { @@ -780,25 +817,26 @@ sub _subtree_list # # returns an empty hash if the subvolume at $url exists, but contains no subvolumes # returns undef if the subvolume at $url does not exists -sub btr_fs_info($;$) +sub vinfo_read_subvolumes($) { - my $url = shift || die; - my $config = shift; - my $detail = vinfo($url, $config); - return undef unless($detail); + my $vol = shift || die; + my $url = $vol->{URL} || die; + + my $tree = btr_tree($vol); + return undef unless($tree); - my $tree = btr_tree($url, $config); my $tree_root; - if($detail->{is_root}) { + if($vol->{is_root}) { $tree_root = $tree; } else { - die unless $uuid_info{$detail->{uuid}}; - $uuid_fs_map{$detail->{uuid}}->{$url} = 1; - $tree_root = $uuid_info{$detail->{uuid}}->{SUBTREE}; + die unless $uuid_info{$vol->{uuid}}; + $uuid_fs_map{$vol->{uuid}}->{$url} = 1; + $tree_root = $uuid_info{$vol->{uuid}}->{SUBTREE}; unless($tree_root) { DEBUG "No subvolumes found in: $url"; - return {}; + $vol->{SUBVOL_INFO} = {}; + return $vol; } } @@ -811,30 +849,36 @@ sub btr_fs_info($;$) my $subvol_path = $_->{SUBVOL_PATH}; die if exists $ret{$subvol_path}; $_->{URL} = $url . '/' . $subvol_path; + $_->{PATH} = $vol->{PATH} . '/' . $subvol_path; + $_->{PRINT} = $vol->{PRINT} . '/' . $subvol_path; + $_->{RSH} = $vol->{RSH}; + # !!! TODO: make real vinfo out of this $uuid_fs_map{$_->{node}->{uuid}}->{$url . '/' . $subvol_path} = 1; $ret{$subvol_path} = $_; } - $vol_btrfs_progs_compat{$url} = config_key($config, "btrfs_progs_compat"); # missing received_uuid in node{} + TRACE(Data::Dumper->Dump([\%ret], ["vol_info{$url}"])); + + $vol->{SUBVOL_INFO} = \%ret; return \%ret; } # returns $target, or undef on error -sub btrfs_snapshot($$;$) +sub btrfs_snapshot($$) { - my $src = shift || die; - my $target = shift || die; - my $config = shift; - my ($rsh, $real_src) = get_rsh($src, $config); - my (undef, $real_target) = get_rsh($target, $config); + my $svol = shift || die; + my $target_path = shift // die; + my $src_path = $svol->{PATH} // die; + my $rsh = $svol->{RSH} || ""; DEBUG "[btrfs] snapshot (ro):"; - DEBUG "[btrfs] source: $src"; - DEBUG "[btrfs] target: $target"; - INFO ">>> $target"; - my $ret = run_cmd("$rsh /sbin/btrfs subvolume snapshot -r $real_src $real_target"); - ERROR "Failed to create btrfs subvolume snapshot: $src -> $target" unless(defined($ret)); - return defined($ret) ? $target : undef; + DEBUG "[btrfs] host : $svol->{HOST}" if($svol->{HOST}); + DEBUG "[btrfs] source: $src_path"; + DEBUG "[btrfs] target: $target_path"; + INFO ">>> " . ($svol->{HOST} ? "$svol->{HOST}:" : "") . $target_path; + my $ret = run_cmd("$rsh /sbin/btrfs subvolume snapshot -r $src_path $target_path"); + ERROR "Failed to create btrfs subvolume snapshot: $svol->{PRINT} -> $target_path" unless(defined($ret)); + return defined($ret) ? $target_path : undef; } @@ -864,34 +908,35 @@ sub btrfs_subvolume_delete($@) } -sub btrfs_send_receive($$$;$) +sub btrfs_send_receive($$$) { - my $src = shift || die; + my $snapshot = shift || die; my $target = shift || die; - my $parent = shift // ""; - my $config = shift; - my ($rsh_src, $real_src) = get_rsh($src, $config); - my ($rsh_target, $real_target) = get_rsh($target, $config); - my (undef, $real_parent) = get_rsh($parent, $config); + my $parent = shift; + my $snapshot_path = $snapshot->{PATH} // die; + my $snapshot_rsh = $snapshot->{RSH} || ""; + my $target_path = $target->{PATH} // die; + my $target_rsh = $target->{RSH} || ""; + my $parent_path = $parent ? $parent->{PATH} : undef; my $now = localtime; - my $src_name = $src; - $src_name =~ s/^.*\///; - INFO ">>> $target/$src_name"; + my $snapshot_name = $snapshot_path; + $snapshot_name =~ s/^.*\///; + INFO ">>> $target->{PRINT}/$snapshot_name"; DEBUG "[btrfs] send/receive" . ($parent ? " (incremental)" : " (complete)") . ":"; - DEBUG "[btrfs] source: $src"; - DEBUG "[btrfs] parent: $parent" if($parent); - DEBUG "[btrfs] target: $target"; + DEBUG "[btrfs] source: $snapshot->{PRINT}"; + DEBUG "[btrfs] parent: $parent->{PRINT}" if($parent); + DEBUG "[btrfs] target: $target->{PRINT}"; - my $parent_option = $real_parent ? "-p $real_parent" : ""; + my $parent_option = $parent_path ? "-p $parent_path" : ""; my $receive_option = ""; $receive_option = "-v" if($loglevel >= 3); - my $cmd = "$rsh_src /sbin/btrfs send $parent_option $real_src | $rsh_target /sbin/btrfs receive $receive_option $real_target/"; + my $cmd = "$snapshot_rsh /sbin/btrfs send $parent_option $snapshot_path | $target_rsh /sbin/btrfs receive $receive_option $target_path/"; my $ret = run_cmd($cmd); unless(defined($ret)) { - ERROR "Failed to send/receive btrfs subvolume: $src " . ($real_parent ? "[$real_parent]" : "") . " -> $target"; + ERROR "Failed to send/receive btrfs subvolume: $snapshot->{PRINT} " . ($parent_path ? "[$parent_path]" : "") . " -> $target->{PRINT}"; return undef; } return 1; @@ -904,28 +949,29 @@ sub macro_send_receive($@) { my $config = shift || die; my %info = @_; + my $snapshot = $info{snapshot} || die; + my $target = $info{target} || die; + my $parent = $info{parent}; my $incremental = config_key($config, "incremental"); - INFO "Receiving from snapshot: $info{src}"; + INFO "Receiving from snapshot: $snapshot->{PRINT}"; # add info to $config->{subvol_received} - my $src_name = $info{src}; - $src_name =~ s/^.*\///; - $info{received_name} = "$info{target}/$src_name"; + $info{received_name} = $snapshot->{PRINT}; $config->{subvol_received} //= []; push(@{$config->{subvol_received}}, \%info); if($incremental) { # create backup from latest common - if($info{parent}) { - INFO "Incremental from parent snapshot: $info{parent}"; + if($parent) { + INFO "Incremental from parent snapshot: $parent"; } elsif($incremental ne "strict") { INFO "No common parent subvolume present, creating full backup"; } else { - WARN "Backup to $info{target} failed: no common parent subvolume found, and option \"incremental\" is set to \"strict\""; + WARN "Backup to $target->{PRINT} failed: no common parent subvolume found, and option \"incremental\" is set to \"strict\""; $info{ERROR} = 1; $config->{ABORTED} = "No common parent subvolume found, and option \"incremental\" is set to \"strict\""; return undef; @@ -936,7 +982,7 @@ sub macro_send_receive($@) delete $info{parent}; } - if(btrfs_send_receive($info{src}, $info{target}, $info{parent}, $config)) { + if(btrfs_send_receive($snapshot, $target, $parent)) { return 1; } else { $info{ERROR} = 1; @@ -961,17 +1007,15 @@ sub get_date_tag($) sub get_snapshot_children($$) { - my $sroot = shift || die; + my $sroot = shift || die; # TODO: this should be second argument, as we return all snap children from svol under sroot my $svol = shift // die; - my $svol_node = subvol($sroot, $svol); - die("subvolume info not present: $sroot/$svol") unless($svol_node); my @ret; - foreach (values %{$vol_info{$sroot}}) { - next unless($_->{node}->{parent_uuid} eq $svol_node->{uuid}); + foreach (values %{$sroot->{SUBVOL_INFO}}) { + next unless($_->{node}->{parent_uuid} eq $svol->{uuid}); TRACE "get_snapshot_children: found: $_->{URL}"; push(@ret, $_); } - DEBUG "Found " . scalar(@ret) . " snapshot children of: $sroot/$svol"; + DEBUG "Found " . scalar(@ret) . " snapshot children of: $svol->{URL}"; return @ret; } @@ -980,16 +1024,16 @@ sub get_receive_targets($$) { my $droot = shift || die; my $src_href = shift || die; - die("root subvolume info not present: $droot") unless($vol_info{$droot}); + die("root subvolume info not present: $droot") unless($droot->{SUBVOL_INFO}); my @ret; - if($vol_btrfs_progs_compat{$droot}) + if($droot->{BTRFS_PROGS_COMPAT}) { # guess matches by subvolume name (node->received_uuid is not available if BTRFS_PROGS_COMPAT is set) DEBUG "Fallback to compatibility mode (get_receive_targets)"; my $src_name = $src_href->{node}->{REL_PATH}; $src_name =~ s/^.*\///; # strip path - foreach my $target (values %{$vol_info{$droot}}) { + foreach my $target (values %{$droot->{SUBVOL_INFO}}) { my $target_name = $target->{node}->{REL_PATH}; $target_name =~ s/^.*\///; # strip path if($target_name eq $src_name) { @@ -1003,13 +1047,13 @@ sub get_receive_targets($$) # find matches by comparing uuid / received_uuid my $uuid = $src_href->{node}->{uuid}; die("subvolume info not present: $uuid") unless($uuid_info{$uuid}); - foreach (values %{$vol_info{$droot}}) { + foreach (values %{$droot->{SUBVOL_INFO}}) { next unless($_->{node}->{received_uuid} eq $uuid); TRACE "get_receive_targets: by-uuid: Found receive target: $_->{SUBVOL_PATH}"; push(@ret, $_); } } - DEBUG "Found " . scalar(@ret) . " receive targets in \"$droot/\" for: $src_href->{URL}"; + DEBUG "Found " . scalar(@ret) . " receive targets in \"$droot->{URL}/\" for: $src_href->{URL}"; return @ret; } @@ -1021,11 +1065,11 @@ sub get_latest_common($$$;$) my $droot = shift || die; my $threshold_gen = shift; # skip all snapshot children with generation >= $threshold_gen - die("source subvolume info not present: $sroot") unless($vol_info{$sroot}); - die("target subvolume info not present: $droot") unless($vol_info{$droot}); + die("source subvolume info not present: $sroot->{URL}") unless($sroot->{URL}); + die("target subvolume info not present: $droot->{URL}") unless($droot->{URL}); - my $debug_src = "$sroot/$svol"; - $debug_src .= "@" . $threshold_gen if($threshold_gen); + my $debug_src = $svol->{URL}; + $debug_src .= "#" . $threshold_gen if($threshold_gen); # sort children of svol descending by generation foreach my $child (sort { $b->{node}->{gen} <=> $a->{node}->{gen} } get_snapshot_children($sroot, $svol)) { @@ -1035,7 +1079,7 @@ sub get_latest_common($$$;$) next; } - if($child->{RECEIVE_TARGET_PRESENT} && ($child->{RECEIVE_TARGET_PRESENT} eq $droot)) { + if($child->{RECEIVE_TARGET_PRESENT} && ($child->{RECEIVE_TARGET_PRESENT} eq $droot->{URL})) { # little hack to keep track of previously received subvolumes DEBUG("Latest common snapshots for: $debug_src: src=$child->{URL} target="); return ($child, undef); @@ -1048,7 +1092,7 @@ sub get_latest_common($$$;$) } TRACE "get_latest_common: no matching targets found for: $child->{URL}"; } - DEBUG("No common snapshots for \"$debug_src\" found in src=$sroot/ target=$droot/"); + DEBUG("No common snapshots for \"$debug_src\" found in src=$sroot->{URL}/ target=$droot->{URL}/"); return (undef, undef); } @@ -1385,35 +1429,35 @@ MAIN: my %processed; foreach my $config_vol (@{$config->{VOLUME}}) { - my $sroot = $config_vol->{sroot} || die; - unless($processed{$sroot}) + my $url = $config_vol->{url} || die; + unless($processed{$url}) { print "\n--------------------------------------------------------------------------------\n"; - print "Source volume: $sroot\n"; + print "Source volume: $url\n"; print "--------------------------------------------------------------------------------\n"; - # print (btr_filesystem_show($sroot, $config_vol) // ""); + # print (btr_filesystem_show(vinfo($url, $config_vol)) // ""); # print "\n\n"; - print (btr_filesystem_usage($sroot, $config_vol) // ""); + print (btr_filesystem_usage(vinfo($url, $config_vol)) // ""); print "\n"; - $processed{$sroot} = 1; + $processed{$url} = 1; } } foreach my $config_vol (@{$config->{VOLUME}}) { - my $sroot = $config_vol->{sroot} || die; + my $sroot_url = $config_vol->{url} || die; foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { foreach my $config_target (@{$config_subvol->{TARGET}}) { - my $droot = $config_target->{droot} || die; - unless($processed{$droot}) + my $droot_url = $config_target->{url} || die; + unless($processed{$droot_url}) { print "\n--------------------------------------------------------------------------------\n"; - print "Target volume: $droot\n"; - print " ^--- $sroot\n"; + print "Target volume: $droot_url\n"; + print " ^--- $sroot_url\n"; print "--------------------------------------------------------------------------------\n"; - print (btr_filesystem_usage($droot, $config_target) // ""); + print (btr_filesystem_usage(vinfo($droot_url, $config_target)) // ""); print "\n"; - $processed{$droot} = 1; + $processed{$droot_url} = 1; } } } @@ -1431,15 +1475,14 @@ MAIN: foreach my $config_vol (@{$config->{VOLUME}}) { my $subvol_filter_count = 0; - my $sroot = $config_vol->{sroot} || die; foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { - my $svol = $config_subvol->{svol} // die; - if(grep(/^$sroot\/$svol$/, @subvol_args)) { + my $svol_url = $config_subvol->{url} // die; + if(grep(/^\Q$svol_url\E$/, @subvol_args)) { $subvol_filter_count++; } else { - DEBUG "No match on subvolume command line argument, skipping: $sroot/$svol"; + DEBUG "No match on subvolume command line argument, skipping: $svol_url"; $config_subvol->{ABORTED} = "No match on subvolume command line arguments"; $config_subvol->{ABORTED_NOERR} = 1; next; @@ -1464,35 +1507,50 @@ MAIN: foreach my $config_vol (@{$config->{VOLUME}}) { next if($config_vol->{ABORTED}); - my $sroot = $config_vol->{sroot} || die; - vinfo($sroot, $config_vol); + my $sroot = vinfo($config_vol->{url}, $config_vol); + unless(vinfo_read_detail($sroot)) { + $config_vol->{ABORTED} = "Failed to fetch subvolume detail"; + WARN "Skipping volume \"$sroot->{URL}\": $config_vol->{ABORTED}"; + next; + } foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { next if($config_subvol->{ABORTED}); - my $svol = $config_subvol->{svol} // die; - vinfo("$sroot/$svol", $config_vol); + my $svol = vinfo($config_subvol->{url}, $config_vol); + unless(vinfo_read_detail($svol)) { + $config_subvol->{ABORTED} = "Failed to fetch subvolume detail"; + WARN "Skipping subvolume \"$svol->{URL}\": $config_subvol->{ABORTED}"; + next; + } + unless(vinfo_read_subvolumes($sroot)) { + $config_subvol->{ABORTED} = "Failed to fetch subvolume list"; + WARN "Skipping subvolume \"$svol->{URL}\": $config_subvol->{ABORTED}"; + next; + } - $vol_info{$sroot} //= btr_fs_info($sroot, $config_vol); - unless(subvol($sroot, $svol)) { - $config_subvol->{ABORTED} = "Subvolume \"$svol\" not present in btrfs subvolume list for \"$sroot\""; - WARN "Skipping subvolume section: $config_subvol->{ABORTED}"; + unless(subvol($sroot, $config_subvol->{rel_path})) { # !!! TODO: maybe check uuid here? + $config_subvol->{ABORTED} = "Subvolume \"$svol->{URL}\" not present in btrfs subvolume list for \"$sroot->{URL}\""; + WARN "Skipping subvolume \"$svol->{URL}\": $config_subvol->{ABORTED}"; next; } foreach my $config_target (@{$config_subvol->{TARGET}}) { - my $droot = $config_target->{droot} || die; - vinfo($droot, $config_target); - $vol_info{$droot} //= btr_fs_info($droot, $config_target); - unless($vol_info{$droot}) { - $config_target->{ABORTED} = "Failed to read btrfs subvolume list for \"$droot\""; - WARN "Skipping target: $config_target->{ABORTED}"; + my $droot = vinfo($config_target->{url}, $config_target); + unless(vinfo_read_detail($droot)) { + $config_target->{ABORTED} = "Failed to fetch subvolume detail"; + WARN "Skipping target \"$droot->{URL}\": $config_target->{ABORTED}"; + next; + } + + unless(vinfo_read_subvolumes($droot)) { + $config_target->{ABORTED} = "Failed to fetch subvolume list"; + WARN "Skipping target \"$droot->{URL}\": $config_target->{ABORTED}"; next; } } } } - TRACE(Data::Dumper->Dump([\%vol_info], ["vol_info"])); if($action_origin) @@ -1500,23 +1558,22 @@ MAIN: # # print origin information # - my $subvol = $subvol_args[0] || die; + my $url = $subvol_args[0] || die; my $dump_uuid = 0; - my $detail = vinfo($subvol); - exit 1 unless($detail); + my $vol = vinfo($url); + exit 1 unless($vol); - if($detail->{is_root}) { - ERROR "Subvolume is btrfs root: $subvol\n"; + if($vol->{is_root}) { + ERROR "Subvolume is btrfs root: $url\n"; exit 1; } - my $uuid = $detail->{uuid} || die; + my $uuid = $vol->{uuid} || die; my $node = $uuid_info{$uuid}; - unless($node) { - DEBUG "Subvolume not parsed yet, fetching info: $subvol"; - vinfo($subvol); - $vol_info{$subvol} //= btr_fs_info($subvol); + unless($node) { # !!! TODO: fix this + DEBUG "Subvolume not parsed yet, fetching info: $url"; +# !!! $vol_info{$url} //= btr_fs_info($vol); $node = $uuid_info{$uuid} || die; } @@ -1552,29 +1609,28 @@ MAIN: foreach my $config_vol (@{$config->{VOLUME}}) { my %droot_compat; - my $sroot = $config_vol->{sroot} || die; - print "$sroot\n"; - next unless $vol_info{$sroot}; + my $sroot = vinfo($config_vol->{url}, $config_vol); + print "$sroot->{URL}\n"; + next unless $sroot->{ERROR}; # !!! TODO: check this foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { - my $svol = $config_subvol->{svol} // die; - print "|-- $svol\n"; - unless($vol_info{$sroot}->{$svol}) { - print " !!! error: no subvolume \"$svol\" found in \"$sroot\"\n"; + my $svol = vinfo($config_subvol->{url}, $config_vol); + print "|-- $svol->{URL}\n"; + unless(subvol($sroot, $config_subvol->{rel_path})) { # !!! TODO: maybe check uuid here? + print " !!! error: no subvolume \"$config_subvol->{rel_path}\" found in \"$sroot->{URL}\"\n"; next; } - my $sroot_uuid = $vol_info{$sroot}->{$svol}->{node}->{uuid} || die; - foreach my $snapshot (sort { $a->{SUBVOL_PATH} cmp $b->{SUBVOL_PATH} } (values %{$vol_info{$sroot}})) + foreach my $snapshot (sort { $a->{SUBVOL_PATH} cmp $b->{SUBVOL_PATH} } (values %{$sroot->{SUBVOL_INFO}})) { - next unless($snapshot->{node}->{parent_uuid} eq $sroot_uuid); + next unless($snapshot->{node}->{parent_uuid} eq $svol->{uuid}); # next unless($snapshot->{SUBVOL_PATH} =~ /^$snapdir/); # don't print non-btrbk snapshots print "| ^-- $snapshot->{SUBVOL_PATH}\n"; foreach my $config_target (@{$config_subvol->{TARGET}}) { - my $droot = $config_target->{droot} || die; - next unless $vol_info{$droot}; - $droot_compat{$droot} = 1 if($vol_btrfs_progs_compat{$droot}); + my $droot = vinfo($config_target->{url}, $config_target); + next unless $droot->{SUBVOL_INFO}; + $droot_compat{$droot} = 1 if($droot->{BTRFS_PROGS_COMPAT}); foreach (sort { $a->{SUBVOL_PATH} cmp $b->{SUBVOL_PATH} } get_receive_targets($droot, $snapshot)) { print "| | ^== $_->{URL}\n"; } @@ -1601,45 +1657,44 @@ MAIN: foreach my $config_vol (@{$config->{VOLUME}}) { next if($config_vol->{ABORTED}); - my $sroot = $config_vol->{sroot} || die; + my $sroot = vinfo($config_vol->{url}); foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { next if($config_subvol->{ABORTED}); - my $svol = $config_subvol->{svol} // die; + my $svol = vinfo($config_subvol->{url}); my $snapdir = config_key($config_subvol, "snapshot_dir") || ""; - my $snapshot; my $snapshot_name; - if($snapshot_cache{"$sroot/$svol"}) + my $snapshot_basename = $config_subvol->{rel_path}; # !!! TODO: add configuration option for this + if($svol->{SNAPSHOT}) # !!! TODO: guess we broke this, rethink what happens if same svol is used on different config lines { - $snapshot = $snapshot_cache{"$sroot/$svol"}->{file}; - $snapshot_name = $snapshot_cache{"$sroot/$svol"}->{name}; + $snapshot_name = $svol->{SNAPSHOT}->{NAME}; } else { # find unique snapshot name - my @lookup = keys %{$vol_info{$sroot}}; + my @lookup = keys %{$sroot->{SUBVOL_INFO}}; @lookup = grep s/^$snapdir// , @lookup; - foreach (@{$config_subvol->{TARGET}}){ - push(@lookup, keys %{$vol_info{$_->{droot}}}); + foreach my $config_target (@{$config_subvol->{TARGET}}) { + my $droot = vinfo($config_target->{url}); + push(@lookup, keys %{$droot->{SUBVOL_INFO}}); } - @lookup = grep /^$svol\.$timestamp(_[0-9]+)?$/ ,@lookup; - TRACE "Present snapshot names for \"$sroot/$svol\": " . join(', ', @lookup); + @lookup = grep /^\Q$snapshot_basename.$timestamp\E(_[0-9]+)?$/ ,@lookup; + TRACE "Present snapshot names for \"$svol->{URL}\": " . join(', ', @lookup); @lookup = map { /_([0-9]+)$/ ? $1 : 0 } @lookup; @lookup = sort { $b <=> $a } @lookup; my $postfix_counter = $lookup[0] // -1; $postfix_counter++; - $snapshot_name = $svol . '.' . $timestamp . ($postfix_counter ? "_$postfix_counter" : ""); - $snapshot = "$sroot/$snapdir$snapshot_name"; + $snapshot_name = $snapshot_basename . '.' . $timestamp . ($postfix_counter ? "_$postfix_counter" : ""); } my $create_snapshot = config_key($config_subvol, "snapshot_create_always"); foreach my $config_target (@{$config_subvol->{TARGET}}) { next if($config_target->{ABORTED}); - my $droot = $config_target->{droot} || die; + my $droot = vinfo($config_target->{url}); if(subvol($droot, $snapshot_name)) { - $config_target->{ABORTED} = "Subvolume already exists at destination: $droot/$snapshot_name"; + $config_target->{ABORTED} = "Subvolume already exists at destination: $droot->{URL}/$snapshot_name"; WARN "Skipping target: $config_target->{ABORTED}"; next; } @@ -1648,25 +1703,26 @@ MAIN: } } unless($create_snapshot) { - $config_subvol->{ABORTED} = "No targets defined for subvolume: $sroot/$svol"; + $config_subvol->{ABORTED} = "No targets defined for subvolume: $svol->{URL}"; WARN "Skipping subvolume section: $config_subvol->{ABORTED}"; next; } # make snapshot of svol, if not already created by another job - unless($snapshot_cache{"$sroot/$svol"}) + unless($svol->{SNAPSHOT}) { - INFO "Creating subvolume snapshot for: $sroot/$svol"; - - unless(btrfs_snapshot("$sroot/$svol", $snapshot, $config_subvol)) { - $config_subvol->{ABORTED} = "Failed to create snapshot, skipping subvolume: $sroot/$svol"; - WARN "Skipping subvolume section: $config_subvol->{ABORTED}"; + INFO "Creating subvolume snapshot for: $svol->{PRINT}"; + if(btrfs_snapshot($svol, "$sroot->{PATH}/$snapdir/$snapshot_name")) { + my $snapvol = vinfo("$sroot->{URL}/$snapdir/$snapshot_name", $config_vol); + $snapvol->{SNAP_BASENAME} = $snapshot_basename; + $svol->{SNAPSHOT} = $snapvol; + } + else { + $config_subvol->{ABORTED} = "Failed to create snapshot: $svol->{PRINT} -> $sroot->{PRINT}/$snapdir/$snapshot_name"; + WARN "Skipping subvolume section: $config_subvol->{ABORTED}"; + $svol->{SNAPSHOT} = { ERROR => $config_subvol->{ABORTED} }; } - $snapshot_cache{"$sroot/$svol"} = { name => $snapshot_name, - file => $snapshot }; } - $config_subvol->{snapshot} = $snapshot; - $config_subvol->{snapshot_name} = $snapshot_name; } } @@ -1676,32 +1732,30 @@ MAIN: foreach my $config_vol (@{$config->{VOLUME}}) { next if($config_vol->{ABORTED}); - my $sroot = $config_vol->{sroot} || die; + my $sroot = vinfo($config_vol->{url}); foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { next if($config_subvol->{ABORTED}); - my $svol = $config_subvol->{svol} // die; - my $snapshot = $config_subvol->{snapshot} || die; - my $snapshot_name = $config_subvol->{snapshot_name} || die; - + my $svol = vinfo($config_subvol->{url}); my $snapdir = config_key($config_subvol, "snapshot_dir") || ""; + my $snapshot_basename = $config_subvol->{rel_path}; # TODO: add configuration option for this, store into svol foreach my $config_target (@{$config_subvol->{TARGET}}) { next if($config_target->{ABORTED}); - my $droot = $config_target->{droot} || die; + my $droot = vinfo($config_target->{url}); my $target_type = $config_target->{target_type} || die; if($target_type eq "send-receive") { if(config_key($config_target, "receive_log")) { - WARN "Ignoring deprecated option \"receive_log\" for target: $droot" + WARN "Ignoring deprecated option \"receive_log\" for target: $droot->{URL}" } # resume missing backups (resume_missing) if(config_key($config_target, "resume_missing")) { - INFO "Checking for missing backups of subvolume \"$sroot/$svol\" in: $droot/"; + INFO "Checking for missing backups of subvolume \"$svol->{URL}\" in: $droot->{URL}/"; my @schedule; my $found_missing = 0; @@ -1716,7 +1770,7 @@ MAIN: # check if the target would be preserved my ($date, $date_ext) = get_date_tag($child->{SUBVOL_PATH}); - next unless($date && ($child->{SUBVOL_PATH} =~ /^$snapdir$svol\./)); + next unless($date && ($child->{SUBVOL_PATH} =~ /^\Q$snapdir$snapshot_basename.\E/)); push(@schedule, { value => $child, date => $date, date_ext => $date_ext }), } } @@ -1726,9 +1780,10 @@ MAIN: DEBUG "Checking schedule for resume candidates"; # add all present backups to schedule, with no value # these are needed for correct results of schedule() - foreach my $vol (keys %{$vol_info{$droot}}) { + foreach my $vol (keys %{$droot->{SUBVOL_INFO}}) { my ($date, $date_ext) = get_date_tag($vol); - next unless($date && ($vol =~ s/^$svol\.//)); # use only the date suffix for sorting + my $snapshot_basename = $config_subvol->{rel_path}; # TODO: add configuration option for this, store into svol + next unless($date && ($vol =~ s/^\Q$snapshot_basename.\E//)); # use only the date suffix for sorting push(@schedule, { value => undef, date => $date, date_ext => $date_ext }); } @@ -1746,15 +1801,15 @@ MAIN: INFO "Resuming subvolume backup (send-receive) for: $child->{URL}"; $found_missing++; my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot, $child->{node}->{gen}); - if(macro_send_receive($config_target, - src => $child->{URL}, - target => $droot, - parent => $latest_common_src ? $latest_common_src->{URL} : undef, - resume => 1, # propagated to $config_target->{subvol_received} + if(macro_send_receive($config_target, # TODO: !!! adapt this function + snapshot => $child, + target => $droot, + parent => $latest_common_src, # this is if no common found + resume => 1, # propagated to $config_target->{subvol_received} )) { # tag the source snapshot, so that get_latest_common() above can make use of the newly received subvolume - $child->{RECEIVE_TARGET_PRESENT} = $droot; + $child->{RECEIVE_TARGET_PRESENT} = $droot->{URL}; } else { # note: ABORTED flag is already set by macro_send_receive() @@ -1773,18 +1828,19 @@ MAIN: # skip creation if resume_missing failed next if($config_target->{ABORTED}); + die unless($svol->{SNAPSHOT}); # finally receive the previously created snapshot - INFO "Creating subvolume backup (send-receive) for: $sroot/$svol"; + INFO "Creating subvolume backup (send-receive) for: $svol->{URL}"; my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot); macro_send_receive($config_target, - src => $snapshot, - target => $droot, - parent => $latest_common_src ? $latest_common_src->{URL} : undef, + snapshot => $svol->{SNAPSHOT}, + target => $droot, + parent => $latest_common_src, # this is if no common found ); } else { - ERROR "Unknown target type \"$target_type\", skipping: $sroot/$svol"; + ERROR "Unknown target type \"$target_type\", skipping: $svol->{URL}"; $config_target->{ABORTED} = "Unknown target type \"$target_type\""; } } @@ -1803,12 +1859,13 @@ MAIN: foreach my $config_vol (@{$config->{VOLUME}}) { next if($config_vol->{ABORTED}); - my $sroot = $config_vol->{sroot} || die; + my $sroot = vinfo($config_vol->{url}); foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { next if($config_subvol->{ABORTED}); - my $svol = $config_subvol->{svol} // die; + my $svol = vinfo($config_subvol->{url}); my $snapdir = config_key($config_subvol, "snapshot_dir") || ""; + my $snapshot_basename = $config_subvol->{rel_path}; # !!! TODO: add configuration option for this, store into svol my $target_aborted = 0; foreach my $config_target (@{$config_subvol->{TARGET}}) { @@ -1816,17 +1873,17 @@ MAIN: $target_aborted = 1; next; } - my $droot = $config_target->{droot} || die; + my $droot = vinfo($config_target->{url}); # # delete backups # - INFO "Cleaning backups of subvolume \"$sroot/$svol\": $droot/$svol.*"; + INFO "Cleaning backups of subvolume \"$svol->{URL}\": $droot->{URL}/$snapshot_basename.*"; my @schedule; - foreach my $vol (keys %{$vol_info{$droot}}) { + foreach my $vol (keys %{$droot->{SUBVOL_INFO}}) { my ($date, $date_ext) = get_date_tag($vol); - next unless($date && ($vol =~ /^$svol\./)); - push(@schedule, { value => "$droot/$vol", name => $vol, date => $date, date_ext => $date_ext }); + next unless($date && ($vol =~ /^\Q$svol.\E/)); + push(@schedule, { value => "$droot->{URL}/$vol", name => $vol, date => $date, date_ext => $date_ext }); } my (undef, $delete) = schedule( schedule => \@schedule, @@ -1839,7 +1896,7 @@ MAIN: ); my $ret = btrfs_subvolume_delete($config_target, @$delete); if(defined($ret)) { - INFO "Deleted $ret subvolumes in: $droot/$svol.*"; + INFO "Deleted $ret subvolumes in: $droot->{URL}/$snapshot_basename.*"; $config_target->{subvol_deleted} = $delete; } else { @@ -1852,15 +1909,15 @@ MAIN: # delete snapshots # if($target_aborted) { - WARN "Skipping cleanup of snapshots for subvolume \"$sroot/$svol\", as at least one target aborted earlier"; + WARN "Skipping cleanup of snapshots for subvolume \"$svol->{URL}\", as at least one target aborted earlier"; next; } - INFO "Cleaning snapshots: $sroot/$snapdir$svol.*"; + INFO "Cleaning snapshots: $sroot->{URL}/$snapdir$snapshot_basename.*"; my @schedule; - foreach my $vol (keys %{$vol_info{$sroot}}) { + foreach my $vol (keys %{$sroot->{SUBVOL_INFO}}) { my ($date, $date_ext) = get_date_tag($vol); - next unless($date && ($vol =~ /^$snapdir$svol\./)); - push(@schedule, { value => "$sroot/$vol", name => $vol, date => $date, date_ext => $date_ext }); + next unless($date && ($vol =~ /^\Q$snapdir$snapshot_basename.\E/)); + push(@schedule, { value => "$sroot->{URL}/$vol", name => $vol, date => $date, date_ext => $date_ext }); } my (undef, $delete) = schedule( schedule => \@schedule, @@ -1873,7 +1930,7 @@ MAIN: ); my $ret = btrfs_subvolume_delete($config_subvol, @$delete); if(defined($ret)) { - INFO "Deleted $ret subvolumes in: $sroot/$snapdir$svol.*"; + INFO "Deleted $ret subvolumes in: $sroot->{URL}/$snapdir$snapshot_basename.*"; $config_subvol->{subvol_deleted} = $delete; } else { @@ -1906,17 +1963,18 @@ MAIN: foreach my $config_vol (@{$config->{VOLUME}}) { if($config_vol->{ABORTED}) { - print "!!! $config_vol->{sroot}: ABORTED: $config_vol->{ABORTED}\n"; + print "!!! $config_vol->{url}: ABORTED: $config_vol->{ABORTED}\n"; $err_count++ unless($config_vol->{ABORTED_NOERR}); } foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { - print "\n$config_vol->{sroot}/$config_subvol->{svol}\n"; + my $svol = vinfo($config_subvol->{url}); + print "\n$svol->{PRINT}\n"; if($config_subvol->{ABORTED}) { - print "!!! Subvolume \"$config_subvol->{svol}\" aborted: $config_subvol->{ABORTED}\n"; + print "!!! Subvolume \"$config_subvol->{rel_path}\" aborted: $config_subvol->{ABORTED}\n"; $err_count++ unless($config_subvol->{ABORTED_NOERR}); } - print "+++ $config_subvol->{snapshot}\n" if($config_subvol->{snapshot}); + print "+++ $svol->{SNAPSHOT}->{PRINT}\n" if($svol->{SNAPSHOT}->{PRINT}); if($config_subvol->{subvol_deleted}) { print "--- $_\n" foreach(sort { $b cmp $a} @{$config_subvol->{subvol_deleted}}); } @@ -1935,7 +1993,7 @@ MAIN: } if($config_target->{ABORTED}) { - print "!!! Target \"$config_target->{droot}\" aborted: $config_target->{ABORTED}\n"; + print "!!! Target \"$config_target->{url}\" aborted: $config_target->{ABORTED}\n"; $err_count++ unless($config_target->{ABORTED_NOERR}); } } From 3413425ed924e39563bd58f603e413cd95893404 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Sat, 18 Apr 2015 20:18:11 +0200 Subject: [PATCH 07/31] btrbk: added "snapshot_name" configuration option --- ChangeLog | 8 +-- btrbk | 173 ++++++++++++++++++++++++++--------------------- doc/btrbk.conf.5 | 15 ++-- 3 files changed, 111 insertions(+), 85 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3198002..2067b53 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,12 +1,12 @@ btrbk-current - - * Bugfix: allow "0" as subvolume name (closes: #10) + * Added configuration option "snapshot_name" (closes: #5). + * Bugfix: allow "0" as subvolume name (closes: #10). * Bugfix: check source AND targets for determining snapshot postfix - (closes: #11) + (closes: #11). btrbk-0.16 - * Bugfix: correctly check retention policy for missing backups + * Bugfix: correctly check retention policy for missing backups. btrbk-0.15 diff --git a/btrbk b/btrbk index 23a511f..45229a3 100755 --- a/btrbk +++ b/btrbk @@ -60,7 +60,8 @@ my %day_of_week_map = ( monday => 1, tuesday => 2, wednesday => 3, thursday => 4 my %config_options = ( # NOTE: the parser always maps "no" to undef # NOTE: keys "volume", "subvolume" and "target" are hardcoded - snapshot_dir => { default => undef, accept_file => { relative => 1 }, append_trailing_slash => 1 }, + snapshot_dir => { default => undef, accept_file => { relative => 1 } }, + snapshot_name => { default => undef, accept_file => { name_only => 1 }, context => [ "subvolume" ] }, receive_log => { default => undef, accept => [ "sidecar", "no" ], accept_file => { absolute => 1 }, deprecated => "removed" }, incremental => { default => "yes", accept => [ "yes", "no", "strict" ] }, snapshot_create_always => { default => undef, accept => [ "yes", "no" ] }, @@ -187,7 +188,12 @@ sub vinfo($;$) die unless($config); - my %info = ( URL => $url ); + my $name = $url; + $name =~ s/^.*\///; + my %info = ( + URL => $url, + NAME => $name, + ); if($url =~ /^ssh:\/\/(\S+?)(\/\S+)$/) { my ($host, $path) = ($1, $2); @@ -256,6 +262,14 @@ sub vinfo_read_detail($) } $vol->{$_} = $detail->{$_}; } + + if($vol->{RSH_TYPE} && ($vol->{RSH_TYPE} eq "ssh")) { + $vol->{REAL_URL} = "ssh://$vol->{HOST}$vol->{REAL_PATH}"; + } else { + $vol->{REAL_URL} = $vol->{REAL_PATH}; + } + + DEBUG "vinfo updated for: $vol->{URL}"; TRACE(Data::Dumper->Dump([$vol], ["vinfo{$vol->{URL}}"])); @@ -325,6 +339,12 @@ sub check_file($$;$$) return undef; } } + elsif($accept->{name_only}) { + if($file =~ /\//) { + ERROR "Option \"$key\" is not a valid file name in \"$config_file\" line $.: $file"; + return undef; + } + } else { die("accept_type must contain either 'relative' or 'absolute'"); } @@ -395,7 +415,7 @@ sub parse_config(@) { while($cur->{CONTEXT} ne "volume") { if(($cur->{CONTEXT} eq "root") || (not $cur->{PARENT})) { - ERROR "subvolume keyword outside volume context, in \"$file\" line $."; + ERROR "Subvolume keyword outside volume context, in \"$file\" line $."; return undef; } $cur = $cur->{PARENT} || die; @@ -406,7 +426,7 @@ sub parse_config(@) $value =~ s/\/+$//; # remove trailing slash $value =~ s/^\/+//; # remove leading slash if($value =~ /\//) { - ERROR "subvolume contains slashes: \"$value\" in \"$file\" line $."; + ERROR "Subvolume contains slashes: \"$value\" in \"$file\" line $."; return undef; } @@ -427,14 +447,14 @@ sub parse_config(@) DEBUG "config: context changed to: $cur->{CONTEXT}"; } if($cur->{CONTEXT} ne "subvolume") { - ERROR "target keyword outside subvolume context, in \"$file\" line $."; + ERROR "Target keyword outside subvolume context, in \"$file\" line $."; return undef; } if($value =~ /^(\S+)\s+(\S+)$/) { my ($target_type, $droot) = ($1, $2); unless(grep(/^$target_type$/, @config_target_types)) { - ERROR "unknown target type \"$target_type\" in \"$file\" line $."; + ERROR "Unknown target type \"$target_type\" in \"$file\" line $."; return undef; } # be very strict about file options, for security sake @@ -474,10 +494,6 @@ sub parse_config(@) TRACE "option \"$key=$value\" is a valid file, accepted"; $value =~ s/\/+$//; # remove trailing slash $value =~ s/^\/+/\//; # sanitize leading slash - if($config_options{$key}->{append_trailing_slash}) { - TRACE "append_trailing_slash is specified for option \"$key\", adding trailing slash"; - $value .= '/'; - } } elsif($config_options{$key}->{accept_regexp}) { my $match = $config_options{$key}->{accept_regexp}; @@ -494,6 +510,12 @@ sub parse_config(@) ERROR "Unsupported value \"$value\" for option \"$key\" in \"$file\" line $."; return undef; } + + if($config_options{$key}->{context} && !grep(/^$cur->{CONTEXT}$/, @{$config_options{$key}->{context}})) { + ERROR "Option \"$key\" is only allowed in " . join(" or ", map("\"$_\"", @{$config_options{$key}->{context}})) . " context, in \"$file\" line $."; + return undef; + } + DEBUG "config: adding option \"$key=$value\" to $cur->{CONTEXT} context"; $value = undef if($value eq "no"); # we don't want to check for "no" all the time $cur->{$key} = $value; @@ -918,7 +940,6 @@ sub btrfs_send_receive($$$) my $target_path = $target->{PATH} // die; my $target_rsh = $target->{RSH} || ""; my $parent_path = $parent ? $parent->{PATH} : undef; - my $now = localtime; my $snapshot_name = $snapshot_path; $snapshot_name =~ s/^.*\///; @@ -1504,6 +1525,7 @@ MAIN: # # fill vol_info hash, basic checks on configuration # + my %snapshot_check; foreach my $config_vol (@{$config->{VOLUME}}) { next if($config_vol->{ABORTED}); @@ -1518,6 +1540,19 @@ MAIN: { next if($config_subvol->{ABORTED}); my $svol = vinfo($config_subvol->{url}, $config_vol); + + # check for duplicate snapshot locations + my $snapdir = config_key($config_subvol, "snapshot_dir") || ""; + my $snapshot_basename = config_key($config_subvol, "snapshot_name") // $svol->{NAME} // die; + my $snapshot_target = "$sroot->{REAL_URL}/$snapdir/$snapshot_basename"; + if(my $prev = $snapshot_check{$snapshot_target}) { + ERROR "Subvolume \"$prev\" and \"$svol->{PRINT}\" will create same snapshot: $snapshot_target"; + ERROR "Please fix \"snapshot_name\" configuration options!"; + exit 1; + } + $snapshot_check{$snapshot_target} = $svol->{PRINT}; + + # read subvolume detail unless(vinfo_read_detail($svol)) { $config_subvol->{ABORTED} = "Failed to fetch subvolume detail"; WARN "Skipping subvolume \"$svol->{URL}\": $config_subvol->{ABORTED}"; @@ -1529,7 +1564,7 @@ MAIN: next; } - unless(subvol($sroot, $config_subvol->{rel_path})) { # !!! TODO: maybe check uuid here? + unless(subvol($sroot, $config_subvol->{rel_path})) { # !!! TODO: check uuid here! $config_subvol->{ABORTED} = "Subvolume \"$svol->{URL}\" not present in btrfs subvolume list for \"$sroot->{URL}\""; WARN "Skipping subvolume \"$svol->{URL}\": $config_subvol->{ABORTED}"; next; @@ -1543,6 +1578,15 @@ MAIN: next; } + # check for duplicate snapshot locations + my $snapshot_backup_target = "$droot->{REAL_URL}/$snapshot_basename"; + if(my $prev = $snapshot_check{$snapshot_backup_target}) { + ERROR "Subvolume \"$prev\" and \"$svol->{PRINT}\" will create same snapshot: $snapshot_target"; + ERROR "Please fix \"snapshot_name\" configuration options!"; + exit 1; + } + $snapshot_check{$snapshot_backup_target} = $svol->{PRINT}; + unless(vinfo_read_subvolumes($droot)) { $config_target->{ABORTED} = "Failed to fetch subvolume list"; WARN "Skipping target \"$droot->{URL}\": $config_target->{ABORTED}"; @@ -1653,7 +1697,6 @@ MAIN: # create snapshots # my $timestamp = sprintf("%04d%02d%02d", @today); - my %snapshot_cache; foreach my $config_vol (@{$config->{VOLUME}}) { next if($config_vol->{ABORTED}); @@ -1663,44 +1706,13 @@ MAIN: next if($config_subvol->{ABORTED}); my $svol = vinfo($config_subvol->{url}); my $snapdir = config_key($config_subvol, "snapshot_dir") || ""; - my $snapshot_name; - my $snapshot_basename = $config_subvol->{rel_path}; # !!! TODO: add configuration option for this - if($svol->{SNAPSHOT}) # !!! TODO: guess we broke this, rethink what happens if same svol is used on different config lines - { - $snapshot_name = $svol->{SNAPSHOT}->{NAME}; - } - else - { - # find unique snapshot name - my @lookup = keys %{$sroot->{SUBVOL_INFO}}; - @lookup = grep s/^$snapdir// , @lookup; - foreach my $config_target (@{$config_subvol->{TARGET}}) { - my $droot = vinfo($config_target->{url}); - push(@lookup, keys %{$droot->{SUBVOL_INFO}}); - } - @lookup = grep /^\Q$snapshot_basename.$timestamp\E(_[0-9]+)?$/ ,@lookup; - TRACE "Present snapshot names for \"$svol->{URL}\": " . join(', ', @lookup); - @lookup = map { /_([0-9]+)$/ ? $1 : 0 } @lookup; - @lookup = sort { $b <=> $a } @lookup; - my $postfix_counter = $lookup[0] // -1; - $postfix_counter++; - - $snapshot_name = $snapshot_basename . '.' . $timestamp . ($postfix_counter ? "_$postfix_counter" : ""); - } + my $snapshot_basename = config_key($config_subvol, "snapshot_name") // $svol->{NAME} // die; + # check if we need to create a snapshot my $create_snapshot = config_key($config_subvol, "snapshot_create_always"); - foreach my $config_target (@{$config_subvol->{TARGET}}) - { + foreach my $config_target (@{$config_subvol->{TARGET}}) { next if($config_target->{ABORTED}); - my $droot = vinfo($config_target->{url}); - if(subvol($droot, $snapshot_name)) { - $config_target->{ABORTED} = "Subvolume already exists at destination: $droot->{URL}/$snapshot_name"; - WARN "Skipping target: $config_target->{ABORTED}"; - next; - } - if($config_target->{target_type} eq "send-receive") { - $create_snapshot = 1; - } + $create_snapshot = 1 if($config_target->{target_type} eq "send-receive"); } unless($create_snapshot) { $config_subvol->{ABORTED} = "No targets defined for subvolume: $svol->{URL}"; @@ -1708,20 +1720,29 @@ MAIN: next; } - # make snapshot of svol, if not already created by another job - unless($svol->{SNAPSHOT}) - { - INFO "Creating subvolume snapshot for: $svol->{PRINT}"; - if(btrfs_snapshot($svol, "$sroot->{PATH}/$snapdir/$snapshot_name")) { - my $snapvol = vinfo("$sroot->{URL}/$snapdir/$snapshot_name", $config_vol); - $snapvol->{SNAP_BASENAME} = $snapshot_basename; - $svol->{SNAPSHOT} = $snapvol; - } - else { - $config_subvol->{ABORTED} = "Failed to create snapshot: $svol->{PRINT} -> $sroot->{PRINT}/$snapdir/$snapshot_name"; - WARN "Skipping subvolume section: $config_subvol->{ABORTED}"; - $svol->{SNAPSHOT} = { ERROR => $config_subvol->{ABORTED} }; - } + # find unique snapshot name + my @lookup = keys %{$sroot->{SUBVOL_INFO}}; + @lookup = grep s/^\Q$snapdir\E\/// , @lookup; + foreach my $config_target (@{$config_subvol->{TARGET}}) { + my $droot = vinfo($config_target->{url}); + push(@lookup, keys %{$droot->{SUBVOL_INFO}}); + } + @lookup = grep /^\Q$snapshot_basename.$timestamp\E(_[0-9]+)?$/ ,@lookup; + TRACE "Present snapshot names for \"$svol->{URL}\": " . join(', ', @lookup); + @lookup = map { /_([0-9]+)$/ ? $1 : 0 } @lookup; + @lookup = sort { $b <=> $a } @lookup; + my $postfix_counter = $lookup[0] // -1; + $postfix_counter++; + my $snapshot_name = $snapshot_basename . '.' . $timestamp . ($postfix_counter ? "_$postfix_counter" : ""); + + # finally create the snapshot + INFO "Creating subvolume snapshot for: $svol->{PRINT}"; + if(btrfs_snapshot($svol, "$sroot->{PATH}/$snapdir/$snapshot_name")) { + $config_subvol->{SNAPSHOT} = vinfo("$sroot->{URL}/$snapdir/$snapshot_name", $config_vol); + } + else { + $config_subvol->{ABORTED} = "Failed to create snapshot: $svol->{PRINT} -> $sroot->{PRINT}/$snapdir/$snapshot_name"; + WARN "Skipping subvolume section: $config_subvol->{ABORTED}"; } } } @@ -1738,7 +1759,7 @@ MAIN: next if($config_subvol->{ABORTED}); my $svol = vinfo($config_subvol->{url}); my $snapdir = config_key($config_subvol, "snapshot_dir") || ""; - my $snapshot_basename = $config_subvol->{rel_path}; # TODO: add configuration option for this, store into svol + my $snapshot_basename = config_key($config_subvol, "snapshot_name") // $svol->{NAME} // die; foreach my $config_target (@{$config_subvol->{TARGET}}) { @@ -1770,7 +1791,7 @@ MAIN: # check if the target would be preserved my ($date, $date_ext) = get_date_tag($child->{SUBVOL_PATH}); - next unless($date && ($child->{SUBVOL_PATH} =~ /^\Q$snapdir$snapshot_basename.\E/)); + next unless($date && ($child->{SUBVOL_PATH} =~ /^\Q$snapdir\/$snapshot_basename.\E/)); push(@schedule, { value => $child, date => $date, date_ext => $date_ext }), } } @@ -1828,13 +1849,13 @@ MAIN: # skip creation if resume_missing failed next if($config_target->{ABORTED}); - die unless($svol->{SNAPSHOT}); + die unless($config_subvol->{SNAPSHOT}); # finally receive the previously created snapshot INFO "Creating subvolume backup (send-receive) for: $svol->{URL}"; my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot); macro_send_receive($config_target, - snapshot => $svol->{SNAPSHOT}, + snapshot => $config_subvol->{SNAPSHOT}, target => $droot, parent => $latest_common_src, # this is if no common found ); @@ -1897,7 +1918,7 @@ MAIN: my $ret = btrfs_subvolume_delete($config_target, @$delete); if(defined($ret)) { INFO "Deleted $ret subvolumes in: $droot->{URL}/$snapshot_basename.*"; - $config_target->{subvol_deleted} = $delete; + $config_target->{SUBVOL_DELETED} = $delete; } else { $config_target->{ABORTED} = "btrfs subvolume delete command failed"; @@ -1912,11 +1933,11 @@ MAIN: WARN "Skipping cleanup of snapshots for subvolume \"$svol->{URL}\", as at least one target aborted earlier"; next; } - INFO "Cleaning snapshots: $sroot->{URL}/$snapdir$snapshot_basename.*"; + INFO "Cleaning snapshots: $sroot->{URL}/$snapdir/$snapshot_basename.*"; my @schedule; foreach my $vol (keys %{$sroot->{SUBVOL_INFO}}) { my ($date, $date_ext) = get_date_tag($vol); - next unless($date && ($vol =~ /^\Q$snapdir$snapshot_basename.\E/)); + next unless($date && ($vol =~ /^\Q$snapdir\/$snapshot_basename.\E/)); push(@schedule, { value => "$sroot->{URL}/$vol", name => $vol, date => $date, date_ext => $date_ext }); } my (undef, $delete) = schedule( @@ -1930,8 +1951,8 @@ MAIN: ); my $ret = btrfs_subvolume_delete($config_subvol, @$delete); if(defined($ret)) { - INFO "Deleted $ret subvolumes in: $sroot->{URL}/$snapdir$snapshot_basename.*"; - $config_subvol->{subvol_deleted} = $delete; + INFO "Deleted $ret subvolumes in: $sroot->{URL}/$snapdir/$snapshot_basename.*"; + $config_subvol->{SUBVOL_DELETED} = $delete; } else { $config_subvol->{ABORTED} = "btrfs subvolume delete command failed"; @@ -1974,9 +1995,9 @@ MAIN: print "!!! Subvolume \"$config_subvol->{rel_path}\" aborted: $config_subvol->{ABORTED}\n"; $err_count++ unless($config_subvol->{ABORTED_NOERR}); } - print "+++ $svol->{SNAPSHOT}->{PRINT}\n" if($svol->{SNAPSHOT}->{PRINT}); - if($config_subvol->{subvol_deleted}) { - print "--- $_\n" foreach(sort { $b cmp $a} @{$config_subvol->{subvol_deleted}}); + print "+++ $config_subvol->{SNAPSHOT}->{PRINT}\n" if($config_subvol->{SNAPSHOT}); + if($config_subvol->{SUBVOL_DELETED}) { + print "--- $_\n" foreach(sort { $b cmp $a} @{$config_subvol->{SUBVOL_DELETED}}); } foreach my $config_target (@{$config_subvol->{TARGET}}) { @@ -1988,8 +2009,8 @@ MAIN: print "$create_mode $_->{received_name}\n"; } - if($config_target->{subvol_deleted}) { - print "--- $_\n" foreach(sort { $b cmp $a} @{$config_target->{subvol_deleted}}); + if($config_target->{SUBVOL_DELETED}) { + print "--- $_\n" foreach(sort { $b cmp $a} @{$config_target->{SUBVOL_DELETED}}); } if($config_target->{ABORTED}) { diff --git a/doc/btrbk.conf.5 b/doc/btrbk.conf.5 index 186eb6a..292c207 100644 --- a/doc/btrbk.conf.5 +++ b/doc/btrbk.conf.5 @@ -13,7 +13,7 @@ generated. The retention policy as well as other options can be defined for each backup. .PP The options specified always apply to the last section encountered, -overriding the same option of the next higher section. This means that +superseding the values set in upper-level sections. This means that global options must be set before any sections are defined. .PP The sections are: @@ -44,14 +44,15 @@ allowed. The configuration options are: .TP \fBsnapshot_dir\fR -Directory in which the btrfs snapshots are created. Relative to +Directory in which the btrfs snapshots are created, relative to \fI\fR of the \fIvolume\fR section. Note that btrbk does not autmatically create this directory, and the snapshot creation will fail if it is not present. .TP -\fBincremental\fR yes|no|strict -Perform incremental backups. Defaults to \(lqyes\(rq. If set to -\(lqstrict\(rq, non-incremental (initial) backups are never created. +\fBsnapshot_name\fR +Base name of the created snapshot (and backup). Defaults to +\fI\fR. This option is only valid in the \fItarget\fR +section. .TP \fBsnapshot_create_always\fR yes|no If set, the snapshots are always created, even if the backup subvolume @@ -62,6 +63,10 @@ is reachable again. Useful for laptop filesystems in order to make sure the snapshots are created even if you are on the road. Defaults to \(lqno\(rq. .TP +\fBincremental\fR yes|no|strict +Perform incremental backups. Defaults to \(lqyes\(rq. If set to +\(lqstrict\(rq, non-incremental (initial) backups are never created. +.TP \fBresume_missing\fR yes|no If set, the backups in the target directory are compared to the source snapshots, and missing backups are created if needed (complying to the From 40d3f27b2ef85115cb78bbfd44a160ad871157da Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Sun, 19 Apr 2015 11:36:40 +0200 Subject: [PATCH 08/31] btrbk: simplified vinfo (SUBVOL_INFO holds copies of btr_tree nodes); cleanup --- btrbk | 320 ++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 186 insertions(+), 134 deletions(-) diff --git a/btrbk b/btrbk index 45229a3..9fc5549 100755 --- a/btrbk +++ b/btrbk @@ -81,7 +81,7 @@ my %config_options = ( my @config_target_types = qw(send-receive); -my %vol_info; +my %vinfo_root; my %uuid_info; my %uuid_fs_map; @@ -166,27 +166,31 @@ sub run_cmd($;$) } -sub subvol($$) +sub vinfo($;$) { - my $root = shift || die; - my $vol = shift // die; - if($root->{SUBVOL_INFO} && $root->{SUBVOL_INFO}->{$vol}) { - return $root->{SUBVOL_INFO}->{$vol}->{node}; + my $root = shift // die; # url or vinfo hash + my $subvol_path = shift; + unless(ref($root)) { + $root = $vinfo_root{$root} || die; + } + + if($subvol_path) { + if($root->{SUBVOL_INFO} && $root->{SUBVOL_INFO}->{$subvol_path}) { + return $root->{SUBVOL_INFO}->{$subvol_path}; + } + return undef; + } + else { + return $root; } - return undef; } -sub vinfo($;$) +sub vinfo_root($$) { my $url = shift // die; - my $config = shift; - if($vol_info{$url}) { - TRACE "vinfo cache hit: $url"; - return $vol_info{$url}; - } - - die unless($config); + my $config = shift || die; + die if($vinfo_root{$url}); my $name = $url; $name =~ s/^.*\///; @@ -231,34 +235,59 @@ sub vinfo($;$) my $btrfs_progs_compat = config_key($config, "btrfs_progs_compat"); $info{BTRFS_PROGS_COMPAT} = $btrfs_progs_compat if($btrfs_progs_compat); - DEBUG "vinfo created for: $url"; + DEBUG "vinfo root created for: $url"; TRACE(Data::Dumper->Dump([\%info], ["vinfo{$url}"])); - $vol_info{$url} = \%info; + $vinfo_root{$url} = \%info; return \%info; } -sub vinfo_read_detail($) +sub vinfo_child($$) +{ + my $parent = shift || die; + my $rel_path = shift // die; + + my $name = $rel_path; + $name =~ s/^.*\///; + my %info = ( + NAME => $name, + URL => "$parent->{URL}/$rel_path", + REAL_URL => "$parent->{REAL_URL}/$rel_path", + PATH => "$parent->{PATH}/$rel_path", + REAL_PATH => "$parent->{REAL_PATH}/$rel_path", + PRINT => "$parent->{PRINT}/$rel_path", + ); + foreach (qw( HOST + RSH_TYPE + SSH_USER + SSH_IDENTITY + RSH + BTRFS_PROGS_COMPAT ) ) + { + $info{$_} = $parent->{$_} if(exists $parent->{$_}); + } + + TRACE "vinfo child \"$rel_path\" created for: $info{URL}"; + TRACE(Data::Dumper->Dump([\%info], ["vinfo{$info{URL}}"])); + return \%info; +} + + +sub vinfo_set_detail($$) { my $vol = shift || die; + my $detail = shift || die; - if($vol->{id}) { - TRACE "vinfo detail cache hit: $vol->{URL}"; - return $vol; - } - - my $detail = btr_subvolume_detail($vol); - unless($detail) { - WARN "Failed to fetch subvolume detail for: $vol->{PRINT}"; - return undef; - } - - # add detail data to vinfo hash + # check and add detail data to vinfo hash foreach(keys %$detail) { if((defined $vol->{$_}) && ($vol->{$_} ne $detail->{$_})) { - WARN "Subvolume detail key \"$_\" is already present, with a different value: old=\"$vol->{$_}\", new=\"$detail->{$_}\""; - WARN "Using new value for \"$_\": $detail->{$_}"; + if($_ eq "REAL_PATH") { + DEBUG "Subvolume REAL_PATH changed (symlink): old=\"$vol->{REAL_PATH}\", new=\"$detail->{REAL_PATH}\""; + } else { + WARN "Subvolume detail key \"$_\" is already present, with a different value: old=\"$vol->{$_}\", new=\"$detail->{$_}\""; + WARN "Using new value for \"$_\": $detail->{$_}"; + } } $vol->{$_} = $detail->{$_}; } @@ -269,14 +298,36 @@ sub vinfo_read_detail($) $vol->{REAL_URL} = $vol->{REAL_PATH}; } - - DEBUG "vinfo updated for: $vol->{URL}"; + TRACE "vinfo updated for: $vol->{URL}"; TRACE(Data::Dumper->Dump([$vol], ["vinfo{$vol->{URL}}"])); - return $vol; } +sub vinfo_read_detail($) +{ + my $vol = shift || die; + + my $detail = btr_subvolume_detail($vol); + unless($detail) { + WARN "Failed to fetch subvolume detail for: $vol->{PRINT}"; + return undef; + } + return vinfo_set_detail($vol, $detail); +} + + +sub vinfo_add_child($$$) +{ + my $root = shift || die; + my $child = shift || die; + my $rel_path = shift // die; + die if($root->{SUBVOL_INFO}->{$rel_path}); + $root->{SUBVOL_INFO}->{$rel_path} = $child; + TRACE "vinfo child \"$rel_path\" added to: $root->{URL}"; +} + + sub get_rsh($$) { my $url = shift // die; @@ -627,7 +678,7 @@ sub btr_subvolume_detail($) WARN "Failed to parse subvolume detail \"$trans{$_}\": $ret"; } } - DEBUG "parsed " . scalar(keys %detail) . " subvolume detail items: $vol_print"; + DEBUG "Parsed " . scalar(keys %detail) . " subvolume detail items: $vol_print"; TRACE "btr_detail for $vol_print: " . Dumper \%detail; } return \%detail; @@ -700,7 +751,7 @@ sub btr_subvolume_list($;@) push @nodes, \%node; # $node{parent_uuid} = undef if($node{parent_uuid} eq '-'); } - DEBUG "parsed " . scalar(@nodes) . " total subvolumes for filesystem at: $vol_print"; + DEBUG "Parsed " . scalar(@nodes) . " total subvolumes for filesystem at: $vol_print"; return \@nodes; } @@ -853,10 +904,10 @@ sub vinfo_read_subvolumes($) } else { die unless $uuid_info{$vol->{uuid}}; - $uuid_fs_map{$vol->{uuid}}->{$url} = 1; + $uuid_fs_map{$vol->{uuid}}->{$vol->{URL}} = 1; $tree_root = $uuid_info{$vol->{uuid}}->{SUBTREE}; unless($tree_root) { - DEBUG "No subvolumes found in: $url"; + DEBUG "No subvolumes found in: $vol->{PRINT}"; $vol->{SUBVOL_INFO} = {}; return $vol; } @@ -870,18 +921,23 @@ sub vinfo_read_subvolumes($) foreach(@$list) { my $subvol_path = $_->{SUBVOL_PATH}; die if exists $ret{$subvol_path}; - $_->{URL} = $url . '/' . $subvol_path; - $_->{PATH} = $vol->{PATH} . '/' . $subvol_path; - $_->{PRINT} = $vol->{PRINT} . '/' . $subvol_path; - $_->{RSH} = $vol->{RSH}; - # !!! TODO: make real vinfo out of this - $uuid_fs_map{$_->{node}->{uuid}}->{$url . '/' . $subvol_path} = 1; - $ret{$subvol_path} = $_; + + my $detail = { %{$_->{node}}, + SUBVOL_PATH => $_->{SUBVOL_PATH}, + }; + delete $detail->{REL_PATH}; + delete $detail->{PARENT}; + my $subvol = vinfo_child($vol, "$subvol_path"); + vinfo_set_detail($subvol, $detail); + vinfo_add_child($vol, $subvol, $subvol_path); + + $uuid_fs_map{$subvol->{uuid}}->{$subvol->{URL}} = 1; + $ret{$subvol_path} = $subvol; } - TRACE(Data::Dumper->Dump([\%ret], ["vol_info{$url}"])); + DEBUG "Added " . scalar(keys %ret) . " subvolume children to: $vol->{PRINT}"; - $vol->{SUBVOL_INFO} = \%ret; + TRACE(Data::Dumper->Dump([\%ret], ["SUBVOL_INFO{$vol->{URL}}"])); return \%ret; } @@ -965,28 +1021,28 @@ sub btrfs_send_receive($$$) # sets $config->{ABORTED} on failure -# sets $config->{subvol_received} +# sets $config->{SUBVOL_RECEIVED} sub macro_send_receive($@) { - my $config = shift || die; + my $config_target = shift || die; my %info = @_; my $snapshot = $info{snapshot} || die; my $target = $info{target} || die; my $parent = $info{parent}; - my $incremental = config_key($config, "incremental"); + my $incremental = config_key($config_target, "incremental"); INFO "Receiving from snapshot: $snapshot->{PRINT}"; - # add info to $config->{subvol_received} - $info{received_name} = $snapshot->{PRINT}; - $config->{subvol_received} //= []; - push(@{$config->{subvol_received}}, \%info); + # add info to $config->{SUBVOL_RECEIVED} + $info{received_name} = "$target->{PRINT}/$snapshot->{NAME}"; + $config_target->{SUBVOL_RECEIVED} //= []; + push(@{$config_target->{SUBVOL_RECEIVED}}, \%info); if($incremental) { # create backup from latest common if($parent) { - INFO "Incremental from parent snapshot: $parent"; + INFO "Incremental from parent snapshot: $parent->{PRINT}"; } elsif($incremental ne "strict") { INFO "No common parent subvolume present, creating full backup"; @@ -994,7 +1050,7 @@ sub macro_send_receive($@) else { WARN "Backup to $target->{PRINT} failed: no common parent subvolume found, and option \"incremental\" is set to \"strict\""; $info{ERROR} = 1; - $config->{ABORTED} = "No common parent subvolume found, and option \"incremental\" is set to \"strict\""; + $config_target->{ABORTED} = "No common parent subvolume found, and option \"incremental\" is set to \"strict\""; return undef; } } @@ -1007,7 +1063,7 @@ sub macro_send_receive($@) return 1; } else { $info{ERROR} = 1; - $config->{ABORTED} = "btrfs send/receive command failed"; + $config_target->{ABORTED} = "btrfs send/receive command failed"; return undef; } } @@ -1032,7 +1088,7 @@ sub get_snapshot_children($$) my $svol = shift // die; my @ret; foreach (values %{$sroot->{SUBVOL_INFO}}) { - next unless($_->{node}->{parent_uuid} eq $svol->{uuid}); + next unless($_->{parent_uuid} eq $svol->{uuid}); TRACE "get_snapshot_children: found: $_->{URL}"; push(@ret, $_); } @@ -1044,7 +1100,7 @@ sub get_snapshot_children($$) sub get_receive_targets($$) { my $droot = shift || die; - my $src_href = shift || die; + my $src_vol = shift || die; die("root subvolume info not present: $droot") unless($droot->{SUBVOL_INFO}); my @ret; @@ -1052,10 +1108,10 @@ sub get_receive_targets($$) { # guess matches by subvolume name (node->received_uuid is not available if BTRFS_PROGS_COMPAT is set) DEBUG "Fallback to compatibility mode (get_receive_targets)"; - my $src_name = $src_href->{node}->{REL_PATH}; + my $src_name = $src_vol->{SUBVOL_PATH}; $src_name =~ s/^.*\///; # strip path foreach my $target (values %{$droot->{SUBVOL_INFO}}) { - my $target_name = $target->{node}->{REL_PATH}; + my $target_name = $target->{SUBVOL_PATH}; $target_name =~ s/^.*\///; # strip path if($target_name eq $src_name) { TRACE "get_receive_targets: by-name: Found receive target: $target->{SUBVOL_PATH}"; @@ -1066,15 +1122,15 @@ sub get_receive_targets($$) else { # find matches by comparing uuid / received_uuid - my $uuid = $src_href->{node}->{uuid}; + my $uuid = $src_vol->{uuid}; die("subvolume info not present: $uuid") unless($uuid_info{$uuid}); foreach (values %{$droot->{SUBVOL_INFO}}) { - next unless($_->{node}->{received_uuid} eq $uuid); + next unless($_->{received_uuid} eq $uuid); TRACE "get_receive_targets: by-uuid: Found receive target: $_->{SUBVOL_PATH}"; push(@ret, $_); } } - DEBUG "Found " . scalar(@ret) . " receive targets in \"$droot->{URL}/\" for: $src_href->{URL}"; + DEBUG "Found " . scalar(@ret) . " receive targets in \"$droot->{URL}/\" for: $src_vol->{URL}"; return @ret; } @@ -1093,10 +1149,10 @@ sub get_latest_common($$$;$) $debug_src .= "#" . $threshold_gen if($threshold_gen); # sort children of svol descending by generation - foreach my $child (sort { $b->{node}->{gen} <=> $a->{node}->{gen} } get_snapshot_children($sroot, $svol)) { + foreach my $child (sort { $b->{gen} <=> $a->{gen} } get_snapshot_children($sroot, $svol)) { TRACE "get_latest_common: checking source snapshot: $child->{SUBVOL_PATH}"; - if($threshold_gen && ($child->{node}->{gen} >= $threshold_gen)) { - TRACE "get_latest_common: skipped gen=$child->{node}->{gen} >= $threshold_gen: $child->{SUBVOL_PATH}"; + if($threshold_gen && ($child->{gen} >= $threshold_gen)) { + TRACE "get_latest_common: skipped gen=$child->{gen} >= $threshold_gen: $child->{SUBVOL_PATH}"; next; } @@ -1329,24 +1385,24 @@ MAIN: # # print snapshot diff # - my $src_vol = $subvol_args[0] || die; - my $target_vol = $subvol_args[1] || die; + my $src_url = $subvol_args[0] || die; + my $target_url = $subvol_args[1] || die; # FIXME: allow ssh:// src/dest (does not work since the configuration is not yet read). - my $src_detail = vinfo($src_vol); - unless($src_detail) { exit 1; } - if($src_detail->{is_root}) { ERROR "subvolume at \"$src_vol\" is btrfs root!"; exit 1; } - unless($src_detail->{cgen}) { ERROR "subvolume at \"$src_vol\" does not provide cgen"; exit 1; } -# if($src_detail->{parent_uuid} eq "-") { ERROR "subvolume at \"$src_vol\" has no parent, aborting."; exit 1; } + my $src_vol = vinfo_create($src_url, {}); + unless($src_vol) { exit 1; } + if($src_vol->{is_root}) { ERROR "subvolume at \"$src_url\" is btrfs root!"; exit 1; } + unless($src_vol->{cgen}) { ERROR "subvolume at \"$src_url\" does not provide cgen"; exit 1; } +# if($src_vol->{parent_uuid} eq "-") { ERROR "subvolume at \"$src_url\" has no parent, aborting."; exit 1; } - my $target_detail = vinfo($target_vol); - unless($target_detail) { exit 1; } - unless($src_detail->{cgen}) { ERROR "subvolume at \"$src_vol\" does not provide cgen"; exit 1; } -# if($src_detail->{parent_uuid} eq "-") { ERROR "subvolume at \"$src_vol\" has no parent, aborting."; exit 1; } + my $target_vol = vinfo_create($target_url, {}); + unless($target_vol) { exit 1; } + unless($src_vol->{cgen}) { ERROR "subvolume at \"$src_url\" does not provide cgen"; exit 1; } +# if($src_vol->{parent_uuid} eq "-") { ERROR "subvolume at \"$src_url\" has no parent, aborting."; exit 1; } - my $info = btr_tree($src_vol); - my $src = $uuid_info{$src_detail->{uuid}} || die; - my $target = $uuid_info{$target_detail->{uuid}}; + my $info = btr_tree($src_url); + my $src = $uuid_info{$src_vol->{uuid}} || die; + my $target = $uuid_info{$target_vol->{uuid}}; unless($target) { ERROR "target subvolume is not on the same btrfs filesystem!"; exit 1; } my $lastgen; @@ -1360,7 +1416,7 @@ MAIN: } else { # TODO: this rule only applies to snapshots. find a way to distinguish snapshots from received backups - # ERROR "subvolumes \"$target_vol\" and \"$src_vol\" do not share the same parents"; + # ERROR "subvolumes \"$target_url\" and \"$src_url\" do not share the same parents"; # exit 1; } @@ -1368,7 +1424,7 @@ MAIN: $lastgen = $src->{cgen} + 1; # dump files, sorted and unique - my $ret = btr_subvolume_find_new($target_vol, $lastgen); + my $ret = btr_subvolume_find_new($target_url, $lastgen); exit 1 unless(ref($ret)); print "--------------------------------------------------------------------------------\n"; @@ -1441,12 +1497,6 @@ MAIN: print " Config: $config->{SRC_FILE}\n"; print "================================================================================\n"; - # print "\n--------------------------------------------------------------------------------\n"; - # print "All local btrfs filesystems\n"; - # print "--------------------------------------------------------------------------------\n"; - # print (btr_filesystem_show_all_local() // ""); - # print "\n"; - my %processed; foreach my $config_vol (@{$config->{VOLUME}}) { @@ -1456,9 +1506,7 @@ MAIN: print "\n--------------------------------------------------------------------------------\n"; print "Source volume: $url\n"; print "--------------------------------------------------------------------------------\n"; - # print (btr_filesystem_show(vinfo($url, $config_vol)) // ""); - # print "\n\n"; - print (btr_filesystem_usage(vinfo($url, $config_vol)) // ""); + print (btr_filesystem_usage(vinfo_create($url, $config_vol)) // ""); print "\n"; $processed{$url} = 1; } @@ -1476,7 +1524,7 @@ MAIN: print "Target volume: $droot_url\n"; print " ^--- $sroot_url\n"; print "--------------------------------------------------------------------------------\n"; - print (btr_filesystem_usage(vinfo($droot_url, $config_target)) // ""); + print (btr_filesystem_usage(vinfo_create($droot_url, $config_target)) // ""); print "\n"; $processed{$droot_url} = 1; } @@ -1523,27 +1571,48 @@ MAIN: # - # fill vol_info hash, basic checks on configuration + # fill vinfo hash, basic checks on configuration # my %snapshot_check; foreach my $config_vol (@{$config->{VOLUME}}) { next if($config_vol->{ABORTED}); - my $sroot = vinfo($config_vol->{url}, $config_vol); + my $sroot = vinfo_root($config_vol->{url}, $config_vol); unless(vinfo_read_detail($sroot)) { $config_vol->{ABORTED} = "Failed to fetch subvolume detail"; WARN "Skipping volume \"$sroot->{URL}\": $config_vol->{ABORTED}"; next; } + unless(vinfo_read_subvolumes($sroot)) { + $config_vol->{ABORTED} = "Failed to fetch subvolume list"; + WARN "Skipping volume \"$sroot->{URL}\": $config_vol->{ABORTED}"; + next; + } foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { next if($config_subvol->{ABORTED}); - my $svol = vinfo($config_subvol->{url}, $config_vol); + my $svol = vinfo($sroot, $config_subvol->{rel_path}); + + unless($svol) { + # configured subvolume is not present in btrfs subvolume list. + # try to read subvolume detail, as configured subvolume could be a symlink. + DEBUG "Subvolume \"$config_subvol->{rel_path}\" not present in btrfs subvolume list for \"$sroot->{PRINT}\""; + $svol = vinfo_child($sroot, $config_subvol->{rel_path}); + unless(vinfo_read_detail($svol)) { + $config_subvol->{ABORTED} = "Failed to fetch subvolume detail"; + WARN "Skipping subvolume \"$svol->{URL}\": $config_subvol->{ABORTED}"; + next; + } + vinfo_add_child($sroot, $svol, $config_subvol->{rel_path}); + } + + # set default for snapshot_name + $config_subvol->{snapshot_name} //= $svol->{NAME}; # check for duplicate snapshot locations my $snapdir = config_key($config_subvol, "snapshot_dir") || ""; - my $snapshot_basename = config_key($config_subvol, "snapshot_name") // $svol->{NAME} // die; + my $snapshot_basename = config_key($config_subvol, "snapshot_name") // die; my $snapshot_target = "$sroot->{REAL_URL}/$snapdir/$snapshot_basename"; if(my $prev = $snapshot_check{$snapshot_target}) { ERROR "Subvolume \"$prev\" and \"$svol->{PRINT}\" will create same snapshot: $snapshot_target"; @@ -1552,26 +1621,9 @@ MAIN: } $snapshot_check{$snapshot_target} = $svol->{PRINT}; - # read subvolume detail - unless(vinfo_read_detail($svol)) { - $config_subvol->{ABORTED} = "Failed to fetch subvolume detail"; - WARN "Skipping subvolume \"$svol->{URL}\": $config_subvol->{ABORTED}"; - next; - } - unless(vinfo_read_subvolumes($sroot)) { - $config_subvol->{ABORTED} = "Failed to fetch subvolume list"; - WARN "Skipping subvolume \"$svol->{URL}\": $config_subvol->{ABORTED}"; - next; - } - - unless(subvol($sroot, $config_subvol->{rel_path})) { # !!! TODO: check uuid here! - $config_subvol->{ABORTED} = "Subvolume \"$svol->{URL}\" not present in btrfs subvolume list for \"$sroot->{URL}\""; - WARN "Skipping subvolume \"$svol->{URL}\": $config_subvol->{ABORTED}"; - next; - } foreach my $config_target (@{$config_subvol->{TARGET}}) { - my $droot = vinfo($config_target->{url}, $config_target); + my $droot = vinfo_root($config_target->{url}, $config_target); unless(vinfo_read_detail($droot)) { $config_target->{ABORTED} = "Failed to fetch subvolume detail"; WARN "Skipping target \"$droot->{URL}\": $config_target->{ABORTED}"; @@ -1653,26 +1705,26 @@ MAIN: foreach my $config_vol (@{$config->{VOLUME}}) { my %droot_compat; - my $sroot = vinfo($config_vol->{url}, $config_vol); + my $sroot = vinfo($config_vol->{url}); print "$sroot->{URL}\n"; next unless $sroot->{ERROR}; # !!! TODO: check this foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { - my $svol = vinfo($config_subvol->{url}, $config_vol); + my $svol = vinfo($sroot, $config_subvol->{rel_path}) || die; print "|-- $svol->{URL}\n"; - unless(subvol($sroot, $config_subvol->{rel_path})) { # !!! TODO: maybe check uuid here? + unless(vinfo($sroot, $config_subvol->{rel_path})) { # !!! TODO: maybe check uuid here? print " !!! error: no subvolume \"$config_subvol->{rel_path}\" found in \"$sroot->{URL}\"\n"; next; } foreach my $snapshot (sort { $a->{SUBVOL_PATH} cmp $b->{SUBVOL_PATH} } (values %{$sroot->{SUBVOL_INFO}})) { - next unless($snapshot->{node}->{parent_uuid} eq $svol->{uuid}); + next unless($snapshot->{parent_uuid} eq $svol->{uuid}); # next unless($snapshot->{SUBVOL_PATH} =~ /^$snapdir/); # don't print non-btrbk snapshots print "| ^-- $snapshot->{SUBVOL_PATH}\n"; foreach my $config_target (@{$config_subvol->{TARGET}}) { - my $droot = vinfo($config_target->{url}, $config_target); + my $droot = vinfo($config_target->{url}); next unless $droot->{SUBVOL_INFO}; $droot_compat{$droot} = 1 if($droot->{BTRFS_PROGS_COMPAT}); foreach (sort { $a->{SUBVOL_PATH} cmp $b->{SUBVOL_PATH} } get_receive_targets($droot, $snapshot)) { @@ -1704,9 +1756,9 @@ MAIN: foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { next if($config_subvol->{ABORTED}); - my $svol = vinfo($config_subvol->{url}); + my $svol = vinfo($sroot, $config_subvol->{rel_path}) || die; my $snapdir = config_key($config_subvol, "snapshot_dir") || ""; - my $snapshot_basename = config_key($config_subvol, "snapshot_name") // $svol->{NAME} // die; + my $snapshot_basename = config_key($config_subvol, "snapshot_name") // die; # check if we need to create a snapshot my $create_snapshot = config_key($config_subvol, "snapshot_create_always"); @@ -1738,7 +1790,7 @@ MAIN: # finally create the snapshot INFO "Creating subvolume snapshot for: $svol->{PRINT}"; if(btrfs_snapshot($svol, "$sroot->{PATH}/$snapdir/$snapshot_name")) { - $config_subvol->{SNAPSHOT} = vinfo("$sroot->{URL}/$snapdir/$snapshot_name", $config_vol); + $config_subvol->{SNAPSHOT} = vinfo_child($sroot, "$snapdir/$snapshot_name"); } else { $config_subvol->{ABORTED} = "Failed to create snapshot: $svol->{PRINT} -> $sroot->{PRINT}/$snapdir/$snapshot_name"; @@ -1757,9 +1809,9 @@ MAIN: foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { next if($config_subvol->{ABORTED}); - my $svol = vinfo($config_subvol->{url}); + my $svol = vinfo($sroot, $config_subvol->{rel_path}) || die; my $snapdir = config_key($config_subvol, "snapshot_dir") || ""; - my $snapshot_basename = config_key($config_subvol, "snapshot_name") // $svol->{NAME} // die; + my $snapshot_basename = config_key($config_subvol, "snapshot_name") // die; foreach my $config_target (@{$config_subvol->{TARGET}}) { @@ -1803,7 +1855,6 @@ MAIN: # these are needed for correct results of schedule() foreach my $vol (keys %{$droot->{SUBVOL_INFO}}) { my ($date, $date_ext) = get_date_tag($vol); - my $snapshot_basename = $config_subvol->{rel_path}; # TODO: add configuration option for this, store into svol next unless($date && ($vol =~ s/^\Q$snapshot_basename.\E//)); # use only the date suffix for sorting push(@schedule, { value => undef, date => $date, date_ext => $date_ext }); } @@ -1818,15 +1869,15 @@ MAIN: ); my @resume = grep defined, @$preserve; # remove entries with no value from list (target subvolumes) - foreach my $child (sort { $a->{node}->{gen} <=> $b->{node}->{gen} } @resume) { + foreach my $child (sort { $a->{gen} <=> $b->{gen} } @resume) { INFO "Resuming subvolume backup (send-receive) for: $child->{URL}"; $found_missing++; - my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot, $child->{node}->{gen}); + my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot, $child->{gen}); if(macro_send_receive($config_target, # TODO: !!! adapt this function snapshot => $child, target => $droot, parent => $latest_common_src, # this is if no common found - resume => 1, # propagated to $config_target->{subvol_received} + resume => 1, # propagated to $config_target->{SUBVOL_RECEIVED} )) { # tag the source snapshot, so that get_latest_common() above can make use of the newly received subvolume @@ -1884,9 +1935,9 @@ MAIN: foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { next if($config_subvol->{ABORTED}); - my $svol = vinfo($config_subvol->{url}); + my $svol = vinfo($sroot, $config_subvol->{rel_path}) || die; my $snapdir = config_key($config_subvol, "snapshot_dir") || ""; - my $snapshot_basename = $config_subvol->{rel_path}; # !!! TODO: add configuration option for this, store into svol + my $snapshot_basename = config_key($config_subvol, "snapshot_name") // die; my $target_aborted = 0; foreach my $config_target (@{$config_subvol->{TARGET}}) { @@ -1983,13 +2034,14 @@ MAIN: print "--------------------------------------------------------------------------------"; foreach my $config_vol (@{$config->{VOLUME}}) { + my $sroot = vinfo($config_vol->{url}); if($config_vol->{ABORTED}) { print "!!! $config_vol->{url}: ABORTED: $config_vol->{ABORTED}\n"; $err_count++ unless($config_vol->{ABORTED_NOERR}); } foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { - my $svol = vinfo($config_subvol->{url}); + my $svol = vinfo($sroot, $config_subvol->{rel_path}) || die; print "\n$svol->{PRINT}\n"; if($config_subvol->{ABORTED}) { print "!!! Subvolume \"$config_subvol->{rel_path}\" aborted: $config_subvol->{ABORTED}\n"; @@ -2001,7 +2053,7 @@ MAIN: } foreach my $config_target (@{$config_subvol->{TARGET}}) { - foreach(@{$config_target->{subvol_received} // []}) { + foreach(@{$config_target->{SUBVOL_RECEIVED} // []}) { my $create_mode = "***"; $create_mode = ">>>" if($_->{parent}); # substr($create_mode, 0, 1, '%') if($_->{resume}); From 19c0733471d85e48ba6f61cb601d0beb2c388410 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Mon, 20 Apr 2015 17:08:59 +0200 Subject: [PATCH 09/31] btrbk: added vinfo_cache; fixed action "origin" --- btrbk | 58 +++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/btrbk b/btrbk index 9fc5549..780c3fc 100755 --- a/btrbk +++ b/btrbk @@ -82,6 +82,7 @@ my %config_options = ( my @config_target_types = qw(send-receive); my %vinfo_root; +my %vinfo_cache; my %uuid_info; my %uuid_fs_map; @@ -166,6 +167,16 @@ sub run_cmd($;$) } +sub vinfo_update_lookup_tables($) +{ + my $vol = shift || die; + $vinfo_cache{$vol->{URL}} = $vol; + $vinfo_cache{$vol->{REAL_URL}} = $vol if($vol->{REAL_URL}); + $uuid_fs_map{$vol->{uuid}}->{$vol->{URL}} = $vol if($vol->{uuid}); + $uuid_fs_map{$vol->{uuid}}->{$vol->{REAL_URL}} = $vol if($vol->{uuid} && $vol->{REAL_URL}); +} + + sub vinfo($;$) { my $root = shift // die; # url or vinfo hash @@ -235,10 +246,11 @@ sub vinfo_root($$) my $btrfs_progs_compat = config_key($config, "btrfs_progs_compat"); $info{BTRFS_PROGS_COMPAT} = $btrfs_progs_compat if($btrfs_progs_compat); - DEBUG "vinfo root created for: $url"; - TRACE(Data::Dumper->Dump([\%info], ["vinfo{$url}"])); $vinfo_root{$url} = \%info; + vinfo_update_lookup_tables(\%info); + TRACE "vinfo root created for: $url"; + TRACE(Data::Dumper->Dump([\%info], ["vinfo{$url}"])); return \%info; } @@ -267,6 +279,7 @@ sub vinfo_child($$) { $info{$_} = $parent->{$_} if(exists $parent->{$_}); } + vinfo_update_lookup_tables(\%info); TRACE "vinfo child \"$rel_path\" created for: $info{URL}"; TRACE(Data::Dumper->Dump([\%info], ["vinfo{$info{URL}}"])); @@ -300,6 +313,7 @@ sub vinfo_set_detail($$) TRACE "vinfo updated for: $vol->{URL}"; TRACE(Data::Dumper->Dump([$vol], ["vinfo{$vol->{URL}}"])); + vinfo_update_lookup_tables($vol); return $vol; } @@ -749,7 +763,6 @@ sub btr_subvolume_list($;@) $node{path} =~ s/^\///; # remove "/" portion from "path". push @nodes, \%node; - # $node{parent_uuid} = undef if($node{parent_uuid} eq '-'); } DEBUG "Parsed " . scalar(@nodes) . " total subvolumes for filesystem at: $vol_print"; return \@nodes; @@ -895,7 +908,7 @@ sub vinfo_read_subvolumes($) my $vol = shift || die; my $url = $vol->{URL} || die; - my $tree = btr_tree($vol); + my $tree = btr_tree($vol); # populates %uuid_info return undef unless($tree); my $tree_root; @@ -904,11 +917,10 @@ sub vinfo_read_subvolumes($) } else { die unless $uuid_info{$vol->{uuid}}; - $uuid_fs_map{$vol->{uuid}}->{$vol->{URL}} = 1; $tree_root = $uuid_info{$vol->{uuid}}->{SUBTREE}; unless($tree_root) { DEBUG "No subvolumes found in: $vol->{PRINT}"; - $vol->{SUBVOL_INFO} = {}; + $vol->{SUBVOL_INFO} //= {}; return $vol; } } @@ -931,7 +943,6 @@ sub vinfo_read_subvolumes($) vinfo_set_detail($subvol, $detail); vinfo_add_child($vol, $subvol, $subvol_path); - $uuid_fs_map{$subvol->{uuid}}->{$subvol->{URL}} = 1; $ret{$subvol_path} = $subvol; } @@ -1169,7 +1180,7 @@ sub get_latest_common($$$;$) } TRACE "get_latest_common: no matching targets found for: $child->{URL}"; } - DEBUG("No common snapshots for \"$debug_src\" found in src=$sroot->{URL}/ target=$droot->{URL}/"); + DEBUG("No common snapshots for \"$debug_src\" found in src=\"$sroot->{URL}/\", target=\"$droot->{URL}/\""); return (undef, undef); } @@ -1185,9 +1196,7 @@ sub _origin_tree return 0; } if($uuid_fs_map{$uuid}) { - foreach(keys %{$uuid_fs_map{$uuid}}) { - push(@$lines, ["$prefix$_", $uuid]); - } + push(@$lines, ["$prefix" . join(" === ", sort keys %{$uuid_fs_map{$uuid}}), $uuid]); } else { push(@$lines, ["$prefix/$node->{path}", $uuid]); } @@ -1657,24 +1666,27 @@ MAIN: my $url = $subvol_args[0] || die; my $dump_uuid = 0; - my $vol = vinfo($url); - exit 1 unless($vol); + my $vol = $vinfo_cache{$url}; + unless($vol) { + # specified volume is not in config + DEBUG "Subvolume not parsed yet, fetching info: $url"; + $vol = vinfo_root($url, { CONTEXT => "cmdline" }); + unless(vinfo_read_detail($vol)) { + ERROR "Failed to fetch subvolume detail: $url"; + exit 1; + } + unless(vinfo_read_subvolumes($vol)) { + ERROR "Failed to fetch subvolume list: $url"; + exit 1; + } + } if($vol->{is_root}) { ERROR "Subvolume is btrfs root: $url\n"; exit 1; } - my $uuid = $vol->{uuid} || die; - my $node = $uuid_info{$uuid}; - - unless($node) { # !!! TODO: fix this - DEBUG "Subvolume not parsed yet, fetching info: $url"; -# !!! $vol_info{$url} //= btr_fs_info($vol); - $node = $uuid_info{$uuid} || die; - } - my $lines = []; - _origin_tree("", $uuid, $lines); + _origin_tree("", $vol->{uuid}, $lines); print "--------------------------------------------------------------------------------\n"; print "Origin Tree\n\n"; From e25312223d947c92ebd7cc315c42480c328d2079 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Mon, 20 Apr 2015 17:47:48 +0200 Subject: [PATCH 10/31] btrbk: fixed action "tree" --- btrbk | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/btrbk b/btrbk index 780c3fc..e0db455 100755 --- a/btrbk +++ b/btrbk @@ -1714,43 +1714,52 @@ MAIN: # print snapshot tree # # TODO: reverse tree: print all backups from $droot and their corresponding source snapshots + my @out; foreach my $config_vol (@{$config->{VOLUME}}) { my %droot_compat; my $sroot = vinfo($config_vol->{url}); - print "$sroot->{URL}\n"; - next unless $sroot->{ERROR}; # !!! TODO: check this + push @out, "$sroot->{PRINT}"; foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { my $svol = vinfo($sroot, $config_subvol->{rel_path}) || die; - print "|-- $svol->{URL}\n"; + push @out, "|-- $svol->{PRINT}"; unless(vinfo($sroot, $config_subvol->{rel_path})) { # !!! TODO: maybe check uuid here? - print " !!! error: no subvolume \"$config_subvol->{rel_path}\" found in \"$sroot->{URL}\"\n"; + push @out, " !!! error: no subvolume \"$config_subvol->{rel_path}\" found in \"$sroot->{PRINT}\""; next; } - - foreach my $snapshot (sort { $a->{SUBVOL_PATH} cmp $b->{SUBVOL_PATH} } (values %{$sroot->{SUBVOL_INFO}})) + foreach my $snapshot (sort { $a->{PATH} cmp $b->{PATH} } (values %{$sroot->{SUBVOL_INFO}})) { next unless($snapshot->{parent_uuid} eq $svol->{uuid}); # next unless($snapshot->{SUBVOL_PATH} =~ /^$snapdir/); # don't print non-btrbk snapshots - print "| ^-- $snapshot->{SUBVOL_PATH}\n"; + push @out, "| ^-- $snapshot->{PATH}"; foreach my $config_target (@{$config_subvol->{TARGET}}) { my $droot = vinfo($config_target->{url}); next unless $droot->{SUBVOL_INFO}; - $droot_compat{$droot} = 1 if($droot->{BTRFS_PROGS_COMPAT}); + $droot_compat{$droot->{URL}} = 1 if($droot->{BTRFS_PROGS_COMPAT}); foreach (sort { $a->{SUBVOL_PATH} cmp $b->{SUBVOL_PATH} } get_receive_targets($droot, $snapshot)) { - print "| | ^== $_->{URL}\n"; + push @out, "| | ^== $_->{PRINT}"; } } } } if(keys %droot_compat) { - print "\nNOTE: Received subvolumes (backups) are guessed by subvolume name for targets:\n"; - print " - " . join("\n - ", (sort keys %droot_compat)); + push @out, "\nNOTE: Received subvolumes (backups) are guessed by subvolume name for targets:"; + push @out, " - " . join("\n - ", (sort keys %droot_compat)); } - print "\n"; + push @out, ""; } + print "--------------------------------------------------------------------------------\n"; + print "Backup Tree\n\n"; + print " Date: " . localtime($start_time) . "\n"; + print " Config: $config->{SRC_FILE}\n"; + print "\nLegend:\n"; + print " ^--- snapshot\n"; + print " ^=== received subvolume (backup)\n"; + print "--------------------------------------------------------------------------------\n"; + + print join("\n", @out); exit 0; } From 6b00ea1741b334870274a9015cc3f3fbdf22f0d6 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Mon, 20 Apr 2015 18:19:55 +0200 Subject: [PATCH 11/31] btrbk: allow multiple identical src/target subvolumes --- btrbk | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/btrbk b/btrbk index e0db455..9bffcfa 100755 --- a/btrbk +++ b/btrbk @@ -201,7 +201,7 @@ sub vinfo_root($$) { my $url = shift // die; my $config = shift || die; - die if($vinfo_root{$url}); + return $vinfo_root{$url} if($vinfo_root{$url}); my $name = $url; $name =~ s/^.*\///; @@ -321,12 +321,14 @@ sub vinfo_set_detail($$) sub vinfo_read_detail($) { my $vol = shift || die; + return $vol if($vol->{VINFO_DETAIL_READ}); my $detail = btr_subvolume_detail($vol); unless($detail) { WARN "Failed to fetch subvolume detail for: $vol->{PRINT}"; return undef; } + $vol->{VINFO_DETAIL_READ} = 1; return vinfo_set_detail($vol, $detail); } @@ -906,6 +908,8 @@ sub _subtree_list sub vinfo_read_subvolumes($) { my $vol = shift || die; + return $vol if($vol->{VINFO_SUBVOLUMES_READ}); + my $url = $vol->{URL} || die; my $tree = btr_tree($vol); # populates %uuid_info @@ -947,6 +951,7 @@ sub vinfo_read_subvolumes($) } DEBUG "Added " . scalar(keys %ret) . " subvolume children to: $vol->{PRINT}"; + $vol->{VINFO_SUBVOLUMES_READ} = 1; TRACE(Data::Dumper->Dump([\%ret], ["SUBVOL_INFO{$vol->{URL}}"])); return \%ret; @@ -1583,6 +1588,7 @@ MAIN: # fill vinfo hash, basic checks on configuration # my %snapshot_check; + my %backup_check; foreach my $config_vol (@{$config->{VOLUME}}) { next if($config_vol->{ABORTED}); @@ -1641,12 +1647,12 @@ MAIN: # check for duplicate snapshot locations my $snapshot_backup_target = "$droot->{REAL_URL}/$snapshot_basename"; - if(my $prev = $snapshot_check{$snapshot_backup_target}) { - ERROR "Subvolume \"$prev\" and \"$svol->{PRINT}\" will create same snapshot: $snapshot_target"; - ERROR "Please fix \"snapshot_name\" configuration options!"; + if(my $prev = $backup_check{$snapshot_backup_target}) { + ERROR "Subvolume \"$prev\" and \"$svol->{PRINT}\" will create same backup target: $snapshot_target"; + ERROR "Please fix \"snapshot_name\" or \"target\" configuration options!"; exit 1; } - $snapshot_check{$snapshot_backup_target} = $svol->{PRINT}; + $backup_check{$snapshot_backup_target} = $svol->{PRINT}; unless(vinfo_read_subvolumes($droot)) { $config_target->{ABORTED} = "Failed to fetch subvolume list"; From 157f9333e7aaa0cb8024b476434a8ae1ac47bda8 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Mon, 20 Apr 2015 18:50:12 +0200 Subject: [PATCH 12/31] btrbk: bugfix: set REAL_PATH to vinfo_child only if it is confirmed --- btrbk | 6 ------ 1 file changed, 6 deletions(-) diff --git a/btrbk b/btrbk index 9bffcfa..c1caaa7 100755 --- a/btrbk +++ b/btrbk @@ -265,9 +265,7 @@ sub vinfo_child($$) my %info = ( NAME => $name, URL => "$parent->{URL}/$rel_path", - REAL_URL => "$parent->{REAL_URL}/$rel_path", PATH => "$parent->{PATH}/$rel_path", - REAL_PATH => "$parent->{REAL_PATH}/$rel_path", PRINT => "$parent->{PRINT}/$rel_path", ); foreach (qw( HOST @@ -295,12 +293,8 @@ sub vinfo_set_detail($$) # check and add detail data to vinfo hash foreach(keys %$detail) { if((defined $vol->{$_}) && ($vol->{$_} ne $detail->{$_})) { - if($_ eq "REAL_PATH") { - DEBUG "Subvolume REAL_PATH changed (symlink): old=\"$vol->{REAL_PATH}\", new=\"$detail->{REAL_PATH}\""; - } else { WARN "Subvolume detail key \"$_\" is already present, with a different value: old=\"$vol->{$_}\", new=\"$detail->{$_}\""; WARN "Using new value for \"$_\": $detail->{$_}"; - } } $vol->{$_} = $detail->{$_}; } From a7d3dac64d7ea96648fb4dff26764840ab674520 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Mon, 20 Apr 2015 18:53:44 +0200 Subject: [PATCH 13/31] btrbk: fixed summary of actions "run" and "dryrun" --- btrbk | 85 ++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 46 insertions(+), 39 deletions(-) diff --git a/btrbk b/btrbk index c1caaa7..80571d8 100755 --- a/btrbk +++ b/btrbk @@ -2041,7 +2041,51 @@ MAIN: # unless($quiet) { + my @out; my $err_count = 0; + foreach my $config_vol (@{$config->{VOLUME}}) + { + my $sroot = vinfo($config_vol->{url}); + foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) + { + my $svol = vinfo_child($sroot, $config_subvol->{rel_path}); # we need {PRINT}, even on errors + push @out, "$svol->{PRINT}"; + if($config_vol->{ABORTED}) { + push @out, "!!! $sroot->{PRINT}: ABORTED: $config_vol->{ABORTED}"; + $err_count++ unless($config_vol->{ABORTED_NOERR}); + } + if($config_subvol->{ABORTED}) { + push @out, "!!! Subvolume \"$svol->{PRINT}\" aborted: $config_subvol->{ABORTED}"; + $err_count++ unless($config_subvol->{ABORTED_NOERR}); + } + push @out, "+++ $config_subvol->{SNAPSHOT}->{PRINT}" if($config_subvol->{SNAPSHOT}); + if($config_subvol->{SUBVOL_DELETED}) { + push @out, "--- $_" foreach(sort { $b cmp $a} @{$config_subvol->{SUBVOL_DELETED}}); + } + foreach my $config_target (@{$config_subvol->{TARGET}}) + { + my $droot = vinfo($config_target->{url}); + foreach(@{$config_target->{SUBVOL_RECEIVED} // []}) { + my $create_mode = "***"; + $create_mode = ">>>" if($_->{parent}); + # substr($create_mode, 0, 1, '%') if($_->{resume}); + $create_mode = "!!!" if($_->{ERROR}); + push @out, "$create_mode $_->{received_name}"; + } + + if($config_target->{SUBVOL_DELETED}) { + push @out, "--- $_" foreach(sort { $b cmp $a} @{$config_target->{SUBVOL_DELETED}}); + } + + if($config_target->{ABORTED}) { + push @out, "!!! Target \"$droot->{PRINT}\" aborted: $config_target->{ABORTED}"; + $err_count++ unless($config_target->{ABORTED_NOERR}); + } + } + push @out, ""; + } + } + print "--------------------------------------------------------------------------------\n"; print "Backup Summary ($version_info)\n\n"; print " Date: " . localtime($start_time) . "\n"; @@ -2052,47 +2096,10 @@ MAIN: print " *** received subvolume (non-incremental)\n"; print " >>> received subvolume (incremental)\n"; # print " %>> received subvolume (incremental, resume_missing)\n"; - print "--------------------------------------------------------------------------------"; - foreach my $config_vol (@{$config->{VOLUME}}) - { - my $sroot = vinfo($config_vol->{url}); - if($config_vol->{ABORTED}) { - print "!!! $config_vol->{url}: ABORTED: $config_vol->{ABORTED}\n"; - $err_count++ unless($config_vol->{ABORTED_NOERR}); - } - foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) - { - my $svol = vinfo($sroot, $config_subvol->{rel_path}) || die; - print "\n$svol->{PRINT}\n"; - if($config_subvol->{ABORTED}) { - print "!!! Subvolume \"$config_subvol->{rel_path}\" aborted: $config_subvol->{ABORTED}\n"; - $err_count++ unless($config_subvol->{ABORTED_NOERR}); - } - print "+++ $config_subvol->{SNAPSHOT}->{PRINT}\n" if($config_subvol->{SNAPSHOT}); - if($config_subvol->{SUBVOL_DELETED}) { - print "--- $_\n" foreach(sort { $b cmp $a} @{$config_subvol->{SUBVOL_DELETED}}); - } - foreach my $config_target (@{$config_subvol->{TARGET}}) - { - foreach(@{$config_target->{SUBVOL_RECEIVED} // []}) { - my $create_mode = "***"; - $create_mode = ">>>" if($_->{parent}); - # substr($create_mode, 0, 1, '%') if($_->{resume}); - $create_mode = "!!!" if($_->{ERROR}); - print "$create_mode $_->{received_name}\n"; - } + print "--------------------------------------------------------------------------------\n"; - if($config_target->{SUBVOL_DELETED}) { - print "--- $_\n" foreach(sort { $b cmp $a} @{$config_target->{SUBVOL_DELETED}}); - } + print join("\n", @out); - if($config_target->{ABORTED}) { - print "!!! Target \"$config_target->{url}\" aborted: $config_target->{ABORTED}\n"; - $err_count++ unless($config_target->{ABORTED_NOERR}); - } - } - } - } if($err_count) { print "\nNOTE: Some errors occurred, which may result in missing backups!\n"; print "Please check warning and error messages above.\n"; From 2f9055634e3198f2eb1440dc0873a0267aa4d3e4 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Mon, 20 Apr 2015 20:35:13 +0200 Subject: [PATCH 14/31] btrbk: adapted btrfs_subvolume_delete() --- btrbk | 66 ++++++++++++++++++++++++++++++----------------------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/btrbk b/btrbk index 80571d8..2876061 100755 --- a/btrbk +++ b/btrbk @@ -972,27 +972,25 @@ sub btrfs_snapshot($$) sub btrfs_subvolume_delete($@) { - my $config = shift; - my @targets = @_; - return 0 unless(scalar(@targets)); - my @real_targets; - my $rsh; - foreach (@targets) { - my ($r, $t) = get_rsh($_, $config); - die if($rsh && ($rsh ne $r)); # make sure all targets share same ssh host - $rsh = $r; - push(@real_targets, $t); + my $targets = shift // die; + my %opts = @_; + my $commit = $opts{commit}; + die if($commit && ($commit ne "after") && ($commit ne "each")); + $targets = [ $targets ] unless(ref($targets)); + return 0 unless(scalar(@$targets)); + my $rsh = $targets->[0]->{RSH} || ""; + foreach (@$targets) { + # make sure all targets share same RSH + my $rsh_check = $_->{RSH} || ""; + die if($rsh ne $rsh_check); } - die if(scalar(@targets) != scalar(@real_targets)); - my $commit_delete = config_key($config, "btrfs_commit_delete") // ""; - DEBUG "[btrfs] delete" . ($commit_delete ? " (commit-$commit_delete):" : ":"); - DEBUG "[btrfs] subvolume: $_" foreach(@targets); + DEBUG "[btrfs] delete" . ($commit ? " (commit-$commit):" : ":"); + DEBUG "[btrfs] subvolume: $_->{PRINT}" foreach(@$targets); my $options = ""; - $options = "--commit-after " if($commit_delete eq "after"); - $options = "--commit-each " if($commit_delete eq "each"); - my $ret = run_cmd("$rsh /sbin/btrfs subvolume delete $options" . join(' ', @real_targets)); - ERROR "Failed to delete btrfs subvolumes: " . join(' ', @targets) unless(defined($ret)); - return defined($ret) ? scalar(@targets) : undef; + $options = "--commit-$commit " if($commit); + my $ret = run_cmd("$rsh /sbin/btrfs subvolume delete $options" . join(' ', map( { $_->{PATH} } @$targets))); + ERROR "Failed to delete btrfs subvolumes: " . join(' ', map( { $_->{URL} } @$targets)) unless(defined($ret)); + return defined($ret) ? scalar(@$targets) : undef; } @@ -1971,12 +1969,14 @@ MAIN: # # delete backups # - INFO "Cleaning backups of subvolume \"$svol->{URL}\": $droot->{URL}/$snapshot_basename.*"; + INFO "Cleaning backups of subvolume \"$svol->{PRINT}\": $droot->{PRINT}/$snapshot_basename.*"; my @schedule; - foreach my $vol (keys %{$droot->{SUBVOL_INFO}}) { - my ($date, $date_ext) = get_date_tag($vol); - next unless($date && ($vol =~ /^\Q$svol.\E/)); - push(@schedule, { value => "$droot->{URL}/$vol", name => $vol, date => $date, date_ext => $date_ext }); + foreach my $vol (values %{$droot->{SUBVOL_INFO}}) { + #!!! TODO: check received_from + next unless($vol->{SUBVOL_PATH} =~ /^\Q$snapshot_basename.\E/); + my ($date, $date_ext) = get_date_tag($vol->{NAME}); + next unless($date); + push(@schedule, { value => $vol, name => $vol->{PRINT}, date => $date, date_ext => $date_ext }); } my (undef, $delete) = schedule( schedule => \@schedule, @@ -1987,7 +1987,7 @@ MAIN: preserve_monthly => config_key($config_target, "target_preserve_monthly"), log_verbose => 1, ); - my $ret = btrfs_subvolume_delete($config_target, @$delete); + my $ret = btrfs_subvolume_delete($delete, commit => config_key($config_target, "btrfs_commit_delete")); if(defined($ret)) { INFO "Deleted $ret subvolumes in: $droot->{URL}/$snapshot_basename.*"; $config_target->{SUBVOL_DELETED} = $delete; @@ -2007,10 +2007,12 @@ MAIN: } INFO "Cleaning snapshots: $sroot->{URL}/$snapdir/$snapshot_basename.*"; my @schedule; - foreach my $vol (keys %{$sroot->{SUBVOL_INFO}}) { - my ($date, $date_ext) = get_date_tag($vol); - next unless($date && ($vol =~ /^\Q$snapdir\/$snapshot_basename.\E/)); - push(@schedule, { value => "$sroot->{URL}/$vol", name => $vol, date => $date, date_ext => $date_ext }); + foreach my $vol (values %{$sroot->{SUBVOL_INFO}}) { + # !!! TODO: skip symlinks + next unless($vol->{SUBVOL_PATH} =~ /^\Q$snapdir\/$snapshot_basename.\E/); + my ($date, $date_ext) = get_date_tag($vol->{NAME}); + next unless($date); + push(@schedule, { value => $vol, name => $vol->{PRINT}, date => $date, date_ext => $date_ext }); } my (undef, $delete) = schedule( schedule => \@schedule, @@ -2021,7 +2023,7 @@ MAIN: preserve_monthly => config_key($config_subvol, "snapshot_preserve_monthly"), log_verbose => 1, ); - my $ret = btrfs_subvolume_delete($config_subvol, @$delete); + my $ret = btrfs_subvolume_delete($delete, commit => config_key($config_subvol, "btrfs_commit_delete")); if(defined($ret)) { INFO "Deleted $ret subvolumes in: $sroot->{URL}/$snapdir/$snapshot_basename.*"; $config_subvol->{SUBVOL_DELETED} = $delete; @@ -2060,7 +2062,7 @@ MAIN: } push @out, "+++ $config_subvol->{SNAPSHOT}->{PRINT}" if($config_subvol->{SNAPSHOT}); if($config_subvol->{SUBVOL_DELETED}) { - push @out, "--- $_" foreach(sort { $b cmp $a} @{$config_subvol->{SUBVOL_DELETED}}); + push @out, "--- $_->{PRINT}" foreach(sort { $b->{PATH} cmp $a->{PATH} } @{$config_subvol->{SUBVOL_DELETED}}); } foreach my $config_target (@{$config_subvol->{TARGET}}) { @@ -2074,7 +2076,7 @@ MAIN: } if($config_target->{SUBVOL_DELETED}) { - push @out, "--- $_" foreach(sort { $b cmp $a} @{$config_target->{SUBVOL_DELETED}}); + push @out, "--- $_->{PRINT}" foreach(sort { $b->{PATH} cmp $a->{PATH} } @{$config_target->{SUBVOL_DELETED}}); } if($config_target->{ABORTED}) { From 6e4e531fbd7f48a49a79ce449b4978fa3a912bc3 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 21 Apr 2015 14:53:31 +0200 Subject: [PATCH 15/31] btrbk: changed vinfo creation and handling; cleanup btr_tree(); fixed action "diff" --- btrbk | 450 +++++++++++++++++++++++++--------------------------------- 1 file changed, 191 insertions(+), 259 deletions(-) diff --git a/btrbk b/btrbk index 2876061..e674070 100755 --- a/btrbk +++ b/btrbk @@ -81,10 +81,10 @@ my %config_options = ( my @config_target_types = qw(send-receive); -my %vinfo_root; -my %vinfo_cache; -my %uuid_info; -my %uuid_fs_map; +my %root_tree_cache; # map URL to SUBTREE (needed since "btrfs subvolume list" does not provide us with the uuid of the btrfs root node) +my %vinfo_cache; # map URL to vinfo +my %uuid_info; # map UUID to btr_tree node +my %uuid_fs_map; # map UUID to URL my $dryrun; my $loglevel = 1; @@ -167,41 +167,10 @@ sub run_cmd($;$) } -sub vinfo_update_lookup_tables($) -{ - my $vol = shift || die; - $vinfo_cache{$vol->{URL}} = $vol; - $vinfo_cache{$vol->{REAL_URL}} = $vol if($vol->{REAL_URL}); - $uuid_fs_map{$vol->{uuid}}->{$vol->{URL}} = $vol if($vol->{uuid}); - $uuid_fs_map{$vol->{uuid}}->{$vol->{REAL_URL}} = $vol if($vol->{uuid} && $vol->{REAL_URL}); -} - - -sub vinfo($;$) -{ - my $root = shift // die; # url or vinfo hash - my $subvol_path = shift; - unless(ref($root)) { - $root = $vinfo_root{$root} || die; - } - - if($subvol_path) { - if($root->{SUBVOL_INFO} && $root->{SUBVOL_INFO}->{$subvol_path}) { - return $root->{SUBVOL_INFO}->{$subvol_path}; - } - return undef; - } - else { - return $root; - } -} - - -sub vinfo_root($$) +sub vinfo(@) { my $url = shift // die; my $config = shift || die; - return $vinfo_root{$url} if($vinfo_root{$url}); my $name = $url; $name =~ s/^.*\///; @@ -246,15 +215,28 @@ sub vinfo_root($$) my $btrfs_progs_compat = config_key($config, "btrfs_progs_compat"); $info{BTRFS_PROGS_COMPAT} = $btrfs_progs_compat if($btrfs_progs_compat); - $vinfo_root{$url} = \%info; - vinfo_update_lookup_tables(\%info); - - TRACE "vinfo root created for: $url"; - TRACE(Data::Dumper->Dump([\%info], ["vinfo{$url}"])); + TRACE "vinfo created: $url"; return \%info; } +sub vinfo_root($$) +{ + my $vol = vinfo(@_); + + my $detail = btr_subvolume_detail($vol); + return undef unless $detail; + vinfo_set_detail($vol, $detail); + + # read (and cache) the subvolume list + return undef unless vinfo_subvol_list($vol); + + TRACE "vinfo root created: $vol->{URL}"; + + return $vol; +} + + sub vinfo_child($$) { my $parent = shift || die; @@ -267,6 +249,7 @@ sub vinfo_child($$) URL => "$parent->{URL}/$rel_path", PATH => "$parent->{PATH}/$rel_path", PRINT => "$parent->{PRINT}/$rel_path", + SUBVOL_PATH => $rel_path, ); foreach (qw( HOST RSH_TYPE @@ -277,10 +260,8 @@ sub vinfo_child($$) { $info{$_} = $parent->{$_} if(exists $parent->{$_}); } - vinfo_update_lookup_tables(\%info); - TRACE "vinfo child \"$rel_path\" created for: $info{URL}"; - TRACE(Data::Dumper->Dump([\%info], ["vinfo{$info{URL}}"])); + TRACE "vinfo child created from \"$parent->{URL}\": $info{URL}"; return \%info; } @@ -290,76 +271,33 @@ sub vinfo_set_detail($$) my $vol = shift || die; my $detail = shift || die; - # check and add detail data to vinfo hash + # add detail data to vinfo hash foreach(keys %$detail) { - if((defined $vol->{$_}) && ($vol->{$_} ne $detail->{$_})) { - WARN "Subvolume detail key \"$_\" is already present, with a different value: old=\"$vol->{$_}\", new=\"$detail->{$_}\""; - WARN "Using new value for \"$_\": $detail->{$_}"; - } + next if($_ eq "REL_PATH"); + next if($_ eq "TOP_LEVEL"); + next if($_ eq "SUBTREE"); + next if($_ eq "path"); $vol->{$_} = $detail->{$_}; } - if($vol->{RSH_TYPE} && ($vol->{RSH_TYPE} eq "ssh")) { - $vol->{REAL_URL} = "ssh://$vol->{HOST}$vol->{REAL_PATH}"; - } else { - $vol->{REAL_URL} = $vol->{REAL_PATH}; + if($vol->{REAL_PATH}) { + if($vol->{RSH_TYPE} && ($vol->{RSH_TYPE} eq "ssh")) { + $vol->{REAL_URL} = "ssh://$vol->{HOST}$detail->{REAL_PATH}"; + } else { + $vol->{REAL_URL} = $vol->{REAL_PATH}; + } } + # update cache + $vinfo_cache{$vol->{URL}} = $vol; + $vinfo_cache{$vol->{REAL_URL}} = $vol if($vol->{REAL_URL}); + TRACE "vinfo updated for: $vol->{URL}"; TRACE(Data::Dumper->Dump([$vol], ["vinfo{$vol->{URL}}"])); - vinfo_update_lookup_tables($vol); return $vol; } -sub vinfo_read_detail($) -{ - my $vol = shift || die; - return $vol if($vol->{VINFO_DETAIL_READ}); - - my $detail = btr_subvolume_detail($vol); - unless($detail) { - WARN "Failed to fetch subvolume detail for: $vol->{PRINT}"; - return undef; - } - $vol->{VINFO_DETAIL_READ} = 1; - return vinfo_set_detail($vol, $detail); -} - - -sub vinfo_add_child($$$) -{ - my $root = shift || die; - my $child = shift || die; - my $rel_path = shift // die; - die if($root->{SUBVOL_INFO}->{$rel_path}); - $root->{SUBVOL_INFO}->{$rel_path} = $child; - TRACE "vinfo child \"$rel_path\" added to: $root->{URL}"; -} - - -sub get_rsh($$) -{ - my $url = shift // die; - my $config = shift; - if($config && ($url =~ /^ssh:\/\/(\S+?)(\/\S+)$/)) { - my ($ssh_host, $path) = ($1, $2); - my $ssh_user = config_key($config, "ssh_user"); - my $ssh_identity = config_key($config, "ssh_identity"); - my $ssh_options = ""; - if($ssh_identity) { - $ssh_options .= " -i $ssh_identity"; - } - else { - WARN "No SSH identity provided (option ssh_identity is not set) for: $url"; - } - my $rsh = "/usr/bin/ssh $ssh_options " . $ssh_user . '@' . $ssh_host; - return ($rsh, $path); - } - return ("", $url); -} - - sub config_key($$) { my $node = shift || die; @@ -767,13 +705,13 @@ sub btr_subvolume_list($;@) sub btr_subvolume_find_new($$;$) { - my $url = shift || die; + my $vol = shift || die; + my $path = $vol->{PATH} // die; + my $rsh = $vol->{RSH} || ""; my $lastgen = shift // die; - my $config = shift; - my ($rsh, $real_vol) = get_rsh($url, $config); - my $ret = run_cmd("$rsh /sbin/btrfs subvolume find-new $real_vol $lastgen"); + my $ret = run_cmd("$rsh /sbin/btrfs subvolume find-new $path $lastgen"); unless(defined($ret)) { - ERROR "Failed to fetch modified files for: $url"; + ERROR "Failed to fetch modified files for: $vol->{PRINT}"; return undef; } @@ -825,9 +763,19 @@ sub btr_subvolume_find_new($$;$) sub btr_tree($) { my $vol = shift; - my %tree; - my %id; - my $subvol_list = btr_subvolume_list($vol, subvol_only => 0); + + # return cached info if present + return $root_tree_cache{$vol->{URL}} if($vol->{is_root} && $root_tree_cache{$vol->{URL}}); + return $root_tree_cache{$vol->{REAL_URL}} if($vol->{is_root} && $vol->{REAL_URL} && $root_tree_cache{$vol->{REAL_URL}}); + return $uuid_info{$vol->{uuid}} if($vol->{uuid} && $uuid_info{$vol->{uuid}}); + + # man btrfs-subvolume: + # Also every btrfs filesystem has a default subvolume as its initially + # top-level subvolume, whose subvolume id is 5(FS_TREE). + my %tree = ( id => 5, SUBTREE => {} ); + my %id = ( 5 => \%tree ); + + my $subvol_list = btr_subvolume_list($vol); return undef unless(ref($subvol_list) eq "ARRAY"); TRACE "btr_tree: processing subvolume list of: $vol->{URL}"; @@ -837,94 +785,66 @@ sub btr_tree($) $id{$node->{id}} = $node; $uuid_info{$node->{uuid}} = $node; + $node->{SUBTREE} //= {}; + + # set SUBTREE / TOP_LEVEL node + die unless exists($id{$node->{top_level}}); + my $top_level = $id{$node->{top_level}}; + + die if exists($top_level->{SUBTREE}->{$node->{id}}); + $top_level->{SUBTREE}->{$node->{id}} = $node; + $node->{TOP_LEVEL} = $top_level; + + # "path" always starts with set REL_PATH my $rel_path = $node->{path}; - if($node->{top_level} == 5) - { - # man btrfs-subvolume: - # Also every btrfs filesystem has a default subvolume as its initially - # top-level subvolume, whose subvolume id is 5(FS_TREE). - - $tree{$node->{id}} = $node; + if($node->{top_level} != 5) { + die unless($rel_path =~ s/^$top_level->{path}\///); } - else - { - # set SUBTREE / PARENT node - die unless exists($id{$node->{top_level}}); - my $parent = $id{$node->{top_level}}; - die if exists($parent->{SUBTREE}->{$node->{id}}); - $parent->{SUBTREE}->{$node->{id}} = $node; - $node->{PARENT} = $parent; - - # "path" always starts with set REL_PATH - die unless($rel_path =~ s/^$parent->{path}\///); - } - $node->{REL_PATH} = $rel_path; # relative to {PARENT}->{path} + $node->{REL_PATH} = $rel_path; # relative to {TOP_LEVEL}->{path} } - # set PARENT node - foreach (values %id){ - $_->{PARENT} = $uuid_info{$_->{parent_uuid}} if($_->{parent_uuid} ne "-"); + if($vol->{is_root}) { + $root_tree_cache{$vol->{URL}} = \%tree; + $root_tree_cache{$vol->{REAL_URL}} = \%tree if($vol->{REAL_URL}); + return \%tree; + } + else { + die unless($uuid_info{$vol->{uuid}}); + return $uuid_info{$vol->{uuid}}; } - return \%tree; } sub _subtree_list { my $tree = shift; - my $list = shift; - my $prefix = shift; - - return $list unless $tree; # silent ignore empty subtrees + my $list = shift // []; + my $prefix = shift // ""; + $tree = $tree->{SUBTREE}; foreach(values %$tree) { my $path = $prefix . $_->{REL_PATH}; push(@$list, { SUBVOL_PATH => $path, node => $_, }); - # recurse into SUBTREE - _subtree_list($_->{SUBTREE}, $list, $path . '/'); + _subtree_list($_, $list, $path . '/'); } return $list; } - -# returns hash of: -# SUBVOL_PATH relative path to URL -# URL absolute path -# node href to tree node -# -# returns an empty hash if the subvolume at $url exists, but contains no subvolumes -# returns undef if the subvolume at $url does not exists -sub vinfo_read_subvolumes($) +sub vinfo_subvol_list($) { my $vol = shift || die; - return $vol if($vol->{VINFO_SUBVOLUMES_READ}); + return $vol->{SUBVOL_LIST} if($vol->{SUBVOL_LIST}); - my $url = $vol->{URL} || die; + my $tree_root = btr_tree($vol); + return undef unless($tree_root); - my $tree = btr_tree($vol); # populates %uuid_info - return undef unless($tree); - - my $tree_root; - if($vol->{is_root}) { - $tree_root = $tree; - } - else { - die unless $uuid_info{$vol->{uuid}}; - $tree_root = $uuid_info{$vol->{uuid}}->{SUBTREE}; - unless($tree_root) { - DEBUG "No subvolumes found in: $vol->{PRINT}"; - $vol->{SUBVOL_INFO} //= {}; - return $vol; - } - } - - # recurse into $tree_root, returns list of href: { URL, node } - my $list = _subtree_list($tree_root, [], ""); + # recurse into $tree_root, returns list of href: { SUBVOL_PATH, node } + my $list = _subtree_list($tree_root); # return a hash of relative subvolume path my %ret; @@ -932,26 +852,46 @@ sub vinfo_read_subvolumes($) my $subvol_path = $_->{SUBVOL_PATH}; die if exists $ret{$subvol_path}; - my $detail = { %{$_->{node}}, - SUBVOL_PATH => $_->{SUBVOL_PATH}, - }; - delete $detail->{REL_PATH}; - delete $detail->{PARENT}; - my $subvol = vinfo_child($vol, "$subvol_path"); - vinfo_set_detail($subvol, $detail); - vinfo_add_child($vol, $subvol, $subvol_path); + my $subvol = vinfo_child($vol, $subvol_path); + vinfo_set_detail($subvol, $_->{node}); + + $uuid_fs_map{$subvol->{uuid}}->{$subvol->{URL}} = $subvol; $ret{$subvol_path} = $subvol; } - DEBUG "Added " . scalar(keys %ret) . " subvolume children to: $vol->{PRINT}"; - $vol->{VINFO_SUBVOLUMES_READ} = 1; + DEBUG "Found " . scalar(keys %ret) . " subvolume children of: $vol->{PRINT}"; + TRACE(Data::Dumper->Dump([\%ret], ["SUBVOL_LIST{$vol->{URL}}"])); + $vol->{SUBVOL_LIST} = \%ret; - TRACE(Data::Dumper->Dump([\%ret], ["SUBVOL_INFO{$vol->{URL}}"])); return \%ret; } +# returns list of uuids for ALL subvolumes in the btrfs filesystem of $vol +sub vinfo_fs_list($) +{ + my $vol = shift || die; + my $tree_root = btr_tree($vol); + return undef unless($tree_root); + + $tree_root = $tree_root->{TOP_LEVEL} while($tree_root->{TOP_LEVEL}); + my $list = _subtree_list($tree_root); + my %ret = map { $_->{node}->{uuid} => $_->{node} } @$list; + return \%ret; +} + + +sub vinfo_subvol($$) +{ + my $vol = shift || die; + my $rel_path = shift // die; + + my $subvols = vinfo_subvol_list($vol); + return $subvols->{$rel_path}; +} + + # returns $target, or undef on error sub btrfs_snapshot($$) { @@ -1092,10 +1032,12 @@ sub get_date_tag($) sub get_snapshot_children($$) { - my $sroot = shift || die; # TODO: this should be second argument, as we return all snap children from svol under sroot + my $sroot = shift || die; my $svol = shift // die; my @ret; - foreach (values %{$sroot->{SUBVOL_INFO}}) { + + my $sroot_subvols = vinfo_subvol_list($sroot); + foreach (values %$sroot_subvols) { next unless($_->{parent_uuid} eq $svol->{uuid}); TRACE "get_snapshot_children: found: $_->{URL}"; push(@ret, $_); @@ -1109,7 +1051,7 @@ sub get_receive_targets($$) { my $droot = shift || die; my $src_vol = shift || die; - die("root subvolume info not present: $droot") unless($droot->{SUBVOL_INFO}); + my $droot_subvols = vinfo_subvol_list($droot); my @ret; if($droot->{BTRFS_PROGS_COMPAT}) @@ -1118,7 +1060,7 @@ sub get_receive_targets($$) DEBUG "Fallback to compatibility mode (get_receive_targets)"; my $src_name = $src_vol->{SUBVOL_PATH}; $src_name =~ s/^.*\///; # strip path - foreach my $target (values %{$droot->{SUBVOL_INFO}}) { + foreach my $target (values %$droot_subvols) { my $target_name = $target->{SUBVOL_PATH}; $target_name =~ s/^.*\///; # strip path if($target_name eq $src_name) { @@ -1132,7 +1074,7 @@ sub get_receive_targets($$) # find matches by comparing uuid / received_uuid my $uuid = $src_vol->{uuid}; die("subvolume info not present: $uuid") unless($uuid_info{$uuid}); - foreach (values %{$droot->{SUBVOL_INFO}}) { + foreach (values %$droot_subvols) { next unless($_->{received_uuid} eq $uuid); TRACE "get_receive_targets: by-uuid: Found receive target: $_->{SUBVOL_PATH}"; push(@ret, $_); @@ -1395,29 +1337,28 @@ MAIN: my $target_url = $subvol_args[1] || die; # FIXME: allow ssh:// src/dest (does not work since the configuration is not yet read). - my $src_vol = vinfo_create($src_url, {}); + my $src_vol = vinfo_root($src_url, { CONTEXT => "cmdline" }); unless($src_vol) { exit 1; } if($src_vol->{is_root}) { ERROR "subvolume at \"$src_url\" is btrfs root!"; exit 1; } unless($src_vol->{cgen}) { ERROR "subvolume at \"$src_url\" does not provide cgen"; exit 1; } -# if($src_vol->{parent_uuid} eq "-") { ERROR "subvolume at \"$src_url\" has no parent, aborting."; exit 1; } - my $target_vol = vinfo_create($target_url, {}); + my $target_vol = vinfo_root($target_url, { CONTEXT => "cmdline" }); unless($target_vol) { exit 1; } unless($src_vol->{cgen}) { ERROR "subvolume at \"$src_url\" does not provide cgen"; exit 1; } -# if($src_vol->{parent_uuid} eq "-") { ERROR "subvolume at \"$src_url\" has no parent, aborting."; exit 1; } - my $info = btr_tree($src_url); - my $src = $uuid_info{$src_vol->{uuid}} || die; - my $target = $uuid_info{$target_vol->{uuid}}; - unless($target) { ERROR "target subvolume is not on the same btrfs filesystem!"; exit 1; } + my $uuid_list = vinfo_fs_list($src_vol); + unless($uuid_list->{$target_vol->{uuid}}) { + ERROR "target subvolume is not on the same btrfs filesystem!"; + exit 1; + } my $lastgen; # check if given src and target share same parent - if(ref($src->{PARENT}) && ($src->{PARENT}->{uuid} eq $target->{uuid})) { + if($src_vol->{parent_uuid} eq $target_vol->{uuid}) { DEBUG "target subvolume is direct parent of source subvolume"; } - elsif(ref($src->{PARENT}) && ref($target->{PARENT}) && ($src->{PARENT}->{uuid} eq $target->{PARENT}->{uuid})) { + elsif($src_vol->{parent_uuid} eq $target_vol->{parent_uuid}) { DEBUG "target subvolume and source subvolume share same parent"; } else { @@ -1427,16 +1368,16 @@ MAIN: } # NOTE: in some cases "cgen" differs from "gen", even for read-only snapshots (observed: gen=cgen+1) - $lastgen = $src->{cgen} + 1; + $lastgen = $src_vol->{cgen} + 1; # dump files, sorted and unique - my $ret = btr_subvolume_find_new($target_url, $lastgen); + my $ret = btr_subvolume_find_new($target_vol, $lastgen); exit 1 unless(ref($ret)); print "--------------------------------------------------------------------------------\n"; - print "Showing changed files for subvolume:\n $target->{path} (gen=$target->{gen})\n"; - print "\nStarting at creation generation from subvolume:\n $src->{path} (cgen=$src->{cgen})\n"; - print "\nThis will show all files modified within generation range: [$lastgen..$target->{gen}]\n"; + print "Showing changed files for subvolume:\n $target_vol->{PRINT} (gen=$target_vol->{gen})\n"; + print "\nStarting at creation generation of subvolume:\n $src_vol->{PRINT} (cgen=$src_vol->{cgen})\n"; + print "\nThis will show all files modified within generation range: [$lastgen..$target_vol->{gen}]\n"; print "Newest file generation (transid marker) was: $ret->{transid_marker}\n"; print "Parse errors: $ret->{parse_errors}\n" if($ret->{parse_errors}); print "\nLegend: \n"; @@ -1512,7 +1453,7 @@ MAIN: print "\n--------------------------------------------------------------------------------\n"; print "Source volume: $url\n"; print "--------------------------------------------------------------------------------\n"; - print (btr_filesystem_usage(vinfo_create($url, $config_vol)) // ""); + print (btr_filesystem_usage(vinfo($url, $config_vol)) // ""); print "\n"; $processed{$url} = 1; } @@ -1530,7 +1471,7 @@ MAIN: print "Target volume: $droot_url\n"; print " ^--- $sroot_url\n"; print "--------------------------------------------------------------------------------\n"; - print (btr_filesystem_usage(vinfo_create($droot_url, $config_target)) // ""); + print (btr_filesystem_usage(vinfo($droot_url, $config_target)) // ""); print "\n"; $processed{$droot_url} = 1; } @@ -1585,34 +1526,43 @@ MAIN: { next if($config_vol->{ABORTED}); my $sroot = vinfo_root($config_vol->{url}, $config_vol); - unless(vinfo_read_detail($sroot)) { + unless($sroot) { $config_vol->{ABORTED} = "Failed to fetch subvolume detail"; - WARN "Skipping volume \"$sroot->{URL}\": $config_vol->{ABORTED}"; - next; - } - unless(vinfo_read_subvolumes($sroot)) { - $config_vol->{ABORTED} = "Failed to fetch subvolume list"; - WARN "Skipping volume \"$sroot->{URL}\": $config_vol->{ABORTED}"; + WARN "Skipping volume \"$config_vol->{url}\": $config_vol->{ABORTED}"; next; } + $config_vol->{sroot} = $sroot; foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { next if($config_subvol->{ABORTED}); - my $svol = vinfo($sroot, $config_subvol->{rel_path}); + my $svol = vinfo_subvol($sroot, $config_subvol->{rel_path}); unless($svol) { # configured subvolume is not present in btrfs subvolume list. # try to read subvolume detail, as configured subvolume could be a symlink. DEBUG "Subvolume \"$config_subvol->{rel_path}\" not present in btrfs subvolume list for \"$sroot->{PRINT}\""; $svol = vinfo_child($sroot, $config_subvol->{rel_path}); - unless(vinfo_read_detail($svol)) { + my $detail = btr_subvolume_detail($svol); + unless($detail) { $config_subvol->{ABORTED} = "Failed to fetch subvolume detail"; WARN "Skipping subvolume \"$svol->{URL}\": $config_subvol->{ABORTED}"; next; } - vinfo_add_child($sroot, $svol, $config_subvol->{rel_path}); + if($detail->{is_root}) { + $config_subvol->{ABORTED} = "Subvolume is btrfs root"; + WARN "Skipping subvolume \"$svol->{URL}\": $config_subvol->{ABORTED}"; + next; + } + if(grep { $_->{uuid} eq $detail->{uuid} } values %{vinfo_subvol_list($sroot)}) { + vinfo_set_detail($svol, $uuid_info{$detail->{uuid}}); + } else { + $config_subvol->{ABORTED} = "Not a child subvolume of: $sroot->{PRINT}"; + WARN "Skipping subvolume \"$svol->{URL}\": $config_subvol->{ABORTED}"; + next; + } } + $config_subvol->{svol} = $svol; # set default for snapshot_name $config_subvol->{snapshot_name} //= $svol->{NAME}; @@ -1631,9 +1581,9 @@ MAIN: foreach my $config_target (@{$config_subvol->{TARGET}}) { my $droot = vinfo_root($config_target->{url}, $config_target); - unless(vinfo_read_detail($droot)) { + unless($droot) { $config_target->{ABORTED} = "Failed to fetch subvolume detail"; - WARN "Skipping target \"$droot->{URL}\": $config_target->{ABORTED}"; + WARN "Skipping target \"$config_target->{url}\": $config_target->{ABORTED}"; next; } @@ -1646,11 +1596,7 @@ MAIN: } $backup_check{$snapshot_backup_target} = $svol->{PRINT}; - unless(vinfo_read_subvolumes($droot)) { - $config_target->{ABORTED} = "Failed to fetch subvolume list"; - WARN "Skipping target \"$droot->{URL}\": $config_target->{ABORTED}"; - next; - } + $config_target->{droot} = $droot; } } } @@ -1669,20 +1615,16 @@ MAIN: # specified volume is not in config DEBUG "Subvolume not parsed yet, fetching info: $url"; $vol = vinfo_root($url, { CONTEXT => "cmdline" }); - unless(vinfo_read_detail($vol)) { + unless($vol) { ERROR "Failed to fetch subvolume detail: $url"; exit 1; } - unless(vinfo_read_subvolumes($vol)) { - ERROR "Failed to fetch subvolume list: $url"; - exit 1; - } } - if($vol->{is_root}) { ERROR "Subvolume is btrfs root: $url\n"; exit 1; } + my $lines = []; _origin_tree("", $vol->{uuid}, $lines); @@ -1716,25 +1658,18 @@ MAIN: foreach my $config_vol (@{$config->{VOLUME}}) { my %droot_compat; - my $sroot = vinfo($config_vol->{url}); + my $sroot = $config_vol->{sroot} || die; push @out, "$sroot->{PRINT}"; foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { - my $svol = vinfo($sroot, $config_subvol->{rel_path}) || die; + my $svol = $config_subvol->{svol} || die; push @out, "|-- $svol->{PRINT}"; - unless(vinfo($sroot, $config_subvol->{rel_path})) { # !!! TODO: maybe check uuid here? - push @out, " !!! error: no subvolume \"$config_subvol->{rel_path}\" found in \"$sroot->{PRINT}\""; - next; - } - foreach my $snapshot (sort { $a->{PATH} cmp $b->{PATH} } (values %{$sroot->{SUBVOL_INFO}})) + foreach my $snapshot (sort { $a->{PATH} cmp $b->{PATH} } get_snapshot_children($sroot, $svol)) { - next unless($snapshot->{parent_uuid} eq $svol->{uuid}); - # next unless($snapshot->{SUBVOL_PATH} =~ /^$snapdir/); # don't print non-btrbk snapshots push @out, "| ^-- $snapshot->{PATH}"; foreach my $config_target (@{$config_subvol->{TARGET}}) { - my $droot = vinfo($config_target->{url}); - next unless $droot->{SUBVOL_INFO}; + my $droot = $config_target->{droot} || die; $droot_compat{$droot->{URL}} = 1 if($droot->{BTRFS_PROGS_COMPAT}); foreach (sort { $a->{SUBVOL_PATH} cmp $b->{SUBVOL_PATH} } get_receive_targets($droot, $snapshot)) { push @out, "| | ^== $_->{PRINT}"; @@ -1771,11 +1706,11 @@ MAIN: foreach my $config_vol (@{$config->{VOLUME}}) { next if($config_vol->{ABORTED}); - my $sroot = vinfo($config_vol->{url}); + my $sroot = $config_vol->{sroot} || die; foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { next if($config_subvol->{ABORTED}); - my $svol = vinfo($sroot, $config_subvol->{rel_path}) || die; + my $svol = $config_subvol->{svol} || die; my $snapdir = config_key($config_subvol, "snapshot_dir") || ""; my $snapshot_basename = config_key($config_subvol, "snapshot_name") // die; @@ -1792,11 +1727,11 @@ MAIN: } # find unique snapshot name - my @lookup = keys %{$sroot->{SUBVOL_INFO}}; + my @lookup = keys %{vinfo_subvol_list($sroot)}; @lookup = grep s/^\Q$snapdir\E\/// , @lookup; foreach my $config_target (@{$config_subvol->{TARGET}}) { - my $droot = vinfo($config_target->{url}); - push(@lookup, keys %{$droot->{SUBVOL_INFO}}); + my $droot = $config_target->{droot} || die; + push(@lookup, keys %{vinfo_subvol_list($droot)}); } @lookup = grep /^\Q$snapshot_basename.$timestamp\E(_[0-9]+)?$/ ,@lookup; TRACE "Present snapshot names for \"$svol->{URL}\": " . join(', ', @lookup); @@ -1824,18 +1759,18 @@ MAIN: foreach my $config_vol (@{$config->{VOLUME}}) { next if($config_vol->{ABORTED}); - my $sroot = vinfo($config_vol->{url}); + my $sroot = $config_vol->{sroot} || die; foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { next if($config_subvol->{ABORTED}); - my $svol = vinfo($sroot, $config_subvol->{rel_path}) || die; + my $svol = $config_subvol->{svol} || die; my $snapdir = config_key($config_subvol, "snapshot_dir") || ""; my $snapshot_basename = config_key($config_subvol, "snapshot_name") // die; foreach my $config_target (@{$config_subvol->{TARGET}}) { next if($config_target->{ABORTED}); - my $droot = vinfo($config_target->{url}); + my $droot = $config_target->{droot} || die; my $target_type = $config_target->{target_type} || die; if($target_type eq "send-receive") @@ -1872,7 +1807,7 @@ MAIN: DEBUG "Checking schedule for resume candidates"; # add all present backups to schedule, with no value # these are needed for correct results of schedule() - foreach my $vol (keys %{$droot->{SUBVOL_INFO}}) { + foreach my $vol (keys %{vinfo_subvol_list($droot)}) { my ($date, $date_ext) = get_date_tag($vol); next unless($date && ($vol =~ s/^\Q$snapshot_basename.\E//)); # use only the date suffix for sorting push(@schedule, { value => undef, date => $date, date_ext => $date_ext }); @@ -1892,7 +1827,7 @@ MAIN: INFO "Resuming subvolume backup (send-receive) for: $child->{URL}"; $found_missing++; my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot, $child->{gen}); - if(macro_send_receive($config_target, # TODO: !!! adapt this function + if(macro_send_receive($config_target, snapshot => $child, target => $droot, parent => $latest_common_src, # this is if no common found @@ -1950,11 +1885,11 @@ MAIN: foreach my $config_vol (@{$config->{VOLUME}}) { next if($config_vol->{ABORTED}); - my $sroot = vinfo($config_vol->{url}); + my $sroot = $config_vol->{sroot} || die; foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { next if($config_subvol->{ABORTED}); - my $svol = vinfo($sroot, $config_subvol->{rel_path}) || die; + my $svol = $config_subvol->{svol} || die; my $snapdir = config_key($config_subvol, "snapshot_dir") || ""; my $snapshot_basename = config_key($config_subvol, "snapshot_name") // die; my $target_aborted = 0; @@ -1964,14 +1899,14 @@ MAIN: $target_aborted = 1; next; } - my $droot = vinfo($config_target->{url}); + my $droot = $config_target->{droot} || die; # # delete backups # INFO "Cleaning backups of subvolume \"$svol->{PRINT}\": $droot->{PRINT}/$snapshot_basename.*"; my @schedule; - foreach my $vol (values %{$droot->{SUBVOL_INFO}}) { + foreach my $vol (values %{vinfo_subvol_list($droot)}) { #!!! TODO: check received_from next unless($vol->{SUBVOL_PATH} =~ /^\Q$snapshot_basename.\E/); my ($date, $date_ext) = get_date_tag($vol->{NAME}); @@ -2007,8 +1942,7 @@ MAIN: } INFO "Cleaning snapshots: $sroot->{URL}/$snapdir/$snapshot_basename.*"; my @schedule; - foreach my $vol (values %{$sroot->{SUBVOL_INFO}}) { - # !!! TODO: skip symlinks + foreach my $vol (values %{vinfo_subvol_list($sroot)}) { next unless($vol->{SUBVOL_PATH} =~ /^\Q$snapdir\/$snapshot_basename.\E/); my ($date, $date_ext) = get_date_tag($vol->{NAME}); next unless($date); @@ -2047,17 +1981,15 @@ MAIN: my $err_count = 0; foreach my $config_vol (@{$config->{VOLUME}}) { - my $sroot = vinfo($config_vol->{url}); foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { - my $svol = vinfo_child($sroot, $config_subvol->{rel_path}); # we need {PRINT}, even on errors - push @out, "$svol->{PRINT}"; + push @out, "$config_subvol->{url}"; if($config_vol->{ABORTED}) { - push @out, "!!! $sroot->{PRINT}: ABORTED: $config_vol->{ABORTED}"; + push @out, "!!! $config_vol->{url}: ABORTED: $config_vol->{ABORTED}"; $err_count++ unless($config_vol->{ABORTED_NOERR}); } if($config_subvol->{ABORTED}) { - push @out, "!!! Subvolume \"$svol->{PRINT}\" aborted: $config_subvol->{ABORTED}"; + push @out, "!!! Subvolume \"$config_subvol->{url}\" aborted: $config_subvol->{ABORTED}"; $err_count++ unless($config_subvol->{ABORTED_NOERR}); } push @out, "+++ $config_subvol->{SNAPSHOT}->{PRINT}" if($config_subvol->{SNAPSHOT}); @@ -2066,7 +1998,7 @@ MAIN: } foreach my $config_target (@{$config_subvol->{TARGET}}) { - my $droot = vinfo($config_target->{url}); + my $droot = $config_target->{droot}; foreach(@{$config_target->{SUBVOL_RECEIVED} // []}) { my $create_mode = "***"; $create_mode = ">>>" if($_->{parent}); From 927b80a3889c343f59b05ef3e625d5447148ac04 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Thu, 23 Apr 2015 14:36:25 +0200 Subject: [PATCH 16/31] btrbk: correctly match snapshots --- btrbk | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/btrbk b/btrbk index e674070..0e4b3d1 100755 --- a/btrbk +++ b/btrbk @@ -93,6 +93,7 @@ my $ip_addr_match = qr/(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([ my $host_name_match = qr/(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])/; my $file_match = qr/[0-9a-zA-Z_@\-\.\/]+/; # note: ubuntu uses '@' in the subvolume layout: my $ssh_prefix_match = qr/ssh:\/\/($ip_addr_match|$host_name_match)/; +my $snapshot_postfix_match = qr/\.[0-9]{8}(_[0-9]+)?/; $SIG{__DIE__} = sub { @@ -1797,7 +1798,7 @@ MAIN: # check if the target would be preserved my ($date, $date_ext) = get_date_tag($child->{SUBVOL_PATH}); - next unless($date && ($child->{SUBVOL_PATH} =~ /^\Q$snapdir\/$snapshot_basename.\E/)); + next unless($date && ($child->{SUBVOL_PATH} =~ /^\Q$snapdir\/$snapshot_basename\E$snapshot_postfix_match$/)); push(@schedule, { value => $child, date => $date, date_ext => $date_ext }), } } @@ -1907,8 +1908,12 @@ MAIN: INFO "Cleaning backups of subvolume \"$svol->{PRINT}\": $droot->{PRINT}/$snapshot_basename.*"; my @schedule; foreach my $vol (values %{vinfo_subvol_list($droot)}) { - #!!! TODO: check received_from - next unless($vol->{SUBVOL_PATH} =~ /^\Q$snapshot_basename.\E/); + next unless($vol->{SUBVOL_PATH} =~ /^\Q$snapshot_basename\E$snapshot_postfix_match$/); + # NOTE: checking received_uuid does not make much sense, as this received_uuid is propagated to snapshots + # if($vol->{received_uuid} && ($vol->{received_uuid} eq '-')) { + # INFO "Target subvolume is not a received backup, skipping deletion of: $vol->{PRINT}"; + # next; + # } my ($date, $date_ext) = get_date_tag($vol->{NAME}); next unless($date); push(@schedule, { value => $vol, name => $vol->{PRINT}, date => $date, date_ext => $date_ext }); @@ -1943,7 +1948,7 @@ MAIN: INFO "Cleaning snapshots: $sroot->{URL}/$snapdir/$snapshot_basename.*"; my @schedule; foreach my $vol (values %{vinfo_subvol_list($sroot)}) { - next unless($vol->{SUBVOL_PATH} =~ /^\Q$snapdir\/$snapshot_basename.\E/); + next unless($vol->{SUBVOL_PATH} =~ /^\Q$snapdir\/$snapshot_basename\E$snapshot_postfix_match$/); my ($date, $date_ext) = get_date_tag($vol->{NAME}); next unless($date); push(@schedule, { value => $vol, name => $vol->{PRINT}, date => $date, date_ext => $date_ext }); From ea59d986d659b516384d6f9b1d066e6e3eb013f9 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Thu, 23 Apr 2015 15:04:28 +0200 Subject: [PATCH 17/31] btrbk: always use {PRINT} instead of {URL} for logging --- btrbk | 110 ++++++++++++++++++++++++++++------------------------------ 1 file changed, 54 insertions(+), 56 deletions(-) diff --git a/btrbk b/btrbk index 0e4b3d1..6377fdc 100755 --- a/btrbk +++ b/btrbk @@ -195,7 +195,7 @@ sub vinfo(@) %info, HOST => $host, PATH => $path, - PRINT => "$host:$path", + PRINT => "{$host}$path", RSH_TYPE => "ssh", SSH_USER => $ssh_user, SSH_IDENTITY => $ssh_identity, @@ -232,7 +232,7 @@ sub vinfo_root($$) # read (and cache) the subvolume list return undef unless vinfo_subvol_list($vol); - TRACE "vinfo root created: $vol->{URL}"; + TRACE "vinfo root created: $vol->{PRINT}"; return $vol; } @@ -262,7 +262,7 @@ sub vinfo_child($$) $info{$_} = $parent->{$_} if(exists $parent->{$_}); } - TRACE "vinfo child created from \"$parent->{URL}\": $info{URL}"; + TRACE "vinfo child created from \"$parent->{PRINT}\": $info{PRINT}"; return \%info; } @@ -293,8 +293,8 @@ sub vinfo_set_detail($$) $vinfo_cache{$vol->{URL}} = $vol; $vinfo_cache{$vol->{REAL_URL}} = $vol if($vol->{REAL_URL}); - TRACE "vinfo updated for: $vol->{URL}"; - TRACE(Data::Dumper->Dump([$vol], ["vinfo{$vol->{URL}}"])); + TRACE "vinfo updated for: $vol->{PRINT}"; + TRACE(Data::Dumper->Dump([$vol], ["vinfo{$vol->{PRINT}}"])); return $vol; } @@ -380,7 +380,7 @@ sub parse_config(@) $root->{$_} = $config_options{$_}->{default}; } - DEBUG "config: parsing file: $file"; + INFO "Using configuration: $file"; open(FILE, '<', $file) or die $!; while () { chomp; @@ -396,13 +396,13 @@ sub parse_config(@) if($key eq "volume") { $cur = $root; - DEBUG "config: context forced to: $cur->{CONTEXT}"; + TRACE "config: context forced to: $cur->{CONTEXT}"; # be very strict about file options, for security sake return undef unless(check_file($value, { absolute => 1, ssh => 1 }, $key, $file)); $value =~ s/\/+$//; # remove trailing slash $value =~ s/^\/+/\//; # sanitize leading slash - DEBUG "config: adding volume \"$value\" to root context"; + TRACE "config: adding volume \"$value\" to root context"; my $volume = { CONTEXT => "volume", PARENT => $cur, url => $value, @@ -419,7 +419,7 @@ sub parse_config(@) return undef; } $cur = $cur->{PARENT} || die; - DEBUG "config: context changed to: $cur->{CONTEXT}"; + TRACE "config: context changed to: $cur->{CONTEXT}"; } # be very strict about file options, for security sake return undef unless(check_file($value, { relative => 1 }, $key, $file)); @@ -430,7 +430,7 @@ sub parse_config(@) return undef; } - DEBUG "config: adding subvolume \"$value\" to volume context: $cur->{url}"; + TRACE "config: adding subvolume \"$value\" to volume context: $cur->{url}"; my $subvolume = { CONTEXT => "subvolume", PARENT => $cur, rel_path => $value, @@ -444,7 +444,7 @@ sub parse_config(@) { if($cur->{CONTEXT} eq "target") { $cur = $cur->{PARENT} || die; - DEBUG "config: context changed to: $cur->{CONTEXT}"; + TRACE "config: context changed to: $cur->{CONTEXT}"; } if($cur->{CONTEXT} ne "subvolume") { ERROR "Target keyword outside subvolume context, in \"$file\" line $."; @@ -462,7 +462,7 @@ sub parse_config(@) $droot =~ s/\/+$//; # remove trailing slash $droot =~ s/^\/+/\//; # sanitize leading slash - DEBUG "config: adding target \"$droot\" (type=$target_type) to subvolume context: $cur->{url}"; + TRACE "config: adding target \"$droot\" (type=$target_type) to subvolume context: $cur->{url}"; my $target = { CONTEXT => "target", PARENT => $cur, target_type => $target_type, @@ -516,7 +516,7 @@ sub parse_config(@) return undef; } - DEBUG "config: adding option \"$key=$value\" to $cur->{CONTEXT} context"; + TRACE "config: adding option \"$key=$value\" to $cur->{CONTEXT} context"; $value = undef if($value eq "no"); # we don't want to check for "no" all the time $cur->{$key} = $value; @@ -539,7 +539,7 @@ sub parse_config(@) } } - TRACE(Data::Dumper->Dump([$root], ["config_root"])); + TRACE(Data::Dumper->Dump([$root], ["config{$file}"])); return $root; } @@ -585,29 +585,28 @@ sub btr_subvolume_detail($) my $vol = shift || die; my $path = $vol->{PATH} // die; my $rsh = $vol->{RSH} || ""; - my $vol_print = $vol->{PRINT} || $path; # used only for logging my $ret = run_cmd("$rsh /sbin/btrfs subvolume show $path 2>/dev/null", 1); if($ret) { my $real_path; if($ret =~ /^($file_match)/) { $real_path = $1; - DEBUG "Real path for subvolume \"$vol_print\" is: $real_path" if($real_path ne $path); + DEBUG "Real path for subvolume \"$vol->{PRINT}\" is: $real_path" if($real_path ne $path); return undef unless(check_file($real_path, { absolute => 1 })); } else { $real_path = $path; - WARN "No real path provided by \"btrfs subvolume show\" for subvolume \"$vol_print\", using: $path"; + WARN "No real path provided by \"btrfs subvolume show\" for subvolume \"$vol->{PRINT}\", using: $path"; } my %detail = ( REAL_PATH => $real_path ); if($ret eq "$real_path is btrfs root") { - DEBUG "found btrfs root: $vol_print"; + DEBUG "found btrfs root: $vol->{PRINT}"; $detail{id} = 5; $detail{is_root} = 1; } elsif($ret =~ /^$real_path/) { - TRACE "btr_detail: found btrfs subvolume: $vol_print"; + TRACE "btr_detail: found btrfs subvolume: $vol->{PRINT}"; my %trans = ( name => "Name", uuid => "uuid", @@ -627,8 +626,8 @@ sub btr_subvolume_detail($) WARN "Failed to parse subvolume detail \"$trans{$_}\": $ret"; } } - DEBUG "Parsed " . scalar(keys %detail) . " subvolume detail items: $vol_print"; - TRACE "btr_detail for $vol_print: " . Dumper \%detail; + DEBUG "Parsed " . scalar(keys %detail) . " subvolume detail items: $vol->{PRINT}"; + TRACE(Data::Dumper->Dump([$vol], ["btr_subvolume_detail($vol->{URL})"])); } return \%detail; } @@ -642,7 +641,6 @@ sub btr_subvolume_list($;@) my %opts = @_; my $path = $vol->{PATH} // die; my $rsh = $vol->{RSH} || ""; - my $vol_print = $vol->{PRINT} || $path; # used only for logging my $btrfs_progs_compat = $vol->{BTRFS_PROGS_COMPAT} || $opts{btrfs_progs_compat}; my $filter_option = "-a"; $filter_option = "-o" if($opts{subvol_only}); @@ -650,7 +648,7 @@ sub btr_subvolume_list($;@) $display_options .= " -R" unless($btrfs_progs_compat); my $ret = run_cmd("$rsh /sbin/btrfs subvolume list $filter_option $display_options $path", 1); unless(defined($ret)) { - WARN "Failed to fetch btrfs subvolume list for: $vol_print"; + WARN "Failed to fetch btrfs subvolume list for: $vol->{PRINT}"; return undef; } my @nodes; @@ -699,7 +697,7 @@ sub btr_subvolume_list($;@) push @nodes, \%node; } - DEBUG "Parsed " . scalar(@nodes) . " total subvolumes for filesystem at: $vol_print"; + DEBUG "Parsed " . scalar(@nodes) . " total subvolumes for filesystem at: $vol->{PRINT}"; return \@nodes; } @@ -779,7 +777,7 @@ sub btr_tree($) my $subvol_list = btr_subvolume_list($vol); return undef unless(ref($subvol_list) eq "ARRAY"); - TRACE "btr_tree: processing subvolume list of: $vol->{URL}"; + TRACE "btr_tree: processing subvolume list of: $vol->{PRINT}"; foreach my $node (@$subvol_list) { @@ -862,9 +860,9 @@ sub vinfo_subvol_list($) } DEBUG "Found " . scalar(keys %ret) . " subvolume children of: $vol->{PRINT}"; - TRACE(Data::Dumper->Dump([\%ret], ["SUBVOL_LIST{$vol->{URL}}"])); - $vol->{SUBVOL_LIST} = \%ret; + TRACE(Data::Dumper->Dump([\%ret], ["vinfo_subvol_list{$vol->{URL}}"])); + $vol->{SUBVOL_LIST} = \%ret; return \%ret; } @@ -1040,10 +1038,10 @@ sub get_snapshot_children($$) my $sroot_subvols = vinfo_subvol_list($sroot); foreach (values %$sroot_subvols) { next unless($_->{parent_uuid} eq $svol->{uuid}); - TRACE "get_snapshot_children: found: $_->{URL}"; + TRACE "get_snapshot_children: found: $_->{PRINT}"; push(@ret, $_); } - DEBUG "Found " . scalar(@ret) . " snapshot children of: $svol->{URL}"; + DEBUG "Found " . scalar(@ret) . " snapshot children of: $svol->{PRINT}"; return @ret; } @@ -1081,7 +1079,7 @@ sub get_receive_targets($$) push(@ret, $_); } } - DEBUG "Found " . scalar(@ret) . " receive targets in \"$droot->{URL}/\" for: $src_vol->{URL}"; + DEBUG "Found " . scalar(@ret) . " receive targets in \"$droot->{PRINT}/\" for: $src_vol->{PRINT}"; return @ret; } @@ -1109,18 +1107,18 @@ sub get_latest_common($$$;$) if($child->{RECEIVE_TARGET_PRESENT} && ($child->{RECEIVE_TARGET_PRESENT} eq $droot->{URL})) { # little hack to keep track of previously received subvolumes - DEBUG("Latest common snapshots for: $debug_src: src=$child->{URL} target="); + DEBUG("Latest common snapshots for: $debug_src: src=$child->{PRINT} target="); return ($child, undef); } foreach (get_receive_targets($droot, $child)) { - TRACE "get_latest_common: found receive target: $_->{URL}"; - DEBUG("Latest common snapshots for: $debug_src: src=$child->{URL} target=$_->{URL}"); + TRACE "get_latest_common: found receive target: $_->{PRINT}"; + DEBUG("Latest common snapshots for: $debug_src: src=$child->{PRINT} target=$_->{PRINT}"); return ($child, $_); } - TRACE "get_latest_common: no matching targets found for: $child->{URL}"; + TRACE "get_latest_common: no matching targets found for: $child->{PRINT}"; } - DEBUG("No common snapshots for \"$debug_src\" found in src=\"$sroot->{URL}/\", target=\"$droot->{URL}/\""); + DEBUG("No common snapshots of \"$debug_src\" found in src=\"$sroot->{PRINT}/\", target=\"$droot->{PRINT}/\""); return (undef, undef); } @@ -1340,16 +1338,16 @@ MAIN: my $src_vol = vinfo_root($src_url, { CONTEXT => "cmdline" }); unless($src_vol) { exit 1; } - if($src_vol->{is_root}) { ERROR "subvolume at \"$src_url\" is btrfs root!"; exit 1; } - unless($src_vol->{cgen}) { ERROR "subvolume at \"$src_url\" does not provide cgen"; exit 1; } + if($src_vol->{is_root}) { ERROR "Subvolume at \"$src_url\" is btrfs root!"; exit 1; } + unless($src_vol->{cgen}) { ERROR "Subvolume at \"$src_url\" does not provide cgen"; exit 1; } my $target_vol = vinfo_root($target_url, { CONTEXT => "cmdline" }); unless($target_vol) { exit 1; } - unless($src_vol->{cgen}) { ERROR "subvolume at \"$src_url\" does not provide cgen"; exit 1; } + unless($src_vol->{cgen}) { ERROR "Subvolume at \"$src_url\" does not provide cgen"; exit 1; } my $uuid_list = vinfo_fs_list($src_vol); unless($uuid_list->{$target_vol->{uuid}}) { - ERROR "target subvolume is not on the same btrfs filesystem!"; + ERROR "Target subvolume is not on the same btrfs filesystem!"; exit 1; } @@ -1364,7 +1362,7 @@ MAIN: } else { # TODO: this rule only applies to snapshots. find a way to distinguish snapshots from received backups - # ERROR "subvolumes \"$target_url\" and \"$src_url\" do not share the same parents"; + # ERROR "Subvolumes \"$target_url\" and \"$src_url\" do not share the same parents"; # exit 1; } @@ -1547,19 +1545,19 @@ MAIN: my $detail = btr_subvolume_detail($svol); unless($detail) { $config_subvol->{ABORTED} = "Failed to fetch subvolume detail"; - WARN "Skipping subvolume \"$svol->{URL}\": $config_subvol->{ABORTED}"; + WARN "Skipping subvolume \"$svol->{PRINT}\": $config_subvol->{ABORTED}"; next; } if($detail->{is_root}) { $config_subvol->{ABORTED} = "Subvolume is btrfs root"; - WARN "Skipping subvolume \"$svol->{URL}\": $config_subvol->{ABORTED}"; + WARN "Skipping subvolume \"$svol->{PRINT}\": $config_subvol->{ABORTED}"; next; } if(grep { $_->{uuid} eq $detail->{uuid} } values %{vinfo_subvol_list($sroot)}) { vinfo_set_detail($svol, $uuid_info{$detail->{uuid}}); } else { $config_subvol->{ABORTED} = "Not a child subvolume of: $sroot->{PRINT}"; - WARN "Skipping subvolume \"$svol->{URL}\": $config_subvol->{ABORTED}"; + WARN "Skipping subvolume \"$svol->{PRINT}\": $config_subvol->{ABORTED}"; next; } } @@ -1722,7 +1720,7 @@ MAIN: $create_snapshot = 1 if($config_target->{target_type} eq "send-receive"); } unless($create_snapshot) { - $config_subvol->{ABORTED} = "No targets defined for subvolume: $svol->{URL}"; + $config_subvol->{ABORTED} = "No targets defined for subvolume: $svol->{PRINT}"; WARN "Skipping subvolume section: $config_subvol->{ABORTED}"; next; } @@ -1735,7 +1733,7 @@ MAIN: push(@lookup, keys %{vinfo_subvol_list($droot)}); } @lookup = grep /^\Q$snapshot_basename.$timestamp\E(_[0-9]+)?$/ ,@lookup; - TRACE "Present snapshot names for \"$svol->{URL}\": " . join(', ', @lookup); + TRACE "Present snapshot names for \"$svol->{PRINT}\": " . join(', ', @lookup); @lookup = map { /_([0-9]+)$/ ? $1 : 0 } @lookup; @lookup = sort { $b <=> $a } @lookup; my $postfix_counter = $lookup[0] // -1; @@ -1777,13 +1775,13 @@ MAIN: if($target_type eq "send-receive") { if(config_key($config_target, "receive_log")) { - WARN "Ignoring deprecated option \"receive_log\" for target: $droot->{URL}" + WARN "Ignoring deprecated option \"receive_log\" for target: $droot->{PRINT}" } # resume missing backups (resume_missing) if(config_key($config_target, "resume_missing")) { - INFO "Checking for missing backups of subvolume \"$svol->{URL}\" in: $droot->{URL}/"; + INFO "Checking for missing backups of subvolume \"$svol->{PRINT}\" in: $droot->{PRINT}/"; my @schedule; my $found_missing = 0; @@ -1791,10 +1789,10 @@ MAIN: foreach my $child (get_snapshot_children($sroot, $svol)) { if(scalar get_receive_targets($droot, $child)) { - DEBUG "Found matching receive target, skipping: $child->{URL}"; + DEBUG "Found matching receive target, skipping: $child->{PRINT}"; } else { - DEBUG "No matching receive targets found, adding resume candidate: $child->{URL}"; + DEBUG "No matching receive targets found, adding resume candidate: $child->{PRINT}"; # check if the target would be preserved my ($date, $date_ext) = get_date_tag($child->{SUBVOL_PATH}); @@ -1825,7 +1823,7 @@ MAIN: my @resume = grep defined, @$preserve; # remove entries with no value from list (target subvolumes) foreach my $child (sort { $a->{gen} <=> $b->{gen} } @resume) { - INFO "Resuming subvolume backup (send-receive) for: $child->{URL}"; + INFO "Resuming subvolume backup (send-receive) for: $child->{PRINT}"; $found_missing++; my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot, $child->{gen}); if(macro_send_receive($config_target, @@ -1858,7 +1856,7 @@ MAIN: die unless($config_subvol->{SNAPSHOT}); # finally receive the previously created snapshot - INFO "Creating subvolume backup (send-receive) for: $svol->{URL}"; + INFO "Creating subvolume backup (send-receive) for: $svol->{PRINT}"; my ($latest_common_src, $latest_common_target) = get_latest_common($sroot, $svol, $droot); macro_send_receive($config_target, snapshot => $config_subvol->{SNAPSHOT}, @@ -1867,7 +1865,7 @@ MAIN: ); } else { - ERROR "Unknown target type \"$target_type\", skipping: $svol->{URL}"; + ERROR "Unknown target type \"$target_type\", skipping: $svol->{PRINT}"; $config_target->{ABORTED} = "Unknown target type \"$target_type\""; } } @@ -1929,7 +1927,7 @@ MAIN: ); my $ret = btrfs_subvolume_delete($delete, commit => config_key($config_target, "btrfs_commit_delete")); if(defined($ret)) { - INFO "Deleted $ret subvolumes in: $droot->{URL}/$snapshot_basename.*"; + INFO "Deleted $ret subvolumes in: $droot->{PRINT}/$snapshot_basename.*"; $config_target->{SUBVOL_DELETED} = $delete; } else { @@ -1942,10 +1940,10 @@ MAIN: # delete snapshots # if($target_aborted) { - WARN "Skipping cleanup of snapshots for subvolume \"$svol->{URL}\", as at least one target aborted earlier"; + WARN "Skipping cleanup of snapshots for subvolume \"$svol->{PRINT}\", as at least one target aborted earlier"; next; } - INFO "Cleaning snapshots: $sroot->{URL}/$snapdir/$snapshot_basename.*"; + INFO "Cleaning snapshots: $sroot->{PRINT}/$snapdir/$snapshot_basename.*"; my @schedule; foreach my $vol (values %{vinfo_subvol_list($sroot)}) { next unless($vol->{SUBVOL_PATH} =~ /^\Q$snapdir\/$snapshot_basename\E$snapshot_postfix_match$/); @@ -1964,7 +1962,7 @@ MAIN: ); my $ret = btrfs_subvolume_delete($delete, commit => config_key($config_subvol, "btrfs_commit_delete")); if(defined($ret)) { - INFO "Deleted $ret subvolumes in: $sroot->{URL}/$snapdir/$snapshot_basename.*"; + INFO "Deleted $ret subvolumes in: $sroot->{PRINT}/$snapdir/$snapshot_basename.*"; $config_subvol->{SUBVOL_DELETED} = $delete; } else { From 8a99adf53fe184a01582ef5624f929f89a1f4042 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Thu, 23 Apr 2015 15:30:33 +0200 Subject: [PATCH 18/31] btrbk: adaptions on vinfo_root: make sure a valid vinfo is in $config on each section --- btrbk | 98 +++++++++++++++++++++++++++++------------------------------ 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/btrbk b/btrbk index 6377fdc..a9dbbde 100755 --- a/btrbk +++ b/btrbk @@ -168,7 +168,7 @@ sub run_cmd($;$) } -sub vinfo(@) +sub vinfo($$) { my $url = shift // die; my $config = shift || die; @@ -221,23 +221,6 @@ sub vinfo(@) } -sub vinfo_root($$) -{ - my $vol = vinfo(@_); - - my $detail = btr_subvolume_detail($vol); - return undef unless $detail; - vinfo_set_detail($vol, $detail); - - # read (and cache) the subvolume list - return undef unless vinfo_subvol_list($vol); - - TRACE "vinfo root created: $vol->{PRINT}"; - - return $vol; -} - - sub vinfo_child($$) { my $parent = shift || die; @@ -267,6 +250,22 @@ sub vinfo_child($$) } +sub vinfo_root($) +{ + my $vol = shift; + + my $detail = btr_subvolume_detail($vol); + return undef unless $detail; + vinfo_set_detail($vol, $detail); + + # read (and cache) the subvolume list + return undef unless vinfo_subvol_list($vol); + + TRACE "vinfo root created: $vol->{PRINT}"; + return $vol; +} + + sub vinfo_set_detail($$) { my $vol = shift || die; @@ -1336,13 +1335,13 @@ MAIN: my $target_url = $subvol_args[1] || die; # FIXME: allow ssh:// src/dest (does not work since the configuration is not yet read). - my $src_vol = vinfo_root($src_url, { CONTEXT => "cmdline" }); - unless($src_vol) { exit 1; } + my $src_vol = vinfo($src_url, { CONTEXT => "cmdline" }); + unless(vinfo_root($src_vol)) { ERROR "Failed to fetch subvolume detail for: $src_vol->{PRINT}"; exit 1; } if($src_vol->{is_root}) { ERROR "Subvolume at \"$src_url\" is btrfs root!"; exit 1; } unless($src_vol->{cgen}) { ERROR "Subvolume at \"$src_url\" does not provide cgen"; exit 1; } - my $target_vol = vinfo_root($target_url, { CONTEXT => "cmdline" }); - unless($target_vol) { exit 1; } + my $target_vol = vinfo($target_url, { CONTEXT => "cmdline" }); + unless(vinfo_root($target_vol)) { ERROR "Failed to fetch subvolume detail for: $src_vol->{PRINT}"; exit 1; } unless($src_vol->{cgen}) { ERROR "Subvolume at \"$src_url\" does not provide cgen"; exit 1; } my $uuid_list = vinfo_fs_list($src_vol); @@ -1446,33 +1445,33 @@ MAIN: my %processed; foreach my $config_vol (@{$config->{VOLUME}}) { - my $url = $config_vol->{url} || die; - unless($processed{$url}) + my $sroot = vinfo($config_vol->{url}, $config_vol); + unless($processed{$sroot->{URL}}) { print "\n--------------------------------------------------------------------------------\n"; - print "Source volume: $url\n"; + print "Source volume: $sroot->{PRINT}\n"; print "--------------------------------------------------------------------------------\n"; - print (btr_filesystem_usage(vinfo($url, $config_vol)) // ""); + print (btr_filesystem_usage($sroot) // ""); print "\n"; - $processed{$url} = 1; + $processed{$sroot->{URL}} = 1; } } foreach my $config_vol (@{$config->{VOLUME}}) { - my $sroot_url = $config_vol->{url} || die; + my $sroot = vinfo($config_vol->{url}, $config_vol); foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { foreach my $config_target (@{$config_subvol->{TARGET}}) { - my $droot_url = $config_target->{url} || die; - unless($processed{$droot_url}) + my $droot = vinfo($config_target->{url}, $config_target); + unless($processed{$droot->{URL}}) { print "\n--------------------------------------------------------------------------------\n"; - print "Target volume: $droot_url\n"; - print " ^--- $sroot_url\n"; + print "Target volume: $droot->{PRINT}\n"; + print " ^--- $sroot->{PRINT}\n"; print "--------------------------------------------------------------------------------\n"; - print (btr_filesystem_usage(vinfo($droot_url, $config_target)) // ""); + print (btr_filesystem_usage($droot) // ""); print "\n"; - $processed{$droot_url} = 1; + $processed{$droot->{URL}} = 1; } } } @@ -1524,10 +1523,10 @@ MAIN: foreach my $config_vol (@{$config->{VOLUME}}) { next if($config_vol->{ABORTED}); - my $sroot = vinfo_root($config_vol->{url}, $config_vol); - unless($sroot) { + my $sroot = vinfo($config_vol->{url}, $config_vol); + unless(vinfo_root($sroot)) { $config_vol->{ABORTED} = "Failed to fetch subvolume detail"; - WARN "Skipping volume \"$config_vol->{url}\": $config_vol->{ABORTED}"; + WARN "Skipping volume \"$sroot->{PRINT}\": $config_vol->{ABORTED}"; next; } $config_vol->{sroot} = $sroot; @@ -1579,12 +1578,13 @@ MAIN: foreach my $config_target (@{$config_subvol->{TARGET}}) { - my $droot = vinfo_root($config_target->{url}, $config_target); - unless($droot) { + my $droot = vinfo($config_target->{url}, $config_target); + unless(vinfo_root($droot)) { $config_target->{ABORTED} = "Failed to fetch subvolume detail"; - WARN "Skipping target \"$config_target->{url}\": $config_target->{ABORTED}"; + WARN "Skipping target \"$droot->{PRINT}\": $config_target->{ABORTED}"; next; } + $config_target->{droot} = $droot; # check for duplicate snapshot locations my $snapshot_backup_target = "$droot->{REAL_URL}/$snapshot_basename"; @@ -1594,8 +1594,6 @@ MAIN: exit 1; } $backup_check{$snapshot_backup_target} = $svol->{PRINT}; - - $config_target->{droot} = $droot; } } } @@ -1613,9 +1611,9 @@ MAIN: unless($vol) { # specified volume is not in config DEBUG "Subvolume not parsed yet, fetching info: $url"; - $vol = vinfo_root($url, { CONTEXT => "cmdline" }); - unless($vol) { - ERROR "Failed to fetch subvolume detail: $url"; + $vol = vinfo($url, { CONTEXT => "cmdline" }); + unless(vinfo_root($vol)) { + ERROR "Failed to fetch subvolume detail for: $url"; exit 1; } } @@ -1984,15 +1982,17 @@ MAIN: my $err_count = 0; foreach my $config_vol (@{$config->{VOLUME}}) { + my $sroot = $config_vol->{sroot} || vinfo($config_vol->{url}, $config_vol); foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { - push @out, "$config_subvol->{url}"; + my $svol = $config_subvol->{svol} || vinfo_child($sroot, $config_subvol->{rel_path}); + push @out, "$svol->{PRINT}"; if($config_vol->{ABORTED}) { - push @out, "!!! $config_vol->{url}: ABORTED: $config_vol->{ABORTED}"; + push @out, "!!! $sroot->{PRINT}: ABORTED: $config_vol->{ABORTED}"; $err_count++ unless($config_vol->{ABORTED_NOERR}); } if($config_subvol->{ABORTED}) { - push @out, "!!! Subvolume \"$config_subvol->{url}\" aborted: $config_subvol->{ABORTED}"; + push @out, "!!! Subvolume \"$svol->{PRINT}\" aborted: $config_subvol->{ABORTED}"; $err_count++ unless($config_subvol->{ABORTED_NOERR}); } push @out, "+++ $config_subvol->{SNAPSHOT}->{PRINT}" if($config_subvol->{SNAPSHOT}); @@ -2001,7 +2001,7 @@ MAIN: } foreach my $config_target (@{$config_subvol->{TARGET}}) { - my $droot = $config_target->{droot}; + my $droot = $config_target->{droot} || vinfo($config_target->{url}, $config_target); foreach(@{$config_target->{SUBVOL_RECEIVED} // []}) { my $create_mode = "***"; $create_mode = ">>>" if($_->{parent}); From 466e066029e906e0e3f970bdcd0207a546397557 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Thu, 23 Apr 2015 16:19:34 +0200 Subject: [PATCH 19/31] btrbk: consistently prefix all functions calling "/sbin/btrfs" with btrfs_ --- btrbk | 272 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 134 insertions(+), 138 deletions(-) diff --git a/btrbk b/btrbk index a9dbbde..d3442b8 100755 --- a/btrbk +++ b/btrbk @@ -254,7 +254,7 @@ sub vinfo_root($) { my $vol = shift; - my $detail = btr_subvolume_detail($vol); + my $detail = btrfs_subvolume_detail($vol); return undef unless $detail; vinfo_set_detail($vol, $detail); @@ -543,13 +543,13 @@ sub parse_config(@) } -sub btr_filesystem_show_all_local() +sub btrfs_filesystem_show_all_local() { return run_cmd("/sbin/btrfs filesystem show", 1); } -sub btr_filesystem_show($) +sub btrfs_filesystem_show($) { my $vol = shift || die; my $path = $vol->{PATH} // die; @@ -559,7 +559,7 @@ sub btr_filesystem_show($) } -sub btr_filesystem_df($) +sub btrfs_filesystem_df($) { my $vol = shift || die; my $path = $vol->{PATH} // die; @@ -569,7 +569,7 @@ sub btr_filesystem_df($) } -sub btr_filesystem_usage($) +sub btrfs_filesystem_usage($) { my $vol = shift || die; my $path = $vol->{PATH} // die; @@ -579,62 +579,60 @@ sub btr_filesystem_usage($) } -sub btr_subvolume_detail($) +sub btrfs_subvolume_detail($) { my $vol = shift || die; my $path = $vol->{PATH} // die; my $rsh = $vol->{RSH} || ""; my $ret = run_cmd("$rsh /sbin/btrfs subvolume show $path 2>/dev/null", 1); - if($ret) - { - my $real_path; - if($ret =~ /^($file_match)/) { - $real_path = $1; - DEBUG "Real path for subvolume \"$vol->{PRINT}\" is: $real_path" if($real_path ne $path); - return undef unless(check_file($real_path, { absolute => 1 })); - } - else { - $real_path = $path; - WARN "No real path provided by \"btrfs subvolume show\" for subvolume \"$vol->{PRINT}\", using: $path"; - } - my %detail = ( REAL_PATH => $real_path ); + return undef unless(defined($ret)); - if($ret eq "$real_path is btrfs root") { - DEBUG "found btrfs root: $vol->{PRINT}"; - $detail{id} = 5; - $detail{is_root} = 1; - } - elsif($ret =~ /^$real_path/) { - TRACE "btr_detail: found btrfs subvolume: $vol->{PRINT}"; - my %trans = ( - name => "Name", - uuid => "uuid", - parent_uuid => "Parent uuid", - creation_time => "Creation time", - id => "Object ID", - gen => "Generation \\(Gen\\)", - cgen => "Gen at creation", - parent_id => "Parent", - top_level => "Top Level", - flags => "Flags", - ); - foreach (keys %trans) { - if($ret =~ /^\s+$trans{$_}:\s+(.*)$/m) { - $detail{$_} = $1; - } else { - WARN "Failed to parse subvolume detail \"$trans{$_}\": $ret"; - } - } - DEBUG "Parsed " . scalar(keys %detail) . " subvolume detail items: $vol->{PRINT}"; - TRACE(Data::Dumper->Dump([$vol], ["btr_subvolume_detail($vol->{URL})"])); - } - return \%detail; + my $real_path; + if($ret =~ /^($file_match)/) { + $real_path = $1; + DEBUG "Real path for subvolume \"$vol->{PRINT}\" is: $real_path" if($real_path ne $path); + return undef unless(check_file($real_path, { absolute => 1 })); } - return undef; + else { + $real_path = $path; + WARN "No real path provided by \"btrfs subvolume show\" for subvolume \"$vol->{PRINT}\", using: $path"; + } + my %detail = ( REAL_PATH => $real_path ); + + if($ret eq "$real_path is btrfs root") { + DEBUG "found btrfs root: $vol->{PRINT}"; + $detail{id} = 5; + $detail{is_root} = 1; + } + elsif($ret =~ /^$real_path/) { + TRACE "btr_detail: found btrfs subvolume: $vol->{PRINT}"; + my %trans = ( + name => "Name", + uuid => "uuid", + parent_uuid => "Parent uuid", + creation_time => "Creation time", + id => "Object ID", + gen => "Generation \\(Gen\\)", + cgen => "Gen at creation", + parent_id => "Parent", + top_level => "Top Level", + flags => "Flags", + ); + foreach (keys %trans) { + if($ret =~ /^\s+$trans{$_}:\s+(.*)$/m) { + $detail{$_} = $1; + } else { + WARN "Failed to parse subvolume detail \"$trans{$_}\": $ret"; + } + } + DEBUG "Parsed " . scalar(keys %detail) . " subvolume detail items: $vol->{PRINT}"; + TRACE(Data::Dumper->Dump([$vol], ["btrfs_subvolume_detail($vol->{URL})"])); + } + return \%detail; } -sub btr_subvolume_list($;@) +sub btrfs_subvolume_list($;@) { my $vol = shift || die; my %opts = @_; @@ -646,10 +644,8 @@ sub btr_subvolume_list($;@) my $display_options = "-c -u -q"; $display_options .= " -R" unless($btrfs_progs_compat); my $ret = run_cmd("$rsh /sbin/btrfs subvolume list $filter_option $display_options $path", 1); - unless(defined($ret)) { - WARN "Failed to fetch btrfs subvolume list for: $vol->{PRINT}"; - return undef; - } + return undef unless(defined($ret)); + my @nodes; foreach (split(/\n/, $ret)) { @@ -701,7 +697,7 @@ sub btr_subvolume_list($;@) } -sub btr_subvolume_find_new($$;$) +sub btrfs_subvolume_find_new($$;$) { my $vol = shift || die; my $path = $vol->{PATH} // die; @@ -758,6 +754,82 @@ sub btr_subvolume_find_new($$;$) } +# returns $target, or undef on error +sub btrfs_subvolume_snapshot($$) +{ + my $svol = shift || die; + my $target_path = shift // die; + my $src_path = $svol->{PATH} // die; + my $rsh = $svol->{RSH} || ""; + DEBUG "[btrfs] snapshot (ro):"; + DEBUG "[btrfs] host : $svol->{HOST}" if($svol->{HOST}); + DEBUG "[btrfs] source: $src_path"; + DEBUG "[btrfs] target: $target_path"; + INFO ">>> " . ($svol->{HOST} ? "$svol->{HOST}:" : "") . $target_path; + my $ret = run_cmd("$rsh /sbin/btrfs subvolume snapshot -r $src_path $target_path"); + ERROR "Failed to create btrfs subvolume snapshot: $svol->{PRINT} -> $target_path" unless(defined($ret)); + return defined($ret) ? $target_path : undef; +} + + +sub btrfs_subvolume_delete($@) +{ + my $targets = shift // die; + my %opts = @_; + my $commit = $opts{commit}; + die if($commit && ($commit ne "after") && ($commit ne "each")); + $targets = [ $targets ] unless(ref($targets)); + return 0 unless(scalar(@$targets)); + my $rsh = $targets->[0]->{RSH} || ""; + foreach (@$targets) { + # make sure all targets share same RSH + my $rsh_check = $_->{RSH} || ""; + die if($rsh ne $rsh_check); + } + DEBUG "[btrfs] delete" . ($commit ? " (commit-$commit):" : ":"); + DEBUG "[btrfs] subvolume: $_->{PRINT}" foreach(@$targets); + my $options = ""; + $options = "--commit-$commit " if($commit); + my $ret = run_cmd("$rsh /sbin/btrfs subvolume delete $options" . join(' ', map( { $_->{PATH} } @$targets))); + ERROR "Failed to delete btrfs subvolumes: " . join(' ', map( { $_->{PRINT} } @$targets)) unless(defined($ret)); + return defined($ret) ? scalar(@$targets) : undef; +} + + +sub btrfs_send_receive($$$) +{ + my $snapshot = shift || die; + my $target = shift || die; + my $parent = shift; + my $snapshot_path = $snapshot->{PATH} // die; + my $snapshot_rsh = $snapshot->{RSH} || ""; + my $target_path = $target->{PATH} // die; + my $target_rsh = $target->{RSH} || ""; + my $parent_path = $parent ? $parent->{PATH} : undef; + + my $snapshot_name = $snapshot_path; + $snapshot_name =~ s/^.*\///; + INFO ">>> $target->{PRINT}/$snapshot_name"; + + DEBUG "[btrfs] send/receive" . ($parent ? " (incremental)" : " (complete)") . ":"; + DEBUG "[btrfs] source: $snapshot->{PRINT}"; + DEBUG "[btrfs] parent: $parent->{PRINT}" if($parent); + DEBUG "[btrfs] target: $target->{PRINT}"; + + my $parent_option = $parent_path ? "-p $parent_path" : ""; + my $receive_option = ""; + $receive_option = "-v" if($loglevel >= 3); + + my $cmd = "$snapshot_rsh /sbin/btrfs send $parent_option $snapshot_path | $target_rsh /sbin/btrfs receive $receive_option $target_path/"; + my $ret = run_cmd($cmd); + unless(defined($ret)) { + ERROR "Failed to send/receive btrfs subvolume: $snapshot->{PRINT} " . ($parent_path ? "[$parent_path]" : "") . " -> $target->{PRINT}"; + return undef; + } + return 1; +} + + sub btr_tree($) { my $vol = shift; @@ -773,7 +845,7 @@ sub btr_tree($) my %tree = ( id => 5, SUBTREE => {} ); my %id = ( 5 => \%tree ); - my $subvol_list = btr_subvolume_list($vol); + my $subvol_list = btrfs_subvolume_list($vol); return undef unless(ref($subvol_list) eq "ARRAY"); TRACE "btr_tree: processing subvolume list of: $vol->{PRINT}"; @@ -890,82 +962,6 @@ sub vinfo_subvol($$) } -# returns $target, or undef on error -sub btrfs_snapshot($$) -{ - my $svol = shift || die; - my $target_path = shift // die; - my $src_path = $svol->{PATH} // die; - my $rsh = $svol->{RSH} || ""; - DEBUG "[btrfs] snapshot (ro):"; - DEBUG "[btrfs] host : $svol->{HOST}" if($svol->{HOST}); - DEBUG "[btrfs] source: $src_path"; - DEBUG "[btrfs] target: $target_path"; - INFO ">>> " . ($svol->{HOST} ? "$svol->{HOST}:" : "") . $target_path; - my $ret = run_cmd("$rsh /sbin/btrfs subvolume snapshot -r $src_path $target_path"); - ERROR "Failed to create btrfs subvolume snapshot: $svol->{PRINT} -> $target_path" unless(defined($ret)); - return defined($ret) ? $target_path : undef; -} - - -sub btrfs_subvolume_delete($@) -{ - my $targets = shift // die; - my %opts = @_; - my $commit = $opts{commit}; - die if($commit && ($commit ne "after") && ($commit ne "each")); - $targets = [ $targets ] unless(ref($targets)); - return 0 unless(scalar(@$targets)); - my $rsh = $targets->[0]->{RSH} || ""; - foreach (@$targets) { - # make sure all targets share same RSH - my $rsh_check = $_->{RSH} || ""; - die if($rsh ne $rsh_check); - } - DEBUG "[btrfs] delete" . ($commit ? " (commit-$commit):" : ":"); - DEBUG "[btrfs] subvolume: $_->{PRINT}" foreach(@$targets); - my $options = ""; - $options = "--commit-$commit " if($commit); - my $ret = run_cmd("$rsh /sbin/btrfs subvolume delete $options" . join(' ', map( { $_->{PATH} } @$targets))); - ERROR "Failed to delete btrfs subvolumes: " . join(' ', map( { $_->{URL} } @$targets)) unless(defined($ret)); - return defined($ret) ? scalar(@$targets) : undef; -} - - -sub btrfs_send_receive($$$) -{ - my $snapshot = shift || die; - my $target = shift || die; - my $parent = shift; - my $snapshot_path = $snapshot->{PATH} // die; - my $snapshot_rsh = $snapshot->{RSH} || ""; - my $target_path = $target->{PATH} // die; - my $target_rsh = $target->{RSH} || ""; - my $parent_path = $parent ? $parent->{PATH} : undef; - - my $snapshot_name = $snapshot_path; - $snapshot_name =~ s/^.*\///; - INFO ">>> $target->{PRINT}/$snapshot_name"; - - DEBUG "[btrfs] send/receive" . ($parent ? " (incremental)" : " (complete)") . ":"; - DEBUG "[btrfs] source: $snapshot->{PRINT}"; - DEBUG "[btrfs] parent: $parent->{PRINT}" if($parent); - DEBUG "[btrfs] target: $target->{PRINT}"; - - my $parent_option = $parent_path ? "-p $parent_path" : ""; - my $receive_option = ""; - $receive_option = "-v" if($loglevel >= 3); - - my $cmd = "$snapshot_rsh /sbin/btrfs send $parent_option $snapshot_path | $target_rsh /sbin/btrfs receive $receive_option $target_path/"; - my $ret = run_cmd($cmd); - unless(defined($ret)) { - ERROR "Failed to send/receive btrfs subvolume: $snapshot->{PRINT} " . ($parent_path ? "[$parent_path]" : "") . " -> $target->{PRINT}"; - return undef; - } - return 1; -} - - # sets $config->{ABORTED} on failure # sets $config->{SUBVOL_RECEIVED} sub macro_send_receive($@) @@ -1369,7 +1365,7 @@ MAIN: $lastgen = $src_vol->{cgen} + 1; # dump files, sorted and unique - my $ret = btr_subvolume_find_new($target_vol, $lastgen); + my $ret = btrfs_subvolume_find_new($target_vol, $lastgen); exit 1 unless(ref($ret)); print "--------------------------------------------------------------------------------\n"; @@ -1451,7 +1447,7 @@ MAIN: print "\n--------------------------------------------------------------------------------\n"; print "Source volume: $sroot->{PRINT}\n"; print "--------------------------------------------------------------------------------\n"; - print (btr_filesystem_usage($sroot) // ""); + print (btrfs_filesystem_usage($sroot) // ""); print "\n"; $processed{$sroot->{URL}} = 1; } @@ -1469,7 +1465,7 @@ MAIN: print "Target volume: $droot->{PRINT}\n"; print " ^--- $sroot->{PRINT}\n"; print "--------------------------------------------------------------------------------\n"; - print (btr_filesystem_usage($droot) // ""); + print (btrfs_filesystem_usage($droot) // ""); print "\n"; $processed{$droot->{URL}} = 1; } @@ -1541,7 +1537,7 @@ MAIN: # try to read subvolume detail, as configured subvolume could be a symlink. DEBUG "Subvolume \"$config_subvol->{rel_path}\" not present in btrfs subvolume list for \"$sroot->{PRINT}\""; $svol = vinfo_child($sroot, $config_subvol->{rel_path}); - my $detail = btr_subvolume_detail($svol); + my $detail = btrfs_subvolume_detail($svol); unless($detail) { $config_subvol->{ABORTED} = "Failed to fetch subvolume detail"; WARN "Skipping subvolume \"$svol->{PRINT}\": $config_subvol->{ABORTED}"; @@ -1740,7 +1736,7 @@ MAIN: # finally create the snapshot INFO "Creating subvolume snapshot for: $svol->{PRINT}"; - if(btrfs_snapshot($svol, "$sroot->{PATH}/$snapdir/$snapshot_name")) { + if(btrfs_subvolume_snapshot($svol, "$sroot->{PATH}/$snapdir/$snapshot_name")) { $config_subvol->{SNAPSHOT} = vinfo_child($sroot, "$snapdir/$snapshot_name"); } else { From 490dab398e49beb6dce3365fd8a31530ce0c4dc7 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Thu, 23 Apr 2015 16:34:07 +0200 Subject: [PATCH 20/31] btrbk: consistent snapshot filtering (cosmetics) --- btrbk | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/btrbk b/btrbk index d3442b8..614bcd7 100755 --- a/btrbk +++ b/btrbk @@ -1800,12 +1800,12 @@ MAIN: DEBUG "Checking schedule for resume candidates"; # add all present backups to schedule, with no value # these are needed for correct results of schedule() - foreach my $vol (keys %{vinfo_subvol_list($droot)}) { - my ($date, $date_ext) = get_date_tag($vol); - next unless($date && ($vol =~ s/^\Q$snapshot_basename.\E//)); # use only the date suffix for sorting + foreach my $vol (values %{vinfo_subvol_list($droot)}) { + next unless($vol->{SUBVOL_PATH} =~ /^\Q$snapshot_basename\E$snapshot_postfix_match$/); + my ($date, $date_ext) = get_date_tag($vol->{NAME}); + next unless($date); push(@schedule, { value => undef, date => $date, date_ext => $date_ext }); } - my ($preserve, undef) = schedule( schedule => \@schedule, today => \@today, From fb912da448ec4090150e9f6c583c43bf6376348e Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 28 Apr 2015 17:46:38 +0200 Subject: [PATCH 21/31] btrbk: bugfix: allow "/" as volume name --- ChangeLog | 1 + btrbk | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 2067b53..71d6609 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,7 @@ btrbk-current * Added configuration option "snapshot_name" (closes: #5). * Bugfix: allow "0" as subvolume name (closes: #10). + * Bugfix: allow "/" as volume name (closes: #15). * Bugfix: check source AND targets for determining snapshot postfix (closes: #11). diff --git a/btrbk b/btrbk index 614bcd7..0399d72 100755 --- a/btrbk +++ b/btrbk @@ -399,7 +399,7 @@ sub parse_config(@) # be very strict about file options, for security sake return undef unless(check_file($value, { absolute => 1, ssh => 1 }, $key, $file)); - $value =~ s/\/+$//; # remove trailing slash + $value =~ s/\/+$// unless($value =~ /^\/+$/); # remove trailing slash $value =~ s/^\/+/\//; # sanitize leading slash TRACE "config: adding volume \"$value\" to root context"; my $volume = { CONTEXT => "volume", From 436b1361ffb813b4f9bbd3d49b8b07ecc6354599 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 28 Apr 2015 19:08:20 +0200 Subject: [PATCH 22/31] btrbk: bugfix: allow relative path for subvolume; adapted documentation --- btrbk | 4 ---- doc/btrbk.conf.5 | 12 +++++++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/btrbk b/btrbk index 0399d72..9b127f6 100755 --- a/btrbk +++ b/btrbk @@ -424,10 +424,6 @@ sub parse_config(@) return undef unless(check_file($value, { relative => 1 }, $key, $file)); $value =~ s/\/+$//; # remove trailing slash $value =~ s/^\/+//; # remove leading slash - if($value =~ /\//) { - ERROR "Subvolume contains slashes: \"$value\" in \"$file\" line $."; - return undef; - } TRACE "config: adding subvolume \"$value\" to volume context: $cur->{url}"; my $subvolume = { CONTEXT => "subvolume", diff --git a/doc/btrbk.conf.5 b/doc/btrbk.conf.5 index 292c207..febcbf8 100644 --- a/doc/btrbk.conf.5 +++ b/doc/btrbk.conf.5 @@ -19,9 +19,10 @@ global options must be set before any sections are defined. The sections are: .TP \fBvolume\fR | -Directory of a btrfs volume (or subvolume) containing the source -subvolume(s) to be backuped (usually the mount-point of a btrfs -filesystem mounted with the \fIsubvolid=0\fR option). +Directory of a btrfs volume containing the source subvolume(s) to be +backuped. \fI\fR must be an absolute path and point +to a btrfs volume (or subvolume). Usually the mount point of a btrfs +filesystem mounted with the \fIsubvolid=0\fR option. .TP \fBsubvolume\fR Subvolume to be backuped, relative to the \fI\fR @@ -29,8 +30,9 @@ specified in the \fIvolume\fR section. .TP \fBtarget\fR | Target type and directory where the backup subvolumes are to be -created. In the current version of btrbk, the only valid \fI\fR -is \(lqsend\-receive\(rq. +created. \fI\fR must be an absolute path and point +to a btrfs volume (or subvolume). Currently the the only valid +\fI\fR is \(lqsend\-receive\(rq. .PP For the \fIvolume\fR and \fItarget\fR sections, you can also specify a ssh-url instead of a local directory. The syntax for \fI\fR is: From 6aa0a84127134379c252333ac86e9b3576fb7890 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 28 Apr 2015 19:22:15 +0200 Subject: [PATCH 23/31] documentation: adapted btrbk.conf.example --- btrbk.conf.example | 54 ++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/btrbk.conf.example b/btrbk.conf.example index 245155e..5237376 100644 --- a/btrbk.conf.example +++ b/btrbk.conf.example @@ -1,15 +1,20 @@ # # Example btrbk configuration file # -# Note that the options can be overridden in the -# volume/subvolume/target sections. Refer to man btrbk.conf(5) for a -# more detailed explanation of this. # +# Please refer to the btrbk.conf(5) man-page for more details. +# +# Note that the options can be overridden in the +# volume/subvolume/target sections. +# + # Directory in which the btrfs snapshots are created. Relative to # of the volume section. -# If not set, the snapshots are directly created in: -# / +# If not set, the snapshots are created in . +# +# If you want to set a custom name for the snapshot (and backups), +# use the "snapshot_name" option within the subvolume section. # # NOTE: btrbk does not autmatically create this directory, and the # snapshot creation will fail if it is not present. @@ -17,10 +22,10 @@ snapshot_dir _btrbk_snap # Perform incremental backups (set to "strict" if you want to prevent -# creation of initial backups if no parent is found) +# creation of initial backups if no parent is found). incremental yes -# Always create snapshots, even if the target volume is unreachable +# Always create snapshots, even if the target volume is unreachable. snapshot_create_always yes # Resume missing backups if the target volume is reachable again. @@ -54,23 +59,22 @@ btrfs_commit_delete after # # Volume section: "volume " -# Directory of a btrfs volume (or subvolume) -# containing the subvolume to be backuped -# (usually the mount-point of a btrfs filesystem -# mounted with subvolid=0 option) +# +# Directory of a btrfs volume (or subvolume) +# containing the subvolume to be backuped +# (usually the mount-point of a btrfs filesystem +# mounted with subvolid=0 option) # # Subvolume section: "subvolume - -# Subvolume to be backuped, relative to -# in volume section +# +# Subvolume to be backuped, relative to +# in volume section. # # Target section: "target " - -# Backup type, currently only "send-receive" - -# Directory of a btrfs volume (or subvolume) -# receiving the backups # +# Backup type, currently only "send-receive". +# Directory of a btrfs volume (or subvolume) +# receiving the backups. # # NOTE: The parser does not care about indentation, this is only for # human readability. The options always apply to the last section @@ -83,7 +87,6 @@ volume /mnt/btr_system subvolume root_gentoo target send-receive /mnt/btr_ext/_btrbk target send-receive /mnt/btr_backup/_btrbk - receive_log sidecar subvolume kvm # use different preserve matrix for kvm backups @@ -106,14 +109,9 @@ volume /mnt/btr_ext subvolume data target send-receive /mnt/btr_backup/_btrbk -volume /mnt/btr_boot - incremental yes - - subvolume boot - target send-receive /mnt/btr_ext/_btrbk - target send-receive /mnt/btr_backup/_btrbk - volume ssh://my-remote-host.com/mnt/btr_pool - subvolume system + subvolume data_0 + snapshot_dir snapshots/btrbk + snapshot_name data_main target send-receive /mnt/btr_backup/_btrbk/my-remote-host.com From d860d9e5dd342750adc1f10ba29930df48642703 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 28 Apr 2015 20:17:47 +0200 Subject: [PATCH 24/31] btrbk: allow '+' character for subvolume names --- btrbk | 2 +- doc/btrbk.conf.5 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/btrbk b/btrbk index 9b127f6..9ef95bc 100755 --- a/btrbk +++ b/btrbk @@ -91,7 +91,7 @@ my $loglevel = 1; my $ip_addr_match = qr/(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/; my $host_name_match = qr/(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])/; -my $file_match = qr/[0-9a-zA-Z_@\-\.\/]+/; # note: ubuntu uses '@' in the subvolume layout: +my $file_match = qr/[0-9a-zA-Z_@\+\-\.\/]+/; # note: ubuntu uses '@' in the subvolume layout: my $ssh_prefix_match = qr/ssh:\/\/($ip_addr_match|$host_name_match)/; my $snapshot_postfix_match = qr/\.[0-9]{8}(_[0-9]+)?/; diff --git a/doc/btrbk.conf.5 b/doc/btrbk.conf.5 index febcbf8..edabe61 100644 --- a/doc/btrbk.conf.5 +++ b/doc/btrbk.conf.5 @@ -40,7 +40,7 @@ ssh-url instead of a local directory. The syntax for \fI\fR is: ssh://host.xz/path/to/volume .PP Note that btrfs is very picky on file names (mainly for security -reasons), only the characters [0-9] [a-z] [A-Z] and "._-@" are +reasons), only the characters [0-9] [a-z] [A-Z] and "._+-@" are allowed. .PP The configuration options are: From faf4a53e516c4fa5f4ca4b2d0fd827c3438ee6e3 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 28 Apr 2015 20:19:22 +0200 Subject: [PATCH 25/31] btrbk: quote filenames on external command execution --- btrbk | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/btrbk b/btrbk index 9ef95bc..011b38b 100755 --- a/btrbk +++ b/btrbk @@ -550,7 +550,7 @@ sub btrfs_filesystem_show($) my $vol = shift || die; my $path = $vol->{PATH} // die; my $rsh = $vol->{RSH} || ""; - my $ret = run_cmd("$rsh /sbin/btrfs filesystem show $path", 1); + my $ret = run_cmd("$rsh /sbin/btrfs filesystem show '$path'", 1); return $ret; } @@ -560,7 +560,7 @@ sub btrfs_filesystem_df($) my $vol = shift || die; my $path = $vol->{PATH} // die; my $rsh = $vol->{RSH} || ""; - my $ret = run_cmd("$rsh /sbin/btrfs filesystem df $path", 1); + my $ret = run_cmd("$rsh /sbin/btrfs filesystem df '$path'", 1); return $ret; } @@ -570,7 +570,7 @@ sub btrfs_filesystem_usage($) my $vol = shift || die; my $path = $vol->{PATH} // die; my $rsh = $vol->{RSH} || ""; - my $ret = run_cmd("$rsh /sbin/btrfs filesystem usage $path", 1); + my $ret = run_cmd("$rsh /sbin/btrfs filesystem usage '$path'", 1); return $ret; } @@ -580,7 +580,7 @@ sub btrfs_subvolume_detail($) my $vol = shift || die; my $path = $vol->{PATH} // die; my $rsh = $vol->{RSH} || ""; - my $ret = run_cmd("$rsh /sbin/btrfs subvolume show $path 2>/dev/null", 1); + my $ret = run_cmd("$rsh /sbin/btrfs subvolume show '$path' 2>/dev/null", 1); return undef unless(defined($ret)); my $real_path; @@ -639,7 +639,7 @@ sub btrfs_subvolume_list($;@) $filter_option = "-o" if($opts{subvol_only}); my $display_options = "-c -u -q"; $display_options .= " -R" unless($btrfs_progs_compat); - my $ret = run_cmd("$rsh /sbin/btrfs subvolume list $filter_option $display_options $path", 1); + my $ret = run_cmd("$rsh /sbin/btrfs subvolume list $filter_option $display_options '$path'", 1); return undef unless(defined($ret)); my @nodes; @@ -699,7 +699,7 @@ sub btrfs_subvolume_find_new($$;$) my $path = $vol->{PATH} // die; my $rsh = $vol->{RSH} || ""; my $lastgen = shift // die; - my $ret = run_cmd("$rsh /sbin/btrfs subvolume find-new $path $lastgen"); + my $ret = run_cmd("$rsh /sbin/btrfs subvolume find-new '$path' $lastgen"); unless(defined($ret)) { ERROR "Failed to fetch modified files for: $vol->{PRINT}"; return undef; @@ -762,7 +762,7 @@ sub btrfs_subvolume_snapshot($$) DEBUG "[btrfs] source: $src_path"; DEBUG "[btrfs] target: $target_path"; INFO ">>> " . ($svol->{HOST} ? "$svol->{HOST}:" : "") . $target_path; - my $ret = run_cmd("$rsh /sbin/btrfs subvolume snapshot -r $src_path $target_path"); + my $ret = run_cmd("$rsh /sbin/btrfs subvolume snapshot -r '$src_path' '$target_path'"); ERROR "Failed to create btrfs subvolume snapshot: $svol->{PRINT} -> $target_path" unless(defined($ret)); return defined($ret) ? $target_path : undef; } @@ -786,7 +786,7 @@ sub btrfs_subvolume_delete($@) DEBUG "[btrfs] subvolume: $_->{PRINT}" foreach(@$targets); my $options = ""; $options = "--commit-$commit " if($commit); - my $ret = run_cmd("$rsh /sbin/btrfs subvolume delete $options" . join(' ', map( { $_->{PATH} } @$targets))); + my $ret = run_cmd("$rsh /sbin/btrfs subvolume delete $options" . join(' ', map( { "'$_->{PATH}'" } @$targets))); ERROR "Failed to delete btrfs subvolumes: " . join(' ', map( { $_->{PRINT} } @$targets)) unless(defined($ret)); return defined($ret) ? scalar(@$targets) : undef; } @@ -812,11 +812,11 @@ sub btrfs_send_receive($$$) DEBUG "[btrfs] parent: $parent->{PRINT}" if($parent); DEBUG "[btrfs] target: $target->{PRINT}"; - my $parent_option = $parent_path ? "-p $parent_path" : ""; + my $parent_option = $parent_path ? "-p '$parent_path'" : ""; my $receive_option = ""; $receive_option = "-v" if($loglevel >= 3); - my $cmd = "$snapshot_rsh /sbin/btrfs send $parent_option $snapshot_path | $target_rsh /sbin/btrfs receive $receive_option $target_path/"; + my $cmd = "$snapshot_rsh /sbin/btrfs send $parent_option '$snapshot_path' | $target_rsh /sbin/btrfs receive $receive_option '$target_path/'"; my $ret = run_cmd($cmd); unless(defined($ret)) { ERROR "Failed to send/receive btrfs subvolume: $snapshot->{PRINT} " . ($parent_path ? "[$parent_path]" : "") . " -> $target->{PRINT}"; From 34e7ad07ece21731a5af79eaa5a37ae339116d0b Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 28 Apr 2015 20:37:39 +0200 Subject: [PATCH 26/31] ssh_filter_btrbk: removed unneeded calls to btrfs-progs --- ssh_filter_btrbk.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ssh_filter_btrbk.sh b/ssh_filter_btrbk.sh index f71ad9c..450f1eb 100644 --- a/ssh_filter_btrbk.sh +++ b/ssh_filter_btrbk.sh @@ -40,8 +40,6 @@ case "$SSH_ORIGINAL_COMMAND" in /sbin/btrfs\ receive\ *) run_cmd ;; # mandatory if this host is backup target /sbin/btrfs\ subvolume\ delete\ *) run_cmd ;; # mandatory if scheduling is active /sbin/btrfs\ subvolume\ find-new\ *) run_cmd ;; # needed for "btrbk diff" - /sbin/btrfs\ filesystem\ show\ *) run_cmd ;; # needed for "btrbk info" - /sbin/btrfs\ filesystem\ df\ *) run_cmd ;; # needed for "btrbk info" /sbin/btrfs\ filesystem\ usage\ *) run_cmd ;; # needed for "btrbk info" *) reject_and_die ;; esac From bd2ad9c25801da883d64c4223f1572dce0cc1bf8 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 28 Apr 2015 20:38:40 +0200 Subject: [PATCH 27/31] btrbk: handle ABORTED flags on "tree" action --- btrbk | 3 +++ 1 file changed, 3 insertions(+) diff --git a/btrbk b/btrbk index 011b38b..0022f09 100755 --- a/btrbk +++ b/btrbk @@ -1646,11 +1646,13 @@ MAIN: my @out; foreach my $config_vol (@{$config->{VOLUME}}) { + next if($config_vol->{ABORTED}); my %droot_compat; my $sroot = $config_vol->{sroot} || die; push @out, "$sroot->{PRINT}"; foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { + next if($config_subvol->{ABORTED}); my $svol = $config_subvol->{svol} || die; push @out, "|-- $svol->{PRINT}"; foreach my $snapshot (sort { $a->{PATH} cmp $b->{PATH} } get_snapshot_children($sroot, $svol)) @@ -1658,6 +1660,7 @@ MAIN: push @out, "| ^-- $snapshot->{PATH}"; foreach my $config_target (@{$config_subvol->{TARGET}}) { + next if($config_target->{ABORTED}); my $droot = $config_target->{droot} || die; $droot_compat{$droot->{URL}} = 1 if($droot->{BTRFS_PROGS_COMPAT}); foreach (sort { $a->{SUBVOL_PATH} cmp $b->{SUBVOL_PATH} } get_receive_targets($droot, $snapshot)) { From 9103cbc79da0a578ba1e8eea2ad19839b836fa15 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 28 Apr 2015 20:49:18 +0200 Subject: [PATCH 28/31] btrbk: output vinfo->{PRINT} instead of {URL} in _origin_tree() --- btrbk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrbk b/btrbk index 0022f09..381d269 100755 --- a/btrbk +++ b/btrbk @@ -1125,7 +1125,7 @@ sub _origin_tree return 0; } if($uuid_fs_map{$uuid}) { - push(@$lines, ["$prefix" . join(" === ", sort keys %{$uuid_fs_map{$uuid}}), $uuid]); + push(@$lines, ["$prefix" . join(" === ", sort map { $_->{PRINT} } values %{$uuid_fs_map{$uuid}}), $uuid]); } else { push(@$lines, ["$prefix/$node->{path}", $uuid]); } From 825fa61eb123ef8f81294f3bed1f0d09d4f0e65b Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 28 Apr 2015 21:22:00 +0200 Subject: [PATCH 29/31] documentation: ChangeLog: added missing changes from refactoring --- ChangeLog | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ChangeLog b/ChangeLog index 71d6609..718c164 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,9 +1,18 @@ btrbk-current + + * New versioning scheme using more common three-level versions. + * Code refactoring: cleanup of data structures and handling of btrfs + subvolume tree, as well as security related code parts. + * Correct handling of symlinks to btrfs subvolumes (closes: #12). * Added configuration option "snapshot_name" (closes: #5). * Bugfix: allow "0" as subvolume name (closes: #10). * Bugfix: allow "/" as volume name (closes: #15). * Bugfix: check source AND targets for determining snapshot postfix (closes: #11). + * Bugfix: fixed "diff" action (colses: #14). + * Allow '+' character for subvolume names. + * Filesystems on remote hosts are now printed as + "{my.remote-host.com}" in summary and logs. btrbk-0.16 From 444fba4a5061c0b3e6363102cdf1bb25ba0b4a22 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 28 Apr 2015 23:47:34 +0200 Subject: [PATCH 30/31] btrbk: log DEBUG instead of WARN if run_cmd() fails. WARN messages are always printed on undef return value --- btrbk | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/btrbk b/btrbk index 381d269..706152a 100755 --- a/btrbk +++ b/btrbk @@ -154,7 +154,7 @@ sub run_cmd($;$) if($?) { my $exitcode= $? >> 8; my $signal = $? & 127; - WARN "Command execution failed (exitcode=$exitcode" . ($signal ? ", signal=$signal" : "") . "): \"$cmd\""; + DEBUG "Command execution failed (exitcode=$exitcode" . ($signal ? ", signal=$signal" : "") . "): \"$cmd\""; return undef; } else { @@ -1001,7 +1001,7 @@ sub macro_send_receive($@) return 1; } else { $info{ERROR} = 1; - $config_target->{ABORTED} = "btrfs send/receive command failed"; + $config_target->{ABORTED} = "Failed to send/receive subvolume"; return undef; } } @@ -1924,7 +1924,7 @@ MAIN: $config_target->{SUBVOL_DELETED} = $delete; } else { - $config_target->{ABORTED} = "btrfs subvolume delete command failed"; + $config_target->{ABORTED} = "Failed to delete subvolume"; $target_aborted = 1; } } @@ -1959,7 +1959,7 @@ MAIN: $config_subvol->{SUBVOL_DELETED} = $delete; } else { - $config_subvol->{ABORTED} = "btrfs subvolume delete command failed"; + $config_subvol->{ABORTED} = "Failed to delete delete subvolume"; } } } From 40201c562be576948cd96a4d5c67f6297958ae68 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Wed, 29 Apr 2015 00:34:11 +0200 Subject: [PATCH 31/31] btrbk: log messages go to stderr instead of stdout (more consistent since btrfs-progs error messages also go to stderr) --- ChangeLog | 2 ++ btrbk | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index 718c164..bf937dd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,8 @@ btrbk-current subvolume tree, as well as security related code parts. * Correct handling of symlinks to btrfs subvolumes (closes: #12). * Added configuration option "snapshot_name" (closes: #5). + * Log messages now go to stderr, only the summary is printed on + stdout. * Bugfix: allow "0" as subvolume name (closes: #10). * Bugfix: allow "/" as volume name (closes: #15). * Bugfix: check source AND targets for determining snapshot postfix diff --git a/btrbk b/btrbk index 706152a..aa425a2 100755 --- a/btrbk +++ b/btrbk @@ -132,11 +132,11 @@ sub HELP_MESSAGE print STDERR "For additional information, see $PROJECT_HOME\n"; } -sub TRACE { my $t = shift; print STDOUT "... $t\n" if($loglevel >= 4); } -sub DEBUG { my $t = shift; print STDOUT "$t\n" if($loglevel >= 3); } -sub INFO { my $t = shift; print STDOUT "$t\n" if($loglevel >= 2); } -sub WARN { my $t = shift; print STDOUT "WARNING: $t\n" if($loglevel >= 1); } -sub ERROR { my $t = shift; print STDOUT "ERROR: $t\n"; } +sub TRACE { my $t = shift; print STDERR "... $t\n" if($loglevel >= 4); } +sub DEBUG { my $t = shift; print STDERR "$t\n" if($loglevel >= 3); } +sub INFO { my $t = shift; print STDERR "$t\n" if($loglevel >= 2); } +sub WARN { my $t = shift; print STDERR "WARNING: $t\n" if($loglevel >= 1); } +sub ERROR { my $t = shift; print STDERR "ERROR: $t\n"; } sub run_cmd($;$)