ambiser
/
btrbk
mirror of https://github.com/digint/btrbk
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

btrbk: refactoring of add global caches: reduce btrfs-progs calls, and make sure all root vinfo (especially targets) with same URL share the same SUBVOL_LIST reference. 

- %btrfs_tree_cache (replaces %root_tree_cache)

- %subvol_list_cache (replaces %vinfo_cache):

- vinfo_init_root() (was: vinfo_root()) now lookups in cache before
  calling btrfs_subvolume_detail()

- vinfo_subvol_list() now lookups in cache before calling
  btrfs_subvolume_list()
pull/73/head
Axel Burri 2016-03-09 19:52:45 +01:00
parent ba90c13320
commit 1d054bf04a
1 changed files with 138 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

226
btrbk
View File

@ -167,10 +167,10 @@ my %table_formats = (
}, },
); );
my %root_tree_cache; # map URL to SUBTREE (needed since "btrfs subvolume list" does not provide us with the uuid of the btrfs root node) my %btrfs_tree_cache; # map URL to btr_tree node
my %vinfo_cache; # map URL to vinfo my %subvol_list_cache; # map URL to subvolume list ( rel_path => vinfo, ... )
my %uuid_info; # map UUID to btr_tree node my %uuid_info; # map UUID to btr_tree node
my %uuid_fs_map; # map UUID to URL my %uuid_url_map; # map UUID to hash ( URL => btr_tree, ... )
my $dryrun; my $dryrun;
my $loglevel = 1; my $loglevel = 1;
@ -408,58 +408,60 @@ sub run_cmd(@)
} }
sub vinfo($$) sub vinfo($;$)
{ {
my $url = shift // die; my $url = shift // die;
my $config = shift || die; my $config = shift;
my %info; my %info;
my $name = $url; my $name = $url;
$name =~ s/^.*\///; $name =~ s/^.*\///;
%info = (
URL => $url,
NAME => $name,
PATH => $url,
PRINT => $url,
);
if($url =~ /^ssh:\/\/(\S+?)(\/\S+)$/) { if($url =~ /^ssh:\/\/(\S+?)(\/\S+)$/) {
my ($host, $path) = ($1, $2); my ($host, $path) = ($1, $2);
my $ssh_port = config_key($config, "ssh_port");
my $ssh_user = config_key($config, "ssh_user");
my $ssh_identity = config_key($config, "ssh_identity");
my $ssh_compression = config_key($config, "ssh_compression");
my $ssh_cipher_spec = config_key($config, "ssh_cipher_spec") // "default";
my @ssh_options;
push(@ssh_options, '-p', $ssh_port) if($ssh_port ne "default");
push(@ssh_options, '-C') if($ssh_compression);
push(@ssh_options, '-c', $ssh_cipher_spec) if($ssh_cipher_spec ne "default");
if($ssh_identity) {
push(@ssh_options, '-i', $ssh_identity);
} else {
WARN "No SSH identity provided (option ssh_identity is not set) for: $url";
}
%info = ( %info = (
URL => $url, %info,
NAME => $name,
HOST => $host, HOST => $host,
PATH => $path, PATH => $path,
PRINT => "$host:$path", PRINT => "$host:$path",
RSH_TYPE => "ssh",
SSH_USER => $ssh_user,
SSH_IDENTITY => $ssh_identity,
SSH_PORT => $ssh_port,
RSH => ['/usr/bin/ssh', @ssh_options, $ssh_user . '@' . $host ],
); );
} if($config) {
elsif(($url =~ /^\//) && ($url =~ /^$file_match$/)) { my $ssh_port = config_key($config, "ssh_port");
%info = ( my $ssh_user = config_key($config, "ssh_user");
URL => $url, my $ssh_identity = config_key($config, "ssh_identity");
NAME => $name, my $ssh_compression = config_key($config, "ssh_compression");
PATH => $url, my $ssh_cipher_spec = config_key($config, "ssh_cipher_spec") // "default";
PRINT => $url, my @ssh_options;
); push(@ssh_options, '-p', $ssh_port) if($ssh_port ne "default");
} push(@ssh_options, '-C') if($ssh_compression);
else { push(@ssh_options, '-c', $ssh_cipher_spec) if($ssh_cipher_spec ne "default");
die "Ambiguous vinfo url: $url"; if($ssh_identity) {
push(@ssh_options, '-i', $ssh_identity);
} else {
WARN "No SSH identity provided (option ssh_identity is not set) for: $url";
}
%info = (
%info,
RSH_TYPE => "ssh",
SSH_USER => $ssh_user,
SSH_IDENTITY => $ssh_identity,
SSH_PORT => $ssh_port,
RSH => ['/usr/bin/ssh', @ssh_options, $ssh_user . '@' . $host ],
);
}
} }
my $btrfs_progs_compat = config_key($config, "btrfs_progs_compat"); if($config) {
$info{BTRFS_PROGS_COMPAT} = $btrfs_progs_compat if($btrfs_progs_compat); my $btrfs_progs_compat = config_key($config, "btrfs_progs_compat");
$info{BTRFS_PROGS_COMPAT} = $btrfs_progs_compat if($btrfs_progs_compat);
}
TRACE "vinfo created: $url"; TRACE "vinfo created: $url";
return \%info; return \%info;
@ -515,15 +517,25 @@ sub vinfo_child($$;$)
} }
sub vinfo_root($) sub vinfo_init_root($)
{ {
my $vol = shift; my $vol = shift;
my $detail = btrfs_subvolume_detail($vol); my $detail = $btrfs_tree_cache{$vol->{URL}};
my $path_verified;
if($detail) {
TRACE "vinfo_init_root: cache HIT: $vol->{URL}";
$path_verified = 1; # all keys from btrfs_tree_cache are real_paths
}
else {
TRACE "vinfo_init_root: cache MISS: $vol->{URL}";
$detail = btrfs_subvolume_detail($vol);
}
return undef unless $detail; return undef unless $detail;
vinfo_set_detail($vol, $detail); vinfo_set_detail($vol, $detail, $path_verified);
# read (and cache) the subvolume list # read (and cache) the subvolume list
return undef unless vinfo_subvol_list($vol); return undef unless vinfo_subvol_list($vol);
TRACE "vinfo root created: $vol->{PRINT}"; TRACE "vinfo root created: $vol->{PRINT}";
@ -531,14 +543,15 @@ sub vinfo_root($)
} }
sub vinfo_set_detail($$) sub vinfo_set_detail($$;$)
{ {
my $vol = shift || die; my $vol = shift || die;
my $detail = shift || die; my $detail = shift || die;
my $path_verified = shift;
# add detail data to vinfo hash # add detail data to vinfo hash
foreach(keys %$detail) { foreach(keys %$detail) {
next if(uc($_) eq $_); # skip UPPER_CASE keys next if(uc($_) eq $_); # skip UPPER_CASE keys (except REAL_PATH below)
next if($_ eq "path"); # skip "path", this comes in wrong by "btrfs subvolume list" next if($_ eq "path"); # skip "path", this comes in wrong by "btrfs subvolume list"
# check if already present matches new # check if already present matches new
@ -546,13 +559,19 @@ sub vinfo_set_detail($$)
$vol->{$_} = $detail->{$_}; $vol->{$_} = $detail->{$_};
} }
# !!! be super-paranoid # be very paranoid, this should never happen
die if(defined($detail->{URL}) && ($detail->{URL} ne $vol->{URL})); die if(defined($detail->{URL}) && ($detail->{URL} ne $vol->{URL}));
die if(defined($detail->{NAME}) && ($detail->{NAME} ne $vol->{NAME})); die if(defined($detail->{NAME}) && ($detail->{NAME} ne $vol->{NAME}));
die if(defined($detail->{SUBVOL_PATH}) && ($detail->{SUBVOL_PATH} ne $vol->{SUBVOL_PATH})); die if(defined($detail->{SUBVOL_PATH}) && defined($vol->{SUBVOL_PATH}) && ($detail->{SUBVOL_PATH} ne $vol->{SUBVOL_PATH}));
if($detail->{REAL_PATH}) { if($detail->{REAL_PATH}) {
$vol->{REAL_PATH} = $detail->{REAL_PATH}; $vol->{REAL_PATH} = $detail->{REAL_PATH};
}
elsif($path_verified) {
$vol->{REAL_PATH} = $vol->{PATH};
}
if($vol->{REAL_PATH}) {
if($vol->{RSH_TYPE} && ($vol->{RSH_TYPE} eq "ssh")) { if($vol->{RSH_TYPE} && ($vol->{RSH_TYPE} eq "ssh")) {
$vol->{REAL_URL} = "ssh://$vol->{HOST}$vol->{REAL_PATH}"; $vol->{REAL_URL} = "ssh://$vol->{HOST}$vol->{REAL_PATH}";
} else { } else {
@ -560,10 +579,6 @@ sub vinfo_set_detail($$)
} }
} }
# update cache
$vinfo_cache{$vol->{URL}} = $vol;
$vinfo_cache{$vol->{REAL_URL}} = $vol if($vol->{REAL_URL});
TRACE "vinfo updated for: $vol->{PRINT}"; TRACE "vinfo updated for: $vol->{PRINT}";
TRACE(vinfo_dump($vol)) if($loglevel >= 4); TRACE(vinfo_dump($vol)) if($loglevel >= 4);
return $vol; return $vol;
@ -1188,7 +1203,7 @@ sub btrfs_subvolume_list($;@)
{ {
my $vol = shift || die; my $vol = shift || die;
my %opts = @_; my %opts = @_;
my $path = $vol->{PATH} // die; my $path = $vol->{PATH} // die; # deliberately NOT using REAL_PATH here!
my $btrfs_progs_compat = $vol->{BTRFS_PROGS_COMPAT} || $opts{btrfs_progs_compat}; my $btrfs_progs_compat = $vol->{BTRFS_PROGS_COMPAT} || $opts{btrfs_progs_compat};
my @filter_options = ('-a'); my @filter_options = ('-a');
push(@filter_options, '-o') if($opts{subvol_only}); push(@filter_options, '-o') if($opts{subvol_only});
@ -1601,13 +1616,36 @@ sub btrfs_send_to_file($$$$;@)
} }
sub _btr_tree_fill_cache
{
my $node = shift;
my $abs_path = shift;
# traverse tree and update tree cache
$btrfs_tree_cache{$abs_path} = $node;
$uuid_url_map{$node->{uuid}}->{$abs_path} = $node if($node->{uuid});
foreach(values %{$node->{SUBTREE}}) {
_btr_tree_fill_cache($_, $abs_path . '/' . $_->{REL_PATH});
}
return undef;
}
sub btr_tree($) sub btr_tree($)
{ {
my $vol = shift; my $vol = shift;
# return cached info if present # return cached info if present
return $root_tree_cache{$vol->{URL}} if($vol->{is_root} && $root_tree_cache{$vol->{URL}}); return $btrfs_tree_cache{$vol->{REAL_URL}} if($vol->{REAL_URL} && $btrfs_tree_cache{$vol->{REAL_URL}});
return $root_tree_cache{$vol->{REAL_URL}} if($vol->{is_root} && $vol->{REAL_URL} && $root_tree_cache{$vol->{REAL_URL}}); return $btrfs_tree_cache{$vol->{URL}} if($btrfs_tree_cache{$vol->{URL}});
# NOTE: make sure to to have either $vol->{uuid} or $vol->{is_root}
# (provided by btrfs_subvolume_show()), or we cannot determine the
# anchor to our root path (since the subvolume path output of "btrfs
# subvolume list" is ambigous, and the uuid of the btrfs root node
# cannot be resolved).
die unless($vol->{uuid} || $vol->{is_root});
die unless($vol->{REAL_URL});
return $uuid_info{$vol->{uuid}} if($vol->{uuid} && $uuid_info{$vol->{uuid}}); return $uuid_info{$vol->{uuid}} if($vol->{uuid} && $uuid_info{$vol->{uuid}});
# man btrfs-subvolume: # man btrfs-subvolume:
@ -1649,15 +1687,19 @@ sub btr_tree($)
$node->{REL_PATH} = $rel_path; # relative to {TOP_LEVEL}->{path} $node->{REL_PATH} = $rel_path; # relative to {TOP_LEVEL}->{path}
} }
my $vol_root;
if($vol->{is_root}) { if($vol->{is_root}) {
$root_tree_cache{$vol->{URL}} = \%tree; $vol_root = \%tree;
$root_tree_cache{$vol->{REAL_URL}} = \%tree if($vol->{REAL_URL});
return \%tree;
} }
else { else {
# TODO: graceful, this might happen on buggy btrfs-progs
die unless($uuid_info{$vol->{uuid}}); die unless($uuid_info{$vol->{uuid}});
return $uuid_info{$vol->{uuid}}; $vol_root = $uuid_info{$vol->{uuid}};
} }
_btr_tree_fill_cache($vol_root, $vol->{REAL_URL});
return $vol_root;
} }
@ -1685,6 +1727,16 @@ sub vinfo_subvol_list($)
my $vol = shift || die; my $vol = shift || die;
return $vol->{SUBVOL_LIST} if($vol->{SUBVOL_LIST}); return $vol->{SUBVOL_LIST} if($vol->{SUBVOL_LIST});
# find cached list
my $subvol_list = $subvol_list_cache{$vol->{URL}};
$subvol_list //= $subvol_list_cache{$vol->{REAL_URL}} if($vol->{REAL_URL} && ($vol->{REAL_URL} ne $vol->{URL}));
if($subvol_list) {
TRACE "vinfo_subvol_list: cache HIT: $vol->{URL}";
$vol->{SUBVOL_LIST} = $subvol_list;
return $subvol_list;
}
TRACE "vinfo_subvol_list: cache MISS: $vol->{URL}";
my $tree_root = btr_tree($vol); my $tree_root = btr_tree($vol);
return undef unless($tree_root); return undef unless($tree_root);
@ -1700,8 +1752,6 @@ sub vinfo_subvol_list($)
my $subvol = vinfo_child($vol, $subvol_path); my $subvol = vinfo_child($vol, $subvol_path);
vinfo_set_detail($subvol, $_->{node}); vinfo_set_detail($subvol, $_->{node});
$uuid_fs_map{$subvol->{uuid}}->{$subvol->{URL}} = $subvol;
$ret{$subvol_path} = $subvol; $ret{$subvol_path} = $subvol;
} }
@ -1709,6 +1759,8 @@ sub vinfo_subvol_list($)
TRACE(Data::Dumper->Dump([\%ret], ["vinfo_subvol_list"])) if($loglevel >= 4); TRACE(Data::Dumper->Dump([\%ret], ["vinfo_subvol_list"])) if($loglevel >= 4);
$vol->{SUBVOL_LIST} = \%ret; $vol->{SUBVOL_LIST} = \%ret;
$subvol_list_cache{$vol->{URL}} = \%ret;
$subvol_list_cache{$vol->{REAL_URL}} = \%ret if($vol->{REAL_URL} && ($vol->{REAL_URL} ne $vol->{URL}));
return \%ret; return \%ret;
} }
@ -2039,8 +2091,8 @@ sub _origin_tree
push(@$lines, ["$prefix<orphaned>", $uuid]); push(@$lines, ["$prefix<orphaned>", $uuid]);
return 0; return 0;
} }
if($uuid_fs_map{$uuid}) { if($uuid_url_map{$uuid}) {
push(@$lines, ["$prefix" . join(" === ", sort map { $_->{PRINT} } values %{$uuid_fs_map{$uuid}}), $uuid]); push(@$lines, ["$prefix" . join(" === ", sort map { vinfo($_)->{PRINT} } keys %{$uuid_url_map{$uuid}}), $uuid]);
} else { } else {
push(@$lines, ["$prefix<BTRFS_ROOT>/$node->{path}", $uuid]); push(@$lines, ["$prefix<BTRFS_ROOT>/$node->{path}", $uuid]);
} }
@ -2598,12 +2650,12 @@ MAIN:
# FIXME: allow ssh:// src/dest (does not work since the configuration is not yet read). # FIXME: allow ssh:// src/dest (does not work since the configuration is not yet read).
my $src_vol = vinfo($src_url, { CONTEXT => "cmdline" }); my $src_vol = vinfo($src_url, { CONTEXT => "cmdline" });
unless(vinfo_root($src_vol)) { ERROR "Failed to fetch subvolume detail for '$src_vol->{PRINT}'" . ($err ? ": $err" : ""); exit 1; } unless(vinfo_init_root($src_vol)) { ERROR "Failed to fetch subvolume detail for '$src_vol->{PRINT}'" . ($err ? ": $err" : ""); exit 1; }
if($src_vol->{is_root}) { ERROR "Subvolume at \"$src_url\" is btrfs root!"; exit 1; } if($src_vol->{is_root}) { ERROR "Subvolume at \"$src_url\" is btrfs root!"; exit 1; }
unless($src_vol->{cgen}) { ERROR "Subvolume at \"$src_url\" does not provide cgen"; exit 1; } unless($src_vol->{cgen}) { ERROR "Subvolume at \"$src_url\" does not provide cgen"; exit 1; }
my $target_vol = vinfo($target_url, { CONTEXT => "cmdline" }); my $target_vol = vinfo($target_url, { CONTEXT => "cmdline" });
unless(vinfo_root($target_vol)) { ERROR "Failed to fetch subvolume detail for '$target_vol->{PRINT}'" . ($err ? ": $err" : ""); exit 1; } unless(vinfo_init_root($target_vol)) { ERROR "Failed to fetch subvolume detail for '$target_vol->{PRINT}'" . ($err ? ": $err" : ""); exit 1; }
unless($target_vol->{cgen}) { ERROR "Subvolume at \"$target_url\" does not provide cgen"; exit 1; } unless($target_vol->{cgen}) { ERROR "Subvolume at \"$target_url\" does not provide cgen"; exit 1; }
my $uuid_list = vinfo_fs_list($src_vol); my $uuid_list = vinfo_fs_list($src_vol);
@ -2940,9 +2992,11 @@ MAIN:
# #
# fill vinfo hash, basic checks on configuration # fill vinfo hash, basic checks on configuration
# #
# read volume btrfs tree, and make sure subvolume exist
foreach my $sroot (vinfo_subsection($config, 'volume')) { foreach my $sroot (vinfo_subsection($config, 'volume')) {
DEBUG "Initializing volume section: $sroot->{PRINT}"; DEBUG "Initializing volume section: $sroot->{PRINT}";
unless(vinfo_root($sroot)) { unless(vinfo_init_root($sroot)) {
ABORTED($sroot, "Failed to fetch subvolume detail" . ($err ? ": $err" : "")); ABORTED($sroot, "Failed to fetch subvolume detail" . ($err ? ": $err" : ""));
WARN "Skipping volume \"$sroot->{PRINT}\": $abrt"; WARN "Skipping volume \"$sroot->{PRINT}\": $abrt";
next; next;
@ -2969,28 +3023,27 @@ MAIN:
WARN "Skipping subvolume \"$svol->{PRINT}\": $abrt"; WARN "Skipping subvolume \"$svol->{PRINT}\": $abrt";
next; next;
} }
# make sure $svol is in subtree of $sroot
if(grep { $_->{uuid} eq $detail->{uuid} } values %{vinfo_subvol_list($sroot)}) { if(grep { $_->{uuid} eq $detail->{uuid} } values %{vinfo_subvol_list($sroot)}) {
vinfo_set_detail($svol, $uuid_info{$detail->{uuid}}); vinfo_set_detail($svol, $detail);
# vinfo_set_detail($svol, $detail);
} else { } else {
ABORTED($svol, "Not a child subvolume of: $sroot->{PRINT}"); ABORTED($svol, "Not a child subvolume of: $sroot->{PRINT}");
WARN "Skipping subvolume \"$svol->{PRINT}\": $abrt"; WARN "Skipping subvolume \"$svol->{PRINT}\": $abrt";
next; next;
} }
} }
}
}
# read target btrfs tree
foreach my $sroot (vinfo_subsection($config, 'volume')) {
foreach my $svol (vinfo_subsection($sroot, 'subvolume')) {
foreach my $droot (vinfo_subsection($svol, 'target')) { foreach my $droot (vinfo_subsection($svol, 'target')) {
DEBUG "Initializing target section: $droot->{PRINT}"; DEBUG "Initializing target section: $droot->{PRINT}";
my $target_type = $droot->{CONFIG}->{target_type} || die; my $target_type = $droot->{CONFIG}->{target_type} || die;
if($target_type eq "send-receive") if($target_type eq "send-receive")
{ {
if(my $vinfo_clone = $vinfo_cache{$droot->{URL}}) { unless(vinfo_init_root($droot)) {
DEBUG "Found previously initialized target with same url, cloning dataset from: $droot->{PRINT}";
vinfo_set_detail($droot, $vinfo_clone);
$droot->{SUBVOL_LIST} = $vinfo_clone->{SUBVOL_LIST};
}
elsif(! vinfo_root($droot)) {
ABORTED($droot, "Failed to fetch subvolume detail" . ($err ? ": $err" : "")); ABORTED($droot, "Failed to fetch subvolume detail" . ($err ? ": $err" : ""));
WARN "Skipping target \"$droot->{PRINT}\": $abrt"; WARN "Skipping target \"$droot->{PRINT}\": $abrt";
next; next;
@ -3038,12 +3091,14 @@ MAIN:
# NOTE: REMOTE_PARENT_UUID in $filename_info is the "parent of the source subvolume", NOT the # NOTE: REMOTE_PARENT_UUID in $filename_info is the "parent of the source subvolume", NOT the
# "parent of the received subvolume". # "parent of the received subvolume".
my $subvol = vinfo_child($droot, $file); my $subvol = vinfo_child($droot, $file);
$filename_info->{uuid} = "FAKE_UUID:" . $subvol->{URL}; my $detail = { uuid => "FAKE_UUID:" . $subvol->{URL},
$filename_info->{parent_uuid} = '-'; # correct value gets inserted below received_uuid => $filename_info->{received_uuid},
$filename_info->{readonly} = 1; # fake subvolume readonly flag # parent_uuid => '-', # correct value gets inserted below
vinfo_set_detail($subvol, $filename_info); readonly => 1, # fake subvolume readonly flag
};
vinfo_set_detail($subvol, $detail);
$uuid_info{$subvol->{uuid}} = $subvol; $uuid_info{$subvol->{uuid}} = $subvol;
$uuid_fs_map{$subvol->{uuid}}->{$subvol->{URL}} = $subvol; $uuid_url_map{$subvol->{uuid}}->{$subvol->{URL}} = $subvol;
$subvol_list{$file} = $subvol; $subvol_list{$file} = $subvol;
if($filename_info->{REMOTE_PARENT_UUID} ne '-') { if($filename_info->{REMOTE_PARENT_UUID} ne '-') {
@ -3072,7 +3127,7 @@ MAIN:
# - svol.<timestamp>--<received_uuid-n>[@<received_uuid_n-1>].btrfs : incremental image # - svol.<timestamp>--<received_uuid-n>[@<received_uuid_n-1>].btrfs : incremental image
foreach my $child (@{$child_uuid_list{$subvol->{received_uuid}}}) { foreach my $child (@{$child_uuid_list{$subvol->{received_uuid}}}) {
$child->{parent_uuid} = $subvol->{uuid}; vinfo_set_detail($child, { parent_uuid => $subvol->{uuid} });
DEBUG "Found parent/child partners, forcing preserve of: \"$subvol->{PRINT}\", \"$child->{PRINT}\""; DEBUG "Found parent/child partners, forcing preserve of: \"$subvol->{PRINT}\", \"$child->{PRINT}\"";
$subvol->{FORCE_PRESERVE} = "preserve forced: parent of another raw target"; $subvol->{FORCE_PRESERVE} = "preserve forced: parent of another raw target";
@ -3128,15 +3183,10 @@ MAIN:
my $url = $filter_args[0] || die; my $url = $filter_args[0] || die;
my $dump_uuid = 0; my $dump_uuid = 0;
my $vol = $vinfo_cache{$url}; my $vol = vinfo($url, { CONTEXT => "cmdline" });
unless($vol) { unless(vinfo_init_root($vol)) {
# specified volume is not in config ERROR "Failed to fetch subvolume detail for: $url" . ($err ? ": $err" : "");
DEBUG "Subvolume not parsed yet, fetching info: $url"; exit 1;
$vol = vinfo($url, { CONTEXT => "cmdline" });
unless(vinfo_root($vol)) {
ERROR "Failed to fetch subvolume detail for: $url" . ($err ? ": $err" : "");
exit 1;
}
} }
if($vol->{is_root}) { if($vol->{is_root}) {
ERROR "Subvolume is btrfs root: $url\n"; ERROR "Subvolume is btrfs root: $url\n";