diff --git a/ChangeLog b/ChangeLog index dde290c..7e8f784 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,8 +1,12 @@ btrbk-current + * MIGRATION + - If "rate_limit" is enabled, update ssh_filter_btrbk.sh on remote + source hosts, and make sure the "pv" command is available there. * Allow converting backup disks to source disks (close #114). * Add "backend btrfs-progs-sudo" configuration option (close #115). * Show aggregate "size" and "used" for "usage" action (close #119). + * Bugfix: rate limiting must be done after compression (close #134). * raw_target_encrypt: Always set "gpg --no-random-seed-file": prevents creation of "~/.gnupg/random_seed" with slight perfomance penalty. diff --git a/btrbk b/btrbk index df86a4c..62a7787 100755 --- a/btrbk +++ b/btrbk @@ -105,7 +105,7 @@ my %config_options = ( ssh_port => { default => "default", accept => [ "default" ], accept_numeric => 1 }, ssh_compression => { default => undef, accept => [ "yes", "no" ] }, ssh_cipher_spec => { default => "default", accept_regexp => qr/^$ssh_cipher_match(,$ssh_cipher_match)*$/ }, - rate_limit => { default => undef, accept => [ "no" ], accept_regexp => qr/^[0-9]+[kmgt]?$/, require_bin => 'pv' }, + rate_limit => { default => undef, accept => [ "no" ], accept_regexp => qr/^[0-9]+[kmgtKMGT]?$/, require_bin => 'pv' }, transaction_log => { default => undef, accept_file => { absolute => 1 } }, transaction_syslog => { default => undef, accept => \@syslog_facilities }, lockfile => { default => undef, accept_file => { absolute => 1 }, context => [ "root" ] }, @@ -477,6 +477,12 @@ sub check_exe($) return 0; } +sub rate_limit_cmd($) +{ + my $rate = shift; + return "pv -q -L " . lc($rate); +} + sub compress_cmd($;$) { my $def = shift; @@ -628,6 +634,10 @@ sub run_cmd(@) } } + if($href->{rsh_rate_limit_in}) { + push @cmd_pipe, { cmd_text => rate_limit_cmd($href->{rsh_rate_limit_in}) }; + } + if($compressed && (not ($href->{compressed_ok}))) { unshift @rsh_cmd_pipe, { cmd_text => decompress_cmd($compressed) }; $compressed = undef; @@ -639,6 +649,10 @@ sub run_cmd(@) $compressed = $href->{rsh_compress_out}; } + if($href->{rsh_rate_limit_out}) { + push @rsh_cmd_pipe, { cmd_text => rate_limit_cmd($href->{rsh_rate_limit_out}) }; + } + if((scalar(@rsh_cmd_pipe) == 1) && ($rsh_cmd_pipe[0]->{redirect_to_file})) { # NOTE: direct redirection in ssh command does not work: "ssh '> outfile'" # we need to assemble: "ssh 'cat > outfile'" @@ -698,21 +712,11 @@ sub run_cmd(@) } -sub add_pv_command($@) +sub add_progress_command($) { my $cmd_pipe = shift || die; - my %opts = @_; - my $rate_limit = $opts{rate_limit}; - - if($opts{show_progress}) { - if($rate_limit) { - push @$cmd_pipe, { cmd => [ 'pv', '-trab', '-L', $rate_limit ], compressed_ok => 1 }; - } else { - push @$cmd_pipe, { cmd => [ 'pv', '-trab' ], compressed_ok => 1 }; - } - } - elsif($rate_limit) { - push @$cmd_pipe, { cmd => [ 'pv', '-q', '-L', $rate_limit ], compressed_ok => 1 }; + if($show_progress) { + push @$cmd_pipe, { cmd => [ 'pv', '-trab' ], compressed_ok => 1 }; } } @@ -1132,7 +1136,7 @@ sub btrfs_send_receive($$$$;@) my $target = shift || die; my $parent = shift; my $ret_vol_received = shift; - my %opts = @_; + # my %opts = @_; my $snapshot_path = $snapshot->{PATH} // die; my $target_path = $target->{PATH} // die; my $parent_path = $parent ? $parent->{PATH} : undef; @@ -1157,15 +1161,17 @@ sub btrfs_send_receive($$$$;@) cmd => vinfo_cmd($snapshot, "btrfs send", @send_options, { unsafe => $snapshot_path } ), rsh => vinfo_rsh($snapshot, disable_compression => config_compress_hash($snapshot, "stream_compress")), rsh_compress_out => config_compress_hash($snapshot, "stream_compress"), + rsh_rate_limit_out => config_key($snapshot, "rate_limit"), name => "btrfs send", catch_stderr => 1, # hack for shell-based run_cmd() }; - add_pv_command(\@cmd_pipe, show_progress => $show_progress, rate_limit => $opts{rate_limit}); + add_progress_command(\@cmd_pipe); push @cmd_pipe, { cmd => vinfo_cmd($target, "btrfs receive", @receive_options, { unsafe => $target_path . '/' } ), rsh => vinfo_rsh($target, disable_compression => config_compress_hash($target, "stream_compress")), name => "btrfs receive", rsh_compress_in => config_compress_hash($target, "stream_compress"), + rsh_rate_limit_in => config_key($target, "rate_limit"), catch_stderr => 1, # hack for shell-based run_cmd() filter_stderr => sub { $err = $_; $_ = undef } }; @@ -1296,8 +1302,9 @@ sub btrfs_send_to_file($$$$;@) rsh => vinfo_rsh($source, disable_compression => $opts{compress} || config_compress_hash($source, "stream_compress")), name => "btrfs send", rsh_compress_out => $opts{compress} || config_compress_hash($source, "stream_compress"), + rsh_rate_limit_out => config_key($source, "rate_limit"), }; - add_pv_command(\@cmd_pipe, show_progress => $show_progress, rate_limit => $opts{rate_limit}); + add_progress_command(\@cmd_pipe); if($opts{compress}) { $target_filename .= '.' . $compression{$opts{compress}->{key}}->{format}; push @cmd_pipe, { compress => $opts{compress} }; # does nothing if already compressed by rsh_compress_out @@ -1340,6 +1347,7 @@ sub btrfs_send_to_file($$$$;@) #redirect_to_file => { unsafe => "${target_path}/${target_filename}.part" }, # alternative (use shell redirection), less overhead on local filesystems (barely measurable): rsh => vinfo_rsh($target, disable_compression => $opts{compress} || config_compress_hash($target, "stream_compress")), rsh_compress_in => $opts{compress} || config_compress_hash($target, "stream_compress"), + rsh_rate_limit_in => config_key($target, "rate_limit"), compressed_ok => ($opts{compress} ? 1 : 0), }; @@ -2928,7 +2936,7 @@ sub macro_send_receive(@) my $vol_received; if($target_type eq "send-receive") { - $ret = btrfs_send_receive($source, $target, $parent, \$vol_received, rate_limit => config_key($config_target, "rate_limit")); + $ret = btrfs_send_receive($source, $target, $parent, \$vol_received); ABORTED($config_target, "Failed to send/receive subvolume") unless($ret); } elsif($target_type eq "raw") @@ -2956,7 +2964,6 @@ sub macro_send_receive(@) $ret = btrfs_send_to_file($source, $target, $parent, \$vol_received, compress => config_compress_hash($config_target, "raw_target_compress"), encrypt => $encrypt, - rate_limit => config_key($config_target, "rate_limit"), ); ABORTED($config_target, "Failed to send subvolume to raw file") unless($ret); } diff --git a/doc/btrbk.conf.5 b/doc/btrbk.conf.5 index 5967820..2952cea 100644 --- a/doc/btrbk.conf.5 +++ b/doc/btrbk.conf.5 @@ -262,7 +262,9 @@ Number of threads to use for . Only supported for .RS 4 Limit the transfer to a maximum of \fI\fR bytes per second. A suffix of "k", "m", "g", or "t" can be added to denote kilobytes -(*1024), megabytes, and so on. Defaults to \[lq]no\[rq]. +(*1024), megabytes, and so on. Defaults to \[lq]no\[rq]. If enabled +for remote sources, make sure that the "pv" command is available on +the source host. .RE .PP \fBlockfile\fR diff --git a/ssh_filter_btrbk.sh b/ssh_filter_btrbk.sh index 349d476..b74677c 100755 --- a/ssh_filter_btrbk.sh +++ b/ssh_filter_btrbk.sh @@ -9,6 +9,7 @@ enable_log= restrict_path_list= allow_list= allow_exact_list= +allow_rate_limit=1 allow_compress= compress_list="gzip|pigz|bzip2|pbzip2|xz|lzo|lz4" @@ -66,8 +67,14 @@ reject_filtered_cmd() compress_match= fi + if [[ -n "$allow_rate_limit" ]]; then + rate_limit_match="( \| pv -q -L [0-9]+[kmgt]?)?" + else + rate_limit_match= + fi + # allow multiple paths (e.g. "btrfs subvolume snapshot ") - btrfs_cmd_match="^${decompress_match}(${allow_list})( ${option_match})*( ${path_match})+${compress_match}$" + btrfs_cmd_match="^${decompress_match}(${allow_list})( ${option_match})*( ${path_match})+${compress_match}${rate_limit_match}$" if [[ $SSH_ORIGINAL_COMMAND =~ $btrfs_cmd_match ]] ; then return 0 @@ -164,7 +171,7 @@ case "$SSH_ORIGINAL_COMMAND" in *\<*) reject_and_die "unsafe character" ;; *\>*) reject_and_die "unsafe character" ;; *\`*) reject_and_die "unsafe character" ;; - *\|*) [[ -n "$allow_compress" ]] || reject_and_die "unsafe character (compression disallowed)" ;; + *\|*) [[ -n "$allow_compress" ]] || [[ -n "$allow_rate_limit" ]] || reject_and_die "unsafe character (compression disallowed)" ;; esac reject_filtered_cmd