btrbk: add config option "stream_buffer <size>". if set, pipes send stream through "mbuffer -m <size>"

ChangeLog

@@ -1,3 +1,7 @@
+btrbk-current
+
+  * Add "stream_buffer" configuration option (close #154).
+
 btrbk-0.25.0
 
   * MIGRATION

btrbk

@@ -46,7 +46,7 @@ use Carp qw(confess);
 use Getopt::Long qw(GetOptions);
 use Time::Local qw( timelocal timegm timegm_nocheck );
 
-our $VERSION         = '0.25.0';
+our $VERSION         = '0.25.1-dev';
 our $AUTHOR          = 'Axel Burri <axel@tty0.ch>';
 our $PROJECT_HOME    = '<http://digint.ch/btrbk/>';
 
@@ -106,6 +106,7 @@ my %config_options = (
   ssh_compression             => { default => undef,     accept => [ "yes", "no" ] },
   ssh_cipher_spec             => { default => "default", accept_regexp => qr/^$ssh_cipher_match(,$ssh_cipher_match)*$/ },
   rate_limit                  => { default => undef,     accept => [ "no" ], accept_regexp => qr/^[0-9]+[kmgtKMGT]?$/, require_bin => 'pv' },
+  stream_buffer               => { default => undef,     accept => [ "no" ], accept_regexp => qr/^[0-9]+[kmgKMG%]?$/ },  # NOTE: requires 'mbuffer' command on target
   transaction_log             => { default => undef,     accept_file => { absolute => 1 } },
   transaction_syslog          => { default => undef,     accept => \@syslog_facilities },
   lockfile                    => { default => undef,     accept_file => { absolute => 1 }, context => [ "root" ] },
@@ -521,6 +522,12 @@ sub decompress_cmd($)
   return compress_cmd($_[0], 1);
 }
 
+sub stream_buffer_cmd($)
+{
+  my $bufsize = shift;
+  return "mbuffer -q -m " . lc($bufsize);
+}
+
 
 sub _assemble_cmd($;$)
 {
@@ -639,6 +646,10 @@ sub run_cmd(@)
           push @cmd_pipe, { cmd_text => rate_limit_cmd($href->{rsh_rate_limit_in}) };
         }
 
+        if($href->{stream_buffer}) {
+          unshift @rsh_cmd_pipe, { cmd_text => stream_buffer_cmd($href->{stream_buffer}) };
+        }
+
         if($compressed && (not ($href->{compressed_ok}))) {
           unshift @rsh_cmd_pipe, { cmd_text => decompress_cmd($compressed) };
           $compressed = undef;
@@ -669,6 +680,11 @@ sub run_cmd(@)
           push @cmd_pipe, { cmd_text => decompress_cmd($compressed) };
           $compressed = undef;
         }
+
+        if($href->{stream_buffer}) {
+          push @cmd_pipe, { cmd_text => stream_buffer_cmd($href->{stream_buffer}) };
+        }
+
       }
       push @cmd_pipe, $href;
     }
@@ -1172,6 +1188,7 @@ sub btrfs_send_receive($$$$)
     name => "btrfs receive",
     rsh_compress_in => config_compress_hash($target, "stream_compress"),
     rsh_rate_limit_in => config_key($target, "rate_limit"),
+    stream_buffer => config_key($target, "stream_buffer"),
     catch_stderr => 1, # hack for shell-based run_cmd()
     filter_stderr => sub { $err = $_; $_ = undef }
   };

doc/btrbk.conf.5

@@ -261,6 +261,18 @@ Number of threads to use for <compress_command>. Only supported for
 "pigz", "pbzip2" and recent versions of "xz".
 .RE
 .PP
+\fBstream_buffer\fR  <size>|no  \fI*experimental*\fR
+.RS 4
+Add a buffer to the btrfs send stream (in front of "btrfs receive"),
+with a maximum size of \fI<size>\fR. This can give a speed improvement
+(measured up to 20%) on both local or remote operations, but also
+increases system load. A suffix of "k", "m", "g", or "%" can be added
+to \fI<size>\fR to denote kilobytes (*1024), megabytes, gigabytes, or
+a percentage of total physical memory. Defaults to \[lq]no\[rq]. If
+enabled, make sure that the "mbuffer" command is available on the
+target host.
+.RE
+.PP
 \fBrate_limit\fR  <rate>|no
 .RS 4
 Limit the transfer to a maximum of \fI<rate>\fR bytes per second. A