Merge 997decd04d into ba3c36c984

ChangeLog

@@ -514,7 +514,7 @@ btrbk-0.17.0
   * Bugfix: allow "/" as volume name (close: #15).
   * Bugfix: check source AND targets for determining snapshot postfix
     (close: #11).
-  * Bugfix: fixed "diff" action (colses: #14).
+  * Bugfix: fixed "diff" action (closes: #14).
   * Allow '+' character for subvolume names.
   * Filesystems on remote hosts are now printed as
     "{my.remote-host.com}" in summary and logs.

btrbk

@@ -332,6 +332,7 @@ my $fake_uuid_prefix = 'XXXXXXXX-XXXX-XXXX-XXXX-'; # plus 0-padded inject_id: XX
 my $program_name; # "btrbk" or "lsbtr", default to "btrbk"
 my $safe_commands;
 my $dryrun;
+my $progress_total = 0;
 my $loglevel = 1;
 my $quiet;
 my @exclude_vf;
@@ -423,6 +424,7 @@ options:
        --format=FORMAT       change output format, FORMAT=table|long|raw
    -S, --print-schedule      print scheduler details (for the "run" command)
        --progress            show progress bar on send-receive operation
+       --progress-total      show progress bar on send-receive operation with ETA
        --lockfile=FILE       create and check lockfile
        --override=KEY=VALUE  globally override a configuration option
 
@@ -682,6 +684,16 @@ sub stream_buffer_cmd_text($)
   return { cmd_text => join(' ', @cmd) };
 }
 
+sub stream_progress_cmd_text($)
+{
+  my $total_len = shift;
+  return () unless($total_len > 0);
+  my @cmd = ( "pv" );
+  push @cmd, ("-s", $total_len);
+  push @cmd, "-b -t -e -a -r -p -i 2";
+  return { cmd_text => join(' ', @cmd) };
+}
+
 sub compress_cmd_text($;$)
 {
   my $def = shift // die;
@@ -808,6 +820,8 @@ sub run_cmd(@)
   $cmd_pipe_in[0]->{stream_source} = 1;
   $cmd_pipe_in[-1]->{stream_sink}  = 1;
 
+  my $cmd_progress_any = 0;
+
   foreach my $href (@cmd_pipe_in)
   {
     die if(defined($href->{cmd_text}));
@@ -876,6 +890,9 @@ sub run_cmd(@)
     if($href->{rsh}) {
       # honor stream_buffer_remote, rate_limit_remote for stream source / sink
       my @rsh_stream_buffer_in  = $href->{stream_sink}   ? stream_buffer_cmd_text($stream_options->{rsh_sink}) : ();
+      
+      my @rsh_total_progress_in  = exists $href->{total_progress} ? stream_progress_cmd_text($href->{total_progress}) : ();
+      
       my @rsh_stream_buffer_out = $href->{stream_source} ? stream_buffer_cmd_text($stream_options->{rsh_source}) : ();
 
       my @rsh_cmd_pipe = (
@@ -883,6 +900,7 @@ sub run_cmd(@)
         @rsh_stream_buffer_in,
         $href,
         @rsh_stream_buffer_out,
+        @rsh_total_progress_in,
         @rsh_compress_out,
        );
       @decompress_in = ();
@@ -902,13 +920,24 @@ sub run_cmd(@)
 
     # local stream_buffer, rate_limit and show_progress in front of stream sink
     my @stream_buffer_in = $href->{stream_sink} ? stream_buffer_cmd_text($stream_options->{local_sink}) : ();
+    my @total_progress_in  = exists $href->{total_progress} ? stream_progress_cmd_text($href->{total_progress}) : ();
+
+    if ( @total_progress_in ){
+      $cmd_progress_any = 1;
+    }
+
+    if (@stream_buffer_in){
+      $cmd_progress_any = 1;
+    }
     
     push @cmd_pipe, (
       @decompress_in,   # empty if rsh
       @stream_buffer_in,
       @rsh_compress_in, # empty if not rsh
       $href,  # command or rsh_cmd_pipe
+      @total_progress_in,
      );
+
   }
 
   my $cmd = _piped_cmd_txt(\@cmd_pipe);
@@ -924,13 +953,19 @@ sub run_cmd(@)
   }
   DEBUG "### $cmd";
 
-
   # execute command
   my ($pid, $out_fh, $err_fh, @stdout);
+  if ($cmd_progress_any){
+    $err_fh = '>&STDERR';
+  }
+  else{
     $err_fh = gensym;
+  }
   if(eval_quiet { $pid = open3(undef, $out_fh, $err_fh, $cmd); }) {
     chomp(@stdout = readline($out_fh));
+    if (!$cmd_progress_any){
       chomp(@stderr = readline($err_fh));
+    }
     waitpid($pid, 0);
     if($do_trace) {
       if($large_output) {
@@ -1470,11 +1505,12 @@ sub _btrfs_send_options($$;$$)
   return \@send_options;
 }
 
-sub btrfs_send_receive($$;$$$)
+sub btrfs_send_receive($$;$$$$)
 {
   my $snapshot = shift || die;
   my $target = shift || die;
   my $parent = shift;
+  my $total_len = shift;
   my $clone_src = shift // [];
   my $ret_vol_received = shift;
   
@@ -1499,6 +1535,7 @@ sub btrfs_send_receive($$;$$$)
   push @cmd_pipe, {
     cmd => vinfo_cmd($snapshot, "btrfs send", @$send_options, { unsafe => $snapshot->{PATH} }),
     rsh => vinfo_rsh($snapshot, disable_compression => $stream_options->{stream_compress}),
+    total_progress => $total_len,
     stream_options => $stream_options,
     filter_stderr => [ \&_btrfs_filter_stderr, sub { $_ = undef if(/^At subvol/) } ],
   };
@@ -4361,6 +4398,7 @@ sub macro_send_receive(@)
   my $config_target = $target->{CONFIG};
   my $target_type = $config_target->{target_type} || die;
   my $incremental = config_key($config_target, "incremental");
+  my $total_len = $info{total_len};
 
   # check for existing target subvolume
   if(my $err_vol = vinfo_subvol($target, $source->{NAME})) {
@@ -4405,7 +4443,7 @@ sub macro_send_receive(@)
   my $raw_info;
   if($target_type eq "send-receive")
   {
-    $ret = btrfs_send_receive($source, $target, $parent, \@clone_src, \$vol_received);
+    $ret = btrfs_send_receive($source, $target, $parent, $total_len, \@clone_src, \$vol_received);
     ABORTED($config_target, "Failed to send/receive subvolume") unless($ret);
   }
   elsif($target_type eq "raw")
@@ -5188,6 +5226,40 @@ sub exit_status
 }
 
 
+sub sub_diff($$) {
+  #
+  # calc snapshot diff (btrfs find-new)
+  #
+  my $src_vol = $_[0];
+  my $target_vol = $_[1];
+
+  if (!defined $src_vol) {
+    return 0;
+  }
+
+  # NOTE: in some cases "cgen" differs from "gen", even for read-only snapshots (observed: gen=cgen+1)
+  my $lastgen = $src_vol->{node}{gen} + 1;
+
+  # dump files, sorted and unique
+  my $ret = btrfs_subvolume_find_new($target_vol, $lastgen);
+  
+  INFO "Listing changed files for subvolume: $target_vol->{PRINT}  (gen=$target_vol->{node}{gen})";
+  INFO "Starting at generation after subvolume: $src_vol->{PRINT}  (gen=$src_vol->{node}{gen})";
+  INFO "Listing files modified within generation range: [$lastgen..$target_vol->{node}{gen}]";
+  DEBUG "Newest file generation (transid marker) was: $ret->{transid_marker}";
+  my $files = $ret->{files};
+  my $total_len = 0;
+  foreach my $name (sort keys %$files) {
+    my $finfo = $files->{$name};
+    $total_len += $finfo->{len};
+  }
+
+  INFO "Total size: " . print_size($total_len) ;
+  
+  return $total_len;
+}
+
+
 
 MAIN:
 {
@@ -5252,6 +5324,7 @@ MAIN:
     'preserve-backups'   => sub { $preserve_backups = "preserve-backups" },
     'wipe'               => \$wipe_snapshots,
     'progress'           => \$show_progress,
+    'progress-total'     => \$progress_total,
     'related'            => \$extents_related,
     'table|t'            => sub { $output_format = "table" },
     'long|L'             => sub { $output_format = "long" },
@@ -6819,11 +6892,20 @@ MAIN:
                   clone_src          => \$clone_src,
                   target_parent_node => \$target_parent_node,
                  );
+                my $total_len;
+                if ($dryrun || !$progress_total) {
+                  $total_len = 0;
+                }
+                else{
+                  $show_progress = 0;
+                  $total_len = sub_diff($parent, $snapshot);
+                }
                 if(macro_send_receive(source    => $snapshot,
                                       target    => $droot,
                                       parent    => $parent,  # this is <undef> if no suitable parent found
                                       clone_src => $clone_src,
                                       target_parent_node => $target_parent_node,
+                                      total_len => $total_len,
                                      ))
                 {
                   $resume_success++;