diff --git a/ChangeLog b/ChangeLog index 935920b..5df2b42 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,6 +8,7 @@ btrbk-current * Allow converting backup disks to source disks (close #114). * Add "backend btrfs-progs-sudo" configuration option (close #115). * Show aggregate "size" and "used" for "usage" action (close #119). + * Add "raw_target_split" configuration option (close #125). * Bugfix: rate limiting must be done after compression (close #134). * raw_target_encrypt: Always set "gpg --no-random-seed-file": prevents creation of "~/.gnupg/random_seed" with slight perfomance diff --git a/btrbk b/btrbk index 62a7787..bbe5c1e 100755 --- a/btrbk +++ b/btrbk @@ -75,7 +75,7 @@ my $file_match = qr/[0-9a-zA-Z_@\+\-\.\/]+/; # note: ubuntu uses '@' in the sub my $glob_match = qr/[0-9a-zA-Z_@\+\-\.\/\*]+/; # file_match plus '*' my $uuid_match = qr/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/; my $timestamp_postfix_match = qr/\.(?[0-9]{4})(?[0-9]{2})(?
[0-9]{2})(T(?[0-9]{2})(?[0-9]{2})((?[0-9]{2})(?(Z|[+-][0-9]{4})))?)?(_(?[0-9]+))?/; # matches "YYYYMMDD[Thhmm[ss+0000]][_NN]" -my $raw_postfix_match = qr/--(?$uuid_match)(\@(?$uuid_match))?\.btrfs?(\.(?($compress_format_alt)))?(\.(?gpg))?(\.(?part))?/; # matches ".btrfs_[@][.gz|bz2|xz][.gpg][.part]" +my $raw_postfix_match = qr/--(?$uuid_match)(\@(?$uuid_match))?\.btrfs?(\.(?($compress_format_alt)))?(\.(?gpg))?(\.(?split))?(\.(?part))?/; # matches ".btrfs_[@][.gz|bz2|xz][.gpg][.split][.part]" my $group_match = qr/[a-zA-Z0-9_:-]+/; my $ssh_cipher_match = qr/[a-z0-9][a-z0-9@.-]+/; @@ -119,6 +119,7 @@ my %config_options = ( raw_target_compress_threads => { default => "default", accept => [ "default" ], accept_numeric => 1 }, raw_target_encrypt => { default => undef, accept => [ "no", "gpg" ] }, raw_target_block_size => { default => "128K", accept_regexp => qr/^[0-9]+(kB|k|K|KiB|MB|M|MiB)?$/ }, + raw_target_split => { default => undef, accept => [ "no" ], accept_regexp => qr/^[0-9]+([kmgtpezyKMGTPEZY][bB]?)?$/ }, gpg_keyring => { default => undef, accept_file => { absolute => 1 } }, gpg_recipient => { default => undef, accept_regexp => qr/^[0-9a-zA-Z_@\+\-\.]+$/ }, @@ -1332,24 +1333,39 @@ sub btrfs_send_to_file($$$$;@) compressed_ok => ($opts{compress} ? 1 : 0), }; } - push @cmd_pipe, { - # NOTE: We use "dd" instead of shell redirections here, as it is - # common to have special filesystems (like NFS, SMB, FUSE) mounted - # on $target_path. By using "dd" we make sure to write in - # reasonably large blocks (default=128K), which is not always the - # case when using redirections (e.g. "gpg > outfile" writes in 8K - # blocks). - # Another approach would be to always pipe through "cat", which - # uses st_blksize from fstat(2) (with a minimum of 128K) to - # determine the block size. - cmd => [ 'dd', 'status=none', 'bs=' . config_key($target, "raw_target_block_size"), "of=${target_path}/${target_filename}.part" ], - check_unsafe => [ { unsafe => "${target_path}/${target_filename}.part" } ], - #redirect_to_file => { unsafe => "${target_path}/${target_filename}.part" }, # alternative (use shell redirection), less overhead on local filesystems (barely measurable): - rsh => vinfo_rsh($target, disable_compression => $opts{compress} || config_compress_hash($target, "stream_compress")), - rsh_compress_in => $opts{compress} || config_compress_hash($target, "stream_compress"), - rsh_rate_limit_in => config_key($target, "rate_limit"), - compressed_ok => ($opts{compress} ? 1 : 0), - }; + + my $split = config_key($target, "raw_target_split"); + if($split) { + $target_filename .= '.split'; + push @cmd_pipe, { + cmd => [ 'split', '-b', uc($split), '-', "${target_path}/${target_filename}_" ], + check_unsafe => [ { unsafe => "${target_path}/${target_filename}_" } ], + rsh => vinfo_rsh($target, disable_compression => $opts{compress} || config_compress_hash($target, "stream_compress")), + rsh_compress_in => $opts{compress} || config_compress_hash($target, "stream_compress"), + rsh_rate_limit_in => config_key($target, "rate_limit"), + compressed_ok => ($opts{compress} ? 1 : 0), + } + } + else { + push @cmd_pipe, { + # NOTE: We use "dd" instead of shell redirections here, as it is + # common to have special filesystems (like NFS, SMB, FUSE) mounted + # on $target_path. By using "dd" we make sure to write in + # reasonably large blocks (default=128K), which is not always the + # case when using redirections (e.g. "gpg > outfile" writes in 8K + # blocks). + # Another approach would be to always pipe through "cat", which + # uses st_blksize from fstat(2) (with a minimum of 128K) to + # determine the block size. + cmd => [ 'dd', 'status=none', 'bs=' . config_key($target, "raw_target_block_size"), "of=${target_path}/${target_filename}.part" ], + check_unsafe => [ { unsafe => "${target_path}/${target_filename}.part" } ], + #redirect_to_file => { unsafe => "${target_path}/${target_filename}.part" }, # alternative (use shell redirection), less overhead on local filesystems (barely measurable): + rsh => vinfo_rsh($target, disable_compression => $opts{compress} || config_compress_hash($target, "stream_compress")), + rsh_compress_in => $opts{compress} || config_compress_hash($target, "stream_compress"), + rsh_rate_limit_in => config_key($target, "rate_limit"), + compressed_ok => ($opts{compress} ? 1 : 0), + }; + } my $vol_received = vinfo_child($target, $target_filename); $$ret_vol_received = $vol_received if(ref $ret_vol_received); @@ -1365,19 +1381,33 @@ sub btrfs_send_to_file($$$$;@) vinfo_prefixed_keys("source", $source), vinfo_prefixed_keys("parent", $parent), ); - my $ret = run_cmd(@cmd_pipe); - if(defined($ret)) { - # Test target file for "exists and size > 0" after writing, - # as we can not rely on the exit status of the command pipe, - # and the shell command always creates the target file. - DEBUG "Testing target file (non-zero size): $target->{PRINT}.part"; + my $ret = ""; + if($split) { + DEBUG "Creating empty part file (split enabled)"; $ret = run_cmd({ - cmd => ['test', '-s', { unsafe => "${target_path}/${target_filename}.part" } ], + cmd => ['touch', { unsafe => "${target_path}/${target_filename}.part" } ], + rsh => vinfo_rsh($target), + name => "touch", + }); + } + if(defined($ret)) { + $ret = run_cmd(@cmd_pipe); + } + + if(defined($ret)) { + # Test target file for "exists and size > 0" after writing, as we + # can not rely on the exit status of the command pipe, and a shell + # redirection as well as "dd" always creates the target file. + # Note that "split" does not create empty files. + my $test_postfix = ($split ? "_aa" : ".part"); + DEBUG "Testing target data file (non-zero size)"; + $ret = run_cmd({ + cmd => ['test', '-s', { unsafe => "${target_path}/${target_filename}${test_postfix}" } ], rsh => vinfo_rsh($target), name => "test", }); if(defined($ret)) { - DEBUG "Renaming target file (remove postfix '.part'): $target->{PRINT}"; + DEBUG "Renaming target file (remove postfix '.part')"; $ret = run_cmd({ cmd => ['mv', { unsafe => "${target_path}/${target_filename}.part" }, { unsafe => "${target_path}/${target_filename}" } ], rsh => vinfo_rsh($target), diff --git a/doc/btrbk.conf.5 b/doc/btrbk.conf.5 index 2952cea..f721b6a 100644 --- a/doc/btrbk.conf.5 +++ b/doc/btrbk.conf.5 @@ -408,7 +408,7 @@ backups (btrbk does not delete any raw files)! Additional options for raw targets: .PP .RS 4 -raw_target_compress gzip|bzip2|xz|no +raw_target_compress gzip|pigz|bzip2|pbzip2|xz|lzo|lz4|no .PD 0 .PP raw_target_compress_level default| @@ -417,6 +417,8 @@ raw_target_compress_threads default| .PP raw_target_encrypt gpg|no .PP +raw_target_split |no +.PP raw_target_block_size (defaults to 128K) .PP gpg_keyring