ambiser
/
btrbk
mirror of https://github.com/digint/btrbk
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

btrbk: add raw_target_split option, using "split" instead of "dd" to write raw files

pull/135/head
Axel Burri 2017-03-18 14:47:43 +01:00
parent 84e5b6243b
commit 28c65e4675
3 changed files with 61 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ChangeLog
View File

@ -8,6 +8,7 @@ btrbk-current
* Allow converting backup disks to source disks (close #114). * Allow converting backup disks to source disks (close #114).
* Add "backend btrfs-progs-sudo" configuration option (close #115). * Add "backend btrfs-progs-sudo" configuration option (close #115).
* Show aggregate "size" and "used" for "usage" action (close #119). * Show aggregate "size" and "used" for "usage" action (close #119).
* Add "raw_target_split" configuration option (close #125).
* Bugfix: rate limiting must be done after compression (close #134). * Bugfix: rate limiting must be done after compression (close #134).
* raw_target_encrypt: Always set "gpg --no-random-seed-file": * raw_target_encrypt: Always set "gpg --no-random-seed-file":
prevents creation of "~/.gnupg/random_seed" with slight perfomance prevents creation of "~/.gnupg/random_seed" with slight perfomance

84
btrbk
View File

@ -75,7 +75,7 @@ my $file_match = qr/[0-9a-zA-Z_@\+\-\.\/]+/; # note: ubuntu uses '@' in the sub
my $glob_match = qr/[0-9a-zA-Z_@\+\-\.\/\*]+/; # file_match plus '*' my $glob_match = qr/[0-9a-zA-Z_@\+\-\.\/\*]+/; # file_match plus '*'
my $uuid_match = qr/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/; my $uuid_match = qr/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/;
my $timestamp_postfix_match = qr/\.(?<YYYY>[0-9]{4})(?<MM>[0-9]{2})(?<DD>[0-9]{2})(T(?<hh>[0-9]{2})(?<mm>[0-9]{2})((?<ss>[0-9]{2})(?<zz>(Z|[+-][0-9]{4})))?)?(_(?<NN>[0-9]+))?/; # matches "YYYYMMDD[Thhmm[ss+0000]][_NN]" my $timestamp_postfix_match = qr/\.(?<YYYY>[0-9]{4})(?<MM>[0-9]{2})(?<DD>[0-9]{2})(T(?<hh>[0-9]{2})(?<mm>[0-9]{2})((?<ss>[0-9]{2})(?<zz>(Z|[+-][0-9]{4})))?)?(_(?<NN>[0-9]+))?/; # matches "YYYYMMDD[Thhmm[ss+0000]][_NN]"
my $raw_postfix_match = qr/--(?<received_uuid>$uuid_match)(\@(?<parent_uuid>$uuid_match))?\.btrfs?(\.(?<compress>($compress_format_alt)))?(\.(?<encrypt>gpg))?(\.(?<incomplete>part))?/; # matches ".btrfs_<received_uuid>[@<parent_uuid>][.gz|bz2|xz][.gpg][.part]" my $raw_postfix_match = qr/--(?<received_uuid>$uuid_match)(\@(?<parent_uuid>$uuid_match))?\.btrfs?(\.(?<compress>($compress_format_alt)))?(\.(?<encrypt>gpg))?(\.(?<split>split))?(\.(?<incomplete>part))?/; # matches ".btrfs_<received_uuid>[@<parent_uuid>][.gz|bz2|xz][.gpg][.split][.part]"
my $group_match = qr/[a-zA-Z0-9_:-]+/; my $group_match = qr/[a-zA-Z0-9_:-]+/;
my $ssh_cipher_match = qr/[a-z0-9][a-z0-9@.-]+/; my $ssh_cipher_match = qr/[a-z0-9][a-z0-9@.-]+/;
@ -119,6 +119,7 @@ my %config_options = (
raw_target_compress_threads => { default => "default", accept => [ "default" ], accept_numeric => 1 }, raw_target_compress_threads => { default => "default", accept => [ "default" ], accept_numeric => 1 },
raw_target_encrypt => { default => undef, accept => [ "no", "gpg" ] }, raw_target_encrypt => { default => undef, accept => [ "no", "gpg" ] },
raw_target_block_size => { default => "128K", accept_regexp => qr/^[0-9]+(kB|k|K|KiB|MB|M|MiB)?$/ }, raw_target_block_size => { default => "128K", accept_regexp => qr/^[0-9]+(kB|k|K|KiB|MB|M|MiB)?$/ },
raw_target_split => { default => undef, accept => [ "no" ], accept_regexp => qr/^[0-9]+([kmgtpezyKMGTPEZY][bB]?)?$/ },
gpg_keyring => { default => undef, accept_file => { absolute => 1 } }, gpg_keyring => { default => undef, accept_file => { absolute => 1 } },
gpg_recipient => { default => undef, accept_regexp => qr/^[0-9a-zA-Z_@\+\-\.]+$/ }, gpg_recipient => { default => undef, accept_regexp => qr/^[0-9a-zA-Z_@\+\-\.]+$/ },
@ -1332,24 +1333,39 @@ sub btrfs_send_to_file($$$$;@)
compressed_ok => ($opts{compress} ? 1 : 0), compressed_ok => ($opts{compress} ? 1 : 0),
}; };
} }
push @cmd_pipe, {
# NOTE: We use "dd" instead of shell redirections here, as it is my $split = config_key($target, "raw_target_split");
# common to have special filesystems (like NFS, SMB, FUSE) mounted if($split) {
# on $target_path. By using "dd" we make sure to write in $target_filename .= '.split';
# reasonably large blocks (default=128K), which is not always the push @cmd_pipe, {
# case when using redirections (e.g. "gpg > outfile" writes in 8K cmd => [ 'split', '-b', uc($split), '-', "${target_path}/${target_filename}_" ],
# blocks). check_unsafe => [ { unsafe => "${target_path}/${target_filename}_" } ],
# Another approach would be to always pipe through "cat", which rsh => vinfo_rsh($target, disable_compression => $opts{compress} || config_compress_hash($target, "stream_compress")),
# uses st_blksize from fstat(2) (with a minimum of 128K) to rsh_compress_in => $opts{compress} || config_compress_hash($target, "stream_compress"),
# determine the block size. rsh_rate_limit_in => config_key($target, "rate_limit"),
cmd => [ 'dd', 'status=none', 'bs=' . config_key($target, "raw_target_block_size"), "of=${target_path}/${target_filename}.part" ], compressed_ok => ($opts{compress} ? 1 : 0),
check_unsafe => [ { unsafe => "${target_path}/${target_filename}.part" } ], }
#redirect_to_file => { unsafe => "${target_path}/${target_filename}.part" }, # alternative (use shell redirection), less overhead on local filesystems (barely measurable): }
rsh => vinfo_rsh($target, disable_compression => $opts{compress} || config_compress_hash($target, "stream_compress")), else {
rsh_compress_in => $opts{compress} || config_compress_hash($target, "stream_compress"), push @cmd_pipe, {
rsh_rate_limit_in => config_key($target, "rate_limit"), # NOTE: We use "dd" instead of shell redirections here, as it is
compressed_ok => ($opts{compress} ? 1 : 0), # common to have special filesystems (like NFS, SMB, FUSE) mounted
}; # on $target_path. By using "dd" we make sure to write in
# reasonably large blocks (default=128K), which is not always the
# case when using redirections (e.g. "gpg > outfile" writes in 8K
# blocks).
# Another approach would be to always pipe through "cat", which
# uses st_blksize from fstat(2) (with a minimum of 128K) to
# determine the block size.
cmd => [ 'dd', 'status=none', 'bs=' . config_key($target, "raw_target_block_size"), "of=${target_path}/${target_filename}.part" ],
check_unsafe => [ { unsafe => "${target_path}/${target_filename}.part" } ],
#redirect_to_file => { unsafe => "${target_path}/${target_filename}.part" }, # alternative (use shell redirection), less overhead on local filesystems (barely measurable):
rsh => vinfo_rsh($target, disable_compression => $opts{compress} || config_compress_hash($target, "stream_compress")),
rsh_compress_in => $opts{compress} || config_compress_hash($target, "stream_compress"),
rsh_rate_limit_in => config_key($target, "rate_limit"),
compressed_ok => ($opts{compress} ? 1 : 0),
};
}
my $vol_received = vinfo_child($target, $target_filename); my $vol_received = vinfo_child($target, $target_filename);
$$ret_vol_received = $vol_received if(ref $ret_vol_received); $$ret_vol_received = $vol_received if(ref $ret_vol_received);
@ -1365,19 +1381,33 @@ sub btrfs_send_to_file($$$$;@)
vinfo_prefixed_keys("source", $source), vinfo_prefixed_keys("source", $source),
vinfo_prefixed_keys("parent", $parent), vinfo_prefixed_keys("parent", $parent),
); );
my $ret = run_cmd(@cmd_pipe); my $ret = "";
if(defined($ret)) { if($split) {
# Test target file for "exists and size > 0" after writing, DEBUG "Creating empty part file (split enabled)";
# as we can not rely on the exit status of the command pipe,
# and the shell command always creates the target file.
DEBUG "Testing target file (non-zero size): $target->{PRINT}.part";
$ret = run_cmd({ $ret = run_cmd({
cmd => ['test', '-s', { unsafe => "${target_path}/${target_filename}.part" } ], cmd => ['touch', { unsafe => "${target_path}/${target_filename}.part" } ],
rsh => vinfo_rsh($target),
name => "touch",
});
}
if(defined($ret)) {
$ret = run_cmd(@cmd_pipe);
}
if(defined($ret)) {
# Test target file for "exists and size > 0" after writing, as we
# can not rely on the exit status of the command pipe, and a shell
# redirection as well as "dd" always creates the target file.
# Note that "split" does not create empty files.
my $test_postfix = ($split ? "_aa" : ".part");
DEBUG "Testing target data file (non-zero size)";
$ret = run_cmd({
cmd => ['test', '-s', { unsafe => "${target_path}/${target_filename}${test_postfix}" } ],
rsh => vinfo_rsh($target), rsh => vinfo_rsh($target),
name => "test", name => "test",
}); });
if(defined($ret)) { if(defined($ret)) {
DEBUG "Renaming target file (remove postfix '.part'): $target->{PRINT}"; DEBUG "Renaming target file (remove postfix '.part')";
$ret = run_cmd({ $ret = run_cmd({
cmd => ['mv', { unsafe => "${target_path}/${target_filename}.part" }, { unsafe => "${target_path}/${target_filename}" } ], cmd => ['mv', { unsafe => "${target_path}/${target_filename}.part" }, { unsafe => "${target_path}/${target_filename}" } ],
rsh => vinfo_rsh($target), rsh => vinfo_rsh($target),

4
doc/btrbk.conf.5
View File

@ -408,7 +408,7 @@ backups (btrbk does not delete any raw files)!
Additional options for raw targets: Additional options for raw targets:
.PP .PP
.RS 4 .RS 4
raw_target_compress gzip|bzip2|xz|no raw_target_compress gzip|pigz|bzip2|pbzip2|xz|lzo|lz4|no
.PD 0 .PD 0
.PP .PP
raw_target_compress_level default|<number> raw_target_compress_level default|<number>
@ -417,6 +417,8 @@ raw_target_compress_threads default|<number>
.PP .PP
raw_target_encrypt gpg|no raw_target_encrypt gpg|no
.PP .PP
raw_target_split <size>|no
.PP
raw_target_block_size <number> (defaults to 128K) raw_target_block_size <number> (defaults to 128K)
.PP .PP
gpg_keyring <file> gpg_keyring <file>