From 823d012fb740ee265b314efac1b502bd55c5cf41 Mon Sep 17 00:00:00 2001 From: Tobias Grosser Date: Sat, 6 Aug 2016 09:23:57 +0200 Subject: [PATCH 1/3] Disable compression in gpg We already perform compression before gpg, such that compressing in gpg is just a waste of time. Interestingly, it seems gpg is not trying to recompress gzip[ed] input streams, as for the default gzip compression this patch does not change performance. However, it is necessary for the upcoming lz4 compression to show its real benefit. --- btrbk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrbk b/btrbk index 6dc7bf9..87a14b1 100755 --- a/btrbk +++ b/btrbk @@ -1092,7 +1092,7 @@ sub btrfs_send_to_file($$$$;@) if($opts{encrypt}) { die unless($opts{encrypt}->{type} eq "gpg"); $target_filename .= '.gpg'; - my @gpg_options = ( '--batch', '--no-tty', '--trust-model', 'always' ); + my @gpg_options = ( '--batch', '--no-tty', '--trust-model', 'always', '--compress-algo', 'none' ); push(@gpg_options, ( '--no-default-keyring', '--keyring', $opts{encrypt}->{keyring} )) if($opts{encrypt}->{keyring}); push(@gpg_options, ( '--default-recipient', $opts{encrypt}->{recipient} )) if($opts{encrypt}->{recipient}); push @cmd_pipe, { From 6f32f5d3c9caa5dc50769b1b0c787236561e0c12 Mon Sep 17 00:00:00 2001 From: Tobias Grosser Date: Sat, 6 Aug 2016 09:23:33 +0200 Subject: [PATCH 2/3] Add lz4 compression support for fast compression speeds Here some statistic for my local backup: Size Time uncompressed 347M 4.4s gzip 105M 40.0s lz4 153M 5.2s Moving from gzip to lz4 increases the backup performance from 9 MB/s to 66 MB/s (measured un the uncompressed stream). As I am backing up over GBit ethernet, this makes a big difference. --- btrbk | 5 +++-- doc/btrbk.conf.5 | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/btrbk b/btrbk index 87a14b1..0c0dcc7 100755 --- a/btrbk +++ b/btrbk @@ -63,7 +63,7 @@ my $file_match = qr/[0-9a-zA-Z_@\+\-\.\/]+/; # note: ubuntu uses '@' in the sub my $glob_match = qr/[0-9a-zA-Z_@\+\-\.\/\*]+/; # file_match plus '*' my $uuid_match = qr/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/; my $timestamp_postfix_match = qr/\.(?[0-9]{4})(?[0-9]{2})(?
[0-9]{2})(T(?[0-9]{2})(?[0-9]{2})((?[0-9]{2})(?(Z|[+-][0-9]{4})))?)?(_(?[0-9]+))?/; # matches "YYYYMMDD[Thhmm[ss+0000]][_NN]" -my $raw_postfix_match = qr/--(?$uuid_match)(\@(?$uuid_match))?\.btrfs?(\.(?(gz|bz2|xz)))?(\.(?gpg))?(\.(?part))?/; # matches ".btrfs_[@][.gz|bz2|xz][.gpg][.part]" +my $raw_postfix_match = qr/--(?$uuid_match)(\@(?$uuid_match))?\.btrfs?(\.(?(gz|bz2|lz4|xz)))?(\.(?gpg))?(\.(?part))?/; # matches ".btrfs_[@][.gz|bz2|lz4|xz][.gpg][.part]" my $group_match = qr/[a-zA-Z0-9_:-]+/; my $ssh_cipher_match = qr/[a-z0-9][a-z0-9@.-]+/; my $safe_cmd_match = qr/[0-9a-zA-Z_@=\+\-\.\/]+/; # $file_match plus '=': good enough for our purpose @@ -98,7 +98,7 @@ my %config_options = ( transaction_syslog => { default => undef, accept => \@syslog_facilities }, lockfile => { default => undef, accept_file => { absolute => 1 }, context => [ "root" ] }, - raw_target_compress => { default => undef, accept => [ "no", "gzip", "bzip2", "xz" ] }, + raw_target_compress => { default => undef, accept => [ "no", "gzip", "bzip2", "lz4", "xz" ] }, raw_target_compress_level => { default => "default", accept => [ "default" ], accept_numeric => 1 }, raw_target_compress_threads => { default => "default", accept => [ "default" ], accept_numeric => 1 }, raw_target_encrypt => { default => undef, accept => [ "no", "gpg" ] }, @@ -1046,6 +1046,7 @@ sub btrfs_send_to_file($$$$;@) my %compress = ( gzip => { name => 'gzip' , cmd => [ 'gzip' ], postfix => '.gz', level_min => 1, level_max => 9 }, bzip2 => { name => 'bzip2', cmd => [ 'bzip2' ], postfix => '.bz2', level_min => 1, level_max => 9 }, + lz4 => { name => 'lz4' , cmd => [ 'lz4' ], postfix => '.lz4', level_min => 1, level_max => 9 }, xz => { name => 'xz' , cmd => [ 'xz' ], postfix => '.xz', level_min => 0, level_max => 9, threads => '--threads=' }, ); diff --git a/doc/btrbk.conf.5 b/doc/btrbk.conf.5 index 68feabc..86be59f 100644 --- a/doc/btrbk.conf.5 +++ b/doc/btrbk.conf.5 @@ -348,7 +348,7 @@ backups (btrbk does not delete any raw files)! Additional options for raw targets: .PP .RS 4 -raw_target_compress gzip|bzip2|xz|no +raw_target_compress gzip|bzip2|lz4|xz|no .PD 0 .PP raw_target_compress_level default| From 7a732ce6cede35c5ba4fa87d076a3376f7a50926 Mon Sep 17 00:00:00 2001 From: Tobias Grosser Date: Sat, 6 Aug 2016 09:30:26 +0200 Subject: [PATCH 3/3] Use 4M block size for dd By default dd uses blocks of size 512 Byte, which does not work well on network mounted fuse file systems (an acd_cli mount). Before this change copying 150MB took more than 5 minutes (I aborted), after this change it takes only 9 seconds. --- btrbk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrbk b/btrbk index 0c0dcc7..ab5e6ea 100755 --- a/btrbk +++ b/btrbk @@ -1102,7 +1102,7 @@ sub btrfs_send_to_file($$$$;@) }; } push @cmd_pipe, { - cmd => [ 'dd', 'status=none', "of=${target_path}/${target_filename}.part" ], + cmd => [ 'dd', 'status=none', 'bs=4M', "of=${target_path}/${target_filename}.part" ], rsh => $target->{RSH}, name => 'dd', };