From d445dd0b120474abe348fd73c992177e4247b1dc Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Sat, 10 Oct 2015 15:13:32 +0200 Subject: [PATCH 01/19] btrbk: print "Dryrun: YES" in backup summary header if dryrun is set --- btrbk | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/btrbk b/btrbk index 39485b5..38d8ba4 100755 --- a/btrbk +++ b/btrbk @@ -1599,19 +1599,20 @@ sub print_header(@) } if($config) { print " Config: $config->{SRC_FILE}\n"; - - if($config->{CMDLINE_FILTER_LIST}) - { - my @list = sort @{$config->{CMDLINE_FILTER_LIST}}; - my @sorted = ( grep(/^group/, @list), - grep(/^volume/, @list), - grep(/^subvolume/, @list), - grep(/^target/, @list) ); - die unless(scalar(@list) == scalar(@sorted)); - print " Filter: "; - print join("\n ", @sorted); - print "\n"; - } + } + if($dryrun) { + print " Dryrun: YES\n"; + } + if($config && $config->{CMDLINE_FILTER_LIST}) { + my @list = sort @{$config->{CMDLINE_FILTER_LIST}}; + my @sorted = ( grep(/^group/, @list), + grep(/^volume/, @list), + grep(/^subvolume/, @list), + grep(/^target/, @list) ); + die unless(scalar(@list) == scalar(@sorted)); + print " Filter: "; + print join("\n ", @sorted); + print "\n"; } if($args{info}) { print "\n" . join("\n", grep(defined, @{$args{info}})) . "\n"; From e87373b0431e826d1630f82f68d3f5857c05c27b Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Sat, 10 Oct 2015 21:26:59 +0200 Subject: [PATCH 02/19] btrbk: add "config print" action: prints internal representation of config --- ChangeLog | 1 + btrbk | 128 ++++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 112 insertions(+), 17 deletions(-) diff --git a/ChangeLog b/ChangeLog index 40f1f47..4c8325f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,7 @@ btrbk-current * Added "--raw-output" command line option, producing raw (machine-readable) output for "(dry)run" and "tree" commands. * Added "config dump" command (experimental). + * Added "config list" command (experimental). * Added configuration option "ssh_cipher_spec" (close: #47). * Added "target raw", with GnuPG and compression support (experimental). diff --git a/btrbk b/btrbk index 38d8ba4..bfb4dd1 100755 --- a/btrbk +++ b/btrbk @@ -390,6 +390,44 @@ sub config_key($$;@) } +sub config_dump_keys($;@) +{ + my $config = shift || die; + my %opts = @_; + my @ret; + my $maxlen = 0; + + foreach my $key (sort keys %config_options) + { + my $val; + if($opts{resolve}) { + $val = config_key($config, $key); + } else { + next unless exists($config->{$key}); + $val = $config->{$key}; + } + if($opts{skip_defaults}) { + if(defined($config_options{$key}->{default}) && defined($val)) { + next if($val eq $config_options{$key}->{default}); + } + if((not defined($config_options{$key}->{default})) && (not (defined($val)))) { + next; # both undef, skip + } + } + if(ref($val) eq "ARRAY") { + my $val2 = join(',', @$val); + $val = $val2; + } + $val //= ""; + my $len = length($key); + $maxlen = $len if($len > $maxlen); + push @ret, { key => $key, val => $val, len => $len }; + } + # print as table + return map { ($opts{prefix} // "") . $_->{key} . (' ' x (1 + $maxlen - $_->{len})) . ' ' . $_->{val} } @ret; +} + + sub check_file($$;$$) { my $file = shift // die; @@ -1679,7 +1717,7 @@ MAIN: WARN 'found option "--progress", but "pv" is not present: (please install "pv")'; $show_progress = 0; } - my ($action_run, $action_info, $action_tree, $action_diff, $action_origin, $action_config); + my ($action_run, $action_info, $action_tree, $action_diff, $action_origin, $action_config_print, $action_config_dump); my @filter_args; my $args_allow_group = 0; my ($args_expected_min, $args_expected_max) = (0, 0); @@ -1716,20 +1754,36 @@ MAIN: @filter_args = @ARGV; } elsif ($command eq "config") { - $action_config = shift @ARGV // ""; - $action_config .= ' ' . (shift @ARGV // ""); - unless(($action_config eq "dump source") || - ($action_config eq "dump target") || - ($action_config eq "dump volume")) - { - ERROR "unknown subcommand for \"config\" command: $action_config"; + my $subcommand = shift @ARGV // ""; + if(($subcommand eq "print") || ($subcommand eq "print-all")) { + $action_config_print = $subcommand; + $args_expected_min = 0; + $args_expected_max = 9999; + $args_allow_group = 1; + @filter_args = @ARGV; + } + elsif($subcommand eq "dump") { + $action_config_dump = shift @ARGV // ""; + if(($action_config_dump eq "source") || + ($action_config_dump eq "target") || + ($action_config_dump eq "volume")) + { + $args_expected_min = 0; + $args_expected_max = 9999; + $args_allow_group = 1; + @filter_args = @ARGV; + } + else { + ERROR "Unknown subcommand for \"config dump\" command: $action_config_dump"; + HELP_MESSAGE(0); + exit 2; + } + } + else { + ERROR "Unknown subcommand for \"config\" command: $subcommand"; HELP_MESSAGE(0); exit 2; } - $args_expected_min = 0; - $args_expected_max = 9999; - $args_allow_group = 1; - @filter_args = @ARGV; } else { ERROR "Unrecognized command: $command"; @@ -1883,7 +1937,7 @@ MAIN: # # filter subvolumes matching command line arguments # - if(($action_run || $action_tree || $action_info || $action_config) && scalar(@filter_args)) + if(($action_run || $action_tree || $action_info || $action_config_dump || $action_config_print) && scalar(@filter_args)) { my %match; foreach my $config_vol (@{$config->{VOLUME}}) { @@ -2009,7 +2063,47 @@ MAIN: } - if($action_config) + if($action_config_print) + { + my $resolve = ($action_config_print eq "print-all"); + # + # print configuration lines, machine readable + # + my @out; + push @out, config_dump_keys($config, skip_defaults => 1); + foreach my $config_vol (@{$config->{VOLUME}}) { + next if($config_vol->{ABORTED}); + my $sroot = vinfo($config_vol->{url}, $config_vol); + push @out, "\nvolume $sroot->{URL}"; + push @out, config_dump_keys($config_vol, prefix => "\t", resolve => $resolve); + + foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { + next if($config_subvol->{ABORTED}); + my $svol = vinfo_child($sroot, $config_subvol->{rel_path}); + # push @out, "\n subvolume $svol->{URL}"; + push @out, "\n\tsubvolume $svol->{SUBVOL_PATH}"; + push @out, config_dump_keys($config_subvol, prefix => "\t\t", resolve => $resolve); + + foreach my $config_target (@{$config_subvol->{TARGET}}) + { + next if($config_target->{ABORTED}); + my $droot = vinfo($config_target->{url}, $config_target); + push @out, "\n\t\ttarget $config_target->{target_type} $droot->{URL}"; + push @out, config_dump_keys($config_target, prefix => "\t\t\t", resolve => $resolve); + } + } + } + + print_header(title => "Configuration Dump", + config => $config, + time => $start_time, + ); + + print join("\n", @out) . "\n"; + exit 0; + } + + if($action_config_dump) { # # print configuration lines, machine readable @@ -2023,7 +2117,7 @@ MAIN: push @vol_info, "volume_path=\"$sroot->{PATH}\""; push @vol_info, "volume_rsh=\"" . ($sroot->{RSH} ? join(" ", @{$sroot->{RSH}}) : "") . "\""; - if($action_config eq "dump volume") { + if($action_config_dump eq "volume") { print join(' ', @vol_info) . "\n"; next; } @@ -2040,10 +2134,10 @@ MAIN: push @sline, "source_host=\"" . ($svol->{HOST} // "") . "\""; push @sline, "source_rsh=\"" . ($svol->{RSH} ? join(" ", @{$svol->{RSH}}) : "") . "\""; - if($action_config eq "dump source") { + if($action_config_dump eq "source") { print join(' ', @sline) . "\n"; } - elsif($action_config eq "dump target") + elsif($action_config_dump eq "target") { foreach my $config_target (@{$config_subvol->{TARGET}}) { From 03db6883c704ce9bc4fcc299512382d259e4275a Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Sun, 11 Oct 2015 01:44:13 +0200 Subject: [PATCH 03/19] btrbk: add framework for printing formatted output (tabular or key="value"...) --- btrbk | 134 ++++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 102 insertions(+), 32 deletions(-) diff --git a/btrbk b/btrbk index bfb4dd1..081f070 100755 --- a/btrbk +++ b/btrbk @@ -1664,6 +1664,49 @@ sub print_header(@) } +sub print_formatted($$$) +{ + my $topic = shift || die; + my $view = shift || die; + my $format = shift; + my $data = $format->{$topic}->{data} || die; + my $keys = $format->{$topic}->{views}->{$view} || die; + + if($view eq "table") + { + # calculate maxlen for each column + my %maxlen; + foreach my $row (@$data) { + foreach (@$keys) { + my $val = $row->{$_} // "-"; + $maxlen{$_} //= length($_); # initialize with size of key + $maxlen{$_} = length($val) if($maxlen{$_} < length($val)); + } + } + + # print keys + print join(" ", map { $_ . (' ' x ($maxlen{$_} - length($_))) } @$keys) . "\n"; + print join(" ", map { '-' x ($maxlen{$_}) } @$keys) . "\n"; + + # print values + foreach my $row (@$data) { + foreach (@$keys) { + my $val = $row->{$_} // "-"; + print $val . (' ' x (2 + $maxlen{$_} - length($val))); + } + print "\n"; + } + } + else + { + # output: key0="value0" key1="value1" ... + foreach my $row (@$data) { + print join(' ', map { "$_=\"" . ($row->{$_} // "") . "\""; } @$keys) . "\n"; + } + } +} + + MAIN: { # set PATH instead of using absolute "/sbin/btrfs" (for now), as @@ -1766,6 +1809,7 @@ MAIN: $action_config_dump = shift @ARGV // ""; if(($action_config_dump eq "source") || ($action_config_dump eq "target") || + ($action_config_dump eq "target_uniq") || ($action_config_dump eq "volume")) { $args_expected_min = 0; @@ -2103,56 +2147,82 @@ MAIN: exit 0; } + if($action_config_dump) { + my @vol; + my @subvol; + my @target; + my %target_uniq; + # # print configuration lines, machine readable # foreach my $config_vol (@{$config->{VOLUME}}) { next if($config_vol->{ABORTED}); my $sroot = vinfo($config_vol->{url}, $config_vol); - - my @vol_info; - push @vol_info, "volume_url=\"$sroot->{URL}\""; - push @vol_info, "volume_path=\"$sroot->{PATH}\""; - push @vol_info, "volume_rsh=\"" . ($sroot->{RSH} ? join(" ", @{$sroot->{RSH}}) : "") . "\""; - - if($action_config_dump eq "volume") { - print join(' ', @vol_info) . "\n"; - next; - } + my $volh = { volume_url => $sroot->{URL}, + volume_path => $sroot->{PATH}, + volume_host => $sroot->{HOST}, + volume_rsh => ($sroot->{RSH} ? join(" ", @{$sroot->{RSH}}) : undef), + }; + push @vol, $volh; foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { next if($config_subvol->{ABORTED}); my $svol = vinfo_child($sroot, $config_subvol->{rel_path}); + my $subvolh = { %$volh, + source_url => $svol->{URL}, + source_path => $svol->{PATH}, + snapshot_path => $sroot->{PATH} . (config_key($config_subvol, "snapshot_dir", prefix => '/') // ""), + snapshot_basename => config_key($config_subvol, "snapshot_name"), + source_host => $svol->{HOST}, + source_rsh => ($svol->{RSH} ? join(" ", @{$svol->{RSH}}) : undef), + }; + push @subvol, $subvolh; - my @sline; - push @sline, "source_url=\"$svol->{URL}\""; - push @sline, "source_path=\"$svol->{PATH}\""; - push @sline, "snapshot_path=\"$sroot->{PATH}" . (config_key($config_subvol, "snapshot_dir", prefix => '/') // "") . "\""; - push @sline, "snapshot_basename=\"" . config_key($config_subvol, "snapshot_name") . "\""; - push @sline, "source_host=\"" . ($svol->{HOST} // "") . "\""; - push @sline, "source_rsh=\"" . ($svol->{RSH} ? join(" ", @{$svol->{RSH}}) : "") . "\""; - - if($action_config_dump eq "source") { - print join(' ', @sline) . "\n"; - } - elsif($action_config_dump eq "target") + foreach my $config_target (@{$config_subvol->{TARGET}}) { - foreach my $config_target (@{$config_subvol->{TARGET}}) - { - next if($config_target->{ABORTED}); - my $droot = vinfo($config_target->{url}, $config_target); - my @dline = @sline; - push @dline, "target_url=\"$droot->{URL}\""; - push @dline, "target_path=\"$droot->{PATH}\""; - push @dline, "target_host=\"" . ($droot->{HOST} // "") . "\""; - push @dline, "target_rsh=\"" . ($droot->{RSH} ? join(" ", @{$droot->{RSH}}) : "") . "\""; - print join(' ', @dline) . "\n"; + next if($config_target->{ABORTED}); + my $droot = vinfo($config_target->{url}, $config_target); + my $targeth = { %$subvolh, + target_url => $droot->{URL}, + target_path => $droot->{PATH}, + target_host => $droot->{HOST}, + target_rsh => ($droot->{RSH} ? join(" ", @{$droot->{RSH}}) : undef), + }; + if($action_config_dump eq "target_uniq") { + next if($target_uniq{$droot->{URL}}); + $target_uniq{$droot->{URL}} = 1; } + push @target, $targeth; } } } + + my @all_vol_keys = qw( volume_url volume_path volume_host volume_rsh ); + my @all_subvol_keys = qw( source_url source_path snapshot_path snapshot_basename source_host source_rsh ); + my @all_target_keys = qw( target_url target_path target_host target_rsh ); + print_formatted( + $action_config_dump, + "table", + { volume => { data => \@vol, + views => { raw => \@all_vol_keys, + table => [ qw( volume_host volume_path ) ] }, + }, + source => { data => \@subvol, + views => { raw => \@all_subvol_keys, + table => [ qw( source_host source_path snapshot_path snapshot_basename ) ] }, + }, + target => { data => \@target, + views => { raw => [ @all_subvol_keys, @all_target_keys ], + table => [ qw( source_host source_path snapshot_path snapshot_basename target_host target_path ) ] }, + }, + target_uniq => { data => [ @target ], + views => { raw => \@all_target_keys, + table => [ qw( target_host target_path ) ] }, + }, + }); exit 0; } From 7bb7883be7a3033763d6b052de49a1c0febb1c8a Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Sun, 11 Oct 2015 02:02:45 +0200 Subject: [PATCH 04/19] btrbk: changed command "config dump" to "list", with new --format=table|raw command line option (needs cleanup) --- btrbk | 70 +++++++++++++++++++++++++---------------------------------- 1 file changed, 30 insertions(+), 40 deletions(-) diff --git a/btrbk b/btrbk index 081f070..70d55b6 100755 --- a/btrbk +++ b/btrbk @@ -1668,9 +1668,9 @@ sub print_formatted($$$) { my $topic = shift || die; my $view = shift || die; - my $format = shift; - my $data = $format->{$topic}->{data} || die; - my $keys = $format->{$topic}->{views}->{$view} || die; + my $spec = shift; + my $data = $spec->{$topic}->{data} || die; + my $keys = $spec->{$topic}->{views}->{$view} || die; if($view eq "table") { @@ -1721,7 +1721,7 @@ MAIN: my @today = @today_and_now[0..2]; - my ($config_cmdline, $quiet, $verbose, $preserve_backups, $resume_only); + my ($config_cmdline, $quiet, $verbose, $preserve_backups, $resume_only, $output_format); unless(GetOptions( 'help|h' => sub { VERSION_MESSAGE(); HELP_MESSAGE(0); exit 0; }, 'version' => sub { VERSION_MESSAGE(); exit 0; }, @@ -1733,6 +1733,7 @@ MAIN: 'loglevel|l=s' => \$loglevel, 'progress' => \$show_progress, 'raw-output' => \$raw_output, + 'format=s' => \$output_format, )) { VERSION_MESSAGE(); @@ -1760,7 +1761,7 @@ MAIN: WARN 'found option "--progress", but "pv" is not present: (please install "pv")'; $show_progress = 0; } - my ($action_run, $action_info, $action_tree, $action_diff, $action_origin, $action_config_print, $action_config_dump); + my ($action_run, $action_info, $action_tree, $action_diff, $action_origin, $action_config_print, $action_list); my @filter_args; my $args_allow_group = 0; my ($args_expected_min, $args_expected_max) = (0, 0); @@ -1796,6 +1797,13 @@ MAIN: $args_expected_min = $args_expected_max = 1; @filter_args = @ARGV; } + elsif($command eq "list") { + $action_list = "target"; + $args_expected_min = 0; + $args_expected_max = 9999; + $args_allow_group = 1; + @filter_args = @ARGV; + } elsif ($command eq "config") { my $subcommand = shift @ARGV // ""; if(($subcommand eq "print") || ($subcommand eq "print-all")) { @@ -1805,24 +1813,6 @@ MAIN: $args_allow_group = 1; @filter_args = @ARGV; } - elsif($subcommand eq "dump") { - $action_config_dump = shift @ARGV // ""; - if(($action_config_dump eq "source") || - ($action_config_dump eq "target") || - ($action_config_dump eq "target_uniq") || - ($action_config_dump eq "volume")) - { - $args_expected_min = 0; - $args_expected_max = 9999; - $args_allow_group = 1; - @filter_args = @ARGV; - } - else { - ERROR "Unknown subcommand for \"config dump\" command: $action_config_dump"; - HELP_MESSAGE(0); - exit 2; - } - } else { ERROR "Unknown subcommand for \"config\" command: $subcommand"; HELP_MESSAGE(0); @@ -1981,7 +1971,7 @@ MAIN: # # filter subvolumes matching command line arguments # - if(($action_run || $action_tree || $action_info || $action_config_dump || $action_config_print) && scalar(@filter_args)) + if(($action_run || $action_tree || $action_info || $action_list || $action_config_print) && scalar(@filter_args)) { my %match; foreach my $config_vol (@{$config->{VOLUME}}) { @@ -2148,7 +2138,7 @@ MAIN: } - if($action_config_dump) + if($action_list) { my @vol; my @subvol; @@ -2191,7 +2181,7 @@ MAIN: target_host => $droot->{HOST}, target_rsh => ($droot->{RSH} ? join(" ", @{$droot->{RSH}}) : undef), }; - if($action_config_dump eq "target_uniq") { + if($action_list eq "target_uniq") { next if($target_uniq{$droot->{URL}}); $target_uniq{$droot->{URL}} = 1; } @@ -2204,24 +2194,24 @@ MAIN: my @all_subvol_keys = qw( source_url source_path snapshot_path snapshot_basename source_host source_rsh ); my @all_target_keys = qw( target_url target_path target_host target_rsh ); print_formatted( - $action_config_dump, - "table", - { volume => { data => \@vol, - views => { raw => \@all_vol_keys, - table => [ qw( volume_host volume_path ) ] }, - }, - source => { data => \@subvol, - views => { raw => \@all_subvol_keys, - table => [ qw( source_host source_path snapshot_path snapshot_basename ) ] }, - }, + $action_list, + $output_format // "table", + { #volume => { data => \@vol, + # views => { raw => \@all_vol_keys, + # table => [ qw( volume_host volume_path ) ] }, + # }, + #source => { data => \@subvol, + # views => { raw => \@all_subvol_keys, + # table => [ qw( source_host source_path snapshot_path snapshot_basename ) ] }, + # }, target => { data => \@target, views => { raw => [ @all_subvol_keys, @all_target_keys ], table => [ qw( source_host source_path snapshot_path snapshot_basename target_host target_path ) ] }, }, - target_uniq => { data => [ @target ], - views => { raw => \@all_target_keys, - table => [ qw( target_host target_path ) ] }, - }, + #target_uniq => { data => [ @target ], + # views => { raw => \@all_target_keys, + # table => [ qw( target_host target_path ) ] }, + # }, }); exit 0; } From b1188484f05214fdc4fb857443960360447557c1 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Sun, 11 Oct 2015 14:20:53 +0200 Subject: [PATCH 05/19] btrbk: use command line option "--format=table|raw" in action tree (needs cleanup) --- btrbk | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/btrbk b/btrbk index 70d55b6..b96ac1c 100755 --- a/btrbk +++ b/btrbk @@ -2462,7 +2462,20 @@ MAIN: $droot_compat{$droot->{URL}} = 1 if($droot->{BTRFS_PROGS_COMPAT}); foreach (sort { $a->{SUBVOL_PATH} cmp $b->{SUBVOL_PATH} } get_receive_targets($droot, $snapshot)) { push @out, "| | >>> $_->{PRINT}"; - push @raw_out, "$svol->{URL} $snapshot->{URL} $_->{URL}" + + push @raw_out, { source_url => $svol->{URL}, + source_path => $svol->{PATH}, + snapshot_url => $snapshot->{URL}, + snapshot_path => $snapshot->{PATH}, +# snapshot_basename => config_key($config_subvol, "snapshot_name"), + source_host => $svol->{HOST}, + source_rsh => ($svol->{RSH} ? join(" ", @{$svol->{RSH}}) : undef), + target_url => $_->{URL}, + target_path => $_->{PATH}, + target_host => $_->{HOST}, + target_rsh => ($_->{RSH} ? join(" ", @{$_->{RSH}}) : undef), + }; +# push @raw_out, { sour"$svol->{URL} $snapshot->{URL} $_->{URL}" } } } @@ -2475,8 +2488,15 @@ MAIN: } if($raw_output) { - print join("\n", @raw_out); - print "\n"; + print_formatted( + "default", + $output_format // "table", + { default => { data => \@raw_out, + views => { raw => [ qw( source_host source_path snapshot_path snapshot_basename target_host target_path ) ], + table => [ qw( source_url snapshot_url target_url ) ], + long => [ qw( source_host source_path snapshot_path target_host target_path ) ] }, + }, + }); } else { print_header(title => "Backup Tree", From e5c629e21800f5ef00a49a483fec801d06e1a4eb Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Sun, 11 Oct 2015 15:38:43 +0200 Subject: [PATCH 06/19] btrbk: use formatted output on run/dryrun actions; cleanup --- ChangeLog | 9 ++- btrbk | 220 ++++++++++++++++++++++++++++++++++-------------------- 2 files changed, 145 insertions(+), 84 deletions(-) diff --git a/ChangeLog b/ChangeLog index 4c8325f..296080e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,10 +2,11 @@ btrbk-current * Added configuration option "group". * Allow filtering subcommands by group as well as targets. - * Added "--raw-output" command line option, producing raw - (machine-readable) output for "(dry)run" and "tree" commands. - * Added "config dump" command (experimental). - * Added "config list" command (experimental). + * Added "config print" command. + * Added "list" command (experimental). + * Added "--format=table|raw" command line option, producing tabular + and raw (machine-readable) output for "(dry)run", "tree" and + "list" commands. * Added configuration option "ssh_cipher_spec" (close: #47). * Added "target raw", with GnuPG and compression support (experimental). diff --git a/btrbk b/btrbk index b96ac1c..78d1b7d 100755 --- a/btrbk +++ b/btrbk @@ -125,7 +125,6 @@ my %uuid_fs_map; # map UUID to URL my $dryrun; my $loglevel = 1; my $show_progress = 0; -my $raw_output = 0; my $err = ""; @@ -1664,34 +1663,51 @@ sub print_header(@) } -sub print_formatted($$$) +sub print_formatted($$$$) { my $topic = shift || die; - my $view = shift || die; + my $format = shift || die; + my $default = shift || die; my $spec = shift; my $data = $spec->{$topic}->{data} || die; - my $keys = $spec->{$topic}->{views}->{$view} || die; + my $keys = $spec->{$topic}->{formats}->{$format}; - if($view eq "table") + unless($keys) { + WARN "Unsupported output format \"$format\", defaulting to \"$default\" format."; + $keys = $spec->{$topic}->{formats}->{$default} || die; + $format = $default; + } + + if($format eq "table") { - # calculate maxlen for each column + # sanitize and calculate maxlen for each column + # NOTE: this is destructive on data! my %maxlen; + my @sane_data; + foreach my $key (@$keys) { + $maxlen{$key} = length($key); # initialize with size of key + } foreach my $row (@$data) { - foreach (@$keys) { - my $val = $row->{$_} // "-"; - $maxlen{$_} //= length($_); # initialize with size of key - $maxlen{$_} = length($val) if($maxlen{$_} < length($val)); + foreach my $key (@$keys) { + my $val = $row->{$key}; + if(ref $val eq "ARRAY") { + $val = join(',', @{$val}); + } + $val //= "-"; + $val = "-" if($val eq ""); + $row->{$key} = $val; # write back the sanitized value + $maxlen{$key} = length($val) if($maxlen{$key} < length($val)); } } - # print keys + # print keys (headings) print join(" ", map { $_ . (' ' x ($maxlen{$_} - length($_))) } @$keys) . "\n"; print join(" ", map { '-' x ($maxlen{$_}) } @$keys) . "\n"; # print values foreach my $row (@$data) { foreach (@$keys) { - my $val = $row->{$_} // "-"; + my $val = $row->{$_}; print $val . (' ' x (2 + $maxlen{$_} - length($val))); } print "\n"; @@ -1732,7 +1748,6 @@ MAIN: 'verbose|v' => sub { $loglevel = 2; }, 'loglevel|l=s' => \$loglevel, 'progress' => \$show_progress, - 'raw-output' => \$raw_output, 'format=s' => \$output_format, )) { @@ -2194,22 +2209,24 @@ MAIN: my @all_subvol_keys = qw( source_url source_path snapshot_path snapshot_basename source_host source_rsh ); my @all_target_keys = qw( target_url target_path target_host target_rsh ); print_formatted( - $action_list, - $output_format // "table", + $action_list, # topic + $output_format, # output format + "table", # default output format { #volume => { data => \@vol, - # views => { raw => \@all_vol_keys, + # formats => { raw => \@all_vol_keys, # table => [ qw( volume_host volume_path ) ] }, # }, #source => { data => \@subvol, - # views => { raw => \@all_subvol_keys, + # formats => { raw => \@all_subvol_keys, # table => [ qw( source_host source_path snapshot_path snapshot_basename ) ] }, # }, target => { data => \@target, - views => { raw => [ @all_subvol_keys, @all_target_keys ], - table => [ qw( source_host source_path snapshot_path snapshot_basename target_host target_path ) ] }, + formats => { raw => [ @all_subvol_keys, @all_target_keys ], + table => [ qw( source_host source_path snapshot_path snapshot_basename target_host target_path ) ], + }, }, #target_uniq => { data => [ @target ], - # views => { raw => \@all_target_keys, + # formats => { raw => \@all_target_keys, # table => [ qw( target_host target_path ) ] }, # }, }); @@ -2435,70 +2452,63 @@ MAIN: # print snapshot tree # # TODO: reverse tree: print all backups from $droot and their corresponding source snapshots - my @out; + my @tree_out; my @raw_out; foreach my $config_vol (@{$config->{VOLUME}}) { next if($config_vol->{ABORTED}); my %droot_compat; my $sroot = $config_vol->{sroot} || die; - push @out, "$sroot->{PRINT}"; + push @tree_out, "$sroot->{PRINT}"; foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { next if($config_subvol->{ABORTED}); my $svol = $config_subvol->{svol} || die; - push @out, "|-- $svol->{PRINT}"; + push @tree_out, "|-- $svol->{PRINT}"; foreach my $snapshot (sort { $a->{cgen} cmp $b->{cgen} } get_snapshot_children($sroot, $svol)) { + my $raw_data = { type => "snapshot", + btrbk_flags => [ ], + source_url => $svol->{URL}, + source_host => $svol->{HOST}, + source_path => $svol->{PATH}, + snapshot_url => $snapshot->{URL}, + snapshot_path => $snapshot->{PATH}, + snapshot_basename => config_key($config_subvol, "snapshot_name"), + }; if($snapshot->{cgen} == $svol->{gen}) { - push @out, "| ^== $snapshot->{PATH}"; + push @tree_out, "| ^== $snapshot->{PATH}"; + push @{$raw_data->{btrbk_flags}}, "up-to-date"; } else { - push @out, "| ^-- $snapshot->{PATH}"; + push @tree_out, "| ^-- $snapshot->{PATH}"; } + push @raw_out, $raw_data; foreach my $config_target (@{$config_subvol->{TARGET}}) { next if($config_target->{ABORTED}); my $droot = $config_target->{droot} || die; $droot_compat{$droot->{URL}} = 1 if($droot->{BTRFS_PROGS_COMPAT}); foreach (sort { $a->{SUBVOL_PATH} cmp $b->{SUBVOL_PATH} } get_receive_targets($droot, $snapshot)) { - push @out, "| | >>> $_->{PRINT}"; - - push @raw_out, { source_url => $svol->{URL}, - source_path => $svol->{PATH}, - snapshot_url => $snapshot->{URL}, - snapshot_path => $snapshot->{PATH}, -# snapshot_basename => config_key($config_subvol, "snapshot_name"), - source_host => $svol->{HOST}, - source_rsh => ($svol->{RSH} ? join(" ", @{$svol->{RSH}}) : undef), - target_url => $_->{URL}, - target_path => $_->{PATH}, - target_host => $_->{HOST}, - target_rsh => ($_->{RSH} ? join(" ", @{$_->{RSH}}) : undef), + push @tree_out, "| | >>> $_->{PRINT}"; + push @raw_out, { %$raw_data, + type => "received", + received_url => $_->{URL}, + received_path => $_->{PATH}, + received_host => $_->{HOST}, }; -# push @raw_out, { sour"$svol->{URL} $snapshot->{URL} $_->{URL}" } } } } if(keys %droot_compat) { - push @out, "\nNOTE: Received subvolumes (backups) are guessed by subvolume name for targets:"; - push @out, " - " . join("\n - ", (sort keys %droot_compat)); + push @tree_out, "\nNOTE: Received subvolumes (backups) are guessed by subvolume name for targets:"; + push @tree_out, " - " . join("\n - ", (sort keys %droot_compat)); } - push @out, ""; + push @tree_out, ""; } - if($raw_output) { - print_formatted( - "default", - $output_format // "table", - { default => { data => \@raw_out, - views => { raw => [ qw( source_host source_path snapshot_path snapshot_basename target_host target_path ) ], - table => [ qw( source_url snapshot_url target_url ) ], - long => [ qw( source_host source_path snapshot_path target_host target_path ) ] }, - }, - }); - } - else { + $output_format //= "tree"; + if($output_format eq "tree") { print_header(title => "Backup Tree", config => $config, time => $start_time, @@ -2508,7 +2518,19 @@ MAIN: ">>> received subvolume (backup)", ] ); - print join("\n", @out); + print join("\n", @tree_out); + } + else { + print_formatted( + "default", + $output_format, + "table", + { default => { data => \@raw_out, + formats => { raw => [ qw( source_host source_path snapshot_path snapshot_basename btrbk_flags received_host received_path ) ], + table => [ qw( source_url snapshot_url btrbk_flags received_url ) ], + }, + }, + }); } exit 0; } @@ -2881,10 +2903,7 @@ MAIN: { my @unrecoverable; my @out; - my @raw_snapshot_out; - my @raw_delete_out; - my @raw_receive_out; - my @raw_err_out; + my @raw_data; my $err_count = 0; foreach my $config_vol (@{$config->{VOLUME}}) { @@ -2899,11 +2918,22 @@ MAIN: } if($config_subvol->{SNAPSHOT}) { push @subvol_out, "+++ $config_subvol->{SNAPSHOT}->{PRINT}"; - push @raw_snapshot_out, "snapshot $config_subvol->{SNAPSHOT}->{URL} $svol->{URL}"; + push @raw_data, { type => "snapshot", + status => $dryrun ? "DRYRUN" : "success", + target_url => $config_subvol->{SNAPSHOT}->{URL}, + source_url => $svol->{URL}, + SORT => 10, + }; } if($config_subvol->{SUBVOL_DELETED}) { - push @subvol_out, "--- $_->{PRINT}" foreach(sort { $a->{PATH} cmp $b->{PATH} } @{$config_subvol->{SUBVOL_DELETED}}); - push @raw_delete_out, "delete $_->{URL}" foreach(@{$config_subvol->{SUBVOL_DELETED}}); + foreach(sort { $a->{PATH} cmp $b->{PATH} } @{$config_subvol->{SUBVOL_DELETED}}) { + push @subvol_out, "--- $_->{PRINT}"; + push @raw_data, { type => "snapshot_delete", + status => $dryrun ? "DRYRUN" : "success", + target_url => $_->{URL}, + SORT => 30, + }; + } } foreach my $config_target (@{$config_subvol->{TARGET}}) { @@ -2914,21 +2944,34 @@ MAIN: # substr($create_mode, 0, 1, '%') if($_->{resume}); $create_mode = "!!!" if($_->{ERROR}); push @subvol_out, "$create_mode $_->{received_subvolume}->{PRINT}"; - if($_->{ERROR}) { - push @raw_err_out, "error receive $_->{received_subvolume}->{URL} $_->{snapshot}->{URL}" . ($_->{parent} ? " $_->{parent}->{URL}" : ""); - } else { - push @raw_receive_out, "receive $_->{received_subvolume}->{URL} $_->{snapshot}->{URL}" . ($_->{parent} ? " $_->{parent}->{URL}" : ""); - } + push @raw_data, { type => "send-receive", + status => $_->{ERROR} ? "ERROR" : ($dryrun ? "DRYRUN" : "success"), + target_url => $_->{received_subvolume}->{URL}, + source_url => $_->{snapshot}->{URL}, + parent_url => $_->{parent}->{URL}, + SORT => 20, + }; } if($config_target->{SUBVOL_DELETED}) { - push @subvol_out, "--- $_->{PRINT}" foreach(sort { $a->{PATH} cmp $b->{PATH} } @{$config_target->{SUBVOL_DELETED}}); - push @raw_delete_out, "delete $_->{URL}" foreach(@{$config_target->{SUBVOL_DELETED}}); + foreach(sort { $a->{PATH} cmp $b->{PATH} } @{$config_target->{SUBVOL_DELETED}}) { + push @subvol_out, "--- $_->{PRINT}"; + push @raw_data, { type => "target_delete", + status => $dryrun ? "DRYRUN" : "success", + target_url => $_->{URL}, + SORT => 40, + }; + } } if($config_target->{ABORTED} && ($config_target->{ABORTED} ne "USER_SKIP")) { push @subvol_out, "!!! Target \"$droot->{PRINT}\" aborted: $config_target->{ABORTED}"; - push @raw_err_out, "aborted target $droot->{URL} -- $config_target->{ABORTED}"; + push @raw_data, { type => "btrbk_target", + status => "ABORT", + target_url => $droot->{URL}, + error_message => $config_target->{ABORTED}, + SORT => 3, + }; $err_count++; } @@ -2940,7 +2983,12 @@ MAIN: } if($config_subvol->{ABORTED} && ($config_subvol->{ABORTED} ne "USER_SKIP")) { push @subvol_out, "!!! Aborted: $config_subvol->{ABORTED}"; - push @raw_err_out, "aborted subvolume $svol->{URL} -- $config_subvol->{ABORTED}"; + push @raw_data, { type => "btrbk_subvolume", + status => "ABORT", + target_url => $svol->{URL}, + error_message => $config_subvol->{ABORTED}, + SORT => 2, + }; $err_count++; } @@ -2955,20 +3003,19 @@ MAIN: } } if($config_vol->{ABORTED} && ($config_vol->{ABORTED} ne "USER_SKIP")) { - push @raw_err_out, "aborted volume $sroot->{URL} -- $config_vol->{ABORTED}"; + push @raw_data, { type => "btrbk_volume", + status => "ABORT", + target_url => $sroot->{URL}, + error_message => $config_vol->{ABORTED}, + SORT => 1, + }; $err_count++; } } - if($raw_output) + $output_format //= "custom"; + if($output_format eq "custom") { - if($dryrun) { - $_ = "DRYRUN $_" foreach(@raw_snapshot_out, @raw_receive_out, @raw_delete_out); - } - print join("\n", @raw_snapshot_out, @raw_receive_out, @raw_delete_out, @raw_err_out); - print "\n"; - } - else { print_header(title => "Backup Summary", config => $config, time => $start_time, @@ -2999,6 +3046,19 @@ MAIN: print "\nNOTE: Dryrun was active, none of the operations above were actually executed!\n"; } } + else + { + print_formatted( + "default", + $output_format, + "table", + { default => { data => [ sort { $a->{SORT} <=> $b->{SORT} } @raw_data ], + formats => { raw => [ qw( type status target_url source_url parent_url ) ], + table => [ qw( type status target_url source_url parent_url ) ], + }, + }, + }); + } } foreach my $config_vol (@{$config->{VOLUME}}) { From b65602f848a97fc158d26a4827c77c893c1cd5e8 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Sun, 11 Oct 2015 19:01:59 +0200 Subject: [PATCH 07/19] btrbk: bugfix/cleanup on formatting --- btrbk | 162 ++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 94 insertions(+), 68 deletions(-) diff --git a/btrbk b/btrbk index 78d1b7d..29b7769 100755 --- a/btrbk +++ b/btrbk @@ -1624,6 +1624,28 @@ sub schedule(@) } +sub format_preserve_matrix($$;$) +{ + my $config = shift || die; + my $prefix = shift || die; + my $format = shift || "long"; + my @out = ""; + my $dow = config_key($config, "preserve_day_of_week"); + my $d = config_key($config, "${prefix}_preserve_daily"); + my $w = config_key($config, "${prefix}_preserve_weekly"); + my $m = config_key($config, "${prefix}_preserve_monthly"); + $d =~ s/^all$/-1/; + $w =~ s/^all$/-1/; + $m =~ s/^all$/-1/; + if($format eq "short") { + # short format + return sprintf("%2sd %2sw %2sm", $d, $w, $m); + } + # long format + return sprintf("%2sd %2sw %2sm ($dow)", $d, $w, $m); +} + + sub print_header(@) { my %args = @_; @@ -1663,22 +1685,28 @@ sub print_header(@) } -sub print_formatted($$$$) +sub print_formatted(@) { - my $topic = shift || die; - my $format = shift || die; - my $default = shift || die; - my $spec = shift; - my $data = $spec->{$topic}->{data} || die; - my $keys = $spec->{$topic}->{formats}->{$format}; + my %args = @_; + my $format = $args{output_format} || die; + my $default = $args{default_format} || die; + my $data = $args{data} || die; + my $keys = $args{formats}->{$format}; unless($keys) { WARN "Unsupported output format \"$format\", defaulting to \"$default\" format."; - $keys = $spec->{$topic}->{formats}->{$default} || die; + $keys = $args{formats}->{$default} || die; $format = $default; } - if($format eq "table") + if($format eq "raw") + { + # output: key0="value0" key1="value1" ... + foreach my $row (@$data) { + print join(' ', map { "$_=\"" . ($row->{$_} // "") . "\""; } @$keys) . "\n"; + } + } + else { # sanitize and calculate maxlen for each column # NOTE: this is destructive on data! @@ -1713,13 +1741,6 @@ sub print_formatted($$$$) print "\n"; } } - else - { - # output: key0="value0" key1="value1" ... - foreach my $row (@$data) { - print join(' ', map { "$_=\"" . ($row->{$_} // "") . "\""; } @$keys) . "\n"; - } - } } @@ -2155,9 +2176,10 @@ MAIN: if($action_list) { - my @vol; - my @subvol; - my @target; + my @vol_data; + my @subvol_data; + my @target_data; + my @mixed_data; my %target_uniq; # @@ -2171,7 +2193,7 @@ MAIN: volume_host => $sroot->{HOST}, volume_rsh => ($sroot->{RSH} ? join(" ", @{$sroot->{RSH}}) : undef), }; - push @vol, $volh; + push @vol_data, $volh; foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { next if($config_subvol->{ABORTED}); @@ -2179,13 +2201,15 @@ MAIN: my $subvolh = { %$volh, source_url => $svol->{URL}, source_path => $svol->{PATH}, - snapshot_path => $sroot->{PATH} . (config_key($config_subvol, "snapshot_dir", prefix => '/') // ""), - snapshot_basename => config_key($config_subvol, "snapshot_name"), source_host => $svol->{HOST}, source_rsh => ($svol->{RSH} ? join(" ", @{$svol->{RSH}}) : undef), + snapshot_path => $sroot->{PATH} . (config_key($config_subvol, "snapshot_dir", prefix => '/') // ""), + snapshot_basename => config_key($config_subvol, "snapshot_name"), + snapshot_preserve => format_preserve_matrix($config_subvol, "snapshot"), }; - push @subvol, $subvolh; + push @subvol_data, $subvolh; + my $found = 0; foreach my $config_target (@{$config_subvol->{TARGET}}) { next if($config_target->{ABORTED}); @@ -2195,41 +2219,47 @@ MAIN: target_path => $droot->{PATH}, target_host => $droot->{HOST}, target_rsh => ($droot->{RSH} ? join(" ", @{$droot->{RSH}}) : undef), + target_preserve => format_preserve_matrix($config_target, "target"), }; if($action_list eq "target_uniq") { next if($target_uniq{$droot->{URL}}); $target_uniq{$droot->{URL}} = 1; } - push @target, $targeth; + push @target_data, $targeth; + push @mixed_data, $targeth; + $found = 1; } + # make sure the subvol is always printed (even if no targets around) + push @mixed_data, $subvolh unless($found); } } - my @all_vol_keys = qw( volume_url volume_path volume_host volume_rsh ); - my @all_subvol_keys = qw( source_url source_path snapshot_path snapshot_basename source_host source_rsh ); - my @all_target_keys = qw( target_url target_path target_host target_rsh ); + my @raw_vol_keys = qw( volume_url volume_path volume_host volume_rsh ); + my @raw_subvol_keys = qw( source_url source_host source_path source_rsh snapshot_path snapshot_basename ); + my @raw_target_keys = qw( target_url target_host target_path target_rsh ); + $output_format ||= "table"; + # TODO: honor $action_list and/or $action_config_dump to add filters print_formatted( - $action_list, # topic - $output_format, # output format - "table", # default output format - { #volume => { data => \@vol, - # formats => { raw => \@all_vol_keys, - # table => [ qw( volume_host volume_path ) ] }, - # }, - #source => { data => \@subvol, - # formats => { raw => \@all_subvol_keys, - # table => [ qw( source_host source_path snapshot_path snapshot_basename ) ] }, - # }, - target => { data => \@target, - formats => { raw => [ @all_subvol_keys, @all_target_keys ], - table => [ qw( source_host source_path snapshot_path snapshot_basename target_host target_path ) ], - }, - }, - #target_uniq => { data => [ @target ], - # formats => { raw => \@all_target_keys, - # table => [ qw( target_host target_path ) ] }, - # }, - }); + output_format => $output_format, + default_format => "table", + data => \@mixed_data, + formats => { raw => [ @raw_subvol_keys, @raw_target_keys ], + table => [ qw( source_host source_path snapshot_path snapshot_basename target_host target_path ) ], + long => [ qw( source_host source_path snapshot_path snapshot_basename snapshot_preserve target_host target_path target_preserve ) ], + }, + # data => \@vol_data, # volume only + # formats => { raw => \@raw_vol_keys, + # table => [ qw( volume_host volume_path ) ], + # }, + # data => \@subvol_data, # source only + # formats => { raw => \@raw_subvol_keys, + # table => [ qw( source_host source_path snapshot_path snapshot_basename ) ], + # }, + # data => \@target_data, + # formats => { raw => \@raw_target_keys, + # table => [ qw( target_host target_path ) ], + # }, + ); exit 0; } @@ -2507,7 +2537,7 @@ MAIN: push @tree_out, ""; } - $output_format //= "tree"; + $output_format ||= "tree"; if($output_format eq "tree") { print_header(title => "Backup Tree", config => $config, @@ -2522,15 +2552,13 @@ MAIN: } else { print_formatted( - "default", - $output_format, - "table", - { default => { data => \@raw_out, - formats => { raw => [ qw( source_host source_path snapshot_path snapshot_basename btrbk_flags received_host received_path ) ], - table => [ qw( source_url snapshot_url btrbk_flags received_url ) ], - }, - }, - }); + output_format => $output_format, + default_format => "table", + data => \@raw_out, + formats => { raw => [ qw( source_host source_path snapshot_path snapshot_basename btrbk_flags received_host received_path ) ], + table => [ qw( source_url snapshot_url btrbk_flags received_url ) ], + }, + ); } exit 0; } @@ -3013,7 +3041,7 @@ MAIN: } } - $output_format //= "custom"; + $output_format ||= "custom"; if($output_format eq "custom") { print_header(title => "Backup Summary", @@ -3049,15 +3077,13 @@ MAIN: else { print_formatted( - "default", - $output_format, - "table", - { default => { data => [ sort { $a->{SORT} <=> $b->{SORT} } @raw_data ], - formats => { raw => [ qw( type status target_url source_url parent_url ) ], - table => [ qw( type status target_url source_url parent_url ) ], - }, - }, - }); + output_format => $output_format, + default_format => "table", + data => [ sort { $a->{SORT} <=> $b->{SORT} } @raw_data ], + formats => { raw => [ qw( type status target_url source_url parent_url ) ], + table => [ qw( type status target_url source_url parent_url ) ], + }, + ); } } From 956b0101435309691928e4e98a4ebac767ba8dfc Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Sun, 11 Oct 2015 20:14:32 +0200 Subject: [PATCH 08/19] documentation: add "--format=table|long|raw" command line option --- ChangeLog | 6 +++--- btrbk | 3 ++- doc/btrbk.1 | 48 ++++++++++++++++++------------------------------ 3 files changed, 23 insertions(+), 34 deletions(-) diff --git a/ChangeLog b/ChangeLog index 296080e..7885b10 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,9 +4,9 @@ btrbk-current * Allow filtering subcommands by group as well as targets. * Added "config print" command. * Added "list" command (experimental). - * Added "--format=table|raw" command line option, producing tabular - and raw (machine-readable) output for "(dry)run", "tree" and - "list" commands. + * Added "--format=table|long|raw" command line option, producing + tabular and raw (machine-readable) output for "(dry)run", "tree" + and "list" commands. * Added configuration option "ssh_cipher_spec" (close: #47). * Added "target raw", with GnuPG and compression support (experimental). diff --git a/btrbk b/btrbk index 29b7769..73102f0 100755 --- a/btrbk +++ b/btrbk @@ -155,13 +155,14 @@ sub HELP_MESSAGE print STDERR " -v, --verbose be verbose (set loglevel=info)\n"; print STDERR " -q, --quiet be quiet (do not print summary for the \"run\" command)\n"; print STDERR " -l, --loglevel=LEVEL set logging level (warn, info, debug, trace)\n"; - print STDERR " --raw-output print raw (machine-readable) output\n"; + print STDERR " --format=FORMAT change output format, FORMAT=table|long|raw\n"; print STDERR " --progress show progress bar on send-receive operation\n"; print STDERR "\n"; print STDERR "commands:\n"; print STDERR " run [filter...] perform backup operations as defined in the config file\n"; print STDERR " dryrun [filter...] don't run btrfs commands; show what would be executed\n"; print STDERR " tree [filter...] shows backup tree\n"; + print STDERR " list [filter...] print source/snapshot/target relations\n"; print STDERR " info [filter...] print useful filesystem information\n"; print STDERR " origin print origin information for subvolume\n"; print STDERR " diff shows new files since subvolume for subvolume \n"; diff --git a/doc/btrbk.1 b/doc/btrbk.1 index 47cdb78..a1636f2 100644 --- a/doc/btrbk.1 +++ b/doc/btrbk.1 @@ -79,10 +79,12 @@ Set the level of verbosity. Accepted levels are warn, info, debug, and trace. .RE .PP -\-\-raw\-output +\-\-format table|long|raw .RS 4 -Print raw (machine-readable) output. Changes output format for -\fBrun\fR, \fBdryrun\fR and \fBtree\fR. Useful for further scripting. +Print output in specified format. If set to "raw", prints +space-separated key="value" pairs (machine-readable). Affects output +format for \fBrun\fR, \fBdryrun\fR, \fBinfo\fR and \fBtree\fR +commands. Useful for further exporting/scripting. .RE .PP \-\-progress @@ -121,25 +123,8 @@ configured retention policy will be deleted. Note that the latest snapshot as well as the latest backup is always preserved, regardless of the retention policy. .PP -If the \-\-raw\-output command line option is set, print the output in -the following format: -.PP -.RS 4 -snapshot -... -.RE -.RS 4 -receive [] -... -.RE -.RS 4 -delete -... -.RE -.RS 4 -aborted [volume|subvolume|target] -- abort_reason -... -.RE +Use the \fI\-\-format\fR command line option to switch between +different output formats. .RE .PP .B dryrun @@ -164,15 +149,18 @@ STATEMENTS\fR below). .RS 4 Print the snapshots and their corresponding backup subvolumes as a tree. Optionally filtered by [filter...] arguments (see \fIFILTER -STATEMENTS\fR below). -.PP -If the \-\-raw\-output command line option is set, print the output in -the following format: -.PP -.RS 4 - -... +STATEMENTS\fR below). Use the \fI\-\-format\fR command line option to +switch between different output formats. .RE +.PP +.B list +[filter...] +.RS 4 +Print the source/snapshot/target relations of the configured +subvolumes in a tabular form. Optionally filtered by [filter...] +arguments (see \fIFILTER STATEMENTS\fR below). Use the +\fI\-\-format\fR command line option to switch between different +output formats. .RE .PP .B origin From 1ac801c0a639167cca237b4c96c12174c50bd34c Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Mon, 12 Oct 2015 14:59:02 +0200 Subject: [PATCH 09/19] btrbk: add "list volume|source|target" actions (special output of configuration list) --- btrbk | 87 +++++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 26 deletions(-) diff --git a/btrbk b/btrbk index 73102f0..1f5dbfc 100755 --- a/btrbk +++ b/btrbk @@ -1835,7 +1835,19 @@ MAIN: @filter_args = @ARGV; } elsif($command eq "list") { - $action_list = "target"; + my $subcommand = shift @ARGV; + $action_list = "target-all"; + if(defined($subcommand)) { + if(($subcommand eq "volume") || + ($subcommand eq "source") || + ($subcommand eq "target")) + { + $action_list = $subcommand; + } + else { + unshift @ARGV, $subcommand; + } + } $args_expected_min = 0; $args_expected_max = 9999; $args_allow_group = 1; @@ -2205,7 +2217,7 @@ MAIN: source_host => $svol->{HOST}, source_rsh => ($svol->{RSH} ? join(" ", @{$svol->{RSH}}) : undef), snapshot_path => $sroot->{PATH} . (config_key($config_subvol, "snapshot_dir", prefix => '/') // ""), - snapshot_basename => config_key($config_subvol, "snapshot_name"), + snapshot_name => config_key($config_subvol, "snapshot_name"), snapshot_preserve => format_preserve_matrix($config_subvol, "snapshot"), }; push @subvol_data, $subvolh; @@ -2235,32 +2247,55 @@ MAIN: } } - my @raw_vol_keys = qw( volume_url volume_path volume_host volume_rsh ); - my @raw_subvol_keys = qw( source_url source_host source_path source_rsh snapshot_path snapshot_basename ); + my @raw_vol_keys = qw( volume_url volume_host volume_path volume_rsh ); + my @raw_subvol_keys = qw( source_url source_host source_path source_rsh snapshot_path snapshot_name ); my @raw_target_keys = qw( target_url target_host target_path target_rsh ); $output_format ||= "table"; - # TODO: honor $action_list and/or $action_config_dump to add filters - print_formatted( - output_format => $output_format, - default_format => "table", - data => \@mixed_data, - formats => { raw => [ @raw_subvol_keys, @raw_target_keys ], - table => [ qw( source_host source_path snapshot_path snapshot_basename target_host target_path ) ], - long => [ qw( source_host source_path snapshot_path snapshot_basename snapshot_preserve target_host target_path target_preserve ) ], - }, - # data => \@vol_data, # volume only - # formats => { raw => \@raw_vol_keys, - # table => [ qw( volume_host volume_path ) ], - # }, - # data => \@subvol_data, # source only - # formats => { raw => \@raw_subvol_keys, - # table => [ qw( source_host source_path snapshot_path snapshot_basename ) ], - # }, - # data => \@target_data, - # formats => { raw => \@raw_target_keys, - # table => [ qw( target_host target_path ) ], - # }, - ); + if($action_list eq "volume") { + print_formatted( + output_format => $output_format, + default_format => "table", + data => \@vol_data, + formats => { raw => \@raw_vol_keys, + table => [ qw( volume_host volume_path ) ], + long => \@raw_vol_keys, + }, + ); + } + elsif($action_list eq "source") { + print_formatted( + output_format => $output_format, + default_format => "table", + data => \@subvol_data, + formats => { raw => \@raw_subvol_keys, + table => [ qw( source_host source_path snapshot_path snapshot_name ) ], + long => \@raw_subvol_keys, + }, + ); + } + elsif($action_list eq "target") { + print_formatted( + output_format => $output_format, + default_format => "table", + data => \@target_data, + formats => { raw => \@raw_target_keys, + table => [ qw( target_host target_path ) ], + long => \@raw_target_keys, + }, + ); + } + else { + # default format + print_formatted( + output_format => $output_format, + default_format => "table", + data => \@mixed_data, + formats => { raw => [ @raw_subvol_keys, @raw_target_keys ], + table => [ qw( source_host source_path snapshot_path snapshot_name target_host target_path ) ], + long => [ qw( source_host source_path snapshot_path snapshot_name snapshot_preserve target_host target_path target_preserve ) ], + }, + ); + } exit 0; } From 07f7bfe3a6333c5870a4bef7365bdd8cec8cf4d9 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Mon, 12 Oct 2015 16:59:08 +0200 Subject: [PATCH 10/19] btrbk: fixed sort order in table format of run/dryrun actions --- btrbk | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/btrbk b/btrbk index 1f5dbfc..51556b3 100755 --- a/btrbk +++ b/btrbk @@ -2986,7 +2986,7 @@ MAIN: status => $dryrun ? "DRYRUN" : "success", target_url => $config_subvol->{SNAPSHOT}->{URL}, source_url => $svol->{URL}, - SORT => 10, + SORT => 10, # sort order: snapshot, send-receive, target_delete, snapshot_delete }; } if($config_subvol->{SUBVOL_DELETED}) { @@ -2995,7 +2995,7 @@ MAIN: push @raw_data, { type => "snapshot_delete", status => $dryrun ? "DRYRUN" : "success", target_url => $_->{URL}, - SORT => 30, + SORT => 40, # sort order: snapshot, send-receive, target_delete, snapshot_delete }; } } @@ -3013,7 +3013,7 @@ MAIN: target_url => $_->{received_subvolume}->{URL}, source_url => $_->{snapshot}->{URL}, parent_url => $_->{parent}->{URL}, - SORT => 20, + SORT => 20, # sort order: snapshot, send-receive, target_delete, snapshot_delete }; } @@ -3023,7 +3023,7 @@ MAIN: push @raw_data, { type => "target_delete", status => $dryrun ? "DRYRUN" : "success", target_url => $_->{URL}, - SORT => 40, + SORT => 30, # sort order: snapshot, send-receive, target_delete, snapshot_delete }; } } From 01b7ab0ebf3e22fc921f1d827a63f20067f3f404 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Mon, 12 Oct 2015 17:13:23 +0200 Subject: [PATCH 11/19] btrbk: add "-t, --table" command line option (shortcut for "--format=table") --- ChangeLog | 6 +++--- btrbk | 2 ++ doc/btrbk.1 | 5 +++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7885b10..d980a7f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,9 +4,9 @@ btrbk-current * Allow filtering subcommands by group as well as targets. * Added "config print" command. * Added "list" command (experimental). - * Added "--format=table|long|raw" command line option, producing - tabular and raw (machine-readable) output for "(dry)run", "tree" - and "list" commands. + * Added "--format=table|long|raw" and "-t, --table" command line + options, producing tabular and raw (machine-readable) output for + "(dry)run", "tree" and "list" commands. * Added configuration option "ssh_cipher_spec" (close: #47). * Added "target raw", with GnuPG and compression support (experimental). diff --git a/btrbk b/btrbk index 51556b3..e907d76 100755 --- a/btrbk +++ b/btrbk @@ -155,6 +155,7 @@ sub HELP_MESSAGE print STDERR " -v, --verbose be verbose (set loglevel=info)\n"; print STDERR " -q, --quiet be quiet (do not print summary for the \"run\" command)\n"; print STDERR " -l, --loglevel=LEVEL set logging level (warn, info, debug, trace)\n"; + print STDERR " -t, --table change output to table format\n"; print STDERR " --format=FORMAT change output format, FORMAT=table|long|raw\n"; print STDERR " --progress show progress bar on send-receive operation\n"; print STDERR "\n"; @@ -1770,6 +1771,7 @@ MAIN: 'verbose|v' => sub { $loglevel = 2; }, 'loglevel|l=s' => \$loglevel, 'progress' => \$show_progress, + 'table|t' => sub { $output_format = "table" }, 'format=s' => \$output_format, )) { diff --git a/doc/btrbk.1 b/doc/btrbk.1 index a1636f2..ac5e6e1 100644 --- a/doc/btrbk.1 +++ b/doc/btrbk.1 @@ -79,6 +79,11 @@ Set the level of verbosity. Accepted levels are warn, info, debug, and trace. .RE .PP +\-t, \-\-table +.RS 4 +Print output in table format (shortcut for "--format=table"). +.RE +.PP \-\-format table|long|raw .RS 4 Print output in specified format. If set to "raw", prints From 93249d1154659110ec0fbdc9c2288239fa4804d2 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Mon, 12 Oct 2015 20:46:05 +0200 Subject: [PATCH 12/19] btrbk: show scheduling summary if -v is set on run/dryrun --- ChangeLog | 3 +- btrbk | 114 ++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 88 insertions(+), 29 deletions(-) diff --git a/ChangeLog b/ChangeLog index d980a7f..072e28c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,9 +4,10 @@ btrbk-current * Allow filtering subcommands by group as well as targets. * Added "config print" command. * Added "list" command (experimental). - * Added "--format=table|long|raw" and "-t, --table" command line + * Added "--format=table|long|raw" and "-t,--table" command line options, producing tabular and raw (machine-readable) output for "(dry)run", "tree" and "list" commands. + * Print scheduler details if -v option is set on action run/dryrun. * Added configuration option "ssh_cipher_spec" (close: #47). * Added "target raw", with GnuPG and compression support (experimental). diff --git a/btrbk b/btrbk index e907d76..aa0872f 100755 --- a/btrbk +++ b/btrbk @@ -1542,13 +1542,12 @@ sub schedule(@) my $preserve_weekly = $args{preserve_weekly} // die; my $preserve_monthly = $args{preserve_monthly} // die; my $preserve_latest = $args{preserve_latest} || 0; - my $log_verbose = $args{log_verbose}; + my $results_list = $args{results}; + my $result_hints = $args{result_hints} // {}; - if($log_verbose) { - INFO "Filter scheme: preserving all within $preserve_daily days"; - INFO "Filter scheme: preserving first in week (starting on $preserve_day_of_week), for $preserve_weekly weeks"; - INFO "Filter scheme: preserving last weekly of month, for $preserve_monthly months"; - } + DEBUG "Filter scheme: preserving all within $preserve_daily days"; + DEBUG "Filter scheme: preserving first in week (starting on $preserve_day_of_week), for $preserve_weekly weeks"; + DEBUG "Filter scheme: preserving last weekly of month, for $preserve_monthly months"; # sort the schedule, ascending by date my @sorted_schedule = sort { ($a->{btrbk_date}->[0] <=> $b->{btrbk_date}->[0]) || @@ -1610,32 +1609,49 @@ sub schedule(@) # assemble results my @delete; my @preserve; + my %preserve_matrix = ( d => $preserve_daily, + w => $preserve_weekly, + m => $preserve_monthly, + dow => $preserve_day_of_week, + ); + my %result_base = ( %preserve_matrix, + scheme => format_preserve_matrix(%preserve_matrix, format => "short"), + %$result_hints, + ); foreach my $href (@sorted_schedule) { if($href->{preserve}) { - INFO "=== $href->{name}: $href->{preserve}" if($href->{name}); push(@preserve, $href->{value}); + DEBUG "=== $href->{name}: $href->{preserve}" if($href->{name}); + push @$results_list, { %result_base, + # action => "preserve", + reason => $href->{preserve}, + value => $href->{value}, + } if($results_list); + } else { - INFO "<<< $href->{name}" if($href->{name}); push(@delete, $href->{value}); + DEBUG "<<< $href->{name}" if($href->{name}); + push @$results_list, { %result_base, + action => "delete", + value => $href->{value}, + } if($results_list);; } } - DEBUG "Preserving " . @preserve . "/" . @$schedule . " items" unless($log_verbose); + DEBUG "Preserving " . @preserve . "/" . @$schedule . " items"; return (\@preserve, \@delete); } -sub format_preserve_matrix($$;$) +sub format_preserve_matrix(@) { - my $config = shift || die; - my $prefix = shift || die; - my $format = shift || "long"; - my @out = ""; - my $dow = config_key($config, "preserve_day_of_week"); - my $d = config_key($config, "${prefix}_preserve_daily"); - my $w = config_key($config, "${prefix}_preserve_weekly"); - my $m = config_key($config, "${prefix}_preserve_monthly"); + my %args = @_; + my $dow = $args{dow} // config_key($args{config}, "preserve_day_of_week"); + my $d = $args{d} // config_key($args{config}, "$args{prefix}_preserve_daily"); + my $w = $args{w} // config_key($args{config}, "$args{prefix}_preserve_weekly"); + my $m = $args{m} // config_key($args{config}, "$args{prefix}_preserve_monthly"); + my $format = $args{format} // "long"; $d =~ s/^all$/-1/; $w =~ s/^all$/-1/; $m =~ s/^all$/-1/; @@ -1690,7 +1706,8 @@ sub print_header(@) sub print_formatted(@) { my %args = @_; - my $format = $args{output_format} || die; + my $title = $args{title}; + my $format = $args{output_format} || "table"; my $default = $args{default_format} || die; my $data = $args{data} || die; my $keys = $args{formats}->{$format}; @@ -1701,6 +1718,7 @@ sub print_formatted(@) $format = $default; } + print "$title\n" if($title); if($format eq "raw") { # output: key0="value0" key1="value1" ... @@ -2220,7 +2238,7 @@ MAIN: source_rsh => ($svol->{RSH} ? join(" ", @{$svol->{RSH}}) : undef), snapshot_path => $sroot->{PATH} . (config_key($config_subvol, "snapshot_dir", prefix => '/') // ""), snapshot_name => config_key($config_subvol, "snapshot_name"), - snapshot_preserve => format_preserve_matrix($config_subvol, "snapshot"), + snapshot_preserve => format_preserve_matrix(config => $config_subvol, prefix => "snapshot"), }; push @subvol_data, $subvolh; @@ -2234,7 +2252,7 @@ MAIN: target_path => $droot->{PATH}, target_host => $droot->{HOST}, target_rsh => ($droot->{RSH} ? join(" ", @{$droot->{RSH}}) : undef), - target_preserve => format_preserve_matrix($config_target, "target"), + target_preserve => format_preserve_matrix(config => $config_target, prefix => "target"), }; if($action_list eq "target_uniq") { next if($target_uniq{$droot->{URL}}); @@ -2832,6 +2850,7 @@ MAIN: # # remove backups following a preserve daily/weekly/monthly scheme # + my $schedule_results = []; if($preserve_backups || $resume_only) { INFO "Preserving all backups (option \"-p\" or \"-r\" present)"; } @@ -2890,7 +2909,7 @@ MAIN: # next; # } push(@schedule, { value => $vol, - name => $vol->{PRINT}, + name => $vol->{PRINT}, # only for logging btrbk_date => $filename_info->{btrbk_date}, preserve => $vol->{FORCE_PRESERVE} }); @@ -2903,7 +2922,8 @@ MAIN: preserve_weekly => config_key($config_target, "target_preserve_weekly"), preserve_monthly => config_key($config_target, "target_preserve_monthly"), preserve_latest => $preserve_latest_backup, - log_verbose => 1, + results => $schedule_results, + result_hints => { topic => "backup", root_path => $droot->{PATH} }, ); my $ret = btrfs_subvolume_delete($delete, commit => config_key($config_target, "btrfs_commit_delete")); if(defined($ret)) { @@ -2933,7 +2953,7 @@ MAIN: my $filename_info = parse_filename($vol->{SUBVOL_PATH}, $snapdir . $snapshot_basename); next unless($filename_info); # ignore non-btrbk files push(@schedule, { value => $vol, - name => $vol->{PRINT}, + name => $vol->{PRINT}, # only for logging btrbk_date => $filename_info->{btrbk_date} }); } @@ -2945,7 +2965,8 @@ MAIN: preserve_weekly => config_key($config_subvol, "snapshot_preserve_weekly"), preserve_monthly => config_key($config_subvol, "snapshot_preserve_monthly"), preserve_latest => $preserve_latest_snapshot, - log_verbose => 1, + results => $schedule_results, + result_hints => { topic => "snapshot", root_path => $sroot->{PATH} }, ); my $ret = btrfs_subvolume_delete($delete, commit => config_key($config_subvol, "btrfs_commit_delete")); if(defined($ret)) { @@ -2962,11 +2983,47 @@ MAIN: my $time_elapsed = time - $start_time; INFO "Completed within: ${time_elapsed}s (" . localtime(time) . ")"; - # - # print summary - # + unless($quiet) { + # + # print scheduling results + # + if($loglevel >= 2) { + my @data = map { $_->{url} = $_->{value}->{URL}; + $_->{host} = $_->{value}->{HOST}; + $_->{path} = $_->{value}->{PATH}; + $_->{name} = $_->{value}->{SUBVOL_PATH}; + $_->{target} = $_->{value}->{PRINT}; + $_; + } @$schedule_results; + my @data_backup = map { $_->{topic} eq "backup" ? $_ : () } @data; + my @data_snapshot = map { $_->{topic} eq "snapshot" ? $_ : () } @data; + + my %format_args = ( + output_format => $output_format, + default_format => "table", + formats => { raw => [ qw( topic action url host path dow d m w) ], + table => [ qw( action target scheme reason ) ], + long => [ qw( action host root_path name scheme reason ) ], + }, + ); + print_formatted( title => "SNAPSHOT SCHEDULE", + data => \@data_snapshot, + %format_args, + ); + print "\n"; + print_formatted( title => "BACKUP SCHEDULE", + data => \@data_backup, + %format_args, + ); + print "\n"; + } + + + # + # print summary + # my @unrecoverable; my @out; my @raw_data; @@ -3115,6 +3172,7 @@ MAIN: else { print_formatted( + title => "SUMMARY", output_format => $output_format, default_format => "table", data => [ sort { $a->{SORT} <=> $b->{SORT} } @raw_data ], From 5356f83dfc692d07cc11180a116660173a383f3d Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Mon, 12 Oct 2015 22:26:36 +0200 Subject: [PATCH 13/19] btrbk: add action_log, a nice way to keep track and list the actions (snapshot/delete/send-receive) --- btrbk | 251 ++++++++++++++++++++++++++++++---------------------------- 1 file changed, 129 insertions(+), 122 deletions(-) diff --git a/btrbk b/btrbk index aa0872f..0020800 100755 --- a/btrbk +++ b/btrbk @@ -122,6 +122,8 @@ my %vinfo_cache; # map URL to vinfo my %uuid_info; # map UUID to btr_tree node my %uuid_fs_map; # map UUID to URL +my @action_log; + my $dryrun; my $loglevel = 1; my $show_progress = 0; @@ -177,6 +179,26 @@ sub INFO { my $t = shift; print STDERR "$t\n" if($loglevel >= 2); } sub WARN { my $t = shift; print STDERR "WARNING: $t\n" if($loglevel >= 1); } sub ERROR { my $t = shift; print STDERR "ERROR: $t\n"; } +sub action($@) +{ + my $type = shift // die; + my %h = @_; + $h{type} = $type; + $h{time} = time; + push @action_log, \%h; +} + +sub ABORTED($$) +{ + my $config = shift; + my $t = shift; + $config->{ABORTED} = $t; + action("abort_" . ($config->{CONTEXT} || "undef"), + status => "ABORT", + target_url => $config->{url}, + error_message => $t, + ); +} sub run_cmd(@) { @@ -932,16 +954,22 @@ sub btrfs_subvolume_find_new($$;$) sub btrfs_subvolume_snapshot($$) { my $svol = shift || die; - my $target_path = shift // die; + my $target_vol = shift // die; + my $target_path = $target_vol->{PATH} // die; my $src_path = $svol->{PATH} // die; DEBUG "[btrfs] snapshot (ro):"; DEBUG "[btrfs] host : $svol->{HOST}" if($svol->{HOST}); DEBUG "[btrfs] source: $src_path"; DEBUG "[btrfs] target: $target_path"; - INFO ">>> " . ($svol->{HOST} ? "{$svol->{HOST}}" : "") . $target_path; + INFO ">>> $target_vol->{PRINT}"; my $ret = run_cmd(cmd => [ qw(btrfs subvolume snapshot), '-r', $src_path, $target_path ], rsh => $svol->{RSH}, ); + action("snapshot", + status => ($dryrun ? "DRYRUN" : (defined($ret) ? "success" : "ERROR")), + target_url => $target_vol->{URL}, + source_url => $svol->{URL}, + ); ERROR "Failed to create btrfs subvolume snapshot: $svol->{PRINT} -> $target_path" unless(defined($ret)); return defined($ret) ? $target_path : undef; } @@ -970,6 +998,11 @@ sub btrfs_subvolume_delete($@) my $ret = run_cmd(cmd => [ qw(btrfs subvolume delete), @options, @target_paths ], rsh => $rsh, ); + action($opts{type} // "delete", + status => ($dryrun ? "DRYRUN" : (defined($ret) ? "success" : "ERROR")), + target_url => $_->{URL}, + # target_vinfo => $_, # TODO !!! resolve this + ) foreach (@$targets); ERROR "Failed to delete btrfs subvolumes: " . join(' ', map( { $_->{PRINT} } @$targets)) unless(defined($ret)); return defined($ret) ? scalar(@$targets) : undef; } @@ -1024,7 +1057,7 @@ sub btrfs_send_receive($$$$) # we need to do this by hand. # TODO: remove this as soon as btrfs-progs handle receive errors correctly. DEBUG "send/received failed, deleting (possibly present and garbled) received subvolume: $vol_received->{PRINT}"; - my $ret = btrfs_subvolume_delete($vol_received, commit => "after"); + my $ret = btrfs_subvolume_delete($vol_received, commit => "after", type => "delete_garbled"); if(defined($ret)) { WARN "Deleted partially received (garbled) subvolume: $vol_received->{PRINT}"; } @@ -1354,6 +1387,12 @@ sub macro_send_receive($@) $config_target->{SUBVOL_RECEIVED} //= []; push(@{$config_target->{SUBVOL_RECEIVED}}, \%info); + action("send-receive", + status => ($dryrun ? "DRYRUN" : ($ret ? "success" : "ERROR")), + target_url => $vol_received->{URL}, + source_url => $snapshot->{URL}, + parent_url => $parent->{URL}, + ); unless($ret) { $info{ERROR} = 1; return undef; @@ -2708,8 +2747,9 @@ MAIN: # finally create the snapshot INFO "Creating subvolume snapshot for: $svol->{PRINT}"; - if(btrfs_subvolume_snapshot($svol, "$sroot->{PATH}/$snapdir$snapshot_name")) { - $config_subvol->{SNAPSHOT} = vinfo_child($sroot, "$snapdir$snapshot_name"); + my $snapshot = vinfo_child($sroot, "$snapdir$snapshot_name"); + if(btrfs_subvolume_snapshot($svol, $snapshot)) { + $config_subvol->{SNAPSHOT} = $snapshot; } else { $config_subvol->{ABORTED} = "Failed to create snapshot: $svol->{PRINT} -> $sroot->{PRINT}/$snapdir$snapshot_name"; @@ -2925,7 +2965,7 @@ MAIN: results => $schedule_results, result_hints => { topic => "backup", root_path => $droot->{PATH} }, ); - my $ret = btrfs_subvolume_delete($delete, commit => config_key($config_target, "btrfs_commit_delete")); + my $ret = btrfs_subvolume_delete($delete, commit => config_key($config_target, "btrfs_commit_delete"), type => "delete_target"); if(defined($ret)) { INFO "Deleted $ret subvolumes in: $droot->{PRINT}/$snapshot_basename.*"; $config_target->{SUBVOL_DELETED} = $delete; @@ -2968,7 +3008,7 @@ MAIN: results => $schedule_results, result_hints => { topic => "snapshot", root_path => $sroot->{PATH} }, ); - my $ret = btrfs_subvolume_delete($delete, commit => config_key($config_subvol, "btrfs_commit_delete")); + my $ret = btrfs_subvolume_delete($delete, commit => config_key($config_subvol, "btrfs_commit_delete"), type => "delete_snapshot"); if(defined($ret)) { INFO "Deleted $ret subvolumes in: $sroot->{PRINT}/$snapdir$snapshot_basename.*"; $config_subvol->{SUBVOL_DELETED} = $delete; @@ -3024,121 +3064,86 @@ MAIN: # # print summary # - my @unrecoverable; - my @out; - my @raw_data; - my $err_count = 0; - foreach my $config_vol (@{$config->{VOLUME}}) - { - my $sroot = $config_vol->{sroot} || vinfo($config_vol->{url}, $config_vol); - foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) - { - my @subvol_out; - my $svol = $config_subvol->{svol} || vinfo_child($sroot, $config_subvol->{rel_path}); - - if($config_subvol->{SNAPSHOT_UP_TO_DATE}) { - push @subvol_out, "=== $config_subvol->{SNAPSHOT_UP_TO_DATE}->{PRINT}"; - } - if($config_subvol->{SNAPSHOT}) { - push @subvol_out, "+++ $config_subvol->{SNAPSHOT}->{PRINT}"; - push @raw_data, { type => "snapshot", - status => $dryrun ? "DRYRUN" : "success", - target_url => $config_subvol->{SNAPSHOT}->{URL}, - source_url => $svol->{URL}, - SORT => 10, # sort order: snapshot, send-receive, target_delete, snapshot_delete - }; - } - if($config_subvol->{SUBVOL_DELETED}) { - foreach(sort { $a->{PATH} cmp $b->{PATH} } @{$config_subvol->{SUBVOL_DELETED}}) { - push @subvol_out, "--- $_->{PRINT}"; - push @raw_data, { type => "snapshot_delete", - status => $dryrun ? "DRYRUN" : "success", - target_url => $_->{URL}, - SORT => 40, # sort order: snapshot, send-receive, target_delete, snapshot_delete - }; - } - } - foreach my $config_target (@{$config_subvol->{TARGET}}) - { - my $droot = $config_target->{droot} || vinfo($config_target->{url}, $config_target); - foreach(@{$config_target->{SUBVOL_RECEIVED} // []}) { - my $create_mode = "***"; - $create_mode = ">>>" if($_->{parent}); - # substr($create_mode, 0, 1, '%') if($_->{resume}); - $create_mode = "!!!" if($_->{ERROR}); - push @subvol_out, "$create_mode $_->{received_subvolume}->{PRINT}"; - push @raw_data, { type => "send-receive", - status => $_->{ERROR} ? "ERROR" : ($dryrun ? "DRYRUN" : "success"), - target_url => $_->{received_subvolume}->{URL}, - source_url => $_->{snapshot}->{URL}, - parent_url => $_->{parent}->{URL}, - SORT => 20, # sort order: snapshot, send-receive, target_delete, snapshot_delete - }; - } - - if($config_target->{SUBVOL_DELETED}) { - foreach(sort { $a->{PATH} cmp $b->{PATH} } @{$config_target->{SUBVOL_DELETED}}) { - push @subvol_out, "--- $_->{PRINT}"; - push @raw_data, { type => "target_delete", - status => $dryrun ? "DRYRUN" : "success", - target_url => $_->{URL}, - SORT => 30, # sort order: snapshot, send-receive, target_delete, snapshot_delete - }; - } - } - - if($config_target->{ABORTED} && ($config_target->{ABORTED} ne "USER_SKIP")) { - push @subvol_out, "!!! Target \"$droot->{PRINT}\" aborted: $config_target->{ABORTED}"; - push @raw_data, { type => "btrbk_target", - status => "ABORT", - target_url => $droot->{URL}, - error_message => $config_target->{ABORTED}, - SORT => 3, - }; - $err_count++; - } - - push(@unrecoverable, $config_target->{UNRECOVERABLE}) if($config_target->{UNRECOVERABLE}); - } - if($config_vol->{ABORTED} && ($config_vol->{ABORTED} ne "USER_SKIP")) { - # repeat volume errors in subvolume context ($err_count is increased in volume context below) - push @subvol_out, "!!! Volume \"$sroot->{PRINT}\" aborted: $config_vol->{ABORTED}"; - } - if($config_subvol->{ABORTED} && ($config_subvol->{ABORTED} ne "USER_SKIP")) { - push @subvol_out, "!!! Aborted: $config_subvol->{ABORTED}"; - push @raw_data, { type => "btrbk_subvolume", - status => "ABORT", - target_url => $svol->{URL}, - error_message => $config_subvol->{ABORTED}, - SORT => 2, - }; - $err_count++; - } - - if(@subvol_out) { - push @out, "$svol->{PRINT}", @subvol_out, ""; - } - elsif($config_subvol->{ABORTED} && ($config_subvol->{ABORTED} eq "USER_SKIP")) { - # don't print "" on USER_SKIP - } - else { - push @out, "$svol->{PRINT}", "", ""; - } - } - if($config_vol->{ABORTED} && ($config_vol->{ABORTED} ne "USER_SKIP")) { - push @raw_data, { type => "btrbk_volume", - status => "ABORT", - target_url => $sroot->{URL}, - error_message => $config_vol->{ABORTED}, - SORT => 1, - }; - $err_count++; - } - } - $output_format ||= "custom"; if($output_format eq "custom") { + my @unrecoverable; + my @out; + my @raw_data; + my $err_count = 0; + foreach my $config_vol (@{$config->{VOLUME}}) + { + my $sroot = $config_vol->{sroot} || vinfo($config_vol->{url}, $config_vol); + foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) + { + my @subvol_out; + my $svol = $config_subvol->{svol} || vinfo_child($sroot, $config_subvol->{rel_path}); + + if($config_subvol->{SNAPSHOT_UP_TO_DATE}) { + push @subvol_out, "=== $config_subvol->{SNAPSHOT_UP_TO_DATE}->{PRINT}"; + } + if($config_subvol->{SNAPSHOT}) { + push @subvol_out, "+++ $config_subvol->{SNAPSHOT}->{PRINT}"; + } + if($config_subvol->{SUBVOL_DELETED}) { + foreach(sort { $a->{PATH} cmp $b->{PATH} } @{$config_subvol->{SUBVOL_DELETED}}) { + push @subvol_out, "--- $_->{PRINT}"; + } + } + foreach my $config_target (@{$config_subvol->{TARGET}}) + { + my $droot = $config_target->{droot} || vinfo($config_target->{url}, $config_target); + foreach(@{$config_target->{SUBVOL_RECEIVED} // []}) { + my $create_mode = "***"; + $create_mode = ">>>" if($_->{parent}); + # substr($create_mode, 0, 1, '%') if($_->{resume}); + $create_mode = "!!!" if($_->{ERROR}); + push @subvol_out, "$create_mode $_->{received_subvolume}->{PRINT}"; + } + + if($config_target->{SUBVOL_DELETED}) { + foreach(sort { $a->{PATH} cmp $b->{PATH} } @{$config_target->{SUBVOL_DELETED}}) { + push @subvol_out, "--- $_->{PRINT}"; + } + } + + if($config_target->{ABORTED} && ($config_target->{ABORTED} ne "USER_SKIP")) { + push @subvol_out, "!!! Target \"$droot->{PRINT}\" aborted: $config_target->{ABORTED}"; + $err_count++; + } + + push(@unrecoverable, $config_target->{UNRECOVERABLE}) if($config_target->{UNRECOVERABLE}); + } + if($config_vol->{ABORTED} && ($config_vol->{ABORTED} ne "USER_SKIP")) { + # repeat volume errors in subvolume context ($err_count is increased in volume context below) + push @subvol_out, "!!! Volume \"$sroot->{PRINT}\" aborted: $config_vol->{ABORTED}"; + } + if($config_subvol->{ABORTED} && ($config_subvol->{ABORTED} ne "USER_SKIP")) { + push @subvol_out, "!!! Aborted: $config_subvol->{ABORTED}"; + $err_count++; + } + + if(@subvol_out) { + push @out, "$svol->{PRINT}", @subvol_out, ""; + } + elsif($config_subvol->{ABORTED} && ($config_subvol->{ABORTED} eq "USER_SKIP")) { + # don't print "" on USER_SKIP + } + else { + push @out, "$svol->{PRINT}", "", ""; + } + } + if($config_vol->{ABORTED} && ($config_vol->{ABORTED} ne "USER_SKIP")) { + push @raw_data, { type => "btrbk_volume", + status => "ABORT", + target_url => $sroot->{URL}, + error_message => $config_vol->{ABORTED}, + SORT => 1, + }; + $err_count++; + } + } + print_header(title => "Backup Summary", config => $config, time => $start_time, @@ -3171,12 +3176,14 @@ MAIN: } else { + # print action log + @action_log = map { $_->{timestamp} = localtime($_->{time}); $_; } @action_log; print_formatted( - title => "SUMMARY", + title => "BACKUP SUMMARY", output_format => $output_format, default_format => "table", - data => [ sort { $a->{SORT} <=> $b->{SORT} } @raw_data ], - formats => { raw => [ qw( type status target_url source_url parent_url ) ], + data => \@action_log, + formats => { raw => [ qw( time type status target_url source_url parent_url ) ], table => [ qw( type status target_url source_url parent_url ) ], }, ); From 0a6599aa782fc9da6bb8275dd9e255d15e9ca622 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Mon, 12 Oct 2015 22:56:52 +0200 Subject: [PATCH 14/19] btrbk: globally use new ABORTED macro, which also adds an entry to the action_log --- btrbk | 46 ++++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/btrbk b/btrbk index 0020800..0a8017f 100755 --- a/btrbk +++ b/btrbk @@ -193,11 +193,13 @@ sub ABORTED($$) my $config = shift; my $t = shift; $config->{ABORTED} = $t; - action("abort_" . ($config->{CONTEXT} || "undef"), - status => "ABORT", - target_url => $config->{url}, - error_message => $t, - ); + unless($t eq "USER_SKIP") { + action("abort_" . ($config->{CONTEXT} || "undef"), + status => "ABORT", + target_url => $config->{url}, + error_message => $t, + ); + } } sub run_cmd(@) @@ -1314,7 +1316,7 @@ sub macro_send_receive($@) # check for existing target subvolume if(my $err_vol = vinfo_subvol($target, $snapshot->{NAME})) { - $config_target->{ABORTED} = "Target subvolume \"$err_vol->{PRINT}\" already exists"; + ABORTED($config_target, "Target subvolume \"$err_vol->{PRINT}\" already exists"); $config_target->{UNRECOVERABLE} = "Please delete stray subvolume: $err_vol->{PRINT}"; ERROR $config_target->{ABORTED} . ", aborting send/receive of: $snapshot->{PRINT}"; ERROR $config_target->{UNRECOVERABLE}; @@ -1334,7 +1336,7 @@ sub macro_send_receive($@) else { WARN "Backup to $target->{PRINT} failed: no common parent subvolume found, and option \"incremental\" is set to \"strict\""; $info{ERROR} = 1; - $config_target->{ABORTED} = "No common parent subvolume found, and option \"incremental\" is set to \"strict\""; + ABORTED($config_target, "No common parent subvolume found, and option \"incremental\" is set to \"strict\""); return undef; } } @@ -1349,7 +1351,7 @@ sub macro_send_receive($@) if($target_type eq "send-receive") { $ret = btrfs_send_receive($snapshot, $target, $parent, \$vol_received); - $config_target->{ABORTED} = "Failed to send/receive subvolume" unless($ret); + ABORTED($config_target, "Failed to send/receive subvolume") unless($ret); } elsif($target_type eq "raw") { @@ -1374,7 +1376,7 @@ sub macro_send_receive($@) } } $ret = btrfs_send_to_file($snapshot, $target, $parent, \$vol_received, compress => $compress, encrypt => $encrypt); - $config_target->{ABORTED} = "Failed to send subvolume to raw file" unless($ret); + ABORTED($config_target, "Failed to send subvolume to raw file") unless($ret); } else { @@ -2128,17 +2130,17 @@ MAIN: } unless($found_target) { DEBUG "No match on filter command line argument, skipping target: $target_url"; - $config_target->{ABORTED} = "USER_SKIP"; + ABORTED($config_target, "USER_SKIP"); } } unless($found_subvol) { DEBUG "No match on filter command line argument, skipping subvolume: $subvol_url"; - $config_subvol->{ABORTED} = "USER_SKIP"; + ABORTED($config_subvol, "USER_SKIP"); } } unless($found_vol) { DEBUG "No match on filter command line argument, skipping volume: $vol_url"; - $config_vol->{ABORTED} = "USER_SKIP"; + ABORTED($config_vol, "USER_SKIP"); } } # make sure all args have a match @@ -2369,7 +2371,7 @@ MAIN: next if($config_vol->{ABORTED}); my $sroot = vinfo($config_vol->{url}, $config_vol); unless(vinfo_root($sroot)) { - $config_vol->{ABORTED} = "Failed to fetch subvolume detail" . ($err ? ": $err" : ""); + ABORTED($config_vol, "Failed to fetch subvolume detail" . ($err ? ": $err" : "")); WARN "Skipping volume \"$sroot->{PRINT}\": $config_vol->{ABORTED}"; next; } @@ -2387,19 +2389,19 @@ MAIN: $svol = vinfo_child($sroot, $config_subvol->{rel_path}); my $detail = btrfs_subvolume_detail($svol); unless($detail) { - $config_subvol->{ABORTED} = "Failed to fetch subvolume detail" . ($err ? ": $err" : ""); + ABORTED($config_subvol, "Failed to fetch subvolume detail" . ($err ? ": $err" : "")); WARN "Skipping subvolume \"$svol->{PRINT}\": $config_subvol->{ABORTED}"; next; } if($detail->{is_root}) { - $config_subvol->{ABORTED} = "Subvolume is btrfs root"; + ABORTED($config_subvol, "Subvolume is btrfs root"); WARN "Skipping subvolume \"$svol->{PRINT}\": $config_subvol->{ABORTED}"; next; } if(grep { $_->{uuid} eq $detail->{uuid} } values %{vinfo_subvol_list($sroot)}) { vinfo_set_detail($svol, $uuid_info{$detail->{uuid}}); } else { - $config_subvol->{ABORTED} = "Not a child subvolume of: $sroot->{PRINT}"; + ABORTED($config_subvol, "Not a child subvolume of: $sroot->{PRINT}"); WARN "Skipping subvolume \"$svol->{PRINT}\": $config_subvol->{ABORTED}"; next; } @@ -2426,7 +2428,7 @@ MAIN: if($target_type eq "send-receive") { unless(vinfo_root($droot)) { - $config_target->{ABORTED} = "Failed to fetch subvolume detail" . ($err ? ": $err" : ""); + ABORTED($config_target, "Failed to fetch subvolume detail" . ($err ? ": $err" : "")); WARN "Skipping target \"$droot->{PRINT}\": $config_target->{ABORTED}"; next; } @@ -2444,7 +2446,7 @@ MAIN: non_destructive => 1, ); unless(defined($ret)) { - $config_target->{ABORTED} = "Failed to list files from: $droot->{PATH}"; + ABORTED($config_target, "Failed to list files from: $droot->{PATH}"); WARN "Skipping target \"$droot->{PRINT}\": $config_target->{ABORTED}"; next; } @@ -2458,7 +2460,7 @@ MAIN: next; } unless($file =~ s/^\Q$droot->{PATH}\E\///) { - $config_target->{ABORTED} = "Unexpected result from 'find': file \"$file\" is not under \"$droot->{PATH}\""; + ABORTED($config_target, "Unexpected result from 'find': file \"$file\" is not under \"$droot->{PATH}\""); last; } my $filename_info = parse_filename($file, $snapshot_basename, 1); @@ -2752,7 +2754,7 @@ MAIN: $config_subvol->{SNAPSHOT} = $snapshot; } else { - $config_subvol->{ABORTED} = "Failed to create snapshot: $svol->{PRINT} -> $sroot->{PRINT}/$snapdir$snapshot_name"; + ABORTED($config_subvol, "Failed to create snapshot: $svol->{PRINT} -> $sroot->{PRINT}/$snapdir$snapshot_name"); WARN "Skipping subvolume section: $config_subvol->{ABORTED}"; } } @@ -2971,7 +2973,7 @@ MAIN: $config_target->{SUBVOL_DELETED} = $delete; } else { - $config_target->{ABORTED} = "Failed to delete subvolume"; + ABORTED($config_target, "Failed to delete subvolume"); $target_aborted = -1; } } @@ -3014,7 +3016,7 @@ MAIN: $config_subvol->{SUBVOL_DELETED} = $delete; } else { - $config_subvol->{ABORTED} = "Failed to delete subvolume"; + ABORTED($config_subvol, "Failed to delete subvolume"); } } } From 10a7f985793004f828f8925b3f6e35702ed4006a Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Mon, 12 Oct 2015 23:58:38 +0200 Subject: [PATCH 15/19] btrbk: add vinfo_prefixed_keys() function, which resolves a vinfo into hash of prefixed (url,path,host,rsh) for formatted lists --- btrbk | 67 +++++++++++++++++++++++++++++------------------------------ 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/btrbk b/btrbk index 0a8017f..717caf0 100755 --- a/btrbk +++ b/btrbk @@ -196,7 +196,7 @@ sub ABORTED($$) unless($t eq "USER_SKIP") { action("abort_" . ($config->{CONTEXT} || "undef"), status => "ABORT", - target_url => $config->{url}, + vinfo_prefixed_keys("target", vinfo($config->{url}, $config)), error_message => $t, ); } @@ -395,6 +395,21 @@ sub vinfo_set_detail($$) } +# returns hash: ( $prefix_{url,path,host,name,subvol_path,rsh} => value, ... ) +sub vinfo_prefixed_keys($$) +{ + my $prefix = shift || die; + my $vinfo = shift || die; + my %ret; + foreach qw( URL PATH HOST NAME SUBVOL_PATH ) { + $ret{$prefix . '_' . lc($_)} = $vinfo->{$_}; + } + $ret{$prefix} = $vinfo->{PRINT}; + $ret{$prefix . "_rsh"} = ($vinfo->{RSH} ? join(" ", @{$vinfo->{RSH}}) : undef), + return %ret; +} + + sub config_key($$;@) { my $node = shift || die; @@ -969,8 +984,8 @@ sub btrfs_subvolume_snapshot($$) ); action("snapshot", status => ($dryrun ? "DRYRUN" : (defined($ret) ? "success" : "ERROR")), - target_url => $target_vol->{URL}, - source_url => $svol->{URL}, + target_vinfo => $target_vol, + source_vinfo => $svol, ); ERROR "Failed to create btrfs subvolume snapshot: $svol->{PRINT} -> $target_path" unless(defined($ret)); return defined($ret) ? $target_path : undef; @@ -1002,8 +1017,7 @@ sub btrfs_subvolume_delete($@) ); action($opts{type} // "delete", status => ($dryrun ? "DRYRUN" : (defined($ret) ? "success" : "ERROR")), - target_url => $_->{URL}, - # target_vinfo => $_, # TODO !!! resolve this + target_vinfo => $_, ) foreach (@$targets); ERROR "Failed to delete btrfs subvolumes: " . join(' ', map( { $_->{PRINT} } @$targets)) unless(defined($ret)); return defined($ret) ? scalar(@$targets) : undef; @@ -1391,9 +1405,9 @@ sub macro_send_receive($@) action("send-receive", status => ($dryrun ? "DRYRUN" : ($ret ? "success" : "ERROR")), - target_url => $vol_received->{URL}, - source_url => $snapshot->{URL}, - parent_url => $parent->{URL}, + target_vinfo => $vol_received, + source_vinfo => $snapshot, + parent_vinfo => $parent, ); unless($ret) { $info{ERROR} = 1; @@ -2262,21 +2276,14 @@ MAIN: foreach my $config_vol (@{$config->{VOLUME}}) { next if($config_vol->{ABORTED}); my $sroot = vinfo($config_vol->{url}, $config_vol); - my $volh = { volume_url => $sroot->{URL}, - volume_path => $sroot->{PATH}, - volume_host => $sroot->{HOST}, - volume_rsh => ($sroot->{RSH} ? join(" ", @{$sroot->{RSH}}) : undef), - }; + my $volh = { vinfo_prefixed_keys("volume", $sroot) }; push @vol_data, $volh; foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { next if($config_subvol->{ABORTED}); my $svol = vinfo_child($sroot, $config_subvol->{rel_path}); my $subvolh = { %$volh, - source_url => $svol->{URL}, - source_path => $svol->{PATH}, - source_host => $svol->{HOST}, - source_rsh => ($svol->{RSH} ? join(" ", @{$svol->{RSH}}) : undef), + vinfo_prefixed_keys("source", $svol), snapshot_path => $sroot->{PATH} . (config_key($config_subvol, "snapshot_dir", prefix => '/') // ""), snapshot_name => config_key($config_subvol, "snapshot_name"), snapshot_preserve => format_preserve_matrix(config => $config_subvol, prefix => "snapshot"), @@ -2289,10 +2296,7 @@ MAIN: next if($config_target->{ABORTED}); my $droot = vinfo($config_target->{url}, $config_target); my $targeth = { %$subvolh, - target_url => $droot->{URL}, - target_path => $droot->{PATH}, - target_host => $droot->{HOST}, - target_rsh => ($droot->{RSH} ? join(" ", @{$droot->{RSH}}) : undef), + vinfo_prefixed_keys("target", $droot), target_preserve => format_preserve_matrix(config => $config_target, prefix => "target"), }; if($action_list eq "target_uniq") { @@ -2352,7 +2356,7 @@ MAIN: default_format => "table", data => \@mixed_data, formats => { raw => [ @raw_subvol_keys, @raw_target_keys ], - table => [ qw( source_host source_path snapshot_path snapshot_name target_host target_path ) ], + table => [ qw( source snapshot_path snapshot_name target ) ], long => [ qw( source_host source_path snapshot_path snapshot_name snapshot_preserve target_host target_path target_preserve ) ], }, ); @@ -2594,13 +2598,10 @@ MAIN: push @tree_out, "|-- $svol->{PRINT}"; foreach my $snapshot (sort { $a->{cgen} cmp $b->{cgen} } get_snapshot_children($sroot, $svol)) { - my $raw_data = { type => "snapshot", - btrbk_flags => [ ], - source_url => $svol->{URL}, - source_host => $svol->{HOST}, - source_path => $svol->{PATH}, - snapshot_url => $snapshot->{URL}, - snapshot_path => $snapshot->{PATH}, + my $raw_data = { type => "snapshot", + btrbk_flags => [ ], + vinfo_prefixed_keys("source", $svol), + vinfo_prefixed_keys("snapshot", $snapshot), snapshot_basename => config_key($config_subvol, "snapshot_name"), }; if($snapshot->{cgen} == $svol->{gen}) { @@ -2618,10 +2619,8 @@ MAIN: foreach (sort { $a->{SUBVOL_PATH} cmp $b->{SUBVOL_PATH} } get_receive_targets($droot, $snapshot)) { push @tree_out, "| | >>> $_->{PRINT}"; push @raw_out, { %$raw_data, - type => "received", - received_url => $_->{URL}, - received_path => $_->{PATH}, - received_host => $_->{HOST}, + type => "received", + vinfo_prefixed_keys("received", $_), }; } } @@ -3186,7 +3185,7 @@ MAIN: default_format => "table", data => \@action_log, formats => { raw => [ qw( time type status target_url source_url parent_url ) ], - table => [ qw( type status target_url source_url parent_url ) ], + table => [ qw( type status target source parent ) ], }, ); } From fa382d5e6d2ba38f17814b354923a6e1520183c2 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 13 Oct 2015 01:10:06 +0200 Subject: [PATCH 16/19] btrbk: add global table format definitions; bugfix on action_log table --- btrbk | 155 ++++++++++++++++++++++++---------------------------------- 1 file changed, 63 insertions(+), 92 deletions(-) diff --git a/btrbk b/btrbk index 717caf0..75c95ae 100755 --- a/btrbk +++ b/btrbk @@ -117,6 +117,40 @@ my %config_options = ( my @config_target_types = qw(send-receive raw); +my %table_formats = ( + list_volume => { table => [ qw( volume ) ], + long => [ qw( volume_host volume_path ) ], + raw => [ qw( volume_url volume_host volume_path volume_rsh ) ], + }, + list_source => { table => [ qw( source_host source_path snapshot_path snapshot_name ) ], + long => [ qw( source_host source_path snapshot_path snapshot_name ) ], + raw => [ qw( source_url source_host source_path snapshot_path snapshot_name source_rsh ) ], + }, + list_target => { table => [ qw( target ) ], + long => [ qw( target_host target_path ) ], + raw => [ qw( target_url target_host target_path target_rsh ) ], + }, + list => { table => [ qw( source snapshot_path snapshot_name target ) ], + long => [ qw( source_host source_path snapshot_path snapshot_name snapshot_preserve target_host target_path target_preserve ) ], + raw => [ qw( source_url source_host source_path snapshot_path snapshot_name snapshot_preserve target_url target_host target_path target_preserve source_rsh target_rsh ) ], + }, + + tree => { table => [ qw( source snapshot btrbk_flags received ) ], + long => [ qw( source_host source_path snapshot_path snapshot_name btrbk_flags received_host received_path ) ], + raw => [ qw( source_host source_path snapshot_path snapshot_name btrbk_flags received_host received_path source_rsh ) ], + }, + + schedule => { table => [ qw( action target scheme reason ) ], + long => [ qw( action host root_path name scheme reason ) ], + raw => [ qw( topic action url host path dow d m w) ], + }, + + action_log => { table => [ qw( type status target source parent ) ], + long => [ qw( localtime type status target_host target_path source_host source_path parent_path error_message ) ], + raw => [ qw( time localtime type status target_url source_url parent_url error_message ) ], + }, +); + my %root_tree_cache; # map URL to SUBTREE (needed since "btrfs subvolume list" does not provide us with the uuid of the btrfs root node) my %vinfo_cache; # map URL to vinfo my %uuid_info; # map UUID to btr_tree node @@ -128,6 +162,7 @@ my $dryrun; my $loglevel = 1; my $show_progress = 0; my $err = ""; +my $output_format = undef; $SIG{__DIE__} = sub { @@ -401,7 +436,7 @@ sub vinfo_prefixed_keys($$) my $prefix = shift || die; my $vinfo = shift || die; my %ret; - foreach qw( URL PATH HOST NAME SUBVOL_PATH ) { + foreach (qw( URL PATH HOST NAME SUBVOL_PATH )) { $ret{$prefix . '_' . lc($_)} = $vinfo->{$_}; } $ret{$prefix} = $vinfo->{PRINT}; @@ -984,8 +1019,8 @@ sub btrfs_subvolume_snapshot($$) ); action("snapshot", status => ($dryrun ? "DRYRUN" : (defined($ret) ? "success" : "ERROR")), - target_vinfo => $target_vol, - source_vinfo => $svol, + vinfo_prefixed_keys("target", $target_vol), + vinfo_prefixed_keys("source", $svol), ); ERROR "Failed to create btrfs subvolume snapshot: $svol->{PRINT} -> $target_path" unless(defined($ret)); return defined($ret) ? $target_path : undef; @@ -1017,7 +1052,7 @@ sub btrfs_subvolume_delete($@) ); action($opts{type} // "delete", status => ($dryrun ? "DRYRUN" : (defined($ret) ? "success" : "ERROR")), - target_vinfo => $_, + vinfo_prefixed_keys("target", $_), ) foreach (@$targets); ERROR "Failed to delete btrfs subvolumes: " . join(' ', map( { $_->{PRINT} } @$targets)) unless(defined($ret)); return defined($ret) ? scalar(@$targets) : undef; @@ -1405,9 +1440,9 @@ sub macro_send_receive($@) action("send-receive", status => ($dryrun ? "DRYRUN" : ($ret ? "success" : "ERROR")), - target_vinfo => $vol_received, - source_vinfo => $snapshot, - parent_vinfo => $parent, + vinfo_prefixed_keys("target", $vol_received), + vinfo_prefixed_keys("source", $snapshot), + vinfo_prefixed_keys("parent", $parent), ); unless($ret) { $info{ERROR} = 1; @@ -1760,17 +1795,18 @@ sub print_header(@) sub print_formatted(@) { + my $format_key = shift || die; + my $data = shift || die; + my $default_format = "table"; my %args = @_; my $title = $args{title}; - my $format = $args{output_format} || "table"; - my $default = $args{default_format} || die; - my $data = $args{data} || die; - my $keys = $args{formats}->{$format}; + my $format = $args{output_format} || $output_format || $default_format; + my $keys = $table_formats{$format_key}->{$format}; unless($keys) { - WARN "Unsupported output format \"$format\", defaulting to \"$default\" format."; - $keys = $args{formats}->{$default} || die; - $format = $default; + WARN "Unsupported output format \"$format\", defaulting to \"$default_format\" format."; + $keys = $table_formats{$format_key}->{$default_format} || die; + $format = $default_format; } print "$title\n" if($title); @@ -1778,6 +1814,7 @@ sub print_formatted(@) { # output: key0="value0" key1="value1" ... foreach my $row (@$data) { + print "list_type=\"$format_key\" "; print join(' ', map { "$_=\"" . ($row->{$_} // "") . "\""; } @$keys) . "\n"; } } @@ -1833,7 +1870,7 @@ MAIN: my @today = @today_and_now[0..2]; - my ($config_cmdline, $quiet, $verbose, $preserve_backups, $resume_only, $output_format); + my ($config_cmdline, $quiet, $verbose, $preserve_backups, $resume_only); unless(GetOptions( 'help|h' => sub { VERSION_MESSAGE(); HELP_MESSAGE(0); exit 0; }, 'version' => sub { VERSION_MESSAGE(); exit 0; }, @@ -2299,7 +2336,7 @@ MAIN: vinfo_prefixed_keys("target", $droot), target_preserve => format_preserve_matrix(config => $config_target, prefix => "target"), }; - if($action_list eq "target_uniq") { + if($action_list eq "target") { next if($target_uniq{$droot->{URL}}); $target_uniq{$droot->{URL}} = 1; } @@ -2311,55 +2348,18 @@ MAIN: push @mixed_data, $subvolh unless($found); } } - - my @raw_vol_keys = qw( volume_url volume_host volume_path volume_rsh ); - my @raw_subvol_keys = qw( source_url source_host source_path source_rsh snapshot_path snapshot_name ); - my @raw_target_keys = qw( target_url target_host target_path target_rsh ); - $output_format ||= "table"; if($action_list eq "volume") { - print_formatted( - output_format => $output_format, - default_format => "table", - data => \@vol_data, - formats => { raw => \@raw_vol_keys, - table => [ qw( volume_host volume_path ) ], - long => \@raw_vol_keys, - }, - ); + print_formatted("list_volume", \@vol_data); } elsif($action_list eq "source") { - print_formatted( - output_format => $output_format, - default_format => "table", - data => \@subvol_data, - formats => { raw => \@raw_subvol_keys, - table => [ qw( source_host source_path snapshot_path snapshot_name ) ], - long => \@raw_subvol_keys, - }, - ); + print_formatted("list_source", \@subvol_data); } elsif($action_list eq "target") { - print_formatted( - output_format => $output_format, - default_format => "table", - data => \@target_data, - formats => { raw => \@raw_target_keys, - table => [ qw( target_host target_path ) ], - long => \@raw_target_keys, - }, - ); + print_formatted("list_target", \@target_data); } else { # default format - print_formatted( - output_format => $output_format, - default_format => "table", - data => \@mixed_data, - formats => { raw => [ @raw_subvol_keys, @raw_target_keys ], - table => [ qw( source snapshot_path snapshot_name target ) ], - long => [ qw( source_host source_path snapshot_path snapshot_name snapshot_preserve target_host target_path target_preserve ) ], - }, - ); + print_formatted("list", \@mixed_data); } exit 0; } @@ -2602,7 +2602,7 @@ MAIN: btrbk_flags => [ ], vinfo_prefixed_keys("source", $svol), vinfo_prefixed_keys("snapshot", $snapshot), - snapshot_basename => config_key($config_subvol, "snapshot_name"), + snapshot_name => config_key($config_subvol, "snapshot_name"), }; if($snapshot->{cgen} == $svol->{gen}) { push @tree_out, "| ^== $snapshot->{PATH}"; @@ -2647,14 +2647,7 @@ MAIN: print join("\n", @tree_out); } else { - print_formatted( - output_format => $output_format, - default_format => "table", - data => \@raw_out, - formats => { raw => [ qw( source_host source_path snapshot_path snapshot_basename btrbk_flags received_host received_path ) ], - table => [ qw( source_url snapshot_url btrbk_flags received_url ) ], - }, - ); + print_formatted("tree", \@raw_out); } exit 0; } @@ -3041,23 +3034,9 @@ MAIN: my @data_backup = map { $_->{topic} eq "backup" ? $_ : () } @data; my @data_snapshot = map { $_->{topic} eq "snapshot" ? $_ : () } @data; - my %format_args = ( - output_format => $output_format, - default_format => "table", - formats => { raw => [ qw( topic action url host path dow d m w) ], - table => [ qw( action target scheme reason ) ], - long => [ qw( action host root_path name scheme reason ) ], - }, - ); - print_formatted( title => "SNAPSHOT SCHEDULE", - data => \@data_snapshot, - %format_args, - ); + print_formatted("schedule", \@data_snapshot, title => "SNAPSHOT SCHEDULE"); print "\n"; - print_formatted( title => "BACKUP SCHEDULE", - data => \@data_backup, - %format_args, - ); + print_formatted("schedule", \@data_backup, title => "BACKUP SCHEDULE"); print "\n"; } @@ -3178,16 +3157,8 @@ MAIN: else { # print action log - @action_log = map { $_->{timestamp} = localtime($_->{time}); $_; } @action_log; - print_formatted( - title => "BACKUP SUMMARY", - output_format => $output_format, - default_format => "table", - data => \@action_log, - formats => { raw => [ qw( time type status target_url source_url parent_url ) ], - table => [ qw( type status target source parent ) ], - }, - ); + @action_log = map { $_->{localtime} = localtime($_->{time}); $_; } @action_log; + print_formatted("action_log", \@action_log, title => "BACKUP SUMMARY"); } } From 282668927d04ab377db91c4f840b8576620b986b Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 13 Oct 2015 01:39:58 +0200 Subject: [PATCH 17/19] btrbk: cosmetics on table format: no trailing whitespace --- btrbk | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/btrbk b/btrbk index 75c95ae..166bee8 100755 --- a/btrbk +++ b/btrbk @@ -1841,14 +1841,21 @@ sub print_formatted(@) } # print keys (headings) - print join(" ", map { $_ . (' ' x ($maxlen{$_} - length($_))) } @$keys) . "\n"; + my $fill = ''; + foreach (@$keys) { + print $fill . $_; + $fill = ' ' x (2 + $maxlen{$_} - length($_)); + } + print "\n"; print join(" ", map { '-' x ($maxlen{$_}) } @$keys) . "\n"; # print values foreach my $row (@$data) { + my $fill = ''; foreach (@$keys) { my $val = $row->{$_}; - print $val . (' ' x (2 + $maxlen{$_} - length($val))); + print $fill . $val; + $fill = ' ' x (2 + $maxlen{$_} - length($val)); } print "\n"; } From 46c7841e9378e427183b53b0228d18ccc49e1d05 Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 13 Oct 2015 18:24:30 +0200 Subject: [PATCH 18/19] btrbk: add transaction log (configuration option "transaction_log " --- ChangeLog | 1 + btrbk | 152 ++++++++++++++++++++++++++++++++------------- btrbk.conf.example | 3 + doc/btrbk.conf.5 | 7 +++ 4 files changed, 119 insertions(+), 44 deletions(-) diff --git a/ChangeLog b/ChangeLog index 072e28c..06abdbd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -12,6 +12,7 @@ btrbk-current * Added "target raw", with GnuPG and compression support (experimental). * Added configuration option "timestamp_format short|long". + * Added transaction log (configuration option "transaction_log"). * Bugfix: correctly handle "incremental no" option. * Bugfix: return exit status 10 instead of 0 if one or more backup tasks aborted. diff --git a/btrbk b/btrbk index 166bee8..bbb761a 100755 --- a/btrbk +++ b/btrbk @@ -45,6 +45,7 @@ use warnings FATAL => qw( all ); use Carp qw(confess); use Date::Calc qw(Today_and_Now Delta_Days Day_of_Week); use Getopt::Long qw(GetOptions); +use POSIX qw(strftime); use Data::Dumper; our $VERSION = "0.21.0-dev"; @@ -89,6 +90,7 @@ my %config_options = ( ssh_port => { default => "default", accept => [ "default" ], accept_numeric => 1 }, ssh_compression => { default => undef, accept => [ "yes", "no" ] }, ssh_cipher_spec => { default => "default", accept_regexp => qr/^$ssh_cipher_match(,$ssh_cipher_match)*$/ }, + transaction_log => { default => undef, accept_file => { absolute => 1 } }, raw_target_compress => { default => undef, accept => [ "no", "gzip", "bzip2", "xz" ] }, raw_target_encrypt => { default => undef, accept => [ "no", "gpg" ] }, @@ -146,8 +148,9 @@ my %table_formats = ( }, action_log => { table => [ qw( type status target source parent ) ], - long => [ qw( localtime type status target_host target_path source_host source_path parent_path error_message ) ], - raw => [ qw( time localtime type status target_url source_url parent_url error_message ) ], + long => [ qw( localtime type status duration target_host target_path source_host source_path parent_path message ) ], + raw => [ qw( time localtime type status duration target_url source_url parent_url message ) ], + tlog => [ qw( localtime type status target_url source_url parent_url message ) ], }, ); @@ -162,7 +165,8 @@ my $dryrun; my $loglevel = 1; my $show_progress = 0; my $err = ""; -my $output_format = undef; +my $output_format; +my $tlog_fh; $SIG{__DIE__} = sub { @@ -214,29 +218,57 @@ sub INFO { my $t = shift; print STDERR "$t\n" if($loglevel >= 2); } sub WARN { my $t = shift; print STDERR "WARNING: $t\n" if($loglevel >= 1); } sub ERROR { my $t = shift; print STDERR "ERROR: $t\n"; } -sub action($@) -{ - my $type = shift // die; - my %h = @_; - $h{type} = $type; - $h{time} = time; - push @action_log, \%h; -} - sub ABORTED($$) { my $config = shift; my $t = shift; $config->{ABORTED} = $t; unless($t eq "USER_SKIP") { + $t =~ s/\n/ /g; action("abort_" . ($config->{CONTEXT} || "undef"), status => "ABORT", vinfo_prefixed_keys("target", vinfo($config->{url}, $config)), - error_message => $t, + message => $t, ); } } + +sub init_transaction_log($) +{ + my $file = shift // die; + if(open($tlog_fh, ">> $file")) + { + # print headers and startup message + print_formatted("action_log", [ ], output_format => "tlog", outfile => $tlog_fh); + INFO "Using transaction log: $file"; + } else { + $tlog_fh = undef; + ERROR "Failed to open transaction log '$file': $!"; + } +} + +sub close_transaction_log() +{ + if($tlog_fh) { + DEBUG "Closing transaction log"; + close $tlog_fh || ERROR "Failed to close transaction log: $!"; + } +} + +sub action($@) +{ + my $type = shift // die; + my $h = { @_ }; + my $time = time; + $h->{type} = $type; + $h->{time} = $time; + $h->{localtime} = strftime("%FT%T%z", localtime($time)); + print_formatted("action_log", [ $h ], output_format => "tlog", no_header => 1, outfile => $tlog_fh); + push @action_log, $h; +} + + sub run_cmd(@) { my @commands = (ref($_[0]) eq "HASH") ? @_ : { @_ }; @@ -1014,11 +1046,13 @@ sub btrfs_subvolume_snapshot($$) DEBUG "[btrfs] source: $src_path"; DEBUG "[btrfs] target: $target_path"; INFO ">>> $target_vol->{PRINT}"; + my $starttime = time; my $ret = run_cmd(cmd => [ qw(btrfs subvolume snapshot), '-r', $src_path, $target_path ], rsh => $svol->{RSH}, ); action("snapshot", status => ($dryrun ? "DRYRUN" : (defined($ret) ? "success" : "ERROR")), + duration => (time - $starttime), vinfo_prefixed_keys("target", $target_vol), vinfo_prefixed_keys("source", $svol), ); @@ -1047,11 +1081,13 @@ sub btrfs_subvolume_delete($@) my @options; @options = ("--commit-$commit") if($commit); my @target_paths = map( { $_->{PATH} } @$targets); + my $starttime = time; my $ret = run_cmd(cmd => [ qw(btrfs subvolume delete), @options, @target_paths ], rsh => $rsh, ); action($opts{type} // "delete", status => ($dryrun ? "DRYRUN" : (defined($ret) ? "success" : "ERROR")), + duration => (time - $starttime), vinfo_prefixed_keys("target", $_), ) foreach (@$targets); ERROR "Failed to delete btrfs subvolumes: " . join(' ', map( { $_->{PRINT} } @$targets)) unless(defined($ret)); @@ -1100,7 +1136,16 @@ sub btrfs_send_receive($$$$) rsh => $target->{RSH}, name => "btrfs receive", }; + my $starttime = time; my $ret = run_cmd(@cmd_pipe); + action("send-receive", + status => ($dryrun ? "DRYRUN" : (defined($ret) ? "success" : "ERROR")), + duration => (time - $starttime), + vinfo_prefixed_keys("target", $vol_received), + vinfo_prefixed_keys("source", $snapshot), + vinfo_prefixed_keys("parent", $parent), + ); + unless(defined($ret)) { ERROR "Failed to send/receive btrfs subvolume: $snapshot->{PRINT} " . ($parent_path ? "[$parent_path]" : "") . " -> $target->{PRINT}"; @@ -1194,6 +1239,7 @@ sub btrfs_send_to_file($$$$;@) DEBUG "[btrfs] parent: $parent->{PRINT}" if($parent); DEBUG "[btrfs] target: $target->{PRINT}"; + my $starttime = time; my $ret = run_cmd(@cmd_pipe); if(defined($ret)) { # Test target file for "exists and size > 0" after writing, @@ -1205,6 +1251,13 @@ sub btrfs_send_to_file($$$$;@) name => "test", }); } + action("send-to-raw", + status => ($dryrun ? "DRYRUN" : (defined($ret) ? "success" : "ERROR")), + duration => (time - $starttime), + vinfo_prefixed_keys("target", $vol_received), + vinfo_prefixed_keys("source", $snapshot), + vinfo_prefixed_keys("parent", $parent), + ); unless(defined($ret)) { ERROR "Failed to send btrfs subvolume to raw file: $snapshot->{PRINT} " . ($parent_path ? "[$parent_path]" : "") . " -> $vol_received->{PRINT}"; return undef; @@ -1438,12 +1491,6 @@ sub macro_send_receive($@) $config_target->{SUBVOL_RECEIVED} //= []; push(@{$config_target->{SUBVOL_RECEIVED}}, \%info); - action("send-receive", - status => ($dryrun ? "DRYRUN" : ($ret ? "success" : "ERROR")), - vinfo_prefixed_keys("target", $vol_received), - vinfo_prefixed_keys("source", $snapshot), - vinfo_prefixed_keys("parent", $parent), - ); unless($ret) { $info{ERROR} = 1; return undef; @@ -1818,6 +1865,18 @@ sub print_formatted(@) print join(' ', map { "$_=\"" . ($row->{$_} // "") . "\""; } @$keys) . "\n"; } } + elsif($format eq "tlog") + { + # output: value0 value1, ... + if($tlog_fh) { + unless($args{no_header}) { + print $tlog_fh join(' ', @$keys) . "\n"; + } + foreach my $row (@$data) { + print $tlog_fh join(' ', map { ($row->{$_} // "-") } @$keys) . "\n"; + } + } + } else { # sanitize and calculate maxlen for each column @@ -2136,6 +2195,15 @@ MAIN: } + # + # open transaction log + # + if($action_run && (not $dryrun) && config_key($config, "transaction_log")) { + init_transaction_log(config_key($config, "transaction_log")); + } + action("startup", status => "v$VERSION", message => "$version_info"); + + # # filter subvolumes matching command line arguments # @@ -3021,8 +3089,24 @@ MAIN: } } + my $err_count = 0; + foreach my $config_vol (@{$config->{VOLUME}}) { + if($config_vol->{ABORTED} && ($config_vol->{ABORTED} ne "USER_SKIP")) { $err_count++; next; } + foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { + if($config_subvol->{ABORTED} && ($config_subvol->{ABORTED} ne "USER_SKIP")) { $err_count++; next; } + foreach my $config_target (@{$config_subvol->{TARGET}}) { + if($config_target->{ABORTED} && ($config_target->{ABORTED} ne "USER_SKIP")) { $err_count++; next; } + } + } + } + my $time_elapsed = time - $start_time; INFO "Completed within: ${time_elapsed}s (" . localtime(time) . ")"; + action("finished", + duration => $time_elapsed, + message => $err_count ? "At least one backup task aborted" : "No errors" + ); + close_transaction_log(); unless($quiet) @@ -3056,8 +3140,6 @@ MAIN: { my @unrecoverable; my @out; - my @raw_data; - my $err_count = 0; foreach my $config_vol (@{$config->{VOLUME}}) { my $sroot = $config_vol->{sroot} || vinfo($config_vol->{url}, $config_vol); @@ -3096,18 +3178,18 @@ MAIN: if($config_target->{ABORTED} && ($config_target->{ABORTED} ne "USER_SKIP")) { push @subvol_out, "!!! Target \"$droot->{PRINT}\" aborted: $config_target->{ABORTED}"; - $err_count++; } - push(@unrecoverable, $config_target->{UNRECOVERABLE}) if($config_target->{UNRECOVERABLE}); + if($config_target->{UNRECOVERABLE}) { + push(@unrecoverable, $config_target->{UNRECOVERABLE}); + } } if($config_vol->{ABORTED} && ($config_vol->{ABORTED} ne "USER_SKIP")) { - # repeat volume errors in subvolume context ($err_count is increased in volume context below) + # repeat volume errors in subvolume context push @subvol_out, "!!! Volume \"$sroot->{PRINT}\" aborted: $config_vol->{ABORTED}"; } if($config_subvol->{ABORTED} && ($config_subvol->{ABORTED} ne "USER_SKIP")) { push @subvol_out, "!!! Aborted: $config_subvol->{ABORTED}"; - $err_count++; } if(@subvol_out) { @@ -3120,15 +3202,6 @@ MAIN: push @out, "$svol->{PRINT}", "", ""; } } - if($config_vol->{ABORTED} && ($config_vol->{ABORTED} ne "USER_SKIP")) { - push @raw_data, { type => "btrbk_volume", - status => "ABORT", - target_url => $sroot->{URL}, - error_message => $config_vol->{ABORTED}, - SORT => 1, - }; - $err_count++; - } } print_header(title => "Backup Summary", @@ -3164,20 +3237,11 @@ MAIN: else { # print action log - @action_log = map { $_->{localtime} = localtime($_->{time}); $_; } @action_log; print_formatted("action_log", \@action_log, title => "BACKUP SUMMARY"); } } - foreach my $config_vol (@{$config->{VOLUME}}) { - exit 10 if($config_vol->{ABORTED} && ($config_vol->{ABORTED} ne "USER_SKIP")); - foreach my $config_subvol (@{$config_vol->{SUBVOLUME}}) { - exit 10 if($config_subvol->{ABORTED} && ($config_subvol->{ABORTED} ne "USER_SKIP")); - foreach my $config_target (@{$config_subvol->{TARGET}}) { - exit 10 if($config_target->{ABORTED} && ($config_target->{ABORTED} ne "USER_SKIP")); - } - } - } + exit 10 if($err_count); } } diff --git a/btrbk.conf.example b/btrbk.conf.example index 72146a0..5d77515 100644 --- a/btrbk.conf.example +++ b/btrbk.conf.example @@ -64,6 +64,9 @@ snapshot_dir _btrbk_snap # Set this either globally or in a specific "target" section. #btrfs_progs_compat no +# Enable transaction log +#transaction_log /var/log/btrbk_transaction.log + # # Volume section: "volume " diff --git a/doc/btrbk.conf.5 b/doc/btrbk.conf.5 index 11f96a7..d31dd2a 100644 --- a/doc/btrbk.conf.5 +++ b/doc/btrbk.conf.5 @@ -258,6 +258,13 @@ can lead to false guesses if the snapshot or target subvolumes are manipulated by hand (moved, deleted). .RE .PP +\fBtransaction_log\fR +.RS 4 +If set, all transactions (snapshot create, subvolume send-receive, +subvolume delete) as well as abort messages are logged to , in a +space-separated table format. +.RE +.PP Lines that contain a hash character (#) in the first column are treated as comments. .SH AVAILABILITY From bca413b7ca0b4f4a67bb19d724e7045b26afb25b Mon Sep 17 00:00:00 2001 From: Axel Burri Date: Tue, 13 Oct 2015 19:32:48 +0200 Subject: [PATCH 19/19] documentation: add missing "config print" command; add predefined "list" filters; cosmetics; fix FAQ --- btrbk | 2 +- doc/FAQ.md | 4 ++-- doc/btrbk.1 | 30 ++++++++++++++++-------------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/btrbk b/btrbk index bbb761a..e03c835 100755 --- a/btrbk +++ b/btrbk @@ -203,8 +203,8 @@ sub HELP_MESSAGE print STDERR "commands:\n"; print STDERR " run [filter...] perform backup operations as defined in the config file\n"; print STDERR " dryrun [filter...] don't run btrfs commands; show what would be executed\n"; - print STDERR " tree [filter...] shows backup tree\n"; print STDERR " list [filter...] print source/snapshot/target relations\n"; + print STDERR " tree [filter...] shows backup tree\n"; print STDERR " info [filter...] print useful filesystem information\n"; print STDERR " origin print origin information for subvolume\n"; print STDERR " diff shows new files since subvolume for subvolume \n"; diff --git a/doc/FAQ.md b/doc/FAQ.md index 1809915..be43768 100644 --- a/doc/FAQ.md +++ b/doc/FAQ.md @@ -37,12 +37,12 @@ valid mount-points, you can loop through the configuration and mount the volumes like this: #!/bin/sh - btrbk config dump volume | while read line; do + btrbk config list volume --format=raw | while read line; do eval $line $volume_rsh mount $volume_path done -Note that the `btrbk config dump volume` command accepts filters (see +Note that the `btrbk config list` command accepts filters (see [btrbk(1)], FILTER STATEMENTS), which means you can e.g. add "group automount" tags in your configuration and dump only the volumes of this group: `btrbk config dump volume automount`. diff --git a/doc/btrbk.1 b/doc/btrbk.1 index ac5e6e1..9c2acbb 100644 --- a/doc/btrbk.1 +++ b/doc/btrbk.1 @@ -88,7 +88,7 @@ Print output in table format (shortcut for "--format=table"). .RS 4 Print output in specified format. If set to "raw", prints space-separated key="value" pairs (machine-readable). Affects output -format for \fBrun\fR, \fBdryrun\fR, \fBinfo\fR and \fBtree\fR +format for \fBrun\fR, \fBdryrun\fR, \fBlist\fR and \fBtree\fR commands. Useful for further exporting/scripting. .RE .PP @@ -141,12 +141,15 @@ in conjunction with \fI\-l debug\fR to see the btrfs commands that would be executed. .RE .PP -.B info +.B list [filter...] .RS 4 -Print filesystem usage information for all source/target -volumes. Optionally filtered by [filter...] arguments (see \fIFILTER -STATEMENTS\fR below). +Print the source/snapshot/target relations of the configured +subvolumes in a tabular form. Optionally filtered by [filter...] +arguments (see \fIFILTER STATEMENTS\fR below). Accepts predefined +filters \fIvolume\fR, \fIsource\fR and \fItarget\fR. Use the +\fI\-\-format\fR command line option to switch between different +output formats. .RE .PP .B tree @@ -158,14 +161,12 @@ STATEMENTS\fR below). Use the \fI\-\-format\fR command line option to switch between different output formats. .RE .PP -.B list +.B info [filter...] .RS 4 -Print the source/snapshot/target relations of the configured -subvolumes in a tabular form. Optionally filtered by [filter...] -arguments (see \fIFILTER STATEMENTS\fR below). Use the -\fI\-\-format\fR command line option to switch between different -output formats. +Print filesystem usage information for all source/target +volumes. Optionally filtered by [filter...] arguments (see \fIFILTER +STATEMENTS\fR below). .RE .PP .B origin @@ -181,10 +182,11 @@ parent-child relationship as well as the received-from information. Print new files since subvolume for subvolume . .RE .PP -.B config dump -volume|source|target [filter...] \fI*experimental*\fR +.B config +print|print-all .RS 4 -Dump parsed content from the configuration file, in format: key="value"... +Prints the parsed configuration file. Use the \fI\-\-format\fR command +line option to switch between different output formats. .RE .SH FILTER STATEMENTS Filter arguments are accepted in form: