From bf1123d93d4f13d462ebfe9363275416cd16090b Mon Sep 17 00:00:00 2001 From: Brian Cheldelin Date: Mon, 19 Sep 2016 16:39:03 -0600 Subject: [PATCH 1/3] Add support for rsync storage --- README.md | 16 ++++++ manifests/job.pp | 44 +++++++++++++-- manifests/params.pp | 4 ++ spec/defines/backup_job_spec.rb | 96 +++++++++++++++++++++++++++++++++ templates/job/rsync.erb | 21 ++++++++ 5 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 templates/job/rsync.erb diff --git a/README.md b/README.md index d2d4581..29fce2d 100644 --- a/README.md +++ b/README.md @@ -276,6 +276,22 @@ String. Bucket to store archive when using S3 storage type String. Region to store archive when using S3 storage type +##### `rsync_mode` + +String. rsync mode to use with rsync storage + +##### `rsync_port` + +Integer. Port for rsync storage model to use + +Default: 22 + +##### `rsync_compress` + +Boolean. Weather or not to enable rsync compression on transfer + +Default: undef (false) + ##### `encryptor` String. Encryptor to use on backup archive diff --git a/manifests/job.pp b/manifests/job.pp index d103c8a..e45575d 100644 --- a/manifests/job.pp +++ b/manifests/job.pp @@ -58,6 +58,10 @@ # FTP $ftp_port = $::backup::ftp_port, $ftp_passive_mode = $::backup::ftp_passive_mode, + # rsync + $rsync_mode = $::backup::rsync_mode, + $rsync_port = $::backup::rsync_port, + $rsync_compress = $::backup::rsync_compress, ## Encryptors $encryptor = $::backup::encryptor, @@ -173,8 +177,8 @@ } # Storage - if !member(['s3', 'local', 'ftp'], $storage_type) { - fail("[Backup::Job::${name}]: Currently supported storage types are: ftp, local, and s3") + if !member(['s3', 'local', 'ftp', 'rsync'], $storage_type) { + fail("[Backup::Job::${name}]: Currently supported storage types are: ftp, local, s3, and rsync") } if $keep and !is_integer($keep) { @@ -186,7 +190,7 @@ } # local and ftp require path parameter - if member(['ftp', 'local'], $storage_type) { + if member(['ftp', 'local', 'rsync'], $storage_type) { if !$path { fail("[Backup::Job::${name}]: Path parameter is required with storage_type => ${storage_type}") } @@ -243,6 +247,27 @@ validate_bool($ftp_passive_mode) } + if $storage_type == 'rsync' { + if $rsync_mode and !member([ + 'ssh', + 'ssh_daemon', + 'rsync_daemon', + ], $rsync_mode ) { + fail("[Backup::Job::${name}]: ${rsync_mode} is not a valid mode") + } + if !$storage_host or !is_string($storage_host) { + fail("[Backup::Job::${name}]: Parameter storage_host is required for rsync storage") + } + if !$rsync_port or !is_integer($rsync_port) { + fail("[Backup::Job::${name}]: rsync_port must be an integer. (Got: ${rsync_port})") + } + + if $rsync_compress { + validate_bool($rsync_compress) + } + + } + # Encryptor if $encryptor and !member(['openssl'], $encryptor) { fail("[Backup::Job::${name}]: Supported encryptors are openssl") @@ -479,6 +504,19 @@ content => template('backup/job/ftp.erb'), order => '35', } + } elsif $storage_type == 'rsync' { + # Template uses + # - $server_username + # - $server_ip + # - $server_port + # - $server_compress + # - $server_mode + # - $path + concat::fragment { "${_name}_rsync": + target => "/etc/backup/models/${_name}.rb", + content => template('backup/job/rsync.erb'), + order => '35', + } } if $logfile_enabled or $syslog_enabled or $console_quiet { diff --git a/manifests/params.pp b/manifests/params.pp index c2d6761..ff0ca0b 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -53,6 +53,10 @@ # FTP $ftp_port = 21 $ftp_passive_mode = false + # rsync + $rsync_mode = undef + $rsync_port = 22 + $rsync_compress = undef ## Encryptors $encryptor = undef diff --git a/spec/defines/backup_job_spec.rb b/spec/defines/backup_job_spec.rb index c518928..d90c409 100644 --- a/spec/defines/backup_job_spec.rb +++ b/spec/defines/backup_job_spec.rb @@ -346,6 +346,68 @@ end end # ftp + context 'rsync' do + context 'bad mode' do + let(:params) { { + :types => 'archive', + :add => 'here', + :storage_type => 'rsync', + :storage_host => 'mysite.example.com', + :path => '/there', + :rsync_mode => 'abcde' + } } + it { expect { is_expected.to compile }.to raise_error(/abcde is not a valid mode/)} + end + + context 'missing host' do + let(:params) { { + :types => 'archive', + :add => 'here', + :storage_type => 'rsync', + :storage_username => 'myuser', + :path => '/there', + } } + it { expect { is_expected.to compile }.to raise_error(/storage_host is required/)} + end + + context 'missing path' do + let(:params) { { + :types => 'archive', + :add => 'here', + :storage_type => 'rsync', + :storage_username => 'myuser', + :storage_host => 'mysite.example.com', + } } + it { expect { is_expected.to compile }.to raise_error(/Path parameter is required/)} + end + + context 'bad port' do + let(:params) { { + :types => 'archive', + :add => 'here', + :storage_type => 'rsync', + :storage_username => 'myuser', + :storage_host => 'mysite.example.com', + :rsync_port => 'abcde', + :path => '/there', + } } + it { expect { is_expected.to compile }.to raise_error(/rsync_port must be an integer/)} + end + + context 'bad rsync_compress' do + let(:params) { { + :types => 'archive', + :add => 'here', + :storage_type => 'rsync', + :storage_username => 'myuser', + :storage_host => 'mysite.example.com', + :path => '/there', + :rsync_compress => 'bob', + } } + it { expect { is_expected.to compile }.to raise_error(/"bob" is not a boolean/)} + end + end # rsync + context 'encryptor generic' do context 'bad encryptor' do let(:params) { { @@ -1014,6 +1076,40 @@ end end # ftp + context 'rsync' do + context 'minimum settings' do + let(:params) { { + :types => 'archive', + :add => '/here', + :storage_type => 'rsync', + :storage_host => 'mysite.example.com', + :path => '/there', + } } + it { should_not contain_concat__fragment('job1_rsync').with(:content => /server\..+user/) } + it { should contain_concat__fragment('job1_rsync').with(:content => /server\.ip\s+=\s"mysite.example.com"/) } + it { should contain_concat__fragment('job1_rsync').with(:content => /server\.port\s+=\s22$/) } + it { should contain_concat__fragment('job1_rsync').with(:content => /server\.path\s+=\s"\/there"/) } + it { should_not contain_concat__fragment('job1_rsync').with(:content => /server\.rsync_compress/) } + end + + context 'all params' do + let(:params) { { + :types => 'archive', + :add => '/here', + :storage_type => 'rsync', + :storage_username => 'myuser', + :storage_host => 'mysite.example.com', + :path => '/there', + :rsync_port => 22, + :rsync_mode => 'ssh', + :keep => 10, + } } + it { should contain_concat__fragment('job1_rsync').with(:content => /server\..+user\s+=\s"myuser"/) } + it { should contain_concat__fragment('job1_rsync').with(:content => /server\.port\s+=\s22/) } + it { should contain_concat__fragment('job1_rsync').with(:content => /server\.mode\s+=\s:ssh/) } + end + end # rsync + context 'logging' do context 'no logging' do let(:params) { { diff --git a/templates/job/rsync.erb b/templates/job/rsync.erb new file mode 100644 index 0000000..85d7cd7 --- /dev/null +++ b/templates/job/rsync.erb @@ -0,0 +1,21 @@ + ## + # rsync [Storage] + # + store_with RSync do |server| +<%if @storage_username and (@rsync_mode == 'ssh') -%> + server.ssh_user = "<%= @storage_username -%>" +<% elsif @storage_username -%> + server.rsync_user = "<%= @storage_username -%>" +<% end -%> + server.host = "<%= @storage_host -%>" +<%if @rsync_port -%> + server.port = <%= @rsync_port %> +<% end -%> + server.path = "<%= @_path -%>" +<% if @rsync_compress -%> + server.compress = "<%= @rsync_compress %>" +<% end -%> +<% if @rsync_mode -%> + server.mode = :<%= @rsync_mode %> +<% end -%> + end From 8d1f4d0260078e007c49c9097b661933091a2c1c Mon Sep 17 00:00:00 2001 From: Brian Cheldelin Date: Tue, 20 Sep 2016 13:15:55 -0600 Subject: [PATCH 2/3] Travis ci fix (#1) * Fixing gem dependencies * Use latest puppet version for tests * Fixes test looking for host not ip --- .travis.yml | 18 ++---------------- Gemfile | 5 +++++ spec/defines/backup_job_spec.rb | 2 +- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 240935c..6146b06 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,14 +7,10 @@ sudo: false matrix: fast_finish: true include: - - rvm: 1.8.7 - env: PUPPET_GEM_VERSION="~> 3.3.0" - - rvm: 1.8.7 - env: PUPPET_GEM_VERSION="~> 3.0" FUTURE_PARSER="yes" - rvm: 1.9.3 - env: PUPPET_GEM_VERSION="~> 3.3.0" + env: PUPPET_GEM_VERSION="~> 3.8.0" JSON_PURE_GEM_VERSION="~> 1.8.3" - rvm: 1.9.3 - env: PUPPET_GEM_VERSION="~> 3.0" FUTURE_PARSER="yes" + env: PUPPET_GEM_VERSION="~> 3.0" JSON_PURE_GEM_VERSION="~> 1.8.3" FUTURE_PARSER="yes" - rvm: 2.0.0 env: PUPPET_GEM_VERSION="~> 3.3.0" - rvm: 2.0.0 @@ -25,13 +21,3 @@ matrix: env: PUPPET_GEM_VERSION="~> 4.1" - rvm: 2.1.0 env: PUPPET_GEM_VERSION="~> 4.2" -notifications: - email: false - hipchat: - rooms: - - secure: CK11caPkzorkB4onwQPDolAFM32RO+Mn/rt3MRBe0xkt+SyRj8Icmv4MLbWUhiIydZQLrZpC+k993U9RuNOgghOcBq9tKF3U1sM6rrWLNJOnvobi6S+WZYe325IcsMchREERd0TUJYeYWCw2YxYlJBS/xRmpXO5gLZAPzm1NI9k= - template: - - '%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message} (Details/Change view)' - format: html - on_success: always - on_failure: always diff --git a/Gemfile b/Gemfile index 007789b..3426dae 100644 --- a/Gemfile +++ b/Gemfile @@ -9,6 +9,11 @@ group :unit_tests do gem 'puppet-syntax', :require => false gem 'metadata-json-lint', :require => false gem 'json', :require => false + if jasonpureversion = ENV['JSON_PURE_GEM_VERSION'] + gem 'json_pure', jasonpureversion, :require => false + else + gem 'json_pure', :require => false + end gem 'puppet-lint-absolute_classname-check', :require => false gem 'puppet-lint-appends-check', :require => false gem 'puppet-lint-empty_string-check', :require => false diff --git a/spec/defines/backup_job_spec.rb b/spec/defines/backup_job_spec.rb index d90c409..b8e1721 100644 --- a/spec/defines/backup_job_spec.rb +++ b/spec/defines/backup_job_spec.rb @@ -1086,7 +1086,7 @@ :path => '/there', } } it { should_not contain_concat__fragment('job1_rsync').with(:content => /server\..+user/) } - it { should contain_concat__fragment('job1_rsync').with(:content => /server\.ip\s+=\s"mysite.example.com"/) } + it { should contain_concat__fragment('job1_rsync').with(:content => /server\.host\s+=\s"mysite.example.com"/) } it { should contain_concat__fragment('job1_rsync').with(:content => /server\.port\s+=\s22$/) } it { should contain_concat__fragment('job1_rsync').with(:content => /server\.path\s+=\s"\/there"/) } it { should_not contain_concat__fragment('job1_rsync').with(:content => /server\.rsync_compress/) } From 25245a05bb714d6fb8c5ecdc40342bf8fec4d49f Mon Sep 17 00:00:00 2001 From: bcheldelin Date: Tue, 25 Oct 2016 09:26:18 -0600 Subject: [PATCH 3/3] Resolves some Rsync archive storage type issues and adds features. * Removes port defaults and lets gem use defaults. * Adds ability to set rsync_password_file. * Adds ability to specify rsync mode. --- manifests/init.pp | 3 +++ manifests/job.pp | 6 ++++- manifests/params.pp | 3 ++- spec/defines/backup_job_spec.rb | 45 ++++++++++++++++++++++++++++++++- templates/job/rsync.erb | 3 +++ 5 files changed, 57 insertions(+), 3 deletions(-) diff --git a/manifests/init.pp b/manifests/init.pp index 239f96f..6d90888 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -37,6 +37,9 @@ # FTP $ftp_port = $::backup::params::ftp_port, $ftp_passive_mode = $::backup::params::ftp_passive_mode, + # Rsync + $rsync_mode = $::backup::params::rsync_mode, + $rsync_password_file = $::backup::params::rsync_password_file, ## Encryptors $encryptor = $::backup::params::encryptor, diff --git a/manifests/job.pp b/manifests/job.pp index e45575d..a7449f0 100644 --- a/manifests/job.pp +++ b/manifests/job.pp @@ -62,6 +62,7 @@ $rsync_mode = $::backup::rsync_mode, $rsync_port = $::backup::rsync_port, $rsync_compress = $::backup::rsync_compress, + $rsync_password_file = $::backup::rsync_password_file, ## Encryptors $encryptor = $::backup::encryptor, @@ -258,13 +259,16 @@ if !$storage_host or !is_string($storage_host) { fail("[Backup::Job::${name}]: Parameter storage_host is required for rsync storage") } - if !$rsync_port or !is_integer($rsync_port) { + if $rsync_port and !is_integer($rsync_port) { fail("[Backup::Job::${name}]: rsync_port must be an integer. (Got: ${rsync_port})") } if $rsync_compress { validate_bool($rsync_compress) } + if $rsync_password_file { + validate_string($rsync_password_file) + } } diff --git a/manifests/params.pp b/manifests/params.pp index ff0ca0b..28dfeb9 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -55,8 +55,9 @@ $ftp_passive_mode = false # rsync $rsync_mode = undef - $rsync_port = 22 + $rsync_port = undef $rsync_compress = undef + $rsync_password_file = undef ## Encryptors $encryptor = undef diff --git a/spec/defines/backup_job_spec.rb b/spec/defines/backup_job_spec.rb index b8e1721..0806d22 100644 --- a/spec/defines/backup_job_spec.rb +++ b/spec/defines/backup_job_spec.rb @@ -406,6 +406,35 @@ } } it { expect { is_expected.to compile }.to raise_error(/"bob" is not a boolean/)} end + + context 'bad rsync_compress' do + let(:params) { { + :types => 'archive', + :add => 'here', + :storage_type => 'rsync', + :storage_username => 'myuser', + :storage_host => 'mysite.example.com', + :path => '/there', + :rsync_compress => 'bob', + } } + it { expect { is_expected.to compile }.to raise_error(/"bob" is not a boolean/)} + end + + context 'rsync_password_file is not a string' do + let(:params) { { + :types => 'archive', + :add => 'here', + :storage_type => 'rsync', + :storage_username => 'myuser', + :storage_host => 'mysite.example.com', + :path => '/there', + :rsync_compress => true, + :rsync_mode => 'rsync_daemon', + :rsync_password_file => true + } } + it { expect { is_expected.to compile }.to raise_error(/true is not a string/)} + end + end # rsync context 'encryptor generic' do @@ -1087,7 +1116,6 @@ } } it { should_not contain_concat__fragment('job1_rsync').with(:content => /server\..+user/) } it { should contain_concat__fragment('job1_rsync').with(:content => /server\.host\s+=\s"mysite.example.com"/) } - it { should contain_concat__fragment('job1_rsync').with(:content => /server\.port\s+=\s22$/) } it { should contain_concat__fragment('job1_rsync').with(:content => /server\.path\s+=\s"\/there"/) } it { should_not contain_concat__fragment('job1_rsync').with(:content => /server\.rsync_compress/) } end @@ -1108,6 +1136,21 @@ it { should contain_concat__fragment('job1_rsync').with(:content => /server\.port\s+=\s22/) } it { should contain_concat__fragment('job1_rsync').with(:content => /server\.mode\s+=\s:ssh/) } end + + context 'rsync_daemon mode' do + let(:params) { { + :types => 'archive', + :add => '/here', + :storage_type => 'rsync', + :storage_host => 'mysite.example.com', + :path => 'there', + :rsync_port => 873, + :rsync_mode => 'rsync_daemon', + :keep => 10, + } } + it { should contain_concat__fragment('job1_rsync').with(:content => /server\.port\s+=\s873/) } + it { should contain_concat__fragment('job1_rsync').with(:content => /server\.mode\s+=\s:rsync_daemon/) } + end end # rsync context 'logging' do diff --git a/templates/job/rsync.erb b/templates/job/rsync.erb index 85d7cd7..1a8a099 100644 --- a/templates/job/rsync.erb +++ b/templates/job/rsync.erb @@ -17,5 +17,8 @@ <% end -%> <% if @rsync_mode -%> server.mode = :<%= @rsync_mode %> +<% end -%> +<% if @rsync_password_file -%> + server.rsync_password_file = "<%= @rsync_password_file %>" <% end -%> end