diff -pruN 3.7.0-2.1/azure-pipelines.yml 3.15.4-1/azure-pipelines.yml --- 3.7.0-2.1/azure-pipelines.yml 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/azure-pipelines.yml 1970-01-01 00:00:00.000000000 +0000 @@ -1,87 +0,0 @@ -# Don't run Azure when a branch is updated, only when a PR is updated. -# Prevents double builds when a PR is made from the main repo and not a fork. -trigger: none -pr: - autoCancel: true - branches: - include: - - '*' - -pool: - # self-hosted agent on Windows 10 1903 environment - # includes newer Docker engine with LCOW enabled, new build of LCOW image - # includes Ruby 2.5, Go 1.10, Node.js 10.10 - name: Default - -variables: - NAMESPACE: puppet - CONTAINER_NAME: r10k - CONTAINER_BUILD_PATH: . - LINT_IGNORES: - DOCKER_BUILDKIT: 1 - BUILD_OPTIONS: --build-arg alpine_version=3.9 - -workspace: - clean: resources - -steps: -- checkout: self # self represents the repo where the initial Pipelines YAML file was found - clean: true # whether to fetch clean each time - -- powershell: | - $gemfile = Join-Path -Path (Get-Location) -ChildPath 'docker/Gemfile' - $gempath = Join-Path -Path (Get-Location) -ChildPath 'docker/.bundle/gems' - bundle config --local gemfile $gemfile - bundle config --local path $gempath - bundle install --with test - displayName: Fetch Dependencies - timeoutInMinutes: 1 - name: fetch_deps - -- powershell: | - . "$(bundle show pupperware)/ci/build.ps1" - Write-HostDiagnostics - displayName: Diagnostic Host Information - timeoutInMinutes: 2 - name: hostinfo - -- powershell: | - . "$(bundle show pupperware)/ci/build.ps1" - Lint-Dockerfile -Name $ENV:CONTAINER_NAME -Ignore ($ENV:LINT_IGNORES -split ' ') - displayName: Lint $(CONTAINER_NAME) Dockerfile - timeoutInMinutes: 1 - name: lint_dockerfile - -- powershell: | - . "$(bundle show pupperware)/ci/build.ps1" - Build-Container -Name $ENV:CONTAINER_NAME -Namespace $ENV:NAMESPACE -PathOrUri $ENV:CONTAINER_BUILD_PATH -AdditionalOptions ($ENV:BUILD_OPTIONS -split ' ') - displayName: Build $(CONTAINER_NAME) Container - timeoutInMinutes: 5 - name: build_container - -- powershell: | - . "$(bundle show pupperware)/ci/build.ps1" - Initialize-TestEnv - displayName: Prepare Test Environment - name: test_prepare - -- powershell: | - . "$(bundle show pupperware)/ci/build.ps1" - Invoke-ContainerTest -Name $ENV:CONTAINER_NAME -Namespace $ENV:NAMESPACE - displayName: Test $(CONTAINER_NAME) - timeoutInMinutes: 5 - name: test_container - -- task: PublishTestResults@2 - displayName: Publish $(CONTAINER_NAME) test results - inputs: - testResultsFormat: 'JUnit' - testResultsFiles: 'docker/TEST-*.xml' - testRunTitle: $(CONTAINER_NAME) Test Results - -- powershell: | - . "$(bundle show pupperware)/ci/build.ps1" - Clear-BuildState -Name $ENV:CONTAINER_NAME -Namespace $ENV:NAMESPACE - displayName: Container Cleanup - timeoutInMinutes: 4 - condition: always() diff -pruN 3.7.0-2.1/CHANGELOG.mkd 3.15.4-1/CHANGELOG.mkd --- 3.7.0-2.1/CHANGELOG.mkd 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/CHANGELOG.mkd 2023-01-19 00:49:17.000000000 +0000 @@ -4,6 +4,130 @@ CHANGELOG Unreleased ---------- +3.15.4 +------ +- Pin dependencies to maintain support for old Ruby versions [#1329](https://github.com/puppetlabs/r10k/pull/1329) + +3.15.3 +------ +- Fix dirty working copy debug logging [#1321](https://github.com/puppetlabs/r10k/pull/1321) +- Allow gettext-setup < 2 for compatibility with Ruby 3.2 and Puppet 8 [#1325](https://github.com/puppetlabs/r10k/pull/1325) + +3.15.2 +------ +- Implement exclude regex for puppetfile install [#1248](https://github.com/puppetlabs/r10k/issues/1248) + +3.15.1 +------ +- Add TOC to configuration docs [#1298](https://github.com/puppetlabs/r10k/issues/1298) +- Remove the spec folder from gemspec [#1316](https://github.com/puppetlabs/r10k/issues/1316) + +3.15.0 +------ + +- Support and test Ruby 3 +- Allow puppet_forge 3.x & newer versions of fast_gettext/gettext [#1302](https://github.com/puppetlabs/r10k/pull/1302) +- Allow newer cri versions [#1302](https://github.com/puppetlabs/r10k/pull/1302) +- Fix error when using install_path from environment module [#1288](https://github.com/puppetlabs/r10k/issues/1288) +- (RK-399) Do not warn about local modifications in the spec directory when `exclude_spec` is set [#1291](https://github.com/puppetlabs/r10k/pull/1291) + +3.14.2 +------ + +- (RK-397) Ensure `--incremental` does not skip undeployed modules [#1278](https://github.com/puppetlabs/r10k/pull/1278) + +3.14.1 +------ + +- (RK-395) Make `exclude_spec` from a Puppetfile the priority override [#1271](https://github.com/puppetlabs/r10k/issues/1271) +- (RK-394) Fix `force` always resolving to true for `puppetfile install` [#1269](https://github.com/puppetlabs/r10k/issues/1265) +- (RK-393) Bug fix: not all spec directories are deleted when :exclude_spec is true [#1267](https://github.com/puppetlabs/r10k/pull/1267) +- Refactor internal module creation to always expect a hash, even for Forge modules, which can be specified in the Puppetfile with just a version string. [#1170](https://github.com/puppetlabs/r10k/pull/1170) + +3.14.0 +------ + +- Record unprocessed environment name, so that `strip_component` does not cause truncated environment names to be used as git branches, resulting in errors or incorrect deploys. [#1240](https://github.com/puppetlabs/r10k/pull/1240) +- (CODEMGMT-1294) Resync repos with unresolvable refs [#1239](https://github.com/puppetlabs/r10k/pull/1239) +- (RK-378) Restore access to the environment name from the Puppetfile [#1241](https://github.com/puppetlabs/r10k/pull/1241) +- (CODEMGMT-1300) Ensure the remote url in rugged cache directories is current [#1245](https://github.com/puppetlabs/r10k/pull/1245) +- Add support for tarball module type, allowing module content to be packaged and sourced from generic fileservers [#1244](https://github.com/puppetlabs/r10k/pull/1244) +- Add experimental support for tarball environment type, allowing whole environments to be packaged and sourced from generic fileservers [#1244](https://github.com/puppetlabs/r10k/pull/1244) + +3.13.0 +------ + +- Restore Ruby 3 compatibility [#1234](https://github.com/puppetlabs/r10k/pull/1234) +- (RK-381) Do not recurse into symlinked dirs when finding files to purge. [#1233](https://github.com/puppetlabs/r10k/pull/1233) +- Purge should remove unmanaged directories, in addition to unmanaged files. [#1222](https://github.com/puppetlabs/r10k/pull/1222) +- Rename experimental environment type "bare" to "plain". [#1228](https://github.com/puppetlabs/r10k/pull/1228) +- Add support for specifying additional logging ouputs. [#1230](https://github.com/puppetlabs/r10k/issues/1230) + +3.12.1 +------ + +- Fix requiring individual R10K::Actions without having already required 'r10k'. [#1223](https://github.com/puppetlabs/r10k/issues/1223) +- Fix evaluation of Puppetfiles that include local modules. [#1224](https://github.com/puppetlabs/r10k/pull/1224) + +3.12.0 +------ + +- (RK-308) Provide a `forge.allow_puppetfile_override` setting that, when true, causes a `forge` declaration in the Puppetfile to override `forge.baseurl`. [#1214](https://github.com/puppetlabs/r10k/pull/1214) +- (CODEMGMT-1415) Provide an `--incremental` flag to only sync those modules in a Puppetfile whose definitions have changed since last sync, or those whose versions could change. [#1200](https://github.com/puppetlabs/r10k/pull/1200) +- (CODEMGMT-1454) Ensure missing repo caches are re-synced [#1210](https://github.com/puppetlabs/r10k/pull/1210) +- (PF-2437) Allow token authentication to be used with the Forge. [#1192](https://github.com/puppetlabs/r10k/pull/1192) +- Only run the module postrun command for environments in which the module was modified. [#1215](https://github.com/puppetlabs/r10k/issues/1215) + +3.11.0 +------ + +- Always sync git cache on `ref: 'HEAD'` [#1182](https://github.com/puppetlabs/r10k/pull/1182) +- (CODEMGMT-1421, CODEMGMT-1422, CODEMGMT-1457) Add setting `exclude_spec` to remove the spec dir from module deployment[#1189](https://github.com/puppetlabs/r10k/pull/1189)[#1198](https://github.com/puppetlabs/r10k/pull/1198)[#1204](https://github.com/puppetlabs/r10k/pull/1204) +- (RK-369) Make module deploys run the postrun command if any environments were updated. [#982](https://github.com/puppetlabs/r10k/issues/982) +- Add support for Github App auth token. This allows r10k to authenticate under strict SSO/2FA guidelines that cannot utilize machine users for code deployment. [#1180](https://github.com/puppetlabs/r10k/pull/1180) +- Restore the ability to load a Puppetfile from a relative `basedir`. [#1202](https://github.com/puppetlabs/r10k/pull/1202), [#1203](https://github.com/puppetlabs/r10k/pull/1203) + +3.10.0 +------ + +- Add `authorization_token` setting to allow authentication to a custom Forge server. [#1181](https://github.com/puppetlabs/r10k/pull/1181) +- (RK-135) Attempting to download the latest version for a module that has no Forge releases will now issue a meaningful error. [#1177](https://github.com/puppetlabs/r10k/pull/1177) +- Added an interface to R10K::Source::Base named `reload!` for updating the environments list for a given deployment; `reload!` is called before deployment purges to make r10k deploy pools more threadsafe. [#1172](https://github.com/puppetlabs/r10k/pull/1172) +- Remove username and password from remote url in cache directory name [#1186](https://github.com/puppetlabs/r10k/pull/1186) +- Purging efficiency is greatly improved. R10K will no longer recurse into directories that match recursive purge exclusions. This should significantly improve the deploy times for those users who enable the "environment" purge level. [#1178](https://github.com/puppetlabs/r10k/pull/1178) + +3.9.3 +----- + +- Fixes a regression when using `--default_branch_override` with Puppetfiles containing Forge modules. [#1173](https://github.com/puppetlabs/r10k/issues/1173) + +3.9.2 +----- + +- Makes the third parameter to R10K::Actions optional, restoring backwards compatability broken in 3.9.1. + +3.9.1 +----- + +- Invalid module specifications in a Puppetfile will cause the R10K run to abort earlier than before. Prior to this release, the R10K run would complete, sync all other modules, and return an exit code of 1. R10K will now stop syncing modules and abort immediately. [#1161](https://github.com/puppetlabs/r10k/pull/1161) + +3.9.0 +----- + +- Add '--modules' flag to `deploy` subcommand as a replacement to '--puppetfile', deprecate '--puppetfile'. [#1147](https://github.com/puppetlabs/r10k/pull/1147) +- Deprecate 'purge_whitelist' and favor usage of 'purge_allowlist'. [#1144](https://github.com/puppetlabs/r10k/pull/1144) +- Add 'strip\_component' environment source configuration setting, to allow deploying Git branches named like "env/production" as Puppet environments named like "production". [#1128](https://github.com/puppetlabs/r10k/pull/1128) +- A warning will be emitted when the user supplies conflicting arguments to module definitions in a Puppetfile, such as when specifying both :commit and :branch [#1130](https://github.com/puppetlabs/r10k/pull/1130) +- Add optional standard module and environment specification interface: name, type, source, version. These options can be used when specifying environments and/or modules in a yaml/exec source, as well as when specifying modules in a Puppetfile. Providing the standard interface simplifies integrations with external services [#1131](https://github.com/puppetlabs/r10k/pull/1131) +- Pin cri to 2.15.10 to maintain support for Ruby 2.3 and 2.4 [#1121](https://github.com/puppetlabs/r10k/issues/1121) + +3.8.0 +----- + +- When a forge module fails name validation the offending name will now be printed in the error message. [#1126](https://github.com/puppetlabs/r10k/pull/1126) +- Module ref resolution will now fall back to the normal default branch if the default branch override cannot be resolved. [#1122](https://github.com/puppetlabs/r10k/pull/1122) +- Experimental feature change: conflicts between environment-defined modules and Puppetfile-defined modules now default to logging a warning and deploying the environment module version, overriding the Puppetfile. Previously, conflicts would result in an error. The behavior is now configurable via the `module_conflicts` environment setting [#1107](https://github.com/puppetlabs/r10k/pull/1107) + 3.7.0 ----- diff -pruN 3.7.0-2.1/CODEOWNERS 3.15.4-1/CODEOWNERS --- 3.7.0-2.1/CODEOWNERS 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/CODEOWNERS 2023-01-19 00:49:17.000000000 +0000 @@ -1,2 +1,2 @@ -* @puppetlabs/puppetserver-maintainers @adrienthebo @dhollinger -/docker/ @puppetlabs/pupperware +# This repo is owned by the dumpling team +* @puppetlabs/dumpling diff -pruN 3.7.0-2.1/debian/changelog 3.15.4-1/debian/changelog --- 3.7.0-2.1/debian/changelog 2021-11-26 23:24:35.000000000 +0000 +++ 3.15.4-1/debian/changelog 2023-02-09 15:44:35.000000000 +0000 @@ -1,3 +1,58 @@ +r10k (3.15.4-1) unstable; urgency=medium + + [ Debian Janitor ] + * Remove constraints unnecessary since buster (oldstable) + * Remove obsolete field Name from debian/upstream/metadata + (already present in machine-readable debian/copyright). + Changes-By: lintian-brush + + [ Sebastien Badia ] + * New upstream version 3.15.4 + * Bump Standards-Version to 4.6.2 (no changes needed) + + -- Sebastien Badia Thu, 09 Feb 2023 16:44:35 +0100 + +r10k (3.15.2-2) unstable; urgency=medium + + * debian/control: + - Drop obsolete field XB-Ruby-Versions. + + -- Georg Faerber Mon, 24 Oct 2022 10:10:22 +0000 + +r10k (3.15.2-1) unstable; urgency=medium + + * New upstream version 3.15.2. + * debian/control: + - Bump Standards-Version to 4.6.1, no changes required. + * debian/patches: + - Refresh patch to relax versions of dependencies in gemspec. + + -- Georg Faerber Sun, 23 Oct 2022 16:14:16 +0000 + +r10k (3.14.2-2) unstable; urgency=medium + + * debian/patches: + - Relax dependencies in gemspec: cri, puppet_forge, fast_gettext, jwt. + - Skip another test which requires Internet access. + + -- Georg Faerber Fri, 15 Apr 2022 10:24:26 +0000 + +r10k (3.14.2-1) unstable; urgency=medium + + * New upstream version 3.14.2. + * debian/control: + - (Build-)Depend on ruby-jwt. + - Drop depending on ruby-interpreter, obsolete. + - Bump Standards-Version to 4.6.0, no changes required. + * debian/patches: + - Drop patch to ensure Ruby 3.0 compatibility, integrated upstream. + - Refresh remaining patches. + * debian/watch: + - Rely on 'special strings' provided by uscan to handle higher version + numbers. + + -- Georg Faerber Thu, 14 Apr 2022 19:00:38 +0000 + r10k (3.7.0-2.1) unstable; urgency=medium * Non-maintainer upload. diff -pruN 3.7.0-2.1/debian/control 3.15.4-1/debian/control --- 3.7.0-2.1/debian/control 2021-11-26 23:24:35.000000000 +0000 +++ 3.15.4-1/debian/control 2023-02-09 15:43:08.000000000 +0000 @@ -6,21 +6,22 @@ Uploaders: Sebastien Badia , Georg Faerber , Build-Depends: debhelper-compat (= 13), - gem2deb (>= 0.6.1~), + gem2deb, git, rake, ronn | ruby-ronn (<< 0.7.3-5.1~), ruby-colored2, - ruby-cri (>= 2.6.1), - ruby-gettext-setup (>= 0.5), - ruby-log4r (>= 1.1.10), - ruby-minitar (>= 0.6.1), - ruby-multi-json (>= 1.10), + ruby-cri, + ruby-gettext-setup, + ruby-jwt (>= 2.2.3~), + ruby-log4r, + ruby-minitar, + ruby-multi-json, ruby-puppet-forge (>= 3.0.0~), - ruby-rspec (>= 3.1), - ruby-rugged (>= 0.24.0), - yard (>= 0.8.7.3), -Standards-Version: 4.5.1 + ruby-rspec, + ruby-rugged, + yard, +Standards-Version: 4.6.2 Vcs-Git: https://salsa.debian.org/puppet-team/r10k.git Vcs-Browser: https://salsa.debian.org/puppet-team/r10k Homepage: https://github.com/puppetlabs/r10k @@ -30,16 +31,16 @@ Rules-Requires-Root: no Package: r10k Architecture: all -XB-Ruby-Versions: ${ruby:Versions} -Depends: ruby | ruby-interpreter, +Depends: ruby, ruby-colored2, - ruby-cri (>= 2.6.1), - ruby-gettext-setup (>= 0.5), - ruby-log4r (>= 1.1.10), - ruby-minitar (>= 0.6.1), - ruby-multi-json (>= 1.10), + ruby-cri, + ruby-gettext-setup, + ruby-jwt (>= 2.2.3~), + ruby-log4r, + ruby-minitar, + ruby-multi-json, ruby-puppet-forge (>= 3.0.0~), - ruby-rugged (>= 0.24.0), + ruby-rugged, ${misc:Depends}, Recommends: git, Description: Puppet environment and module deployment diff -pruN 3.7.0-2.1/debian/copyright 3.15.4-1/debian/copyright --- 3.7.0-2.1/debian/copyright 2021-11-26 23:24:35.000000000 +0000 +++ 3.15.4-1/debian/copyright 2023-02-09 15:44:35.000000000 +0000 @@ -7,7 +7,7 @@ Copyright: 2013, 2015 Adrien Thebo +Copyright: 2014-2023 Sebastien Badia License: Apache-2.0 Comment: the Debian packaging is licensed under the same terms as the original package. diff -pruN 3.7.0-2.1/debian/patches/10-gemspec-relax-deps.patch 3.15.4-1/debian/patches/10-gemspec-relax-deps.patch --- 3.7.0-2.1/debian/patches/10-gemspec-relax-deps.patch 2021-11-26 23:24:35.000000000 +0000 +++ 3.15.4-1/debian/patches/10-gemspec-relax-deps.patch 2023-02-09 15:31:45.000000000 +0000 @@ -1,25 +1,19 @@ -Description: gemspec: relax dep on fast_gettext +Description: gemspec: relax deps Author: Georg Faerber Forwarded: not-needed -Last-Update: 2020-03-10 +Last-Update: 2022-10-23 --- This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ ---- a/r10k.gemspec -+++ b/r10k.gemspec -@@ -28,13 +28,13 @@ Gem::Specification.new do |s| - s.add_dependency 'log4r', '1.1.10' - s.add_dependency 'multi_json', '~> 1.10' +Index: r10k/r10k.gemspec +=================================================================== +--- r10k.orig/r10k.gemspec 2022-10-23 14:27:07.299637598 +0000 ++++ r10k/r10k.gemspec 2022-10-23 16:13:52.415575723 +0000 +@@ -34,7 +34,7 @@ + s.add_dependency 'fast_gettext', ['>= 1.1.0', '< 3.0.0'] + s.add_dependency 'gettext', ['>= 3.0.2', '< 4.0.0'] -- s.add_dependency 'puppet_forge', '~> 2.3.0' -+ s.add_dependency 'puppet_forge', '>= 3.0.0' - - s.add_dependency 'gettext-setup', '~>0.24' - # These two pins narrow what is allowed by gettext-setup, - # to preserver compatability with Ruby 2.4 -- s.add_dependency 'fast_gettext', '~> 1.1.0' -- s.add_dependency 'gettext', ['>= 3.0.2', '< 3.3.0'] -+ s.add_dependency 'fast_gettext', '>= 1.1.0' -+ s.add_dependency 'gettext', ['>= 3.0.2'] +- s.add_dependency 'jwt', '~> 2.2.3' ++ s.add_dependency 'jwt', '>= 2.2.3' + s.add_dependency 'minitar', '~> 0.9' s.add_development_dependency 'rspec', '~> 3.1' - diff -pruN 3.7.0-2.1/debian/patches/12_disable_test_with_network_access 3.15.4-1/debian/patches/12_disable_test_with_network_access --- 3.7.0-2.1/debian/patches/12_disable_test_with_network_access 2021-11-26 23:24:35.000000000 +0000 +++ 3.15.4-1/debian/patches/12_disable_test_with_network_access 2022-06-12 23:22:58.000000000 +0000 @@ -3,25 +3,27 @@ Description: Disable tests that require in order to fetch module version. Author: Sebastien Badia Forwarded: not-needed -Last-Update: 2020-02-07 +Last-Update: 2022-04-15 ---- a/spec/unit/module/forge_spec.rb -+++ b/spec/unit/module/forge_spec.rb -@@ -63,7 +63,7 @@ describe R10K::Module::Forge do +Index: r10k/spec/unit/module/forge_spec.rb +=================================================================== +--- r10k.orig/spec/unit/module/forge_spec.rb 2022-04-15 09:52:53.191051602 +0000 ++++ r10k/spec/unit/module/forge_spec.rb 2022-04-15 09:53:54.916301787 +0000 +@@ -93,7 +93,7 @@ context "when a module is deprecated" do - subject { described_class.new('puppetlabs/corosync', fixture_modulepath, :latest) } + subject { described_class.new('puppetlabs/corosync', fixture_modulepath, { version: :latest }) } - it "warns on sync if module is not already insync" do + xit "warns on sync if module is not already insync" do allow(subject).to receive(:status).and_return(:absent) allow(R10K::Forge::ModuleRelease).to receive(:new).and_return(double('mod_release', install: true)) -@@ -179,7 +179,7 @@ describe R10K::Module::Forge do +@@ -243,7 +243,7 @@ end describe '#install' do - it 'installs the module from the forge' do + xit 'installs the module from the forge' do - subject = described_class.new('branan/eight_hundred', fixture_modulepath, '8.0.0') + subject = described_class.new('branan/eight_hundred', fixture_modulepath, { version: '8.0.0' }) release = instance_double('R10K::Forge::ModuleRelease') expect(R10K::Forge::ModuleRelease).to receive(:new).with('branan-eight_hundred', '8.0.0').and_return(release) diff -pruN 3.7.0-2.1/debian/patches/13_fix_ruby_30_kwargs.patch 3.15.4-1/debian/patches/13_fix_ruby_30_kwargs.patch --- 3.7.0-2.1/debian/patches/13_fix_ruby_30_kwargs.patch 2021-11-26 23:24:35.000000000 +0000 +++ 3.15.4-1/debian/patches/13_fix_ruby_30_kwargs.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,36 +0,0 @@ -From: Daniel Leidert -Date: Sat, 27 Nov 2021 00:20:48 +0100 -Subject: Fix Ruby 3.0 keyword arguments - -Bug-Debian: https://bugs.debian.org/996116 ---- - lib/r10k/git/rugged/bare_repository.rb | 2 +- - lib/r10k/git/rugged/working_repository.rb | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) - -diff --git a/lib/r10k/git/rugged/bare_repository.rb b/lib/r10k/git/rugged/bare_repository.rb -index 9dd25cc..b712be1 100644 ---- a/lib/r10k/git/rugged/bare_repository.rb -+++ b/lib/r10k/git/rugged/bare_repository.rb -@@ -64,7 +64,7 @@ class R10K::Git::Rugged::BareRepository < R10K::Git::Rugged::BaseRepository - results = nil - - R10K::Git.with_proxy(proxy) do -- results = with_repo { |repo| repo.fetch(remote_name, refspecs, options) } -+ results = with_repo { |repo| repo.fetch(remote_name, refspecs, **options) } - end - - report_transfer(results, remote_name) -diff --git a/lib/r10k/git/rugged/working_repository.rb b/lib/r10k/git/rugged/working_repository.rb -index f78391f..c5a26fe 100644 ---- a/lib/r10k/git/rugged/working_repository.rb -+++ b/lib/r10k/git/rugged/working_repository.rb -@@ -93,7 +93,7 @@ class R10K::Git::Rugged::WorkingRepository < R10K::Git::Rugged::BaseRepository - results = nil - - R10K::Git.with_proxy(proxy) do -- results = with_repo { |repo| repo.fetch(remote_name, refspecs, options) } -+ results = with_repo { |repo| repo.fetch(remote_name, refspecs, **options) } - end - - report_transfer(results, remote) diff -pruN 3.7.0-2.1/debian/patches/series 3.15.4-1/debian/patches/series --- 3.7.0-2.1/debian/patches/series 2021-11-26 23:24:35.000000000 +0000 +++ 3.15.4-1/debian/patches/series 2021-02-03 18:47:39.000000000 +0000 @@ -1,4 +1,3 @@ 10-gemspec-relax-deps.patch 11_locales_path 12_disable_test_with_network_access -13_fix_ruby_30_kwargs.patch diff -pruN 3.7.0-2.1/debian/upstream/metadata 3.15.4-1/debian/upstream/metadata --- 3.7.0-2.1/debian/upstream/metadata 2021-11-26 23:24:35.000000000 +0000 +++ 3.15.4-1/debian/upstream/metadata 2023-02-09 15:31:45.000000000 +0000 @@ -3,6 +3,5 @@ Archive: GitHub Bug-Database: https://github.com/puppetlabs/r10k/issues Bug-Submit: https://github.com/puppetlabs/r10k/issues Changelog: https://github.com/puppetlabs/r10k/tags -Name: r10k Repository: https://github.com/puppetlabs/r10k.git Repository-Browse: https://github.com/puppetlabs/r10k diff -pruN 3.7.0-2.1/debian/watch 3.15.4-1/debian/watch --- 3.7.0-2.1/debian/watch 2021-11-26 23:24:35.000000000 +0000 +++ 3.15.4-1/debian/watch 2022-06-12 23:22:58.000000000 +0000 @@ -1,2 +1,2 @@ version=4 -https://github.com/puppetlabs/r10k/tags .*/(\d\.\d\.\d)\.tar\.gz +https://github.com/puppetlabs/r10k/tags .*/@ANY_VERSION@@ARCHIVE_EXT@ diff -pruN 3.7.0-2.1/doc/common-patterns.mkd 3.15.4-1/doc/common-patterns.mkd --- 3.7.0-2.1/doc/common-patterns.mkd 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/doc/common-patterns.mkd 2023-01-19 00:49:17.000000000 +0000 @@ -42,3 +42,4 @@ publicly available hook. These include: * [Reaktor](https://github.com/pzim/reaktor) * [zack/r10k's Webhooks](https://forge.puppetlabs.com/zack/r10k#webhook-support) (Puppet Enterprise only) +* [Simple Puppet Provisioner](https://github.com/mbaynton/SimplePuppetProvisioner) diff -pruN 3.7.0-2.1/doc/dynamic-environments/configuration.mkd 3.15.4-1/doc/dynamic-environments/configuration.mkd --- 3.7.0-2.1/doc/dynamic-environments/configuration.mkd 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/doc/dynamic-environments/configuration.mkd 2023-01-19 00:49:17.000000000 +0000 @@ -1,6 +1,70 @@ Dynamic Environment Configuration ================================= + + +* [Config file location](#config-file-location) + * [Manual configuration](#manual-configuration) + * [Automatic configuration](#automatic-configuration) +* [General options](#general-options) + * [cachedir](#cachedir) + * [proxy](#proxy) + * [pool_size](#pool_size) + * [git](#git) + * [provider](#provider) + * [proxy](#proxy-1) + * [username](#username) + * [private_key](#private_key) + * [oauth_token](#oauth_token) + * [repositories](#repositories) + * [private_key](#private_key-1) + * [oauth_token](#oauth_token-1) + * [proxy](#proxy-2) + * [forge](#forge) + * [proxy](#proxy-3) + * [baseurl](#baseurl) + * [authorization_token](#authorization_token) + * [allow_puppetfile_override](#allow_puppetfile_override) +* [Deployment options](#deployment-options) + * [postrun](#postrun) + * [sources](#sources) + * [deploy](#deploy) + * [purge\_levels](#purge_levels) + * [deployment](#deployment) + * [environment](#environment) + * [puppetfile](#puppetfile) + * [purge\_allowlist](#purge_allowlist) + * [write\_lock](#write_lock) + * [generate\_types](#generate_types) + * [puppet\_path](#puppet_path) + * [puppet\_conf](#puppet_conf) + * [exclude_spec](#exclude_spec) +* [Source options](#source-options) + * [remote](#remote) + * [basedir](#basedir) + * [prefix](#prefix) + * [prefix behaviour](#prefix-behaviour) + * [strip\_component](#strip_component) + * [strip\_component behaviour](#strip_component-behaviour) + * [ignore_branch_prefixes](#ignore_branch_prefixes) + * [ignore_branch_prefixes behaviour](#ignore_branch_prefixes-behaviour) + * [filter_command](#filter_command) +* [Examples](#examples) + * [Minimal example](#minimal-example) + * [Separate hiera data](#separate-hiera-data) + * [Multiple tenancy](#multiple-tenancy) + * [Multiple tenancy with external hieradata](#multiple-tenancy-with-external-hieradata) +* [Experimental Features](#experimental-features) + * [YAML Environment Source](#yaml-environment-source) + * [YAMLdir Environment Source](#yamldir-environment-source) + * [Exec environment Source](#exec-environment-source) + * [Environment Modules](#environment-modules) + * [Puppetfile module conflicts](#puppetfile-module-conflicts) + * [Plain Environment Type](#plain-environment-type) + * [Tarball Environment Type](#tarball-environment-type) + + + R10k uses a configuration file to determine how dynamic environments should be deployed. @@ -128,6 +192,18 @@ git: private_key: "/etc/puppetlabs/r10k/ssh/id_rsa" ``` +#### oauth_token + +The oauth_token setting is only used by the Rugged git provider. + +The oauth_token option specifies the path to the default access token for Git HTTPS remotes. +Public git repositories can be accessed via HTTPS without authentication, but the oauth_token setting may be set if any non-public HTTPS remotes are used. + +```yaml +git: + oauth_token: "/etc/puppetlabs/r10k/token" +``` + #### repositories The repositories option allows configuration to be set on a per-remote basis. Each entry is a map of @@ -145,6 +221,18 @@ git: private_key: "/etc/puppetlabs/r10k/ssh/id_rsa-protected-repo-deploy-key" ``` +##### oauth_token + +A repository specific access token to use for HTTPS connections for the given repository URL. This +overrides the global oauth_token setting. + +```yaml +git: + repositories: + - remote: "https://tessier-ashpool.freeside/protected-repo.git" + oauth_token: "/etc/puppetlabs/r10k/protected-repo-deploy-token" +``` + ##### proxy The 'proxy' setting allows you to set or override the global proxy setting for a single, specific @@ -165,11 +253,22 @@ interactions. See the global proxy setti The 'baseurl' setting indicates where Forge modules should be installed from. This defaults to 'https://forgeapi.puppetlabs.com' +#### authorization_token + +The 'authorization_token' setting allows you to provide a token for authenticating to a Forge server. +You will need to prepend your token with 'Bearer ' to authenticate to the Forge or when using your own Artifactory server. + ```yaml forge: baseurl: 'https://private-forge.mysite' + authorization_token: 'Bearer mysupersecretauthtoken' ``` +#### allow_puppetfile_override + +The `allow_puppetfile_override` setting causes r10k to respect [`forge` declarations](https://github.com/puppetlabs/r10k/blob/main/doc/puppetfile.mkd#forge) +in Puppetfiles, overriding the `baseurl` setting and allowing per-environment configuration of the Forge URL. + Deployment options ------------------ @@ -191,7 +290,7 @@ postrun: ['/usr/bin/curl', '-F', 'deploy The postrun setting can only be set once. Occurrences of the string `$modifiedenvs` in the postrun command will be -replaced with the current environment(s) being deployed. +replaced with the current environment(s) being deployed, space separated. ### sources @@ -249,9 +348,9 @@ found which is neither committed to the environment, nor declared in a Puppetfile committed to that branch. Enabling this purge level will cause r10k to load and parse the Puppetfile for -the environment even without the `--puppetfile` flag being set. However, +the environment even without the `--modules` flag being set. However, Puppetfile content will still only be deployed if the environment is new or -the `--puppetfile` flag is set. Additionally, no environment-level content +the `--modules` flag is set. Additionally, no environment-level content will be purged if any errors are encountered while evaluating the Puppetfile or deploying its contents. @@ -266,31 +365,31 @@ managed by a Puppetfile include the conf "modules") as well as alternate directories specified as an `install_path` option to any Puppetfile content declarations. -#### purge\_whitelist +#### purge\_allowlist -The `purge_whitelist` setting exempts the specified filename patterns from +The `purge_allowlist` setting exempts the specified filename patterns from being purged. This setting is currently only considered during `environment` level purging. (See above.) Given value must be a list of shell style filename patterns in string format. See the Ruby [documentation for the `fnmatch` method](http://ruby-doc.org/core-2.2.0/File.html#method-c-fnmatch) for more details on valid patterns. Note that the `FNM_PATHNAME` and -`FNM_DOTMATCH` flags are in effect when r10k considers the whitelist. +`FNM_DOTMATCH` flags are in effect when r10k considers the allowlist. Patterns are relative to the root of the environment being purged and *do -not match recursively* by default. For example, a whitelist value of +not match recursively* by default. For example, a allowlist value of `*myfile*` would only preserve a matching file at the root of the environment. To preserve the file throughout the deployed environment, a recursive pattern such as `**/*myfile*` would be required. -Files matching a whitelist pattern may still be removed if they exist in +Files matching a allowlist pattern may still be removed if they exist in a folder that is otherwise subject to purging. In this case, an additional -whitelist rule to preserve the containing folder is required. +allowlist rule to preserve the containing folder is required. ```yaml --- deploy: - purge_whitelist: [ 'custom.json', '**/*.xpp' ] + purge_allowlist: [ 'custom.json', '**/*.xpp' ] ``` @@ -336,6 +435,21 @@ deploy: puppet_conf: '/opt/puppet/conf/puppet.conf' ``` +#### exclude_spec + +During module deployment, r10k's default behavior is to deploy the spec directory. Setting +`exclude_spec` to true will deploy modules without their spec directory. This behavior +can be configured for all modules using the `exclude_spec` setting in the r10k config. +It can also be passed as a CLI argument for `deploy environment/module`, overriding the +r10k config. Setting this per module in a `Puppetfile` will override the default, r10k config, +and cli flag for that module. The following example sets all modules to not deploy the spec +dir via the r10k config. + +```yaml +deploy: + exclude_spec: true +``` + Source options -------------- @@ -348,13 +462,16 @@ specific documentation for more informat The 'remote' setting specifies where the source repository should be fetched from. It may be any valid URL that the source may check out or clone. The remote must be able to be fetched without any interactive input, eg usernames or -passwords cannot be prompted for in order to fetch the remote. +passwords cannot be prompted for in order to fetch the remote. We support the +`git`, `ssh`, and `https` transport protocols. An SSH private key or access +token must be provided for authentication. Only `https` may be used without +authentication. See [GitHub's blog on protocol security](https://github.blog/2021-09-01-improving-git-protocol-security-github/) for more info. ```yaml --- sources: mysource: - remote: 'git://git-server.site/my-org/main-modules' + remote: 'https://git-server.site/my-org/main-modules' ``` ### basedir @@ -394,6 +511,23 @@ sources: * if `false` (default) environment folder will not be prefixed * if `String` environment folder will be prefixed with the `prefix` value. +### strip\_component + +The 'strip\_component' setting allows parts of environment names from a source to have a transformation applied, removing a part of the name before turning them into Puppet environments. This is primarily useful for VCS sources (e.g. Git), because it allows branch names to use prefixes or organizing name components such as "env/production", "env/development", but deploy Puppet environments from these branches named without the leading "env/" component. E.g. "production", "development". + +```yaml +--- +sources: + mysource: + basedir: '/etc/puppet/environments' + strip_component: 'env/' +``` + +#### strip\_component behaviour + +* if `string` environment names will have this prefix removed, if the prefix is present. Note that when string values are used, names can only have prefix components removed. +* if `/regex/` the regex will be matched against environment names and if a match is found, the matching name component will be removed. + ### ignore_branch_prefixes The 'ignore_branch_prefixes' setting causes environments to be ignored which match in part or whole @@ -455,7 +589,7 @@ hiera data files are kept. In this case --- sources: operations: - remote: 'git://git-server.site/my-org/org-modules' + remote: 'https://git-server.site/my-org/org-modules' basedir: '/etc/puppet/environments' ``` @@ -468,10 +602,10 @@ repository and your modules in another r --- sources: operations: - remote: 'git://git-server.site/my-org/org-modules' + remote: 'https://git-server.site/my-org/org-modules' basedir: '/etc/puppet/environments' hiera: - remote: 'git://git-server.site/my-org/org-hiera-data' + remote: 'https://git-server.site/my-org/org-hiera-data' basedir: '/etc/puppet/hiera-data' ``` @@ -486,15 +620,15 @@ not the modules of other groups. --- sources: main: - remote: 'git://git-server.site/my-org/main-modules' + remote: 'https://git-server.site/my-org/main-modules' basedir: '/etc/puppet/environments' prefix: false # Prefix defaults to false so this is only here for clarity qa: - remote: 'git://git-server.site/my-org/qa-puppet-modules' + remote: 'https://git-server.site/my-org/qa-puppet-modules' basedir: '/etc/puppet/environments' prefix: true dev: - remote: 'git://git-server.site/my-org/dev-puppet-modules' + remote: 'https://git-server.site/my-org/dev-puppet-modules' basedir: '/etc/puppet/environments' prefix: true ``` @@ -520,11 +654,11 @@ must override the `prefix` so environmen --- sources: app1_data: - remote: 'git://git-server.site/my-org/app1-hieradata' + remote: 'https://git-server.site/my-org/app1-hieradata' basedir: '/etc/puppet/hieradata' prefix: "app1" app1_modules: - remote: 'git://git-server.site/my-org/app1-puppet-modules' + remote: 'https://git-server.site/my-org/app1-puppet-modules' basedir: '/etc/puppet/environments' prefix: "app1" ``` @@ -569,13 +703,13 @@ When using the YAML source type, every e --- production: type: git - remote: git@github.com:puppetlabs/control-repo.git - ref: 8820892 + source: git@github.com:puppetlabs/control-repo.git + version: 8820892 development: type: git - remote: git@github.com:puppetlabs/control-repo.git - ref: 8820892 + source: git@github.com:puppetlabs/control-repo.git + version: 8820892 ``` ### YAMLdir Environment Source @@ -606,8 +740,8 @@ The contents of the file should be a has # production.yaml --- type: git -remote: git@github.com:puppetlabs/control-repo.git -ref: 8820892 +source: git@github.com:puppetlabs/control-repo.git +version: 8820892 ``` ### Exec environment Source @@ -628,7 +762,7 @@ sources: The environment modules feature allows module content to be attached to an environment at environment definition time. This happens before modules specified in a Puppetfile are attached to an environment, which does not happen until deploy time. Environment module implementation depends on the environment source type. -For the YAML environment source type, attach modules to an environment by specifying a modules key for the environment, and providing a hash of modules to attach. Each module accepts the same arguments accepted by the `mod` method in a Puppetfile. +For the YAML environment source type, attach modules to an environment by specifying a modules key for the environment, and providing a hash of modules to attach. Each module accepts the same arguments accepted by the `mod` method in a Puppetfile. For ease of reading and consistency, however, it is perferred to use the generic type, source, and version options over implementation-specific formats and options such as "ref" and "git". The example below includes two Forge modules and one module sourced from a Git repository. The two environments are almost identical. However, a new version of the stdlib module has been deployed in development (6.2.0), that has not yet been deployed to production. @@ -636,25 +770,35 @@ The example below includes two Forge mod --- production: type: git - remote: git@github.com:puppetlabs/control-repo.git - ref: 8820892 + source: git@github.com:puppetlabs/control-repo.git + version: 8820892 modules: - puppetlabs-stdlib: 6.0.0 - puppetlabs-concat: 6.1.0 + puppetlabs-stdlib: + type: forge + version: 6.0.0 + puppetlabs-concat: + type: forge + version: 6.1.0 reidmv-xampl: - git: https://github.com/reidmv/reidmv-xampl.git - ref: 62d07f2 + type: git + source: https://github.com/reidmv/reidmv-xampl.git + version: 62d07f2 development: type: git - remote: git@github.com:puppetlabs/control-repo.git - ref: 8820892 + source: git@github.com:puppetlabs/control-repo.git + version: 8820892 modules: - puppetlabs-stdlib: 6.2.0 - puppetlabs-concat: 6.1.0 + puppetlabs-stdlib: + type: forge + version: 6.2.0 + puppetlabs-concat: + type: forge + version: 6.1.0 reidmv-xampl: - git: https://github.com/reidmv/reidmv-xampl.git - ref: 62d07f2 + type: git + source: https://github.com/reidmv/reidmv-xampl.git + version: 62d07f2 ``` An example of a single environment definition for the YAMLdir environment source type: @@ -663,39 +807,108 @@ An example of a single environment defin # production.yaml --- type: git -remote: git@github.com:puppetlabs/control-repo.git -ref: 8820892 +source: git@github.com:puppetlabs/control-repo.git +version: 8820892 modules: - puppetlabs-stdlib: 6.0.0 - puppetlabs-concat: 6.1.0 + puppetlabs-stdlib: + type: forge + version: 6.0.0 + puppetlabs-concat: + type: forge + version: 6.1.0 reidmv-xampl: - git: https://github.com/reidmv/reidmv-xampl.git - ref: 62d07f2 + type: git + source: https://github.com/reidmv/reidmv-xampl.git + version: 62d07f2 ``` -### Bare Environment Type +#### Puppetfile module conflicts + +When a module is defined in an environment and also in a Puppetfile, the default behavior is for the environment definition of the module to take precedence, a warning to be logged, and the Puppetfile definition to be ignored. The behavior is configurable to optionally skip the warning, or allow a hard failure instead. Use the `module_conflicts` option in an environment definition to control this. + +Available `module_conflicts` options: + +* `override_and_warn` (default): the version of the module defined by the environment will be used, and the version defined in the Puppetfile will be ignored. A warning will be printed. +* `override`: the version of the module defined by the environment will be used, and the version defined in the Puppetfile will be ignored. +* `error`: an error will be raised alerting the user to the conflict. The environment will not be deployed. + +```yaml +# production.yaml +--- +type: git +source: git@github.com:puppetlabs/control-repo.git +version: 8820892 +module_conflicts: override_and_warn +modules: + puppetlabs-stdlib: + type: forge + version: 6.0.0 + puppetlabs-concat: + type: forge + version: 6.1.0 + reidmv-xampl: + type: git + source: https://github.com/reidmv/reidmv-xampl.git + version: 62d07f2 +``` + +### Plain Environment Type A "control repository" typically contains a hiera.yaml, an environment.conf, a manifests/site.pp file, and a few other things. However, none of these are strictly necessary for an environment to be functional if modules can be deployed to it. -The bare environment type allows sources that support environment modules to operate without a control repo being required. Modules can be deployed directly. +The plain environment type allows sources that support environment modules to operate without a control repo being required. Modules can be deployed directly. ```yaml --- production: - type: bare + type: plain + modules: + puppetlabs-stdlib: + type: forge + version: 6.0.0 + puppetlabs-concat: + type: forge + version: 6.1.0 + reidmv-xampl: + type: git + source: https://github.com/reidmv/reidmv-xampl.git + version: 62d07f2 + +development: + type: plain modules: - puppetlabs-stdlib: 6.0.0 - puppetlabs-concat: 6.1.0 + puppetlabs-stdlib: + type: forge + version: 6.0.0 + puppetlabs-concat: + type: forge + version: 6.1.0 reidmv-xampl: - git: https://github.com/reidmv/reidmv-xampl.git - ref: 62d07f2 + type: git + source: https://github.com/reidmv/reidmv-xampl.git + version: 62d07f2 +``` + +### Tarball Environment Type + +The tarball environment type allows an environment to be deployed from a tarball archive, rather than a Git repository. When using a tarball environment type, a source location for the tarball is required. Optionally, the tarball's sha256 checksum may be specified as the version. It is highly recommended to include a version specifier. If a version specifier is not included, r10k will never invalidate a cached copy of the tarball's source. + +Tarball environment sources will be unpacked directly into the environment root. + +```yaml +--- +production: + type: tarball + source: https://repo.example.com/projects/puppet/env-2.36.1.tar.gz + version: 99a906c99c2f144de43f2ae500509a7474ed11c583fb623efa8e5b377a3157f0 # sha256digest development: - type: bare + type: tarball + source: https://repo.example.com/projects/puppet/env-6128ada.tar.gz + version: 6128ada158622cd90f8e1360fb7c2c3830a812d1ec26ddf0db7eb16d61b7293f # sha256digest modules: - puppetlabs-stdlib: 6.0.0 - puppetlabs-concat: 6.1.0 reidmv-xampl: - git: https://github.com/reidmv/reidmv-xampl.git - ref: 62d07f2 + type: git + source: https://github.com/reidmv/reidmv-xampl.git + version: 62d07f2 ``` diff -pruN 3.7.0-2.1/doc/dynamic-environments/usage.mkd 3.15.4-1/doc/dynamic-environments/usage.mkd --- 3.7.0-2.1/doc/dynamic-environments/usage.mkd 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/doc/dynamic-environments/usage.mkd 2023-01-19 00:49:17.000000000 +0000 @@ -18,16 +18,16 @@ Command line invocation Recursively update all environments: - r10k deploy environment --puppetfile + r10k deploy environment --modules The simplest way to use r10k is by simply updating all environments and modules and takes the brute force approach of "update everything, ever." When this command is run r10k will update all sources, create new environments and delete old environments, and recursively update all environment modules specified in -environment Puppetfiles. While this is the simplest method for running r10k, is -is also the slowest by a very large degree because it does the maximum possible -work. This should not be something you run interactively, or use on a regular -basis. +environment Puppetfiles, yamldirs, etc. While this is the simplest method for +running r10k, it is also the slowest by a very large degree because it does the +maximum possible work. This should not be something you run interactively, or +use on a regular basis. - - - @@ -55,22 +55,53 @@ only the environment itself will be upda Update a single environment and force an update of modules: - r10k deploy environment my_working_environment --puppetfile + r10k deploy environment my_working_environment --modules This will update the given environment and update all contained modules. This is useful if you want to make sure that a given environment is fully up to date. - - - +There is also a middle ground between updating all modules and updating no modules. +It is often desirable to update the environment and then update only those modules +whose definitions have changed in the Puppetfile, or whose content _could_ have +changed since the last deployment (eg, Forge modules with their version set to +`:latest` or Git modules who point to a `branch` ref). + +This can be achieved by assuming content is unchanged locally on disk. This is the +opposite of what one would assume during a module development cycle, when a user +might be making local edits to test code changes. However, in production, access +to puppet code is usually locked down, and updates are deployed through automated +invocations of R10K. + +In these cases, deploys where most modules are unchanged and reference exact +versions (ie, not `:latest` or a branch as mentioned above), this invocation +may shorten deployment times dozens of seconds if not minutes depending on how +many modules meet the above criteria (approximately 1 minute for every 400 modules). + +To take advantage of this, set as many modules as possible in the Puppetfile to +explicit, static version. These are released Forge versions, or Git modules using +the `:tag`, or `:commit` keys. Git `:ref`s containing only the full 40 character +commit SHA will also be treated as static versions. Then invoke a deploy with: + +There may be issues with deployments apparently successful after an initial errored +deployment. If this is happening, try running without the `--incremental` flag +to run a full deployment. + + r10k deploy environment production --modules --incremental + +- - - + Update a single environment and specify a default branch override: - r10k deploy environment my_working_environment --puppetfile --default-branch-override default_branch_override + r10k deploy environment my_working_environment --modules --default-branch-override default_branch_override -This will update the given environment and update all contained modules, overrideing -the :default_branch entry in the Puppetfile of each module. This is used primarily to allow +This will update the given environment and update all contained modules, overriding +the :default_branch entry in the Puppetfile of each module. If the specified override branch is not +found, it will fall back to the normal default branch and attempt to use that. This is used primarily to allow automated r10k solutions using the control_branch pattern with a temporary branch deployment to -ensure the deployment is pushed to the correct module repository branch. Note that the :default_branch -is only ever utilized if the desired ref cannot be located. +ensure the deployment is pushed to the correct module repository branch. Note that the :default_branch and its +override are only ever used if the specific desired ref cannot be located. ### Deploying modules diff -pruN 3.7.0-2.1/doc/dynamic-environments/workflow-guide.mkd 3.15.4-1/doc/dynamic-environments/workflow-guide.mkd --- 3.7.0-2.1/doc/dynamic-environments/workflow-guide.mkd 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/doc/dynamic-environments/workflow-guide.mkd 2023-01-19 00:49:17.000000000 +0000 @@ -38,7 +38,7 @@ mod "puppetlabs/ntp" # Your modules: mod "custom_facts", - :git => "git://github.com/user/custom_facts" + :git => "https://github.com/user/custom_facts" ``` For any existing modules that you branched, add a reference to the new branch @@ -46,7 +46,7 @@ name. Don't forget the comma at the end ``` mod "other_module", - :git => "git://github.com/user/other_module", + :git => "https://github.com/user/other_module", :ref => "feature" ``` @@ -159,7 +159,7 @@ the *:git* value. ``` mod "other_module", - :git => "git://github.com/user/other_module", + :git => "https://github.com/user/other_module", :ref => "feature" ``` diff -pruN 3.7.0-2.1/doc/faq.mkd 3.15.4-1/doc/faq.mkd --- 3.7.0-2.1/doc/faq.mkd 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/doc/faq.mkd 2023-01-19 00:49:17.000000000 +0000 @@ -17,7 +17,7 @@ modify. For example creating the script: ``` $ cat /usr/local/bin/generate-puppet-types.sh -!#/bin/bash +#!/bin/bash for environment in $1; do /opt/puppetlabs/bin/puppet generate types --environment $environment diff -pruN 3.7.0-2.1/doc/git/providers.mkd 3.15.4-1/doc/git/providers.mkd --- 3.7.0-2.1/doc/git/providers.mkd 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/doc/git/providers.mkd 2023-01-19 00:49:17.000000000 +0000 @@ -56,6 +56,28 @@ git: private_key: '/root/.ssh/private_repo_id' ``` +### HTTPS Configuration + +Public HTTPS based Git repositories can be accessed with no additional settings. +For repos that do require authentication, the 'oauth_token' option may be provided. + +```yaml +git: + oauth_token: '/etc/puppetlabs/r10k/token' +``` + +If you have per repository access tokens you can add them with the repositories list. + +```yaml +git: + # default access token + oauth_token: '/etc/puppetlabs/r10k/token' + repositories: + - remote: "https://github.com/my_org/private_repo.git" + # access token for this repo only + oauth_token: '/etc/puppetlabs/r10k/private_repo_token' +``` + #### Supported transports with Rugged Rugged compiles libgit2 and and the Ruby bindings when the gem is installed. You diff -pruN 3.7.0-2.1/doc/puppetfile.mkd 3.15.4-1/doc/puppetfile.mkd --- 3.7.0-2.1/doc/puppetfile.mkd 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/doc/puppetfile.mkd 2023-01-19 00:49:17.000000000 +0000 @@ -51,10 +51,9 @@ handles modules. ### forge The `forge` setting specifies which server that Forge based modules are fetched -from. This is currently a noop and is provided for compatibility with -librarian-puppet. - -R10k supports setting the Forge to use _globally_ in `r10k.yaml`. see [Configuration](/doc/dynamic-environments/configuration.mkd#baseurl) for details. +from. This declaration is only respected if [`forge.allow_puppetfile_override`](/dynamic-environments/configuration.mkd#allow_puppetfile_override) +is set to true in the main `r10k.yaml`. Otherwise, use [`forge.baseurl`](/doc/dynamic-environments/configuration.mkd#baseurl) +to globally configure where modules should be downloaded from. ### moduledir @@ -109,11 +108,19 @@ latest version available. mod 'puppetlabs/apache', :latest +An explicit type and/or version can be specified using the standard interface, +`:type` and `:version`. The `:source` parameter is not supported for individual +forge modules and will be ignored. + + mod 'puppetlabs/apache', + type: 'forge', + version: '6.0.0' + ### Git Git repositories that contain a Puppet module can be cloned and used as modules. When Git is used, the module version can be specified by using `:ref`, `:tag`, -`:commit`, and `:branch`. +`:commit`, `:branch`, or the standard interface parameter `:version`. When a module is installed using `:ref`, r10k uses some simple heuristics to determine the type of Git object that should be checked out. This can be used @@ -122,7 +129,8 @@ with a git commit, branch reference, or When a module is installed using `:tag` or `:commit`, r10k assumes that the given object is a tag or commit and can do some optimizations around fetching the object. If the tag or commit is already available r10k will skip network -operations when updating the repo, which can speed up install times. +operations when updating the repo, which can speed up install times. When +`:ref` is set to track `HEAD`, it will synchronize the module on each run. Module versions can also be specified using `:branch` to track a specific branch reference. @@ -153,6 +161,13 @@ mod 'apache', mod 'apache', :git => 'https://github.com/puppetlabs/puppetlabs-apache', :branch => 'docs_experiment' + +# Install puppetlabs/apache and use standard interface parameters pinned to the +# '2098a17' commit. +mod 'puppetlabs-apache', + type: 'git', + source: 'https://github.com/puppetlabs/puppetlabs-apache', + version: '2098a17' ``` #### Control Repo Branch Tracking @@ -195,8 +210,8 @@ the latest version available in the main mod 'apache', :svn => 'https://github.com/puppetlabs/puppetlabs-apache/trunk' -If an SVN revision number is specified with `:rev` (or `:revision`), that -SVN revision will be kept checked out. +If an SVN revision number is specified with `:rev`, `:revision`, or `:version`, +that SVN revision will be kept checked out. mod 'apache', :svn => 'https://github.com/puppetlabs/puppetlabs-apache/trunk', @@ -206,6 +221,11 @@ SVN revision will be kept checked out. :svn => 'https://github.com/puppetlabs/puppetlabs-apache/trunk', :revision => '154' + mod 'apache', + type: 'svn', + source: 'https://github.com/puppetlabs/puppetlabs-apache/trunk', + version: '154' + If the SVN repository requires credentials, you can supply the `:username` and `:password` options. @@ -219,6 +239,19 @@ credentials may be visible in the proces choose to supply SVN credentials make sure that the system running r10k is appropriately secured. +### Tarball + +Modules can be installed from tarball archives. A tarball module must specify a source URL to retreive the tarball content from. A tarball module may optionally specify a sha256 checksum as the module version. + + mod 'puppetlabs-apache', + type: 'tarball', + source: 'https://repo.example.com/puppet/modules/puppetlabs-apache-7.0.0.tar.gz', + version: 'aedd6dc1a5136c6a1a1ec2f285df2a70b0fe4c9effb254b5a1f58116e4c1659e' # sha256 digest + +If no version is specified, a tarball will be downloaded from the given source and cached. The cache will not be invalidated until the source URL is changed, or a sha256 checksum version is provided. + +Tarball module content will be unpacked directly into an appropriately named module directory. For example, the puppetlabs-apache-7.0.0.tar.gz archive in the example above will be unpacked into `/modules/apache/`. + ### Local In the event you want to store locally written modules in your r10k-managed @@ -292,6 +325,19 @@ module. For more information see the [FAQ entry](faq.mkd#how-do-i-prevent-r10k-from-removing-modules-in-the-modules-directory-of-my-git-repository) on managing internal and external modules in the same directory. +### Per-Item spec dir deployment + +During deployment, r10k's default behavior is to deploy the spec directory. The +Puppetfile can modify this per module, overriding settings from the default +r10k config. The following example sets the module to not deploy the spec +directory. + +``` +mod 'apache', + :git => 'git@github.com:puppetlabs/puppetlabs-apache.git', + :exclude_spec => true +``` + ### Per-Item Install Path Git and SVN content types support installing into an alternate path without changing diff -pruN 3.7.0-2.1/docker/docker-compose.yml 3.15.4-1/docker/docker-compose.yml --- 3.7.0-2.1/docker/docker-compose.yml 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/docker/docker-compose.yml 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,18 @@ +version: '3.7' + +services: + r10k_check: + image: ${R10K_IMAGE:-puppet/r10k} + environment: + - PUPPERWARE_ANALYTICS_ENABLED=${PUPPERWARE_ANALYTICS_ENABLED:-false} + command: 'puppetfile check --verbose --trace --puppetfile test/Puppetfile' + volumes: + - ${SPEC_DIRECTORY}/fixtures:/home/puppet/test + + r10k_install: + image: ${R10K_IMAGE:-puppet/r10k} + environment: + - PUPPERWARE_ANALYTICS_ENABLED=${PUPPERWARE_ANALYTICS_ENABLED:-false} + command: 'puppetfile install --verbose --trace --puppetfile test/Puppetfile' + volumes: + - ${SPEC_DIRECTORY}/fixtures:/home/puppet/test diff -pruN 3.7.0-2.1/docker/Gemfile 3.15.4-1/docker/Gemfile --- 3.7.0-2.1/docker/Gemfile 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/docker/Gemfile 2023-01-19 00:49:17.000000000 +0000 @@ -2,7 +2,7 @@ source "https://rubygems.org" gem 'pupperware', :git => 'https://github.com/puppetlabs/pupperware.git', - :branch => 'master', + :branch => 'main', :glob => 'gem/*.gemspec' group :test do diff -pruN 3.7.0-2.1/docker/Makefile 3.15.4-1/docker/Makefile --- 3.7.0-2.1/docker/Makefile 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/docker/Makefile 2023-01-19 00:49:17.000000000 +0000 @@ -5,12 +5,16 @@ vcs_ref := $(shell git rev-parse HEAD) build_date := $(shell date -u +%FT%T) hadolint_available := $(shell hadolint --help > /dev/null 2>&1; echo $$?) hadolint_command := hadolint -hadolint_container := hadolint/hadolint:latest -alpine_version := 3.9 +hadolint_container := ghcr.io/hadolint/hadolint:latest +alpine_version := 3.14 +# --load (--output=type=docker) can only be used with a single arch / platform +# https://github.com/docker/buildx/issues/59 +output_type := docker +platform := linux/amd64 export BUNDLE_PATH = $(PWD)/.bundle/gems export BUNDLE_BIN = $(PWD)/.bundle/bin export GEMFILE = $(PWD)/Gemfile -export DOCKER_BUILDKIT = 1 +export DOCKER_BUILDKIT ?= 1 ifeq ($(IS_RELEASE),true) VERSION ?= $(shell echo $(git_describe) | sed 's/-.*//') @@ -32,6 +36,10 @@ else dockerfile_context := $(PWD)/.. endif +ifeq ($(IS_LATEST),true) + latest_tag := --tag $(NAMESPACE)/r10k:$(LATEST_VERSION) +endif + prep: @git fetch --unshallow 2> /dev/null ||: @git fetch origin 'refs/tags/*:refs/tags/*' @@ -50,18 +58,17 @@ endif build: prep docker pull alpine:$(alpine_version) - docker build \ + docker buildx build \ ${DOCKER_BUILD_FLAGS} \ + --output=type=$(output_type) \ + --platform $(platform) \ --build-arg alpine_version=$(alpine_version) \ --build-arg vcs_ref=$(vcs_ref) \ --build-arg build_date=$(build_date) \ --build-arg version=$(VERSION) \ --build-arg pupperware_analytics_stream=$(PUPPERWARE_ANALYTICS_STREAM) \ --file r10k/$(dockerfile) \ - --tag $(NAMESPACE)/r10k:$(VERSION) $(dockerfile_context) -ifeq ($(IS_LATEST),true) - @docker tag $(NAMESPACE)/r10k:$(VERSION) puppet/r10k:$(LATEST_VERSION) -endif + --tag $(NAMESPACE)/r10k:$(VERSION) $(latest_tag) $(dockerfile_context) test: prep @bundle install --path $$BUNDLE_PATH --gemfile $$GEMFILE --with test @@ -70,11 +77,12 @@ test: prep bundle exec --gemfile $$GEMFILE \ rspec spec -push-image: prep - @docker push $(NAMESPACE)/r10k:$(VERSION) -ifeq ($(IS_LATEST),true) - @docker push $(NAMESPACE)/r10k:$(LATEST_VERSION) -endif +# call build to produce multiple architectures +# uses cached output from amd64 build target if it exists +# registry output is equivalent to --push +push-image: platform=linux/amd64,linux/arm64 +push-image: output_type=registry +push-image: prep build push-readme: @docker pull sheogorath/readme-to-dockerhub diff -pruN 3.7.0-2.1/docker/r10k/docker-entrypoint.sh 3.15.4-1/docker/r10k/docker-entrypoint.sh --- 3.7.0-2.1/docker/r10k/docker-entrypoint.sh 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/docker/r10k/docker-entrypoint.sh 2023-01-19 00:49:17.000000000 +0000 @@ -4,7 +4,6 @@ set -e for f in /docker-entrypoint.d/*.sh; do # Don't print out any messages here since this is a CLI container - chmod +x "$f" "$f" done diff -pruN 3.7.0-2.1/docker/r10k/Dockerfile 3.15.4-1/docker/r10k/Dockerfile --- 3.7.0-2.1/docker/r10k/Dockerfile 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/docker/r10k/Dockerfile 2023-01-19 00:49:17.000000000 +0000 @@ -1,4 +1,4 @@ -ARG alpine_version=3.9 +ARG alpine_version=3.14 FROM alpine:${alpine_version} as build # hadolint ignore=DL3018 @@ -49,7 +49,7 @@ COPY --from=build /workspace/r10k.gem / SHELL ["/bin/ash", "-eo", "pipefail", "-c"] # ignore apk and gem pinning # hadolint ignore=DL3018,DL3028 -RUN chmod a+x /adduser.sh /docker-entrypoint.sh && \ +RUN chmod a+x /adduser.sh /docker-entrypoint.sh /docker-entrypoint.d/*.sh && \ # Add a puppet user to run r10k as for consistency with puppetserver /adduser.sh && \ chown -R puppet: /docker-entrypoint.d /docker-entrypoint.sh && \ diff -pruN 3.7.0-2.1/docker/r10k/release.Dockerfile 3.15.4-1/docker/r10k/release.Dockerfile --- 3.7.0-2.1/docker/r10k/release.Dockerfile 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/docker/r10k/release.Dockerfile 2023-01-19 00:49:17.000000000 +0000 @@ -1,4 +1,4 @@ -ARG alpine_version=3.9 +ARG alpine_version=3.14 FROM alpine:${alpine_version} ARG vcs_ref @@ -38,7 +38,7 @@ LABEL org.label-schema.version="$version SHELL ["/bin/ash", "-eo", "pipefail", "-c"] # ignore apk and gem pinning # hadolint ignore=DL3018,DL3028 -RUN chmod a+x /adduser.sh /docker-entrypoint.sh && \ +RUN chmod a+x /adduser.sh /docker-entrypoint.sh /docker-entrypoint.d/*.sh && \ /adduser.sh && \ chown -R puppet: /docker-entrypoint.d /docker-entrypoint.sh && \ apk add --no-cache ruby openssh-client git ruby-rugged curl ruby-dev make gcc musl-dev && \ diff -pruN 3.7.0-2.1/docker/spec/dockerfile_spec.rb 3.15.4-1/docker/spec/dockerfile_spec.rb --- 3.7.0-2.1/docker/spec/dockerfile_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/docker/spec/dockerfile_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,43 +1,37 @@ require 'rspec/core' require 'fileutils' require 'open3' +include Pupperware::SpecHelpers -SPEC_DIRECTORY = File.dirname(__FILE__) - -describe 'r10k container' do - include Pupperware::SpecHelpers - def run_r10k(command) - run_command("docker run --detach \ - --volume #{File.join(SPEC_DIRECTORY, 'fixtures')}:/home/puppet/test \ - #{@image} #{command} \ - --verbose \ - --trace \ - --puppetfile test/Puppetfile") - end - - before(:all) do - @image = require_test_image +ENV['SPEC_DIRECTORY'] = File.dirname(__FILE__) +# unifies volume naming +ENV['COMPOSE_PROJECT_NAME'] ||= 'r10k' + +RSpec.configure do |c| + c.before(:suite) do + ENV['R10K_IMAGE'] = require_test_image + pull_images(['r10k_check','r10k_install']) + teardown_cluster() + # no certs to preload, but if the suite adds puppetserver, be explicit + docker_compose_up(preload_certs: true) end - after(:all) do - FileUtils.rm_rf(File.join(SPEC_DIRECTORY, 'fixtures', 'modules')) - end - - it 'should validate the Puppetfile' do - result = run_r10k('puppetfile check') - container = result[:stdout].chomp - wait_on_container_exit(container) - expect(get_container_exit_code(container)).to eq(0) - emit_log(container) - teardown_container(container) + c.after(:suite) do + teardown_cluster() + FileUtils.rm_rf(File.join(ENV['SPEC_DIRECTORY'], 'fixtures', 'modules')) end +end - it 'should install the Puppetfile' do - result = run_r10k('puppetfile install') - container = result[:stdout].chomp - wait_on_container_exit(container) - expect(get_container_exit_code(container)).to eq(0) - emit_log(container) - teardown_container(container) +describe 'r10k container' do + { + 'r10k_check': 'validate', + 'r10k_install': 'install', + }.each do |container, op| + it "should #{op} the Puppetfile" do + container = get_service_container(container) + wait_on_container_exit(container) + expect(get_container_exit_code(container)).to eq(0) + emit_log(container) + end end end diff -pruN 3.7.0-2.1/.github/pull_request_template.md 3.15.4-1/.github/pull_request_template.md --- 3.7.0-2.1/.github/pull_request_template.md 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/.github/pull_request_template.md 2023-01-19 00:49:17.000000000 +0000 @@ -1,4 +1,4 @@ Please add all notable changes to the "Unreleased" section of the CHANGELOG in the format: ``` -- Summary of changes. [Issue or PR #](link to issue or PR) +- (JIRA ticket) Summary of changes. [Issue or PR #](link to issue or PR) ``` diff -pruN 3.7.0-2.1/.github/workflows/docker.yml 3.15.4-1/.github/workflows/docker.yml --- 3.7.0-2.1/.github/workflows/docker.yml 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/.github/workflows/docker.yml 2023-01-19 00:49:17.000000000 +0000 @@ -3,7 +3,7 @@ name: Docker test and publish on: push: branches: - - master + - main jobs: build-and-publish: @@ -27,9 +27,17 @@ jobs: with: ruby-version: 2.6.x - run: gem install bundler + - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 - name: Build container working-directory: docker - run: make lint build test + run: | + docker system prune --all --force --volumes + docker builder prune --force --keep-storage=10GB + make lint build test - name: Publish container working-directory: docker run: | diff -pruN 3.7.0-2.1/.github/workflows/release.yml 3.15.4-1/.github/workflows/release.yml --- 3.7.0-2.1/.github/workflows/release.yml 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/.github/workflows/release.yml 2023-01-19 00:49:17.000000000 +0000 @@ -3,7 +3,7 @@ name: Tag and release on: push: branches: - - master + - main paths: - 'lib/r10k/version.rb' @@ -15,10 +15,11 @@ jobs: with: fetch-depth: '0' - name: Bump version and push tag - uses: anothrNick/github-tag-action@1.17.2 + uses: anothrNick/github-tag-action@1.35.0 env: GITHUB_TOKEN: ${{ secrets.PUPPET_RELEASE_GH_TOKEN }} DEFAULT_BUMP: patch + TAG_CONTEXT: branch WITH_V: false # Uncomment this if the tag and version file become out-of-sync and # you need to tag at a specific version. diff -pruN 3.7.0-2.1/.github/workflows/rspec_tests.yml 3.15.4-1/.github/workflows/rspec_tests.yml --- 3.7.0-2.1/.github/workflows/rspec_tests.yml 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/.github/workflows/rspec_tests.yml 2023-01-19 00:49:17.000000000 +0000 @@ -3,7 +3,7 @@ name: Rspec tests on: pull_request: branches: - - master + - main jobs: rspec_tests: @@ -15,10 +15,11 @@ jobs: - {os: ubuntu-18.04, ruby: 2.5} - {os: ubuntu-18.04, ruby: 2.6} - {os: ubuntu-18.04, ruby: 2.7} - - {os: ubuntu-18.04, ruby: jruby-9.2.9.0} - - {os: windows-2016, ruby: 2.5} - - {os: windows-2016, ruby: 2.6} - - {os: windows-2016, ruby: 2.7} + - {os: ubuntu-18.04, ruby: 3.1} + - {os: ubuntu-18.04, ruby: jruby-9.2.10.0} + - {os: windows-2019, ruby: 2.5} + - {os: windows-2019, ruby: 2.6} + - {os: windows-2019, ruby: 2.7} runs-on: ${{ matrix.cfg.os }} steps: @@ -32,7 +33,8 @@ jobs: - name: Install bundler and gems run: | - gem install bundler + # Pin bundler to maintain support for Ruby 2.4 and 2.5 + gem install bundler -v 2.3.26 bundle config set without packaging documentation bundle install --jobs 4 --retry 3 @@ -59,7 +61,7 @@ jobs: bundle --version # Run tests - bundle exec rspec --color --format documentation spec/unit + bundle exec rspec --color --format documentation spec - name: Run tests on Linux if: runner.os == 'Linux' @@ -78,4 +80,4 @@ jobs: fi # Run tests - bundle exec rspec --color --format documentation spec/unit + bundle exec rspec --color --format documentation spec diff -pruN 3.7.0-2.1/.github/workflows/stale.yml 3.15.4-1/.github/workflows/stale.yml --- 3.7.0-2.1/.github/workflows/stale.yml 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/.github/workflows/stale.yml 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,21 @@ +name: Mark stale issues + +on: + schedule: + - cron: "30 1 * * *" + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + days-before-stale: 60 + days-before-close: 7 + stale-issue-message: 'This issue has been marked stale because it has had no activity for 60 days. The Puppet Team is actively prioritizing existing bugs and new features, if this issue is still important to you please comment and we will add this to our backlog to complete. Otherwise, it will be closed in 7 days.' + stale-issue-label: 'stale' + exempt-issue-labels: 'community interest' + stale-pr-message: "This PR has been marked stale because it has had no activity for 60 days. If you are still interested in getting this merged, please comment and we'll try to move it forward. Otherwise, it will be closed in 7 days." + stale-pr-label: 'stale' + exempt-pr-labels: 'community interest' diff -pruN 3.7.0-2.1/.gitignore 3.15.4-1/.gitignore --- 3.7.0-2.1/.gitignore 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/.gitignore 2023-01-19 00:49:17.000000000 +0000 @@ -7,3 +7,4 @@ coverage integration/log integration/junit integration/configs +r10k.log diff -pruN 3.7.0-2.1/integration/Rakefile 3.15.4-1/integration/Rakefile --- 3.7.0-2.1/integration/Rakefile 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/Rakefile 2023-01-19 00:49:17.000000000 +0000 @@ -1,4 +1,5 @@ require 'rototiller' +require 'fileutils' namespace :ci do namespace :test do @@ -59,13 +60,14 @@ end desc 'Generate a host configuration used by Beaker' rototiller_task :beaker_hostgenerator do |t| if ENV['BEAKER_HOST'].nil? + FileUtils.mkdir_p 'configs' t.add_command do |c| c.name = 'beaker-hostgenerator' c.argument = '> configs/generated' end # This is a hack :( - t.add_flag(:name => '', :default => 'centos6-64mdca-64.fa', :override_env => 'TEST_TARGET') + t.add_flag(:name => '', :default => 'centos7-64mdca-64.fa', :override_env => 'TEST_TARGET') t.add_flag(:name => '--global-config', :default => '{forge_host=forge-aio01-petest.puppetlabs.com}', :override_env => 'BHG_GLOBAL_CONFIG') end diff -pruN 3.7.0-2.1/integration/tests/basic_functionality/basic_deployment.rb 3.15.4-1/integration/tests/basic_functionality/basic_deployment.rb --- 3.7.0-2.1/integration/tests/basic_functionality/basic_deployment.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/integration/tests/basic_functionality/basic_deployment.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,176 @@ +require 'git_utils' +require 'r10k_utils' +require 'master_manipulator' + + +test_name 'Basic Environment Deployment Workflows' + +# This isn't a block because we want to use the local variables throughout the file +step 'init' + @env_path = on(master, puppet('config print environmentpath')).stdout.rstrip + r10k_fqp = get_r10k_fqp(master) + + control_repo_gitdir = '/git_repos/environments.git' + control_repo_worktree = '/root/environments' + last_commit = git_last_commit(master, control_repo_worktree) + git_provider = ENV['GIT_PROVIDER'] + + config_path = get_r10k_config_file_path(master) + config_backup_path = "#{config_path}.bak" + + puppetfile1 =<<-EOS + mod 'puppetlabs/apache', '0.10.0' + mod 'puppetlabs/stdlib', '8.0.0' + EOS + + r10k_conf = <<-CONF + cachedir: '/var/cache/r10k' + git: + provider: '#{git_provider}' + sources: + control: + basedir: "#{@env_path}" + remote: "#{control_repo_gitdir}" + deploy: + purge_levels: ['deployment','environment','puppetfile'] + + CONF + + +def and_stdlib_is_correct + metadata_path = "#{@env_path}/production/modules/stdlib/metadata.json" + on(master, "test -f #{metadata_path}", accept_all_exit_codes: true) do |result| + assert(result.exit_code == 0, 'stdlib content has been inappropriately purged') + end + metadata_info = JSON.parse(on(master, "cat #{metadata_path}").stdout) + assert(metadata_info['version'] == '8.0.0', 'stdlib deployed to wrong version') +end + +teardown do + on(master, "mv #{config_backup_path} #{config_path}") + clean_up_r10k(master, last_commit, control_repo_worktree) +end + +step 'Set up r10k and control repo' do + + # Backup and replace r10k config + on(master, "mv #{config_path} #{config_backup_path}") + create_remote_file(master, config_path, r10k_conf) + + # Place our Puppetfile in the control repo's production branch + git_on(master, 'checkout production', control_repo_worktree) + create_remote_file(master, "#{control_repo_worktree}/Puppetfile", puppetfile1) + git_add_commit_push(master, 'production', 'add Puppetfile for Basic Deployment test', control_repo_worktree) + + # Ensure the production environment will be deployed anew + on(master, "rm -rf #{@env_path}/production") +end + +test_path = "#{@env_path}/production/modules/apache/metadata.json" +step 'Test initial environment deploy works' do + on(master, "#{r10k_fqp} deploy environment production --verbose=info") do |result| + assert(result.output =~ /.*Deploying module to .*apache.*/, 'Did not log apache deployment') + assert(result.output =~ /.*Deploying module to .*stdlib.*/, 'Did not log stdlib deployment') + end + on(master, "test -f #{test_path}", accept_all_exit_codes: true) do |result| + assert(result.exit_code == 0, 'Expected module in Puppetfile was not installed') + end + + and_stdlib_is_correct +end + +original_apache_info = JSON.parse(on(master, "cat #{test_path}").stdout) + +step 'Test second run of deploy updates control repo, but leaves moduledir untouched' do + puppetfile2 =<<-EOS + # Current latest of apache is 6.5.1 as of writing this test + mod 'puppetlabs/apache', :latest + mod 'puppetlabs/stdlib', '8.0.0' + mod 'puppetlabs/concat', '7.0.0' + EOS + + git_on(master, 'checkout production', control_repo_worktree) + create_remote_file(master, "#{control_repo_worktree}/Puppetfile", puppetfile2) + git_add_commit_push(master, 'production', 'add Puppetfile for Basic Deployment test', control_repo_worktree) + + on(master, "#{r10k_fqp} deploy environment production --verbose=info") do |result| + refute(result.output =~ /.*Deploying module to .*apache.*/, 'Inappropriately updated apache') + refute(result.output =~ /.*Deploying module to .*stdlib.*/, 'Inappropriately updated stdlib') + end + + on(master, "test -f #{test_path}", accept_all_exit_codes: true) do |result| + assert(result.exit_code == 0, 'Expected module content in Puppetfile was inappropriately purged') + end + + new_apache_info = JSON.parse(on(master, "cat #{test_path}").stdout) + on(master, "cat #{@env_path}/production/Puppetfile | grep ':latest'", accept_all_exit_codes: true) do |result| + assert(result.exit_code == 0, 'Puppetfile not updated on subsequent r10k deploys') + end + + assert(original_apache_info['version'] == new_apache_info['version'] && + new_apache_info['version'] == '0.10.0', + 'Module content updated on subsequent r10k invocations w/o providing --modules') + + on(master, "test -f #{@env_path}/production/modules/concat/metadata.json", accept_all_exit_codes: true) do |result| + assert(result.exit_code == 1, 'Module content deployed on subsequent r10k invocation w/o providing --modules') + end + + and_stdlib_is_correct +end + +step 'Test --modules updates modules' do + on(master, "#{r10k_fqp} deploy environment production --modules --verbose=info") do |result| + assert(result.output =~ /.*Deploying module to .*apache.*/, 'Did not log apache deployment') + assert(result.output =~ /.*Deploying module to .*stdlib.*/, 'Did not log stdlib deployment') + assert(result.output =~ /.*Deploying module to .*concat.*/, 'Did not log concat deployment') + end + + on(master, "test -f #{test_path}", accept_all_exit_codes: true) do |result| + assert(result.exit_code == 0, 'Expected module content in Puppetfile was inappropriately purged') + end + + on(master, "test -f #{@env_path}/production/modules/concat/metadata.json", accept_all_exit_codes: true) do |result| + assert(result.exit_code == 0, 'New module content was not deployed when providing --modules') + end + + new_apache_info = JSON.parse(on(master, "cat #{test_path}").stdout) + apache_major_version = new_apache_info['version'].split('.').first.to_i + assert(apache_major_version > 5, 'Module not updated correctly using --modules') + + and_stdlib_is_correct +end + +step 'Test --modules --incremental deploys changed & dynamic modules, but not unchanged, static modules' do + puppetfile3 =<<-EOS + # Current latest of apache is 6.5.1 as of writing this test + mod 'puppetlabs/apache', :latest + mod 'puppetlabs/stdlib', '8.0.0' + mod 'puppetlabs/concat', '7.1.0' + EOS + + git_on(master, 'checkout production', control_repo_worktree) + create_remote_file(master, "#{control_repo_worktree}/Puppetfile", puppetfile3) + git_add_commit_push(master, 'production', 'add Puppetfile for Basic Deployment test', control_repo_worktree) + + on(master, "#{r10k_fqp} deploy environment production --modules --incremental --verbose=debug1") do |result| + assert(result.output =~ /.*Deploying module to .*apache.*/, 'Did not log apache deployment') + assert(result.output =~ /.*Deploying module to .*concat.*/, 'Did not log concat deployment') + assert(result.output =~ /.*Not updating module stdlib, assuming content unchanged.*/, 'Did not log notice of skipping stdlib') + end + + on(master, "test -f #{test_path}", accept_all_exit_codes: true) do |result| + assert(result.exit_code == 0, 'Expected module content in Puppetfile was inappropriately purged') + end + + new_apache_info = JSON.parse(on(master, "cat #{test_path}").stdout) + apache_major_version = new_apache_info['version'].split('.').first.to_i + assert(apache_major_version > 5, 'Module not updated correctly using --modules & --incremental') + + concat_info = JSON.parse(on(master, "cat #{@env_path}/production/modules/concat/metadata.json").stdout) + concat_minor_version = concat_info['version'].split('.')[1].to_i + assert(concat_minor_version == 1, 'Module not updated correctly using --modules & --incremental') + + and_stdlib_is_correct +end + + diff -pruN 3.7.0-2.1/integration/tests/git_source/git_source_repeated_remote.rb 3.15.4-1/integration/tests/git_source/git_source_repeated_remote.rb --- 3.7.0-2.1/integration/tests/git_source/git_source_repeated_remote.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/tests/git_source/git_source_repeated_remote.rb 2023-01-19 00:49:17.000000000 +0000 @@ -29,12 +29,12 @@ CONF # Install the same module in two different places puppetfile = <<-EOS mod 'prod_apache', - :git => 'git://github.com/puppetlabs/puppetlabs-apache.git', - :branch => 'master' + :git => 'https://github.com/puppetlabs/puppetlabs-apache.git', + :tag => 'v6.0.0' mod 'test_apache', - :git => 'git://github.com/puppetlabs/puppetlabs-apache.git', - :branch => 'master' + :git => 'https://github.com/puppetlabs/puppetlabs-apache.git', + :tag => 'v6.0.0' EOS teardown do diff -pruN 3.7.0-2.1/integration/tests/git_source/git_source_submodule.rb 3.15.4-1/integration/tests/git_source/git_source_submodule.rb --- 3.7.0-2.1/integration/tests/git_source/git_source_submodule.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/tests/git_source/git_source_submodule.rb 2023-01-19 00:49:17.000000000 +0000 @@ -44,7 +44,7 @@ scp_to(master, helloworld_module_path, F git_add_commit_push(master, 'master', 'Add module.', git_clone_module_path) step 'Add "helloworld" Module Git Repo as Submodule' -on(master, "cd #{git_environments_path};git submodule add file://#{git_repo_module_path} dist") +on(master, "cd #{git_environments_path};git -c protocol.file.allow=always submodule add file://#{git_repo_module_path} dist") step 'Checkout "production" Branch' git_on(master, 'checkout production', git_environments_path) diff -pruN 3.7.0-2.1/integration/tests/git_source/HTTP_proxy_and_git_source.rb 3.15.4-1/integration/tests/git_source/HTTP_proxy_and_git_source.rb --- 3.7.0-2.1/integration/tests/git_source/HTTP_proxy_and_git_source.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/tests/git_source/HTTP_proxy_and_git_source.rb 2023-01-19 00:49:17.000000000 +0000 @@ -16,7 +16,7 @@ r10k_config_bak_path = "#{r10k_config_pa puppetfile =<<-EOS mod 'motd', - :git => 'git://github.com/puppetlabs/puppetlabs-motd' + :git => 'https://github.com/puppetlabs/puppetlabs-motd' EOS proxy_env_value = 'http://cattastic.net:3219' diff -pruN 3.7.0-2.1/integration/tests/purging/content_not_purged_at_root.rb 3.15.4-1/integration/tests/purging/content_not_purged_at_root.rb --- 3.7.0-2.1/integration/tests/purging/content_not_purged_at_root.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/tests/purging/content_not_purged_at_root.rb 2023-01-19 00:49:17.000000000 +0000 @@ -30,12 +30,12 @@ CONF puppetfile = <<-EOS mod 'non_module_object_1', :install_path => './', - :git => 'git://github.com/puppetlabs/control-repo.git', + :git => 'https://github.com/puppetlabs/control-repo.git', :branch => 'production' mod 'non_module_object_2', :install_path => '', - :git => 'git://github.com/puppetlabs/control-repo.git', + :git => 'https://github.com/puppetlabs/control-repo.git', :branch => 'production' EOS diff -pruN 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module.rb 3.15.4-1/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module.rb --- 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module.rb 2023-01-19 00:49:17.000000000 +0000 @@ -27,7 +27,8 @@ stdlib_notify_message_regex = /The test puppet_file = <<-PUPPETFILE mod "puppetlabs/motd" mod 'puppetlabs/stdlib', - :git => 'git://github.com/puppetlabs/puppetlabs-stdlib.git' + :git => 'https://github.com/puppetlabs/puppetlabs-stdlib.git', + :tag => 'v7.0.1' PUPPETFILE puppet_file_path = File.join(git_environments_path, 'Puppetfile') diff -pruN 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module_static.rb 3.15.4-1/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module_static.rb --- 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module_static.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module_static.rb 2023-01-19 00:49:17.000000000 +0000 @@ -31,7 +31,8 @@ puppet_file = <<-PUPPETFILE moduledir '#{@module_path}' mod "puppetlabs/motd" mod 'puppetlabs/stdlib', - :git => 'git://github.com/puppetlabs/puppetlabs-stdlib.git' + :git => 'https://github.com/puppetlabs/puppetlabs-stdlib.git', + :tag => 'v7.0.1' PUPPETFILE puppet_file_path = File.join(git_environments_path, 'Puppetfile') diff -pruN 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/multi_source_custom_forge_git_module.rb 3.15.4-1/integration/tests/user_scenario/basic_workflow/multi_source_custom_forge_git_module.rb --- 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/multi_source_custom_forge_git_module.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/tests/user_scenario/basic_workflow/multi_source_custom_forge_git_module.rb 2023-01-19 00:49:17.000000000 +0000 @@ -65,7 +65,7 @@ env_structs = {:production => GitEnv.new '/git_repos_alt/environments_alt.git', '/root/environments_alt', '/root/environments_alt/Puppetfile', - 'mod "puppetlabs/stdlib", :git => "git://github.com/puppetlabs/puppetlabs-stdlib.git"', + 'mod "puppetlabs/stdlib", :git => "https://github.com/puppetlabs/puppetlabs-stdlib.git", :tag => "v7.0.1"', '/root/environments_alt/manifests/site.pp', create_site_pp(master_certname, stage_env_manifest) ), diff -pruN 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module.rb 3.15.4-1/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module.rb --- 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module.rb 2023-01-19 00:49:17.000000000 +0000 @@ -15,7 +15,7 @@ r10k_fqp = get_r10k_fqp(master) #File puppet_file = <<-PUPPETFILE -mod 'broken', :git => 'git://github.com/puppetlabs/puppetlabs-broken' +mod 'broken', :git => 'https://github.com/puppetlabs/puppetlabs-broken' PUPPETFILE puppet_file_path = File.join(git_environments_path, 'Puppetfile') diff -pruN 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module_ref.rb 3.15.4-1/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module_ref.rb --- 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module_ref.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module_ref.rb 2023-01-19 00:49:17.000000000 +0000 @@ -12,7 +12,7 @@ r10k_fqp = get_r10k_fqp(master) #File puppet_file = <<-PUPPETFILE mod 'broken', - :git => 'git://github.com/puppetlabs/puppetlabs-motd', + :git => 'https://github.com/puppetlabs/puppetlabs-motd', :ref => 'does_not_exist' PUPPETFILE diff -pruN 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/negative/neg_specify_deleted_forge_module.rb 3.15.4-1/integration/tests/user_scenario/basic_workflow/negative/neg_specify_deleted_forge_module.rb --- 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/negative/neg_specify_deleted_forge_module.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/tests/user_scenario/basic_workflow/negative/neg_specify_deleted_forge_module.rb 2023-01-19 00:49:17.000000000 +0000 @@ -10,7 +10,7 @@ last_commit = git_last_commit(master, gi r10k_fqp = get_r10k_fqp(master) #Verification -error_notification_regex = /Does 'puppetlabs-regret' have at least one published release?/ +error_notification_regex = /The module puppetlabs-regret does not appear to have any published releases/ #File puppet_file = <<-PUPPETFILE @@ -40,12 +40,6 @@ git_add_commit_push(master, 'production' #Tests step "Deploy production environment via r10k with specified module deleted" -on(master, "#{r10k_fqp} deploy environment -p -v", :acceptable_exit_codes => 1) do |result| - if get_puppet_version(master) < 4.0 - assert_match(error_notification_regex, result.stderr, 'Unexpected error was detected!') - else - expect_failure('expected to fail due to RK-135') do - assert_match(error_notification_regex, result.stderr, 'Unexpected error was detected!') - end - end +on(master, "#{r10k_fqp} deploy environment -p -v --trace", :acceptable_exit_codes => 1) do |result| + assert_match(error_notification_regex, result.stderr, 'Unexpected error was detected!') end diff -pruN 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/single_env_custom_forge_git_module.rb 3.15.4-1/integration/tests/user_scenario/basic_workflow/single_env_custom_forge_git_module.rb --- 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/single_env_custom_forge_git_module.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/tests/user_scenario/basic_workflow/single_env_custom_forge_git_module.rb 2023-01-19 00:49:17.000000000 +0000 @@ -28,7 +28,8 @@ notify_message_regex = /I am in the prod puppet_file = <<-PUPPETFILE mod "puppetlabs/motd" mod 'puppetlabs/inifile', - :git => 'git://github.com/puppetlabs/puppetlabs-inifile' + :git => 'https://github.com/puppetlabs/puppetlabs-inifile', + :tag => 'v5.0.1' PUPPETFILE puppet_file_path = File.join(git_environments_path, 'Puppetfile') diff -pruN 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/single_env_purge_unmanaged_modules.rb 3.15.4-1/integration/tests/user_scenario/basic_workflow/single_env_purge_unmanaged_modules.rb --- 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/single_env_purge_unmanaged_modules.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/tests/user_scenario/basic_workflow/single_env_purge_unmanaged_modules.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,12 +1,15 @@ require 'git_utils' require 'r10k_utils' require 'master_manipulator' -test_name 'CODEMGMT-73 - C63184 - Single Environment Purge Unmanaged Modules' +test_name 'CODEMGMT-78 - Puppetfile Purge --puppetfile & --moduledir flag usage' #Init master_certname = on(master, puppet('config', 'print', 'certname')).stdout.rstrip -git_environments_path = '/root/environments' -last_commit = git_last_commit(master, git_environments_path) +environments_path = on(master, puppet('config', 'print', 'environmentpath')).stdout.strip +moduledir = File.join(environments_path, 'production', 'modules') +puppetfile_path = File.join(environments_path, 'production', 'Puppetfile') +git_remote_environments_path = '/root/environments' +last_commit = git_last_commit(master, git_remote_environments_path) r10k_fqp = get_r10k_fqp(master) #Verification @@ -14,14 +17,12 @@ motd_path = '/etc/motd' motd_contents = 'Hello!' motd_contents_regex = /\A#{motd_contents}\z/ -error_message_regex = /Blah/ - #File -puppet_file = <<-PUPPETFILE +puppetfile = <<-PUPPETFILE mod "puppetlabs/xinetd" PUPPETFILE -puppet_file_path = File.join(git_environments_path, 'Puppetfile') +remote_puppetfile_path = File.join(git_remote_environments_path, 'Puppetfile') #Manifest manifest = <<-MANIFEST @@ -30,12 +31,12 @@ manifest = <<-MANIFEST } MANIFEST -site_pp_path = File.join(git_environments_path, 'manifests', 'site.pp') +remote_site_pp_path = File.join(git_remote_environments_path, 'manifests', 'site.pp') site_pp = create_site_pp(master_certname, manifest) #Teardown teardown do - clean_up_r10k(master, last_commit, git_environments_path) + clean_up_r10k(master, last_commit, git_remote_environments_path) step 'Remove "/etc/motd" File' on(agents, "rm -rf #{motd_path}") @@ -46,27 +47,24 @@ step 'Stub Forge on Master' stub_forge_on(master) step 'Checkout "production" Branch' -git_on(master, 'checkout production', git_environments_path) - -step 'Manually Install the "motd" Module from the Forge' -on(master, puppet('module install puppetlabs-motd')) +git_on(master, 'checkout production', git_remote_environments_path) step 'Create "Puppetfile" for the "production" Environment' -create_remote_file(master, puppet_file_path, puppet_file) +create_remote_file(master, remote_puppetfile_path, puppetfile) step 'Inject New "site.pp" to the "production" Environment' -inject_site_pp(master, site_pp_path, site_pp) +inject_site_pp(master, remote_site_pp_path, site_pp) step 'Push Changes' -git_add_commit_push(master, 'production', 'Update site.pp and add module.', git_environments_path) +git_add_commit_push(master, 'production', 'Update site.pp and add module.', git_remote_environments_path) -#Tests step 'Deploy Environments via r10k' -on(master, "#{r10k_fqp} deploy environment -v -p") +on(master, "#{r10k_fqp} deploy environment --modules --verbose debug --trace") -step 'Plug-in Sync Agents' -on(agents, puppet("plugin download --server #{master}")) +step 'Manually Install the "motd" Module from the Forge' +on(master, puppet("module install puppetlabs-motd --modulepath #{moduledir}")) +#Tests agents.each do |agent| step 'Run Puppet Agent Against "production" Environment' on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => 2) do |result| @@ -80,14 +78,12 @@ agents.each do |agent| end step 'Use r10k to Purge Unmanaged Modules' -on(master, "#{r10k_fqp} puppetfile purge -v", :acceptable_exit_codes => 1) +on(master, "#{r10k_fqp} puppetfile purge --puppetfile #{puppetfile_path} --moduledir #{moduledir} --verbose debug --trace") #Agent will fail because r10k will purge the "motd" module agents.each do |agent| step 'Attempt to Run Puppet Agent' - on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => 0) do |result| - expect_failure('Expected to fail due to CODEMGMT-78') do - assert_match(error_message_regex, result.stderr, 'Expected error was not detected!') - end + on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => 1) do |result| + assert_match(/Could not find declared class motd/, result.stderr, 'Module was not purged') end end diff -pruN 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/single_env_switch_forge_git_module.rb 3.15.4-1/integration/tests/user_scenario/basic_workflow/single_env_switch_forge_git_module.rb --- 3.7.0-2.1/integration/tests/user_scenario/basic_workflow/single_env_switch_forge_git_module.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/tests/user_scenario/basic_workflow/single_env_switch_forge_git_module.rb 2023-01-19 00:49:17.000000000 +0000 @@ -27,7 +27,7 @@ PUPPETFILE puppet_file_git = <<-PUPPETFILE mod "puppetlabs/motd", - :git => 'git://github.com/puppetlabs/puppetlabs-motd', + :git => 'https://github.com/puppetlabs/puppetlabs-motd', :tag => '1.2.0' PUPPETFILE diff -pruN 3.7.0-2.1/integration/tests/user_scenario/complex_workflow/multi_env_add_change_remove.rb 3.15.4-1/integration/tests/user_scenario/complex_workflow/multi_env_add_change_remove.rb --- 3.7.0-2.1/integration/tests/user_scenario/complex_workflow/multi_env_add_change_remove.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/tests/user_scenario/complex_workflow/multi_env_add_change_remove.rb 2023-01-19 00:49:17.000000000 +0000 @@ -24,7 +24,7 @@ stage_env_notify_message = 'This is a di stage_env_notify_message_regex = /#{stage_env_notify_message}/ #Verification for "test" Environment -test_env_error_message_regex = /Error:.*Could not.*environment '?test'?/ +test_env_message_regex = /Environment 'test' not found on server/ #Verification for "temp" Environment test_env_notify_message_regex = /I am in the temp environment/ @@ -157,7 +157,7 @@ agents.each do |agent| end step 'Attempt to Run Puppet Agent Against "test" Environment' - on(agent, puppet('agent', '--test', '--environment test'), :acceptable_exit_codes => 1) do |result| - assert_match(test_env_error_message_regex, result.stderr, 'Expected error was not detected!') + on(agent, puppet('agent', '--test', '--environment test'), :acceptable_exit_codes => 2) do |result| + assert_match(test_env_message_regex, result.stdout, 'Expected message not found!') end end diff -pruN 3.7.0-2.1/integration/tests/user_scenario/complex_workflow/multi_env_remove_re-add.rb 3.15.4-1/integration/tests/user_scenario/complex_workflow/multi_env_remove_re-add.rb --- 3.7.0-2.1/integration/tests/user_scenario/complex_workflow/multi_env_remove_re-add.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/tests/user_scenario/complex_workflow/multi_env_remove_re-add.rb 2023-01-19 00:49:17.000000000 +0000 @@ -15,7 +15,7 @@ initial_env_names = ['production', 'stag #Verification notify_message_regex = /I am in the production environment/ -stage_env_error_message_regex = /Error:.*Could not.*environment '?stage'?/ +stage_env_message_regex = /Environment 'stage' not found on server/ #Manifest site_pp_path = File.join(git_environments_path, 'manifests', 'site.pp') @@ -83,8 +83,8 @@ agents.each do |agent| end step 'Attempt to Run Puppet Agent Against "stage" Environment' - on(agent, puppet('agent', '--test', '--environment stage'), :acceptable_exit_codes => 1) do |result| - assert_match(stage_env_error_message_regex, result.stderr, 'Expected error was not detected!') + on(agent, puppet('agent', '--test', '--environment stage'), :acceptable_exit_codes => 2) do |result| + assert_match(stage_env_message_regex, result.stdout, 'Expected message not found!') end end diff -pruN 3.7.0-2.1/integration/tests/user_scenario/complex_workflow/multi_env_unamanaged.rb 3.15.4-1/integration/tests/user_scenario/complex_workflow/multi_env_unamanaged.rb --- 3.7.0-2.1/integration/tests/user_scenario/complex_workflow/multi_env_unamanaged.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/integration/tests/user_scenario/complex_workflow/multi_env_unamanaged.rb 2023-01-19 00:49:17.000000000 +0000 @@ -22,7 +22,7 @@ site_pp = create_site_pp(master_certname notify_message_prod_env_regex = /I am in the production environment/ notify_message_test_env_regex = /I am in the test environment/ removal_message_test_env_regex = /Removing unmanaged path.*test/ -error_message_regex = /Could not retrieve (catalog from remote server|information from environment test)/ +missing_message_regex = /Environment 'test' not found on server/ #Teardown teardown do @@ -72,7 +72,7 @@ end agents.each do |agent| step 'Run Puppet Agent Against "test" Environment' - on(agent, puppet('agent', '--test', '--environment test'), :acceptable_exit_codes => 1) do |result| - assert_match(error_message_regex, result.stderr, 'Expected message not found!') + on(agent, puppet('agent', '--test', '--environment test'), :acceptable_exit_codes => 2) do |result| + assert_match(missing_message_regex, result.stdout, 'Expected message not found!') end end diff -pruN 3.7.0-2.1/lib/r10k/action/base.rb 3.15.4-1/lib/r10k/action/base.rb --- 3.7.0-2.1/lib/r10k/action/base.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/action/base.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,5 +1,5 @@ -require 'r10k/util/setopts' require 'r10k/logging' +require 'r10k/util/setopts' module R10K module Action @@ -10,6 +10,16 @@ module R10K attr_accessor :settings + # @param opts [Hash] A hash of options defined in #allowed_initialized_opts + # and managed by the SetOps mixin within the Action::Base class. + # Corresponds to the CLI flags and options. + # @param argv [Enumerable] Typically CRI::ArgumentList or Array. A list-like + # collection of the remaining arguments to the CLI invocation (after + # removing flags and options). + # @param settings [Hash] A hash of configuration loaded from the relevant + # config (r10k.yaml). + # + # @note All arguments will be required in the next major version def initialize(opts, argv, settings = {}) @opts = opts @argv = argv diff -pruN 3.7.0-2.1/lib/r10k/action/deploy/deploy_helpers.rb 3.15.4-1/lib/r10k/action/deploy/deploy_helpers.rb --- 3.7.0-2.1/lib/r10k/action/deploy/deploy_helpers.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/action/deploy/deploy_helpers.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,8 +1,12 @@ +require 'r10k/logging' + module R10K module Action module Deploy module DeployHelpers + include R10K::Logging + # Ensure that a config file has been found (and presumably loaded) and exit # with a helpful error if it hasn't. # diff -pruN 3.7.0-2.1/lib/r10k/action/deploy/display.rb 3.15.4-1/lib/r10k/action/deploy/display.rb --- 3.7.0-2.1/lib/r10k/action/deploy/display.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/action/deploy/display.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,6 +1,6 @@ -require 'r10k/deployment' require 'r10k/action/base' require 'r10k/action/deploy/deploy_helpers' +require 'r10k/deployment' module R10K module Action @@ -9,17 +9,48 @@ module R10K include R10K::Action::Deploy::DeployHelpers + # @param opts [Hash] A hash of options defined in #allowed_initialized_opts + # and managed by the SetOps mixin within the Action::Base class. + # Corresponds to the CLI flags and options. + # @param argv [Enumerable] Typically CRI::ArgumentList or Array. A list-like + # collection of the remaining arguments to the CLI invocation (after + # removing flags and options). + # @param settings [Hash] A hash of configuration loaded from the relevant + # config (r10k.yaml). + # + # @note All arguments will be required in the next major version + def initialize(opts, argv, settings = {}) + super + + @settings = @settings.merge({ + overrides: { + environments: { + preload_environments: @fetch, + requested_environments: @argv.map { |arg| arg.gsub(/\W/, '_') } + }, + modules: {}, + output: { + format: @format, + trace: @trace, + detail: @detail + }, + purging: {} + } + }) + end + def call expect_config! deployment = R10K::Deployment.new(@settings) - if @fetch + if @settings.dig(:overrides, :environments, :preload_environments) deployment.preload! + deployment.validate! end - output = { :sources => deployment.sources.map { |source| source_info(source, @argv) } } + output = { :sources => deployment.sources.map { |source| source_info(source, @settings.dig(:overrides, :environments, :requested_environments)) } } - case @format + case @settings.dig(:overrides, :output, :format) when 'json' then json_format(output) else yaml_format(output) end @@ -27,7 +58,7 @@ module R10K # exit 0 true rescue => e - logger.error R10K::Errors::Formatting.format_exception(e, @trace) + logger.error R10K::Errors::Formatting.format_exception(e, @settings.dig(:overrides, :output, :trace)) false end @@ -43,7 +74,7 @@ module R10K puts output.to_yaml end - def source_info(source, argv=[]) + def source_info(source, requested_environments = []) source_info = { :name => source.name, :basedir => source.basedir, @@ -52,28 +83,30 @@ module R10K source_info[:prefix] = source.prefix if source.prefix source_info[:remote] = source.remote if source.respond_to?(:remote) - env_list = source.environments.select { |env| argv.empty? || argv.include?(env.name) } + select_all_envs = requested_environments.empty? + env_list = source.environments.select { |env| select_all_envs || requested_environments.include?(env.name) } source_info[:environments] = env_list.map { |env| environment_info(env) } source_info end def environment_info(env) - if !@puppetfile && !@detail + modules = @settings.dig(:overrides, :environments, :deploy_modules) + if !modules && !@settings.dig(:overrides, :output, :detail) env.dirname else env_info = env.info.merge({ :status => (env.status rescue nil), }) - env_info[:modules] = env.modules.map { |mod| module_info(mod) } if @puppetfile + env_info[:modules] = env.modules.map { |mod| module_info(mod) } if modules env_info end end def module_info(mod) - if @detail + if @settings.dig(:overrides, :output, :detail) { :name => mod.title, :properties => mod.properties } else mod.title @@ -81,7 +114,13 @@ module R10K end def allowed_initialize_opts - super.merge(puppetfile: :self, detail: :self, format: :self, fetch: :self) + super.merge({ + puppetfile: :modules, + modules: :self, + detail: :self, + format: :self, + fetch: :self + }) end end end diff -pruN 3.7.0-2.1/lib/r10k/action/deploy/environment.rb 3.15.4-1/lib/r10k/action/deploy/environment.rb --- 3.7.0-2.1/lib/r10k/action/deploy/environment.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/action/deploy/environment.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,9 +1,9 @@ -require 'r10k/util/setopts' -require 'r10k/deployment' -require 'r10k/logging' -require 'r10k/action/visitor' require 'r10k/action/base' require 'r10k/action/deploy/deploy_helpers' +require 'r10k/action/visitor' +require 'r10k/deployment' +require 'r10k/util/setopts' + require 'json' module R10K @@ -12,46 +12,105 @@ module R10K class Environment < R10K::Action::Base include R10K::Action::Deploy::DeployHelpers + include R10K::Action::Visitor + # Deprecated attr_reader :force - def initialize(opts, argv, settings = nil) - settings ||= {} - @purge_levels = settings.fetch(:deploy, {}).fetch(:purge_levels, []) - @user_purge_whitelist = settings.fetch(:deploy, {}).fetch(:purge_whitelist, []) - @generate_types = settings.fetch(:deploy, {}).fetch(:generate_types, false) + attr_reader :settings + # @param opts [Hash] A hash of options defined in #allowed_initialized_opts + # and managed by the SetOps mixin within the Action::Base class. + # Corresponds to the CLI flags and options. + # @param argv [Enumerable] Typically CRI::ArgumentList or Array. A list-like + # collection of the remaining arguments to the CLI invocation (after + # removing flags and options). + # @param settings [Hash] A hash of configuration loaded from the relevant + # config (r10k.yaml). + # + # @note All arguments will be required in the next major version + def initialize(opts, argv, settings = {}) super - # @force here is used to make it easier to reason about - @force = !@no_force - @argv = @argv.map { |arg| arg.gsub(/\W/,'_') } + # instance variables below are set by the super class based on the + # spec of #allowed_initialize_opts and any command line flags. This + # gives a preference order of cli flags > config files > defaults. + @settings = @settings.merge({ + overrides: { + environments: { + requested_environments: @argv.map { |arg| arg.gsub(/\W/,'_') }, + default_branch_override: @default_branch_override, + generate_types: @generate_types || settings.dig(:deploy, :generate_types) || false, + preload_environments: true, + incremental: @incremental + }, + modules: { + exclude_spec: settings.dig(:deploy, :exclude_spec), + requested_modules: [], + deploy_modules: @modules, + pool_size: @settings[:pool_size] || 4, + force: !@no_force, # force here is used to make it easier to reason about + }, + purging: { + purge_levels: settings.dig(:deploy, :purge_levels) || [], + purge_allowlist: read_purge_allowlist(settings.dig(:deploy, :purge_whitelist) || [], + settings.dig(:deploy, :purge_allowlist) || []) + }, + forge: { + allow_puppetfile_override: settings.dig(:forge, :allow_puppetfile_override) || false + }, + output: {} + } + }) end def call @visit_ok = true - expect_config! - deployment = R10K::Deployment.new(@settings) - check_write_lock!(@settings) + begin + expect_config! + deployment = R10K::Deployment.new(@settings) + check_write_lock!(@settings) + + deployment.accept(self) + rescue => e + @visit_ok = false + logger.error R10K::Errors::Formatting.format_exception(e, @trace) + end - deployment.accept(self) @visit_ok end - include R10K::Action::Visitor - private + def read_purge_allowlist (whitelist, allowlist) + whitelist_has_content = !whitelist.empty? + allowlist_has_content = !allowlist.empty? + case + when whitelist_has_content == false && allowlist_has_content == false + [] + when whitelist_has_content && allowlist_has_content + raise R10K::Error.new "Values found for both purge_whitelist and purge_allowlist. Setting " << + "purge_whitelist is deprecated, please only use purge_allowlist." + when allowlist_has_content + allowlist + else + logger.warn "Setting purge_whitelist is deprecated; please use purge_allowlist instead." + whitelist + end + end + def visit_deployment(deployment) # Ensure that everything can be preloaded. If we cannot preload all # sources then we can't fully enumerate all environments which # could be dangerous. If this fails then an exception will be raised # and execution will be halted. - deployment.preload! - deployment.validate! + if @settings.dig(:overrides, :environments, :preload_environments) + deployment.preload! + deployment.validate! + end - undeployable = undeployable_environment_names(deployment.environments, @argv) + undeployable = undeployable_environment_names(deployment.environments, @settings.dig(:overrides, :environments, :requested_environments)) if !undeployable.empty? @visit_ok = false logger.error _("Environment(s) \'%{environments}\' cannot be found in any source and will not be deployed.") % {environments: undeployable.join(", ")} @@ -59,17 +118,22 @@ module R10K yield - if @purge_levels.include?(:deployment) + if @settings.dig(:overrides, :purging, :purge_levels).include?(:deployment) logger.debug("Purging unmanaged environments for deployment...") + deployment.sources.each do |source| + source.reload! + end deployment.purge! end ensure if (postcmd = @settings[:postrun]) if postcmd.grep('$modifiedenvs').any? envs = deployment.environments.map { |e| e.dirname } - envs.reject! { |e| !@argv.include?(e) } if @argv.any? + requested_envs = @settings.dig(:overrides, :environments, :requested_environments) + envs.reject! { |e| !requested_envs.include?(e) } if requested_envs.any? postcmd = postcmd.map { |e| e.gsub('$modifiedenvs', envs.join(' ')) } end + logger.debug _("Executing postrun command.") subproc = R10K::Util::Subprocess.new(postcmd) subproc.logger = logger subproc.execute @@ -81,7 +145,8 @@ module R10K end def visit_environment(environment) - if !(@argv.empty? || @argv.any? { |name| environment.dirname == name }) + requested_envs = @settings.dig(:overrides, :environments, :requested_environments) + if !(requested_envs.empty? || requested_envs.any? { |name| environment.dirname == name }) logger.debug1(_("Environment %{env_dir} does not match environment name filter, skipping") % {env_dir: environment.dirname}) return end @@ -95,28 +160,31 @@ module R10K environment.sync logger.info _("Environment %{env_dir} is now at %{env_signature}") % {env_dir: environment.dirname, env_signature: environment.signature} - if status == :absent || @puppetfile + if status == :absent || @settings.dig(:overrides, :modules, :deploy_modules) if status == :absent logger.debug(_("Environment %{env_dir} is new, updating all modules") % {env_dir: environment.dirname}) end previous_ok = @visit_ok @visit_ok = true - yield + + environment.deploy + @environment_ok = @visit_ok @visit_ok &&= previous_ok end - if @purge_levels.include?(:environment) + + if @settings.dig(:overrides, :purging, :purge_levels).include?(:environment) if @visit_ok logger.debug("Purging unmanaged content for environment '#{environment.dirname}'...") - environment.purge!(:recurse => true, :whitelist => environment.whitelist(@user_purge_whitelist)) + environment.purge!(:recurse => true, :whitelist => environment.whitelist(@settings.dig(:overrides, :purging, :purge_allowlist))) else logger.debug("Not purging unmanaged content for environment '#{environment.dirname}' due to prior deploy failures.") end end - if @generate_types + if @settings.dig(:overrides, :environments, :generate_types) if @environment_ok logger.debug("Generating puppet types for environment '#{environment.dirname}'...") environment.generate_types! @@ -128,34 +196,21 @@ module R10K write_environment_info!(environment, started_at, @visit_ok) end - def visit_puppetfile(puppetfile) - puppetfile.load(@opts[:'default-branch-override']) - - yield - - if @purge_levels.include?(:puppetfile) - logger.debug("Purging unmanaged Puppetfile content for environment '#{puppetfile.environment.dirname}'...") - puppetfile.purge! - end - end - - def visit_module(mod) - logger.info _("Deploying %{origin} content %{path}") % {origin: mod.origin, path: mod.path} - mod.sync(force: @force) - end - def write_environment_info!(environment, started_at, success) - module_deploys = [] - begin - environment.modules.each do |mod| - name = mod.name - version = mod.version - sha = mod.repo.head rescue nil - module_deploys.push({:name => name, :version => version, :sha => sha}) + module_deploys = + begin + environment.modules.map do |mod| + props = mod.properties + { + name: mod.name, + version: props[:expected], + sha: props[:type] == :git ? props[:actual] : nil + } + end + rescue + logger.debug("Unable to get environment module deploy data for .r10k-deploy.json at #{environment.path}") + [] end - rescue - logger.debug("Unable to get environment module deploy data for .r10k-deploy.json at #{environment.path}") - end # make this file write as atomic as possible in pure ruby final = "#{environment.path}/.r10k-deploy.json" @@ -183,13 +238,21 @@ module R10K end def allowed_initialize_opts - super.merge(puppetfile: :self, + super.merge(puppetfile: :modules, + modules: :self, cachedir: :self, + incremental: :self, 'no-force': :self, + 'exclude-spec': :self, 'generate-types': :self, 'puppet-path': :self, 'puppet-conf': :self, - 'default-branch-override': :self) + 'private-key': :self, + 'oauth-token': :self, + 'default-branch-override': :self, + 'github-app-id': :self, + 'github-app-key': :self, + 'github-app-ttl': :self) end end end diff -pruN 3.7.0-2.1/lib/r10k/action/deploy/module.rb 3.15.4-1/lib/r10k/action/deploy/module.rb --- 3.7.0-2.1/lib/r10k/action/deploy/module.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/action/deploy/module.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,7 +1,7 @@ -require 'r10k/deployment' -require 'r10k/action/visitor' require 'r10k/action/base' require 'r10k/action/deploy/deploy_helpers' +require 'r10k/action/visitor' +require 'r10k/deployment' module R10K module Action @@ -9,35 +9,88 @@ module R10K class Module < R10K::Action::Base include R10K::Action::Deploy::DeployHelpers + include R10K::Action::Visitor + # Deprecated attr_reader :force - def initialize(opts, argv, settings = nil) - settings ||= {} + attr_reader :settings + # @param opts [Hash] A hash of options defined in #allowed_initialized_opts + # and managed by the SetOps mixin within the Action::Base class. + # Corresponds to the CLI flags and options. + # @param argv [Enumerable] Typically CRI::ArgumentList or Array. A list-like + # collection of the remaining arguments to the CLI invocation (after + # removing flags and options). + # @param settings [Hash] A hash of configuration loaded from the relevant + # config (r10k.yaml). + # + # @note All arguments will be required in the next major version + def initialize(opts, argv, settings = {}) super - # @force here is used to make it easier to reason about - @force = !@no_force + requested_env = @opts[:environment] ? [@opts[:environment].gsub(/\W/, '_')] : [] + @modified_envs = [] + + @settings = @settings.merge({ + overrides: { + environments: { + requested_environments: requested_env, + generate_types: @generate_types + }, + modules: { + exclude_spec: settings.dig(:deploy, :exclude_spec), + pool_size: @settings[:pool_size] || 4, + requested_modules: @argv.map.to_a, + # force here is used to make it easier to reason about + force: !@no_force + }, + forge: { + allow_puppetfile_override: settings.dig(:forge, :allow_puppetfile_override) || false + }, + purging: {}, + output: {} + } + }) end def call @visit_ok = true + begin + expect_config! + deployment = R10K::Deployment.new(@settings) + check_write_lock!(@settings) + + deployment.accept(self) + rescue => e + @visit_ok = false + logger.error R10K::Errors::Formatting.format_exception(e, @trace) + end - expect_config! - deployment = R10K::Deployment.new(@settings) - check_write_lock!(@settings) - - deployment.accept(self) @visit_ok end - include R10K::Action::Visitor - private def visit_deployment(deployment) yield + ensure + if (postcmd = @settings[:postrun]) + if @modified_envs.any? + envs_to_run = @modified_envs.join(' ') + logger.debug _("Running postrun command for environments: %{envs_to_run}.") % { envs_to_run: envs_to_run } + + if postcmd.grep('$modifiedenvs').any? + postcmd = postcmd.map { |e| e.gsub('$modifiedenvs', envs_to_run) } + end + + subproc = R10K::Util::Subprocess.new(postcmd) + subproc.logger = logger + subproc.execute + else + logger.debug _("No environments were modified, not executing postrun command.") + end + end end def visit_source(source) @@ -45,39 +98,40 @@ module R10K end def visit_environment(environment) - if @opts[:environment] && (@opts[:environment] != environment.dirname) - logger.debug1(_("Only updating modules in environment %{opt_env} skipping environment %{env_path}") % {opt_env: @opts[:environment], env_path: environment.path}) + requested_envs = @settings.dig(:overrides, :environments, :requested_environments) + if !requested_envs.empty? && !requested_envs.include?(environment.dirname) + logger.debug1(_("Only updating modules in environment(s) %{opt_env} skipping environment %{env_path}") % {opt_env: requested_envs.inspect, env_path: environment.path}) else - logger.debug1(_("Updating modules %{modules} in environment %{env_path}") % {modules: @argv.inspect, env_path: environment.path}) - yield - end - end + logger.debug1(_("Updating modules %{modules} in environment %{env_path}") % {modules: @settings.dig(:overrides, :modules, :requested_modules).inspect, env_path: environment.path}) - def visit_puppetfile(puppetfile) - puppetfile.load - yield - end + updated_modules = environment.deploy - def visit_module(mod) - if @argv.include?(mod.name) - logger.info _("Deploying module %{mod_path}") % {mod_path: mod.path} - mod.sync(force: @force) - if mod.environment && @generate_types - logger.debug("Generating puppet types for environment '#{mod.environment.dirname}'...") - mod.environment.generate_types! + # We actually synced a module in this env + if !updated_modules.nil? && !updated_modules.empty? + # Record modified environment for postrun command + @modified_envs << environment.dirname + + if generate_types = @settings.dig(:overrides, :environments, :generate_types) + logger.debug("Generating puppet types for environment '#{environment.dirname}'...") + environment.generate_types! + end end - else - logger.debug1(_("Only updating modules %{modules}, skipping module %{mod_name}") % {modules: @argv.inspect, mod_name: mod.name}) end end def allowed_initialize_opts super.merge(environment: true, cachedir: :self, + 'exclude-spec': :self, 'no-force': :self, 'generate-types': :self, 'puppet-path': :self, - 'puppet-conf': :self) + 'puppet-conf': :self, + 'private-key': :self, + 'oauth-token': :self, + 'github-app-id': :self, + 'github-app-key': :self, + 'github-app-ttl': :self) end end end diff -pruN 3.7.0-2.1/lib/r10k/action/puppetfile/check.rb 3.15.4-1/lib/r10k/action/puppetfile/check.rb --- 3.7.0-2.1/lib/r10k/action/puppetfile/check.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/action/puppetfile/check.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,6 +1,6 @@ -require 'r10k/puppetfile' require 'r10k/action/base' require 'r10k/errors/formatting' +require 'r10k/module_loader/puppetfile' module R10K module Action @@ -8,9 +8,13 @@ module R10K class Check < R10K::Action::Base def call - pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile) + options = { basedir: @root } + options[:moduledir] = @moduledir if @moduledir + options[:puppetfile] = @puppetfile if @puppetfile + + loader = R10K::ModuleLoader::Puppetfile.new(**options) begin - pf.load! + loader.load! $stderr.puts _("Syntax OK") true rescue => e diff -pruN 3.7.0-2.1/lib/r10k/action/puppetfile/install.rb 3.15.4-1/lib/r10k/action/puppetfile/install.rb --- 3.7.0-2.1/lib/r10k/action/puppetfile/install.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/action/puppetfile/install.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,7 +1,8 @@ -require 'r10k/puppetfile' -require 'r10k/errors/formatting' -require 'r10k/action/visitor' require 'r10k/action/base' +require 'r10k/content_synchronizer' +require 'r10k/errors/formatting' +require 'r10k/module_loader/puppetfile' +require 'r10k/util/cleaner' module R10K module Action @@ -9,35 +10,38 @@ module R10K class Install < R10K::Action::Base def call - @visit_ok = true - pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile, nil , @force) - pf.accept(self) - @visit_ok + begin + options = { basedir: @root, overrides: { force: @force || false } } + options[:moduledir] = @moduledir if @moduledir + options[:puppetfile] = @puppetfile if @puppetfile + options[:module_exclude_regex] = @module_exclude_regex if @module_exclude_regex + + loader = R10K::ModuleLoader::Puppetfile.new(**options) + loaded_content = loader.load! + + pool_size = @settings[:pool_size] || 4 + modules = loaded_content[:modules] + if pool_size > 1 + R10K::ContentSynchronizer.concurrent_sync(modules, pool_size, logger) + else + R10K::ContentSynchronizer.serial_sync(modules, logger) + end + + R10K::Util::Cleaner.new(loaded_content[:managed_directories], + loaded_content[:desired_contents], + loaded_content[:purge_exclusions]).purge! + + true + rescue => e + logger.error R10K::Errors::Formatting.format_exception(e, @trace) + false + end end private - include R10K::Action::Visitor - - def visit_puppetfile(pf) - pf.load! - yield - pf.purge! - end - - def visit_module(mod) - @force ||= false - logger.info _("Updating module %{mod_path}") % {mod_path: mod.path} - - if mod.respond_to?(:desired_ref) && mod.desired_ref == :control_branch - logger.warn _("Cannot track control repo branch for content '%{name}' when not part of a 'deploy' action, will use default if available." % {name: mod.name}) - end - - mod.sync(force: @force) - end - def allowed_initialize_opts - super.merge(root: :self, puppetfile: :self, moduledir: :self, force: :self ) + super.merge(root: :self, puppetfile: :self, moduledir: :self, :'module-exclude-regex' => :self, force: :self ) end end end diff -pruN 3.7.0-2.1/lib/r10k/action/puppetfile/purge.rb 3.15.4-1/lib/r10k/action/puppetfile/purge.rb --- 3.7.0-2.1/lib/r10k/action/puppetfile/purge.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/action/puppetfile/purge.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,6 +1,7 @@ -require 'r10k/puppetfile' require 'r10k/action/base' require 'r10k/errors/formatting' +require 'r10k/module_loader/puppetfile' +require 'r10k/util/cleaner' module R10K module Action @@ -8,9 +9,17 @@ module R10K class Purge < R10K::Action::Base def call - pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile) - pf.load! - pf.purge! + options = { basedir: @root } + + options[:moduledir] = @moduledir if @moduledir + options[:puppetfile] = @puppetfile if @puppetfile + + loader = R10K::ModuleLoader::Puppetfile.new(**options) + loaded_content = loader.load! + R10K::Util::Cleaner.new(loaded_content[:managed_directories], + loaded_content[:desired_contents], + loaded_content[:purge_exclusions]).purge! + true rescue => e logger.error R10K::Errors::Formatting.format_exception(e, @trace) diff -pruN 3.7.0-2.1/lib/r10k/action/runner.rb 3.15.4-1/lib/r10k/action/runner.rb --- 3.7.0-2.1/lib/r10k/action/runner.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/action/runner.rb 2023-01-19 00:49:17.000000000 +0000 @@ -44,10 +44,19 @@ module R10K overrides = {} overrides[:cachedir] = @opts[:cachedir] if @opts.key?(:cachedir) - overrides[:deploy] = {} if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types') - overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path') - overrides[:deploy][:puppet_conf] = @opts[:'puppet-conf'] unless @opts[:'puppet-conf'].nil? - overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types') + if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types') || @opts.key?(:'exclude-spec') || @opts.key?(:'puppet-conf') + overrides[:deploy] = {} + overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path') + overrides[:deploy][:puppet_conf] = @opts[:'puppet-conf'] if @opts.key?(:'puppet-conf') + overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types') + overrides[:deploy][:exclude_spec] = @opts[:'exclude-spec'] if @opts.key?(:'exclude-spec') + end + # If the log level has been given as an argument, ensure that output happens on stderr + if @opts.key?(:loglevel) + overrides[:logging] = {} + overrides[:logging][:level] = @opts[:loglevel] + overrides[:logging][:disable_default_stderr] = false + end with_overrides = config_settings.merge(overrides) do |key, oldval, newval| newval = oldval.merge(newval) if oldval.is_a? Hash @@ -55,6 +64,10 @@ module R10K newval end + # Credentials from the CLI override both the global and per-repo + # credentials from the config, and so need to be handled specially + with_overrides = add_credential_overrides(with_overrides) + @settings = R10K::Settings.global_settings.evaluate(with_overrides) R10K::Initializers::GlobalInitializer.new(@settings).call @@ -63,15 +76,20 @@ module R10K exit(8) end + # Set up authorization from license file if it wasn't + # already set via the config def setup_authorization - begin - license = R10K::Util::License.load - - if license.respond_to?(:authorization_token) - PuppetForge::Connection.authorization = license.authorization_token + if PuppetForge::Connection.authorization.nil? + begin + license = R10K::Util::License.load + + if license.respond_to?(:authorization_token) + logger.debug "Using token from license to connect to the Forge." + PuppetForge::Connection.authorization = license.authorization_token + end + rescue R10K::Error => e + logger.warn e.message end - rescue R10K::Error => e - logger.warn e.message end end @@ -92,6 +110,62 @@ module R10K results end + + def add_credential_overrides(overrides) + sshkey_path = @opts[:'private-key'] + token_path = @opts[:'oauth-token'] + app_id = @opts[:'github-app-id'] + app_private_key_path = @opts[:'github-app-key'] + app_ttl = @opts[:'github-app-ttl'] + + if sshkey_path && token_path + raise R10K::Error, "Cannot specify both an SSH key and a token to use with this deploy." + end + + if sshkey_path && (app_private_key_path || app_id) + raise R10K::Error, "Cannot specify both an SSH key and an SSL key or Github App id to use with this deploy." + end + + if token_path && (app_private_key_path || app_id) + raise R10K::Error, "Cannot specify both an OAuth token and an SSL key or Github App id to use with this deploy." + end + + if app_id && ! app_private_key_path || app_private_key_path && ! app_id + raise R10K::Error, "Must specify both id and SSL private key to use Github App for this deploy." + end + + if sshkey_path + overrides[:git] ||= {} + overrides[:git][:private_key] = sshkey_path + if repo_settings = overrides[:git][:repositories] + repo_settings.each do |repo| + repo[:private_key] = sshkey_path + end + end + elsif token_path + overrides[:git] ||= {} + overrides[:git][:oauth_token] = token_path + if repo_settings = overrides[:git][:repositories] + repo_settings.each do |repo| + repo[:oauth_token] = token_path + end + end + elsif app_id + overrides[:git] ||= {} + overrides[:git][:github_app_id] = app_id + overrides[:git][:github_app_key] = app_private_key_path + overrides[:git][:github_app_ttl] = app_ttl + if repo_settings = overrides[:git][:repositories] + repo_settings.each do |repo| + repo[:github_app_id] = app_id + repo[:github_app_key] = app_private_key_path + repo[:github_app_ttl] = app_ttl + end + end + end + + overrides + end end end end diff -pruN 3.7.0-2.1/lib/r10k/action/visitor.rb 3.15.4-1/lib/r10k/action/visitor.rb --- 3.7.0-2.1/lib/r10k/action/visitor.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/action/visitor.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,4 +1,5 @@ require 'r10k/errors/formatting' +require 'r10k/logging' module R10K module Action @@ -13,6 +14,8 @@ module R10K # @api private module Visitor + include R10K::Logging + # Dispatch to the type specific visitor method # # @param type [Symbol] The object type to dispatch for diff -pruN 3.7.0-2.1/lib/r10k/cli/deploy.rb 3.15.4-1/lib/r10k/cli/deploy.rb --- 3.7.0-2.1/lib/r10k/cli/deploy.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/cli/deploy.rb 2023-01-19 00:49:17.000000000 +0000 @@ -21,9 +21,10 @@ module R10K::CLI (https://puppet.com/docs/puppet/latest/environments_about.html). DESCRIPTION - required nil, :cachedir, 'Specify a cachedir, overriding the value in config' + option nil, :cachedir, 'Specify a cachedir, overriding the value in config', argument: :required flag nil, :'no-force', 'Prevent the overwriting of local module modifications' flag nil, :'generate-types', 'Run `puppet generate types` after updating an environment' + flag nil, :'exclude-spec', 'Exclude the module\'s spec dir from deployment' option nil, :'puppet-path', 'Path to puppet executable', argument: :required do |value, cmd| unless File.executable? value $stderr.puts "The specified puppet executable #{value} is not executable." @@ -32,6 +33,11 @@ module R10K::CLI end end option nil, :'puppet-conf', 'Path to puppet.conf', argument: :required + option nil, :'private-key', 'Path to SSH key to use when cloning. Only valid with rugged provider', argument: :required + option nil, :'oauth-token', 'Path to OAuth token to use when cloning. Only valid with rugged provider', argument: :required + option nil, :'github-app-id', 'Github App id. Only valid with rugged provider', argument: :required + option nil, :'github-app-key', 'Github App private key. Only valid with rugged provider', argument: :required + option nil, :'github-app-ttl', 'Github App token expiration, in seconds. Only valid with rugged provider', default: "120", argument: :optional run do |opts, args, cmd| puts cmd.help(:verbose => opts[:verbose]) @@ -53,7 +59,7 @@ branches. Environments can provide a Puppetfile at the root of the directory to deploy independent Puppet modules. To recursively deploy an environment, pass the -`--puppetfile` flag to the command. +`--modules` flag to the command. **NOTE**: If an environment has a Puppetfile when it is instantiated a recursive update will be forced. It is assumed that environments are dependent @@ -61,8 +67,11 @@ on modules specified in the Puppetfile a scheduled. On subsequent deployments, Puppetfile deployment will default to off. DESCRIPTION - flag :p, :puppetfile, 'Deploy modules from a puppetfile' - required nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile' + flag :p, :puppetfile, 'Deploy modules (deprecated, use -m)' + flag :m, :modules, 'Deploy modules' + flag nil, :incremental, 'Used with the --modules flag, only update those modules whose definition has changed or whose definition allows the version to float' + option nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile', + argument: :required runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Environment) end @@ -82,7 +91,7 @@ It will load the Puppetfile configuratio try to deploy the given module names in all environments. DESCRIPTION - required :e, :environment, 'Update the modules in the given environment' + option :e, :environment, 'Update the modules in the given environment', argument: :required runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Module) end @@ -97,10 +106,12 @@ try to deploy the given module names in usage 'display' summary 'Display environments and modules in the deployment' - flag :p, :puppetfile, 'Display Puppetfile modules' + flag :p, :puppetfile, 'Display modules (deprecated, use -m)' + flag :m, :modules, 'Display modules' flag nil, :detail, 'Display detailed information' flag nil, :fetch, 'Update available environment lists from all remote sources' - required nil, :format, 'Display output in a specific format. Valid values: json, yaml. Default: yaml' + option nil, :format, 'Display output in a specific format. Valid values: json, yaml. Default: yaml', + argument: :required runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Display) end diff -pruN 3.7.0-2.1/lib/r10k/cli/puppetfile.rb 3.15.4-1/lib/r10k/cli/puppetfile.rb --- 3.7.0-2.1/lib/r10k/cli/puppetfile.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/cli/puppetfile.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,5 +1,4 @@ require 'r10k/cli' -require 'r10k/puppetfile' require 'r10k/action/puppetfile' require 'cri' @@ -30,8 +29,9 @@ Puppetfile (http://bombasticmonkey.com/l name 'install' usage 'install' summary 'Install all modules from a Puppetfile' - required nil, :moduledir, 'Path to install modules to' - required nil, :puppetfile, 'Path to puppetfile' + option nil, :moduledir, 'Path to install modules to', argument: :required + option nil, :puppetfile, 'Path to puppetfile', argument: :required + option nil, :'module-exclude-regex', 'A regex to exclude modules from installation. Helpful in CI environments.', argument: :required flag nil, :force, 'Force locally changed files to be overwritten' runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Install) end @@ -45,7 +45,7 @@ Puppetfile (http://bombasticmonkey.com/l usage 'check' summary 'Try and load the Puppetfile to verify the syntax is correct.' - required nil, :puppetfile, 'Path to Puppetfile' + option nil, :puppetfile, 'Path to Puppetfile', argument: :required runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Check) end end @@ -58,8 +58,8 @@ Puppetfile (http://bombasticmonkey.com/l usage 'purge' summary 'Purge unmanaged modules from a Puppetfile managed directory' - required nil, :moduledir, 'Path to install modules to' - required nil, :puppetfile, 'Path to Puppetfile' + option nil, :moduledir, 'Path to install modules to', argument: :required + option nil, :puppetfile, 'Path to Puppetfile', argument: :required runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Purge) end end diff -pruN 3.7.0-2.1/lib/r10k/content_synchronizer.rb 3.15.4-1/lib/r10k/content_synchronizer.rb --- 3.7.0-2.1/lib/r10k/content_synchronizer.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/lib/r10k/content_synchronizer.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,95 @@ +module R10K + module ContentSynchronizer + + def self.serial_accept(modules, visitor, loader) + visitor.visit(:puppetfile, loader) do + serial_sync(modules) + end + end + + def self.serial_sync(modules) + updated_modules = [] + modules.each do |mod| + updated = mod.sync + updated_modules << mod.name if updated + end + updated_modules + end + + # Returns a Queue of the names of modules actually updated + def self.concurrent_accept(modules, visitor, loader, pool_size, logger) + mods_queue = modules_visit_queue(modules, visitor, loader) + sync_queue(mods_queue, pool_size, logger) + end + + # Returns a Queue of the names of modules actually updated + def self.concurrent_sync(modules, pool_size, logger) + mods_queue = modules_sync_queue(modules) + sync_queue(mods_queue, pool_size, logger) + end + + # Returns a Queue of the names of modules actually updated + def self.sync_queue(mods_queue, pool_size, logger) + logger.debug _("Updating modules with %{pool_size} threads") % {pool_size: pool_size} + updated_modules = Queue.new + thread_pool = pool_size.times.map { sync_thread(mods_queue, logger, updated_modules) } + thread_exception = nil + + # If any threads raise an exception the deployment is considered a failure. + # In that event clear the queue, wait for other threads to finish their + # current work, then re-raise the first exception caught. + begin + thread_pool.each(&:join) + # Return the list of all modules that were actually updated + updated_modules + rescue => e + logger.error _("Error during concurrent deploy of a module: %{message}") % {message: e.message} + mods_queue.clear + thread_exception ||= e + retry + ensure + raise thread_exception unless thread_exception.nil? + end + end + + def self.modules_visit_queue(modules, visitor, loader) + Queue.new.tap do |queue| + visitor.visit(:puppetfile, loader) do + enqueue_modules(queue, modules) + end + end + end + + def self.modules_sync_queue(modules) + Queue.new.tap do |queue| + enqueue_modules(queue, modules) + end + end + + def self.enqueue_modules(queue, modules) + modules_by_cachedir = modules.group_by { |mod| mod.cachedir } + modules_without_vcs_cachedir = modules_by_cachedir.delete(:none) || [] + + modules_without_vcs_cachedir.each {|mod| queue << Array(mod) } + modules_by_cachedir.values.each {|mods| queue << mods } + end + + def self.sync_thread(mods_queue, logger, updated_modules) + Thread.new do + begin + while mods = mods_queue.pop(true) do + mods.each do |mod| + updated = mod.sync + updated_modules << mod.name if updated + end + end + rescue ThreadError => e + logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id} + Thread.exit + rescue => e + Thread.main.raise(e) + end + end + end + end +end diff -pruN 3.7.0-2.1/lib/r10k/deployment.rb 3.15.4-1/lib/r10k/deployment.rb --- 3.7.0-2.1/lib/r10k/deployment.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/deployment.rb 2023-01-19 00:49:17.000000000 +0000 @@ -118,7 +118,7 @@ module R10K raise R10K::Error, _("Unable to load sources; the supplied configuration does not define the 'sources' key") end @_sources = sources.map do |(name, hash)| - R10K::Source.from_hash(name, hash) + R10K::Source.from_hash(name, hash.merge({overrides: @config[:overrides]})) end end diff -pruN 3.7.0-2.1/lib/r10k/environment/bare.rb 3.15.4-1/lib/r10k/environment/bare.rb --- 3.7.0-2.1/lib/r10k/environment/bare.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/environment/bare.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,13 +1,10 @@ -class R10K::Environment::Bare < R10K::Environment::WithModules +class R10K::Environment::Bare < R10K::Environment::Plain R10K::Environment.register(:bare, self) - def sync - path.mkpath - end - - def status - :not_applicable + def initialize(name, basedir, dirname, options = {}) + logger.warn _('"bare" environment type is deprecated; please use "plain" instead (environment: %{name})') % {name: name} + super end def signature diff -pruN 3.7.0-2.1/lib/r10k/environment/base.rb 3.15.4-1/lib/r10k/environment/base.rb --- 3.7.0-2.1/lib/r10k/environment/base.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/environment/base.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,3 +1,7 @@ +require 'r10k/content_synchronizer' +require 'r10k/logging' +require 'r10k/module_loader/puppetfile' +require 'r10k/util/cleaner' require 'r10k/util/subprocess' # This class defines a common interface for environment implementations. @@ -5,6 +9,8 @@ require 'r10k/util/subprocess' # @since 1.3.0 class R10K::Environment::Base + include R10K::Logging + # @!attribute [r] name # @return [String] A name for this environment that is unique to the given source attr_reader :name @@ -31,6 +37,10 @@ class R10K::Environment::Base # @return [String] The puppetfile name (relative) attr_reader :puppetfile_name + attr_reader :managed_directories, :desired_contents + + attr_reader :loader + # Initialize the given environment. # # @param name [String] The unique name describing this environment. @@ -43,13 +53,31 @@ class R10K::Environment::Base @basedir = basedir @dirname = dirname @options = options - @puppetfile_name = options[:puppetfile_name] + @puppetfile_name = options.delete(:puppetfile_name) + @overrides = options.delete(:overrides) || {} @full_path = File.join(@basedir, @dirname) @path = Pathname.new(File.join(@basedir, @dirname)) - @puppetfile = R10K::Puppetfile.new(@full_path, nil, nil, @puppetfile_name) + @puppetfile = R10K::Puppetfile.new(@full_path, + {overrides: @overrides, + force: @overrides.dig(:modules, :force), + puppetfile_name: @puppetfile_name}) @puppetfile.environment = self + + loader_options = { basedir: @full_path, overrides: @overrides, environment: self } + loader_options[:puppetfile] = @puppetfile_name if @puppetfile_name + + @loader = R10K::ModuleLoader::Puppetfile.new(**loader_options) + + if @overrides.dig(:environments, :incremental) + @loader.load_metadata + end + + @base_modules = nil + @purge_exclusions = nil + @managed_directories = [ @full_path ] + @desired_contents = [] end # Synchronize the given environment. @@ -99,8 +127,18 @@ class R10K::Environment::Base # @return [Array] All modules defined in the Puppetfile # associated with this environment. def modules - @puppetfile.load - @puppetfile.modules + if @base_modules.nil? + load_puppetfile_modules + end + + @base_modules + end + + # @return [Array] Whether or not the given module + # conflicts with any modules already defined in the r10k environment + # object. + def module_conflicts?(mod) + false end def accept(visitor) @@ -109,16 +147,50 @@ class R10K::Environment::Base end end + + # Returns a Queue of the names of modules actually updated + def deploy + if @base_modules.nil? + load_puppetfile_modules + end + + if ! @base_modules.empty? + pool_size = @overrides.dig(:modules, :pool_size) + updated_modules = R10K::ContentSynchronizer.concurrent_sync(@base_modules, pool_size, logger) + end + + if (@overrides.dig(:purging, :purge_levels) || []).include?(:puppetfile) + logger.debug("Purging unmanaged Puppetfile content for environment '#{dirname}'...") + @puppetfile_cleaner.purge! + end + + updated_modules + end + + def load_puppetfile_modules + loaded_content = @loader.load + @base_modules = loaded_content[:modules] + + @purge_exclusions = determine_purge_exclusions(loaded_content[:managed_directories], + loaded_content[:desired_contents]) + + @puppetfile_cleaner = R10K::Util::Cleaner.new(loaded_content[:managed_directories], + loaded_content[:desired_contents], + loaded_content[:purge_exclusions]) + end + def whitelist(user_whitelist=[]) user_whitelist.collect { |pattern| File.join(@full_path, pattern) } end - def purge_exclusions + def determine_purge_exclusions(pf_managed_dirs = @puppetfile.managed_directories, + pf_desired_contents = @puppetfile.desired_contents) + list = [File.join(@full_path, '.r10k-deploy.json')].to_set - list += @puppetfile.managed_directories + list += pf_managed_dirs - list += @puppetfile.desired_contents.flat_map do |item| + list += pf_desired_contents.flat_map do |item| desired_tree = [] if File.directory?(item) @@ -136,6 +208,14 @@ class R10K::Environment::Base list.to_a end + def purge_exclusions + if @purge_exclusions.nil? + load_puppetfile_modules + end + + @purge_exclusions + end + def generate_types! argv = [R10K::Settings.puppet_path, 'generate', 'types', '--environment', dirname, '--environmentpath', basedir, '--config', R10K::Settings.puppet_conf] subproc = R10K::Util::Subprocess.new(argv) diff -pruN 3.7.0-2.1/lib/r10k/environment/git.rb 3.15.4-1/lib/r10k/environment/git.rb --- 3.7.0-2.1/lib/r10k/environment/git.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/environment/git.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,4 +1,3 @@ -require 'r10k/logging' require 'r10k/puppetfile' require 'r10k/git/stateful_repository' require 'forwardable' @@ -8,8 +7,6 @@ require 'forwardable' # @since 1.3.0 class R10K::Environment::Git < R10K::Environment::WithModules - include R10K::Logging - R10K::Environment.register(:git, self) # Register git as the default environment type R10K::Environment.register(nil, self) @@ -27,6 +24,8 @@ class R10K::Environment::Git < R10K::Env # @return [R10K::Git::StatefulRepository] The git repo backing this environment attr_reader :repo + include R10K::Util::Setopts + # Initialize the given Git environment. # # @param name [String] The unique name describing this environment. @@ -38,8 +37,21 @@ class R10K::Environment::Git < R10K::Env # @param options [String] :ref The git reference to use for this environment def initialize(name, basedir, dirname, options = {}) super - @remote = options[:remote] - @ref = options[:ref] + setopts(options, { + # Standard option interface + :version => :ref, + :source => :remote, + :type => ::R10K::Util::Setopts::Ignore, + + # Type-specific options + :ref => :self, + :remote => :self, + + }, raise_on_unhandled: false) + # TODO: in r10k 4.0.0, a major version bump, stop allowing garbage options. + # We only allow them now, here, on this object, because prior to adopting + # setopts in the constructor, this object type didn't do any validation + # checking of options passed, and would permit garbage parameters. @repo = R10K::Git::StatefulRepository.new(@remote, @basedir, @dirname) end diff -pruN 3.7.0-2.1/lib/r10k/environment/name.rb 3.15.4-1/lib/r10k/environment/name.rb --- 3.7.0-2.1/lib/r10k/environment/name.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/environment/name.rb 2023-01-19 00:49:17.000000000 +0000 @@ -6,19 +6,24 @@ module R10K class Name # @!attribute [r] name - # @return [String] The unmodified name of the environment + # @return [String] The functional name of the environment derived from inputs and options. attr_reader :name - INVALID_CHARACTERS = %r[\W] + # @!attribute [r] original_name + # @return [String] The unmodified name originally given to create the object. + attr_reader :original_name - def initialize(name, opts) - @name = name - @opts = opts + INVALID_CHARACTERS = %r[\W] + def initialize(original_name, opts) @source = opts[:source] @prefix = opts[:prefix] @invalid = opts[:invalid] + @name = derive_name(original_name, opts[:strip_component]) + @original_name = original_name + @opts = opts + case @invalid when 'correct_and_warn' @validate = true @@ -71,8 +76,26 @@ module R10K private - def derive_prefix(source,prefix) + def derive_name(original_name, strip_component) + return original_name unless strip_component + + unless strip_component.is_a?(String) + raise _('Improper configuration value given for strip_component setting in %{src} source. ' \ + 'Value must be a string, a /regex/, false, or omitted. Got "%{val}" (%{type})' \ + % {src: @source, val: strip_component, type: strip_component.class}) + end + if %r{^/.*/$}.match(strip_component) + regex = Regexp.new(strip_component[1..-2]) + original_name.gsub(regex, '') + elsif original_name.start_with?(strip_component) + original_name[strip_component.size..-1] + else + original_name + end + end + + def derive_prefix(source,prefix) if prefix == true "#{source}_" elsif prefix.is_a? String diff -pruN 3.7.0-2.1/lib/r10k/environment/plain.rb 3.15.4-1/lib/r10k/environment/plain.rb --- 3.7.0-2.1/lib/r10k/environment/plain.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/lib/r10k/environment/plain.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,16 @@ +class R10K::Environment::Plain < R10K::Environment::WithModules + + R10K::Environment.register(:plain, self) + + def sync + path.mkpath + end + + def status + :not_applicable + end + + def signature + 'plain-default' + end +end diff -pruN 3.7.0-2.1/lib/r10k/environment/svn.rb 3.15.4-1/lib/r10k/environment/svn.rb --- 3.7.0-2.1/lib/r10k/environment/svn.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/environment/svn.rb 2023-01-19 00:49:17.000000000 +0000 @@ -7,8 +7,6 @@ require 'r10k/util/setopts' # @since 1.3.0 class R10K::Environment::SVN < R10K::Environment::Base - include R10K::Logging - R10K::Environment.register(:svn, self) # @!attribute [r] remote @@ -44,8 +42,17 @@ class R10K::Environment::SVN < R10K::Env # @option options [String] :password The SVN password def initialize(name, basedir, dirname, options = {}) super - - setopts(options, {:remote => :self, :username => :self, :password => :self, :puppetfile_name => :self }) + setopts(options, { + # Standard option interface + :source => :remote, + :version => :expected_revision, + :type => ::R10K::Util::Setopts::Ignore, + + # Type-specific options + :remote => :self, + :username => :self, + :password => :self, + }) @working_dir = R10K::SVN::WorkingDir.new(Pathname.new(@full_path), :username => @username, :password => @password) end @@ -61,7 +68,7 @@ class R10K::Environment::SVN < R10K::Env if @working_dir.is_svn? @working_dir.update else - @working_dir.checkout(@remote) + @working_dir.checkout(@remote, @expected_revision) end @synced = true end diff -pruN 3.7.0-2.1/lib/r10k/environment/tarball.rb 3.15.4-1/lib/r10k/environment/tarball.rb --- 3.7.0-2.1/lib/r10k/environment/tarball.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/lib/r10k/environment/tarball.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,78 @@ +require 'r10k/util/setopts' +require 'r10k/tarball' +require 'r10k/environment' + +class R10K::Environment::Tarball < R10K::Environment::WithModules + + R10K::Environment.register(:tarball, self) + + # @!attribute [r] tarball + # @api private + # @return [R10K::Tarball] + attr_reader :tarball + + include R10K::Util::Setopts + + # Initialize the given tarball environment. + # + # @param name [String] The unique name describing this environment. + # @param basedir [String] The base directory where this environment will be created. + # @param dirname [String] The directory name for this environment. + # @param options [Hash] An additional set of options for this environment. + # + # @param options [String] :source Where to get the tarball from + # @param options [String] :version The sha256 digest of the tarball + def initialize(name, basedir, dirname, options = {}) + super + setopts(options, { + # Standard option interface + :type => ::R10K::Util::Setopts::Ignore, + :source => :self, + :version => :checksum, + + # Type-specific options + :checksum => :self, + }) + + @tarball = R10K::Tarball.new(name, @source, checksum: @checksum) + end + + def path + @path ||= Pathname.new(File.join(@basedir, @dirname)) + end + + def sync + tarball.get unless tarball.cache_valid? + case status + when :absent, :mismatched + tarball.unpack(path.to_s) + # Untracked files left behind from previous extractions are expected to + # be deleted by r10k's purge facility. + end + end + + def status + if not path.exist? + :absent + elsif not (tarball.cache_valid? && tarball.insync?(path.to_s, ignore_untracked_files: true)) + :mismatched + else + :insync + end + end + + def signature + @checksum || @tarball.cache_checksum + end + + include R10K::Util::Purgeable + + # Returns an array of the full paths to all the content being managed. + # @note This implements a required method for the Purgeable mixin + # @return [Array] + def desired_contents + desired = [] + desired += @tarball.paths.map { |entry| File.join(@full_path, entry) } + desired += super + end +end diff -pruN 3.7.0-2.1/lib/r10k/environment/with_modules.rb 3.15.4-1/lib/r10k/environment/with_modules.rb --- 3.7.0-2.1/lib/r10k/environment/with_modules.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/environment/with_modules.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,4 +1,3 @@ -require 'r10k/logging' require 'r10k/util/purgeable' # This abstract base class implements an environment that can include module @@ -7,8 +6,6 @@ require 'r10k/util/purgeable' # @since 3.4.0 class R10K::Environment::WithModules < R10K::Environment::Base - include R10K::Logging - # @!attribute [r] moduledir # @return [String] The directory to install environment-defined modules # into (default: #{basedir}/modules) @@ -24,8 +21,9 @@ class R10K::Environment::WithModules < R # @param options [String] :moduledir The path to install modules to # @param options [Hash] :modules Modules to add to the environment def initialize(name, basedir, dirname, options = {}) - super(name, basedir, dirname, options) + super + @all_modules = nil @managed_content = {} @modules = [] @moduledir = case options[:moduledir] @@ -46,39 +44,97 @@ class R10K::Environment::WithModules < R # - The r10k environment object # - A Puppetfile in the environment's content def modules - return @modules if @puppetfile.nil? + if @all_modules.nil? + puppetfile_modules = super() + @all_modules = @modules + puppetfile_modules + end + + @all_modules + end + + def module_conflicts?(mod_b) + conflict = @modules.any? { |mod_a| mod_a.name == mod_b.name } + return false unless conflict + + msg_vars = {src: mod_b.origin, name: mod_b.name} + msg_error = _('Environment and %{src} both define the "%{name}" module' % msg_vars) + msg_continue = _("#{msg_error}. The %{src} definition will be ignored" % msg_vars) + + case conflict_opt = @options[:module_conflicts] + when 'override_and_warn', nil + logger.warn msg_continue + when 'override' + logger.debug msg_continue + when 'error' + raise R10K::Error, msg_error + else + raise R10K::Error, _('Unexpected value for `module_conflicts` setting in %{env} ' \ + 'environment: %{val}' % {env: self.name, val: conflict_opt}) + end - @puppetfile.load unless @puppetfile.loaded? - @modules + @puppetfile.modules + true end def accept(visitor) visitor.visit(:environment, self) do @modules.each do |mod| - mod.accept(visitor) + mod.sync end puppetfile.accept(visitor) - validate_no_module_conflicts end end + def deploy + @modules.each do |mod| + mod.sync + end + + super + end + def load_modules(module_hash) module_hash.each do |name, args| + if !args.is_a?(Hash) + args = { type: 'forge', version: args } + end + add_module(name, args) end end + def resolve_path(base, dirname, path) + if Pathname.new(path).absolute? + cleanpath(path) + else + cleanpath(File.join(base, dirname, path)) + end + end + + # .cleanpath is as good as we can do without touching the filesystem. + # The .realpath methods will choke if some of the intermediate paths + # are missing, even though in some cases we will create them later as + # needed. + def cleanpath(path) + Pathname.new(path).cleanpath.to_s + end + + def validate_install_path(path, modname) + unless /^#{Regexp.escape(@basedir)}.*/ =~ path + raise R10K::Error.new("Environment cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{@basedir}") + end + true + end + # @param [String] name - # @param [*Object] args + # @param [Hash] args def add_module(name, args) - if args.is_a?(Hash) - # symbolize keys in the args hash - args = args.inject({}) { |memo,(k,v)| memo[k.to_sym] = v; memo } - end + # symbolize keys in the args hash + args = args.inject({}) { |memo,(k,v)| memo[k.to_sym] = v; memo } + args[:overrides] = @overrides - if args.is_a?(Hash) && install_path = args.delete(:install_path) - install_path = resolve_install_path(install_path) + if install_path = args.delete(:install_path) + install_path = resolve_path(@basedir, @dirname, install_path) validate_install_path(install_path, name) else install_path = @moduledir @@ -88,35 +144,14 @@ class R10K::Environment::WithModules < R @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path) mod = R10K::Module.new(name, install_path, args, self.name) - mod.origin = 'Environment' + mod.origin = :environment @managed_content[install_path] << mod.name @modules << mod end - def validate_no_module_conflicts - @puppetfile.load unless @puppetfile.loaded? - conflicts = (@modules + @puppetfile.modules) - .group_by { |mod| mod.name } - .select { |_, v| v.size > 1 } - .map(&:first) - unless conflicts.empty? - msg = _('Puppetfile cannot contain module names defined by environment %{name}') % {name: self.name} - msg += ' ' - msg += _("Remove the conflicting definitions of the following modules: %{conflicts}" % { conflicts: conflicts.join(' ') }) - raise R10K::Error.new(msg) - end - end - include R10K::Util::Purgeable - # Returns an array of the full paths that can be purged. - # @note This implements a required method for the Purgeable mixin - # @return [Array] - def managed_directories - [@full_path] - end - # Returns an array of the full paths of filenames that should exist. Files # inside managed_directories that are not listed in desired_contents will # be purged. diff -pruN 3.7.0-2.1/lib/r10k/environment.rb 3.15.4-1/lib/r10k/environment.rb --- 3.7.0-2.1/lib/r10k/environment.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/environment.rb 2023-01-19 00:49:17.000000000 +0000 @@ -30,8 +30,10 @@ module R10K require 'r10k/environment/base' require 'r10k/environment/with_modules' + require 'r10k/environment/plain' require 'r10k/environment/bare' require 'r10k/environment/git' require 'r10k/environment/svn' + require 'r10k/environment/tarball' end end diff -pruN 3.7.0-2.1/lib/r10k/errors.rb 3.15.4-1/lib/r10k/errors.rb --- 3.7.0-2.1/lib/r10k/errors.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/errors.rb 2023-01-19 00:49:17.000000000 +0000 @@ -58,4 +58,9 @@ module R10K str.gsub(/^/, prefix) end end + + # An error class for configuration errors + # + class ConfigError < Error + end end diff -pruN 3.7.0-2.1/lib/r10k/forge/module_release.rb 3.15.4-1/lib/r10k/forge/module_release.rb --- 3.7.0-2.1/lib/r10k/forge/module_release.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/forge/module_release.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,5 +1,6 @@ require 'r10k/logging' require 'r10k/settings/mixin' +require 'r10k/util/cacheable' require 'fileutils' require 'tmpdir' require 'puppet_forge' @@ -13,7 +14,7 @@ module R10K def_setting_attr :proxy def_setting_attr :baseurl - def_setting_attr :cache_root, File.expand_path(ENV['HOME'] ? '~/.r10k/cache': '/root/.r10k/cache') + def_setting_attr :cache_root, R10K::Util::Cacheable.default_cachedir include R10K::Logging diff -pruN 3.7.0-2.1/lib/r10k/git/cache.rb 3.15.4-1/lib/r10k/git/cache.rb --- 3.7.0-2.1/lib/r10k/git/cache.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/git/cache.rb 2023-01-19 00:49:17.000000000 +0000 @@ -3,6 +3,7 @@ require 'r10k/git' require 'r10k/settings' require 'r10k/instance_cache' require 'forwardable' +require 'r10k/util/cacheable' # Cache Git repository mirrors for object database reuse. # @@ -15,18 +16,9 @@ require 'forwardable' class R10K::Git::Cache include R10K::Settings::Mixin + include R10K::Util::Cacheable - #@api private - def self.determine_cache_root - if R10K::Util::Platform.windows? - File.join(ENV['LOCALAPPDATA'], 'r10k', 'git') - else - File.expand_path(ENV['HOME'] ? '~/.r10k/git': '/root/.r10k/git') - end - end - private_class_method :determine_cache_root - - def_setting_attr :cache_root, determine_cache_root + def_setting_attr :cache_root, R10K::Util::Cacheable.default_cachedir('git') @instance_cache = R10K::InstanceCache.new(self) @@ -109,8 +101,7 @@ class R10K::Git::Cache alias cached? exist? - # Reformat the remote name into something that can be used as a directory def sanitized_dirname - @sanitized_dirname ||= @remote.gsub(/[^@\w\.-]/, '-') + @sanitized_dirname ||= super(@remote) end end diff -pruN 3.7.0-2.1/lib/r10k/git/rugged/bare_repository.rb 3.15.4-1/lib/r10k/git/rugged/bare_repository.rb --- 3.7.0-2.1/lib/r10k/git/rugged/bare_repository.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/git/rugged/bare_repository.rb 2023-01-19 00:49:17.000000000 +0000 @@ -64,7 +64,7 @@ class R10K::Git::Rugged::BareRepository results = nil R10K::Git.with_proxy(proxy) do - results = with_repo { |repo| repo.fetch(remote_name, refspecs, options) } + results = with_repo { |repo| repo.fetch(remote_name, refspecs, **options) } end report_transfer(results, remote_name) diff -pruN 3.7.0-2.1/lib/r10k/git/rugged/base_repository.rb 3.15.4-1/lib/r10k/git/rugged/base_repository.rb --- 3.7.0-2.1/lib/r10k/git/rugged/base_repository.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/git/rugged/base_repository.rb 2023-01-19 00:49:17.000000000 +0000 @@ -20,7 +20,8 @@ class R10K::Git::Rugged::BaseRepository else object.oid end - rescue ::Rugged::ReferenceError + rescue ::Rugged::ReferenceError, ::Rugged::OdbError => e + logger.debug2(_("Unable to resolve %{pattern}: %{e} ") % {pattern: pattern, e: e }) nil end @@ -60,6 +61,16 @@ class R10K::Git::Rugged::BaseRepository remotes_hash end + # Update a remote URL + # @param [String] The remote URL of the git repository + # @param [String] An optional remote name for the git repository + def update_remote(remote, remote_name='origin') + if @_rugged_repo + logger.debug2(_("Remote URL is different from cache, updating %{orig} to %{update}") % {orig: remotes[remote_name], update: remote}) + @_rugged_repo.remotes.set_url(remote_name, remote) + end + end + private def with_repo(opts={}) diff -pruN 3.7.0-2.1/lib/r10k/git/rugged/cache.rb 3.15.4-1/lib/r10k/git/rugged/cache.rb --- 3.7.0-2.1/lib/r10k/git/rugged/cache.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/git/rugged/cache.rb 2023-01-19 00:49:17.000000000 +0000 @@ -8,4 +8,12 @@ class R10K::Git::Rugged::Cache < R10K::G def self.bare_repository R10K::Git::Rugged::BareRepository end + + # Update the remote URL if the cache differs from the current configuration + def sync! + if cached? && @repo.remotes['origin'] != @remote + @repo.update_remote(@remote) + end + super + end end diff -pruN 3.7.0-2.1/lib/r10k/git/rugged/credentials.rb 3.15.4-1/lib/r10k/git/rugged/credentials.rb --- 3.7.0-2.1/lib/r10k/git/rugged/credentials.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/git/rugged/credentials.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,6 +1,10 @@ require 'r10k/git/rugged' require 'r10k/git/errors' require 'r10k/logging' +require 'json' +require 'jwt' +require 'net/http' +require 'openssl' # Generate credentials for secured remote connections. # @@ -61,11 +65,62 @@ class R10K::Git::Rugged::Credentials end def get_plaintext_credentials(url, username_from_url) - user = get_git_username(url, username_from_url) - password = URI.parse(url).password || '' + per_repo_oauth_token = nil + per_repo_github_app_id = nil + per_repo_github_app_key = nil + per_repo_github_app_ttl = nil + + if per_repo_settings = R10K::Git.get_repo_settings(url) + per_repo_oauth_token = per_repo_settings[:oauth_token] + per_repo_github_app_id = per_repo_settings[:github_app_id] + per_repo_github_app_key = per_repo_settings[:github_app_key] + per_repo_github_app_ttl = per_repo_settings[:github_app_ttl] + end + + app_id = per_repo_github_app_id || R10K::Git.settings[:github_app_id] + app_key = per_repo_github_app_key || R10K::Git.settings[:github_app_key] + app_ttl = per_repo_github_app_ttl || R10K::Git.settings[:github_app_ttl] + + if token_path = per_repo_oauth_token || R10K::Git.settings[:oauth_token] + @oauth_token ||= extract_token(token_path, url) + + user = 'x-oauth-token' + password = @oauth_token + elsif app_id && app_key && app_ttl + user = 'x-access-token' + password = github_app_token(app_id, app_key, app_ttl) + else + user = get_git_username(url, username_from_url) + password = URI.parse(url).password || '' + end Rugged::Credentials::UserPassword.new(username: user, password: password) end + def extract_token(token_path, url) + if token_path == '-' + token = $stdin.read.strip + logger.debug2 _("Using OAuth token from stdin for URL %{url}") % { url: url } + elsif File.readable?(token_path) + token = File.read(token_path).strip + logger.debug2 _("Using OAuth token from %{token_path} for URL %{url}") % { token_path: token_path, url: url } + else + raise R10K::Git::GitError, _("%{path} is missing or unreadable, cannot load OAuth token") % { path: token_path } + end + + unless valid_token?(token) + raise R10K::Git::GitError, _("Supplied OAuth token contains invalid characters.") + end + + token + end + + # This regex is the only real requirement for OAuth token format, + # per https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/ + # Bitbucket's tokens also can include an underscore, so that is added here. + def valid_token?(token) + return token =~ /^[\w\-\.~_\+\/]+$/ + end + def get_default_credentials(url, username_from_url) Rugged::Credentials::Default.new end @@ -75,7 +130,7 @@ class R10K::Git::Rugged::Credentials user = nil - if !username_from_url.nil? + if !username_from_url.nil? && !username_from_url.empty? user = username_from_url logger.debug2 _("URL %{url} includes the username %{username}, using that user for authentication.") % {url: url.inspect, username: username_from_url} elsif git_user @@ -88,4 +143,63 @@ class R10K::Git::Rugged::Credentials user end + + def github_app_token(app_id, private_key, ttl) + raise R10K::Git::GitError, _('Github App id contains invalid characters.') unless app_id =~ /^\d+$/ + raise R10K::Git::GitError, _('Github App token ttl contains invalid characters.') unless ttl =~ /^\d+$/ + raise R10K::Git::GitError, _('Github App key is missing or unreadable') unless File.readable?(private_key) + + begin + ssl_key = OpenSSL::PKey::RSA.new(File.read(private_key).strip) + unless ssl_key.private? + raise R10K::Git::GitError, _('Github App key is not a valid SSL private key') + end + rescue OpenSSL::PKey::RSAError + raise R10K::Git::GitError, _('Github App key is not a valid SSL key') + end + + logger.debug2 _("Using Github App id %{app_id} with SSL key from %{key_path}") % { key_path: private_key, app_id: app_id } + + jwt_issue_time = Time.now.to_i - 60 + jwt_exp_time = (jwt_issue_time + 60) + ttl.to_i + payload = { iat: jwt_issue_time, exp: jwt_exp_time, iss: app_id } + jwt = JWT.encode(payload, ssl_key, "RS256") + + get = URI.parse("https://api.github.com/app/installations") + get_request = Net::HTTP::Get.new(get) + get_request["Authorization"] = "Bearer #{jwt}" + get_request["Accept"] = "application/vnd.github.v3+json" + get_req_options = { use_ssl: get.scheme == "https", } + get_response = Net::HTTP.start(get.hostname, get.port, get_req_options) do |http| + http.request(get_request) + end + + unless (get_response.class < Net::HTTPSuccess) + logger.debug2 _("Unexpected response code: #{get_response.code}\nResponse body: #{get_response.body}") + raise R10K::Git::GitError, _("Error using private key to get Github App access token from url") + end + + access_tokens_url = JSON.parse(get_response.body)[0]['access_tokens_url'] + + post = URI.parse(access_tokens_url) + post_request = Net::HTTP::Post.new(post) + post_request["Authorization"] = "Bearer #{jwt}" + post_request["Accept"] = "application/vnd.github.v3+json" + post_req_options = { use_ssl: post.scheme == "https", } + post_response = Net::HTTP.start(post.hostname, post.port, post_req_options) do |http| + http.request(post_request) + end + + unless (post_response.class < Net::HTTPSuccess) + logger.debug2 _("Unexpected response code: #{post_response.code}\nResponse body: #{post_response.body}") + raise R10K::Git::GitError, _("Error using private key to generate access token from #{access_token_url}") + end + + token = JSON.parse(post_response.body)['token'] + + raise R10K::Git::GitError, _("Github App token contains invalid characters.") unless valid_token?(token) + + logger.debug2 _("Github App token generated, expires at: %{expire}") % {expire: JSON.parse(post_response.body)['expires_at']} + token + end end diff -pruN 3.7.0-2.1/lib/r10k/git/rugged/thin_repository.rb 3.15.4-1/lib/r10k/git/rugged/thin_repository.rb --- 3.7.0-2.1/lib/r10k/git/rugged/thin_repository.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/git/rugged/thin_repository.rb 2023-01-19 00:49:17.000000000 +0000 @@ -75,6 +75,13 @@ class R10K::Git::Rugged::ThinRepository end end + def stage_files(files=['.']) + with_repo do |repo| + index = repo.index + files.each { |p| index.add( :path => p ) } + end + end + private # Override the parent class repo setup so that we can make sure the alternates file is up to date diff -pruN 3.7.0-2.1/lib/r10k/git/rugged/working_repository.rb 3.15.4-1/lib/r10k/git/rugged/working_repository.rb --- 3.7.0-2.1/lib/r10k/git/rugged/working_repository.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/git/rugged/working_repository.rb 2023-01-19 00:49:17.000000000 +0000 @@ -93,7 +93,7 @@ class R10K::Git::Rugged::WorkingReposito results = nil R10K::Git.with_proxy(proxy) do - results = with_repo { |repo| repo.fetch(remote_name, refspecs, options) } + results = with_repo { |repo| repo.fetch(remote_name, refspecs, **options) } end report_transfer(results, remote) @@ -117,11 +117,15 @@ class R10K::Git::Rugged::WorkingReposito with_repo { |repo| repo.config['remote.origin.url'] } end - def dirty? + def dirty?(exclude_spec=false) with_repo do |repo| - diff = repo.diff_workdir('HEAD') + if exclude_spec + diff = repo.diff_workdir('HEAD').select { |d| ! d.delta.old_file[:path].start_with?('spec/') } + else + diff = repo.diff_workdir('HEAD').to_a + end - diff.each_patch do |p| + diff.each do |p| logger.debug(_("Found local modifications in %{file_path}" % {file_path: File.join(@path, p.delta.old_file[:path])})) logger.debug1(p.to_s) end diff -pruN 3.7.0-2.1/lib/r10k/git/shellgit/thin_repository.rb 3.15.4-1/lib/r10k/git/shellgit/thin_repository.rb --- 3.7.0-2.1/lib/r10k/git/shellgit/thin_repository.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/git/shellgit/thin_repository.rb 2023-01-19 00:49:17.000000000 +0000 @@ -43,6 +43,10 @@ class R10K::Git::ShellGit::ThinRepositor git(['ls-tree', '-t', '-r', '--name-only', ref], :path => @path.to_s).stdout.split("\n") end + def stage_files(files=['.']) + git(['add', files].flatten, :path => @path.to_s) + end + private def setup_cache_remote diff -pruN 3.7.0-2.1/lib/r10k/git/shellgit/working_repository.rb 3.15.4-1/lib/r10k/git/shellgit/working_repository.rb --- 3.7.0-2.1/lib/r10k/git/shellgit/working_repository.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/git/shellgit/working_repository.rb 2023-01-19 00:49:17.000000000 +0000 @@ -90,11 +90,12 @@ class R10K::Git::ShellGit::WorkingReposi end # does the working tree have local modifications to tracked files? - def dirty? + def dirty?(exclude_spec=false) result = git(['diff-index', '--exit-code', '--name-only', 'HEAD'], :path => @path.to_s, :raise_on_fail => false) if result.exit_code != 0 - dirty_files = result.stdout.split('\n') + dirty_files = result.stdout.split("\n") + dirty_files.delete_if { |f| f.start_with?('spec/') } if exclude_spec dirty_files.each do |file| logger.debug(_("Found local modifications in %{file_path}" % {file_path: File.join(@path, file)})) @@ -103,7 +104,7 @@ class R10K::Git::ShellGit::WorkingReposi logger.debug1 { git(['diff-index', '-p', 'HEAD', file], :path => @path.to_s, :raise_on_fail => false).stdout } end - return true + return dirty_files.size > 0 else return false end diff -pruN 3.7.0-2.1/lib/r10k/git/stateful_repository.rb 3.15.4-1/lib/r10k/git/stateful_repository.rb --- 3.7.0-2.1/lib/r10k/git/stateful_repository.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/git/stateful_repository.rb 2023-01-19 00:49:17.000000000 +0000 @@ -35,7 +35,8 @@ class R10K::Git::StatefulRepository @cache.resolve(ref) end - def sync(ref, force=true) + # Returns true if the sync actually updated the repo, false otherwise + def sync(ref, force=true, exclude_spec=false) @cache.sync if sync_cache?(ref) sha = @cache.resolve(ref) @@ -44,8 +45,9 @@ class R10K::Git::StatefulRepository raise R10K::Git::UnresolvableRefError.new(_("Unable to sync repo to unresolvable ref '%{ref}'") % {ref: ref}, :git_dir => @repo.git_dir) end - workdir_status = status(ref) + workdir_status = status(ref, exclude_spec) + updated = true case workdir_status when :absent logger.debug(_("Cloning %{repo_path} and checking out %{ref}") % {repo_path: @repo.path, ref: ref }) @@ -64,22 +66,29 @@ class R10K::Git::StatefulRepository @repo.checkout(sha, {:force => force}) else logger.warn(_("Skipping %{repo_path} due to local modifications") % {repo_path: @repo.path}) + updated = false end else logger.debug(_("%{repo_path} is already at Git ref %{ref}") % {repo_path: @repo.path, ref: ref }) + updated = false end + updated end - def status(ref) + def status(ref, exclude_spec=false) if !@repo.exist? :absent + elsif !@cache.exist? + :mismatched elsif !@repo.git_dir.exist? :mismatched elsif !@repo.git_dir.directory? :mismatched elsif !(@repo.origin == @remote) :mismatched - elsif @repo.dirty? + elsif @repo.head.nil? + :mismatched + elsif @repo.dirty?(exclude_spec) :dirty elsif !(@repo.head == @cache.resolve(ref)) :outdated @@ -93,6 +102,7 @@ class R10K::Git::StatefulRepository # @api private def sync_cache?(ref) return true if !@cache.exist? + return true if ref == 'HEAD' return true if !([:commit, :tag].include? @cache.ref_type(ref)) return false end diff -pruN 3.7.0-2.1/lib/r10k/git.rb 3.15.4-1/lib/r10k/git.rb --- 3.7.0-2.1/lib/r10k/git.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/git.rb 2023-01-19 00:49:17.000000000 +0000 @@ -134,6 +134,10 @@ module R10K extend R10K::Settings::Mixin::ClassMethods def_setting_attr :private_key + def_setting_attr :oauth_token + def_setting_attr :github_app_id + def_setting_attr :github_app_key + def_setting_attr :github_app_ttl def_setting_attr :proxy def_setting_attr :username def_setting_attr :repositories, {} diff -pruN 3.7.0-2.1/lib/r10k/initializers.rb 3.15.4-1/lib/r10k/initializers.rb --- 3.7.0-2.1/lib/r10k/initializers.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/initializers.rb 2023-01-19 00:49:17.000000000 +0000 @@ -4,6 +4,7 @@ require 'r10k/git' require 'r10k/git/cache' require 'r10k/forge/module_release' +require 'r10k/tarball' module R10K module Initializers @@ -30,14 +31,27 @@ module R10K logger.warn(_("the purgedirs key in r10k.yaml is deprecated. it is currently ignored.")) end + with_setting(:logging) { |value| LoggingInitializer.new(value).call } + with_setting(:deploy) { |value| DeployInitializer.new(value).call } with_setting(:cachedir) { |value| R10K::Git::Cache.settings[:cache_root] = value } with_setting(:cachedir) { |value| R10K::Forge::ModuleRelease.settings[:cache_root] = value } + with_setting(:cachedir) { |value| R10K::Tarball.settings[:cache_root] = value } with_setting(:pool_size) { |value| R10K::Puppetfile.settings[:pool_size] = value } + with_setting(:proxy) { |value| R10K::Tarball.settings[:proxy] = value } with_setting(:git) { |value| GitInitializer.new(value).call } with_setting(:forge) { |value| ForgeInitializer.new(value).call } + with_setting(:tarball) { |value| TarballInitializer.new(value).call } + end + end + + class LoggingInitializer < BaseInitializer + def call + with_setting(:level) { |value| R10K::Logging.level = value } + with_setting(:disable_default_stderr) { |value| R10K::Logging.disable_default_stderr = value } + with_setting(:outputs) { |value| R10K::Logging.add_outputters(value) } end end @@ -55,6 +69,10 @@ module R10K with_setting(:private_key) { |value| R10K::Git.settings[:private_key] = value } with_setting(:proxy) { |value| R10K::Git.settings[:proxy] = value } with_setting(:repositories) { |value| R10K::Git.settings[:repositories] = value } + with_setting(:oauth_token) { |value| R10K::Git.settings[:oauth_token] = value } + with_setting(:github_app_id) { |value| R10K::Git.settings[:github_app_id] = value } + with_setting(:github_app_key) { |value| R10K::Git.settings[:github_app_key] = value } + with_setting(:github_app_ttl) { |value| R10K::Git.settings[:github_app_ttl] = value } end end @@ -62,6 +80,13 @@ module R10K def call with_setting(:baseurl) { |value| PuppetForge.host = value } with_setting(:proxy) { |value| PuppetForge::Connection.proxy = value } + with_setting(:authorization_token) { |value| PuppetForge::Connection.authorization = value } + end + end + + class TarballInitializer < BaseInitializer + def call + with_setting(:proxy) { |value| R10K::Tarball.settings[:proxy] = value } end end end diff -pruN 3.7.0-2.1/lib/r10k/logging.rb 3.15.4-1/lib/r10k/logging.rb --- 3.7.0-2.1/lib/r10k/logging.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/logging.rb 2023-01-19 00:49:17.000000000 +0000 @@ -8,6 +8,16 @@ require 'r10k/logging/terminaloutputter' module R10K::Logging LOG_LEVELS = %w{DEBUG2 DEBUG1 DEBUG INFO NOTICE WARN ERROR FATAL} + SYSLOG_LEVELS_MAP = { + 'DEBUG2' => 'DEBUG', + 'DEBUG1' => 'DEBUG', + 'DEBUG' => 'DEBUG', + 'INFO' => 'INFO', + 'NOTICE' => 'INFO', + 'WARN' => 'WARN', + 'ERROR' => 'ERROR', + 'FATAL' => 'FATAL', + }.freeze def logger_name self.class.to_s @@ -21,6 +31,9 @@ module R10K::Logging else @logger = Log4r::Logger.new(name) @logger.add(R10K::Logging.outputter) + R10K::Logging.outputters.each do |output| + @logger.add(output) + end end end @logger @@ -59,7 +72,7 @@ module R10K::Logging if level.nil? raise ArgumentError, _("Invalid log level '%{val}'. Valid levels are %{log_levels}") % {val: val, log_levels: LOG_LEVELS.map(&:downcase).inspect} end - outputter.level = level + outputter.level = level unless @disable_default_stderr @level = level if level < Log4r::INFO @@ -69,6 +82,58 @@ module R10K::Logging end end + def disable_default_stderr=(val) + @disable_default_stderr = val + outputter.level = val ? Log4r::OFF : @level + end + + def add_outputters(outputs) + outputs.each do |output| + type = output.fetch(:type) + # Support specifying both short as well as full names + type = type.to_s[0..-10] if type.to_s.downcase.end_with? 'outputter' + + name = output.fetch(:name, 'r10k') + if output[:level] + level = parse_level(output[:level]) + if level.nil? + raise ArgumentError, _("Invalid log level '%{val}'. Valid levels are %{log_levels}") % { val: output[:level], log_levels: LOG_LEVELS.map(&:downcase).inspect } + end + else + level = self.level + end + only_at = output[:only_at] + only_at&.map! do |val| + lv = parse_level(val) + if lv.nil? + raise ArgumentError, _("Invalid log level '%{val}'. Valid levels are %{log_levels}") % { val: val, log_levels: LOG_LEVELS.map(&:downcase).inspect } + end + + lv + end + parameters = output.fetch(:parameters, {}).merge({ level: level }) + + begin + # Try to load the outputter file if possible + require "log4r/outputter/#{type.to_s.downcase}outputter" + rescue LoadError + false + end + outputtertype = Log4r.constants + .select { |klass| klass.to_s.end_with? 'Outputter' } + .find { |klass| klass.to_s.downcase == "#{type.to_s.downcase}outputter" } + raise ArgumentError, "Unable to find a #{output[:type]} outputter." unless outputtertype + + outputter = Log4r.const_get(outputtertype).new(name, parameters) + outputter.only_at(*only_at) if only_at + # Handle log4r's syslog mapping correctly + outputter.map_levels_by_name_to_syslog(SYSLOG_LEVELS_MAP) if outputter.respond_to? :map_levels_by_name_to_syslog + + @outputters << outputter + Log4r::Logger.global.add outputter + end + end + extend Forwardable def_delegators :@outputter, :use_color, :use_color= @@ -87,6 +152,16 @@ module R10K::Logging # @return [Log4r::Outputter] attr_reader :outputter + # @!attribute [r] outputters + # @api private + # @return [Array[Log4r::Outputter]] + attr_reader :outputters + + # @!attribute [r] disable_default_stderr + # @api private + # @return [Boolean] + attr_reader :disable_default_stderr + def default_formatter Log4r::PatternFormatter.new(:pattern => '%l\t -> %m') end @@ -106,4 +181,6 @@ module R10K::Logging @level = Log4r::WARN @formatter = default_formatter @outputter = default_outputter + @outputters = [] + @disable_default_stderr = false end diff -pruN 3.7.0-2.1/lib/r10k/module/base.rb 3.15.4-1/lib/r10k/module/base.rb --- 3.7.0-2.1/lib/r10k/module/base.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/module/base.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,9 +1,12 @@ require 'r10k/module' +require 'r10k/logging' require 'puppet_forge' # This class defines a common interface for module implementations. class R10K::Module::Base + include R10K::Logging + # @!attribute [r] title # @return [String] The forward slash separated owner and name of the module attr_reader :title @@ -35,6 +38,10 @@ class R10K::Module::Base # @return [String] Where the module was sourced from. E.g., "Puppetfile" attr_accessor :origin + # @!attribute [rw] spec_deletable + # @return [Boolean] set this to true if the spec dir can be safely removed, ie in the moduledir + attr_accessor :spec_deletable + # There's been some churn over `author` vs `owner` and `full_name` over # `title`, so in the short run it's easier to support both and deprecate one # later. @@ -43,7 +50,7 @@ class R10K::Module::Base # @param title [String] # @param dirname [String] - # @param args [Array] + # @param args [Hash] def initialize(title, dirname, args, environment=nil) @title = PuppetForge::V3.normalize_name(title) @dirname = dirname @@ -51,7 +58,18 @@ class R10K::Module::Base @owner, @name = parse_title(@title) @path = Pathname.new(File.join(@dirname, @name)) @environment = environment + @overrides = args.delete(:overrides) || {} + @spec_deletable = true + @exclude_spec = false + @exclude_spec = @overrides.dig(:modules, :exclude_spec) if @overrides.dig(:modules, :exclude_spec) + if args.has_key?(:exclude_spec) + logger.debug2 _("Overriding :exclude_spec setting with per module setting for #{@title}") + @exclude_spec = args.delete(:exclude_spec) + end @origin = 'external' # Expect Puppetfile or R10k::Environment to set this to a specific value + + @requested_modules = @overrides.dig(:modules, :requested_modules) || [] + @should_sync = (@requested_modules.empty? || @requested_modules.include?(@name)) end # @deprecated @@ -60,12 +78,54 @@ class R10K::Module::Base path.to_s end + # Delete the spec dir if @exclude_spec has been set to true and @spec_deletable is also true + def maybe_delete_spec_dir + if @exclude_spec + if @spec_deletable + delete_spec_dir + else + logger.info _("Spec dir for #{@title} will not be deleted because it is not in the moduledir") + end + end + end + + # Actually remove the spec dir + def delete_spec_dir + spec_path = @path + 'spec' + if spec_path.symlink? + spec_path = spec_path.realpath + end + if spec_path.directory? + logger.debug2 _("Deleting spec data at #{spec_path}") + # Use the secure flag for the #rm_rf method to avoid security issues + # involving TOCTTOU(time of check to time of use); more details here: + # https://ruby-doc.org/stdlib-2.7.0/libdoc/fileutils/rdoc/FileUtils.html#method-c-rm_rf + # Additionally, #rm_rf also has problems in windows with with symlink targets + # also being deleted; this should be revisted if Windows becomes higher priority. + FileUtils.rm_rf(spec_path, secure: true) + else + logger.debug2 _("No spec dir detected at #{spec_path}, skipping deletion") + end + end + # Synchronize this module with the indicated state. - # @abstract + # @param [Hash] opts Deprecated + # @return [Boolean] true if the module was updated, false otherwise def sync(opts={}) raise NotImplementedError end + def should_sync? + if @should_sync + logger.info _("Deploying module to %{path}") % {path: path} + true + else + logger.debug1(_("Only updating modules %{modules}, skipping module %{name}") % {modules: @requested_modules.inspect, name: name}) + false + end + end + + # Return the desired version of this module # @abstract def version @@ -87,6 +147,7 @@ class R10K::Module::Base raise NotImplementedError end + # Deprecated def accept(visitor) visitor.visit(:module, self) end diff -pruN 3.7.0-2.1/lib/r10k/module/definition.rb 3.15.4-1/lib/r10k/module/definition.rb --- 3.7.0-2.1/lib/r10k/module/definition.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/lib/r10k/module/definition.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,64 @@ +require 'r10k/module' + +class R10K::Module::Definition < R10K::Module::Base + + attr_reader :version + + def initialize(name, dirname:, args:, implementation:, environment: nil) + @original_name = name + @original_args = args.dup + @implementation = implementation + @version = implementation.statically_defined_version(name, args) + + super(name, dirname, args, environment) + end + + def to_implementation + mod = @implementation.new(@title, @dirname, @original_args, @environment) + + mod.origin = origin + mod.spec_deletable = spec_deletable + + mod + end + + # syncing is a noop for module definitions + # Returns false to inidicate the module was not updated + def sync(args = {}) + logger.debug1(_("Not updating module %{name}, assuming content unchanged") % {name: name}) + false + end + + def status + :insync + end + + def properties + type = nil + + if @args[:type] + type = @args[:type] + elsif @args[:ref] || @args[:commit] || @args[:branch] || @args[:tag] + type = 'git' + elsif @args[:svn] + # This logic is clear and included for completeness sake, though at + # this time module definitions do not support SVN versions. + type = 'svn' + else + type = 'forge' + end + + { + expected: version, + # We can't get the value for `actual` here because that requires the + # implementation (and potentially expensive operations by the + # implementation). Some consumers will check this value, if it exists + # and if not, fall back to the expected version. That is the correct + # behavior when assuming modules are unchanged, and why `actual` is set + # to `nil` here. + actual: nil, + type: type + } + end +end + diff -pruN 3.7.0-2.1/lib/r10k/module/forge.rb 3.15.4-1/lib/r10k/module/forge.rb --- 3.7.0-2.1/lib/r10k/module/forge.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/module/forge.rb 2023-01-19 00:49:17.000000000 +0000 @@ -13,11 +13,11 @@ class R10K::Module::Forge < R10K::Module R10K::Module.register(self) def self.implement?(name, args) - !!(name.match %r[\w+[/-]\w+]) && valid_version?(args) + args[:type].to_s == 'forge' end - def self.valid_version?(expected_version) - expected_version == :latest || expected_version.nil? || PuppetForge::Util.version_valid?(expected_version) + def self.statically_defined_version(name, args) + args[:version] if args[:version].is_a?(String) end # @!attribute [r] metadata @@ -30,27 +30,54 @@ class R10K::Module::Forge < R10K::Module # @return [PuppetForge::V3::Module] The Puppet Forge module metadata attr_reader :v3_module - include R10K::Logging + include R10K::Util::Setopts - def initialize(title, dirname, expected_version, environment=nil) + def initialize(title, dirname, opts, environment=nil) super @metadata_file = R10K::Module::MetadataFile.new(path + 'metadata.json') @metadata = @metadata_file.read - @expected_version = expected_version || current_version || :latest + setopts(opts, { + # Standard option interface + :version => :expected_version, + :source => ::R10K::Util::Setopts::Ignore, + :type => ::R10K::Util::Setopts::Ignore, + }, :raise_on_unhandled => false) + + # Validate version and raise on issue. Title is validated by base class. + unless valid_version?(@expected_version) + raise ArgumentError, _("Module version %{ver} is not a valid Forge module version") % {ver: @expected_version} + end + + @expected_version ||= current_version || :latest + @v3_module = PuppetForge::V3::Module.new(:slug => @title) end + def valid_version?(version) + version == :latest || version.nil? || PuppetForge::Util.version_valid?(version) + end + + # @param [Hash] opts Deprecated + # @return [Boolean] true if the module was updated, false otherwise def sync(opts={}) - case status - when :absent - install - when :outdated - upgrade - when :mismatched - reinstall + updated = false + if should_sync? + case status + when :absent + install + updated = true + when :outdated + upgrade + updated = true + when :mismatched + reinstall + updated = true + end + maybe_delete_spec_dir end + updated end def properties @@ -65,7 +92,11 @@ class R10K::Module::Forge < R10K::Module def expected_version if @expected_version == :latest begin - @expected_version = @v3_module.current_release.version + if @v3_module.current_release + @expected_version = @v3_module.current_release.version + else + raise PuppetForge::ReleaseNotFound, _("The module %{title} does not appear to have any published releases, cannot determine latest version.") % { title: @title } + end rescue Faraday::ResourceNotFound => e raise PuppetForge::ReleaseNotFound, _("The module %{title} does not exist on %{url}.") % {title: @title, url: PuppetForge::V3::Release.conn.url_prefix}, e.backtrace end @@ -171,7 +202,7 @@ class R10K::Module::Forge < R10K::Module if (match = title.match(/\A(\w+)[-\/](\w+)\Z/)) [match[1], match[2]] else - raise ArgumentError, _("Forge module names must match 'owner/modulename'") + raise ArgumentError, _("Forge module names must match 'owner/modulename', instead got #{title}") end end end diff -pruN 3.7.0-2.1/lib/r10k/module/git.rb 3.15.4-1/lib/r10k/module/git.rb --- 3.7.0-2.1/lib/r10k/module/git.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/module/git.rb 2023-01-19 00:49:17.000000000 +0000 @@ -8,9 +8,22 @@ class R10K::Module::Git < R10K::Module:: R10K::Module.register(self) def self.implement?(name, args) - args.is_a? Hash and args.has_key?(:git) - rescue - false + args.has_key?(:git) || args[:type].to_s == 'git' + end + + # Will be called if self.implement? above returns true. Will return + # the version info, if version is statically defined in the modules + # declaration. + def self.statically_defined_version(name, args) + if !args[:type] && (args[:ref] || args[:tag] || args[:commit]) + if args[:ref] && args[:ref].to_s.match(/[0-9a-f]{40}/) + args[:ref] + else + args[:tag] || args[:commit] + end + elsif args[:type].to_s == 'git' && args[:version] && args[:version].to_s.match(/[0-9a-f]{40}/) + args[:version] + end end # @!attribute [r] repo @@ -28,16 +41,50 @@ class R10K::Module::Git < R10K::Module:: # @return [String] attr_reader :default_ref - def initialize(title, dirname, args, environment=nil) - super + # @!attribute [r] default_override_ref + # @api private + # @return [String] + attr_reader :default_override_ref + + include R10K::Util::Setopts - parse_options(@args) + def initialize(title, dirname, opts, environment=nil) + + super + setopts(opts, { + # Standard option interface + :version => :desired_ref, + :source => :remote, + :type => ::R10K::Util::Setopts::Ignore, + + # Type-specific options + :branch => :desired_ref, + :tag => :desired_ref, + :commit => :desired_ref, + :ref => :desired_ref, + :git => :remote, + :default_branch => :default_ref, + :default_branch_override => :default_override_ref, + }, :raise_on_unhandled => false) + + force = @overrides[:force] + @force = force == false ? false : true + + @desired_ref ||= 'master' + + if @desired_ref == :control_branch + if @environment && @environment.respond_to?(:ref) + @desired_ref = @environment.ref + else + logger.warn _("Cannot track control repo branch for content '%{name}' when not part of a git-backed environment, will use default if available." % {name: name}) + end + end @repo = R10K::Git::StatefulRepository.new(@remote, @dirname, @name) end def version - validate_ref(@desired_ref, @default_ref) + validate_ref(@desired_ref, @default_ref, @default_override_ref) end def properties @@ -48,9 +95,17 @@ class R10K::Module::Git < R10K::Module:: } end + # @param [Hash] opts Deprecated + # @return [Boolean] true if the module was updated, false otherwise def sync(opts={}) - force = opts && opts.fetch(:force, true) - @repo.sync(version, force) + force = opts[:force] || @force + if should_sync? + updated = @repo.sync(version, force, @exclude_spec) + else + updated = false + end + maybe_delete_spec_dir + updated end def status @@ -63,9 +118,11 @@ class R10K::Module::Git < R10K::Module:: private - def validate_ref(desired, default) + def validate_ref(desired, default, default_override) if desired && desired != :control_branch && @repo.resolve(desired) return desired + elsif default_override && @repo.resolve(default_override) + return default_override elsif default && @repo.resolve(default) return default else @@ -81,6 +138,11 @@ class R10K::Module::Git < R10K::Module:: msg << "Could not determine desired ref" end + if default_override + msg << "or resolve the default branch override '%{default_override}'," + vars[:default_override] = default_override + end + if default msg << "or resolve default ref '%{default}'" vars[:default] = default @@ -91,23 +153,4 @@ class R10K::Module::Git < R10K::Module:: raise ArgumentError, _(msg.join(' ')) % vars end end - - def parse_options(options) - ref_opts = [:branch, :tag, :commit, :ref] - known_opts = [:git, :default_branch] + ref_opts - - unhandled = options.keys - known_opts - unless unhandled.empty? - raise ArgumentError, _("Unhandled options %{unhandled} specified for %{class}") % {unhandled: unhandled, class: self.class} - end - - @remote = options[:git] - - @desired_ref = ref_opts.find { |key| break options[key] if options.has_key?(key) } || 'master' - @default_ref = options[:default_branch] - - if @desired_ref == :control_branch && @environment && @environment.respond_to?(:ref) - @desired_ref = @environment.ref - end - end end diff -pruN 3.7.0-2.1/lib/r10k/module/local.rb 3.15.4-1/lib/r10k/module/local.rb --- 3.7.0-2.1/lib/r10k/module/local.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/module/local.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,5 +1,4 @@ require 'r10k/module' -require 'r10k/logging' # A dummy module type that can be used to "protect" Puppet modules that exist # inside of the Puppetfile "moduledir" location. Local modules will not be @@ -9,13 +8,15 @@ class R10K::Module::Local < R10K::Module R10K::Module.register(self) def self.implement?(name, args) - args.is_a?(Hash) && args[:local] + args[:local] || args[:type].to_s == 'local' end - include R10K::Logging + def self.statically_defined_version(*) + "0.0.0" + end def version - "0.0.0" + self.class.statically_defined_version end def properties @@ -30,7 +31,10 @@ class R10K::Module::Local < R10K::Module :insync end + # @param [Hash] opts Deprecated + # @return [Boolean] false, because local modules are always considered in-sync def sync(opts={}) logger.debug1 _("Module %{title} is a local module, always indicating synced.") % {title: title} + false end end diff -pruN 3.7.0-2.1/lib/r10k/module/svn.rb 3.15.4-1/lib/r10k/module/svn.rb --- 3.7.0-2.1/lib/r10k/module/svn.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/module/svn.rb 2023-01-19 00:49:17.000000000 +0000 @@ -7,7 +7,11 @@ class R10K::Module::SVN < R10K::Module:: R10K::Module.register(self) def self.implement?(name, args) - args.is_a? Hash and args.has_key? :svn + args.has_key?(:svn) || args[:type].to_s == 'svn' + end + + def self.statically_defined_version(name, args) + nil end # @!attribute [r] expected_revision @@ -36,18 +40,21 @@ class R10K::Module::SVN < R10K::Module:: include R10K::Util::Setopts - INITIALIZE_OPTS = { - :svn => :url, - :rev => :expected_revision, - :revision => :expected_revision, - :username => :self, - :password => :self - } - def initialize(name, dirname, opts, environment=nil) super - - setopts(opts, INITIALIZE_OPTS) + setopts(opts, { + # Standard option interface + :source => :url, + :version => :expected_revision, + :type => ::R10K::Util::Setopts::Ignore, + + # Type-specific options + :svn => :url, + :rev => :expected_revision, + :revision => :expected_revision, + :username => :self, + :password => :self + }, :raise_on_unhandled => false) @working_dir = R10K::SVN::WorkingDir.new(@path, :username => @username, :password => @password) end @@ -66,15 +73,25 @@ class R10K::Module::SVN < R10K::Module:: end end + # @param [Hash] opts Deprecated + # @return [Boolean] true if the module was updated, false otherwise def sync(opts={}) - case status - when :absent - install - when :mismatched - reinstall - when :outdated - update + updated = false + if should_sync? + case status + when :absent + install + updated = true + when :mismatched + reinstall + updated = true + when :outdated + update + updated = true + end + maybe_delete_spec_dir end + updated end def exist? diff -pruN 3.7.0-2.1/lib/r10k/module/tarball.rb 3.15.4-1/lib/r10k/module/tarball.rb --- 3.7.0-2.1/lib/r10k/module/tarball.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/lib/r10k/module/tarball.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,101 @@ +require 'r10k/module' +require 'r10k/util/setopts' +require 'r10k/tarball' + +# This class defines a tarball source module implementation +class R10K::Module::Tarball < R10K::Module::Base + + R10K::Module.register(self) + + def self.implement?(name, args) + args.is_a?(Hash) && args[:type].to_s == 'tarball' + rescue + false + end + + def self.statically_defined_version(name, args) + args[:version] || args[:checksum] + end + + # @!attribute [r] tarball + # @api private + # @return [R10K::Tarball] + attr_reader :tarball + + include R10K::Util::Setopts + + def initialize(name, dirname, opts, environment=nil) + super + setopts(opts, { + # Standard option interface + :source => :self, + :version => :checksum, + :type => ::R10K::Util::Setopts::Ignore, + :overrides => :self, + + # Type-specific options + :checksum => :self, + }) + + @tarball = R10K::Tarball.new(name, @source, checksum: @checksum) + end + + # Return the status of the currently installed module. + # + # @return [Symbol] + def status + if not path.exist? + :absent + elsif not (tarball.cache_valid? && tarball.insync?(path.to_s)) + :mismatched + else + :insync + end + end + + # Synchronize this module with the indicated state. + # @param [Hash] opts Deprecated + # @return [Boolean] true if the module was updated, false otherwise + def sync(opts={}) + tarball.get unless tarball.cache_valid? + if should_sync? + case status + when :absent + tarball.unpack(path.to_s) + when :mismatched + path.rmtree + tarball.unpack(path.to_s) + end + maybe_delete_spec_dir + true + else + false + end + end + + # Return the desired version of this module + def version + @checksum || '(present)' + end + + # Return the properties of the module + # + # @return [Hash] + # @abstract + def properties + { + :expected => version, + :actual => ((state = status) == :insync) ? version : state, + :type => :tarball, + } + end + + # Tarball caches are files, not directories. An important purpose of this + # method is to indicate where the cache "path" is, for locking/parallelism, + # so for the Tarball module type, the relevant path location is returned. + # + # @return [String] The path this module will cache its tarball source to + def cachedir + tarball.cache_path + end +end diff -pruN 3.7.0-2.1/lib/r10k/module_loader/puppetfile/dsl.rb 3.15.4-1/lib/r10k/module_loader/puppetfile/dsl.rb --- 3.7.0-2.1/lib/r10k/module_loader/puppetfile/dsl.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/lib/r10k/module_loader/puppetfile/dsl.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,42 @@ +module R10K + module ModuleLoader + class Puppetfile + class DSL + # A barebones implementation of the Puppetfile DSL + # + # @api private + + def initialize(librarian, metadata_only: false) + @librarian = librarian + @metadata_only = metadata_only + end + + def mod(name, args = nil) + if args.is_a?(Hash) + opts = args + else + opts = { type: 'forge', version: args } + end + + if @metadata_only + @librarian.add_module_metadata(name, opts) + else + @librarian.add_module(name, opts) + end + end + + def forge(location) + @librarian.set_forge(location) + end + + def moduledir(location) + @librarian.set_moduledir(location) + end + + def method_missing(method, *args) + raise NoMethodError, _("unrecognized declaration '%{method}'") % {method: method} + end + end + end + end +end diff -pruN 3.7.0-2.1/lib/r10k/module_loader/puppetfile.rb 3.15.4-1/lib/r10k/module_loader/puppetfile.rb --- 3.7.0-2.1/lib/r10k/module_loader/puppetfile.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/lib/r10k/module_loader/puppetfile.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,299 @@ +require 'r10k/errors' +require 'r10k/logging' +require 'r10k/module' +require 'r10k/module_loader/puppetfile/dsl' + +require 'pathname' + +module R10K + module ModuleLoader + class Puppetfile + + include R10K::Logging + + DEFAULT_MODULEDIR = 'modules' + DEFAULT_PUPPETFILE_NAME = 'Puppetfile' + + attr_accessor :default_branch_override, :environment + attr_reader :modules, :moduledir, :puppetfile_path, + :managed_directories, :desired_contents, :purge_exclusions, + :environment_name + + # @param basedir [String] The path that contains the moduledir & + # Puppetfile by default. May be an environment, project, or + # simple directory. + # @param puppetfile [String] The path to the Puppetfile, either an + # absolute full path or a relative path with regards to the basedir. + # @param moduledir [String] The path to the moduledir, either an + # absolute full path or a relative path with regards to the basedir. + # @param forge [String] The url (without protocol) to the Forge + # @param overrides [Hash] Configuration for loaded modules' behavior + # @param environment [R10K::Environment] When provided, the environment + # in which loading takes place + # @param module_exclude_regex [Regex] A regex to exclude modules from + # installation. Helpful in CI environments. + def initialize(basedir:, + moduledir: DEFAULT_MODULEDIR, + puppetfile: DEFAULT_PUPPETFILE_NAME, + overrides: {}, + environment: nil, + module_exclude_regex: nil) + + @basedir = cleanpath(basedir) + @moduledir = resolve_path(@basedir, moduledir) + @puppetfile_path = resolve_path(@basedir, puppetfile) + @overrides = overrides + @environment = environment + @module_exclude_regex = module_exclude_regex + @environment_name = @environment&.name + @default_branch_override = @overrides.dig(:environments, :default_branch_override) + @allow_puppetfile_forge = @overrides.dig(:forge, :allow_puppetfile_override) + + @existing_module_metadata = [] + @existing_module_versions_by_name = {} + @modules = [] + + @managed_directories = [] + @desired_contents = [] + @purge_exclusions = [] + end + + def load + with_readable_puppetfile(@puppetfile_path) do + self.load! + end + end + + def load! + logger.info _("Using Puppetfile '%{puppetfile}'") % {puppetfile: @puppetfile_path} + logger.debug _("Using moduledir '%{moduledir}'") % {moduledir: @moduledir} + + dsl = R10K::ModuleLoader::Puppetfile::DSL.new(self) + dsl.instance_eval(puppetfile_content(@puppetfile_path), @puppetfile_path) + + validate_no_duplicate_names(@modules) + @modules = filter_modules(@modules, @module_exclude_regex) if @module_exclude_regex + + managed_content = @modules.group_by(&:dirname) + + @managed_directories = determine_managed_directories(managed_content) + @desired_contents = determine_desired_contents(managed_content) + @purge_exclusions = determine_purge_exclusions(@managed_directories) + + { + modules: @modules, + managed_directories: @managed_directories, + desired_contents: @desired_contents, + purge_exclusions: @purge_exclusions + } + + rescue SyntaxError, LoadError, ArgumentError, NameError => e + raise R10K::Error.wrap(e, _("Failed to evaluate %{path}") % {path: @puppetfile_path}) + end + + def load_metadata + with_readable_puppetfile(@puppetfile_path) do + self.load_metadata! + end + end + + def load_metadata! + dsl = R10K::ModuleLoader::Puppetfile::DSL.new(self, metadata_only: true) + dsl.instance_eval(puppetfile_content(@puppetfile_path), @puppetfile_path) + + @existing_module_versions_by_name = @existing_module_metadata.map {|mod| [ mod.name, mod.version ] }.to_h + empty_load_output.merge(modules: @existing_module_metadata) + + rescue SyntaxError, LoadError, ArgumentError, NameError => e + logger.warn _("Unable to preload Puppetfile because of %{msg}" % { msg: e.message }) + end + + def add_module_metadata(name, info) + install_path, metadata_info, _ = parse_module_definition(name, info) + + mod = R10K::Module.from_metadata(name, install_path, metadata_info, @environment) + + @existing_module_metadata << mod + end + + ## + ## set_forge, set_moduledir, and add_module are used directly by the DSL class + ## + + # @param [String] forge + def set_forge(forge) + if @allow_puppetfile_forge + logger.debug _("Using Forge from Puppetfile: %{forge}") % { forge: forge } + PuppetForge.host = forge + else + logger.debug _("Ignoring Forge declaration in Puppetfile, using value from settings: %{forge}.") % { forge: PuppetForge.host } + end + end + + # @param [String] moduledir + def set_moduledir(moduledir) + @moduledir = resolve_path(@basedir, moduledir) + end + + # @param [String] name + # @param [Hash, String, Symbol, nil] info Calling with + # anything but a Hash is deprecated. The DSL will now convert + # String and Symbol versions to Hashes of the shape + # { version: } + # + # String inputs should be valid module versions, the Symbol + # `:latest` is allowed, as well as `nil`. + # + # Non-Hash inputs are only ever used by Forge modules. In + # future versions this method will require the caller (the + # DSL class, not the Puppetfile author) to do this conversion + # itself. + # + def add_module(name, info) + install_path, metadata_info, spec_deletable = parse_module_definition(name, info) + + mod = R10K::Module.from_metadata(name, install_path, metadata_info, @environment) + mod.origin = :puppetfile + mod.spec_deletable = spec_deletable + + # Do not save modules if they would conflict with the attached + # environment + if @environment && @environment.module_conflicts?(mod) + return @modules + end + + # If this module's metadata has a static version, and that version + # matches the existing module declaration, and it ostensibly + # has already has been deployed to disk, use it. Otherwise create a + # regular module to sync. + unless mod.version && + mod.version == @existing_module_versions_by_name[mod.name] && + File.directory?(mod.path) + mod = mod.to_implementation + end + + @modules << mod + end + + # @deprecated + # @return [String] The base directory that contains the Puppetfile + def basedir + logger.warn _('"basedir" is deprecated. Please use "environment_name" instead. "basedir" will be removed in a future version.') + @basedir + end + + private + + def empty_load_output + { + modules: [], + managed_directories: [], + desired_contents: [], + purge_exclusions: [] + } + end + + def with_readable_puppetfile(puppetfile_path, &block) + if File.readable?(puppetfile_path) + block.call + else + logger.debug _("Puppetfile %{path} missing or unreadable") % {path: puppetfile_path.inspect} + + empty_load_output + end + end + + def parse_module_definition(name, info) + # The only valid (deprecated) way a module can be defined with a + # non-hash info is if it is a Forge module. + if !info.is_a?(Hash) + info = { type: 'forge', version: info } + end + + info[:overrides] = @overrides + + if @default_branch_override + info[:default_branch_override] = @default_branch_override + end + + spec_deletable = false + if install_path = info.delete(:install_path) + install_path = resolve_path(@basedir, install_path) + validate_install_path(install_path, name) + else + install_path = @moduledir + spec_deletable = true + end + + return [ install_path, info, spec_deletable ] + end + + def filter_modules(modules, exclude_regex) + modules.reject { |mod| mod.name =~ /#{exclude_regex}/ } + end + + # @param [Array] modules + def validate_no_duplicate_names(modules) + dupes = modules + .group_by { |mod| mod.name } + .select { |_, mods| mods.size > 1 } + .map(&:first) + unless dupes.empty? + msg = _('Puppetfiles cannot contain duplicate module names.') + msg += ' ' + msg += _("Remove the duplicates of the following modules: %{dupes}" % { dupes: dupes.join(' ') }) + raise R10K::Error.new(msg) + end + end + + def resolve_path(base, path) + if Pathname.new(path).absolute? + cleanpath(path) + else + cleanpath(File.join(base, path)) + end + end + + def validate_install_path(path, modname) + unless /^#{Regexp.escape(@basedir)}.*/ =~ path + raise R10K::Error.new("Puppetfile cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{@basedir}") + end + + true + end + + def determine_managed_directories(managed_content) + managed_content.keys.reject { |dir| dir == @basedir } + end + + # Returns an array of the full paths to all the content being managed. + # @return [Array] + def determine_desired_contents(managed_content) + managed_content.flat_map do |install_path, mods| + mods.collect { |mod| File.join(install_path, mod.name) } + end + end + + def determine_purge_exclusions(managed_dirs) + if environment && environment.respond_to?(:desired_contents) + managed_dirs + environment.desired_contents + else + managed_dirs + end + end + + # .cleanpath is as close to a canonical path as we can do without touching + # the filesystem. The .realpath methods will choke if some of the + # intermediate paths are missing, even though in some cases we will create + # them later as needed. + def cleanpath(path) + Pathname.new(path).cleanpath.to_s + end + + # For testing purposes only + def puppetfile_content(path) + File.read(path) + end + end + end +end diff -pruN 3.7.0-2.1/lib/r10k/module.rb 3.15.4-1/lib/r10k/module.rb --- 3.7.0-2.1/lib/r10k/module.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/module.rb 2023-01-19 00:49:17.000000000 +0000 @@ -17,22 +17,41 @@ module R10K::Module # # @param [String] name The unique name of the module # @param [String] basedir The root to install the module in - # @param [Object] args An arbitary value or set of values that specifies the implementation + # @param [Hash] args An arbitary Hash that specifies the implementation # @param [R10K::Environment] environment Optional environment that this module is a part of # # @return [Object < R10K::Module] A member of the implementing subclass def self.new(name, basedir, args, environment=nil) + with_implementation(name, args) do |implementation| + implementation.new(name, basedir, args, environment) + end + end + + # Takes the same signature as Module.new but returns an metadata module + def self.from_metadata(name, basedir, args, environment=nil) + with_implementation(name, args) do |implementation| + R10K::Module::Definition.new(name, + dirname: basedir, + args: args, + implementation: implementation, + environment: environment) + end + end + + def self.with_implementation(name, args, &block) if implementation = @klasses.find { |klass| klass.implement?(name, args) } - obj = implementation.new(name, basedir, args, environment) - obj + block.call(implementation) else raise _("Module %{name} with args %{args} doesn't have an implementation. (Are you using the right arguments?)") % {name: name, args: args.inspect} end end + require 'r10k/module/base' require 'r10k/module/git' require 'r10k/module/svn' require 'r10k/module/local' require 'r10k/module/forge' + require 'r10k/module/definition' + require 'r10k/module/tarball' end diff -pruN 3.7.0-2.1/lib/r10k/puppetfile.rb 3.15.4-1/lib/r10k/puppetfile.rb --- 3.7.0-2.1/lib/r10k/puppetfile.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/puppetfile.rb 2023-01-19 00:49:17.000000000 +0000 @@ -3,10 +3,18 @@ require 'pathname' require 'r10k/module' require 'r10k/util/purgeable' require 'r10k/errors' +require 'r10k/content_synchronizer' +require 'r10k/module_loader/puppetfile/dsl' +require 'r10k/module_loader/puppetfile' module R10K + +# Deprecated, use R10K::ModuleLoader::Puppetfile#load to load content, +# provide the `:modules` key of the returned Hash to +# R10K::ContentSynchronizer (either the `serial_sync` or `concurrent_sync`) +# and the remaining keys (`:managed_directories`, `:desired_contents`, and +# `:purge_exclusions`) to R10K::Util::Cleaner. class Puppetfile - # Defines the data members of a Puppetfile include R10K::Settings::Mixin @@ -18,289 +26,186 @@ class Puppetfile # @return [String] The URL to use for the Puppet Forge attr_reader :forge - # @!attribute [r] modules - # @return [Array] - attr_reader :modules - # @!attribute [r] basedir # @return [String] The base directory that contains the Puppetfile attr_reader :basedir - # @!attribute [r] moduledir - # @return [String] The directory to install the modules #{basedir}/modules - attr_reader :moduledir - - # @!attrbute [r] puppetfile_path - # @return [String] The path to the Puppetfile - attr_reader :puppetfile_path - - # @!attribute [rw] environment + # @!attribute [r] environment # @return [R10K::Environment] Optional R10K::Environment that this Puppetfile belongs to. - attr_accessor :environment + attr_reader :environment # @!attribute [rw] force # @return [Boolean] Overwrite any locally made changes attr_accessor :force - # @!attribute [r] modules_by_vcs_cachedir - # @api private Only exposed for testing purposes - # @return [Hash{:none, String => Array}] - attr_reader :modules_by_vcs_cachedir + # @!attribute [r] overrides + # @return [Hash] Various settings overridden from normal configs + attr_reader :overrides + + # @!attribute [r] loader + # @return [R10K::ModuleLoader::Puppetfile] The internal module loader + attr_reader :loader # @param [String] basedir - # @param [String] moduledir The directory to install the modules, default to #{basedir}/modules - # @param [String] puppetfile_path The path to the Puppetfile, default to #{basedir}/Puppetfile - # @param [String] puppetfile_name The name of the Puppetfile, default to 'Puppetfile' - # @param [Boolean] force Shall we overwrite locally made changes? - def initialize(basedir, moduledir = nil, puppetfile_path = nil, puppetfile_name = nil, force = nil ) + # @param [Hash, String, nil] options_or_moduledir The directory to install the modules or a Hash of options. + # Usage as moduledir is deprecated. Only use as options, defaults to nil + # @param [String, nil] puppetfile_path Deprecated - The path to the Puppetfile, defaults to nil + # @param [String, nil] puppetfile_name Deprecated - The name of the Puppetfile, defaults to nil + # @param [Boolean, nil] force Deprecated - Shall we overwrite locally made changes? + def initialize(basedir, options_or_moduledir = nil, deprecated_path_arg = nil, deprecated_name_arg = nil, deprecated_force_arg = nil) @basedir = basedir - @force = force || false - @moduledir = moduledir || File.join(basedir, 'modules') - @puppetfile_name = puppetfile_name || 'Puppetfile' - @puppetfile_path = puppetfile_path || File.join(basedir, @puppetfile_name) - - logger.info _("Using Puppetfile '%{puppetfile}'") % {puppetfile: @puppetfile_path} - - @modules = [] - @managed_content = {} - @modules_by_vcs_cachedir = {} + if options_or_moduledir.is_a? Hash + options = options_or_moduledir + deprecated_moduledir_arg = nil + else + options = {} + deprecated_moduledir_arg = options_or_moduledir + end + + @force = deprecated_force_arg || options.delete(:force) || false + @moduledir = deprecated_moduledir_arg || options.delete(:moduledir) || File.join(basedir, 'modules') + puppetfile_name = deprecated_name_arg || options.delete(:puppetfile_name) || 'Puppetfile' + puppetfile_path = deprecated_path_arg || options.delete(:puppetfile_path) + @puppetfile = puppetfile_path || puppetfile_name + @environment = options.delete(:environment) + + @overrides = options.delete(:overrides) || {} + @default_branch_override = @overrides.dig(:environments, :default_branch_override) + @forge = 'forgeapi.puppetlabs.com' + @loader = ::R10K::ModuleLoader::Puppetfile.new( + basedir: @basedir, + moduledir: @moduledir, + puppetfile: @puppetfile, + overrides: @overrides, + environment: @environment + ) + + @loaded_content = { + modules: [], + managed_directories: [], + desired_contents: [], + purge_exclusions: [] + } + @loaded = false end + # @param [String] default_branch_override The default branch to use + # instead of one specified in the module declaration, if applicable. + # Deprecated, use R10K::ModuleLoader::Puppetfile directly and pass + # the default_branch_override as an option on initialization. def load(default_branch_override = nil) - return true if self.loaded? - if File.readable? @puppetfile_path - self.load!(default_branch_override) + if self.loaded? + return @loaded_content else - logger.debug _("Puppetfile %{path} missing or unreadable") % {path: @puppetfile_path.inspect} + if !File.readable?(puppetfile_path) + logger.debug _("Puppetfile %{path} missing or unreadable") % {path: puppetfile_path.inspect} + else + self.load!(default_branch_override) + end end end + # @param [String] default_branch_override The default branch to use + # instead of one specified in the module declaration, if applicable. + # Deprecated, use R10K::ModuleLoader::Puppetfile directly and pass + # the default_branch_override as an option on initialization. def load!(default_branch_override = nil) - @default_branch_override = default_branch_override - dsl = R10K::Puppetfile::DSL.new(self) - dsl.instance_eval(puppetfile_contents, @puppetfile_path) - - validate_no_duplicate_names(@modules) + if default_branch_override && (default_branch_override != @default_branch_override) + logger.warn("Mismatch between passed and initialized default branch overrides, preferring passed value.") + @loader.default_branch_override = default_branch_override + end + + @loaded_content = @loader.load! @loaded = true - rescue SyntaxError, LoadError, ArgumentError, NameError => e - raise R10K::Error.wrap(e, _("Failed to evaluate %{path}") % {path: @puppetfile_path}) + + @loaded_content end def loaded? @loaded end - # @param [Array] modules - def validate_no_duplicate_names(modules) - dupes = modules - .group_by { |mod| mod.name } - .select { |_, v| v.size > 1 } - .map(&:first) - unless dupes.empty? - msg = _('Puppetfiles cannot contain duplicate module names.') - msg += ' ' - msg += _("Remove the duplicates of the following modules: %{dupes}" % { dupes: dupes.join(' ') }) - raise R10K::Error.new(msg) - end + def modules + @loaded_content[:modules] end - # @param [String] forge - def set_forge(forge) - @forge = forge + # @see R10K::ModuleLoader::Puppetfile#add_module for upcoming signature changes + def add_module(name, args) + @loader.add_module(name, args) end - # @param [String] moduledir - def set_moduledir(moduledir) - @moduledir = if Pathname.new(moduledir).absolute? - moduledir - else - File.join(basedir, moduledir) - end + def set_moduledir(dir) + @loader.set_moduledir(dir) end - # @param [String] name - # @param [*Object] args - def add_module(name, args) - if args.is_a?(Hash) && install_path = args.delete(:install_path) - install_path = resolve_install_path(install_path) - validate_install_path(install_path, name) - else - install_path = @moduledir - end - - if args.is_a?(Hash) && @default_branch_override != nil - args[:default_branch] = @default_branch_override - end + def set_forge(forge) + @loader.set_forge(forge) + end - # Keep track of all the content this Puppetfile is managing to enable purging. - @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path) + def moduledir + @loader.moduledir + end - mod = R10K::Module.new(name, install_path, args, @environment) - mod.origin = 'Puppetfile' + def puppetfile_path + @loader.puppetfile_path + end - @managed_content[install_path] << mod.name - cachedir = mod.cachedir - @modules_by_vcs_cachedir[cachedir] ||= [] - @modules_by_vcs_cachedir[cachedir] << mod - @modules << mod + def environment=(env) + @loader.environment = env + @environment = env end include R10K::Util::Purgeable def managed_directories - self.load unless @loaded + self.load - dirs = @managed_content.keys - dirs.delete(real_basedir) - dirs + @loaded_content[:managed_directories] end # Returns an array of the full paths to all the content being managed. # @note This implements a required method for the Purgeable mixin # @return [Array] def desired_contents - self.load unless @loaded + self.load - @managed_content.flat_map do |install_path, modnames| - modnames.collect { |name| File.join(install_path, name) } - end + @loaded_content[:desired_contents] end def purge_exclusions - exclusions = managed_directories - - if environment && environment.respond_to?(:desired_contents) - exclusions += environment.desired_contents - end + self.load - exclusions + @loaded_content[:purge_exclusions] end def accept(visitor) pool_size = self.settings[:pool_size] if pool_size > 1 - concurrent_accept(visitor, pool_size) + R10K::ContentSynchronizer.concurrent_accept(modules, visitor, self, pool_size, logger) else - serial_accept(visitor) - end - end - - private - - def serial_accept(visitor) - visitor.visit(:puppetfile, self) do - modules.each do |mod| - mod.accept(visitor) - end - end - end - - def concurrent_accept(visitor, pool_size) - logger.debug _("Updating modules with %{pool_size} threads") % {pool_size: pool_size} - mods_queue = modules_queue(visitor) - thread_pool = pool_size.times.map { visitor_thread(visitor, mods_queue) } - thread_exception = nil - - # If any threads raise an exception the deployment is considered a failure. - # In that event clear the queue, wait for other threads to finish their - # current work, then re-raise the first exception caught. - begin - thread_pool.each(&:join) - rescue => e - logger.error _("Error during concurrent deploy of a module: %{message}") % {message: e.message} - mods_queue.clear - thread_exception ||= e - retry - ensure - raise thread_exception unless thread_exception.nil? - end - end - - def modules_queue(visitor) - Queue.new.tap do |queue| - visitor.visit(:puppetfile, self) do - modules_by_cachedir = modules_by_vcs_cachedir.clone - modules_without_vcs_cachedir = modules_by_cachedir.delete(:none) || [] - - modules_without_vcs_cachedir.each {|mod| queue << Array(mod) } - modules_by_cachedir.values.each {|mods| queue << mods } - end + R10K::ContentSynchronizer.serial_accept(modules, visitor, self) end end - public :modules_queue - def visitor_thread(visitor, mods_queue) - Thread.new do - begin - while mods = mods_queue.pop(true) do - mods.each {|mod| mod.accept(visitor) } - end - rescue ThreadError => e - logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id} - Thread.exit - rescue => e - Thread.main.raise(e) - end - end - end - - def puppetfile_contents - File.read(@puppetfile_path) - end - - def resolve_install_path(path) - pn = Pathname.new(path) - - unless pn.absolute? - pn = Pathname.new(File.join(basedir, path)) + def sync + pool_size = self.settings[:pool_size] + if pool_size > 1 + R10K::ContentSynchronizer.concurrent_sync(modules, pool_size, logger) + else + R10K::ContentSynchronizer.serial_sync(modules) end - - # .cleanpath is as good as we can do without touching the filesystem. - # The .realpath methods will also choke if some of the intermediate - # paths are missing, even though we will create them later as needed. - pn.cleanpath.to_s end - def validate_install_path(path, modname) - unless /^#{Regexp.escape(real_basedir)}.*/ =~ path - raise R10K::Error.new("Puppetfile cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{real_basedir}") - end - - true - end + private def real_basedir Pathname.new(basedir).cleanpath.to_s end - class DSL - # A barebones implementation of the Puppetfile DSL - # - # @api private - - def initialize(librarian) - @librarian = librarian - end - - def mod(name, args = nil) - @librarian.add_module(name, args) - end - - def forge(location) - @librarian.set_forge(location) - end - - def moduledir(location) - @librarian.set_moduledir(location) - end - - def method_missing(method, *args) - raise NoMethodError, _("unrecognized declaration '%{method}'") % {method: method} - end - end + DSL = R10K::ModuleLoader::Puppetfile::DSL end end diff -pruN 3.7.0-2.1/lib/r10k/settings/container.rb 3.15.4-1/lib/r10k/settings/container.rb --- 3.7.0-2.1/lib/r10k/settings/container.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/settings/container.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,3 +1,4 @@ +require 'set' # Defines a collection for application settings # # This implements a hierarchical interface to application settings. Containers diff -pruN 3.7.0-2.1/lib/r10k/settings/definition.rb 3.15.4-1/lib/r10k/settings/definition.rb --- 3.7.0-2.1/lib/r10k/settings/definition.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/settings/definition.rb 2023-01-19 00:49:17.000000000 +0000 @@ -90,7 +90,7 @@ module R10K def resolve if !@value.nil? @value - elsif @default + elsif !@default.nil? if @default == :inherit # walk all the way up to root, starting with grandparent ancestor = parent diff -pruN 3.7.0-2.1/lib/r10k/settings.rb 3.15.4-1/lib/r10k/settings.rb --- 3.7.0-2.1/lib/r10k/settings.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/settings.rb 2023-01-19 00:49:17.000000000 +0000 @@ -37,6 +37,27 @@ module R10K Only used by the 'rugged' Git provider.", }), + Definition.new(:oauth_token, { + :desc => "The path to a token file for Git OAuth remotes. + Only used by the 'rugged' Git provider." + }), + + Definition.new(:github_app_id, { + :desc => "The Github App id for Git SSL remotes. + Only used by the 'rugged' Git provider." + }), + + Definition.new(:github_app_key, { + :desc => "The Github App private key for Git SSL remotes. + Only used by the 'rugged' Git provider." + }), + + Definition.new(:github_app_ttl, { + :desc => "The ttl expiration for SSL tokens. + Only used by the 'rugged' Git provider.", + :default => "120", + }), + URIDefinition.new(:proxy, { :desc => "An optional proxy server to use when interacting with Git sources via HTTP(S).", :default => :inherit, @@ -54,11 +75,35 @@ module R10K :default => :inherit, }), + Definition.new(:oauth_token, { + :desc => "The path to a token file for Git OAuth remotes. + Only used by the 'rugged' Git provider.", + :default => :inherit + }), + + Definition.new(:github_app_id, { + :desc => "The Github App id for Git SSL remotes. + Only used by the 'rugged' Git provider.", + :default => :inherit + }), + + Definition.new(:github_app_key, { + :desc => "The Github App private key for Git SSL remotes. + Only used by the 'rugged' Git provider.", + :default => :inherit + }), + + Definition.new(:github_app_ttl, { + :desc => "The ttl expiration for Git SSL tokens. + Only used by the 'rugged' Git provider.", + :default => :inherit + }), + URIDefinition.new(:proxy, { :desc => "An optional proxy server to use when interacting with Git sources via HTTP(S).", :default => :inherit, }), - + Definition.new(:ignore_branch_prefixes, { :desc => "Array of strings used to prefix branch names that will not be deployed as environments.", }), @@ -81,6 +126,20 @@ module R10K URIDefinition.new(:baseurl, { :desc => "The URL to the Puppet Forge to use for downloading modules." }), + + Definition.new(:authorization_token, { + :desc => "The token for Puppet Forge authorization. Leave blank for unauthorized or license-based connections." + }), + + Definition.new(:allow_puppetfile_override, { + :desc => "Whether to use `forge` declarations in the Puppetfile as an override of `baseurl`.", + :default => false, + :validate => lambda do |value| + unless !!value == value + raise ArgumentError, "`allow_puppetfile_override` can only be a boolean value, not '#{value}'" + end + end + }) ]) end @@ -111,11 +170,16 @@ module R10K end, }), - Definition.new(:purge_whitelist, { + Definition.new(:purge_allowlist, { :desc => "A list of filename patterns to be excluded from any purge operations. Patterns are matched relative to the root of each deployed environment, if you want a pattern to match recursively you need to use the '**' glob in your pattern. Basic shell style globs are supported.", :default => [], }), + Definition.new(:purge_whitelist, { + :desc => "Deprecated; please use purge_allowlist instead. This setting will be removed in a future version.", + :default => [], + }), + Definition.new(:generate_types, { :desc => "Controls whether to generate puppet types after deploying an environment. Defaults to false.", :default => false, @@ -142,6 +206,47 @@ module R10K end end }), + Definition.new(:exclude_spec, { + :desc => "Whether or not to deploy the spec dir of a module. Defaults to false.", + :default => false, + :validate => lambda do |value| + unless !!value == value + raise ArgumentError, "`exclude_spec` can only be a boolean value, not '#{value}'" + end + end + })]) + end + + def self.logging_settings + R10K::Settings::Collection.new(:logging, [ + Definition.new(:level, { + desc: 'What logging level should R10k run on if not specified at runtime.', + validate: lambda do |value| + if R10K::Logging.parse_level(value).nil? + raise ArgumentError, "`level` must be a valid log level. + Valid levels are #{R10K::Logging::LOG_LEVELS.map(&:downcase).inspect}" + end + end + }), + + Definition.new(:outputs, { + desc: 'Additional log outputs to use.', + validate: lambda do |value| + unless value.is_a?(Array) + raise ArgumentError, "The `outputs` setting should be an array of outputs, not a #{value.class}" + end + end + }), + + Definition.new(:disable_default_stderr, { + desc: 'Disable the default stderr logging output', + default: false, + validate: lambda do |value| + unless !!value == value + raise ArgumentError, "`disable_default_stderr` can only be a boolean value, not '#{value}'" + end + end + }) ]) end @@ -161,7 +266,7 @@ module R10K }), Definition.new(:postrun, { - :desc => "The command r10k should run after deploying environments.", + :desc => "The command r10k should run after deploying environments or modules.", :validate => lambda do |value| if !value.is_a?(Array) raise ArgumentError, "The postrun setting should be an array of strings, not a #{value.class}" @@ -199,6 +304,8 @@ module R10K R10K::Settings.git_settings, R10K::Settings.deploy_settings, + + R10K::Settings.logging_settings ]) end end diff -pruN 3.7.0-2.1/lib/r10k/source/base.rb 3.15.4-1/lib/r10k/source/base.rb --- 3.7.0-2.1/lib/r10k/source/base.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/source/base.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,8 +1,12 @@ +require 'r10k/logging' + # This class defines a common interface for source implementations. # # @since 1.3.0 class R10K::Source::Base + include R10K::Logging + # @!attribute [r] basedir # @return [String] The path this source will place environments in attr_reader :basedir @@ -31,10 +35,15 @@ class R10K::Source::Base # @option options [Boolean, String] :prefix If a String this becomes the prefix. # If true, will use the source name as the prefix. All sources should respect this option. # Defaults to false for no environment prefix. + # @option options [String] :strip_component If a string, this value will be + # removed from the beginning of each generated environment's name, if + # present. If the string is contained within two "/" characters, it will + # be treated as a regular expression. def initialize(name, basedir, options = {}) @name = name @basedir = Pathname.new(basedir).cleanpath.to_s @prefix = options.delete(:prefix) + @strip_component = options.delete(:strip_component) @puppetfile_name = options.delete(:puppetfile_name) @options = options end @@ -54,6 +63,16 @@ class R10K::Source::Base end + # Perform actions to reload environments after the `preload!`. Similar + # to preload!, and likely to include network queries and rerunning + # environment generation. + # + # @api public + # @abstract + # @return [void] + def reload! + end + # Enumerate the environments associated with this SVN source. # # @api public diff -pruN 3.7.0-2.1/lib/r10k/source/git.rb 3.15.4-1/lib/r10k/source/git.rb --- 3.7.0-2.1/lib/r10k/source/git.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/source/git.rb 2023-01-19 00:49:17.000000000 +0000 @@ -11,8 +11,6 @@ require 'r10k/environment/name' # @since 1.3.0 class R10K::Source::Git < R10K::Source::Base - include R10K::Logging - R10K::Source.register(:git, self) # Register git as the default source R10K::Source.register(nil, self) @@ -95,18 +93,33 @@ class R10K::Source::Git < R10K::Source:: end end + def reload! + @cache.sync! + @environments = generate_environments() + end + def generate_environments envs = [] - branch_names.each do |bn| - if bn.valid? - envs << R10K::Environment::Git.new(bn.name, @basedir, bn.dirname, - {:remote => remote, :ref => bn.name, :puppetfile_name => puppetfile_name }) - elsif bn.correct? - logger.warn _("Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}") % {env_name: bn.name.inspect, corrected_env_name: bn.dirname} - envs << R10K::Environment::Git.new(bn.name, @basedir, bn.dirname, - {:remote => remote, :ref => bn.name, :puppetfile_name => puppetfile_name}) - elsif bn.validate? - logger.error _("Environment %{env_name} contained non-word characters, ignoring it.") % {env_name: bn.name.inspect} + environment_names.each do |en| + if en.valid? + envs << R10K::Environment::Git.new(en.name, + @basedir, + en.dirname, + {remote: remote, + ref: en.original_name, + puppetfile_name: puppetfile_name, + overrides: @options[:overrides]}) + elsif en.correct? + logger.warn _("Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}") % {env_name: en.name.inspect, corrected_env_name: en.dirname} + envs << R10K::Environment::Git.new(en.name, + @basedir, + en.dirname, + {remote: remote, + ref: en.original_name, + puppetfile_name: puppetfile_name, + overrides: @options[:overrides]}) + elsif en.validate? + logger.error _("Environment %{env_name} contained non-word characters, ignoring it.") % {env_name: en.name.inspect} end end @@ -144,19 +157,22 @@ class R10K::Source::Git < R10K::Source:: private - def branch_names - opts = {:prefix => @prefix, :invalid => @invalid_branches, :source => @name} - branches = @cache.branches + def environment_names + opts = {prefix: @prefix, + invalid: @invalid_branches, + source: @name, + strip_component: @strip_component} + branch_names = @cache.branches if @ignore_branch_prefixes && !@ignore_branch_prefixes.empty? - branches = filter_branches_by_regexp(branches, @ignore_branch_prefixes) + branch_names = filter_branches_by_regexp(branch_names, @ignore_branch_prefixes) end if @filter_command && !@filter_command.empty? - branches = filter_branches_by_command(branches, @filter_command) + branch_names = filter_branches_by_command(branch_names, @filter_command) end - branches.map do |branch| - R10K::Environment::Name.new(branch, opts) + branch_names.map do |branch_name| + R10K::Environment::Name.new(branch_name, opts) end end end diff -pruN 3.7.0-2.1/lib/r10k/source/hash.rb 3.15.4-1/lib/r10k/source/hash.rb --- 3.7.0-2.1/lib/r10k/source/hash.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/source/hash.rb 2023-01-19 00:49:17.000000000 +0000 @@ -120,8 +120,6 @@ # class R10K::Source::Hash < R10K::Source::Base - include R10K::Logging - # @param hash [Hash] A hash to validate. # @return [Boolean] False if the hash is obviously invalid. A true return # means _maybe_ it's valid. @@ -152,8 +150,10 @@ class R10K::Source::Hash < R10K::Source: R10K::Util::SymbolizeKeys.symbolize_keys!(opts) memo.merge({ name => opts.merge({ - :basedir => @basedir, - :dirname => R10K::Environment::Name.new(name, {prefix: @prefix, source: @name}).dirname + basedir: @basedir, + dirname: R10K::Environment::Name.new(name, {prefix: @prefix, + source: @name, + strip_component: @strip_component}).dirname }) }) end @@ -168,7 +168,7 @@ class R10K::Source::Hash < R10K::Source: def environments @environments ||= environments_hash.map do |name, hash| - R10K::Environment.from_hash(name, hash) + R10K::Environment.from_hash(name, hash.merge({overrides: @options[:overrides]})) end end diff -pruN 3.7.0-2.1/lib/r10k/source/svn.rb 3.15.4-1/lib/r10k/source/svn.rb --- 3.7.0-2.1/lib/r10k/source/svn.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/source/svn.rb 2023-01-19 00:49:17.000000000 +0000 @@ -65,6 +65,10 @@ class R10K::Source::SVN < R10K::Source:: @ignore_branch_prefixes = options[:ignore_branch_prefixes] end + def reload! + @environments = generate_environments() + end + # Enumerate the environments associated with this SVN source. # # @return [Array] An array of environments created @@ -103,8 +107,6 @@ class R10K::Source::SVN < R10K::Source:: @environments.map {|env| env.dirname } end - include R10K::Logging - def filter_branches(branches, ignore_prefixes) filter = Regexp.new("^(#{ignore_prefixes.join('|')})") branches = branches.reject do |branch| @@ -121,7 +123,11 @@ class R10K::Source::SVN < R10K::Source:: def names_and_paths branches = [] - opts = {:prefix => @prefix, :correct => false, :validate => false, :source => @name} + opts = {prefix: @prefix, + correct: false, + validate: false, + source: @name, + strip_component: @strip_component} branches << [R10K::Environment::Name.new('production', opts), "#{@remote}/trunk"] additional_branch_names = @svn_remote.branches if @ignore_branch_prefixes && !@ignore_branch_prefixes.empty? diff -pruN 3.7.0-2.1/lib/r10k/source/yaml.rb 3.15.4-1/lib/r10k/source/yaml.rb --- 3.7.0-2.1/lib/r10k/source/yaml.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/source/yaml.rb 2023-01-19 00:49:17.000000000 +0000 @@ -7,7 +7,7 @@ class R10K::Source::Yaml < R10K::Source: begin contents = ::YAML.load_file(config) rescue => e - raise ConfigError, _("Couldn't open environments file %{file}: %{err}") % {file: config, err: e.message} + raise R10K::ConfigError, _("Couldn't open environments file %{file}: %{err}") % {file: config, err: e.message} end # Set the environments key for the parent class to consume diff -pruN 3.7.0-2.1/lib/r10k/tarball.rb 3.15.4-1/lib/r10k/tarball.rb --- 3.7.0-2.1/lib/r10k/tarball.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/lib/r10k/tarball.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,183 @@ +require 'fileutils' +require 'find' +require 'minitar' +require 'tempfile' +require 'uri' +require 'zlib' +require 'r10k/settings' +require 'r10k/settings/mixin' +require 'r10k/util/platform' +require 'r10k/util/cacheable' +require 'r10k/util/downloader' + +module R10K + class Tarball + + include R10K::Settings::Mixin + include R10K::Util::Cacheable + include R10K::Util::Downloader + + def_setting_attr :proxy # Defaults to global proxy setting + def_setting_attr :cache_root, R10K::Util::Cacheable.default_cachedir + + # @!attribute [rw] name + # @return [String] The tarball's name + attr_accessor :name + + # @!attribute [rw] source + # @return [String] The tarball's source + attr_accessor :source + + # @!attribute [rw] checksum + # @return [String] The tarball's expected sha256 digest + attr_accessor :checksum + + # @param name [String] The name of the tarball content + # @param source [String] The source for the tarball content + # @param checksum [String] The sha256 digest of the tarball content + def initialize(name, source, checksum: nil) + @name = name + @source = source + @checksum = checksum + + # At this time, the only checksum type supported is sha256. In the future, + # we may decide to support other algorithms if a use case arises. TBD. + checksum_algorithm = :SHA256 + end + + # @return [String] Directory. Where the cache_basename file will be created. + def cache_dirname + File.join(settings[:cache_root], 'tarball') + end + + # The final cache_path should match one of the templates: + # + # - {cachedir}/{checksum}.tar.gz + # - {cachedir}/{source}.tar.gz + # + # @return [String] File. The full file path the tarball will be cached to. + def cache_path + File.join(cache_dirname, cache_basename) + end + + # @return [String] The basename of the tarball cache file. + def cache_basename + if checksum.nil? + sanitized_dirname(source) + '.tar.gz' + else + checksum + '.tar.gz' + end + end + + # Extract the cached tarball to the target directory. + # + # @param target_dir [String] Where to unpack the tarball + def unpack(target_dir) + file = File.open(cache_path, 'rb') + reader = Zlib::GzipReader.new(file) + begin + Minitar.unpack(reader, target_dir) + ensure + reader.close + end + end + + # @param target_dir [String] The directory to check if is in sync with the + # tarball content + # @param ignore_untracked_files [Boolean] If true, consider the target + # dir to be in sync as long as all tracked content matches. + # + # @return [Boolean] + def insync?(target_dir, ignore_untracked_files: false) + target_tree_entries = Find.find(target_dir).map(&:to_s) - [target_dir] + each_tarball_entry do |entry| + found = target_tree_entries.delete(File.join(target_dir, entry.full_name.chomp('/'))) + return false if found.nil? + next if entry.directory? + return false unless file_digest(found) == reader_digest(entry) + end + + if ignore_untracked_files + # We wouldn't have gotten this far if there were discrepancies in + # tracked content + true + else + # If there are still files in target_tree_entries, then there is + # untracked content present in the target tree. If not, we're in sync. + target_tree_entries.empty? + end + end + + # Download the tarball from @source to @cache_path + def get + Tempfile.open(cache_basename) do |tempfile| + tempfile.binmode + src_uri = URI.parse(source) + + temp_digest = case src_uri.scheme + when 'file', nil + copy(src_uri.path, tempfile) + when %r{^[a-z]$} # Windows drive letter + copy(src_uri.to_s, tempfile) + when %r{^https?$} + download(src_uri, tempfile) + else + raise "Unexpected source scheme #{src_uri.scheme}" + end + + # Verify the download + unless (checksum == temp_digest) || checksum.nil? + raise 'Downloaded file does not match checksum' + end + + # Move the download to cache_path + FileUtils::mkdir_p(cache_dirname) + begin + FileUtils.mv(tempfile.path, cache_path) + rescue Errno::EACCES + # It may be the case that permissions don't permit moving the file + # into place, but do permit overwriting an existing in-place file. + FileUtils.cp(tempfile.path, cache_path) + end + end + end + + # Checks the cached tarball's digest against the expected checksum. Returns + # false if no cached file is present. If the tarball has no expected + # checksum, any cached file is assumed to be valid. + # + # @return [Boolean] + def cache_valid? + return false unless File.exist?(cache_path) + return true if checksum.nil? + checksum == file_digest(cache_path) + end + + # List all of the files contained in the tarball and their paths. This is + # useful for implementing R10K::Purgable + # + # @return [Array] A normalized list of file paths contained in the archive + def paths + names = Array.new + each_tarball_entry { |entry| names << Pathname.new(entry).cleanpath.to_s } + names - ['.'] + end + + def cache_checksum + raise R10K::Error, _("Cache not present at %{path}") % {path: cache_path} unless File.exist?(cache_path) + file_digest(cache_path) + end + + private + + def each_tarball_entry(&block) + File.open(cache_path, 'rb') do |file| + Zlib::GzipReader.wrap(file) do |reader| + Archive::Tar::Minitar::Input.each_entry(reader) do |entry| + yield entry + end + end + end + end + end +end diff -pruN 3.7.0-2.1/lib/r10k/util/cacheable.rb 3.15.4-1/lib/r10k/util/cacheable.rb --- 3.7.0-2.1/lib/r10k/util/cacheable.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/lib/r10k/util/cacheable.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,31 @@ +module R10K + module Util + + # Utility mixin for classes that need to implement caches + # + # @abstract Classes using this mixin need to implement {#managed_directory} and + # {#desired_contents} + module Cacheable + + # Provide a default cachedir location. This is consumed by R10K::Settings + # for appropriate global default values. + # + # @return [String] Path to the default cache directory + def self.default_cachedir(basename = 'cache') + if R10K::Util::Platform.windows? + File.join(ENV['LOCALAPPDATA'], 'r10k', basename) + else + File.join(ENV['HOME'] || '/root', '.r10k', basename) + end + end + + # Reformat a string into something that can be used as a directory + # + # @param string [String] An identifier to create a sanitized dirname for + # @return [String] A sanitized dirname for the given string + def sanitized_dirname(string) + string.gsub(/(\w+:\/\/)(.*)(@)/, '\1').gsub(/[^@\w\.-]/, '-') + end + end + end +end diff -pruN 3.7.0-2.1/lib/r10k/util/cleaner.rb 3.15.4-1/lib/r10k/util/cleaner.rb --- 3.7.0-2.1/lib/r10k/util/cleaner.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/lib/r10k/util/cleaner.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,21 @@ +require 'r10k/logging' +require 'r10k/util/purgeable' + +module R10K + module Util + class Cleaner + + include R10K::Logging + include R10K::Util::Purgeable + + attr_reader :managed_directories, :desired_contents, :purge_exclusions + + def initialize(managed_directories, desired_contents, purge_exclusions = []) + @managed_directories = managed_directories + @desired_contents = desired_contents + @purge_exclusions = purge_exclusions + end + + end + end +end diff -pruN 3.7.0-2.1/lib/r10k/util/downloader.rb 3.15.4-1/lib/r10k/util/downloader.rb --- 3.7.0-2.1/lib/r10k/util/downloader.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/lib/r10k/util/downloader.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,134 @@ +require 'digest' +require 'net/http' + +module R10K + module Util + + # Utility mixin for classes that need to download files + module Downloader + + # Downloader objects need to checksum downloaded or saved content. The + # algorithm used to perform this checksumming (and therefore the kinds of + # checksums returned by various methods) is reported by this method. + # + # @return [Symbol] The checksum algorithm the downloader uses + def checksum_algorithm + @checksum_algorithm ||= :SHA256 + end + + private + + # Set the checksum algorithm the downloader should use. It should be a + # symbol, and a valid Ruby 'digest' library algorithm. The default is + # :SHA256. + # + # @param algorithm [Symbol] The checksum algorithm the downloader should use + def checksum_algorithm=(algorithm) + @checksum_algorithm = algorithm + end + + CHUNK_SIZE = 64 * 1024 # 64 kb + + # @param src_uri [URI] The URI to download from + # @param dst_file [String] The file or path to save to + # @return [String] The downloaded file's hex digest + def download(src_uri, dst_file) + digest = Digest(checksum_algorithm).new + http_get(src_uri) do |resp| + File.open(dst_file, 'wb') do |output_stream| + resp.read_body do |chunk| + output_stream.write(chunk) + digest.update(chunk) + end + end + end + + digest.hexdigest + end + + # @param src_file The file or path to copy from + # @param dst_file The file or path to copy to + # @return [String] The copied file's sha256 hex digest + def copy(src_file, dst_file) + digest = Digest(checksum_algorithm).new + File.open(src_file, 'rb') do |input_stream| + File.open(dst_file, 'wb') do |output_stream| + until input_stream.eof? + chunk = input_stream.read(CHUNK_SIZE) + output_stream.write(chunk) + digest.update(chunk) + end + end + end + + digest.hexdigest + end + + # Start a Net::HTTP::Get connection, then yield the Net::HTTPSuccess object + # to the caller's block. Follow redirects if Net::HTTPRedirection responses + # are encountered, and use a proxy if directed. + # + # @param uri [URI] The URI to download the file from + # @param redirect_limit [Integer] How many redirects to permit before failing + # @param proxy [URI, String] The URI to use as a proxy + def http_get(uri, redirect_limit: 10, proxy: nil, &block) + raise "HTTP redirect too deep" if redirect_limit.zero? + + session = Net::HTTP.new(uri.host, uri.port, *proxy_to_array(proxy)) + session.use_ssl = true if uri.scheme == 'https' + session.start + + begin + session.request_get(uri) do |response| + case response + when Net::HTTPRedirection + redirect = response['location'] + session.finish + return http_get(URI.parse(redirect), redirect_limit: redirect_limit - 1, proxy: proxy, &block) + when Net::HTTPSuccess + yield response + else + raise "Unexpected response code #{response.code}: #{response}" + end + end + ensure + session.finish if session.active? + end + end + + # Helper method to translate a proxy URI to array arguments for + # Net::HTTP#new. A nil argument returns nil array elements. + def proxy_to_array(proxy_uri) + if proxy_uri + px = proxy_uri.is_a?(URI) ? proxy_uri : URI.parse(proxy_uri) + [px.host, px.port, px.user, px.password] + else + [nil, nil, nil, nil] + end + end + + # Return the sha256 digest of the file at the given path + # + # @param path [String] The path to the file + # @return [String] The file's sha256 hex digest + def file_digest(path) + File.open(path) do |file| + reader_digest(file) + end + end + + # Return the sha256 digest of the readable data + # + # @param reader [String] An object that responds to #read + # @return [String] The read data's sha256 hex digest + def reader_digest(reader) + digest = Digest(checksum_algorithm).new + while chunk = reader.read(CHUNK_SIZE) + digest.update(chunk) + end + + digest.hexdigest + end + end + end +end diff -pruN 3.7.0-2.1/lib/r10k/util/purgeable.rb 3.15.4-1/lib/r10k/util/purgeable.rb --- 3.7.0-2.1/lib/r10k/util/purgeable.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/util/purgeable.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,3 +1,5 @@ +require 'r10k/logging' + require 'fileutils' module R10K @@ -9,6 +11,14 @@ module R10K # {#desired_contents} module Purgeable + include R10K::Logging + + HIDDEN_FILE = /\.[^.]+/ + + FN_MATCH_OPTS = File::FNM_PATHNAME | File::FNM_DOTMATCH + + # @deprecated + # # @!method logger # @abstract Including classes must provide a logger method # @return [Log4r::Logger] @@ -38,23 +48,79 @@ module R10K end end + # @deprecated Unused helper function + # # @return [Array] Directory contents that are expected but not present def pending_contents(recurse) desired_contents - current_contents(recurse) end + def matches?(test, path) + if test == path + true + elsif File.fnmatch?(test, path, FN_MATCH_OPTS) + true + else + false + end + end + + # A method to collect potentially purgeable content without searching into + # ignored directories when recursively searching. + # + # @param dir [String, Pathname] The directory to search for purgeable content + # @param exclusion_gobs [Array] A list of file paths or File globs + # to exclude from recursion (These are generated by the classes that + # mix this module into them and are typically programatically generated) + # @param allowed_gobs [Array] A list of file paths or File globs to exclude + # from recursion (These are passed in by the caller of purge! and typically + # are user supplied configuration values) + # @param desireds_not_to_recurse_into [Array] A list of file paths not to + # recurse into. These are programatically generated, these exist to maintain + # backwards compatibility with previous implementations that used File.globs + # for "recursion", ie "**/{*,.[^.]*}" which would not recurse into dot directories. + # @param recurse [Boolean] Whether or not to recurse into child directories that do + # not match other filters. + # + # @return [Array] Contents which may be purged. + def potentially_purgeable(dir, exclusion_globs, allowed_globs, desireds_not_to_recurse_into, recurse) + children = Pathname.new(dir).children.reject do |path| + path = path.to_s + + if exclusion_match = exclusion_globs.find { |exclusion| matches?(exclusion, path) } + logger.debug2 _("Not purging %{path} due to internal exclusion match: %{exclusion_match}") % {path: path, exclusion_match: exclusion_match} + elsif allowlist_match = allowed_globs.find { |allowed| matches?(allowed, path) } + logger.debug _("Not purging %{path} due to whitelist match: %{allowlist_match}") % {path: path, allowlist_match: allowlist_match} + else + desired_match = desireds_not_to_recurse_into.grep(path).first + end + + !!exclusion_match || !!allowlist_match || !!desired_match + end + + children.flat_map do |child| + if File.directory?(child) && !File.symlink?(child) && recurse + potentially_purgeable(child, exclusion_globs, allowed_globs, desireds_not_to_recurse_into, recurse) << child.to_s + else + child.to_s + end + end + end + # @return [Array] Directory contents that are present but not expected def stale_contents(recurse, exclusions, whitelist) - fn_match_opts = File::FNM_PATHNAME | File::FNM_DOTMATCH + dirs = self.managed_directories + desireds = self.desired_contents + hidden_desireds, regular_desireds = desireds.partition do |desired| + HIDDEN_FILE.match(File.basename(desired)) + end - (current_contents(recurse) - desired_contents).reject do |item| - if exclusion_match = exclusions.find { |ex_item| (ex_item == item) || File.fnmatch?(ex_item, item, fn_match_opts) } - logger.debug2 _("Not purging %{item} due to internal exclusion match: %{exclusion_match}") % {item: item, exclusion_match: exclusion_match} - elsif whitelist_match = whitelist.find { |wl_item| (wl_item == item) || File.fnmatch?(wl_item, item, fn_match_opts) } - logger.debug _("Not purging %{item} due to whitelist match: %{whitelist_match}") % {item: item, whitelist_match: whitelist_match} - end + initial_purgelist = dirs.flat_map do |dir| + potentially_purgeable(dir, exclusions, whitelist, hidden_desireds, recurse) + end - !!exclusion_match || !!whitelist_match + initial_purgelist.reject do |path| + regular_desireds.any? { |desired| matches?(desired, path) } end end diff -pruN 3.7.0-2.1/lib/r10k/util/setopts.rb 3.15.4-1/lib/r10k/util/setopts.rb --- 3.7.0-2.1/lib/r10k/util/setopts.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/util/setopts.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,3 +1,5 @@ +require 'r10k/logging' + module R10K module Util @@ -7,6 +9,10 @@ module R10K # supports Ruby 1.8.7+ we cannot use that functionality. module Setopts + class Ignore; end + + include R10K::Logging + private # @param opts [Hash] @@ -31,22 +37,39 @@ module R10K # setopts(opts, allowed) # @trace # => nil # - def setopts(opts, allowed) + def setopts(opts, allowed, raise_on_unhandled: true) + processed_vars = {} opts.each_pair do |key, value| if allowed.key?(key) - rhs = allowed[key] - case rhs - when NilClass, FalseClass - # Ignore nil options - when :self, TrueClass - # tr here is because instance variables cannot have hyphens in their names. - instance_variable_set("@#{key}".tr('-','_').to_sym, value) - else - # tr here same as previous - instance_variable_set("@#{rhs}".tr('-','_').to_sym, value) + # Ignore nil options and explicit ignore param + next unless rhs = allowed[key] + next if rhs == ::R10K::Util::Setopts::Ignore + + var = case rhs + when :self, TrueClass + # tr here is because instance variables cannot have hyphens in their names. + "@#{key}".tr('-','_').to_sym + else + # tr here same as previous + "@#{rhs}".tr('-','_').to_sym + end + + if processed_vars.include?(var) + # This should be a raise, but that would be a behavior change and + # should happen on a SemVer boundry. + logger.warn _("%{class_name} parameters '%{a}' and '%{b}' conflict. Specify one or the other, but not both" \ + % {class_name: self.class.name, a: processed_vars[var], b: key}) end + + instance_variable_set(var, value) + processed_vars[var] = key else - raise ArgumentError, _("%{class_name} cannot handle option '%{key}'") % {class_name: self.class.name, key: key} + err_str = _("%{class_name} cannot handle option '%{key}'") % {class_name: self.class.name, key: key} + if raise_on_unhandled + raise ArgumentError, err_str + else + logger.warn(err_str) + end end end end diff -pruN 3.7.0-2.1/lib/r10k/util/subprocess.rb 3.15.4-1/lib/r10k/util/subprocess.rb --- 3.7.0-2.1/lib/r10k/util/subprocess.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/util/subprocess.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,3 +1,4 @@ +require 'r10k/logging' require 'r10k/util/platform' module R10K diff -pruN 3.7.0-2.1/lib/r10k/version.rb 3.15.4-1/lib/r10k/version.rb --- 3.7.0-2.1/lib/r10k/version.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/lib/r10k/version.rb 2023-01-19 00:49:17.000000000 +0000 @@ -2,5 +2,5 @@ module R10K # When updating to a new major (X) or minor (Y) version, include `#major` or # `#minor` (respectively) in your commit message to trigger the appropriate # release. Otherwise, a new patch (Z) version will be released. - VERSION = '3.7.0' + VERSION = '3.15.4' end diff -pruN 3.7.0-2.1/locales/r10k.pot 3.15.4-1/locales/r10k.pot --- 3.7.0-2.1/locales/r10k.pot 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/locales/r10k.pot 2023-01-19 00:49:17.000000000 +0000 @@ -1,16 +1,16 @@ # SOME DESCRIPTIVE TITLE. -# Copyright (C) 2020 Puppet, Inc. +# Copyright (C) 2022 Puppet, Inc. # This file is distributed under the same license as the r10k package. -# FIRST AUTHOR , 2020. +# FIRST AUTHOR , 2022. # #, fuzzy msgid "" msgstr "" -"Project-Id-Version: r10k 3.4.1-57-g2eb088a\n" +"Project-Id-Version: r10k 3.9.3-265-gcb0a3463\n" "\n" "Report-Msgid-Bugs-To: docs@puppetlabs.com\n" -"POT-Creation-Date: 2020-07-22 16:41+0000\n" -"PO-Revision-Date: 2020-07-22 16:41+0000\n" +"POT-Creation-Date: 2022-12-08 22:58+0000\n" +"PO-Revision-Date: 2022-12-08 22:58+0000\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" "Language: \n" @@ -19,80 +19,84 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;\n" -#: ../lib/r10k/action/deploy/deploy_helpers.rb:12 ../lib/r10k/settings/loader.rb:63 +#: ../lib/r10k/action/deploy/deploy_helpers.rb:16 ../lib/r10k/settings/loader.rb:63 msgid "No configuration file given, no config file found in current directory, and no global config present" msgstr "" -#: ../lib/r10k/action/deploy/deploy_helpers.rb:26 +#: ../lib/r10k/action/deploy/deploy_helpers.rb:30 msgid "Making changes to deployed environments has been administratively disabled." msgstr "" -#: ../lib/r10k/action/deploy/deploy_helpers.rb:27 +#: ../lib/r10k/action/deploy/deploy_helpers.rb:31 msgid "Reason: %{write_lock}" msgstr "" -#: ../lib/r10k/action/deploy/environment.rb:57 +#: ../lib/r10k/action/deploy/environment.rb:116 msgid "Environment(s) \\'%{environments}\\' cannot be found in any source and will not be deployed." msgstr "" -#: ../lib/r10k/action/deploy/environment.rb:85 +#: ../lib/r10k/action/deploy/environment.rb:136 +msgid "Executing postrun command." +msgstr "" + +#: ../lib/r10k/action/deploy/environment.rb:150 msgid "Environment %{env_dir} does not match environment name filter, skipping" msgstr "" -#: ../lib/r10k/action/deploy/environment.rb:93 +#: ../lib/r10k/action/deploy/environment.rb:158 msgid "Deploying environment %{env_path}" msgstr "" -#: ../lib/r10k/action/deploy/environment.rb:96 +#: ../lib/r10k/action/deploy/environment.rb:161 msgid "Environment %{env_dir} is now at %{env_signature}" msgstr "" -#: ../lib/r10k/action/deploy/environment.rb:100 +#: ../lib/r10k/action/deploy/environment.rb:165 msgid "Environment %{env_dir} is new, updating all modules" msgstr "" -#: ../lib/r10k/action/deploy/environment.rb:143 -msgid "Deploying %{origin} content %{path}" +#: ../lib/r10k/action/deploy/module.rb:81 +msgid "Running postrun command for environments: %{envs_to_run}." msgstr "" -#: ../lib/r10k/action/deploy/module.rb:49 -msgid "Only updating modules in environment %{opt_env} skipping environment %{env_path}" +#: ../lib/r10k/action/deploy/module.rb:91 +msgid "No environments were modified, not executing postrun command." msgstr "" -#: ../lib/r10k/action/deploy/module.rb:51 -msgid "Updating modules %{modules} in environment %{env_path}" +#: ../lib/r10k/action/deploy/module.rb:103 +msgid "Only updating modules in environment(s) %{opt_env} skipping environment %{env_path}" msgstr "" -#: ../lib/r10k/action/deploy/module.rb:63 -msgid "Deploying module %{mod_path}" +#: ../lib/r10k/action/deploy/module.rb:105 +msgid "Updating modules %{modules} in environment %{env_path}" msgstr "" -#: ../lib/r10k/action/deploy/module.rb:70 -msgid "Only updating modules %{modules}, skipping module %{mod_name}" +#: ../lib/r10k/action/puppetfile/check.rb:18 +msgid "Syntax OK" msgstr "" -#: ../lib/r10k/action/puppetfile/check.rb:14 -msgid "Syntax OK" +#: ../lib/r10k/action/runner.rb:63 ../lib/r10k/deployment/config.rb:42 +msgid "Overriding config file setting '%{key}': '%{old_val}' -> '%{new_val}'" msgstr "" -#: ../lib/r10k/action/puppetfile/install.rb:30 -msgid "Updating module %{mod_path}" +#: ../lib/r10k/action/runner.rb:105 +msgid "Reading configuration from %{config_path}" msgstr "" -#: ../lib/r10k/action/puppetfile/install.rb:33 -msgid "Cannot track control repo branch for content '%{name}' when not part of a 'deploy' action, will use default if available." +#: ../lib/r10k/action/runner.rb:108 +msgid "No config file explicitly given and no default config file could be found, default settings will be used." msgstr "" -#: ../lib/r10k/action/runner.rb:53 ../lib/r10k/deployment/config.rb:42 -msgid "Overriding config file setting '%{key}': '%{old_val}' -> '%{new_val}'" +#: ../lib/r10k/content_synchronizer.rb:33 +msgid "Updating modules with %{pool_size} threads" msgstr "" -#: ../lib/r10k/action/runner.rb:86 -msgid "Reading configuration from %{config_path}" +#: ../lib/r10k/content_synchronizer.rb:46 +msgid "Error during concurrent deploy of a module: %{message}" msgstr "" -#: ../lib/r10k/action/runner.rb:89 -msgid "No config file explicitly given and no default config file could be found, default settings will be used." +#: ../lib/r10k/content_synchronizer.rb:87 +msgid "Module thread %{id} exiting: %{message}" msgstr "" #: ../lib/r10k/deployment.rb:90 @@ -103,16 +107,24 @@ msgstr "" msgid "Unable to load sources; the supplied configuration does not define the 'sources' key" msgstr "" -#: ../lib/r10k/environment/base.rb:61 ../lib/r10k/environment/base.rb:77 ../lib/r10k/environment/base.rb:86 ../lib/r10k/source/base.rb:64 +#: ../lib/r10k/environment/bare.rb:6 +msgid "\"bare\" environment type is deprecated; please use \"plain\" instead (environment: %{name})" +msgstr "" + +#: ../lib/r10k/environment/base.rb:89 ../lib/r10k/environment/base.rb:105 ../lib/r10k/environment/base.rb:114 ../lib/r10k/source/base.rb:83 msgid "%{class} has not implemented method %{method}" msgstr "" -#: ../lib/r10k/environment/with_modules.rb:104 -msgid "Puppetfile cannot contain module names defined by environment %{name}" +#: ../lib/r10k/environment/name.rb:83 +msgid "Improper configuration value given for strip_component setting in %{src} source. Value must be a string, a /regex/, false, or omitted. Got \"%{val}\" (%{type})" msgstr "" -#: ../lib/r10k/environment/with_modules.rb:106 -msgid "Remove the conflicting definitions of the following modules: %{conflicts}" +#: ../lib/r10k/environment/with_modules.rb:60 +msgid "Environment and %{src} both define the \"%{name}\" module" +msgstr "" + +#: ../lib/r10k/environment/with_modules.rb:71 +msgid "Unexpected value for `module_conflicts` setting in %{env} environment: %{val}" msgstr "" #: ../lib/r10k/feature.rb:27 @@ -139,19 +151,19 @@ msgstr "" msgid "Proc %{block} for feature %{name} returned %{output}" msgstr "" -#: ../lib/r10k/forge/module_release.rb:196 +#: ../lib/r10k/forge/module_release.rb:197 msgid "Unpacking %{tarball_cache_path} to %{target_dir} (with tmpdir %{tmp_path})" msgstr "" -#: ../lib/r10k/forge/module_release.rb:198 +#: ../lib/r10k/forge/module_release.rb:199 msgid "Valid files unpacked: %{valid_files}" msgstr "" -#: ../lib/r10k/forge/module_release.rb:200 +#: ../lib/r10k/forge/module_release.rb:201 msgid "These files existed in the module's tar file, but are invalid filetypes and were not unpacked: %{invalid_files}" msgstr "" -#: ../lib/r10k/forge/module_release.rb:203 +#: ../lib/r10k/forge/module_release.rb:204 msgid "Symlinks are unsupported and were not unpacked from the module tarball. %{release_slug} contained these ignored symlinks: %{symlinks}" msgstr "" @@ -187,11 +199,11 @@ msgstr "" msgid "Cannot write %{file}; parent directory does not exist" msgstr "" -#: ../lib/r10k/git/cache.rb:55 +#: ../lib/r10k/git/cache.rb:57 msgid "%{class}#path is deprecated; use #git_dir" msgstr "" -#: ../lib/r10k/git/cache.rb:84 +#: ../lib/r10k/git/cache.rb:86 msgid "Creating new git cache for %{remote}" msgstr "" @@ -207,39 +219,99 @@ msgstr "" msgid "Rugged versions prior to 0.24.0 do not support pruning stale branches during fetch, please upgrade your \\'rugged\\' gem. (Current version is: %{version})" msgstr "" -#: ../lib/r10k/git/rugged/credentials.rb:24 +#: ../lib/r10k/git/rugged/base_repository.rb:24 +msgid "Unable to resolve %{pattern}: %{e} " +msgstr "" + +#: ../lib/r10k/git/rugged/base_repository.rb:69 +msgid "Remote URL is different from cache, updating %{orig} to %{update}" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:28 msgid "Authentication failed for Git remote %{url}." msgstr "" -#: ../lib/r10k/git/rugged/credentials.rb:48 +#: ../lib/r10k/git/rugged/credentials.rb:52 msgid "Using per-repository private key %{key} for URL %{url}" msgstr "" -#: ../lib/r10k/git/rugged/credentials.rb:51 +#: ../lib/r10k/git/rugged/credentials.rb:55 msgid "URL %{url} has no per-repository private key using '%{key}'." msgstr "" -#: ../lib/r10k/git/rugged/credentials.rb:53 +#: ../lib/r10k/git/rugged/credentials.rb:57 msgid "Git remote %{url} uses the SSH protocol but no private key was given" msgstr "" -#: ../lib/r10k/git/rugged/credentials.rb:57 +#: ../lib/r10k/git/rugged/credentials.rb:61 msgid "Unable to use SSH key auth for %{url}: private key %{private_key} is missing or unreadable" msgstr "" -#: ../lib/r10k/git/rugged/credentials.rb:80 +#: ../lib/r10k/git/rugged/credentials.rb:102 +msgid "Using OAuth token from stdin for URL %{url}" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:105 +msgid "Using OAuth token from %{token_path} for URL %{url}" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:107 +msgid "%{path} is missing or unreadable, cannot load OAuth token" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:111 +msgid "Supplied OAuth token contains invalid characters." +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:135 msgid "URL %{url} includes the username %{username}, using that user for authentication." msgstr "" -#: ../lib/r10k/git/rugged/credentials.rb:83 +#: ../lib/r10k/git/rugged/credentials.rb:138 msgid "URL %{url} did not specify a user, using %{user} from configuration" msgstr "" -#: ../lib/r10k/git/rugged/credentials.rb:86 +#: ../lib/r10k/git/rugged/credentials.rb:141 msgid "URL %{url} did not specify a user, using current user %{user}" msgstr "" -#: ../lib/r10k/git/rugged/thin_repository.rb:85 ../lib/r10k/git/shellgit/thin_repository.rb:65 +#: ../lib/r10k/git/rugged/credentials.rb:148 +msgid "Github App id contains invalid characters." +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:149 +msgid "Github App token ttl contains invalid characters." +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:150 +msgid "Github App key is missing or unreadable" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:155 +msgid "Github App key is not a valid SSL private key" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:158 +msgid "Github App key is not a valid SSL key" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:161 +msgid "Using Github App id %{app_id} with SSL key from %{key_path}" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:179 +msgid "Error using private key to get Github App access token from url" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:200 +msgid "Github App token contains invalid characters." +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:202 +msgid "Github App token generated, expires at: %{expire}" +msgstr "" + +#: ../lib/r10k/git/rugged/thin_repository.rb:92 ../lib/r10k/git/shellgit/thin_repository.rb:69 msgid "Updated repo %{path} to include alternate object db path %{objects_dir}" msgstr "" @@ -251,39 +323,39 @@ msgstr "" msgid "Fetching remote '%{remote}' at %{path}" msgstr "" -#: ../lib/r10k/git/rugged/working_repository.rb:125 ../lib/r10k/git/shellgit/working_repository.rb:100 +#: ../lib/r10k/git/rugged/working_repository.rb:129 ../lib/r10k/git/shellgit/working_repository.rb:101 msgid "Found local modifications in %{file_path}" msgstr "" -#: ../lib/r10k/git/stateful_repository.rb:40 +#: ../lib/r10k/git/stateful_repository.rb:45 msgid "Unable to sync repo to unresolvable ref '%{ref}'" msgstr "" -#: ../lib/r10k/git/stateful_repository.rb:47 +#: ../lib/r10k/git/stateful_repository.rb:53 msgid "Cloning %{repo_path} and checking out %{ref}" msgstr "" -#: ../lib/r10k/git/stateful_repository.rb:50 +#: ../lib/r10k/git/stateful_repository.rb:56 msgid "Replacing %{repo_path} and checking out %{ref}" msgstr "" -#: ../lib/r10k/git/stateful_repository.rb:54 ../lib/r10k/git/stateful_repository.rb:59 +#: ../lib/r10k/git/stateful_repository.rb:60 ../lib/r10k/git/stateful_repository.rb:65 msgid "Updating %{repo_path} to %{ref}" msgstr "" -#: ../lib/r10k/git/stateful_repository.rb:58 +#: ../lib/r10k/git/stateful_repository.rb:64 msgid "Overwriting local modifications to %{repo_path}" msgstr "" -#: ../lib/r10k/git/stateful_repository.rb:62 +#: ../lib/r10k/git/stateful_repository.rb:68 msgid "Skipping %{repo_path} due to local modifications" msgstr "" -#: ../lib/r10k/git/stateful_repository.rb:65 +#: ../lib/r10k/git/stateful_repository.rb:72 msgid "%{repo_path} is already at Git ref %{ref}" msgstr "" -#: ../lib/r10k/initializers.rb:30 +#: ../lib/r10k/initializers.rb:31 msgid "the purgedirs key in r10k.yaml is deprecated. it is currently ignored." msgstr "" @@ -295,31 +367,47 @@ msgstr "" msgid "No class registered for %{key}" msgstr "" -#: ../lib/r10k/logging.rb:60 +#: ../lib/r10k/logging.rb:73 ../lib/r10k/logging.rb:100 ../lib/r10k/logging.rb:109 msgid "Invalid log level '%{val}'. Valid levels are %{log_levels}" msgstr "" -#: ../lib/r10k/module.rb:29 +#: ../lib/r10k/module.rb:45 msgid "Module %{name} with args %{args} doesn't have an implementation. (Are you using the right arguments?)" msgstr "" -#: ../lib/r10k/module/base.rb:110 +#: ../lib/r10k/module/base.rb:120 +msgid "Deploying module to %{path}" +msgstr "" + +#: ../lib/r10k/module/base.rb:123 +msgid "Only updating modules %{modules}, skipping module %{name}" +msgstr "" + +#: ../lib/r10k/module/base.rb:179 msgid "Module name (%{title}) must match either 'modulename' or 'owner/modulename'" msgstr "" -#: ../lib/r10k/module/forge.rb:70 ../lib/r10k/module/forge.rb:99 -msgid "The module %{title} does not exist on %{url}." +#: ../lib/r10k/module/definition.rb:28 +msgid "Not updating module %{name}, assuming content unchanged" +msgstr "" + +#: ../lib/r10k/module/forge.rb:50 +msgid "Module version %{ver} is not a valid Forge module version" +msgstr "" + +#: ../lib/r10k/module/forge.rb:98 +msgid "The module %{title} does not appear to have any published releases, cannot determine latest version." msgstr "" -#: ../lib/r10k/module/forge.rb:174 -msgid "Forge module names must match 'owner/modulename'" +#: ../lib/r10k/module/forge.rb:101 ../lib/r10k/module/forge.rb:130 +msgid "The module %{title} does not exist on %{url}." msgstr "" -#: ../lib/r10k/module/git.rb:97 -msgid "Unhandled options %{unhandled} specified for %{class}" +#: ../lib/r10k/module/git.rb:79 +msgid "Cannot track control repo branch for content '%{name}' when not part of a git-backed environment, will use default if available." msgstr "" -#: ../lib/r10k/module/local.rb:34 +#: ../lib/r10k/module/local.rb:37 msgid "Module %{title} is a local module, always indicating synced." msgstr "" @@ -327,39 +415,47 @@ msgstr "" msgid "Could not read metadata.json" msgstr "" -#: ../lib/r10k/puppetfile.rb:57 +#: ../lib/r10k/module_loader/puppetfile.rb:68 msgid "Using Puppetfile '%{puppetfile}'" msgstr "" -#: ../lib/r10k/puppetfile.rb:71 -msgid "Puppetfile %{path} missing or unreadable" +#: ../lib/r10k/module_loader/puppetfile.rb:69 +msgid "Using moduledir '%{moduledir}'" msgstr "" -#: ../lib/r10k/puppetfile.rb:84 +#: ../lib/r10k/module_loader/puppetfile.rb:91 msgid "Failed to evaluate %{path}" msgstr "" -#: ../lib/r10k/puppetfile.rb:98 -msgid "Puppetfiles cannot contain duplicate module names." +#: ../lib/r10k/module_loader/puppetfile.rb:108 +msgid "Unable to preload Puppetfile because of %{msg}" msgstr "" -#: ../lib/r10k/puppetfile.rb:100 -msgid "Remove the duplicates of the following modules: %{dupes}" +#: ../lib/r10k/module_loader/puppetfile.rb:126 +msgid "Using Forge from Puppetfile: %{forge}" msgstr "" -#: ../lib/r10k/puppetfile.rb:192 -msgid "Updating modules with %{pool_size} threads" +#: ../lib/r10k/module_loader/puppetfile.rb:129 +msgid "Ignoring Forge declaration in Puppetfile, using value from settings: %{forge}." msgstr "" -#: ../lib/r10k/puppetfile.rb:203 -msgid "Error during concurrent deploy of a module: %{message}" +#: ../lib/r10k/module_loader/puppetfile.rb:181 +msgid "\"basedir\" is deprecated. Please use \"environment_name\" instead. \"basedir\" will be removed in a future version." msgstr "" -#: ../lib/r10k/puppetfile.rb:225 -msgid "Module thread %{id} exiting: %{message}" +#: ../lib/r10k/module_loader/puppetfile.rb:200 ../lib/r10k/puppetfile.rb:104 +msgid "Puppetfile %{path} missing or unreadable" +msgstr "" + +#: ../lib/r10k/module_loader/puppetfile.rb:242 +msgid "Puppetfiles cannot contain duplicate module names." msgstr "" -#: ../lib/r10k/puppetfile.rb:282 +#: ../lib/r10k/module_loader/puppetfile.rb:244 +msgid "Remove the duplicates of the following modules: %{dupes}" +msgstr "" + +#: ../lib/r10k/module_loader/puppetfile/dsl.rb:37 msgid "unrecognized declaration '%{method}'" msgstr "" @@ -371,7 +467,7 @@ msgstr "" msgid "Validation failed for settings group" msgstr "" -#: ../lib/r10k/settings/container.rb:91 +#: ../lib/r10k/settings/container.rb:92 msgid "Key %{key} is not a valid key" msgstr "" @@ -415,10 +511,6 @@ msgstr "" msgid "Couldn't load config file: %{error_msg}" msgstr "" -#: ../lib/r10k/settings/loader.rb:73 -msgid "File exists at #{path} but doesn't contain any YAML" -msgstr "" - #: ../lib/r10k/settings/uri_definition.rb:12 msgid "Setting %{name} requires a URL but '%{value}' could not be parsed as a URL" msgstr "" @@ -442,27 +534,27 @@ msgid "" "Returned: %{data}" msgstr "" -#: ../lib/r10k/source/git.rb:77 +#: ../lib/r10k/source/git.rb:75 msgid "Fetching '%{remote}' to determine current branches." msgstr "" -#: ../lib/r10k/source/git.rb:80 +#: ../lib/r10k/source/git.rb:78 msgid "Unable to determine current branches for Git source '%{name}' (%{basedir})" msgstr "" -#: ../lib/r10k/source/git.rb:105 +#: ../lib/r10k/source/git.rb:113 msgid "Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}" msgstr "" -#: ../lib/r10k/source/git.rb:109 +#: ../lib/r10k/source/git.rb:122 msgid "Environment %{env_name} contained non-word characters, ignoring it." msgstr "" -#: ../lib/r10k/source/git.rb:128 ../lib/r10k/source/svn.rb:113 +#: ../lib/r10k/source/git.rb:141 ../lib/r10k/source/svn.rb:115 msgid "Branch %{branch} filtered out by ignore_branch_prefixes %{ibp}" msgstr "" -#: ../lib/r10k/source/git.rb:139 +#: ../lib/r10k/source/git.rb:152 msgid "Branch `%{name}:%{branch}` filtered out by filter_command %{cmd}" msgstr "" @@ -486,6 +578,10 @@ msgstr "" msgid "Both username and password must be specified" msgstr "" +#: ../lib/r10k/tarball.rb:167 +msgid "Cache not present at %{path}" +msgstr "" + #: ../lib/r10k/util/basedir.rb:34 msgid "Expected Array<#desired_contents>, got R10K::Deployment" msgstr "" @@ -506,41 +602,45 @@ msgstr "" msgid "pe_license feature is not available, PE only Puppet modules will not be downloadable." msgstr "" -#: ../lib/r10k/util/purgeable.rb:52 -msgid "Not purging %{item} due to internal exclusion match: %{exclusion_match}" +#: ../lib/r10k/util/purgeable.rb:91 +msgid "Not purging %{path} due to internal exclusion match: %{exclusion_match}" msgstr "" -#: ../lib/r10k/util/purgeable.rb:54 -msgid "Not purging %{item} due to whitelist match: %{whitelist_match}" +#: ../lib/r10k/util/purgeable.rb:93 +msgid "Not purging %{path} due to whitelist match: %{allowlist_match}" msgstr "" -#: ../lib/r10k/util/purgeable.rb:71 +#: ../lib/r10k/util/purgeable.rb:137 msgid "No unmanaged contents in %{managed_dirs}, nothing to purge" msgstr "" -#: ../lib/r10k/util/purgeable.rb:76 +#: ../lib/r10k/util/purgeable.rb:142 msgid "Removing unmanaged path %{path}" msgstr "" -#: ../lib/r10k/util/purgeable.rb:81 +#: ../lib/r10k/util/purgeable.rb:147 msgid "Unable to remove unmanaged path: %{path}" msgstr "" -#: ../lib/r10k/util/setopts.rb:49 +#: ../lib/r10k/util/setopts.rb:60 +msgid "%{class_name} parameters '%{a}' and '%{b}' conflict. Specify one or the other, but not both" +msgstr "" + +#: ../lib/r10k/util/setopts.rb:67 msgid "%{class_name} cannot handle option '%{key}'" msgstr "" -#: ../lib/r10k/util/subprocess.rb:69 +#: ../lib/r10k/util/subprocess.rb:70 msgid "Starting process: %{args}" msgstr "" -#: ../lib/r10k/util/subprocess.rb:74 +#: ../lib/r10k/util/subprocess.rb:75 msgid "" "Finished process:\n" "%{result}" msgstr "" -#: ../lib/r10k/util/subprocess.rb:77 +#: ../lib/r10k/util/subprocess.rb:78 msgid "Command exited with non-zero exit code" msgstr "" diff -pruN 3.7.0-2.1/r10k.gemspec 3.15.4-1/r10k.gemspec --- 3.7.0-2.1/r10k.gemspec 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/r10k.gemspec 2023-01-19 00:49:17.000000000 +0000 @@ -23,27 +23,27 @@ Gem::Specification.new do |s| s.license = 'Apache-2.0' s.add_dependency 'colored2', '3.1.2' - s.add_dependency 'cri', ['>= 2.15.10', '< 3.0.0'] + s.add_dependency 'cri', '>= 2.15.10' s.add_dependency 'log4r', '1.1.10' s.add_dependency 'multi_json', '~> 1.10' - s.add_dependency 'puppet_forge', '~> 2.3.0' + s.add_dependency 'puppet_forge', ['>= 2.3.0', '< 4.0.0'] - s.add_dependency 'gettext-setup', '~>0.24' - # These two pins narrow what is allowed by gettext-setup, - # to preserver compatability with Ruby 2.4 - s.add_dependency 'fast_gettext', '~> 1.1.0' - s.add_dependency 'gettext', ['>= 3.0.2', '< 3.3.0'] + s.add_dependency 'gettext-setup', ['>=0.24', '< 2.0.0'] + s.add_dependency 'fast_gettext', ['>= 1.1.0', '< 3.0.0'] + s.add_dependency 'gettext', ['>= 3.0.2', '< 4.0.0'] + + s.add_dependency 'jwt', '~> 2.2.3' + s.add_dependency 'minitar', '~> 0.9' s.add_development_dependency 'rspec', '~> 3.1' s.add_development_dependency 'rake' s.add_development_dependency 'yard', '~> 0.9.11' - s.add_development_dependency 'minitar', '~> 0.9.0' - s.files = %x[git ls-files].split($/) + s.files = %x[git ls-files].split($/).reject { |f| f.match(%r{^spec}) } s.require_path = 'lib' s.bindir = 'bin' s.executables = 'r10k' diff -pruN 3.7.0-2.1/r10k.yaml.example 3.15.4-1/r10k.yaml.example --- 3.7.0-2.1/r10k.yaml.example 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/r10k.yaml.example 2023-01-19 00:49:17.000000000 +0000 @@ -110,3 +110,31 @@ forge: # The 'baseurl' setting indicates where Forge modules should be installed # from. This defaults to 'https://forgeapi.puppetlabs.com' #baseurl: 'https://forgemirror.example.com' + +# Configuration options on how R10k should log its actions +logging: + # The 'level' setting sets the default log level to run R10k actions at. + # This value will be overridden by any value set through the command line. + #level: warn + + # Specify additional log outputs here, any log4r outputter can be used. + # If no log level is specified then the output will use the global level. + #outputs: + # - type: file + # level: debug + # parameters: + # filename: /var/log/r10k.log + # trunc: true + # - type: syslog + # - type: email + # only_at: [fatal] + # parameters: + # from: r10k@example.com + # to: sysadmins@example.com + # server: smtp.example.com + # subject: Fatal R10k error occurred + + # The 'disable_default_stderr' setting specifies if the default output on + # stderr should be active or not, in case R10k is to be run entirely + # through scripts or cronjobs where console output is unwelcome. + #disable_default_stderr: false diff -pruN 3.7.0-2.1/README.mkd 3.15.4-1/README.mkd --- 3.7.0-2.1/README.mkd 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/README.mkd 2023-01-19 00:49:17.000000000 +0000 @@ -5,6 +5,12 @@ Puppet environment and module deployment [![Build Status](https://travis-ci.org/puppetlabs/r10k.png?branch=master)](https://travis-ci.org/puppetlabs/r10k) +> R10k is supported and maintained by Puppet, but we consider it to be feature +> complete and currently have no plans for any new development. We will keep it +> working within the context of Puppet Enterprise, but we cannot make any other +> maintenance promises at this time. + + Description ----------- @@ -14,11 +20,15 @@ R10k provides a general purpose toolset modules. It implements the [Puppetfile](doc/puppetfile.mkd) format and provides a native implementation of Puppet [environments][workflow]. +You might also consider [g10k](https://github.com/xorpaul/g10k) as a non-ruby +based alternative. + + Requirements ------------ R10k supports the Ruby versions `>= 2.4.0`. It's tested on Ruby 2.4.0 up to -Ruby 2.7.0 + Jruby. +Ruby 3.1.0 + Jruby. R10k requires additional components, depending on how you plan on managing environments and modules. @@ -61,11 +71,17 @@ for installation. If you have more specific needs or plan on modifying r10k you can run it out of a git repository using Bundler for dependencies: - git clone git://github.com/puppetlabs/r10k + git clone https://github.com/puppetlabs/r10k cd r10k bundle install bundle exec r10k help +### Arch Linux + +Arch Linux provides a [system package](https://archlinux.org/packages/community/any/r10k/) for r10k. +This is built against the [system Ruby](https://archlinux.org/packages/extra/x86_64/ruby/) (which is Ruby 3.0.2 as of 2021-08-03). +This package is maintained by [Tim Meusel](https://github.com/bastelfreak). + Usage ----- @@ -104,24 +120,13 @@ To release a new version of the r10k gem By default, a patch (Z) release will be triggered. To release a new major (X) or minor (Y) version, include `#major` or `#minor` (respectively) in your commit message to trigger the appropriate release. -NOTE: This currently only works for the default branch. If you would like to release from a different branch, please contact the [CODEOWNERS](CODEOWNERS). - Getting help ------------ - * IRC: r10k has a dedicated channel, `#r10k`, on Freenode where r10k questions - can be directed. Questions about r10k can frequently be asked in `#puppet` as well. + * [Puppet Community Slack](https://puppetcommunity.slack.com/) * Mailing lists: [puppet-users](https://groups.google.com/forum/#!forum/puppet-users) * Q&A: [Puppet Ask](https://ask.puppetlabs.com/questions/) -Contributors ------------- - -Please see the CHANGELOG for a listing of the (very awesome) contributors. - ## Maintenance -See [CODEOWNERS](CODEOWNERS) for active repo maintainers. - -Open [issues](https://github.com/puppetlabs/r10k/issues) directly in the r10k repo. - +See [CODEOWNERS](CODEOWNERS) for current project owners. Binary files 3.7.0-2.1/spec/fixtures/tarball/tarball.tar.gz and 3.15.4-1/spec/fixtures/tarball/tarball.tar.gz differ diff -pruN 3.7.0-2.1/spec/fixtures/unit/action/r10k_creds.yaml 3.15.4-1/spec/fixtures/unit/action/r10k_creds.yaml --- 3.7.0-2.1/spec/fixtures/unit/action/r10k_creds.yaml 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/fixtures/unit/action/r10k_creds.yaml 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,9 @@ +--- + git: + private_key: '/global/config/private/key' + oauth_token: '/global/config/oauth/token' + repositories: + - remote: 'git@myfakegitserver.com:user/repo.git' + private_key: '/config/private/key' + - remote: 'https://myfakegitserver.com/user/repo.git' + oauth_token: '/config/oauth/token' diff -pruN 3.7.0-2.1/spec/fixtures/unit/action/r10k_forge_auth_no_url.yaml 3.15.4-1/spec/fixtures/unit/action/r10k_forge_auth_no_url.yaml --- 3.7.0-2.1/spec/fixtures/unit/action/r10k_forge_auth_no_url.yaml 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/fixtures/unit/action/r10k_forge_auth_no_url.yaml 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,3 @@ +--- +forge: + authorization_token: 'faketoken' diff -pruN 3.7.0-2.1/spec/fixtures/unit/action/r10k_forge_auth.yaml 3.15.4-1/spec/fixtures/unit/action/r10k_forge_auth.yaml --- 3.7.0-2.1/spec/fixtures/unit/action/r10k_forge_auth.yaml 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/fixtures/unit/action/r10k_forge_auth.yaml 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,4 @@ +--- +forge: + baseurl: 'http://private-forge.com' + authorization_token: 'faketoken' diff -pruN 3.7.0-2.1/spec/fixtures/unit/action/r10k_logging.yaml 3.15.4-1/spec/fixtures/unit/action/r10k_logging.yaml --- 3.7.0-2.1/spec/fixtures/unit/action/r10k_logging.yaml 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/fixtures/unit/action/r10k_logging.yaml 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,12 @@ +--- + logging: + level: FATAL + + outputs: + - type: file + parameters: + filename: r10k.log + + - type: syslog + + disable_default_stderr: true diff -pruN 3.7.0-2.1/spec/fixtures/unit/puppetfile/forge-override/Puppetfile 3.15.4-1/spec/fixtures/unit/puppetfile/forge-override/Puppetfile --- 3.7.0-2.1/spec/fixtures/unit/puppetfile/forge-override/Puppetfile 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/fixtures/unit/puppetfile/forge-override/Puppetfile 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,8 @@ +forge "my.custom.forge.com" + +mod "puppetlabs/stdlib", '4.12.0' +mod "puppetlabs/concat", '2.1.0' + +mod 'apache', + :git => 'https://github.com/puppetlabs/puppetlabs-apache', + :branch => 'docs_experiment' diff -pruN 3.7.0-2.1/spec/fixtures/unit/puppetfile/various-modules/modules/apt/.gitkeep 3.15.4-1/spec/fixtures/unit/puppetfile/various-modules/modules/apt/.gitkeep --- 3.7.0-2.1/spec/fixtures/unit/puppetfile/various-modules/modules/apt/.gitkeep 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/fixtures/unit/puppetfile/various-modules/modules/apt/.gitkeep 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1 @@ +This only exists so the directory can be committed to git for testing purposes. \ No newline at end of file diff -pruN 3.7.0-2.1/spec/fixtures/unit/puppetfile/various-modules/modules/baz/.gitkeep 3.15.4-1/spec/fixtures/unit/puppetfile/various-modules/modules/baz/.gitkeep --- 3.7.0-2.1/spec/fixtures/unit/puppetfile/various-modules/modules/baz/.gitkeep 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/fixtures/unit/puppetfile/various-modules/modules/baz/.gitkeep 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1 @@ +This only exists so the directory can be committed to git for testing purposes. \ No newline at end of file diff -pruN 3.7.0-2.1/spec/fixtures/unit/puppetfile/various-modules/modules/buzz/.gitkeep 3.15.4-1/spec/fixtures/unit/puppetfile/various-modules/modules/buzz/.gitkeep --- 3.7.0-2.1/spec/fixtures/unit/puppetfile/various-modules/modules/buzz/.gitkeep 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/fixtures/unit/puppetfile/various-modules/modules/buzz/.gitkeep 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1 @@ +This only exists so the directory can be committed to git for testing purposes. \ No newline at end of file diff -pruN 3.7.0-2.1/spec/fixtures/unit/puppetfile/various-modules/modules/canary/.gitkeep 3.15.4-1/spec/fixtures/unit/puppetfile/various-modules/modules/canary/.gitkeep --- 3.7.0-2.1/spec/fixtures/unit/puppetfile/various-modules/modules/canary/.gitkeep 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/fixtures/unit/puppetfile/various-modules/modules/canary/.gitkeep 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1 @@ +This only exists so the directory can be committed to git for testing purposes. \ No newline at end of file diff -pruN 3.7.0-2.1/spec/fixtures/unit/puppetfile/various-modules/modules/fizz/.gitkeep 3.15.4-1/spec/fixtures/unit/puppetfile/various-modules/modules/fizz/.gitkeep --- 3.7.0-2.1/spec/fixtures/unit/puppetfile/various-modules/modules/fizz/.gitkeep 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/fixtures/unit/puppetfile/various-modules/modules/fizz/.gitkeep 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1 @@ +This only exists so the directory can be committed to git for testing purposes. \ No newline at end of file diff -pruN 3.7.0-2.1/spec/fixtures/unit/puppetfile/various-modules/modules/rpm/.gitkeep 3.15.4-1/spec/fixtures/unit/puppetfile/various-modules/modules/rpm/.gitkeep --- 3.7.0-2.1/spec/fixtures/unit/puppetfile/various-modules/modules/rpm/.gitkeep 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/fixtures/unit/puppetfile/various-modules/modules/rpm/.gitkeep 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1 @@ +This only exists so the directory can be committed to git for testing purposes. \ No newline at end of file diff -pruN 3.7.0-2.1/spec/fixtures/unit/puppetfile/various-modules/Puppetfile 3.15.4-1/spec/fixtures/unit/puppetfile/various-modules/Puppetfile --- 3.7.0-2.1/spec/fixtures/unit/puppetfile/various-modules/Puppetfile 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/fixtures/unit/puppetfile/various-modules/Puppetfile 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,10 @@ +mod 'puppetlabs/apt', '2.1.1' +mod 'puppetlabs/stdlib', :latest +mod 'puppetlabs/concat' +mod 'puppetlabs/rpm', '2.1.1-pre1' +mod 'foo', git: 'this/remote', branch: 'main' +mod 'bar', git: 'this/remote', tag: 'v1.2.3' +mod 'baz', git: 'this/remote', commit: '123abc456' +mod 'fizz', git: 'this/remote', ref: '1234567890abcdef1234567890abcdef12345678' +mod 'buzz', git: 'this/remote', ref: 'refs/heads/main' +mod 'canary', local: true diff -pruN 3.7.0-2.1/spec/fixtures/unit/puppetfile/various-modules/Puppetfile.new 3.15.4-1/spec/fixtures/unit/puppetfile/various-modules/Puppetfile.new --- 3.7.0-2.1/spec/fixtures/unit/puppetfile/various-modules/Puppetfile.new 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/fixtures/unit/puppetfile/various-modules/Puppetfile.new 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,10 @@ +mod 'puppetlabs/apt', '3.0.0' +mod 'puppetlabs/stdlib', :latest +mod 'puppetlabs/concat' +mod 'puppetlabs/rpm', '2.1.1-pre1' +mod 'foo', git: 'this/remote', branch: 'main' +mod 'bar', git: 'this/remote', tag: 'v1.2.3' +mod 'baz', git: 'this/remote', commit: '123abc456' +mod 'fizz', git: 'this/remote', ref: '1234567890abcdef1234567890abcdef12345678' +mod 'buzz', git: 'this/remote', ref: 'refs/heads/main' +mod 'canary', local: true diff -pruN 3.7.0-2.1/spec/integration/git/rugged/cache_spec.rb 3.15.4-1/spec/integration/git/rugged/cache_spec.rb --- 3.7.0-2.1/spec/integration/git/rugged/cache_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/integration/git/rugged/cache_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,33 @@ +require 'spec_helper' +require 'r10k/git/rugged/cache' + +describe R10K::Git::Rugged::Cache, :if => R10K::Features.available?(:rugged) do + include_context 'Git integration' + + let(:dirname) { 'working-repo' } + let(:remote_name) { 'origin' } + + subject { described_class.new(remote) } + + context "syncing with the remote" do + before(:each) do + subject.reset! + end + + describe "with the correct configuration" do + it "is able to sync with the remote" do + subject.sync + expect(subject.synced?).to eq(true) + end + end + + describe "with a out of date cached remote" do + it "updates the cached remote configuration" do + subject.repo.update_remote('foo', remote_name) + expect(subject.repo.remotes[remote_name]).to eq('foo') + subject.sync + expect(subject.repo.remotes[remote_name]).to eq(remote) + end + end + end +end diff -pruN 3.7.0-2.1/spec/integration/git/stateful_repository_spec.rb 3.15.4-1/spec/integration/git/stateful_repository_spec.rb --- 3.7.0-2.1/spec/integration/git/stateful_repository_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/integration/git/stateful_repository_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -83,6 +83,22 @@ describe R10K::Git::StatefulRepository d end end + describe "when the workdir has spec dir modifications" do + before(:each) do + thinrepo.clone(remote, {:ref => ref}) + FileUtils.mkdir_p(File.join(thinrepo.path, 'spec')) + File.open(File.join(thinrepo.path, 'spec', 'file_spec.rb'), 'a') { |f| f.write('local modifications!') } + thinrepo.stage_files(['spec/file_spec.rb']) + end + it "is dirty with exclude_spec false" do + expect(subject.status(ref, false)).to eq :dirty + end + + it "is insync with exclude_spec true" do + expect(subject.status(ref, true)).to eq :insync + end + end + describe "if the right ref is checked out" do it "is insync" do thinrepo.clone(remote, {:ref => ref}) diff -pruN 3.7.0-2.1/spec/integration/util/purageable_spec.rb 3.15.4-1/spec/integration/util/purageable_spec.rb --- 3.7.0-2.1/spec/integration/util/purageable_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/integration/util/purageable_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,41 @@ +require 'spec_helper' +require 'r10k/util/purgeable' +require 'r10k/util/cleaner' + +require 'tmpdir' + +RSpec.describe R10K::Util::Purgeable do + it 'purges only unmanaged files' do + Dir.mktmpdir do |envdir| + managed_directory = "#{envdir}/managed_one" + desired_contents = [ + "#{managed_directory}/expected_1", + "#{managed_directory}/managed_subdir_1", + "#{managed_directory}/managed_symlink_dir", + "#{managed_directory}/managed_subdir_1/subdir_expected_1", + "#{managed_directory}/managed_subdir_1/managed_symlink_file", + ] + + FileUtils.cp_r('spec/fixtures/unit/util/purgeable/managed_one/', + managed_directory) + + cleaner = R10K::Util::Cleaner.new([managed_directory], desired_contents) + + cleaner.purge!({ recurse: true, whitelist: ["**/subdir_allowlisted_2"] }) + + # Files present after purge + expect(File.exist?("#{managed_directory}/expected_1")).to be true + expect(File.exist?("#{managed_directory}/managed_subdir_1")).to be true + expect(File.exist?("#{managed_directory}/managed_symlink_dir")).to be true + expect(File.exist?("#{managed_directory}/managed_subdir_1/subdir_expected_1")).to be true + expect(File.exist?("#{managed_directory}/managed_subdir_1/managed_symlink_file")).to be true + expect(File.exist?("#{managed_directory}/managed_subdir_1/subdir_allowlisted_2")).to be true + + # Purged files + expect(File.exist?("#{managed_directory}/unmanaged_1")).to be false + expect(File.exist?("#{managed_directory}/managed_subdir_1/unmanaged_symlink_dir")).to be false + expect(File.exist?("#{managed_directory}/unmanaged_symlink_file")).to be false + expect(File.exist?("#{managed_directory}/managed_subdir_1/subdir_unmanaged_1")).to be false + end + end +end diff -pruN 3.7.0-2.1/spec/r10k-mocks/mock_env.rb 3.15.4-1/spec/r10k-mocks/mock_env.rb --- 3.7.0-2.1/spec/r10k-mocks/mock_env.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/r10k-mocks/mock_env.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,6 +1,9 @@ require 'r10k/environment' +require 'r10k/util/purgeable' class R10K::Environment::Mock < R10K::Environment::Base + include R10K::Util::Purgeable + def sync "synced" end diff -pruN 3.7.0-2.1/spec/r10k-mocks/mock_source.rb 3.15.4-1/spec/r10k-mocks/mock_source.rb --- 3.7.0-2.1/spec/r10k-mocks/mock_source.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/r10k-mocks/mock_source.rb 2023-01-19 00:49:17.000000000 +0000 @@ -5,9 +5,13 @@ class R10K::Source::Mock < R10K::Source: R10K::Source.register(:mock, self) def environments - corrected_environment_names = @options[:environments].map do |env| - R10K::Environment::Name.new(env, :prefix => @prefix, :invalid => 'correct_and_warn') + if @_environments.nil? + corrected_environment_names = @options[:environments].map do |env| + R10K::Environment::Name.new(env, :prefix => @prefix, :invalid => 'correct_and_warn') + end + @_environments = corrected_environment_names.map { |env| R10K::Environment::Mock.new(env.name, @basedir, env.dirname, { overrides: @options[:overrides] }) } end - corrected_environment_names.map { |env| R10K::Environment::Mock.new(env.name, @basedir, env.dirname) } + + @_environments end end diff -pruN 3.7.0-2.1/spec/shared-contexts/tarball.rb 3.15.4-1/spec/shared-contexts/tarball.rb --- 3.7.0-2.1/spec/shared-contexts/tarball.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/shared-contexts/tarball.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,32 @@ +require 'tmpdir' +require 'fileutils' + +shared_context "Tarball" do + # Suggested subject: + # + # subject { described_class.new('fixture-tarball', fixture_tarball, checksum: fixture_checksum) } + # + let(:fixture_tarball) do + File.expand_path('spec/fixtures/tarball/tarball.tar.gz', PROJECT_ROOT) + end + + let(:fixture_checksum) { '292e692ad18faabd4f9b21037d51f0185e04b69f82c522a54af91fb5b88c2d3b' } + + # Use tmpdir for cached tarballs + let(:tmpdir) { Dir.mktmpdir } + + # `moduledir` and `cache_root` are available for examples to use in creating + # their subjects + let(:moduledir) { File.join(tmpdir, 'modules').tap { |path| Dir.mkdir(path) } } + let(:cache_root) { File.join(tmpdir, 'cache').tap { |path| Dir.mkdir(path) } } + + around(:each) do |example| + if subject.is_a?(R10K::Tarball) + subject.settings[:cache_root] = cache_root + elsif subject.respond_to?(:tarball) && subject.tarball.is_a?(R10K::Tarball) + subject.tarball.settings[:cache_root] = cache_root + end + example.run + FileUtils.remove_entry_secure(tmpdir) + end +end diff -pruN 3.7.0-2.1/spec/shared-examples/git/working_repository.rb 3.15.4-1/spec/shared-examples/git/working_repository.rb --- 3.7.0-2.1/spec/shared-examples/git/working_repository.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/shared-examples/git/working_repository.rb 2023-01-19 00:49:17.000000000 +0000 @@ -194,11 +194,13 @@ RSpec.shared_examples "a git working rep context "with local changes" do before(:each) do File.open(File.join(subject.path, 'README.markdown'), 'a') { |f| f.write('local modifications!') } + File.open(File.join(subject.path, 'CHANGELOG'), 'a') { |f| f.write('local modifications to the changelog too') } end it "logs and reports worktree as dirty" do expect(subject.logger).to receive(:debug).with(/found local modifications in.*README\.markdown/i) - expect(subject.logger).to receive(:debug1) + expect(subject.logger).to receive(:debug).with(/found local modifications in.*CHANGELOG/i) + expect(subject.logger).to receive(:debug1).twice expect(subject.dirty?).to be true end diff -pruN 3.7.0-2.1/spec/shared-examples/puppetfile-action.rb 3.15.4-1/spec/shared-examples/puppetfile-action.rb --- 3.7.0-2.1/spec/shared-examples/puppetfile-action.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/shared-examples/puppetfile-action.rb 2023-01-19 00:49:17.000000000 +0000 @@ -3,15 +3,15 @@ require 'spec_helper' shared_examples_for "a puppetfile action" do describe "initializing" do it "accepts the :root option" do - described_class.new({root: "/some/nonexistent/path"}, []) + described_class.new({root: "/some/nonexistent/path"}, [], {}) end it "accepts the :puppetfile option" do - described_class.new({puppetfile: "/some/nonexistent/path/Puppetfile"}, []) + described_class.new({puppetfile: "/some/nonexistent/path/Puppetfile"}, [], {}) end it "accepts the :moduledir option" do - described_class.new({moduledir: "/some/nonexistent/path/modules"}, []) + described_class.new({moduledir: "/some/nonexistent/path/modules"}, [], {}) end end @@ -20,19 +20,19 @@ end shared_examples_for "a puppetfile install action" do describe "initializing" do it "accepts the :root option" do - described_class.new({root: "/some/nonexistent/path"}, []) + described_class.new({root: "/some/nonexistent/path"}, [], {}) end it "accepts the :puppetfile option" do - described_class.new({puppetfile: "/some/nonexistent/path/Puppetfile"}, []) + described_class.new({puppetfile: "/some/nonexistent/path/Puppetfile"}, [], {}) end it "accepts the :moduledir option" do - described_class.new({moduledir: "/some/nonexistent/path/modules"}, []) + described_class.new({moduledir: "/some/nonexistent/path/modules"}, [], {}) end it "accepts the :force option" do - described_class.new({force: true}, []) + described_class.new({force: true}, [], {}) end end diff -pruN 3.7.0-2.1/spec/spec_helper.rb 3.15.4-1/spec/spec_helper.rb --- 3.7.0-2.1/spec/spec_helper.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/spec_helper.rb 2023-01-19 00:49:17.000000000 +0000 @@ -19,6 +19,7 @@ require 'r10k' Dir.glob(File.expand_path('spec/shared-examples/**/*.rb', PROJECT_ROOT)).each { |file| require file } require 'shared-contexts/git-fixtures' +require 'shared-contexts/tarball' require 'matchers/exit_with' require 'matchers/match_realpath' require 'r10k-mocks' diff -pruN 3.7.0-2.1/spec/unit/action/deploy/display_spec.rb 3.15.4-1/spec/unit/action/deploy/display_spec.rb --- 3.7.0-2.1/spec/unit/action/deploy/display_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/action/deploy/display_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -5,27 +5,57 @@ require 'r10k/action/deploy/display' describe R10K::Action::Deploy::Display do describe "initializing" do it "accepts a puppetfile option" do - described_class.new({puppetfile: true}, []) + described_class.new({puppetfile: true}, [], {}) + end + + it "accepts a modules option" do + described_class.new({modules: true}, [], {}) end it "accepts a detail option" do - described_class.new({detail: true}, []) + described_class.new({detail: true}, [], {}) end it "accepts a format option" do - described_class.new({format: "json"}, []) + described_class.new({format: "json"}, [], {}) end it "accepts a fetch option" do - described_class.new({fetch: true}, []) + described_class.new({fetch: true}, [], {}) end end - subject { described_class.new({config: "/some/nonexistent/path"}, []) } + subject { described_class.new({config: "/some/nonexistent/path"}, [], {}) } before do allow(subject).to receive(:puts) end it_behaves_like "a deploy action that requires a config file" + + describe "collecting info" do + subject { described_class.new({config: "/some/nonexistent/path", format: 'json', puppetfile: true, detail: true}, ['first'], {}) } + + let(:mock_config) do + R10K::Deployment::MockConfig.new( + :sources => { + :control => { + :type => :mock, + :basedir => '/some/nonexistent/path/control', + :environments => %w[first second third env-that/will-be-corrected], + :prefix => 'PREFIX' + } + } + ) + end + + let(:deployment) { R10K::Deployment.new(mock_config) } + + it "gathers environment info" do + source_info = subject.send(:source_info, deployment.sources.first, ['first']) + expect(source_info[:name]).to eq(:control) + expect(source_info[:environments].length).to eq(1) + expect(source_info[:environments][0][:name]).to eq('first') + end + end end diff -pruN 3.7.0-2.1/spec/unit/action/deploy/environment_spec.rb 3.15.4-1/spec/unit/action/deploy/environment_spec.rb --- 3.7.0-2.1/spec/unit/action/deploy/environment_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/action/deploy/environment_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -5,37 +5,78 @@ require 'r10k/action/deploy/environment' describe R10K::Action::Deploy::Environment do - subject { described_class.new({config: "/some/nonexistent/path"}, []) } + subject { described_class.new({config: "/some/nonexistent/path"}, [], {}) } it_behaves_like "a deploy action that can be write locked" it_behaves_like "a deploy action that requires a config file" describe "initializing" do it "can accept a cachedir option" do - described_class.new({cachedir: "/some/nonexistent/cachedir"}, []) + described_class.new({cachedir: "/some/nonexistent/cachedir"}, [], {}) end it "can accept a puppetfile option" do - described_class.new({puppetfile: true}, []) + described_class.new({puppetfile: true}, [], {}) + end + + it "can accept a modules option" do + described_class.new({modules: true}, [], {}) end it "can accept a default_branch_override option" do - described_class.new({:'default-branch-override' => 'default_branch_override_name'}, []) + described_class.new({:'default-branch-override' => 'default_branch_override_name'}, [], {}) end it "can accept a no-force option" do - described_class.new({:'no-force' => true}, []) + described_class.new({:'no-force' => true}, [], {}) end - it "normalizes environment names in the arg vector" - it 'can accept a generate-types option' do - described_class.new({ 'generate-types': true }, []) + described_class.new({ 'generate-types': true }, [], {}) end it 'can accept a puppet-path option' do - described_class.new({ 'puppet-path': '/nonexistent' }, []) + described_class.new({ 'puppet-path': '/nonexistent' }, [], {}) + end + + it 'can accept a private-key option' do + described_class.new({ 'private-key': '/nonexistent' }, [], {}) + end + + it 'can accept a token option' do + described_class.new({ 'oauth-token': '/nonexistent' }, [], {}) + end + + it 'can accept an app id option' do + described_class.new({ 'github-app-id': '/nonexistent' }, [], {}) + end + + it 'can accept a ttl option' do + described_class.new({ 'github-app-ttl': '/nonexistent' }, [], {}) + end + + it 'can accept a ssl private key option' do + described_class.new({ 'github-app-key': '/nonexistent' }, [], {}) end + + it 'can accept a exclude-spec option' do + described_class.new({ :'exclude-spec' => true }, [], {}) + end + + it 'can accept an incremental option' do + described_class.new({ :incremental => true }, [], {}) + end + + describe "initializing errors" do + let (:settings) { { deploy: { purge_levels: [:environment], + purge_whitelist: ['coolfile', 'coolfile2'], + purge_allowlist: ['anothercoolfile']}}} + + subject { described_class.new({config: "/some/nonexistent/path"}, [], settings)} + it 'errors out when both purge_whitelist and purge_allowlist are set' do + expect{subject}.to raise_error(R10K::Error, /Values found for both purge_whitelist and purge_allowlist./) + end + end end describe "when called" do @@ -52,6 +93,72 @@ describe R10K::Action::Deploy::Environme ) end + describe "with puppetfile or modules flag" do + let(:deployment) { R10K::Deployment.new(mock_config) } + let(:loader) do + instance_double("R10K::ModuleLoader::Puppetfile", + :load => { + :modules => ['foo'], + :purge_exclusions => [], + :managed_directories => [], + :desired_contents => [] + } + ).as_null_object + end + + before do + expect(R10K::Deployment).to receive(:new).and_return(deployment) + expect(R10K::ModuleLoader::Puppetfile).to receive(:new). + and_return(loader).at_least(:once) + end + + it "syncs the puppetfile content when given the puppetfile flag" do + expect(loader).to receive(:load).exactly(4).times + expect(R10K::ContentSynchronizer).to receive(:concurrent_sync).exactly(4).times + action = described_class.new({config: "/some/nonexistent/path", puppetfile: true}, [], {}) + action.call + end + + it "syncs the puppetfile when given the modules flag" do + expect(loader).to receive(:load).exactly(4).times + expect(R10K::ContentSynchronizer).to receive(:concurrent_sync).exactly(4).times + action = described_class.new({config: "/some/nonexistent/path", modules: true}, [], {}) + action.call + end + end + + describe "with incremental flag" do + let(:loader) do + instance_double("R10K::ModuleLoader::Puppetfile", + :load => { + :modules => ['foo'], + :purge_exclusions => [], + :managed_directories => [], + :desired_contents => [] + } + ).as_null_object + end + + before do + expect(R10K::Deployment).to receive(:new).and_wrap_original do |original, settings| + original.call(mock_config.merge(settings)) + end + expect(R10K::ModuleLoader::Puppetfile).to receive(:new). + and_return(loader).at_least(:once) + end + + it "incremental flag causes the module definitons to be preloaded by the loader" do + expect(loader).to receive(:load_metadata).exactly(4).times + action = described_class.new({:config => "/some/nonexistent/path", + :modules => true, + :incremental => true}, + [], + {}) + action.call + end + end + + describe "with an environment that doesn't exist" do let(:deployment) do R10K::Deployment.new(mock_config) @@ -61,7 +168,7 @@ describe R10K::Action::Deploy::Environme expect(R10K::Deployment).to receive(:new).and_return(deployment) end - subject { described_class.new({config: "/some/nonexistent/path"}, %w[not_an_environment]) } + subject { described_class.new({config: "/some/nonexistent/path"}, %w[not_an_environment], {}) } it "logs that the environments can't be deployed and returns false" do expect(subject.logger).to receive(:error).with("Environment(s) 'not_an_environment' cannot be found in any source and will not be deployed.") @@ -71,10 +178,10 @@ describe R10K::Action::Deploy::Environme end describe "with no-force" do - subject { described_class.new({ config: "/some/nonexistent/path", puppetfile: true, :'no-force' => true}, %w[first]) } + subject { described_class.new({ config: "/some/nonexistent/path", modules: true, :'no-force' => true}, %w[first], {}) } it "tries to preserve local modifications" do - expect(subject.force).to equal(false) + expect(subject.settings[:overrides][:modules][:force]).to equal(false) end end @@ -162,26 +269,85 @@ describe R10K::Action::Deploy::Environme end end + describe "Purging white/allowlist" do + + let(:settings) { { pool_size: 4, deploy: { purge_levels: [:environment], purge_allowlist: ['coolfile', 'coolfile2'] } } } + let(:overrides) { { environments: {}, modules: { pool_size: 4 }, purging: { purge_levels: [:environment], purge_allowlist: ['coolfile', 'coolfile2'] } } } + let(:deployment) do + R10K::Deployment.new(mock_config.merge({overrides: overrides})) + end + before do + expect(R10K::Deployment).to receive(:new).and_return(deployment) + allow_any_instance_of(R10K::Environment::Base).to receive(:purge!) + end + + subject { described_class.new({ config: "/some/nonexistent/path", modules: true }, %w[PREFIX_first], settings) } + + it "reads in the purge_allowlist setting and purges accordingly" do + expect(subject.logger).to receive(:debug).with(/Purging unmanaged content for environment/) + expect(subject.settings[:overrides][:purging][:purge_allowlist]).to eq(['coolfile', 'coolfile2']) + subject.call + end + + describe "purge_whitelist" do + let (:settings) { { deploy: { purge_levels: [:environment], purge_whitelist: ['coolfile', 'coolfile2'] } } } + + it "reads in the purge_whitelist setting and still sets it to purge_allowlist and purges accordingly" do + expect(subject.logger).to receive(:debug).with(/Purging unmanaged content for environment/) + expect(subject.settings[:overrides][:purging][:purge_allowlist]).to eq(['coolfile', 'coolfile2']) + subject.call + end + end + end + describe "purge_levels" do let(:settings) { { deploy: { purge_levels: purge_levels } } } + let(:overrides) do + { + environments: { + requested_environments: ['PREFIX_first'] + }, + modules: { + deploy_modules: true, + pool_size: 4 + }, + purging: { + purge_levels: purge_levels + } + } + end let(:deployment) do - R10K::Deployment.new(mock_config.merge(settings)) + R10K::Deployment.new(mock_config.merge({ overrides: overrides })) end before do expect(R10K::Deployment).to receive(:new).and_return(deployment) + allow_any_instance_of(R10K::Environment::Base).to receive(:purge!) end - subject { described_class.new({ config: "/some/nonexistent/path", puppetfile: true }, %w[PREFIX_first], settings) } + subject { described_class.new({ config: "/some/nonexistent/path", modules: true }, %w[PREFIX_first], settings) } describe "deployment purge level" do let(:purge_levels) { [:deployment] } + + it "updates the source's cache before it purges environments" do + deployment.sources.each do |source| + expect(source).to receive(:reload!).ordered + end + expect(deployment).to receive(:purge!).ordered + subject.call + end + it "only logs about purging deployment" do - expect(subject.logger).to receive(:debug).with(/purging unmanaged environments for deployment/i) - expect(subject.logger).to_not receive(:debug).with(/purging unmanaged content for environment/i) - expect(subject.logger).to_not receive(:debug).with(/purging unmanaged puppetfile content/i) + expect(subject).to receive(:visit_environment).and_wrap_original do |original, env, &block| + expect(env.logger).to_not receive(:debug).with(/Purging unmanaged puppetfile content/) + original.call(env) + end.at_least(:once) + + expect(subject.logger).to receive(:debug).with(/Purging unmanaged environments for deployment/) + expect(subject.logger).to_not receive(:debug).with(/Purging unmanaged content for environment/) subject.call end @@ -191,17 +357,25 @@ describe R10K::Action::Deploy::Environme let(:purge_levels) { [:environment] } it "only logs about purging environment" do - expect(subject.logger).to receive(:debug).with(/purging unmanaged content for environment/i) - expect(subject.logger).to_not receive(:debug).with(/purging unmanaged environments for deployment/i) - expect(subject.logger).to_not receive(:debug).with(/purging unmanaged puppetfile content/i) + expect(subject).to receive(:visit_environment).and_wrap_original do |original, env, &block| + expect(env.logger).to_not receive(:debug).with(/Purging unmanaged puppetfile content/) + original.call(env) + end.at_least(:once) + expect(subject.logger).to receive(:debug).with(/Purging unmanaged content for environment/) + expect(subject.logger).to_not receive(:debug).with(/Purging unmanaged environments for deployment/) subject.call end it "logs that environment was not purged if deploy failed" do - expect(subject).to receive(:visit_puppetfile) { subject.instance_variable_set(:@visit_ok, false) } + expect(subject).to receive(:visit_environment).and_wrap_original do |original, env, &block| + if env.name =~ /first/ + expect(env).to receive(:deploy) { subject.instance_variable_set(:@visit_ok, false) } + end + original.call(env) + end.at_least(:once) - expect(subject.logger).to receive(:debug).with(/not purging unmanaged content for environment/i) + expect(subject.logger).to receive(:debug).with(/Not purging unmanaged content for environment/) subject.call end @@ -211,14 +385,22 @@ describe R10K::Action::Deploy::Environme let(:purge_levels) { [:puppetfile] } it "only logs about purging puppetfile" do - expect(subject.logger).to receive(:debug).with(/purging unmanaged puppetfile content/i) - expect(subject.logger).to_not receive(:debug).with(/purging unmanaged environments for deployment/i) - expect(subject.logger).to_not receive(:debug).with(/purging unmanaged content for environment/i) + allow(R10K::ContentSynchronizer).to receive(:concurrent_sync) + expect(subject).to receive(:visit_environment).and_wrap_original do |original, env, &block| + if env.name =~ /first/ + expect(env.logger).to receive(:debug).with(/Purging unmanaged Puppetfile content/) + end + original.call(env) + end.at_least(:once) + + expect(subject.logger).to_not receive(:debug).with(/Purging unmanaged environments for deployment/) + expect(subject.logger).to_not receive(:debug).with(/Purging unmanaged content for environment/) subject.call end end end + describe "generate-types" do let(:deployment) do R10K::Deployment.new( @@ -229,6 +411,11 @@ describe R10K::Action::Deploy::Environme basedir: '/some/nonexistent/path/control', environments: %w[first second] } + }, + overrides: { + modules: { + pool_size: 4 + } } ) ) @@ -236,9 +423,8 @@ describe R10K::Action::Deploy::Environme before do allow(R10K::Deployment).to receive(:new).and_return(deployment) - end + allow_any_instance_of(R10K::Environment::Base).to receive(:purge!) - before(:each) do allow(subject).to receive(:write_environment_info!) expect(subject.logger).not_to receive(:error) end @@ -248,19 +434,22 @@ describe R10K::Action::Deploy::Environme described_class.new( { config: '/some/nonexistent/path', - puppetfile: true, + modules: true, 'generate-types': true }, - %w[first second] + %w[first second], + {} ) end it 'generate_types is true' do - expect(subject.instance_variable_get(:@generate_types)).to eq(true) + expect(subject.settings[:overrides][:environments][:generate_types]).to eq(true) end it 'only calls puppet generate types on specified environment' do - subject.instance_variable_set(:@argv, %w[first]) + settings = subject.instance_variable_get(:@settings) + settings[:overrides][:environments][:requested_environments] = %w{first} + subject.instance_variable_set(:@settings, settings) expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| if environment.dirname == 'first' expect(environment).to receive(:generate_types!) @@ -273,8 +462,8 @@ describe R10K::Action::Deploy::Environme end it 'does not call puppet generate types on puppetfile failure' do - allow(subject).to receive(:visit_puppetfile) { subject.instance_variable_set(:@visit_ok, false) } expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + allow(environment).to receive(:deploy) { subject.instance_variable_set(:@visit_ok, false) } expect(environment).not_to receive(:generate_types!) original.call(environment, &block) end.twice @@ -282,10 +471,11 @@ describe R10K::Action::Deploy::Environme end it 'calls puppet generate types on previous puppetfile failure' do - allow(subject).to receive(:visit_puppetfile) do |puppetfile| - subject.instance_variable_set(:@visit_ok, false) if puppetfile.environment.dirname == 'first' - end expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + allow(environment).to receive(:deploy) do + subject.instance_variable_set(:@visit_ok, false) if environment.dirname == 'first' + end + if environment.dirname == 'second' expect(environment).to receive(:generate_types!) else @@ -302,15 +492,16 @@ describe R10K::Action::Deploy::Environme described_class.new( { config: '/some/nonexistent/path', - puppetfile: true, + modules: true, 'generate-types': false }, - %w[first] + %w[first], + {} ) end it 'generate_types is false' do - expect(subject.instance_variable_get(:@generate_types)).to eq(false) + expect(subject.settings[:overrides][:environments][:generate_types]).to eq(false) end it 'does not call puppet generate types' do @@ -325,7 +516,7 @@ describe R10K::Action::Deploy::Environme describe 'with puppet-path' do - subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-path': '/nonexistent' }, []) } + subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-path': '/nonexistent' }, [], {}) } it 'sets puppet_path' do expect(subject.instance_variable_get(:@puppet_path)).to eq('/nonexistent') @@ -334,12 +525,30 @@ describe R10K::Action::Deploy::Environme describe 'with puppet-conf' do - subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-conf': '/nonexistent' }, []) } + subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-conf': '/nonexistent' }, [], {}) } it 'sets puppet_conf' do expect(subject.instance_variable_get(:@puppet_conf)).to eq('/nonexistent') end end + + describe 'with private-key' do + + subject { described_class.new({ config: '/some/nonexistent/path', 'private-key': '/nonexistent' }, [], {}) } + + it 'sets private_key' do + expect(subject.instance_variable_get(:@private_key)).to eq('/nonexistent') + end + end + + describe 'with oauth-token' do + + subject { described_class.new({ config: '/some/nonexistent/path', 'oauth-token': '/nonexistent' }, [], {}) } + + it 'sets oauth_token' do + expect(subject.instance_variable_get(:@oauth_token)).to eq('/nonexistent') + end + end end describe "write_environment_info!" do @@ -352,16 +561,31 @@ describe R10K::Action::Deploy::Environme def initialize(path, info) @path = path @info = info - @puppetfile = R10K::Puppetfile.new + @puppetfile = R10K::Puppetfile.new("", {}) end end let(:mock_stateful_repo_1) { instance_double("R10K::Git::StatefulRepository", :head => "123456") } let(:mock_stateful_repo_2) { instance_double("R10K::Git::StatefulRepository", :head => "654321") } - let(:mock_git_module_1) { instance_double("R10K::Module::Git", :name => "my_cool_module", :version => "1.0", :repo => mock_stateful_repo_1) } - let(:mock_git_module_2) { instance_double("R10K::Module::Git", :name => "my_lame_module", :version => "0.0.1", :repo => mock_stateful_repo_2) } - let(:mock_forge_module_1) { double(:name => "their_shiny_module", :version => "2.0.0") } - let(:mock_puppetfile) { instance_double("R10K::Puppetfile", :modules => [mock_git_module_1, mock_git_module_2, mock_forge_module_1]) } + let(:mock_git_module_1) do + instance_double("R10K::Module::Git", + :name => "my_cool_module", + :properties => { + :type => :git, + :expected => "1.0", + :actual => mock_stateful_repo_1.head + }) + end + let(:mock_git_module_2) do + instance_double("R10K::Module::Git", + :name => "my_uncool_module", + :properties => { + :type => :git, + :expected => "0.0.1", + :actual => mock_stateful_repo_2.head + }) + end + let(:mock_forge_module_1) { double(:name => "their_shiny_module", :properties => { :expected => "2.0.0" }) } before(:all) do @tmp_path = "./tmp-r10k-test-dir/" @@ -373,12 +597,9 @@ describe R10K::Action::Deploy::Environme Dir.delete(@tmp_path) end - it "writes the .r10k-deploy file correctly" do - allow(R10K::Puppetfile).to receive(:new).and_return(mock_puppetfile) - allow(mock_forge_module_1).to receive(:repo).and_raise(NoMethodError) - + it "writes the .r10k-deploy file correctly if all goes well" do fake_env = Fake_Environment.new(@tmp_path, {:name => "my_cool_environment", :signature => "pablo picasso"}) - allow(fake_env).to receive(:modules).and_return(mock_puppetfile.modules) + allow(fake_env).to receive(:modules).and_return([mock_git_module_1, mock_git_module_2, mock_forge_module_1]) subject.send(:write_environment_info!, fake_env, "2019-01-01 23:23:22 +0000", true) file_contents = File.read("#{@tmp_path}/.r10k-deploy.json") @@ -392,13 +613,28 @@ describe R10K::Action::Deploy::Environme expect(r10k_deploy['module_deploys'][0]['name']).to eq("my_cool_module") expect(r10k_deploy['module_deploys'][0]['version']).to eq("1.0") expect(r10k_deploy['module_deploys'][0]['sha']).to eq("123456") - expect(r10k_deploy['module_deploys'][1]['name']).to eq("my_lame_module") + expect(r10k_deploy['module_deploys'][1]['name']).to eq("my_uncool_module") expect(r10k_deploy['module_deploys'][1]['version']).to eq("0.0.1") expect(r10k_deploy['module_deploys'][1]['sha']).to eq("654321") expect(r10k_deploy['module_deploys'][2]['name']).to eq("their_shiny_module") expect(r10k_deploy['module_deploys'][2]['version']).to eq("2.0.0") expect(r10k_deploy['module_deploys'][2]['sha']).to eq(nil) + end + + it "writes the .r10k-deploy file correctly if there's a failure" do + fake_env = Fake_Environment.new(@tmp_path, {:name => "my_cool_environment", :signature => "pablo picasso"}) + allow(fake_env).to receive(:modules).and_return([mock_git_module_1, mock_git_module_2, mock_forge_module_1]) + allow(mock_forge_module_1).to receive(:properties).and_raise(StandardError) + subject.send(:write_environment_info!, fake_env, "2019-01-01 23:23:22 +0000", true) + + file_contents = File.read("#{@tmp_path}/.r10k-deploy.json") + r10k_deploy = JSON.parse(file_contents) + expect(r10k_deploy['name']).to eq("my_cool_environment") + expect(r10k_deploy['signature']).to eq("pablo picasso") + expect(r10k_deploy['started_at']).to eq("2019-01-01 23:23:22 +0000") + expect(r10k_deploy['deploy_success']).to eq(true) + expect(r10k_deploy['module_deploys'].length).to eq(0) end end end diff -pruN 3.7.0-2.1/spec/unit/action/deploy/module_spec.rb 3.15.4-1/spec/unit/action/deploy/module_spec.rb --- 3.7.0-2.1/spec/unit/action/deploy/module_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/action/deploy/module_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -4,43 +4,67 @@ require 'r10k/action/deploy/module' describe R10K::Action::Deploy::Module do - subject { described_class.new({config: "/some/nonexistent/path"}, []) } + subject { described_class.new({config: "/some/nonexistent/path"}, [], {}) } it_behaves_like "a deploy action that requires a config file" it_behaves_like "a deploy action that can be write locked" describe "initializing" do it "accepts an environment option" do - described_class.new({environment: "production"}, []) + described_class.new({environment: "production"}, [], {}) end it "can accept a no-force option" do - described_class.new({:'no-force' => true}, []) + described_class.new({:'no-force' => true}, [], {}) end it 'can accept a generate-types option' do - described_class.new({ 'generate-types': true }, []) + described_class.new({ 'generate-types': true }, [], {}) end it 'can accept a puppet-path option' do - described_class.new({ 'puppet-path': '/nonexistent' }, []) + described_class.new({ 'puppet-path': '/nonexistent' }, [], {}) end it 'can accept a puppet-conf option' do - described_class.new({ 'puppet-conf': '/nonexistent' }, []) + described_class.new({ 'puppet-conf': '/nonexistent' }, [], {}) end it 'can accept a cachedir option' do - described_class.new({ cachedir: '/nonexistent' }, []) + described_class.new({ cachedir: '/nonexistent' }, [], {}) + end + + it 'can accept a private-key option' do + described_class.new({ 'private-key': '/nonexistent' }, [], {}) + end + + it 'can accept a token option' do + described_class.new({ 'oauth-token': '/nonexistent' }, [], {}) + end + + it 'can accept an app id option' do + described_class.new({ 'github-app-id': '/nonexistent' }, [], {}) + end + + it 'can accept a ttl option' do + described_class.new({ 'github-app-ttl': '/nonexistent' }, [], {}) + end + + it 'can accept a ssl private key option' do + described_class.new({ 'github-app-key': '/nonexistent' }, [], {}) + end + + it 'can accept a exclude-spec option' do + described_class.new({ :'exclude-spec' => true }, [], {}) end end describe "with no-force" do - subject { described_class.new({ config: "/some/nonexistent/path", :'no-force' => true}, [] )} + subject { described_class.new({ config: "/some/nonexistent/path", :'no-force' => true}, [], {}) } it "tries to preserve local modifications" do - expect(subject.force).to equal(false) + expect(subject.settings[:overrides][:modules][:force]).to equal(false) end end @@ -53,6 +77,11 @@ describe R10K::Action::Deploy::Module do basedir: '/some/nonexistent/path/control', environments: %w[first second] } + }, + overrides: { + modules: { + pool_size: 4 + } } ) end @@ -68,32 +97,26 @@ describe R10K::Action::Deploy::Module do config: '/some/nonexistent/path', 'generate-types': true }, - %w[first] + %w[first], + {} ) end - before do - allow(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| - expect(environment.puppetfile).to receive(:modules_by_vcs_cachedir).and_return( - {none: [R10K::Module::Local.new(environment.name, '/fakedir', [], environment)]} - ) - original.call(environment, &block) - end - end - it 'generate_types is true' do - expect(subject.instance_variable_get(:@generate_types)).to eq(true) + expect(subject.settings[:overrides][:environments][:generate_types]).to eq(true) end - it 'only calls puppet generate types on environments with specified module' do - expect(subject).to receive(:visit_module).and_wrap_original do |original, mod, &block| - if mod.name == 'first' - expect(mod.environment).to receive(:generate_types!) + it 'only calls puppet generate types on environments where the specified module was updated' do + allow(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + if environment.name == 'first' + expect(environment).to receive(:deploy).and_return(['first']) + expect(environment).to receive(:generate_types!) else - expect(mod.environment).not_to receive(:generate_types!) + expect(environment).to receive(:deploy).and_return([]) + expect(environment).not_to receive(:generate_types!) end - original.call(mod, &block) - end.twice + original.call(environment, &block) + end subject.call end end @@ -105,12 +128,13 @@ describe R10K::Action::Deploy::Module do config: '/some/nonexistent/path', 'generate-types': false }, - %w[first] + %w[first], + {} ) end it 'generate_types is false' do - expect(subject.instance_variable_get(:@generate_types)).to eq(false) + expect(subject.settings[:overrides][:environments][:generate_types]).to eq(false) end it 'does not call puppet generate types' do |it| @@ -125,7 +149,7 @@ describe R10K::Action::Deploy::Module do describe 'with puppet-path' do - subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-path': '/nonexistent' }, []) } + subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-path': '/nonexistent' }, [], {}) } it 'sets puppet_path' do expect(subject.instance_variable_get(:@puppet_path)).to eq('/nonexistent') @@ -134,7 +158,7 @@ describe R10K::Action::Deploy::Module do describe 'with puppet-conf' do - subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-conf': '/nonexistent' }, []) } + subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-conf': '/nonexistent' }, [], {}) } it 'sets puppet_conf' do expect(subject.instance_variable_get(:@puppet_conf)).to eq('/nonexistent') @@ -143,10 +167,310 @@ describe R10K::Action::Deploy::Module do describe 'with cachedir' do - subject { described_class.new({ config: '/some/nonexistent/path', cachedir: '/nonexistent' }, []) } + subject { described_class.new({ config: '/some/nonexistent/path', cachedir: '/nonexistent' }, [], {}) } - it 'sets puppet_path' do + it 'sets cachedir' do expect(subject.instance_variable_get(:@cachedir)).to eq('/nonexistent') end end + + describe 'with private-key' do + + subject { described_class.new({ config: '/some/nonexistent/path', 'private-key': '/nonexistent' }, [], {}) } + + it 'sets private_key' do + expect(subject.instance_variable_get(:@private_key)).to eq('/nonexistent') + end + end + + describe 'with oauth-token' do + + subject { described_class.new({ config: '/some/nonexistent/path', 'oauth-token': '/nonexistent' }, [], {}) } + + it 'sets token_path' do + expect(subject.instance_variable_get(:@oauth_token)).to eq('/nonexistent') + end + end + + describe 'with github-app-id' do + + subject { described_class.new({ config: '/some/nonexistent/path', 'github-app-id': '/nonexistent' }, [], {}) } + + it 'sets github-app-id' do + expect(subject.instance_variable_get(:@github_app_id)).to eq('/nonexistent') + end + end + + describe 'with github-app-key' do + + subject { described_class.new({ config: '/some/nonexistent/path', 'github-app-key': '/nonexistent' }, [], {}) } + + it 'sets github-app-key' do + expect(subject.instance_variable_get(:@github_app_key)).to eq('/nonexistent') + end + end + + describe 'with github-app-ttl' do + + subject { described_class.new({ config: '/some/nonexistent/path', 'github-app-ttl': '/nonexistent' }, [], {}) } + + it 'sets github-app-ttl' do + expect(subject.instance_variable_get(:@github_app_ttl)).to eq('/nonexistent') + end + end + + describe 'with modules' do + + subject { described_class.new({ config: '/some/nonexistent/path' }, ['mod1', 'mod2'], {}) } + + let(:cache) { instance_double("R10K::Git::Cache", 'sanitized_dirname' => 'foo', 'cached?' => true, 'sync' => true) } + let(:repo) { instance_double("R10K::Git::StatefulRepository", cache: cache, resolve: 'main', tracked_paths: []) } + + it 'does not sync modules not given' do + allow(R10K::Deployment).to receive(:new).and_wrap_original do |original, settings, &block| + original.call(settings.merge({ + sources: { + main: { + remote: 'https://not/a/remote', + basedir: '/not/a/basedir', + type: 'git' + } + } + })) + end + + allow(R10K::Git::StatefulRepository).to receive(:new).and_return(repo) + allow(R10K::Git).to receive_message_chain(:cache, :generate).and_return(cache) + allow_any_instance_of(R10K::Source::Git).to receive(:environment_names).and_return([R10K::Environment::Name.new('first', {})]) + + expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + # For this test we want to have realistic Modules and access to + # their internal Repos to validate the sync. Unfortunately, to + # do so we do some invasive mocking, effectively implementing + # our own R10K::ModuleLoader::Puppetfile#load. We directly update + # the Environment's internal ModuleLoader and then call `load` on + # it so it will create the correct loaded_content. + loader = environment.loader + allow(loader).to receive(:puppetfile_content).and_return('') + expect(loader).to receive(:load) do + loader.add_module('mod1', { git: 'https://remote' }) + loader.add_module('mod2', { git: 'https://remote' }) + loader.add_module('mod3', { git: 'https://remote' }) + + loaded_content = loader.load! + loaded_content[:modules].each do |mod| + if ['mod1', 'mod2'].include?(mod.name) + expect(mod.should_sync?).to be(true) + else + expect(mod.should_sync?).to be(false) + end + expect(mod).to receive(:sync).and_call_original + end + + loaded_content + end + + original.call(environment, &block) + end + + expect(repo).to receive(:sync).twice + + subject.call + end + end + + describe 'with environments' do + subject { described_class.new({ config: '/some/nonexistent/path', environment: 'first' }, ['mod1'], {}) } + + let(:cache) { instance_double("R10K::Git::Cache", 'sanitized_dirname' => 'foo', 'cached?' => true, 'sync' => true) } + let(:repo) { instance_double("R10K::Git::StatefulRepository", cache: cache, resolve: 'main', tracked_paths: []) } + + it 'only syncs to the given environments' do + allow(R10K::Deployment).to receive(:new).and_wrap_original do |original, settings, &block| + original.call(settings.merge({ + sources: { + main: { + remote: 'https://not/a/remote', + basedir: '/not/a/basedir', + type: 'git' + } + } + })) + end + + allow(R10K::Git::StatefulRepository).to receive(:new).and_return(repo) + allow(R10K::Git).to receive_message_chain(:cache, :generate).and_return(cache) + allow_any_instance_of(R10K::Source::Git).to receive(:environment_names).and_return([R10K::Environment::Name.new('first', {}), + R10K::Environment::Name.new('second', {})]) + + expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + loader = environment.loader + + if environment.name == 'first' + # For this test we want to have realistic Modules and access to + # their internal Repos to validate the sync. Unfortunately, to + # do so we do some invasive mocking, effectively implementing + # our own R10K::ModuleLoader::Puppetfile#load. We directly update + # the Environment's internal ModuleLoader and then call `load` on + # it so it will create the correct loaded_content. + allow(loader).to receive(:puppetfile_content).and_return('') + expect(loader).to receive(:load) do + loader.add_module('mod1', { git: 'https://remote' }) + loader.add_module('mod2', { git: 'https://remote' }) + + loaded_content = loader.load! + loaded_content[:modules].each do |mod| + if mod.name == 'mod1' + expect(mod.should_sync?).to be(true) + else + expect(mod.should_sync?).to be(false) + end + expect(mod).to receive(:sync).and_call_original + end + + loaded_content + end + + else + expect(loader).not_to receive(:load) + end + + original.call(environment, &block) + end.twice + + expect(repo).to receive(:sync).once + expect(subject.logger).to receive(:debug1).with(/Updating modules.*in environment.*first/i) + expect(subject.logger).to receive(:debug1).with(/skipping environment.*second/i) + + subject.call + end + end + + + describe "postrun" do + let(:mock_config) do + R10K::Deployment::MockConfig.new( + :sources => { + :control => { + :type => :mock, + :basedir => '/some/nonexistent/path/control', + :environments => %w[first second third], + } + } + ) + end + + context "basic postrun hook" do + let(:settings) { { postrun: ["/path/to/executable", "arg1", "arg2"] } } + let(:deployment) { R10K::Deployment.new(mock_config.merge(settings)) } + + before do + expect(R10K::Deployment).to receive(:new).and_return(deployment) + end + + subject do + described_class.new({config: "/some/nonexistent/path" }, + ['mod1'], settings) + end + + it "is passed to Subprocess" do + mock_subprocess = double + allow(mock_subprocess).to receive(:logger=) + expect(mock_subprocess).to receive(:execute) + + expect(R10K::Util::Subprocess).to receive(:new). + with(["/path/to/executable", "arg1", "arg2"]). + and_return(mock_subprocess) + + expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + modified = subject.instance_variable_get(:@modified_envs) << environment + subject.instance_variable_set(:modified_envs, modified) + end.exactly(3).times + + subject.call + end + end + + context "supports environments" do + context "with one environment" do + let(:settings) { { postrun: ["/generate/types/wrapper", "$modifiedenvs"] } } + let(:deployment) { R10K::Deployment.new(mock_config.merge(settings)) } + + before do + expect(R10K::Deployment).to receive(:new).and_return(deployment) + end + + subject do + described_class.new({ config: '/some/nonexistent/path', + environment: 'first' }, + ['mod1'], settings) + end + + it "properly substitutes the environment" do + mock_subprocess = double + allow(mock_subprocess).to receive(:logger=) + expect(mock_subprocess).to receive(:execute) + + expect(R10K::Util::Subprocess).to receive(:new). + with(["/generate/types/wrapper", "first"]). + and_return(mock_subprocess) + + expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + if environment.name == 'first' + expect(environment).to receive(:deploy).and_return(['first']) + end + original.call(environment, &block) + end.exactly(3).times + + subject.call + end + end + + context "with all environments" do + let(:settings) { { postrun: ["/generate/types/wrapper", "$modifiedenvs"] } } + let(:deployment) { R10K::Deployment.new(mock_config.merge(settings)) } + + before do + expect(R10K::Deployment).to receive(:new).and_return(deployment) + end + + subject do + described_class.new({ config: '/some/nonexistent/path' }, + ['mod1'], settings) + end + + it "properly substitutes the environment where modules were deployed" do + mock_subprocess = double + allow(mock_subprocess).to receive(:logger=) + expect(mock_subprocess).to receive(:execute) + + expect(R10K::Util::Subprocess).to receive(:new). + with(["/generate/types/wrapper", "first third"]). + and_return(mock_subprocess) + + expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + if ['first', 'third'].include?(environment.name) + expect(environment).to receive(:deploy).and_return(['mod1']) + end + original.call(environment, &block) + end.exactly(3).times + + subject.call + end + + it "does not execute the command if no envs had the module" do + expect(R10K::Util::Subprocess).not_to receive(:new) + + mock_mod2 = double('mock_mod', name: 'mod2') + expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + expect(environment).to receive(:deploy).and_return([]) + original.call(environment, &block) + end.exactly(3).times + + subject.call + end + end + end + end end + diff -pruN 3.7.0-2.1/spec/unit/action/puppetfile/check_spec.rb 3.15.4-1/spec/unit/action/puppetfile/check_spec.rb --- 3.7.0-2.1/spec/unit/action/puppetfile/check_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/action/puppetfile/check_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -3,7 +3,7 @@ require 'r10k/action/puppetfile/check' describe R10K::Action::Puppetfile::Check do let(:default_opts) { {root: "/some/nonexistent/path"} } - let(:puppetfile) { instance_double('R10K::Puppetfile', :load! => true) } + let(:loader) { instance_double('R10K::ModuleLoader::Puppetfile', :load! => {}) } def checker(opts = {}, argv = [], settings = {}) opts = default_opts.merge(opts) @@ -11,7 +11,11 @@ describe R10K::Action::Puppetfile::Check end before(:each) do - allow(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, nil).and_return(puppetfile) + allow(R10K::ModuleLoader::Puppetfile). + to receive(:new). + with({ + basedir: "/some/nonexistent/path", + }).and_return(loader) end it_behaves_like "a puppetfile action" @@ -23,8 +27,11 @@ describe R10K::Action::Puppetfile::Check end it "prints an error message when validating the Puppetfile syntax raised an error" do - allow(puppetfile).to receive(:load!).and_raise(R10K::Error.new("Boom!")) - allow(R10K::Errors::Formatting).to receive(:format_exception).with(instance_of(R10K::Error), anything).and_return("Formatted error message") + allow(loader).to receive(:load!).and_raise(R10K::Error.new("Boom!")) + allow(R10K::Errors::Formatting). + to receive(:format_exception). + with(instance_of(R10K::Error), anything). + and_return("Formatted error message") expect($stderr).to receive(:puts).with("Formatted error message") @@ -34,7 +41,12 @@ describe R10K::Action::Puppetfile::Check it "respects --puppetfile option" do allow($stderr).to receive(:puts) - expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, "/custom/puppetfile/path").and_return(puppetfile) + expect(R10K::ModuleLoader::Puppetfile). + to receive(:new). + with({ + basedir: "/some/nonexistent/path", + puppetfile: "/custom/puppetfile/path" + }).and_return(loader) checker({puppetfile: "/custom/puppetfile/path"}).call end diff -pruN 3.7.0-2.1/spec/unit/action/puppetfile/install_spec.rb 3.15.4-1/spec/unit/action/puppetfile/install_spec.rb --- 3.7.0-2.1/spec/unit/action/puppetfile/install_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/action/puppetfile/install_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -2,8 +2,12 @@ require 'spec_helper' require 'r10k/action/puppetfile/install' describe R10K::Action::Puppetfile::Install do - let(:default_opts) { {root: "/some/nonexistent/path"} } - let(:puppetfile) { R10K::Puppetfile.new('/some/nonexistent/path', nil, nil) } + let(:default_opts) { { root: "/some/nonexistent/path" } } + let(:loader) { + R10K::ModuleLoader::Puppetfile.new( + basedir: '/some/nonexistent/path', + overrides: {force: false}) + } def installer(opts = {}, argv = [], settings = {}) opts = default_opts.merge(opts) @@ -11,8 +15,11 @@ describe R10K::Action::Puppetfile::Insta end before(:each) do - allow(puppetfile).to receive(:load!).and_return(nil) - allow(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, nil, nil, nil).and_return(puppetfile) + allow(loader).to receive(:load!).and_return({}) + allow(R10K::ModuleLoader::Puppetfile).to receive(:new). + with({basedir: "/some/nonexistent/path", + overrides: {force: false}}). + and_return(loader) end it_behaves_like "a puppetfile install action" @@ -20,14 +27,19 @@ describe R10K::Action::Puppetfile::Insta describe "installing modules" do let(:modules) do (1..4).map do |idx| - R10K::Module::Base.new("author/modname#{idx}", "/some/nonexistent/path/modname#{idx}", nil) + R10K::Module::Base.new("author/modname#{idx}", + "/some/nonexistent/path/modname#{idx}", + {}) end end before do - allow(puppetfile).to receive(:purge!) - allow(puppetfile).to receive(:modules).and_return(modules) - allow(puppetfile).to receive(:modules_by_vcs_cachedir).and_return({none: modules}) + allow(loader).to receive(:load!).and_return({ + modules: modules, + managed_directories: [], + desired_contents: [], + purge_exclusions: [] + }) end it "syncs each module in the Puppetfile" do @@ -45,39 +57,54 @@ describe R10K::Action::Puppetfile::Insta end describe "purging" do - before do - allow(puppetfile).to receive(:modules).and_return([]) - end - it "purges the moduledir after installation" do - expect(puppetfile).to receive(:purge!) + allow(loader).to receive(:load!).and_return({ + modules: [], + desired_contents: [ 'root/foo' ], + managed_directories: [ 'root' ], + purge_exclusions: [ 'root/**/**.rb' ] + }) + + mock_cleaner = double("cleaner") + + expect(R10K::Util::Cleaner).to receive(:new). + with(["root"], ["root/foo"], ["root/**/**.rb"]). + and_return(mock_cleaner) + expect(mock_cleaner).to receive(:purge!) installer.call end end describe "using custom paths" do - it "can use a custom puppetfile path" do - expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, "/some/other/path/Puppetfile", nil, nil).and_return(puppetfile) + it "can use a custom moduledir path" do + expect(R10K::ModuleLoader::Puppetfile).to receive(:new). + with({basedir: "/some/nonexistent/path", + overrides: { force: false }, + puppetfile: "/some/other/path/Puppetfile"}). + and_return(loader) installer({puppetfile: "/some/other/path/Puppetfile"}).call - end - it "can use a custom moduledir path" do - expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", "/some/other/path/site-modules", nil, nil, nil).and_return(puppetfile) + expect(R10K::ModuleLoader::Puppetfile).to receive(:new). + with({basedir: "/some/nonexistent/path", + overrides: { force: false }, + moduledir: "/some/other/path/site-modules"}). + and_return(loader) installer({moduledir: "/some/other/path/site-modules"}).call end end describe "forcing to overwrite local changes" do - before do - allow(puppetfile).to receive(:modules).and_return([]) - end - it "can use the force overwrite option" do - subject = described_class.new({root: "/some/nonexistent/path", force: true}, []) - expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, nil, nil, true).and_return(puppetfile) + allow(loader).to receive(:load!).and_return({ modules: [] }) + + subject = described_class.new({root: "/some/nonexistent/path", force: true}, [], {}) + expect(R10K::ModuleLoader::Puppetfile).to receive(:new). + with({basedir: "/some/nonexistent/path", + overrides: { force: true }}). + and_return(loader) subject.call end diff -pruN 3.7.0-2.1/spec/unit/action/puppetfile/purge_spec.rb 3.15.4-1/spec/unit/action/puppetfile/purge_spec.rb --- 3.7.0-2.1/spec/unit/action/puppetfile/purge_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/action/puppetfile/purge_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -3,7 +3,15 @@ require 'r10k/action/puppetfile/purge' describe R10K::Action::Puppetfile::Purge do let(:default_opts) { {root: "/some/nonexistent/path"} } - let(:puppetfile) { instance_double('R10K::Puppetfile', :load! => nil) } + let(:puppetfile) do + instance_double('R10K::ModuleLoader::Puppetfile', + :load! => { + :modules => %w{mod}, + :managed_directories => %w{foo}, + :desired_contents => %w{bar}, + :purge_exclusions => %w{baz} + }) + end def purger(opts = {}, argv = [], settings = {}) opts = default_opts.merge(opts) @@ -11,30 +19,40 @@ describe R10K::Action::Puppetfile::Purge end before(:each) do - allow(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, nil).and_return(puppetfile) + allow(R10K::ModuleLoader::Puppetfile).to receive(:new). + with({basedir: "/some/nonexistent/path"}). + and_return(puppetfile) end it_behaves_like "a puppetfile action" it "purges unmanaged entries in the Puppetfile moduledir" do - expect(puppetfile).to receive(:purge!) + mock_cleaner = double("cleaner") + + expect(R10K::Util::Cleaner).to receive(:new). + with(["foo"], ["bar"], ["baz"]). + and_return(mock_cleaner) + + expect(mock_cleaner).to receive(:purge!) purger.call end describe "using custom paths" do - before(:each) do - allow(puppetfile).to receive(:purge!) - end - it "can use a custom puppetfile path" do - expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, "/some/other/path/Puppetfile").and_return(puppetfile) + expect(R10K::ModuleLoader::Puppetfile).to receive(:new). + with({basedir: "/some/nonexistent/path", + puppetfile: "/some/other/path/Puppetfile"}). + and_return(puppetfile) purger({puppetfile: "/some/other/path/Puppetfile"}).call end it "can use a custom moduledir path" do - expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", "/some/other/path/site-modules", nil).and_return(puppetfile) + expect(R10K::ModuleLoader::Puppetfile).to receive(:new). + with({basedir: "/some/nonexistent/path", + moduledir: "/some/other/path/site-modules"}). + and_return(puppetfile) purger({moduledir: "/some/other/path/site-modules"}).call end diff -pruN 3.7.0-2.1/spec/unit/action/runner_spec.rb 3.15.4-1/spec/unit/action/runner_spec.rb --- 3.7.0-2.1/spec/unit/action/runner_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/action/runner_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -10,11 +10,12 @@ describe R10K::Action::Runner do Class.new do attr_reader :opts attr_reader :argv + attr_reader :settings def initialize(opts, argv, settings = {}) @opts = opts @argv = argv - @settings = {} + @settings = settings end def call @@ -158,55 +159,248 @@ describe R10K::Action::Runner do end describe "configuring logging" do + before(:each) do + R10K::Logging.outputters.clear + end + it "sets the log level if :loglevel is provided" do runner = described_class.new({:opts => :yep, :loglevel => 'FATAL'}, %w[args yes], action_class) - expect(R10K::Logging).to receive(:level=).with('FATAL') + # The settings/overrides system causes the level to be set twice + expect(R10K::Logging).to receive(:level=).with('FATAL').twice runner.call end + # The logging fixture tests require a platform with syslog + if !R10K::Util::Platform.windows? + it "sets the log level if the logging.level setting is provided" do + runner = described_class.new({ opts: :yep, config: 'spec/fixtures/unit/action/r10k_logging.yaml'}, %w[args yes], action_class) + expect(R10K::Logging).to receive(:level=).with('FATAL') + runner.call + end + + it "sets the outputters if logging.outputs is provided" do + runner = described_class.new({ opts: :yep, config: 'spec/fixtures/unit/action/r10k_logging.yaml' }, %w[args yes], action_class) + expect(R10K::Logging).to receive(:add_outputters).with([ + { type: 'file', parameters: { filename: 'r10k.log' } }, + { type: 'syslog' } + ]) + runner.call + end + + it "disables the default outputter if the logging.disable_default_stderr setting is provided" do + runner = described_class.new({ opts: :yep, config: 'spec/fixtures/unit/action/r10k_logging.yaml'}, %w[args yes], action_class) + expect(R10K::Logging).to receive(:disable_default_stderr=).with(true) + runner.call + end + + it "adds additional log outputs if the logging.outputs setting is provided" do + runner = described_class.new({ opts: :yep, config: 'spec/fixtures/unit/action/r10k_logging.yaml'}, %w[args yes], action_class) + runner.call + expect(R10K::Logging.outputters).to_not be_empty + end + + it "disables the default output if the logging.disable_default_stderr setting is provided" do + runner = described_class.new({ opts: :yep, config: 'spec/fixtures/unit/action/r10k_logging.yaml'}, %w[args yes], action_class) + runner.call + expect(runner.logger.outputters).to satisfy { |outputs| outputs.any? { |output| output.is_a?(R10K::Logging::TerminalOutputter) && output.level == Log4r::OFF } } + end + end + + it "doesn't add additional log outputs if the logging.outputs setting is not provided" do + runner.call + expect(R10K::Logging.outputters).to be_empty + end + + it "includes the default stderr outputter" do + runner.call + expect(runner.logger.outputters).to satisfy { |outputs| outputs.any? { |output| output.is_a? R10K::Logging::TerminalOutputter } } + end + it "does not modify the loglevel if :loglevel is not provided" do expect(R10K::Logging).to_not receive(:level=) runner.call end end - describe "configuration authorization" do - context "when license is not present" do - before(:each) do - expect(R10K::Util::License).to receive(:load).and_return(nil) - end + describe "configuring github app credentials" do + it 'errors if app id is passed without ssl key' do + runner = described_class.new( + { 'github-app-id': '/nonexistent', }, + %w[args yes], + action_class + ) + expect{ runner.call }.to raise_error(R10K::Error, /Must specify both id and SSL private key/) + end + + it 'errors if ssl key is passed without app id' do + runner = described_class.new( + { 'github-app-key': '/nonexistent', }, + %w[args yes], + action_class + ) + expect{ runner.call }.to raise_error(R10K::Error, /Must specify both id and SSL private key/) + end + + it 'errors if both app id and token paths are passed' do + runner = described_class.new( + { 'github-app-id': '/nonexistent', 'oauth-token': '/also/fake' }, + %w[args yes], + action_class + ) + expect{ runner.call }.to raise_error(R10K::Error, /Cannot specify both/) + end + + it 'errors if both ssl key and token paths are passed' do + runner = described_class.new( + { 'github-app-key': '/nonexistent', 'oauth-token': '/also/fake' }, + %w[args yes], + action_class + ) + expect{ runner.call }.to raise_error(R10K::Error, /Cannot specify both/) + end + + it 'errors if both ssl key and ssh key paths are passed' do + runner = described_class.new( + { 'github-app-key': '/nonexistent', 'private-key': '/also/fake' }, + %w[args yes], + action_class + ) + expect{ runner.call }.to raise_error(R10K::Error, /Cannot specify both/) + end + + it 'errors if both app id and ssh key are passed' do + runner = described_class.new( + { 'github-app-id': '/nonexistent', 'private-key': '/also/fake' }, + %w[args yes], + action_class + ) + expect{ runner.call }.to raise_error(R10K::Error, /Cannot specify both/) + end + + it 'saves the parameters in settings hash' do + runner = described_class.new( + { 'github-app-id': '123456', 'github-app-key': '/my/ssl/key', 'github-app-ttl': '600' }, + %w[args yes], + action_class + ) + runner.call + expect(runner.instance.settings[:git][:github_app_id]).to eq('123456') + expect(runner.instance.settings[:git][:github_app_key]).to eq('/my/ssl/key') + expect(runner.instance.settings[:git][:github_app_ttl]).to eq('600') + end - it "does not set authorization header on connection class" do - expect(PuppetForge::Connection).not_to receive(:authorization=) - runner.setup_authorization - end + it 'saves the parameters in settings hash without ttl and uses its default value' do + runner = described_class.new( + { 'github-app-id': '123456', 'github-app-key': '/my/ssl/key', }, + %w[args yes], + action_class + ) + runner.call + expect(runner.instance.settings[:git][:github_app_id]).to eq('123456') + expect(runner.instance.settings[:git][:github_app_key]).to eq('/my/ssl/key') + expect(runner.instance.settings[:git][:github_app_ttl]).to eq('120') end + end - context "when license is present but invalid" do - before(:each) do - expect(R10K::Util::License).to receive(:load).and_raise(R10K::Error.new('invalid license')) + describe "configuring git credentials" do + it 'errors if both token and key paths are passed' do + runner = described_class.new({ 'oauth-token': '/nonexistent', + 'private-key': '/also/fake' }, %w[args yes], action_class) + expect{ runner.call }.to raise_error(R10K::Error, /Cannot specify both/) + end + + it 'saves the sshkey path in settings hash' do + runner = described_class.new({ 'private-key': '/my/ssh/key' }, %w[args yes], action_class) + runner.call + expect(runner.instance.settings[:git][:private_key]).to eq('/my/ssh/key') + end + + it 'overrides per-repo sshkey in settings hash' do + runner = described_class.new({ config: "spec/fixtures/unit/action/r10k_creds.yaml", + 'private-key': '/my/ssh/key' }, + %w[args yes], + action_class) + runner.call + expect(runner.instance.settings[:git][:private_key]).to eq('/my/ssh/key') + expect(runner.instance.settings[:git][:repositories].count).to eq(2) + runner.instance.settings[:git][:repositories].each do |repo_settings| + expect(repo_settings[:private_key]).to eq('/my/ssh/key') end + end - it "issues warning to logger" do - expect(runner.logger).to receive(:warn).with(/invalid license/) - runner.setup_authorization + it 'saves the token path in settings hash' do + runner = described_class.new({ 'oauth-token': '/my/token/path' }, %w[args yes], action_class) + runner.call + expect(runner.instance.settings[:git][:oauth_token]).to eq('/my/token/path') + end + + it 'overrides per-repo oauth token in settings hash' do + runner = described_class.new({ config: "spec/fixtures/unit/action/r10k_creds.yaml", + 'oauth-token': '/my/token' }, + %w[args yes], + action_class) + runner.call + expect(runner.instance.settings[:git][:oauth_token]).to eq('/my/token') + expect(runner.instance.settings[:git][:repositories].count).to eq(2) + runner.instance.settings[:git][:repositories].each do |repo_settings| + expect(repo_settings[:oauth_token]).to eq('/my/token') end + end + end - it "does not set authorization header on connection class" do - expect(PuppetForge::Connection).not_to receive(:authorization=) + describe "configuration authorization" do + context "settings auth" do + it "sets the configured token as the forge authorization header" do + options = { config: "spec/fixtures/unit/action/r10k_forge_auth.yaml" } + runner = described_class.new(options, %w[args yes], action_class) + + expect(PuppetForge).to receive(:host=).with('http://private-forge.com') + expect(PuppetForge::Connection).to receive(:authorization=).with('faketoken') + expect(PuppetForge::Connection).to receive(:authorization).and_return('faketoken') + expect(R10K::Util::License).not_to receive(:load) + runner.setup_settings runner.setup_authorization end end - context "when license is present and valid" do - before(:each) do - mock_license = double('pe-license', :authorization_token => 'test token') - expect(R10K::Util::License).to receive(:load).and_return(mock_license) + context "license auth" do + context "when license is not present" do + before(:each) do + expect(R10K::Util::License).to receive(:load).and_return(nil) + end + + it "does not set authorization header on connection class" do + expect(PuppetForge::Connection).not_to receive(:authorization=) + runner.setup_authorization + end end - it "sets authorization header on connection class" do - expect(PuppetForge::Connection).to receive(:authorization=).with('test token') - runner.setup_authorization + context "when license is present but invalid" do + before(:each) do + expect(R10K::Util::License).to receive(:load).and_raise(R10K::Error.new('invalid license')) + end + + it "issues warning to logger" do + expect(runner.logger).to receive(:warn).with(/invalid license/) + runner.setup_authorization + end + + it "does not set authorization header on connection class" do + expect(PuppetForge::Connection).not_to receive(:authorization=) + runner.setup_authorization + end + end + + context "when license is present and valid" do + before(:each) do + mock_license = double('pe-license', :authorization_token => 'test token') + expect(R10K::Util::License).to receive(:load).and_return(mock_license) + end + + it "sets authorization header on connection class" do + expect(PuppetForge::Connection).to receive(:authorization=).with('test token') + runner.setup_authorization + end end end end diff -pruN 3.7.0-2.1/spec/unit/environment/bare_spec.rb 3.15.4-1/spec/unit/environment/bare_spec.rb --- 3.7.0-2.1/spec/unit/environment/bare_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/unit/environment/bare_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,13 @@ +require 'spec_helper' +require 'r10k/environment' + +describe R10K::Environment::Bare do + it "warns on initialization" do + logger_spy = spy('logger') + allow_any_instance_of(described_class).to receive(:logger).and_return(logger_spy) + + described_class.new('envname', '/basedir', 'dirname', {}) + + expect(logger_spy).to have_received(:warn).with(%r{deprecated.*envname}) + end +end diff -pruN 3.7.0-2.1/spec/unit/environment/base_spec.rb 3.15.4-1/spec/unit/environment/base_spec.rb --- 3.7.0-2.1/spec/unit/environment/base_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/environment/base_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -3,10 +3,13 @@ require 'r10k/environment' describe R10K::Environment::Base do - subject(:environment) { described_class.new('envname', '/some/imaginary/path', 'env_name', {}) } + let(:basepath) { '/some/imaginary/path' } + let(:envname) { 'env_name' } + let(:path) { File.join(basepath, envname) } + subject(:environment) { described_class.new('envname', basepath, envname, {}) } it "can return the fully qualified path" do - expect(environment.path).to eq(Pathname.new('/some/imaginary/path/env_name')) + expect(environment.path).to eq(Pathname.new(path)) end it "raises an exception when #sync is called" do @@ -49,45 +52,55 @@ describe R10K::Environment::Base do describe "#purge_exclusions" do let(:mock_env) { instance_double("R10K::Environment::Base") } let(:mock_puppetfile) { instance_double("R10K::Puppetfile", :environment= => true, :environment => mock_env) } + let(:loader) do + instance_double("R10K::ModuleLoader::Puppetfile", + :environment= => nil, + :load => { :modules => @modules, + :managed_directories => @managed_dirs, + :desired_contents => @desired_contents, + :purge_exclusions => @purge_ex }) + end before(:each) do - allow(mock_puppetfile).to receive(:managed_directories).and_return([]) - allow(mock_puppetfile).to receive(:desired_contents).and_return([]) - allow(R10K::Puppetfile).to receive(:new).and_return(mock_puppetfile) + @modules = [] + @managed_dirs = [] + @desired_contents = [] + @purge_exclusions = [] end it "excludes .r10k-deploy.json" do + allow(R10K::ModuleLoader::Puppetfile).to receive(:new).and_return(loader) + subject.deploy + expect(subject.purge_exclusions).to include(/r10k-deploy\.json/) end it "excludes puppetfile managed directories" do - managed_dirs = [ + @managed_dirs = [ '/some/imaginary/path/env_name/modules', '/some/imaginary/path/env_name/data', ] - expect(mock_puppetfile).to receive(:managed_directories).and_return(managed_dirs) + allow(R10K::ModuleLoader::Puppetfile).to receive(:new).and_return(loader) + subject.deploy exclusions = subject.purge_exclusions - managed_dirs.each do |dir| + @managed_dirs.each do |dir| expect(exclusions).to include(dir) end end describe "puppetfile desired contents" do - let(:desired_contents) do - basedir = subject.path.to_s - - [ 'modules/apache', 'data/local/site' ].collect do |c| - File.join(basedir, c) - end - end before(:each) do - allow(File).to receive(:directory?).with(/^\/some\/imaginary\/path/).and_return(true) + @desired_contents = [ 'modules/apache', 'data/local/site' ].collect do |c| + File.join(path, c) + end - expect(mock_puppetfile).to receive(:desired_contents).and_return(desired_contents) + allow(File).to receive(:directory?).and_return true + allow(R10K::ModuleLoader::Puppetfile).to receive(:new).and_return(loader) + subject.deploy end it "excludes desired directory contents with glob" do diff -pruN 3.7.0-2.1/spec/unit/environment/git_spec.rb 3.15.4-1/spec/unit/environment/git_spec.rb --- 3.7.0-2.1/spec/unit/environment/git_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/environment/git_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -9,12 +9,28 @@ describe R10K::Environment::Git do '/some/nonexistent/environmentdir', 'gitref', { - :remote => 'git://git-server.site/my-repo.git', + :remote => 'https://git-server.site/my-repo.git', :ref => 'd026ea677116424d2968edb9cee8cbc24d09322b', } ) end + describe "initializing" do + subject do + described_class.new('name', '/dir', 'ref', { + :remote => 'url', + :ref => 'value', + :puppetfile_name => 'Puppetfile', + :moduledir => 'modules', + :modules => { }, + }) + end + + it "accepts valid base class initialization arguments" do + expect(subject.name).to eq 'name' + end + end + describe "storing attributes" do it "can return the environment name" do expect(subject.name).to eq 'myenv' @@ -29,7 +45,7 @@ describe R10K::Environment::Git do end it "can return the environment remote" do - expect(subject.remote).to eq 'git://git-server.site/my-repo.git' + expect(subject.remote).to eq 'https://git-server.site/my-repo.git' end it "can return the environment ref" do @@ -62,9 +78,10 @@ describe R10K::Environment::Git do describe "enumerating modules" do it "loads the Puppetfile and returns modules in that puppetfile" do - expect(subject.puppetfile).to receive(:load) - expect(subject.puppetfile).to receive(:modules).and_return [:modules] - expect(subject.modules).to eq([:modules]) + loaded = { desired_contents: [], managed_directories: [], purge_exclusions: [] } + mod = double('A module', :name => 'dbl') + expect(subject.loader).to receive(:load).and_return(loaded.merge(modules: [mod])) + expect(subject.modules).to eq([mod]) end end diff -pruN 3.7.0-2.1/spec/unit/environment/name_spec.rb 3.15.4-1/spec/unit/environment/name_spec.rb --- 3.7.0-2.1/spec/unit/environment/name_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/environment/name_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -2,25 +2,69 @@ require 'spec_helper' require 'r10k/environment/name' describe R10K::Environment::Name do + describe "strip_component" do + it "does not modify the given name when no strip_component is given" do + bn = described_class.new('myenv', source: 'source', prefix: false) + expect(bn.dirname).to eq 'myenv' + expect(bn.name).to eq 'myenv' + expect(bn.original_name).to eq 'myenv' + end + + it "removes the first occurance of a regex match when a regex is given" do + bn = described_class.new('myenv', source: 'source', prefix: false, strip_component: '/env/') + expect(bn.dirname).to eq 'my' + expect(bn.name).to eq 'my' + expect(bn.original_name).to eq 'myenv' + end + + it "does not modify the given name when there is no regex match" do + bn = described_class.new('myenv', source: 'source', prefix: false, strip_component: '/bar/') + expect(bn.dirname).to eq 'myenv' + expect(bn.name).to eq 'myenv' + expect(bn.original_name).to eq 'myenv' + end + + it "removes the given name's prefix when it matches strip_component" do + bn = described_class.new('env/prod', source: 'source', prefix: false, strip_component: 'env/') + expect(bn.dirname).to eq 'prod' + expect(bn.name).to eq 'prod' + expect(bn.original_name).to eq 'env/prod' + end + + it "raises an error when given an integer" do + expect { + described_class.new('env/prod', source: 'source', prefix: false, strip_component: 4) + }.to raise_error(%r{Improper.*"4"}) + end + end + describe "prefixing" do it "uses the branch name as the dirname when prefixing is off" do bn = described_class.new('mybranch', :source => 'source', :prefix => false) expect(bn.dirname).to eq 'mybranch' + expect(bn.name).to eq 'mybranch' + expect(bn.original_name).to eq 'mybranch' end it "prepends the source name when prefixing is on" do bn = described_class.new('mybranch', :source => 'source', :prefix => true) expect(bn.dirname).to eq 'source_mybranch' + expect(bn.name).to eq 'mybranch' + expect(bn.original_name).to eq 'mybranch' end it "prepends the prefix name when prefixing is overridden" do bn = described_class.new('mybranch', {:prefix => "bar", :sourcename => 'foo'}) expect(bn.dirname).to eq 'bar_mybranch' + expect(bn.name).to eq 'mybranch' + expect(bn.original_name).to eq 'mybranch' end it "uses the branch name as the dirname when prefixing is nil" do bn = described_class.new('mybranch', {:prefix => nil, :sourcename => 'foo'}) expect(bn.dirname).to eq 'mybranch' + expect(bn.name).to eq 'mybranch' + expect(bn.original_name).to eq 'mybranch' end end @@ -121,6 +165,8 @@ describe R10K::Environment::Name do it "replaces invalid characters in #{branch} with underscores" do bn = described_class.new(branch.dup, {:correct => true}) expect(bn.dirname).to eq branch.gsub(/\W/, '_') + expect(bn.name).to eq branch + expect(bn.original_name).to eq branch end end diff -pruN 3.7.0-2.1/spec/unit/environment/plain_spec.rb 3.15.4-1/spec/unit/environment/plain_spec.rb --- 3.7.0-2.1/spec/unit/environment/plain_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/unit/environment/plain_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,8 @@ +require 'spec_helper' +require 'r10k/environment' + +describe R10K::Environment::Plain do + it "initializes successfully" do + expect(described_class.new('envname', '/basedir', 'dirname', {})).to be_a_kind_of(described_class) + end +end diff -pruN 3.7.0-2.1/spec/unit/environment/svn_spec.rb 3.15.4-1/spec/unit/environment/svn_spec.rb --- 3.7.0-2.1/spec/unit/environment/svn_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/environment/svn_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -16,6 +16,18 @@ describe R10K::Environment::SVN do let(:working_dir) { subject.working_dir } + describe "initializing" do + subject do + described_class.new('name', '/dir', 'ref', { + :puppetfile_name => 'Puppetfile', + }) + end + + it "accepts valid base class initialization arguments" do + expect(subject.name).to eq 'name' + end + end + describe "storing attributes" do it "can return the environment name" do expect(subject.name).to eq 'myenv' @@ -66,9 +78,10 @@ describe R10K::Environment::SVN do describe "enumerating modules" do it "loads the Puppetfile and returns modules in that puppetfile" do - expect(subject.puppetfile).to receive(:load) - expect(subject.puppetfile).to receive(:modules).and_return [:modules] - expect(subject.modules).to eq([:modules]) + loaded = { managed_directories: [], desired_contents: [], purge_exclusions: [] } + mod = double('A module', :name => 'dbl') + expect(subject.loader).to receive(:load).and_return(loaded.merge(modules: [mod])) + expect(subject.modules).to eq([mod]) end end diff -pruN 3.7.0-2.1/spec/unit/environment/tarball_spec.rb 3.15.4-1/spec/unit/environment/tarball_spec.rb --- 3.7.0-2.1/spec/unit/environment/tarball_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/unit/environment/tarball_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,45 @@ +require 'spec_helper' +require 'r10k/environment' + +describe R10K::Environment::Tarball do + let(:tgz_path) do + File.expand_path('spec/fixtures/tarball/tarball.tar.gz', PROJECT_ROOT) + end + + let(:checksum) { '36afcfc2378b8235902d6e647fce7479da6898354d620388646c595a1155ed67' } + let(:base_params) { { source: tgz_path, version: checksum, modules: { } } } + + subject { described_class.new('envname', '/some/imaginary/path', 'dirname', base_params) } + + describe "initializing" do + it "accepts valid base class initialization arguments" do + expect(subject.name).to eq 'envname' + end + end + + describe "storing attributes" do + it "can return the environment name" do + expect(subject.name).to eq 'envname' + end + + it "can return the environment basedir" do + expect(subject.basedir).to eq '/some/imaginary/path' + end + + it "can return the environment dirname" do + expect(subject.dirname).to eq 'dirname' + end + + it "can return the environment path" do + expect(subject.path.to_s).to eq '/some/imaginary/path/dirname' + end + + it "can return the environment source" do + expect(subject.tarball.source).to eq tgz_path + end + + it "can return the environment version" do + expect(subject.tarball.checksum).to eq checksum + end + end +end diff -pruN 3.7.0-2.1/spec/unit/environment/with_modules_spec.rb 3.15.4-1/spec/unit/environment/with_modules_spec.rb --- 3.7.0-2.1/spec/unit/environment/with_modules_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/unit/environment/with_modules_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,122 @@ +require 'spec_helper' +require 'r10k/environment' + +describe R10K::Environment::WithModules do + subject do + described_class.new( + 'release42', + '/some/nonexistent/environmentdir', + 'prefix_release42', + { + :type => 'plain', + :modules => { + 'puppetlabs-stdlib' => { local: true }, + 'puppetlabs-concat' => { local: true }, + 'puppetlabs-exec' => { local: true }, + } + }.merge(subject_params) + ) + end + + # Default no additional params + let(:subject_params) { {} } + + describe "dealing with module conflicts" do + context "with no module conflicts" do + it "validates when there are no conflicts" do + mod = instance_double('R10K::Module::Base', name: 'nonconflict', origin: :puppetfile) + expect(subject.module_conflicts?(mod)).to eq false + end + end + + context "with module conflicts and default behavior" do + it "does not raise an error" do + mod = instance_double('R10K::Module::Base', name: 'stdlib', origin: :puppetfile) + expect(subject.logger).to receive(:warn).with(/Puppetfile.*both define.*ignored/i) + expect(subject.module_conflicts?(mod)).to eq true + end + end + + context "with module conflicts and 'error' behavior" do + let(:subject_params) {{ :module_conflicts => 'error' }} + it "raises an error" do + mod = instance_double('R10K::Module::Base', name: 'stdlib', origin: :puppetfile) + expect { subject.module_conflicts?(mod) }.to raise_error(R10K::Error, /Puppetfile.*both define.*/i) + end + end + + context "with module conflicts and 'override' behavior" do + let(:subject_params) {{ :module_conflicts => 'override' }} + it "does not raise an error" do + mod = instance_double('R10K::Module::Base', name: 'stdlib', origin: :puppetfile) + expect(subject.logger).to receive(:debug).with(/Puppetfile.*both define.*ignored/i) + expect(subject.module_conflicts?(mod)).to eq true + end + end + + context "with module conflicts and invalid configuration" do + let(:subject_params) {{ :module_conflicts => 'batman' }} + it "raises an error" do + mod = instance_double('R10K::Module::Base', name: 'stdlib', origin: :puppetfile) + expect { subject.module_conflicts?(mod) }.to raise_error(R10K::Error, /Unexpected value.*module_conflicts.*/i) + end + end + end + + describe "modules method" do + it "returns the configured modules, and Puppetfile modules" do + loaded = { managed_directories: [], desired_contents: [], purge_exclusions: [] } + puppetfile_mod = instance_double('R10K::Module::Base', name: 'zebra') + expect(subject.loader).to receive(:load).and_return(loaded.merge(modules: [puppetfile_mod])) + returned_modules = subject.modules + expect(returned_modules.map(&:name).sort).to eq(%w[concat exec stdlib zebra]) + end + end + + describe "module options" do + let(:subject_params) {{ + :modules => { + 'hieradata' => { + :type => 'git', + :source => 'git@git.example.com:site_data.git', + :install_path => '' + }, + 'site_data_2' => { + :type => 'git', + :source => 'git@git.example.com:site_data.git', + :install_path => 'subdir' + }, + + } + }} + + it "should support empty install_path" do + modules = subject.modules + expect(modules[0].title).to eq 'hieradata' + expect(modules[0].path).to eq Pathname.new('/some/nonexistent/environmentdir/prefix_release42/hieradata') + + end + + it "should support install_path" do + modules = subject.modules + expect(modules[1].title).to eq 'site_data_2' + expect(modules[1].path).to eq Pathname.new('/some/nonexistent/environmentdir/prefix_release42/subdir/site_data_2') + end + + context "with invalid configuration" do + let(:subject_params) {{ + :modules => { + 'site_data_2' => { + :type => 'git', + :source => 'git@git.example.com:site_data.git', + :install_path => '/absolute_path_outside_of_containing_environment' + } + } + }} + + it "raises an error" do + expect{ subject.modules }.to raise_error(R10K::Error, /Environment cannot.*outside of containing environment.*/i) + end + end + end +end diff -pruN 3.7.0-2.1/spec/unit/git/alternates_spec.rb 3.15.4-1/spec/unit/git/alternates_spec.rb --- 3.7.0-2.1/spec/unit/git/alternates_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/git/alternates_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -13,13 +13,13 @@ describe R10K::Git::Alternates do it "reads the alternates file and splits on lines" do expect(subject.file).to receive(:file?).and_return true expect(subject.file).to receive(:readlines).and_return([ - "/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git\n", - "/vagrant/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git\n", + "/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git\n", + "/vagrant/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git\n", ]) expect(subject.read).to eq([ - "/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", - "/vagrant/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", + "/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", + "/vagrant/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", ]) end @@ -33,17 +33,17 @@ describe R10K::Git::Alternates do describe "determining if an entry is already present" do before do allow(subject).to receive(:to_a).and_return([ - "/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", - "/vagrant/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", + "/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", + "/vagrant/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", ]) end it "is true if the element is in the array of read entries" do - expect(subject).to include("/vagrant/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git") + expect(subject).to include("/vagrant/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git") end it "is false if the element is not in the array of read entries" do - expect(subject).to_not include("/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git") + expect(subject).to_not include("/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git") end end @@ -52,7 +52,7 @@ describe R10K::Git::Alternates do describe "and the git objects/info directory does not exist" do it "raises an error when the parent directory does not exist" do expect { - subject.write(["/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git"]) + subject.write(["/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git"]) }.to raise_error(R10K::Git::GitError,"Cannot write /some/nonexistent/path/.git/objects/info/alternates; parent directory does not exist") end end @@ -66,51 +66,51 @@ describe R10K::Git::Alternates do end it "creates the alternates file with the new entry when not present" do - subject.write(["/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git"]) - expect(io.string).to eq("/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git\n") + subject.write(["/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git"]) + expect(io.string).to eq("/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git\n") end it "rewrites the file with all alternate entries" do - subject.write(["/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", - "/vagrant/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", - "/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git"]) + subject.write(["/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", + "/vagrant/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", + "/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git"]) expect(io.string).to eq(<<-EOD) -/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git -/vagrant/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git -/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git +/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git +/vagrant/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git +/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git EOD end end describe "appending a new alternate object entry" do it "re-writes the file with the new entry concatenated to the file" do - expect(subject).to receive(:to_a).and_return(["/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", - "/vagrant/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git"]) + expect(subject).to receive(:to_a).and_return(["/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", + "/vagrant/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git"]) - expect(subject).to receive(:write).with(["/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", - "/vagrant/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", - "/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git"]) + expect(subject).to receive(:write).with(["/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", + "/vagrant/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", + "/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git"]) - subject.add("/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git") + subject.add("/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git") end end end describe "conditionally appending a new alternate object entry" do before do - expect(subject).to receive(:read).and_return(%w[/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git]) + expect(subject).to receive(:read).and_return(%w[/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git]) end it "adds the entry and returns true when the entry doesn't exist" do - expect(subject).to receive(:write).with(["/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", - "/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git"]) - expect(subject.add?("/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git")).to eq true + expect(subject).to receive(:write).with(["/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", + "/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git"]) + expect(subject.add?("/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git")).to eq true end it "doesn't modify the file and returns false when the entry exists" do expect(subject).to_not receive(:write) - expect(subject.add?("/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git")).to eq false + expect(subject.add?("/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git")).to eq false end end end diff -pruN 3.7.0-2.1/spec/unit/git/cache_spec.rb 3.15.4-1/spec/unit/git/cache_spec.rb --- 3.7.0-2.1/spec/unit/git/cache_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/git/cache_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -21,7 +21,8 @@ describe R10K::Git::Cache do end end - subject { subclass.new('git://some/git/remote') } + let(:remote) { 'https://some/git/remote' } + subject { subclass.new(remote) } describe "updating the cache" do it "only updates the cache once" do diff -pruN 3.7.0-2.1/spec/unit/git/rugged/cache_spec.rb 3.15.4-1/spec/unit/git/rugged/cache_spec.rb --- 3.7.0-2.1/spec/unit/git/rugged/cache_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/git/rugged/cache_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -5,7 +5,7 @@ describe R10K::Git::Rugged::Cache, :unle require 'r10k/git/rugged/cache' end - subject(:cache) { described_class.new('git://some/git/remote') } + subject(:cache) { described_class.new('https://some/git/remote') } it "wraps a Rugged::BareRepository instance" do expect(cache.repo).to be_a_kind_of R10K::Git::Rugged::BareRepository @@ -26,4 +26,23 @@ describe R10K::Git::Rugged::Cache, :unle expect(described_class.settings[:cache_root]).to eq '/some/path' end end + + describe "remote url updates" do + before do + allow(subject.repo).to receive(:exist?).and_return true + allow(subject.repo).to receive(:fetch) + allow(subject.repo).to receive(:remotes).and_return({ 'origin' => 'https://some/git/remote' }) + end + + it "does not update the URLs if they match" do + expect(subject.repo).to_not receive(:update_remote) + subject.sync! + end + + it "updates the remote URL if they do not match" do + allow(subject.repo).to receive(:remotes).and_return({ 'origin' => 'foo'}) + expect(subject.repo).to receive(:update_remote) + subject.sync! + end + end end diff -pruN 3.7.0-2.1/spec/unit/git/rugged/credentials_spec.rb 3.15.4-1/spec/unit/git/rugged/credentials_spec.rb --- 3.7.0-2.1/spec/unit/git/rugged/credentials_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/git/rugged/credentials_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -10,7 +10,7 @@ describe R10K::Git::Rugged::Credentials, subject { described_class.new(repo) } - after(:all) { R10K::Git.settings.reset! } + after(:each) { R10K::Git.settings.reset! } describe "determining the username" do before { R10K::Git.settings[:username] = "moderns" } @@ -39,6 +39,7 @@ describe R10K::Git::Rugged::Credentials, it "prefers a per-repository SSH private key" do allow(File).to receive(:readable?).with("/etc/puppetlabs/r10k/ssh/tessier-ashpool-id_rsa").and_return true + R10K::Git.settings[:private_key] = "/etc/puppetlabs/r10k/ssh/id_rsa" R10K::Git.settings[:repositories] = [{ remote: "ssh://git@tessier-ashpool.freeside/repo.git", private_key: "/etc/puppetlabs/r10k/ssh/tessier-ashpool-id_rsa"}] creds = subject.get_ssh_key_credentials("ssh://git@tessier-ashpool.freeside/repo.git", nil) @@ -78,6 +79,111 @@ describe R10K::Git::Rugged::Credentials, end end + describe "generating github app tokens" do + it 'errors if app id has invalid characters' do + expect { subject.github_app_token("123A567890", "fake", "300") + }.to raise_error(R10K::Git::GitError, /App id contains invalid characters/) + end + it 'errors if app ttl has invalid characters' do + expect { subject.github_app_token("123456", "fake", "abc") + }.to raise_error(R10K::Git::GitError, /Github App token ttl contains/) + end + it 'errors if private file does not exist' do + R10K::Git.settings[:github_app_key] = "/missing/token/file" + expect(File).to receive(:readable?).with(R10K::Git.settings[:github_app_key]).and_return false + expect { + subject.github_app_token("123456", R10K::Git.settings[:github_app_key], "300") + }.to raise_error(R10K::Git::GitError, /App key is missing or unreadable/) + end + it 'errors if file is not a valid SSL key' do + token_file = Tempfile.new('token') + token_file.write('my_token') + token_file.close + R10K::Git.settings[:github_app_key] = token_file.path + expect(File).to receive(:readable?).with(token_file.path).and_return true + expect { + subject.github_app_token("123456", R10K::Git.settings[:github_app_key], "300") + }.to raise_error(R10K::Git::GitError, /App key is not a valid SSL key/) + token_file.unlink + end + end + + describe "generating token credentials" do + it 'errors if token file does not exist' do + R10K::Git.settings[:oauth_token] = "/missing/token/file" + expect(File).to receive(:readable?).with("/missing/token/file").and_return false + R10K::Git.settings[:repositories] = [{remote: "https://tessier-ashpool.freeside/repo.git"}] + expect { + subject.get_plaintext_credentials("https://tessier-ashpool.freeside/repo.git", nil) + }.to raise_error(R10K::Git::GitError, /cannot load OAuth token/) + end + + it 'errors if the token on stdin is not a valid OAuth token' do + allow($stdin).to receive(:read).and_return("token") + R10K::Git.settings[:oauth_token] = "-" + R10K::Git.settings[:repositories] = [{remote: "https://tessier-ashpool.freeside/repo.git"}] + expect { + subject.get_plaintext_credentials("https://tessier-ashpool.freeside/repo.git", nil) + }.to raise_error(R10K::Git::GitError, /invalid characters/) + end + + it 'errors if the token in the file is not a valid OAuth token' do + token_file = Tempfile.new('token') + token_file.write('my bad \ntoken') + token_file.close + R10K::Git.settings[:oauth_token] = token_file.path + R10K::Git.settings[:repositories] = [{remote: "https://tessier-ashpool.freeside/repo.git"}] + expect { + subject.get_plaintext_credentials("https://tessier-ashpool.freeside/repo.git", nil) + }.to raise_error(R10K::Git::GitError, /invalid characters/) + end + + it 'prefers per-repo token file' do + token_file = Tempfile.new('token') + token_file.write('my_token') + token_file.close + R10K::Git.settings[:oauth_token] = "/do/not/use" + R10K::Git.settings[:repositories] = [{remote: "https://tessier-ashpool.freeside/repo.git", + oauth_token: token_file.path }] + creds = subject.get_plaintext_credentials("https://tessier-ashpool.freeside/repo.git", nil) + expect(creds).to be_a_kind_of(Rugged::Credentials::UserPassword) + expect(creds.instance_variable_get(:@password)).to eq("my_token") + expect(creds.instance_variable_get(:@username)).to eq("x-oauth-token") + end + + it 'uses the token from a file as a password' do + token_file = Tempfile.new('token') + token_file.write('my_token') + token_file.close + R10K::Git.settings[:oauth_token] = token_file.path + R10K::Git.settings[:repositories] = [{remote: "https://tessier-ashpool.freeside/repo.git"}] + creds = subject.get_plaintext_credentials("https://tessier-ashpool.freeside/repo.git", nil) + expect(creds).to be_a_kind_of(Rugged::Credentials::UserPassword) + expect(creds.instance_variable_get(:@password)).to eq("my_token") + expect(creds.instance_variable_get(:@username)).to eq("x-oauth-token") + end + + it 'uses the token from stdin as a password' do + allow($stdin).to receive(:read).and_return("my_token") + R10K::Git.settings[:oauth_token] = '-' + R10K::Git.settings[:repositories] = [{remote: "https://tessier-ashpool.freeside/repo.git"}] + creds = subject.get_plaintext_credentials("https://tessier-ashpool.freeside/repo.git", nil) + expect(creds).to be_a_kind_of(Rugged::Credentials::UserPassword) + expect(creds.instance_variable_get(:@password)).to eq("my_token") + expect(creds.instance_variable_get(:@username)).to eq("x-oauth-token") + end + + it 'only reads the token in once' do + expect($stdin).to receive(:read).and_return("my_token").once + R10K::Git.settings[:oauth_token] = '-' + R10K::Git.settings[:repositories] = [{remote: "https://tessier-ashpool.freeside/repo.git"}] + creds = subject.get_plaintext_credentials("https://tessier-ashpool.freeside/repo.git", nil) + expect(creds.instance_variable_get(:@password)).to eq("my_token") + creds = subject.get_plaintext_credentials("https://tessier-ashpool.freeside/repo.git", nil) + expect(creds.instance_variable_get(:@password)).to eq("my_token") + end + end + describe "generating default credentials" do it "generates the rugged default credential type" do creds = subject.get_default_credentials("https://azurediamond:hunter2@tessier-ashpool.freeside/repo.git", "azurediamond") diff -pruN 3.7.0-2.1/spec/unit/git/shellgit/cache_spec.rb 3.15.4-1/spec/unit/git/shellgit/cache_spec.rb --- 3.7.0-2.1/spec/unit/git/shellgit/cache_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/git/shellgit/cache_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -3,7 +3,7 @@ require 'r10k/git/shellgit/cache' describe R10K::Git::ShellGit::Cache do - subject { described_class.new('git://some/git/remote') } + subject { described_class.new('https://some/git/remote') } it "wraps a ShellGit::BareRepository instance" do expect(subject.repo).to be_a_kind_of R10K::Git::ShellGit::BareRepository diff -pruN 3.7.0-2.1/spec/unit/git/stateful_repository_spec.rb 3.15.4-1/spec/unit/git/stateful_repository_spec.rb --- 3.7.0-2.1/spec/unit/git/stateful_repository_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/git/stateful_repository_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -4,7 +4,7 @@ require 'r10k/git/stateful_repository' describe R10K::Git::StatefulRepository do - let(:remote) { 'git://some.site/some-repo.git' } + let(:remote) { 'https://some.site/some-repo.git' } let(:ref) { '0.9.x' } subject { described_class.new(remote, '/some/nonexistent/basedir', 'some-dirname') } @@ -19,6 +19,11 @@ describe R10K::Git::StatefulRepository d expect(subject.sync_cache?(ref)).to eq true end + it "is true if the ref is HEAD" do + expect(cache).to receive(:exist?).and_return true + expect(subject.sync_cache?('HEAD')).to eq true + end + it "is true if the ref is unresolvable" do expect(cache).to receive(:exist?).and_return true expect(cache).to receive(:ref_type).with('0.9.x').and_return(:unknown) diff -pruN 3.7.0-2.1/spec/unit/module/base_spec.rb 3.15.4-1/spec/unit/module/base_spec.rb --- 3.7.0-2.1/spec/unit/module/base_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/module/base_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -4,39 +4,85 @@ require 'r10k/module/base' describe R10K::Module::Base do describe "parsing the title" do it "parses titles with no owner" do - m = described_class.new('eight_hundred', '/moduledir', []) + m = described_class.new('eight_hundred', '/moduledir', {}) expect(m.name).to eq 'eight_hundred' expect(m.owner).to be_nil end it "parses forward slash separated titles" do - m = described_class.new('branan/eight_hundred', '/moduledir', []) + m = described_class.new('branan/eight_hundred', '/moduledir', {}) expect(m.name).to eq 'eight_hundred' expect(m.owner).to eq 'branan' end it "parses hyphen separated titles" do - m = described_class.new('branan-eight_hundred', '/moduledir', []) + m = described_class.new('branan-eight_hundred', '/moduledir', {}) expect(m.name).to eq 'eight_hundred' expect(m.owner).to eq 'branan' end it "raises an error when the title is not correctly formatted" do expect { - described_class.new('branan!eight_hundred', '/moduledir', []) + described_class.new('branan!eight_hundred', '/moduledir', {}) }.to raise_error(ArgumentError, "Module name (branan!eight_hundred) must match either 'modulename' or 'owner/modulename'") end end + describe 'deleting the spec dir' do + let(:module_org) { "coolorg" } + let(:module_name) { "coolmod" } + let(:title) { "#{module_org}-#{module_name}" } + let(:dirname) { Pathname.new(Dir.mktmpdir) } + let(:spec_path) { dirname + module_name + 'spec' } + + before(:each) do + logger = double("logger") + allow_any_instance_of(described_class).to receive(:logger).and_return(logger) + allow(logger).to receive(:debug2).with(any_args) + allow(logger).to receive(:info).with(any_args) + end + + it 'does not remove the spec directory by default' do + FileUtils.mkdir_p(spec_path) + m = described_class.new(title, dirname, {}) + m.maybe_delete_spec_dir + expect(Dir.exist?(spec_path)).to eq true + end + + it 'detects a symlink and deletes the target' do + Dir.mkdir(dirname + module_name) + target_dir = Dir.mktmpdir + FileUtils.ln_s(target_dir, spec_path) + m = described_class.new(title, dirname, {exclude_spec: true}) + m.maybe_delete_spec_dir + expect(Dir.exist?(target_dir)).to eq false + end + + it 'removes the spec directory if exclude_spec is set' do + FileUtils.mkdir_p(spec_path) + m = described_class.new(title, dirname, {exclude_spec: true}) + m.maybe_delete_spec_dir + expect(Dir.exist?(spec_path)).to eq false + end + + it 'does not remove the spec directory if spec_deletable is false' do + FileUtils.mkdir_p(spec_path) + m = described_class.new(title, dirname, {}) + m.spec_deletable = false + m.maybe_delete_spec_dir + expect(Dir.exist?(spec_path)).to eq true + end + end + describe "path variables" do it "uses the module name as the name" do - m = described_class.new('eight_hundred', '/moduledir', []) + m = described_class.new('eight_hundred', '/moduledir', {}) expect(m.dirname).to eq '/moduledir' expect(m.path).to eq(Pathname.new('/moduledir/eight_hundred')) end it "does not include the owner in the path" do - m = described_class.new('branan/eight_hundred', '/moduledir', []) + m = described_class.new('branan/eight_hundred', '/moduledir', {}) expect(m.dirname).to eq '/moduledir' expect(m.path).to eq(Pathname.new('/moduledir/eight_hundred')) end @@ -44,7 +90,7 @@ describe R10K::Module::Base do describe "with alternate variable names" do subject do - described_class.new('branan/eight_hundred', '/moduledir', []) + described_class.new('branan/eight_hundred', '/moduledir', {}) end it "aliases full_name to title" do @@ -61,7 +107,7 @@ describe R10K::Module::Base do end describe "accepting a visitor" do - subject { described_class.new('branan-eight_hundred', '/moduledir', []) } + subject { described_class.new('branan-eight_hundred', '/moduledir', {}) } it "passes itself to the visitor" do visitor = spy('visitor') diff -pruN 3.7.0-2.1/spec/unit/module/forge_spec.rb 3.15.4-1/spec/unit/module/forge_spec.rb --- 3.7.0-2.1/spec/unit/module/forge_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/module/forge_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -9,22 +9,46 @@ describe R10K::Module::Forge do let(:fixture_modulepath) { File.expand_path('spec/fixtures/module/forge', PROJECT_ROOT) } let(:empty_modulepath) { File.expand_path('spec/fixtures/empty', PROJECT_ROOT) } + describe "statically determined version support" do + it 'returns explicitly released forge versions' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { version: '8.0.0' }) + expect(static_version).to eq('8.0.0') + end + + it 'returns explicit pre-released forge versions' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { version: '8.0.0-pre1' }) + expect(static_version).to eq('8.0.0-pre1') + end + + it 'retuns nil for latest versions' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { version: :latest }) + expect(static_version).to eq(nil) + end + + it 'retuns nil for undefined versions' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { version: nil }) + expect(static_version).to eq(nil) + end + end + describe "implementing the Puppetfile spec" do it "should implement 'branan/eight_hundred', '8.0.0'" do - expect(described_class).to be_implement('branan/eight_hundred', '8.0.0') + expect(described_class).to be_implement('branan/eight_hundred', { type: 'forge', version: '8.0.0' }) end it "should implement 'branan-eight_hundred', '8.0.0'" do - expect(described_class).to be_implement('branan-eight_hundred', '8.0.0') + expect(described_class).to be_implement('branan-eight_hundred', { type: 'forge', version: '8.0.0' }) end + end - it "should fail with an invalid title" do - expect(described_class).to_not be_implement('branan!eight_hundred', '8.0.0') + describe "implementing the standard options interface" do + it "should implement {type: forge}" do + expect(described_class).to be_implement('branan-eight_hundred', { type: 'forge', version: '8.0.0', source: 'not implemented' }) end end describe "setting attributes" do - subject { described_class.new('branan/eight_hundred', '/moduledir', '8.0.0') } + subject { described_class.new('branan/eight_hundred', '/moduledir', { version: '8.0.0' }) } it "sets the name" do expect(subject.name).to eq 'eight_hundred' @@ -43,8 +67,14 @@ describe R10K::Module::Forge do end end + describe "invalid attributes" do + it "errors on invalid versions" do + expect { described_class.new('branan/eight_hundred', '/moduledir', { version: '_8.0.0_' }) }.to raise_error ArgumentError, /version/ + end + end + describe "properties" do - subject { described_class.new('branan/eight_hundred', fixture_modulepath, '8.0.0') } + subject { described_class.new('branan/eight_hundred', fixture_modulepath, { version: '8.0.0' }) } it "sets the module type to :forge" do expect(subject.properties).to include(:type => :forge) @@ -61,7 +91,7 @@ describe R10K::Module::Forge do end context "when a module is deprecated" do - subject { described_class.new('puppetlabs/corosync', fixture_modulepath, :latest) } + subject { described_class.new('puppetlabs/corosync', fixture_modulepath, { version: :latest }) } it "warns on sync if module is not already insync" do allow(subject).to receive(:status).and_return(:absent) @@ -71,6 +101,8 @@ describe R10K::Module::Forge do logger_dbl = double(Log4r::Logger) allow_any_instance_of(described_class).to receive(:logger).and_return(logger_dbl) + allow(logger_dbl).to receive(:info).with(/Deploying module to.*/) + allow(logger_dbl).to receive(:debug2).with(/No spec dir detected/) expect(logger_dbl).to receive(:warn).with(/puppet forge module.*puppetlabs-corosync.*has been deprecated/i) subject.sync @@ -82,6 +114,8 @@ describe R10K::Module::Forge do logger_dbl = double(Log4r::Logger) allow_any_instance_of(described_class).to receive(:logger).and_return(logger_dbl) + allow(logger_dbl).to receive(:info).with(/Deploying module to.*/) + allow(logger_dbl).to receive(:debug2).with(/No spec dir detected/) expect(logger_dbl).to_not receive(:warn).with(/puppet forge module.*puppetlabs-corosync.*has been deprecated/i) subject.sync @@ -90,20 +124,27 @@ describe R10K::Module::Forge do describe '#expected_version' do it "returns an explicitly given expected version" do - subject = described_class.new('branan/eight_hundred', fixture_modulepath, '8.0.0') + subject = described_class.new('branan/eight_hundred', fixture_modulepath, { version: '8.0.0' }) expect(subject.expected_version).to eq '8.0.0' end it "uses the latest version from the forge when the version is :latest" do - subject = described_class.new('branan/eight_hundred', fixture_modulepath, :latest) - expect(subject.v3_module).to receive_message_chain(:current_release, :version).and_return('8.8.8') + subject = described_class.new('branan/eight_hundred', fixture_modulepath, { version: :latest }) + release = double("Module Release", version: '8.8.8') + expect(subject.v3_module).to receive(:current_release).and_return(release).twice expect(subject.expected_version).to eq '8.8.8' end + + it "throws when there are no available versions" do + subject = described_class.new('branan/eight_hundred', fixture_modulepath, { version: :latest }) + expect(subject.v3_module).to receive(:current_release).and_return(nil) + expect { subject.expected_version }.to raise_error(PuppetForge::ReleaseNotFound) + end end describe "determining the status" do - subject { described_class.new('branan/eight_hundred', fixture_modulepath, '8.0.0') } + subject { described_class.new('branan/eight_hundred', fixture_modulepath, { version: '8.0.0' }) } it "is :absent if the module directory is absent" do allow(subject).to receive(:exist?).and_return false @@ -148,7 +189,24 @@ describe R10K::Module::Forge do end describe "#sync" do - subject { described_class.new('branan/eight_hundred', fixture_modulepath, '8.0.0') } + subject { described_class.new('branan/eight_hundred', fixture_modulepath, { version: '8.0.0' }) } + + context "syncing the repo" do + let(:module_org) { "coolorg" } + let(:module_name) { "coolmod" } + let(:title) { "#{module_org}-#{module_name}" } + let(:dirname) { Pathname.new(Dir.mktmpdir) } + let(:spec_path) { dirname + module_name + 'spec' } + subject { described_class.new(title, dirname, {}) } + + it 'defaults to keeping the spec dir' do + FileUtils.mkdir_p(spec_path) + expect(subject).to receive(:status).and_return(:absent) + expect(subject).to receive(:install) + subject.sync + expect(Dir.exist?(spec_path)).to eq true + end + end it 'does nothing when the module is in sync' do allow(subject).to receive(:status).and_return :insync @@ -156,31 +214,37 @@ describe R10K::Module::Forge do expect(subject).to receive(:install).never expect(subject).to receive(:upgrade).never expect(subject).to receive(:reinstall).never - subject.sync + expect(subject.sync).to be false end it 'reinstalls the module when it is mismatched' do allow(subject).to receive(:status).and_return :mismatched expect(subject).to receive(:reinstall) - subject.sync + expect(subject.sync).to be true end it 'upgrades the module when it is outdated' do allow(subject).to receive(:status).and_return :outdated expect(subject).to receive(:upgrade) - subject.sync + expect(subject.sync).to be true end it 'installs the module when it is absent' do allow(subject).to receive(:status).and_return :absent expect(subject).to receive(:install) - subject.sync + expect(subject.sync).to be true + end + + it 'returns false if `should_sync?` is false' do + # modules do not sync if they are not requested + mod = described_class.new('my_org/my_mod', '/path/to/mod', { overrides: { modules: { requested_modules: ['other_mod'] } } }) + expect(mod.sync).to be false end end describe '#install' do it 'installs the module from the forge' do - subject = described_class.new('branan/eight_hundred', fixture_modulepath, '8.0.0') + subject = described_class.new('branan/eight_hundred', fixture_modulepath, { version: '8.0.0' }) release = instance_double('R10K::Forge::ModuleRelease') expect(R10K::Forge::ModuleRelease).to receive(:new).with('branan-eight_hundred', '8.0.0').and_return(release) expect(release).to receive(:install).with(subject.path) @@ -190,7 +254,7 @@ describe R10K::Module::Forge do describe '#uninstall' do it 'removes the module path' do - subject = described_class.new('branan/eight_hundred', fixture_modulepath, '8.0.0') + subject = described_class.new('branan/eight_hundred', fixture_modulepath, { version: '8.0.0' }) expect(FileUtils).to receive(:rm_rf).with(subject.path.to_s) subject.uninstall end @@ -198,7 +262,7 @@ describe R10K::Module::Forge do describe '#reinstall' do it 'uninstalls and then installs the module' do - subject = described_class.new('branan/eight_hundred', fixture_modulepath, '8.0.0') + subject = described_class.new('branan/eight_hundred', fixture_modulepath, { version: '8.0.0' }) expect(subject).to receive(:uninstall) expect(subject).to receive(:install) subject.reinstall diff -pruN 3.7.0-2.1/spec/unit/module/git_spec.rb 3.15.4-1/spec/unit/module/git_spec.rb --- 3.7.0-2.1/spec/unit/module/git_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/module/git_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -10,15 +10,41 @@ describe R10K::Module::Git do allow(R10K::Git::StatefulRepository).to receive(:new).and_return(mock_repo) end + + describe "statically determined version support" do + it 'returns a given commit' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { git: 'my/remote', commit: '123adf' }) + expect(static_version).to eq('123adf') + end + + it 'returns a given tag' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { git: 'my/remote', tag: 'v1.2.3' }) + expect(static_version).to eq('v1.2.3') + end + + it 'returns a ref if it looks like a full commit sha' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { git: 'my/remote', ref: '1234567890abcdef1234567890abcdef12345678' }) + expect(static_version).to eq('1234567890abcdef1234567890abcdef12345678') + end + + it 'returns nil for any non-sha-like ref' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { git: 'my/remote', ref: 'refs/heads/main' }) + expect(static_version).to eq(nil) + end + + it 'returns nil for branches' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { git: 'my/remote', branch: 'main' }) + expect(static_version).to eq(nil) + end + end + describe "setting the owner and name" do describe "with a title of 'branan/eight_hundred'" do subject do described_class.new( 'branan/eight_hundred', '/moduledir', - { - :git => 'git://git-server.site/branan/puppet-eight_hundred', - } + { :git => 'https://git-server.site/branan/puppet-eight_hundred' } ) end @@ -40,9 +66,7 @@ describe R10K::Module::Git do described_class.new( 'eight_hundred', '/moduledir', - { - :git => 'git://git-server.site/branan/puppet-eight_hundred', - } + { :git => 'https://git-server.site/branan/puppet-eight_hundred' } ) end @@ -62,7 +86,7 @@ describe R10K::Module::Git do describe "properties" do subject do - described_class.new('boolean', '/moduledir', {:git => 'git://git.example.com/adrienthebo/puppet-boolean'}) + described_class.new('boolean', '/moduledir', {:git => 'https://git.example.com/adrienthebo/puppet-boolean'}) end before(:each) do @@ -89,14 +113,48 @@ describe R10K::Module::Git do end end + describe 'syncing the repo' do + let(:module_org) { "coolorg" } + let(:module_name) { "coolmod" } + let(:title) { "#{module_org}-#{module_name}" } + let(:dirname) { Pathname.new(Dir.mktmpdir) } + let(:spec_path) { dirname + module_name + 'spec' } + subject { described_class.new(title, dirname, {}) } + + before(:each) do + allow(mock_repo).to receive(:resolve).with('master').and_return('abc123') + end + + it 'defaults to keeping the spec dir' do + FileUtils.mkdir_p(spec_path) + allow(mock_repo).to receive(:sync) + subject.sync + expect(Dir.exist?(spec_path)).to eq true + end + + it 'returns true if repo was updated' do + expect(mock_repo).to receive(:sync).and_return(true) + expect(subject.sync).to be true + end + + it 'returns false if repo was not updated (in-sync)' do + expect(mock_repo).to receive(:sync).and_return(false) + expect(subject.sync).to be false + end + + it 'returns false if `should_sync?` is false' do + # modules do not sync if they are not requested + mod = described_class.new(title, dirname, { overrides: { modules: { requested_modules: ['other_mod'] } } }) + expect(mod.sync).to be false + end + end + describe "determining the status" do subject do described_class.new( 'boolean', '/moduledir', - { - :git => 'git://git.example.com/adrienthebo/puppet-boolean' - } + { :git => 'https://git.example.com/adrienthebo/puppet-boolean' } ) end @@ -113,20 +171,12 @@ describe R10K::Module::Git do described_class.new('boolean', '/moduledir', base_opts.merge(extra_opts), env) end - let(:base_opts) { { git: 'git://git.example.com/adrienthebo/puppet-boolean' } } + let(:base_opts) { { git: 'https://git.example.com/adrienthebo/puppet-boolean' } } before(:each) do allow(mock_repo).to receive(:head).and_return('abc123') end - context "when option is unrecognized" do - let(:opts) { { unrecognized: true } } - - it "raises an error" do - expect { test_module(opts) }.to raise_error(ArgumentError, /unhandled options.*unrecognized/i) - end - end - describe "desired ref" do context "when no desired ref is given" do it "defaults to master" do @@ -210,6 +260,14 @@ describe R10K::Module::Git do expect(mod.desired_ref).to eq(:control_branch) end + it "warns control branch may be unresolvable" do + logger = double("logger") + allow_any_instance_of(described_class).to receive(:logger).and_return(logger) + expect(logger).to receive(:warn).with(/Cannot track control repo branch.*boolean.*/) + + test_module(branch: :control_branch) + end + context "when default ref is provided and resolvable" do it "uses default ref" do expect(mock_repo).to receive(:resolve).with('default').and_return('abc123') @@ -268,6 +326,61 @@ describe R10K::Module::Git do end end end + + context "when using default_branch_override" do + before(:each) do + allow(mock_repo).to receive(:resolve).with(mock_env.ref).and_return(nil) + end + + context "and the default branch override is resolvable" do + it "uses the override" do + expect(mock_repo).to receive(:resolve).with('default_override').and_return('5566aabb') + mod = test_module({branch: :control_branch, + default_branch: 'default', + default_branch_override: 'default_override'}, + mock_env) + expect(mod.properties).to include(expected: 'default_override') + end + end + + context "and the default branch override is not resolvable" do + context "and default branch is provided" do + it "falls back to the default" do + expect(mock_repo).to receive(:resolve).with('default_override').and_return(nil) + expect(mock_repo).to receive(:resolve).with('default').and_return('5566aabb') + mod = test_module({branch: :control_branch, + default_branch: 'default', + default_branch_override: 'default_override'}, + mock_env) + expect(mod.properties).to include(expected: 'default') + end + end + + context "and default branch is not provided" do + it "raises the appropriate error" do + expect(mock_repo).to receive(:resolve).with('default_override').and_return(nil) + mod = test_module({branch: :control_branch, + default_branch_override: 'default_override'}, + mock_env) + + expect { mod.properties }.to raise_error(ArgumentError, /unable to manage.*or resolve the default branch override.*no default provided/i) + end + end + + context "and default branch is not resolvable" do + it "raises the appropriate error" do + expect(mock_repo).to receive(:resolve).with('default_override').and_return(nil) + expect(mock_repo).to receive(:resolve).with('default').and_return(nil) + mod = test_module({branch: :control_branch, + default_branch: 'default', + default_branch_override: 'default_override'}, + mock_env) + + expect { mod.properties }.to raise_error(ArgumentError, /unable to manage.*or resolve the default branch override.*or resolve default/i) + end + end + end + end end end end diff -pruN 3.7.0-2.1/spec/unit/module/svn_spec.rb 3.15.4-1/spec/unit/module/svn_spec.rb --- 3.7.0-2.1/spec/unit/module/svn_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/module/svn_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -6,6 +6,13 @@ describe R10K::Module::SVN do include_context 'fail on execution' + describe "statically determined version support" do + it 'is unsupported by svn backed modules' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { svn: 'my/remote', revision: '123adf' }) + expect(static_version).to eq(nil) + end + end + describe "determining it implements a Puppetfile mod" do it "implements mods with the :svn hash key" do implements = described_class.implement?('r10k-fixture-repo', :svn => 'https://github.com/adrienthebo/r10k-fixture-repo') @@ -119,6 +126,23 @@ describe R10K::Module::SVN do end end + describe 'the default spec dir' do + let(:module_org) { "coolorg" } + let(:module_name) { "coolmod" } + let(:title) { "#{module_org}-#{module_name}" } + let(:dirname) { Pathname.new(Dir.mktmpdir) } + let(:spec_path) { dirname + module_name + 'spec' } + subject { described_class.new(title, dirname, {}) } + + it 'is kept by default' do + FileUtils.mkdir_p(spec_path) + expect(subject).to receive(:status).and_return(:absent) + expect(subject).to receive(:install).and_return(nil) + subject.sync + expect(Dir.exist?(spec_path)).to eq true + end + end + describe "synchronizing" do subject { described_class.new('foo', '/moduledir', :svn => 'https://github.com/adrienthebo/r10k-fixture-repo', :rev => 123) } @@ -132,7 +156,7 @@ describe R10K::Module::SVN do it "installs the SVN module" do expect(subject).to receive(:install) - subject.sync + expect(subject.sync).to be true end end @@ -142,14 +166,14 @@ describe R10K::Module::SVN do it "reinstalls the module" do expect(subject).to receive(:reinstall) - subject.sync + expect(subject.sync).to be true end it "removes the existing directory" do expect(subject.path).to receive(:rmtree) allow(subject).to receive(:install) - subject.sync + expect(subject.sync).to be true end end @@ -159,7 +183,7 @@ describe R10K::Module::SVN do it "upgrades the repository" do expect(subject).to receive(:update) - subject.sync + expect(subject.sync).to be true end end @@ -171,8 +195,14 @@ describe R10K::Module::SVN do expect(subject).to receive(:reinstall).never expect(subject).to receive(:update).never - subject.sync + expect(subject.sync).to be false end end + + it 'and `should_sync?` is false' do + # modules do not sync if they are not requested + mod = described_class.new('my_mod', '/path/to/mod', { overrides: { modules: { requested_modules: ['other_mod'] } } }) + expect(mod.sync).to be false + end end end diff -pruN 3.7.0-2.1/spec/unit/module/tarball_spec.rb 3.15.4-1/spec/unit/module/tarball_spec.rb --- 3.7.0-2.1/spec/unit/module/tarball_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/unit/module/tarball_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,70 @@ +require 'spec_helper' +require 'r10k/module' +require 'fileutils' + +describe R10K::Module::Tarball do + include_context 'Tarball' + + let(:base_params) { { type: 'tarball', source: fixture_tarball, version: fixture_checksum } } + + subject do + described_class.new( + 'fixture-tarball', + moduledir, + base_params, + ) + end + + describe "setting the owner and name" do + describe "with a title of 'fixture-tarball'" do + it "sets the owner to 'fixture'" do + expect(subject.owner).to eq 'fixture' + end + + it "sets the name to 'tarball'" do + expect(subject.name).to eq 'tarball' + end + + it "sets the path to the given moduledir + modname" do + expect(subject.path.to_s).to eq(File.join(moduledir, 'tarball')) + end + end + end + + describe "properties" do + it "sets the module type to :tarball" do + expect(subject.properties).to include(type: :tarball) + end + + it "sets the version" do + expect(subject.properties).to include(expected: fixture_checksum) + end + end + + describe 'syncing the module' do + it 'defaults to keeping the spec dir' do + subject.sync + expect(Dir.exist?(File.join(moduledir, 'tarball', 'spec'))).to be(true) + end + end + + describe "determining the status" do + it "delegates to R10K::Tarball" do + expect(subject).to receive(:tarball).twice.and_return instance_double('R10K::Tarball', cache_valid?: true, insync?: true) + expect(subject).to receive(:path).twice.and_return instance_double('Pathname', exist?: true) + + expect(subject.status).to eq(:insync) + end + end + + describe "option parsing" do + describe "version" do + context "when no version is given" do + subject { described_class.new('fixture-tarball', moduledir, base_params.reject { |k| k.eql?(:version) }) } + it "does not require a version" do + expect(subject).to be_kind_of(described_class) + end + end + end + end +end diff -pruN 3.7.0-2.1/spec/unit/module_loader/puppetfile_spec.rb 3.15.4-1/spec/unit/module_loader/puppetfile_spec.rb --- 3.7.0-2.1/spec/unit/module_loader/puppetfile_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/unit/module_loader/puppetfile_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,439 @@ +require 'spec_helper' +require 'r10k/module_loader/puppetfile' +require 'tmpdir' + +describe R10K::ModuleLoader::Puppetfile do + describe 'initial parameters' do + describe 'honor' do + let(:options) do + { + basedir: '/test/basedir/env', + overrides: { modules: { deploy_modules: true } }, + environment: R10K::Environment::Git.new('env', + '/test/basedir/', + 'env', + { remote: 'https://foo/remote', + ref: 'env' }) + } + end + + subject { R10K::ModuleLoader::Puppetfile.new(**options) } + + describe 'the moduledir' do + it 'respects absolute paths' do + absolute_options = options.merge({moduledir: '/opt/puppetlabs/special/modules'}) + puppetfile = R10K::ModuleLoader::Puppetfile.new(**absolute_options) + expect(puppetfile.instance_variable_get(:@moduledir)).to eq('/opt/puppetlabs/special/modules') + end + + it 'roots the moduledir in the basepath if a relative path is specified' do + relative_options = options.merge({moduledir: 'my/special/modules'}) + puppetfile = R10K::ModuleLoader::Puppetfile.new(**relative_options) + expect(puppetfile.instance_variable_get(:@moduledir)).to eq('/test/basedir/env/my/special/modules') + end + end + + describe 'the Puppetfile' do + it 'respects absolute paths' do + absolute_options = options.merge({puppetfile: '/opt/puppetlabs/special/Puppetfile'}) + puppetfile = R10K::ModuleLoader::Puppetfile.new(**absolute_options) + expect(puppetfile.instance_variable_get(:@puppetfile_path)).to eq('/opt/puppetlabs/special/Puppetfile') + end + + it 'roots the Puppetfile in the basepath if a relative path is specified' do + relative_options = options.merge({puppetfile: 'Puppetfile.global'}) + puppetfile = R10K::ModuleLoader::Puppetfile.new(**relative_options) + expect(puppetfile.instance_variable_get(:@puppetfile_path)).to eq('/test/basedir/env/Puppetfile.global') + end + end + + it 'the overrides' do + expect(subject.instance_variable_get(:@overrides)).to eq({ modules: { deploy_modules: true }}) + end + + it 'the environment' do + expect(subject.instance_variable_get(:@environment).name).to eq('env') + end + end + + describe 'sane defaults' do + subject { R10K::ModuleLoader::Puppetfile.new(basedir: '/test/basedir') } + + it 'has a moduledir rooted in the basedir' do + expect(subject.instance_variable_get(:@moduledir)).to eq('/test/basedir/modules') + end + + it 'has a Puppetfile rooted in the basedir' do + expect(subject.instance_variable_get(:@puppetfile_path)).to eq('/test/basedir/Puppetfile') + end + + it 'creates an empty overrides' do + expect(subject.instance_variable_get(:@overrides)).to eq({}) + end + + it 'does not require an environment' do + expect(subject.instance_variable_get(:@environment)).to eq(nil) + end + end + end + + describe 'adding modules' do + let(:basedir) { '/test/basedir' } + + subject { R10K::ModuleLoader::Puppetfile.new(basedir: basedir, + overrides: {modules: {exclude_spec: true}}) } + + it 'should transform Forge modules with a string arg to have a version key' do + expect(R10K::Module).to receive(:from_metadata).with('puppet/test_module', subject.moduledir, hash_including(version: '1.2.3'), anything).and_call_original + + expect { subject.add_module('puppet/test_module', '1.2.3') }.to change { subject.modules } + expect(subject.modules.collect(&:name)).to include('test_module') + end + + it 'should not accept Forge modules with a version comparison' do + expect(R10K::Module).to receive(:from_metadata).with('puppet/test_module', subject.moduledir, hash_including(version: '< 1.2.0'), anything).and_call_original + + expect { + subject.add_module('puppet/test_module', '< 1.2.0') + }.to raise_error(ArgumentError, /module version .* is not a valid forge module version/i) + + expect(subject.modules.collect(&:name)).not_to include('test_module') + end + + it 'should not modify the overrides when adding modules' do + module_opts = { git: 'git@example.com:puppet/test_module.git' } + subject.add_module('puppet/test_module', module_opts) + expect(subject.instance_variable_get("@overrides")[:modules]).to eq({exclude_spec: true}) + end + + it 'should read the `exclude_spec` setting in the module definition and override the overrides' do + module_opts = { git: 'git@example.com:puppet/test_module.git', exclude_spec: false } + subject.add_module('puppet/test_module', module_opts) + expect(subject.modules[0].instance_variable_get("@exclude_spec")).to be false + end + + it 'should set :spec_deletable to true for modules in the basedir' do + module_opts = { git: 'git@example.com:puppet/test_module.git' } + subject.add_module('puppet/test_module', module_opts) + expect(subject.modules[0].spec_deletable).to be true + end + + it 'should set :spec_deletable to false for modules outside the basedir' do + module_opts = { git: 'git@example.com:puppet/test_module.git', install_path: 'some/path' } + subject.add_module('puppet/test_module', module_opts) + expect(subject.modules[0].spec_deletable).to be false + end + + it 'should accept non-Forge modules with a hash arg' do + module_opts = { git: 'git@example.com:puppet/test_module.git' } + + expect(R10K::Module).to receive(:from_metadata).with('puppet/test_module', subject.moduledir, module_opts, anything).and_call_original + + expect { subject.add_module('puppet/test_module', module_opts) }.to change { subject.modules } + expect(subject.modules.collect(&:name)).to include('test_module') + end + + it 'should accept non-Forge modules with a valid relative :install_path option' do + module_opts = { + install_path: 'vendor', + git: 'git@example.com:puppet/test_module.git', + } + + expect(R10K::Module).to receive(:from_metadata).with('puppet/test_module', File.join(basedir, 'vendor'), module_opts, anything).and_call_original + + expect { subject.add_module('puppet/test_module', module_opts) }.to change { subject.modules } + expect(subject.modules.collect(&:name)).to include('test_module') + end + + it 'should accept non-Forge modules with a valid absolute :install_path option' do + install_path = File.join(basedir, 'vendor') + + module_opts = { + install_path: install_path, + git: 'git@example.com:puppet/test_module.git', + } + + expect(R10K::Module).to receive(:from_metadata).with('puppet/test_module', install_path, module_opts, anything).and_call_original + + expect { subject.add_module('puppet/test_module', module_opts) }.to change { subject.modules } + expect(subject.modules.collect(&:name)).to include('test_module') + end + + it 'should reject non-Forge modules with an invalid relative :install_path option' do + module_opts = { + install_path: '../../vendor', + git: 'git@example.com:puppet/test_module.git', + } + + expect { subject.add_module('puppet/test_module', module_opts) }.to raise_error(R10K::Error, /cannot manage content.*is not within/i).and not_change { subject.modules } + end + + it 'should reject non-Forge modules with an invalid absolute :install_path option' do + module_opts = { + install_path: '/tmp/mydata/vendor', + git: 'git@example.com:puppet/test_module.git', + } + + expect { subject.add_module('puppet/test_module', module_opts) }.to raise_error(R10K::Error, /cannot manage content.*is not within/i).and not_change { subject.modules } + end + + it 'should disable and not add modules that conflict with the environment' do + env = instance_double('R10K::Environment::Base') + mod = instance_double('R10K::Module::Base', name: 'conflict', origin: :puppetfile, 'origin=': nil) + allow(env).to receive(:name).and_return('conflict') + loader = R10K::ModuleLoader::Puppetfile.new(basedir: basedir, environment: env) + allow(env).to receive(:'module_conflicts?').with(mod).and_return(true) + allow(mod).to receive(:spec_deletable=) + + expect(R10K::Module).to receive(:from_metadata).with('conflict', anything, anything, anything).and_return(mod) + expect { loader.add_module('conflict', {}) }.not_to change { loader.modules } + end + end + + describe '#purge_exclusions' do + let(:managed_dirs) { ['dir1', 'dir2'] } + subject { R10K::ModuleLoader::Puppetfile.new(basedir: '/test/basedir') } + + it 'includes managed_directories' do + expect(subject.send(:determine_purge_exclusions, managed_dirs)).to match_array(managed_dirs) + end + + context 'when belonging to an environment' do + let(:env_contents) { ['env1', 'env2' ] } + let(:env) { double(:environment, desired_contents: env_contents) } + before { + allow(env).to receive(:name).and_return('env1') + } + subject { R10K::ModuleLoader::Puppetfile.new(basedir: '/test/basedir', environment: env) } + + it "includes environment's desired_contents" do + expect(subject.send(:determine_purge_exclusions, managed_dirs)).to match_array(managed_dirs + env_contents) + end + end + end + + describe '#managed_directories' do + + let(:basedir) { '/test/basedir' } + subject { R10K::ModuleLoader::Puppetfile.new(basedir: basedir) } + + before do + allow(subject).to receive(:puppetfile_content).and_return('') + end + + it 'returns an array of paths that #purge! will operate within' do + expect(R10K::Module).to receive(:from_metadata).with('puppet/test_module', subject.moduledir, hash_including(version: '1.2.3'), anything).and_call_original + subject.add_module('puppet/test_module', '1.2.3') + subject.load! + + expect(subject.modules.length).to be 1 + expect(subject.managed_directories).to match_array([subject.moduledir]) + end + + context "with a module with install_path == ''" do + it "basedir isn't in the list of paths to purge" do + module_opts = { install_path: '', git: 'git@example.com:puppet/test_module.git' } + + expect(R10K::Module).to receive(:from_metadata).with('puppet/test_module', basedir, module_opts, anything).and_call_original + subject.add_module('puppet/test_module', module_opts) + subject.load! + + expect(subject.modules.length).to be 1 + expect(subject.managed_directories).to be_empty + end + end + end + + describe 'evaluating a Puppetfile' do + def expect_wrapped_error(error, pf_path, error_type) + expect(error).to be_a_kind_of(R10K::Error) + expect(error.message).to eq("Failed to evaluate #{pf_path}") + expect(error.original).to be_a_kind_of(error_type) + end + + subject { described_class.new(basedir: @path) } + + it 'wraps and re-raises syntax errors' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'invalid-syntax') + pf_path = File.join(@path, 'Puppetfile') + expect { + subject.load! + }.to raise_error do |e| + expect_wrapped_error(e, pf_path, SyntaxError) + end + end + + it 'wraps and re-raises load errors' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'load-error') + pf_path = File.join(@path, 'Puppetfile') + expect { + subject.load! + }.to raise_error do |e| + expect_wrapped_error(e, pf_path, LoadError) + end + end + + it 'wraps and re-raises argument errors' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'argument-error') + pf_path = File.join(@path, 'Puppetfile') + expect { + subject.load! + }.to raise_error do |e| + expect_wrapped_error(e, pf_path, ArgumentError) + end + end + + describe 'forge declaration' do + before(:each) do + PuppetForge.host = "" + end + + it 'is respected if `allow_puppetfile_override` is true' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'forge-override') + puppetfile = R10K::ModuleLoader::Puppetfile.new(basedir: @path, overrides: { forge: { allow_puppetfile_override: true } }) + puppetfile.load! + expect(PuppetForge.host).to eq("my.custom.forge.com/") + end + + it 'is ignored if `allow_puppetfile_override` is false' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'forge-override') + puppetfile = R10K::ModuleLoader::Puppetfile.new(basedir: @path, overrides: { forge: { allow_puppetfile_override: false } }) + expect(PuppetForge).not_to receive(:host=).with("my.custom.forge.com") + puppetfile.load! + expect(PuppetForge.host).to eq("/") + end + end + + it 'rejects Puppetfiles with duplicate module names' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'duplicate-module-error') + pf_path = File.join(@path, 'Puppetfile') + expect { + subject.load! + }.to raise_error(R10K::Error, /Puppetfiles cannot contain duplicate module names/i) + end + + it 'wraps and re-raises name errors' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'name-error') + pf_path = File.join(@path, 'Puppetfile') + expect { + subject.load! + }.to raise_error do |e| + expect_wrapped_error(e, pf_path, NameError) + end + end + + it 'accepts a forge module with a version' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-with-version') + pf_path = File.join(@path, 'Puppetfile') + expect { subject.load! }.not_to raise_error + end + + describe 'setting a custom moduledir' do + it 'allows setting an absolute moduledir' do + @path = '/fake/basedir' + allow(subject).to receive(:puppetfile_content).and_return('moduledir "/fake/moduledir"') + subject.load! + expect(subject.instance_variable_get(:@moduledir)).to eq('/fake/moduledir') + end + + it 'roots relative moduledirs in the basedir' do + @path = '/fake/basedir' + allow(subject).to receive(:puppetfile_content).and_return('moduledir "my/moduledir"') + subject.load! + expect(subject.instance_variable_get(:@moduledir)).to eq(File.join(@path, 'my/moduledir')) + end + end + + it 'accepts a forge module without a version' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-without-version') + pf_path = File.join(@path, 'Puppetfile') + expect { subject.load! }.not_to raise_error + end + + it 'creates a git module and applies the default branch specified in the Puppetfile' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'default-branch-override') + pf_path = File.join(@path, 'Puppetfile') + expect { subject.load! }.not_to raise_error + git_module = subject.modules[0] + expect(git_module.default_ref).to eq 'here_lies_the_default_branch' + end + + it 'creates a git module and applies the provided default_branch_override' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'default-branch-override') + pf_path = File.join(@path, 'Puppetfile') + default_branch_override = 'default_branch_override_name' + subject.default_branch_override = default_branch_override + expect { subject.load! }.not_to raise_error + git_module = subject.modules[0] + expect(git_module.default_override_ref).to eq default_branch_override + expect(git_module.default_ref).to eq 'here_lies_the_default_branch' + end + + describe 'using module metadata' do + it 'properly loads module metadata' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'various-modules') + metadata = subject.load_metadata[:modules].map { |mod| [ mod.name, mod.version ] }.to_h + expect(metadata['apt']).to eq('2.1.1') + expect(metadata['stdlib']).to eq(nil) + expect(metadata['concat']).to eq(nil) + expect(metadata['rpm']).to eq('2.1.1-pre1') + expect(metadata['foo']).to eq(nil) + expect(metadata['bar']).to eq('v1.2.3') + expect(metadata['baz']).to eq('123abc456') + expect(metadata['fizz']).to eq('1234567890abcdef1234567890abcdef12345678') + expect(metadata['buzz']).to eq(nil) + expect(metadata['canary']).to eq('0.0.0') + end + + it 'does not load module implementations for static versions unless the module install path does not exist on disk' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'various-modules') + subject.load_metadata + modules = subject.load[:modules].map { |mod| [ mod.name, mod ] }.to_h + expect(modules['apt']).to be_a_kind_of(R10K::Module::Definition) + expect(modules['stdlib']).to be_a_kind_of(R10K::Module::Forge) + expect(modules['concat']).to be_a_kind_of(R10K::Module::Forge) + expect(modules['rpm']).to be_a_kind_of(R10K::Module::Definition) + expect(modules['foo']).to be_a_kind_of(R10K::Module::Git) + expect(modules['bar']).to be_a_kind_of(R10K::Module::Git) + expect(modules['baz']).to be_a_kind_of(R10K::Module::Definition) + expect(modules['fizz']).to be_a_kind_of(R10K::Module::Definition) + expect(modules['buzz']).to be_a_kind_of(R10K::Module::Git) + expect(modules['canary']).to be_a_kind_of(R10K::Module::Definition) + end + + it 'loads module implementations whose static versions are different' do + fixture_path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'various-modules') + @path = Dir.mktmpdir + unsynced_pf_path = File.join(fixture_path, 'Puppetfile') + FileUtils.cp(unsynced_pf_path, @path) + + subject.load_metadata + + synced_pf_path = File.join(fixture_path, 'Puppetfile.new') + FileUtils.cp(synced_pf_path, File.join(@path, 'Puppetfile')) + + modules = subject.load[:modules].map { |mod| [ mod.name, mod ] }.to_h + + expect(modules['apt']).to be_a_kind_of(R10K::Module::Forge) + end + + end + + describe 'using module-exclude-regex' do + it 'can exclude a module from being installed' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'various-modules') + puppetfile = R10K::ModuleLoader::Puppetfile.new(basedir: @path, module_exclude_regex: '^concat$') + puppetfile.load! + expect(puppetfile.modules.collect(&:name)).not_to include('concat') + end + + it 'can exclude multiple modules from being installed' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'various-modules') + puppetfile = R10K::ModuleLoader::Puppetfile.new(basedir: @path, module_exclude_regex: '^ba[rz]$') + puppetfile.load! + expect(puppetfile.modules.collect(&:name)).not_to include('bar') + expect(puppetfile.modules.collect(&:name)).not_to include('baz') + end + end + end +end diff -pruN 3.7.0-2.1/spec/unit/module_spec.rb 3.15.4-1/spec/unit/module_spec.rb --- 3.7.0-2.1/spec/unit/module_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/module_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -3,27 +3,112 @@ require 'r10k/module' describe R10K::Module do describe 'delegating to R10K::Module::Git' do - it "accepts args {:git => 'git url}" do - obj = R10K::Module.new('foo', '/modulepath', :git => 'git url') - expect(obj).to be_a_kind_of(R10K::Module::Git) + [ {git: 'git url'}, + {type: 'git', source: 'git url'}, + ].each do |scenario| + it "accepts a name matching 'test' and args #{scenario.inspect}" do + obj = R10K::Module.new('test', '/modulepath', scenario) + expect(obj).to be_a_kind_of(R10K::Module::Git) + expect(obj.send(:instance_variable_get, :'@remote')).to eq('git url') + end + end + end + + describe 'delegating to R10K::Module::Svn' do + [ {svn: 'svn url'}, + {type: 'svn', source: 'svn url'}, + ].each do |scenario| + it "accepts a name matching 'test' and args #{scenario.inspect}" do + obj = R10K::Module.new('test', '/modulepath', scenario) + expect(obj).to be_a_kind_of(R10K::Module::SVN) + expect(obj.send(:instance_variable_get, :'@url')).to eq('svn url') + end end end describe 'delegating to R10K::Module::Forge' do - [ - ['bar/quux', nil], - ['bar-quux', nil], - ['bar/quux', '8.0.0'], + [ 'bar/quux', + 'bar-quux', + ].each do |scenario| + it "accepts a name matching #{scenario} and version nil" do + obj = R10K::Module.new(scenario, '/modulepath', { type: 'forge', version: nil }) + expect(obj).to be_a_kind_of(R10K::Module::Forge) + end + end + [ {type: 'forge', version: '8.0.0'}, ].each do |scenario| - it "accepts a name matching #{scenario[0]} and args #{scenario[1].inspect}" do - expect(R10K::Module.new(scenario[0], '/modulepath', scenario[1])).to be_a_kind_of(R10K::Module::Forge) + it "accepts a name matching bar-quux and args #{scenario.inspect}" do + obj = R10K::Module.new('bar-quux', '/modulepath', scenario) + expect(obj).to be_a_kind_of(R10K::Module::Forge) + expect(obj.send(:instance_variable_get, :'@expected_version')).to eq('8.0.0') + end + end + + describe 'when the module is ostensibly on disk' do + before do + owner = 'theowner' + module_name = 'themodulename' + @title = "#{owner}-#{module_name}" + metadata = <<~METADATA + { + "name": "#{@title}", + "version": "1.2.0" + } + METADATA + @dirname = Dir.mktmpdir + module_path = File.join(@dirname, module_name) + FileUtils.mkdir(module_path) + File.open("#{module_path}/metadata.json", 'w') do |file| + file.write(metadata) + end + end + + it 'sets the expected version to what is found in the metadata' do + obj = R10K::Module.new(@title, @dirname, {type: 'forge', version: nil}) + expect(obj.send(:instance_variable_get, :'@expected_version')).to eq('1.2.0') end end end it "raises an error if delegation fails" do expect { - R10K::Module.new('bar!quux', '/modulepath', ["NOPE NOPE NOPE NOPE!"]) + R10K::Module.new('bar!quux', '/modulepath', {version: ["NOPE NOPE NOPE NOPE!"]}) }.to raise_error RuntimeError, /doesn't have an implementation/ end + + describe 'Given a set of initialization parameters for R10K::Module' do + [ ['name', {git: 'git url'}], + ['name', {type: 'git', source: 'git url'}], + ['name', {svn: 'svn url'}], + ['name', {type: 'svn', source: 'svn url'}], + ['namespace-name', {type: 'forge', version: '8.0.0'}] + ].each do |(name, options)| + it 'can handle the default_branch_override option' do + expect { + obj = R10K::Module.new(name, '/modulepath', options.merge({default_branch_override: 'foo'})) + expect(obj).to be_a_kind_of(R10K::Module::Base) + }.not_to raise_error + end + describe 'the exclude_spec setting' do + it 'sets the exclude_spec instance variable to false by default' do + obj = R10K::Module.new(name, '/modulepath', options) + expect(obj.instance_variable_get("@exclude_spec")).to eq(false) + end + it 'sets the exclude_spec instance variable' do + obj = R10K::Module.new(name, '/modulepath', options.merge({exclude_spec: true})) + expect(obj.instance_variable_get("@exclude_spec")).to eq(true) + end + it 'cannot be overridden by the settings from the cli, r10k.yaml, or settings default' do + options = options.merge({exclude_spec: true, overrides: {modules: {exclude_spec: false}}}) + obj = R10K::Module.new(name, '/modulepath', options) + expect(obj.instance_variable_get("@exclude_spec")).to eq(true) + end + it 'reads the setting from the cli, r10k.yaml, or settings default when not provided directly' do + options = options.merge({overrides: {modules: {exclude_spec: true}}}) + obj = R10K::Module.new(name, '/modulepath', options) + expect(obj.instance_variable_get("@exclude_spec")).to eq(true) + end + end + end + end end diff -pruN 3.7.0-2.1/spec/unit/puppetfile_spec.rb 3.15.4-1/spec/unit/puppetfile_spec.rb --- 3.7.0-2.1/spec/unit/puppetfile_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/puppetfile_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -6,13 +6,11 @@ describe R10K::Puppetfile do subject do described_class.new( '/some/nonexistent/basedir', - nil, - nil, - 'Puppetfile.r10k' + {puppetfile_name: 'Puppetfile.r10k'} ) end - describe "a custom puppetfile Puppetfile.r10k" do + describe "a custom puppetfile_name" do it "is the basedir joined with '/Puppetfile.r10k' path" do expect(subject.puppetfile_path).to eq '/some/nonexistent/basedir/Puppetfile.r10k' end @@ -22,10 +20,45 @@ end describe R10K::Puppetfile do + describe "a custom relative puppetfile_path" do + it "is the basedir joined with the puppetfile_path" do + relative_subject = described_class.new('/some/nonexistent/basedir', + {puppetfile_path: 'relative/Puppetfile'}) + expect(relative_subject.puppetfile_path).to eq '/some/nonexistent/basedir/relative/Puppetfile' + end + end + + describe "a custom absolute puppetfile_path" do + it "is the puppetfile_path as given" do + absolute_subject = described_class.new('/some/nonexistent/basedir', + {puppetfile_path: '/some/absolute/custom/Puppetfile'}) + expect(absolute_subject.puppetfile_path).to eq '/some/absolute/custom/Puppetfile' + end + end +end + +describe R10K::Puppetfile do + subject do - described_class.new( - '/some/nonexistent/basedir' - ) + described_class.new( '/some/nonexistent/basedir', {}) + end + + describe "backwards compatibility with older calling conventions" do + it "honors all arguments correctly" do + puppetfile = described_class.new('/some/nonexistant/basedir', '/some/nonexistant/basedir/site-modules', nil, 'Pupupupetfile', true) + expect(puppetfile.force).to eq(true) + expect(puppetfile.moduledir).to eq('/some/nonexistant/basedir/site-modules') + expect(puppetfile.puppetfile_path).to eq('/some/nonexistant/basedir/Pupupupetfile') + expect(puppetfile.overrides).to eq({}) + end + + it "handles defaults correctly" do + puppetfile = described_class.new('/some/nonexistant/basedir', nil, nil, nil) + expect(puppetfile.force).to eq(false) + expect(puppetfile.moduledir).to eq('/some/nonexistant/basedir/modules') + expect(puppetfile.puppetfile_path).to eq('/some/nonexistant/basedir/Puppetfile') + expect(puppetfile.overrides).to eq({}) + end end describe "the default moduledir" do @@ -53,237 +86,147 @@ describe R10K::Puppetfile do end end - describe "adding modules" do - it "should accept Forge modules with a string arg" do - allow(R10K::Module).to receive(:new).with('puppet/test_module', subject.moduledir, '1.2.3', anything).and_call_original - - expect { subject.add_module('puppet/test_module', '1.2.3') }.to change { subject.modules } - expect(subject.modules.collect(&:name)).to include('test_module') - end - - it "should not accept Forge modules with a version comparison" do - allow(R10K::Module).to receive(:new).with('puppet/test_module', subject.moduledir, '< 1.2.0', anything).and_call_original - - expect { - subject.add_module('puppet/test_module', '< 1.2.0') - }.to raise_error(RuntimeError, /module puppet\/test_module.*doesn't have an implementation/i) - - expect(subject.modules.collect(&:name)).not_to include('test_module') - end - - it "should accept non-Forge modules with a hash arg" do - module_opts = { git: 'git@example.com:puppet/test_module.git' } - - allow(R10K::Module).to receive(:new).with('puppet/test_module', subject.moduledir, module_opts, anything).and_call_original - - expect { subject.add_module('puppet/test_module', module_opts) }.to change { subject.modules } - expect(subject.modules.collect(&:name)).to include('test_module') - end - - it "should accept non-Forge modules with a valid relative :install_path option" do - module_opts = { - install_path: 'vendor', - git: 'git@example.com:puppet/test_module.git', - } - - allow(R10K::Module).to receive(:new).with('puppet/test_module', File.join(subject.basedir, 'vendor'), module_opts, anything).and_call_original - - expect { subject.add_module('puppet/test_module', module_opts) }.to change { subject.modules } - expect(subject.modules.collect(&:name)).to include('test_module') - end - - it "should accept non-Forge modules with a valid absolute :install_path option" do - install_path = File.join(subject.basedir, 'vendor') - - module_opts = { - install_path: install_path, - git: 'git@example.com:puppet/test_module.git', - } - - allow(R10K::Module).to receive(:new).with('puppet/test_module', install_path, module_opts, anything).and_call_original + describe "loading a Puppetfile" do + context 'using load' do + it "returns the loaded content" do + path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-with-version') + subject = described_class.new(path, {}) - expect { subject.add_module('puppet/test_module', module_opts) }.to change { subject.modules } - expect(subject.modules.collect(&:name)).to include('test_module') - end - - it "should reject non-Forge modules with an invalid relative :install_path option" do - module_opts = { - install_path: '../../vendor', - git: 'git@example.com:puppet/test_module.git', - } - - allow(R10K::Module).to receive(:new).with('puppet/test_module', File.join(subject.basedir, 'vendor'), module_opts, anything).and_call_original - - expect { subject.add_module('puppet/test_module', module_opts) }.to raise_error(R10K::Error, /cannot manage content.*is not within/i).and not_change { subject.modules } - end + loaded_content = subject.load + expect(loaded_content).to be_an_instance_of(Hash) - it "should reject non-Forge modules with an invalid absolute :install_path option" do - module_opts = { - install_path: '/tmp/mydata/vendor', - git: 'git@example.com:puppet/test_module.git', - } + has_some_data = loaded_content.values.none?(&:empty?) + expect(has_some_data).to be true + end - allow(R10K::Module).to receive(:new).with('puppet/test_module', File.join(subject.basedir, 'vendor'), module_opts, anything).and_call_original + it "handles a relative basedir" do + path = File.join('spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-with-version') + subject = described_class.new(path, {}) - expect { subject.add_module('puppet/test_module', module_opts) }.to raise_error(R10K::Error, /cannot manage content.*is not within/i).and not_change { subject.modules } - end + loaded_content = subject.load + expect(loaded_content).to be_an_instance_of(Hash) - it "groups modules by vcs cache location" do - module_opts = { install_path: File.join(subject.basedir, 'vendor') } - opts1 = module_opts.merge(git: 'git@example.com:puppet/test_module.git') - opts2 = module_opts.merge(git: 'git@example.com:puppet/test_module_c.git') - sanitized_name1 = "git@example.com-puppet-test_module.git" - sanitized_name2 = "git@example.com-puppet-test_module_c.git" + has_some_data = loaded_content.values.none?(&:empty?) + expect(has_some_data).to be true + end - subject.add_module('puppet/test_module_a', opts1) - subject.add_module('puppet/test_module_b', opts1) - subject.add_module('puppet/test_module_c', opts2) - subject.add_module('puppet/test_module_d', '1.2.3') + it "is idempotent" do + path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-with-version') + subject = described_class.new(path, {}) - mods_by_cachedir = subject.modules_by_vcs_cachedir + expect(subject.loader).to receive(:load!).and_call_original.once - expect(mods_by_cachedir[:none].length).to be 1 - expect(mods_by_cachedir[sanitized_name1].length).to be 2 - expect(mods_by_cachedir[sanitized_name2].length).to be 1 - end - end + loaded_content1 = subject.load + expect(subject.loaded?).to be true + loaded_content2 = subject.load - describe "#purge_exclusions" do - let(:managed_dirs) { ['dir1', 'dir2'] } + expect(loaded_content2).to eq(loaded_content1) + end - before(:each) do - allow(subject).to receive(:managed_directories).and_return(managed_dirs) + it "returns nil if Puppetfile doesn't exist" do + path = '/rando/path/that/wont/exist' + subject = described_class.new(path, {}) + expect(subject.load).to eq nil + end end - it "includes managed_directories" do - expect(subject.purge_exclusions).to match_array(managed_dirs) - end + context 'using load!' do + it "returns the loaded content" do + path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-with-version') + subject = described_class.new(path, {}) - context "when belonging to an environment" do - let(:env_contents) { ['env1', 'env2' ] } + loaded_content = subject.load! + expect(loaded_content).to be_an_instance_of(Hash) - before(:each) do - mock_env = double(:environment, desired_contents: env_contents) - allow(subject).to receive(:environment).and_return(mock_env) + has_some_data = loaded_content.values.none?(&:empty?) + expect(has_some_data).to be true end - it "includes environment's desired_contents" do - expect(subject.purge_exclusions).to match_array(managed_dirs + env_contents) + it "raises if Puppetfile doesn't exist" do + path = '/rando/path/that/wont/exist' + subject = described_class.new(path, {}) + expect { + subject.load! + }.to raise_error(/No such file or directory.*\/rando\/path\/.*/) end end end - describe '#managed_directories' do - it 'returns an array of paths that can be purged' do - allow(R10K::Module).to receive(:new).with('puppet/test_module', subject.moduledir, '1.2.3', anything).and_call_original - - subject.add_module('puppet/test_module', '1.2.3') - expect(subject.managed_directories).to match_array(["/some/nonexistent/basedir/modules"]) - end - - context 'with a module with install_path == \'\'' do - it 'basedir isn\'t in the list of paths to purge' do - module_opts = { install_path: '', git: 'git@example.com:puppet/test_module.git' } + describe 'default_branch_override' do + it 'is passed correctly to module loader init' do + # This path doesn't matter so long as it has a Puppetfile within it + path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-with-version') + subject = described_class.new(path, {overrides: {environments: {default_branch_override: 'foo'}}}) - allow(R10K::Module).to receive(:new).with('puppet/test_module', subject.basedir, module_opts, anything).and_call_original + repo = instance_double('R10K::Git::StatefulRepository') + allow(repo).to receive(:resolve).with('foo').and_return(true) + allow(R10K::Git::StatefulRepository).to receive(:new).and_return(repo) - subject.add_module('puppet/test_module', module_opts) - expect(subject.managed_directories).to be_empty - end - end - end + allow(subject.loader).to receive(:puppetfile_content).and_return <<-EOPF + # Track control branch and fall-back to main if no matching branch. + mod 'hieradata', + :git => 'git@git.example.com:organization/hieradata.git', + :branch => :control_branch, + :default_branch => 'main' + EOPF - describe "evaluating a Puppetfile" do - def expect_wrapped_error(orig, pf_path, wrapped_error) - expect(orig).to be_a_kind_of(R10K::Error) - expect(orig.message).to eq("Failed to evaluate #{pf_path}") - expect(orig.original).to be_a_kind_of(wrapped_error) - end + expect(subject.logger).not_to receive(:warn). + with(/Mismatch between passed and initialized.*preferring passed value/) - it "wraps and re-raises syntax errors" do - path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'invalid-syntax') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - expect { - subject.load! - }.to raise_error do |e| - expect_wrapped_error(e, pf_path, SyntaxError) - end - end + subject.load - it "wraps and re-raises load errors" do - path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'load-error') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - expect { - subject.load! - }.to raise_error do |e| - expect_wrapped_error(e, pf_path, LoadError) - end + loaded_module = subject.modules.first + expect(loaded_module.version).to eq('foo') end - it "wraps and re-raises argument errors" do - path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'argument-error') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - expect { - subject.load! - }.to raise_error do |e| - expect_wrapped_error(e, pf_path, ArgumentError) - end - end + it 'overrides module loader init if needed' do + # This path doesn't matter so long as it has a Puppetfile within it + path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-with-version') + subject = described_class.new(path, {overrides: {environments: {default_branch_override: 'foo'}}}) - it "rejects Puppetfiles with duplicate module names" do - path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'duplicate-module-error') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - expect { - subject.load! - }.to raise_error(R10K::Error, /Puppetfiles cannot contain duplicate module names/i) - end - - it "wraps and re-raises name errors" do - path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'name-error') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - expect { - subject.load! - }.to raise_error do |e| - expect_wrapped_error(e, pf_path, NameError) - end + repo = instance_double('R10K::Git::StatefulRepository') + allow(repo).to receive(:resolve).with('bar').and_return(true) + allow(R10K::Git::StatefulRepository).to receive(:new).and_return(repo) + + allow(subject.loader).to receive(:puppetfile_content).and_return <<-EOPF + # Track control branch and fall-back to main if no matching branch. + mod 'hieradata', + :git => 'git@git.example.com:organization/hieradata.git', + :branch => :control_branch, + :default_branch => 'main' + EOPF + + expect(subject.logger).to receive(:warn). + with(/Mismatch between passed and initialized.*preferring passed value/) + + subject.load('bar') + loaded_module = subject.modules.first + expect(loaded_module.version).to eq('bar') end - it "accepts a forge module with a version" do + it 'does not warn if passed and initialized default_branch_overrides match' do + # This path doesn't matter so long as it has a Puppetfile within it path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-with-version') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - expect { subject.load! }.not_to raise_error - end - - it "accepts a forge module without a version" do - path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-without-version') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - expect { subject.load! }.not_to raise_error - end - - it "creates a git module and applies the default branch sepcified in the Puppetfile" do - path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'default-branch-override') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - expect { subject.load! }.not_to raise_error - git_module = subject.modules[0] - expect(git_module.default_ref).to eq 'here_lies_the_default_branch' - end - - it "creates a git module and applies the provided default_branch_override" do - path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'default-branch-override') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - default_branch_override = 'default_branch_override_name' - expect { subject.load!(default_branch_override) }.not_to raise_error - git_module = subject.modules[0] - expect(git_module.default_ref).to eq default_branch_override + subject = described_class.new(path, {overrides: {environments: {default_branch_override: 'foo'}}}) + + repo = instance_double('R10K::Git::StatefulRepository') + allow(repo).to receive(:resolve).with('foo').and_return(true) + allow(R10K::Git::StatefulRepository).to receive(:new).and_return(repo) + + allow(subject.loader).to receive(:puppetfile_content).and_return <<-EOPF + # Track control branch and fall-back to main if no matching branch. + mod 'hieradata', + :git => 'git@git.example.com:organization/hieradata.git', + :branch => :control_branch, + :default_branch => 'main' + EOPF + + expect(subject.logger).not_to receive(:warn). + with(/Mismatch between passed and initialized.*preferring passed value/) + + subject.load('foo') + loaded_module = subject.modules.first + expect(loaded_module.version).to eq('foo') end end @@ -294,7 +237,7 @@ describe R10K::Puppetfile do subject.accept(visitor) end - it "passes the visitor to each module if the visitor yields" do + it "synchronizes each module if the visitor yields" do visitor = spy('visitor') expect(visitor).to receive(:visit) do |type, other, &block| expect(type).to eq :puppetfile @@ -302,12 +245,12 @@ describe R10K::Puppetfile do block.call end - mod1 = spy('module') - expect(mod1).to receive(:accept).with(visitor) - mod2 = spy('module') - expect(mod2).to receive(:accept).with(visitor) + mod1 = instance_double('R10K::Module::Base', :cachedir => :none) + mod2 = instance_double('R10K::Module::Base', :cachedir => :none) + expect(mod1).to receive(:sync) + expect(mod2).to receive(:sync) + expect(subject).to receive(:modules).and_return([mod1, mod2]) - expect(subject).to receive(:modules_by_vcs_cachedir).and_return({none: [mod1, mod2]}) subject.accept(visitor) end @@ -323,15 +266,14 @@ describe R10K::Puppetfile do block.call end - mod1 = spy('module') - expect(mod1).to receive(:accept).with(visitor) - mod2 = spy('module') - expect(mod2).to receive(:accept).with(visitor) - - expect(subject).to receive(:modules_by_vcs_cachedir).and_return({none: [mod1, mod2]}) + mod1 = instance_double('R10K::Module::Base', :cachedir => :none) + mod2 = instance_double('R10K::Module::Base', :cachedir => :none) + expect(mod1).to receive(:sync) + expect(mod2).to receive(:sync) + expect(subject).to receive(:modules).and_return([mod1, mod2]) expect(Thread).to receive(:new).exactly(pool_size).and_call_original - expect(Queue).to receive(:new).and_call_original + expect(Queue).to receive(:new).and_call_original.twice subject.accept(visitor) end @@ -344,22 +286,19 @@ describe R10K::Puppetfile do block.call end - mod1 = spy('module1') - mod2 = spy('module2') - mod3 = spy('module3') - mod4 = spy('module4') - mod5 = spy('module5') - mod6 = spy('module6') - - expect(subject).to receive(:modules_by_vcs_cachedir) - .and_return({:none => [mod1, mod2], - "foo-cachedir" => [mod3, mod4], - "bar-cachedir" => [mod5, mod6]}) + m1 = instance_double('R10K::Module::Base', :cachedir => '/dev/null/A') + m2 = instance_double('R10K::Module::Base', :cachedir => '/dev/null/B') + m3 = instance_double('R10K::Module::Base', :cachedir => '/dev/null/C') + m4 = instance_double('R10K::Module::Base', :cachedir => '/dev/null/C') + m5 = instance_double('R10K::Module::Base', :cachedir => '/dev/null/D') + m6 = instance_double('R10K::Module::Base', :cachedir => '/dev/null/D') + + modules = [m1, m2, m3, m4, m5, m6] - queue = subject.modules_queue(visitor) + queue = R10K::ContentSynchronizer.modules_visit_queue(modules, visitor, subject) expect(queue.length).to be 4 queue_array = 4.times.map { queue.pop } - expect(queue_array).to match_array([[mod1], [mod2], [mod3, mod4], [mod5, mod6]]) + expect(queue_array).to match_array([[m1], [m2], [m3, m4], [m5, m6]]) end end end diff -pruN 3.7.0-2.1/spec/unit/settings_spec.rb 3.15.4-1/spec/unit/settings_spec.rb --- 3.7.0-2.1/spec/unit/settings_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/settings_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -90,11 +90,50 @@ describe R10K::Settings do end end end + + describe "allow_puppetfile_override" do + it 'is false by default' do + expect(subject.evaluate({})[:allow_puppetfile_override]).to eq(false) + end + + it 'can be set to true' do + expect(subject.evaluate({"allow_puppetfile_override" => true})[:allow_puppetfile_override]).to eq(true) + end + + it "raises an error for non-boolean values" do + expect { + subject.evaluate({"allow_puppetfile_override" => 'invalid_string'}) + }.to raise_error do |err| + expect(err.message).to match(/Validation failed for 'forge' settings group/) + expect(err.errors.size).to eq 1 + expect(err.errors[:allow_puppetfile_override]).to be_a_kind_of(ArgumentError) + expect(err.errors[:allow_puppetfile_override].message).to match(/`allow_puppetfile_override` can only be a boolean value, not 'invalid_string'/) + end + end + end end describe "deploy settings" do subject { described_class.deploy_settings } + describe 'exclude_spec' do + it 'is false by default' do + expect(subject.evaluate({})[:exclude_spec]).to eq(false) + end + it 'can be set to true' do + expect(subject.evaluate({"exclude_spec" => true})[:exclude_spec]).to eq(true) + end + it "raises an error for non-boolean values" do + expect { + subject.evaluate({"exclude_spec" => 'invalid_string'}) + }.to raise_error do |err| + expect(err.message).to match(/Validation failed for 'deploy' settings group/) + expect(err.errors.size).to eq 1 + expect(err.errors[:exclude_spec]).to be_a_kind_of(ArgumentError) + expect(err.errors[:exclude_spec].message).to match(/`exclude_spec` can only be a boolean value, not 'invalid_string'/) + end + end + end describe "write_lock" do it "accepts a string with a reason for the write lock" do output = subject.evaluate("write_lock" => "No maintenance window active, code freeze till 2038-01-19") @@ -250,8 +289,14 @@ describe R10K::Settings do describe "forge settings" do it "passes settings through to the forge settings" do - output = subject.evaluate("forge" => {"baseurl" => "https://forge.tessier-ashpool.freeside", "proxy" => "https://proxy.tessier-ashpool.freesize:3128"}) - expect(output[:forge]).to eq(:baseurl => "https://forge.tessier-ashpool.freeside", :proxy => "https://proxy.tessier-ashpool.freesize:3128") + output = subject.evaluate("forge" => {"baseurl" => "https://forge.tessier-ashpool.freeside", + "proxy" => "https://proxy.tessier-ashpool.freesize:3128", + "authorization_token" => "faketoken", + "allow_puppetfile_override" => true}) + expect(output[:forge]).to eq(:baseurl => "https://forge.tessier-ashpool.freeside", + :proxy => "https://proxy.tessier-ashpool.freesize:3128", + :authorization_token => "faketoken", + :allow_puppetfile_override => true) end end end diff -pruN 3.7.0-2.1/spec/unit/tarball_spec.rb 3.15.4-1/spec/unit/tarball_spec.rb --- 3.7.0-2.1/spec/unit/tarball_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/unit/tarball_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,57 @@ +require 'spec_helper' +require 'r10k/tarball' + +describe R10K::Tarball do + include_context 'Tarball' + + subject { described_class.new('fixture-tarball', fixture_tarball, checksum: fixture_checksum) } + + describe 'initialization' do + it 'initializes' do + expect(subject).to be_kind_of(described_class) + end + end + + describe 'downloading and caching' do + it 'downloads the source to the cache' do + # No cache present initially + expect(File.exist?(subject.cache_path)).to be(false) + expect(subject.cache_valid?).to be(false) + + subject.get + + expect(subject.cache_valid?).to be(true) + expect(File.exist?(subject.cache_path)).to be(true) + end + + let(:raw_content) {[ + './', + './Puppetfile', + './metadata.json', + './spec/', + './environment.conf', + './spec/1', + ]} + + let(:clean_content) {[ + 'Puppetfile', + 'metadata.json', + 'spec', + 'environment.conf', + 'spec/1', + ]} + + it 'returns clean paths when listing cached tarball content' do + iterator = allow(subject).to receive(:each_tarball_entry) + raw_content.each { |entry| iterator.and_yield(entry) } + + expect(subject.paths).to eq(clean_content) + end + end + + describe 'http sources' + + describe 'file sources' + + describe 'syncing' +end diff -pruN 3.7.0-2.1/spec/unit/util/cacheable_spec.rb 3.15.4-1/spec/unit/util/cacheable_spec.rb --- 3.7.0-2.1/spec/unit/util/cacheable_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/unit/util/cacheable_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,23 @@ +require 'spec_helper' +require 'r10k/util/cacheable' + +RSpec.describe R10K::Util::Cacheable do + + subject { Object.new.extend(R10K::Util::Cacheable) } + + describe "dirname sanitization" do + let(:input) { 'https://some/git/remote' } + + it 'sanitizes URL to directory name' do + expect(subject.sanitized_dirname(input)).to eq('https---some-git-remote') + end + + context 'with username and password' do + let(:input) { 'https://"user:pa$$w0rd:@authenticated/git/remote' } + + it 'sanitizes authenticated URL to directory name' do + expect(subject.sanitized_dirname(input)).to eq('https---authenticated-git-remote') + end + end + end +end diff -pruN 3.7.0-2.1/spec/unit/util/downloader_spec.rb 3.15.4-1/spec/unit/util/downloader_spec.rb --- 3.7.0-2.1/spec/unit/util/downloader_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ 3.15.4-1/spec/unit/util/downloader_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -0,0 +1,98 @@ +require 'spec_helper' +require 'r10k/util/downloader' + +describe R10K::Util::Downloader do + + subject(:downloader) do + subj = Object.new + subj.extend(R10K::Util::Downloader) + subj.singleton_class.class_eval { public :download } + subj.singleton_class.class_eval { public :http_get } + subj.singleton_class.class_eval { public :file_digest } + subj + end + + let(:tmpdir) { Dir.mktmpdir } + after(:each) { FileUtils.remove_entry_secure(tmpdir) } + + describe 'http_get' do + let(:src_url) { 'https://example.com' } + let(:dst_file) { File.join(tmpdir, 'test.out') } + let(:tarball_uri) { URI('http://tarball.example.com/tarball.tar.gz') } + let(:redirect_uri) { URI('http://redirect.example.com/redirect') } + let(:proxy_uri) { URI('http://user:password@proxy.example.com') } + + it 'downloads a simple file' do + mock_session = instance_double('Net::HTTP', active?: true) + tarball_response = instance_double('Net::HTTPSuccess') + + expect(Net::HTTP).to receive(:new).with(tarball_uri.host, any_args).and_return(mock_session) + expect(Net::HTTPSuccess).to receive(:===).with(tarball_response).and_return(true) + + expect(mock_session).to receive(:request_get).and_yield(tarball_response) + expect(mock_session).to receive(:start).once + expect(mock_session).to receive(:finish).once + + expect { |b| downloader.http_get(tarball_uri, &b) }.to yield_with_args(tarball_response) + end + + it 'follows redirects' do + mock_session_1 = instance_double('Net::HTTP', active?: false) + mock_session_2 = instance_double('Net::HTTP', active?: true) + redirect_response = instance_double('Net::HTTPRedirection') + tarball_response = instance_double('Net::HTTPSuccess') + + expect(Net::HTTP).to receive(:new).with(redirect_uri.host, any_args).and_return(mock_session_1).once + expect(Net::HTTP).to receive(:new).with(tarball_uri.host, any_args).and_return(mock_session_2).once + expect(Net::HTTPRedirection).to receive(:===).with(redirect_response).and_return(true) + expect(Net::HTTPSuccess).to receive(:===).with(tarball_response).and_return(true) + allow(Net::HTTPRedirection).to receive(:===).and_call_original + + expect(mock_session_1).to receive(:request_get).and_yield(redirect_response) + expect(mock_session_2).to receive(:request_get).and_yield(tarball_response) + + # The redirect response should be queried for the redirect location + expect(redirect_response).to receive(:[]).with('location').and_return(tarball_uri.to_s) + + # Both sessions should start and finish cleanly + expect(mock_session_1).to receive(:start).once + expect(mock_session_1).to receive(:finish).once + expect(mock_session_2).to receive(:start).once + expect(mock_session_2).to receive(:finish).once + + expect { |b| downloader.http_get(redirect_uri, &b) }.to yield_with_args(tarball_response) + end + + it 'can use a proxy' do + mock_session = instance_double('Net::HTTP', active?: true) + + expect(Net::HTTP).to receive(:new) + .with(tarball_uri.host, + tarball_uri.port, + proxy_uri.host, + proxy_uri.port, + proxy_uri.user, + proxy_uri.password, + any_args) + .and_return(mock_session) + + expect(mock_session).to receive(:request_get).and_return(:not_yielded) + expect(mock_session).to receive(:start).once + expect(mock_session).to receive(:finish).once + + downloader.http_get(tarball_uri, proxy: proxy_uri) + end + end + + describe 'checksums' do + let(:fixture_checksum) { '0bcea17aa0c5e868c18f0fa042feda770e47c1a4223229f82116ccb3ca33c6e3' } + let(:fixture_tarball) do + File.expand_path('spec/fixtures/integration/git/puppet-boolean-bare.tar', PROJECT_ROOT) + end + + it 'checksums files' do + expect(downloader.file_digest(fixture_tarball)).to eql(fixture_checksum) + end + end +end + diff -pruN 3.7.0-2.1/spec/unit/util/purgeable_spec.rb 3.15.4-1/spec/unit/util/purgeable_spec.rb --- 3.7.0-2.1/spec/unit/util/purgeable_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/util/purgeable_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -1,5 +1,6 @@ require 'spec_helper' require 'r10k/util/purgeable' +require 'r10k/util/cleaner' RSpec.describe R10K::Util::Purgeable do let(:managed_directories) do @@ -14,28 +15,25 @@ RSpec.describe R10K::Util::Purgeable do 'spec/fixtures/unit/util/purgeable/managed_one/expected_1', 'spec/fixtures/unit/util/purgeable/managed_one/new_1', 'spec/fixtures/unit/util/purgeable/managed_one/managed_subdir_1', + 'spec/fixtures/unit/util/purgeable/managed_one/managed_symlink_dir', 'spec/fixtures/unit/util/purgeable/managed_one/managed_subdir_1/subdir_expected_1', 'spec/fixtures/unit/util/purgeable/managed_one/managed_subdir_1/subdir_new_1', + 'spec/fixtures/unit/util/purgeable/managed_one/managed_subdir_1/managed_symlink_file', 'spec/fixtures/unit/util/purgeable/managed_two/expected_2', 'spec/fixtures/unit/util/purgeable/managed_two/new_2', + 'spec/fixtures/unit/util/purgeable/managed_two/.hidden', ] end - let(:test_class) do - Struct.new(:managed_directories, :desired_contents) do - include R10K::Util::Purgeable - include R10K::Logging - end - end - - subject { test_class.new(managed_directories, desired_contents) } + subject { R10K::Util::Cleaner.new(managed_directories, desired_contents) } context 'without recurse option' do let(:recurse) { false } describe '#current_contents' do it 'collects direct contents of all managed directories' do - expect(subject.current_contents(recurse)).to contain_exactly(/\/expected_1/, /\/expected_2/, /\/unmanaged_1/, /\/unmanaged_2/, /\/managed_subdir_1/) + expect(subject.current_contents(recurse)).to contain_exactly(/\/expected_1/, /\/expected_2/, /\/unmanaged_1/, /\/unmanaged_2/, + /\/managed_subdir_1/, /\/managed_symlink_dir/, /\/unmanaged_symlink_file/) end end @@ -51,7 +49,7 @@ RSpec.describe R10K::Util::Purgeable do let(:whitelist) { [] } it 'collects current_contents that should not exist' do - expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_1/, /\/unmanaged_2/) + expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_1/, /\/unmanaged_2/, /\/unmanaged_symlink_file/) end end @@ -61,7 +59,7 @@ RSpec.describe R10K::Util::Purgeable do it 'collects current_contents that should not exist except whitelisted items' do expect(subject.logger).to receive(:debug).with(/unmanaged_1.*whitelist match/i) - expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_2/) + expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_2/, /\/unmanaged_symlink_file/) end end @@ -71,7 +69,7 @@ RSpec.describe R10K::Util::Purgeable do it 'collects current_contents that should not exist except excluded items' do expect(subject.logger).to receive(:debug2).with(/unmanaged_2.*internal exclusion match/i) - expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_1/) + expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_1/, /\/unmanaged_symlink_file/) end end end @@ -104,7 +102,17 @@ RSpec.describe R10K::Util::Purgeable do describe '#current_contents' do it 'collects contents of all managed directories recursively' do - expect(subject.current_contents(recurse)).to contain_exactly(/\/expected_1/, /\/expected_2/, /\/unmanaged_1/, /\/unmanaged_2/, /\/managed_subdir_1/, /\/subdir_expected_1/, /\/subdir_unmanaged_1/) + expect(subject.current_contents(recurse)). + to contain_exactly(/\/expected_1/, /\/expected_2/, + /\/unmanaged_1/, /\/unmanaged_2/, + /\/managed_symlink_dir/, + /\/unmanaged_symlink_file/, + /\/managed_subdir_1/, + /\/subdir_expected_1/, /\/subdir_unmanaged_1/, + /\/managed_symlink_file/, + /\/unmanaged_symlink_dir/, + /\/subdir_allowlisted_2/, /\/ignored_1/, + /\/\.hidden/) end end @@ -120,7 +128,9 @@ RSpec.describe R10K::Util::Purgeable do let(:whitelist) { [] } it 'collects current_contents that should not exist recursively' do - expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_1/, /\/unmanaged_2/, /\/subdir_unmanaged_1/) + expect(subject.stale_contents(recurse, exclusions, whitelist)). + to contain_exactly(/\/unmanaged_1/, /\/unmanaged_2/, /\/unmanaged_symlink_file/, /\/subdir_unmanaged_1/, + /\/ignored_1/, /\/subdir_allowlisted_2/, /\/unmanaged_symlink_dir/) end end @@ -130,7 +140,18 @@ RSpec.describe R10K::Util::Purgeable do it 'collects current_contents that should not exist except whitelisted items' do expect(subject.logger).to receive(:debug).with(/unmanaged_1.*whitelist match/i) - expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_2/, /\/subdir_unmanaged_1/) + + expect(subject.stale_contents(recurse, exclusions, whitelist)). + to contain_exactly(/\/unmanaged_2/, /\/subdir_unmanaged_1/, /\/unmanaged_symlink_file/, /\/ignored_1/, + /\/subdir_allowlisted_2/, /\/unmanaged_symlink_dir/) + end + + it 'does not collect contents that match recursive globbed whitelist items as intermediate values' do + recursive_whitelist = ['**/managed_subdir_1/**/*'] + expect(subject.logger).not_to receive(:debug).with(/ignored_1/) + + expect(subject.stale_contents(recurse, exclusions, recursive_whitelist)). + to contain_exactly(/\/unmanaged_2/, /\/managed_one\/unmanaged_1/, /\/managed_one\/unmanaged_symlink_file/) end end @@ -140,7 +161,18 @@ RSpec.describe R10K::Util::Purgeable do it 'collects current_contents that should not exist except excluded items' do expect(subject.logger).to receive(:debug2).with(/unmanaged_2.*internal exclusion match/i) - expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_1/, /\/subdir_unmanaged_1/) + + expect(subject.stale_contents(recurse, exclusions, whitelist)). + to contain_exactly(/\/unmanaged_1/, /\/unmanaged_symlink_file/, /\/subdir_unmanaged_1/, /\/ignored_1/, + /\/subdir_allowlisted_2/, /\/unmanaged_symlink_dir/) + end + + it 'does not collect contents that match recursive globbed exclusion items as intermediate values' do + recursive_exclusions = ['**/managed_subdir_1/**/*'] + expect(subject.logger).not_to receive(:debug).with(/ignored_1/) + + expect(subject.stale_contents(recurse, recursive_exclusions, whitelist)). + to contain_exactly(/\/unmanaged_2/, /\/unmanaged_symlink_file/, /\/managed_one\/unmanaged_1/) end end end @@ -177,6 +209,7 @@ RSpec.describe R10K::Util::Purgeable do it 'does not purge items matching glob at root level' do allow(FileUtils).to receive(:rm_r) expect(FileUtils).to_not receive(:rm_r).with(/\/unmanaged_[12]/, anything) + expect(FileUtils).to_not receive(:rm_r).with(/\/unmanaged_symlink_file/, anything) expect(subject.logger).to receive(:debug).with(/whitelist match/i).at_least(:once) subject.purge!(purge_opts) @@ -185,7 +218,11 @@ RSpec.describe R10K::Util::Purgeable do end context "recursive whitelist glob" do - let(:whitelist) { managed_directories.collect { |dir| File.join(dir, "**", "*unmanaged*") } } + let(:whitelist) do + managed_directories.flat_map do |dir| + [File.join(dir, "**", "*unmanaged*"), File.join(dir, "**", "subdir_allowlisted_2")] + end + end let(:purge_opts) { { recurse: true, whitelist: whitelist } } describe '#purge!' do @@ -220,7 +257,7 @@ RSpec.describe R10K::Util::Purgeable do context "when class does not implement #purge_exclusions" do describe '#purge!' do it 'purges normally' do - expect(FileUtils).to receive(:rm_r).at_least(3).times + expect(FileUtils).to receive(:rm_r).at_least(4).times subject.purge!(purge_opts) end diff -pruN 3.7.0-2.1/spec/unit/util/setopts_spec.rb 3.15.4-1/spec/unit/util/setopts_spec.rb --- 3.7.0-2.1/spec/unit/util/setopts_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/spec/unit/util/setopts_spec.rb 2023-01-19 00:49:17.000000000 +0000 @@ -10,7 +10,10 @@ describe R10K::Util::Setopts do def initialize(opts = {}) setopts(opts, { - :valid => :self, :alsovalid => :self, :truthyvalid => true, + :valid => :self, + :duplicate => :valid, + :alsovalid => :self, + :truthyvalid => true, :validalias => :valid, :ignoreme => nil }) @@ -53,7 +56,28 @@ describe R10K::Util::Setopts do }.to raise_error(ArgumentError, /cannot handle option 'notvalid'/) end + it "warns when given an unhandled option and raise_on_unhandled=false" do + test = Class.new { include R10K::Util::Setopts }.new + allow(test).to receive(:logger).and_return(spy) + + test.send(:setopts, {valid: :value, invalid: :value}, + {valid: :self}, + raise_on_unhandled: false) + + expect(test.logger).to have_received(:warn).with(%r{cannot handle option 'invalid'}) + end + it "ignores values that are marked as unhandled" do klass.new(:ignoreme => "IGNORE ME!") end + + it "warns when given conflicting options" do + test = Class.new { include R10K::Util::Setopts }.new + allow(test).to receive(:logger).and_return(spy) + + test.send(:setopts, {valid: :one, duplicate: :two}, + {valid: :arg, duplicate: :arg}) + + expect(test.logger).to have_received(:warn).with(%r{valid.*duplicate.*conflict.*not both}) + end end diff -pruN 3.7.0-2.1/.travis.yml 3.15.4-1/.travis.yml --- 3.7.0-2.1/.travis.yml 2020-11-13 22:27:51.000000000 +0000 +++ 3.15.4-1/.travis.yml 1970-01-01 00:00:00.000000000 +0000 @@ -1,45 +0,0 @@ ---- -language: ruby -bundler_args: "--without system" -script: "bundle exec rspec --color --format documentation spec/unit" -notifications: - email: false -sudo: false -jdk: - - openjdk11 -before_install: gem install bundler -v '< 2' --no-document -matrix: - include: - - stage: r10k tests - rvm: 2.7.0 - - stage: r10k tests - rvm: 2.6.5 - - stage: r10k tests - rvm: 2.5.0 - - stage: r10k tests - rvm: 2.4.0 - - stage: r10k tests - rvm: 2.3.0 - - stage: r10k tests - rvm: jruby - - stage: r10k container tests - dist: focal - language: ruby - services: - - docker - rvm: 2.6.6 - env: - - DOCKER_COMPOSE_VERSION=1.25.5 - # necessary to prevent overwhelming TravisCI build output limits - - DOCKER_BUILD_FLAGS="--progress plain" - before_install: - - sudo rm /usr/local/bin/docker-compose - - curl --location https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname --kernel-name`-`uname --machine` > docker-compose - - chmod +x docker-compose - - sudo mv docker-compose /usr/local/bin - script: - - set -e - - cd docker - - make lint - - make build - - make test