From 4c14df6d8848618bf7d24c4eb14d24a49041da86 Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Thu, 17 Dec 2020 12:27:07 +0530 Subject: [PATCH 001/498] bitbucket_pipeline_variable: Change pagination logic (#1498) Bitbucket's pagination for pipeline variables is flawed. Refactor bitbucket_pipeline_variable code to correct this logic. Fixes: #1425 Signed-off-by: Abhijeet Kasurde --- .../1425_bitbucket_pipeline_variable.yml | 2 ++ .../bitbucket/bitbucket_pipeline_variable.py | 24 +++++++++++-------- 2 files changed, 16 insertions(+), 10 deletions(-) create mode 100644 changelogs/fragments/1425_bitbucket_pipeline_variable.yml diff --git a/changelogs/fragments/1425_bitbucket_pipeline_variable.yml b/changelogs/fragments/1425_bitbucket_pipeline_variable.yml new file mode 100644 index 00000000000..b284ca54d36 --- /dev/null +++ b/changelogs/fragments/1425_bitbucket_pipeline_variable.yml @@ -0,0 +1,2 @@ +bugfixes: +- bitbucket_pipeline_variable - change pagination logic for pipeline variable get API (https://github.com/ansible-collections/community.general/issues/1425). diff --git a/plugins/modules/source_control/bitbucket/bitbucket_pipeline_variable.py b/plugins/modules/source_control/bitbucket/bitbucket_pipeline_variable.py index d0eb86e6dac..c4ca59a3f10 100644 --- a/plugins/modules/source_control/bitbucket/bitbucket_pipeline_variable.py +++ b/plugins/modules/source_control/bitbucket/bitbucket_pipeline_variable.py @@ -72,7 +72,7 @@ secured: '{{ item.secured }}' state: present with_items: - - { name: AWS_ACCESS_KEY, value: ABCD1234 } + - { name: AWS_ACCESS_KEY, value: ABCD1234, secured: False } - { name: AWS_SECRET, value: qwe789poi123vbn0, secured: True } - name: Remove pipeline variable @@ -119,17 +119,16 @@ def get_existing_pipeline_variable(module, bitbucket): The `value` key in dict is absent in case of secured variable. """ - content = { - 'next': BITBUCKET_API_ENDPOINTS['pipeline-variable-list'].format( - username=module.params['username'], - repo_slug=module.params['repository'], - ) - } - + variables_base_url = BITBUCKET_API_ENDPOINTS['pipeline-variable-list'].format( + username=module.params['username'], + repo_slug=module.params['repository'], + ) # Look through the all response pages in search of variable we need - while 'next' in content: + page = 1 + while True: + next_url = "%s?page=%s" % (variables_base_url, page) info, content = bitbucket.request( - api_url=content['next'], + api_url=next_url, method='GET', ) @@ -139,6 +138,11 @@ def get_existing_pipeline_variable(module, bitbucket): if info['status'] != 200: module.fail_json(msg='Failed to retrieve the list of pipeline variables: {0}'.format(info)) + # We are at the end of list + if 'pagelen' in content and content['pagelen'] == 0: + return None + + page += 1 var = next(filter(lambda v: v['key'] == module.params['name'], content['values']), None) if var is not None: From 33126b7267f0cf30e8ac5bc1bf54f1a8f2a8e154 Mon Sep 17 00:00:00 2001 From: John Barker Date: Thu, 17 Dec 2020 18:13:12 +0000 Subject: [PATCH 002/498] AZP Bootstrap --- .azure-pipelines/README.md | 3 + .azure-pipelines/azure-pipelines.yml | 309 ++++++++++++++++++ .../scripts/aggregate-coverage.sh | 20 ++ .azure-pipelines/scripts/combine-coverage.py | 60 ++++ .azure-pipelines/scripts/process-results.sh | 24 ++ .azure-pipelines/scripts/publish-codecov.sh | 27 ++ .azure-pipelines/scripts/report-coverage.sh | 15 + .azure-pipelines/scripts/run-tests.sh | 34 ++ .azure-pipelines/scripts/time-command.py | 25 ++ .azure-pipelines/templates/coverage.yml | 39 +++ .azure-pipelines/templates/matrix.yml | 55 ++++ .azure-pipelines/templates/test.yml | 45 +++ tests/utils/shippable/shippable.sh | 22 +- 13 files changed, 669 insertions(+), 9 deletions(-) create mode 100644 .azure-pipelines/README.md create mode 100644 .azure-pipelines/azure-pipelines.yml create mode 100755 .azure-pipelines/scripts/aggregate-coverage.sh create mode 100755 .azure-pipelines/scripts/combine-coverage.py create mode 100755 .azure-pipelines/scripts/process-results.sh create mode 100755 .azure-pipelines/scripts/publish-codecov.sh create mode 100755 .azure-pipelines/scripts/report-coverage.sh create mode 100755 .azure-pipelines/scripts/run-tests.sh create mode 100755 .azure-pipelines/scripts/time-command.py create mode 100644 .azure-pipelines/templates/coverage.yml create mode 100644 .azure-pipelines/templates/matrix.yml create mode 100644 .azure-pipelines/templates/test.yml diff --git a/.azure-pipelines/README.md b/.azure-pipelines/README.md new file mode 100644 index 00000000000..385e70bac59 --- /dev/null +++ b/.azure-pipelines/README.md @@ -0,0 +1,3 @@ +## Azure Pipelines Configuration + +Please see the [Documentation](https://github.com/ansible/community/wiki/Testing:-Azure-Pipelines) for more information. diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml new file mode 100644 index 00000000000..3c9cf03dc5f --- /dev/null +++ b/.azure-pipelines/azure-pipelines.yml @@ -0,0 +1,309 @@ +trigger: + batch: true + branches: + include: + - main + - stable-* + +pr: + autoCancel: true + branches: + include: + - main + - stable-* + +schedules: + - cron: 0 9 * * * + displayName: Nightly + always: true + branches: + include: + - main + - stable-* + +variables: + - name: checkoutPath + value: ansible_collections/community/general + - name: coverageBranches + value: main + - name: pipelinesCoverage + value: coverage + - name: entryPoint + value: tests/utils/shippable/shippable.sh + - name: fetchDepth + value: 0 + +resources: + containers: + - container: default + image: quay.io/ansible/azure-pipelines-test-container:1.7.1 + +pool: Standard + +stages: +### Sanity + - stage: Sanity_devel + displayName: Sanity devel + dependsOn: [] + jobs: + - template: templates/matrix.yml + parameters: + nameFormat: Test {0} + testFormat: devel/sanity/{0} + targets: + - test: 1 + - test: 2 + - test: 3 + - test: 4 + - test: extra + - stage: Sanity_2_10 + displayName: Sanity 2.10 + dependsOn: [] + jobs: + - template: templates/matrix.yml + parameters: + nameFormat: Test {0} + testFormat: 2.10/sanity/{0} + targets: + - test: 1 + - test: 2 + - test: 3 + - test: 4 + - stage: Sanity_2_9 + displayName: Sanity 2.9 + dependsOn: [] + jobs: + - template: templates/matrix.yml + parameters: + nameFormat: Test {0} + testFormat: 2.9/sanity/{0} + targets: + - test: 1 + - test: 2 + - test: 3 + - test: 4 +### Units + - stage: Units_devel + displayName: Units devel + dependsOn: [] + jobs: + - template: templates/matrix.yml + parameters: + nameFormat: Python {0} + testFormat: devel/units/{0}/1 + targets: + - test: 2.6 + - test: 2.7 + - test: 3.5 + - test: 3.6 + - test: 3.7 + - test: 3.8 + - test: 3.9 + - stage: Units_2_10 + displayName: Units 2.10 + dependsOn: [] + jobs: + - template: templates/matrix.yml + parameters: + nameFormat: Python {0} + testFormat: 2.10/units/{0}/1 + targets: + - test: 2.6 + - test: 2.7 + - test: 3.5 + - test: 3.6 + - test: 3.7 + - test: 3.8 + - test: 3.9 + - stage: Units_2_9 + displayName: Units 2.9 + dependsOn: [] + jobs: + - template: templates/matrix.yml + parameters: + nameFormat: Python {0} + testFormat: 2.9/units/{0}/1 + targets: + - test: 2.6 + - test: 2.7 + - test: 3.5 + - test: 3.6 + - test: 3.7 + - test: 3.8 + +## Remote + - stage: Remote_devel + displayName: Remote devel + dependsOn: [] + jobs: + - template: templates/matrix.yml + parameters: + testFormat: devel/{0} + targets: + - name: OS X 10.11 + test: osx/10.11 + - name: macOS 10.15 + test: macos/10.15 + - name: RHEL 7.8 + test: rhel/7.8 + - name: RHEL 8.2 + test: rhel/8.2 + - name: FreeBSD 11.1 + test: freebsd/11.1 + - name: FreeBSD 12.1 + test: freebsd/12.1 + groups: + - 1 + - 2 + - 3 + - stage: Remote_2_10 + displayName: Remote 2.10 + dependsOn: [] + jobs: + - template: templates/matrix.yml + parameters: + testFormat: 2.10/{0} + targets: + - name: OS X 10.11 + test: osx/10.11 + - name: RHEL 8.2 + test: rhel/8.2 + - name: FreeBSD 12.1 + test: freebsd/12.1 + groups: + - 1 + - 2 + - stage: Remote_2_9 + displayName: Remote 2.9 + dependsOn: [] + jobs: + - template: templates/matrix.yml + parameters: + testFormat: 2.9/{0} + targets: + - name: RHEL 8.2 + test: rhel/8.2 + - name: FreeBSD 12.0 + test: freebsd/12.0 + groups: + - 1 + - 2 + +### Docker + - stage: Docker_devel + displayName: Docker devel + dependsOn: [] + jobs: + - template: templates/matrix.yml + parameters: + testFormat: devel/linux/{0} + targets: + - name: CentOS 6 + test: centos6 + - name: CentOS 7 + test: centos7 + - name: CentOS 8 + test: centos8 + - name: Fedora 31 + test: fedora31 + - name: Fedora 32 + test: fedora32 + - name: openSUSE 15 py2 + test: opensuse15py2 + - name: openSUSE 15 py3 + test: opensuse15 + - name: Ubuntu 16.04 + test: ubuntu1604 + - name: Ubuntu 18.04 + test: ubuntu1804 + groups: + - 1 + - 2 + - 3 + - stage: Docker_2_10 + displayName: Docker 2.10 + dependsOn: [] + jobs: + - template: templates/matrix.yml + parameters: + testFormat: 2.10/linux/{0} + targets: + - name: CentOS 8 + test: centos8 + - name: Fedora 32 + test: fedora32 + - name: openSUSE 15 py3 + test: opensuse15 + groups: + - 2 + - 3 + - stage: Docker_2_9 + displayName: Docker 2.9 + dependsOn: [] + jobs: + - template: templates/matrix.yml + parameters: + testFormat: 2.9/linux/{0} + targets: + - name: CentOS 8 + test: centos8 + - name: Fedora 31 + test: fedora31 + - name: openSUSE 15 py3 + test: opensuse15 + groups: + - 2 + - 3 + +### Cloud + - stage: Cloud_devel + displayName: Cloud devel + dependsOn: [] + jobs: + - template: templates/matrix.yml + parameters: + nameFormat: Python {0} + testFormat: devel/cloud/{0}/1 + targets: + - test: 2.7 + - test: 3.6 + - stage: Cloud_2_10 + displayName: Cloud 2.10 + dependsOn: [] + jobs: + - template: templates/matrix.yml + parameters: + nameFormat: Python 3.6 {0} + testFormat: 2.10/cloud/3.6/{0}/1 + targets: + - test: '' + - stage: Cloud_2_9 + displayName: Cloud 2.9 + dependsOn: [] + jobs: + - template: templates/matrix.yml + parameters: + nameFormat: Python 3.6 {0} + testFormat: 2.9/cloud/3.6/{0}/1 + targets: + - test: '' + - stage: Summary + condition: succeededOrFailed() + dependsOn: + - Sanity_devel + - Sanity_2_9 + - Sanity_2_10 + - Units_devel + - Units_2_9 + - Units_2_10 + - Remote_devel + - Remote_2_9 + - Remote_2_10 + - Docker_devel + - Docker_2_9 + - Docker_2_10 + - Cloud_devel + - Cloud_2_9 + - Cloud_2_10 + jobs: + - template: templates/coverage.yml diff --git a/.azure-pipelines/scripts/aggregate-coverage.sh b/.azure-pipelines/scripts/aggregate-coverage.sh new file mode 100755 index 00000000000..f3113dd0a9f --- /dev/null +++ b/.azure-pipelines/scripts/aggregate-coverage.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Aggregate code coverage results for later processing. + +set -o pipefail -eu + +agent_temp_directory="$1" + +PATH="${PWD}/bin:${PATH}" + +mkdir "${agent_temp_directory}/coverage/" + +options=(--venv --venv-system-site-packages --color -v) + +ansible-test coverage combine --export "${agent_temp_directory}/coverage/" "${options[@]}" + +if ansible-test coverage analyze targets generate --help >/dev/null 2>&1; then + # Only analyze coverage if the installed version of ansible-test supports it. + # Doing so allows this script to work unmodified for multiple Ansible versions. + ansible-test coverage analyze targets generate "${agent_temp_directory}/coverage/coverage-analyze-targets.json" "${options[@]}" +fi diff --git a/.azure-pipelines/scripts/combine-coverage.py b/.azure-pipelines/scripts/combine-coverage.py new file mode 100755 index 00000000000..506ade6460c --- /dev/null +++ b/.azure-pipelines/scripts/combine-coverage.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +""" +Combine coverage data from multiple jobs, keeping the data only from the most recent attempt from each job. +Coverage artifacts must be named using the format: "Coverage $(System.JobAttempt) {StableUniqueNameForEachJob}" +The recommended coverage artifact name format is: Coverage $(System.JobAttempt) $(System.StageDisplayName) $(System.JobDisplayName) +Keep in mind that Azure Pipelines does not enforce unique job display names (only names). +It is up to pipeline authors to avoid name collisions when deviating from the recommended format. +""" + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import re +import shutil +import sys + + +def main(): + """Main program entry point.""" + source_directory = sys.argv[1] + + if '/ansible_collections/' in os.getcwd(): + output_path = "tests/output" + else: + output_path = "test/results" + + destination_directory = os.path.join(output_path, 'coverage') + + if not os.path.exists(destination_directory): + os.makedirs(destination_directory) + + jobs = {} + count = 0 + + for name in os.listdir(source_directory): + match = re.search('^Coverage (?P[0-9]+) (?P