From e286d768c0c83e01cb2fd87ec5e7e4a0c7d3d905 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 19 Apr 2025 13:07:53 +0200 Subject: [PATCH] Use antsibull-nox for CI. (#361) --- .github/workflows/ansible-test.yml | 125 ------------------ .github/workflows/extra-tests.yml | 52 -------- .github/workflows/import-galaxy.yml | 20 --- .github/workflows/{reuse.yml => nox.yml} | 28 ++-- README.md | 2 +- antsibull-nox.toml | 57 ++++++++ noxfile.py | 50 +++++++ tests/sanity/extra/action-group.json | 12 -- tests/sanity/extra/action-group.json.license | 3 - tests/sanity/extra/action-group.py | 119 ----------------- tests/sanity/extra/extra-docs.json | 13 -- tests/sanity/extra/extra-docs.json.license | 3 - tests/sanity/extra/extra-docs.py | 29 ---- tests/sanity/extra/licenses.json | 4 - tests/sanity/extra/licenses.json.license | 3 - tests/sanity/extra/licenses.py | 110 --------------- tests/sanity/extra/licenses.py.license | 3 - tests/sanity/extra/no-unwanted-files.json | 7 - .../extra/no-unwanted-files.json.license | 3 - tests/sanity/extra/no-unwanted-files.py | 58 -------- tests/sanity/extra/update-docs.json | 11 -- tests/sanity/extra/update-docs.json.license | 3 - tests/sanity/extra/update-docs.py | 21 --- tests/sanity/ignore-2.10.txt | 12 +- tests/sanity/ignore-2.11.txt | 12 +- tests/sanity/ignore-2.12.txt | 2 +- tests/sanity/ignore-2.13.txt | 2 +- tests/sanity/ignore-2.14.txt | 2 +- tests/sanity/ignore-2.15.txt | 2 +- tests/sanity/ignore-2.16.txt | 2 +- tests/sanity/ignore-2.17.txt | 2 +- tests/sanity/ignore-2.18.txt | 2 +- tests/sanity/ignore-2.19.txt | 2 +- tests/sanity/ignore-2.9.txt | 12 +- tests/unit/requirements.yml | 2 +- update-docs.py => tests/update-docs.py | 29 ++-- 36 files changed, 168 insertions(+), 651 deletions(-) delete mode 100644 .github/workflows/ansible-test.yml delete mode 100644 .github/workflows/extra-tests.yml delete mode 100644 .github/workflows/import-galaxy.yml rename .github/workflows/{reuse.yml => nox.yml} (50%) create mode 100644 antsibull-nox.toml create mode 100644 noxfile.py delete mode 100644 tests/sanity/extra/action-group.json delete mode 100644 tests/sanity/extra/action-group.json.license delete mode 100755 tests/sanity/extra/action-group.py delete mode 100644 tests/sanity/extra/extra-docs.json delete mode 100644 tests/sanity/extra/extra-docs.json.license delete mode 100755 tests/sanity/extra/extra-docs.py delete mode 100644 tests/sanity/extra/licenses.json delete mode 100644 tests/sanity/extra/licenses.json.license delete mode 100755 tests/sanity/extra/licenses.py delete mode 100644 tests/sanity/extra/licenses.py.license delete mode 100644 tests/sanity/extra/no-unwanted-files.json delete mode 100644 tests/sanity/extra/no-unwanted-files.json.license delete mode 100755 tests/sanity/extra/no-unwanted-files.py delete mode 100644 tests/sanity/extra/update-docs.json delete mode 100644 tests/sanity/extra/update-docs.json.license delete mode 100644 tests/sanity/extra/update-docs.py rename update-docs.py => tests/update-docs.py (62%) mode change 100755 => 100644 diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml deleted file mode 100644 index 45ca774..0000000 --- a/.github/workflows/ansible-test.yml +++ /dev/null @@ -1,125 +0,0 @@ ---- -# Copyright (c) Ansible Project -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) -# SPDX-License-Identifier: GPL-3.0-or-later - -# For the comprehensive list of the inputs supported by the ansible-community/ansible-test-gh-action GitHub Action, see -# https://github.com/marketplace/actions/ansible-test - -name: CI -on: - # Run CI against all pushes (direct commits, also merged PRs), Pull Requests - push: - branches: - - main - - stable-* - pull_request: - # Run CI once per day (at 05:15 UTC) - schedule: - - cron: '15 5 * * *' - -jobs: - sanity: - name: Sanity (Ⓐ${{ matrix.ansible }}) - strategy: - matrix: - ansible: - # It's important that Sanity is tested against all stable-X.Y branches - # Testing against `devel` may fail as new tests are added. - - stable-2.15 - - stable-2.16 - - stable-2.17 - - stable-2.18 - - devel - runs-on: ubuntu-latest - steps: - - name: Perform sanity testing - uses: felixfontein/ansible-test-gh-action@main - with: - ansible-core-github-repository-slug: ${{ contains(fromJson('["stable-2.14"]'), matrix.ansible) && 'ansible-community/eol-ansible' || 'ansible/ansible' }} - ansible-core-version: ${{ matrix.ansible }} - codecov-token: ${{ secrets.CODECOV_TOKEN }} - testing-type: sanity - test-deps: >- - git+https://github.com/ansible-collections/ansible.utils.git,main - git+https://github.com/ansible-collections/ansible.netcommon.git,main - git+https://github.com/ansible-collections/community.internal_test_tools.git,main - - units: - runs-on: ubuntu-latest - name: Units (Ⓐ${{ matrix.ansible }}) - strategy: - # As soon as the first unit test fails, cancel the others to free up the CI queue - fail-fast: true - matrix: - ansible: - - stable-2.16 - - stable-2.17 - - stable-2.18 - - devel - - steps: - - name: >- - Perform unit testing against - Ansible version ${{ matrix.ansible }} - uses: felixfontein/ansible-test-gh-action@main - with: - ansible-core-github-repository-slug: ${{ contains(fromJson('["stable-2.14"]'), matrix.ansible) && 'ansible-community/eol-ansible' || 'ansible/ansible' }} - ansible-core-version: ${{ matrix.ansible }} - codecov-token: ${{ secrets.CODECOV_TOKEN }} - testing-type: units - test-deps: >- - git+https://github.com/ansible-collections/ansible.utils.git,main - git+https://github.com/ansible-collections/ansible.netcommon.git,main - git+https://github.com/ansible-collections/community.internal_test_tools.git,main - - integration: - runs-on: ubuntu-latest - name: I (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }}) - strategy: - fail-fast: false - matrix: - ansible: - - devel - python: - - "3.10" - - "3.11" - - "3.12" - - "3.13" - include: - # 2.15 - - ansible: stable-2.15 - python: "2.7" - - ansible: stable-2.15 - python: "3.6" - - ansible: stable-2.15 - python: "3.7" - # 2.16 - - ansible: stable-2.16 - python: "3.10" - # 2.17 - - ansible: stable-2.17 - python: "3.8" - # 2.18 - - ansible: stable-2.18 - python: "3.9" - - steps: - - name: >- - Perform integration testing against - Ansible version ${{ matrix.ansible }} - under Python ${{ matrix.python }} - uses: felixfontein/ansible-test-gh-action@main - with: - ansible-core-github-repository-slug: ${{ contains(fromJson('["stable-2.14"]'), matrix.ansible) && 'ansible-community/eol-ansible' || 'ansible/ansible' }} - ansible-core-version: ${{ matrix.ansible }} - codecov-token: ${{ secrets.CODECOV_TOKEN }} - integration-continue-on-error: 'false' - integration-diff: 'false' - integration-retry-on-error: 'true' - test-deps: >- - git+https://github.com/ansible-collections/ansible.utils.git,main - git+https://github.com/ansible-collections/ansible.netcommon.git,main - git+https://github.com/ansible-collections/community.internal_test_tools.git,main - target-python-version: ${{ matrix.python }} - testing-type: integration diff --git a/.github/workflows/extra-tests.yml b/.github/workflows/extra-tests.yml deleted file mode 100644 index 0fda77b..0000000 --- a/.github/workflows/extra-tests.yml +++ /dev/null @@ -1,52 +0,0 @@ ---- -# Copyright (c) Ansible Project -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) -# SPDX-License-Identifier: GPL-3.0-or-later - -name: extra-tests -on: - # Run CI against all pushes (direct commits, also merged PRs), Pull Requests - push: - branches: - - main - - stable-* - pull_request: - # Run CI once per day (at 05:15 UTC) - # This ensures that even if there haven't been commits that we are still testing against latest version of ansible-test for each ansible-base version - schedule: - - cron: '15 5 * * *' -env: - NAMESPACE: community - COLLECTION_NAME: routeros - -jobs: - extra-sanity: - name: Extra Sanity - runs-on: ubuntu-latest - steps: - - - name: Check out code - uses: actions/checkout@v4 - with: - path: ansible_collections/${{env.NAMESPACE}}/${{env.COLLECTION_NAME}} - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install ansible-core - run: pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check - - - name: Install collection dependencies - run: | - git clone --depth=1 --single-branch https://github.com/ansible-collections/community.internal_test_tools.git ./ansible_collections/community/internal_test_tools - git clone --depth=1 --single-branch https://github.com/ansible-collections/ansible.netcommon.git ./ansible_collections/ansible/netcommon - git clone --depth=1 --single-branch https://github.com/ansible-collections/ansible.utils.git ./ansible_collections/ansible/utils - # NOTE: we're installing with git to work around Galaxy being a huge PITA (https://github.com/ansible/galaxy/issues/2429) - # run: ansible-galaxy collection install community.internal_test_tools ansible.netcommon -p . - - - name: Run sanity tests - run: ../../community/internal_test_tools/tools/run.py --color - working-directory: ./ansible_collections/${{env.NAMESPACE}}/${{env.COLLECTION_NAME}} diff --git a/.github/workflows/import-galaxy.yml b/.github/workflows/import-galaxy.yml deleted file mode 100644 index 0c0ee40..0000000 --- a/.github/workflows/import-galaxy.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -# Copyright (c) Ansible Project -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) -# SPDX-License-Identifier: GPL-3.0-or-later - -name: import-galaxy -'on': - # Run CI against all pushes (direct commits, also merged PRs) to main, and all Pull Requests - push: - branches: - - main - - stable-* - pull_request: - -jobs: - import-galaxy: - permissions: - contents: read - name: Test to import built collection artifact with Galaxy importer - uses: ansible-community/github-action-test-galaxy-import/.github/workflows/test-galaxy-import.yml@main diff --git a/.github/workflows/reuse.yml b/.github/workflows/nox.yml similarity index 50% rename from .github/workflows/reuse.yml rename to .github/workflows/nox.yml index 78cc533..43cce99 100644 --- a/.github/workflows/reuse.yml +++ b/.github/workflows/nox.yml @@ -3,31 +3,33 @@ # GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) # SPDX-License-Identifier: GPL-3.0-or-later -name: Verify REUSE - -on: +name: nox +'on': push: branches: - main - stable-* pull_request: - branches: - - main - - stable-* # Run CI once per day (at 05:15 UTC) schedule: - cron: '15 5 * * *' + workflow_dispatch: jobs: - check: - permissions: - contents: read + nox: runs-on: ubuntu-latest - + name: "Run extra sanity tests" steps: - - uses: actions/checkout@v4 + - name: Check out collection + uses: actions/checkout@v4 with: persist-credentials: false + - name: Run nox + uses: ansible-community/antsibull-nox@main - - name: REUSE Compliance Check - uses: fsfe/reuse-action@v5 + ansible-test: + uses: ansible-community/antsibull-nox/.github/workflows/reusable-nox-matrix.yml@main + with: + upload-codecov: true + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/README.md b/README.md index bdea3c3..2d6aec1 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ SPDX-License-Identifier: GPL-3.0-or-later # Community RouterOS Collection [![Documentation](https://img.shields.io/badge/docs-brightgreen.svg)](https://docs.ansible.com/ansible/devel/collections/community/routeros/) -[![CI](https://github.com/ansible-collections/community.routeros/actions/workflows/ansible-test.yml/badge.svg?branch=main)](https://github.com/ansible-collections/community.routeros/actions) +[![CI](https://github.com/ansible-collections/community.routeros/actions/workflows/nox.yml/badge.svg?branch=main)](https://github.com/ansible-collections/community.routeros/actions) [![Codecov](https://img.shields.io/codecov/c/github/ansible-collections/community.routeros)](https://codecov.io/gh/ansible-collections/community.routeros) [![REUSE status](https://api.reuse.software/badge/github.com/ansible-collections/community.routeros)](https://api.reuse.software/info/github.com/ansible-collections/community.routeros) diff --git a/antsibull-nox.toml b/antsibull-nox.toml new file mode 100644 index 0000000..6d73110 --- /dev/null +++ b/antsibull-nox.toml @@ -0,0 +1,57 @@ +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +[collection_sources] +"community.internal_test_tools" = "git+https://github.com/ansible-collections/community.internal_test_tools.git,main" +"community.netcommon" = "git+https://github.com/ansible-collections/ansible.netcommon.git,main" +"community.utils" = "git+https://github.com/ansible-collections/ansible.utils.git,main" + +[sessions] + +[sessions.lint] +# https://ansible.readthedocs.io/projects/antsibull-nox/config-file/#basic-linting-sessions + +# disable reformatting for now +run_isort = false +run_black = false + +# disable most linters +run_flake8 = false +run_pylint = false +run_yamllint = false +run_mypy = false + +[sessions.docs_check] +validate_collection_refs="all" + +[sessions.extra_checks] +run_no_unwanted_files = true +no_unwanted_files_module_extensions = [".py"] +no_unwanted_files_yaml_extensions = [".yml"] +run_action_groups = true + +[[sessions.extra_checks.action_groups_config]] +name = "api" +pattern = "^api.*$" +exclusions = [] +doc_fragment = "community.routeros.attributes.actiongroup_api" + +[sessions.build_import_check] +run_galaxy_importer = true + +[sessions.ansible_test_sanity] +include_devel = true + +[sessions.ansible_test_units] +include_devel = true + +[sessions.ansible_test_integration_w_default_container] +include_devel = true +controller_python_versions_only = true + +[sessions.ansible_test_integration_w_default_container.core_python_versions] +"2.15" = ["2.7", "3.6", "3.7"] +"2.16" = ["3.10"] +"2.17" = ["3.8"] +"2.18" = ["3.9"] diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000..d8c5173 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,50 @@ + +# The following metadata allows Python runners and nox to install the required +# dependencies for running this Python script: +# +# /// script +# dependencies = ["nox>=2025.02.09", "antsibull-nox"] +# /// + +import os +import sys + +import nox + + +# We try to import antsibull-nox, and if that doesn't work, provide a more useful +# error message to the user. +try: + import antsibull_nox +except ImportError: + print("You need to install antsibull-nox in the same Python environment as nox.") + sys.exit(1) + + +IN_CI = os.environ.get("CI") == "true" + + +antsibull_nox.load_antsibull_nox_toml() + + +@nox.session(name="update-docs", default=True) +def update_docs_fragments(session: nox.Session) -> None: + """ + Update/check auto-generated parts of docs fragments. + """ + session.install("ansible-core") + prepare = antsibull_nox.sessions.prepare_collections( + session, install_in_site_packages=True + ) + if not prepare: + return + data = ["python", "tests/update-docs.py"] + if IN_CI: + data.append("--lint") + session.run(*data) + + +# Allow to run the noxfile with `python noxfile.py`, `pipx run noxfile.py`, or similar. +# Requires nox >= 2025.02.09 +if __name__ == "__main__": + nox.main() diff --git a/tests/sanity/extra/action-group.json b/tests/sanity/extra/action-group.json deleted file mode 100644 index db6a92b..0000000 --- a/tests/sanity/extra/action-group.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "include_symlinks": true, - "prefixes": [ - "meta/runtime.yml", - "plugins/modules/", - "tests/sanity/extra/action-group." - ], - "output": "path-message", - "requirements": [ - "pyyaml" - ] -} diff --git a/tests/sanity/extra/action-group.json.license b/tests/sanity/extra/action-group.json.license deleted file mode 100644 index edff8c7..0000000 --- a/tests/sanity/extra/action-group.json.license +++ /dev/null @@ -1,3 +0,0 @@ -GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) -SPDX-License-Identifier: GPL-3.0-or-later -SPDX-FileCopyrightText: Ansible Project diff --git a/tests/sanity/extra/action-group.py b/tests/sanity/extra/action-group.py deleted file mode 100755 index cc00945..0000000 --- a/tests/sanity/extra/action-group.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2024, Felix Fontein -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) -# SPDX-License-Identifier: GPL-3.0-or-later -"""Make sure all modules that should show up in the action group.""" - -from __future__ import annotations - -import os -import re -import yaml - - -ACTION_GROUPS = { - # The format is as follows: - # * 'pattern': a regular expression matching all module names potentially belonging to the action group; - # * 'exclusions': a list of modules that are not part of the action group; all other modules matching 'pattern' must be part of it; - # * 'doc_fragment': the docs fragment that documents membership of the action group. - 'api': { - 'pattern': re.compile('^api.*$'), - 'exclusions': [], - 'doc_fragment': 'community.routeros.attributes.actiongroup_api', - }, -} - - -def main(): - """Main entry point.""" - - # Load redirects - meta_runtime = 'meta/runtime.yml' - self_path = 'tests/sanity/extra/action-group.py' - try: - with open(meta_runtime, 'rb') as f: - data = yaml.safe_load(f) - action_groups = data['action_groups'] - except Exception as exc: - print(f'{meta_runtime}: cannot load action groups: {exc}') - return - - for action_group in action_groups: - if action_group not in ACTION_GROUPS: - print(f'{meta_runtime}: found unknown action group {action_group!r}; likely {self_path} needs updating') - for action_group, action_group_data in list(ACTION_GROUPS.items()): - if action_group not in action_groups: - print(f'{meta_runtime}: cannot find action group {action_group!r}; likely {self_path} needs updating') - - modules_directory = 'plugins/modules/' - modules_suffix = '.py' - - for file in os.listdir(modules_directory): - if not file.endswith(modules_suffix): - continue - module_name = file[:-len(modules_suffix)] - - for action_group, action_group_data in ACTION_GROUPS.items(): - action_group_content = action_groups.get(action_group) or [] - path = os.path.join(modules_directory, file) - - if not action_group_data['pattern'].match(module_name): - if module_name in action_group_content: - print(f'{path}: module is in action group {action_group!r} despite not matching its pattern as defined in {self_path}') - continue - - should_be_in_action_group = module_name not in action_group_data['exclusions'] - - if should_be_in_action_group: - if module_name not in action_group_content: - print(f'{meta_runtime}: module {module_name!r} is not part of {action_group!r} action group') - else: - action_group_content.remove(module_name) - - documentation = [] - in_docs = False - with open(path, 'r', encoding='utf-8') as f: - for line in f: - if line.startswith('DOCUMENTATION ='): - in_docs = True - elif line.startswith(("'''", '"""')) and in_docs: - in_docs = False - elif in_docs: - documentation.append(line) - if in_docs: - print(f'{path}: cannot find DOCUMENTATION end') - if not documentation: - print(f'{path}: cannot find DOCUMENTATION') - continue - - try: - docs = yaml.safe_load('\n'.join(documentation)) - if not isinstance(docs, dict): - raise Exception('is not a top-level dictionary') - except Exception as exc: - print(f'{path}: cannot load DOCUMENTATION as YAML: {exc}') - continue - - docs_fragments = docs.get('extends_documentation_fragment') or [] - is_in_action_group = action_group_data['doc_fragment'] in docs_fragments - - if should_be_in_action_group != is_in_action_group: - if should_be_in_action_group: - print( - f'{path}: module does not document itself as part of action group {action_group!r}, but it should;' - f' you need to add {action_group_data["doc_fragment"]} to "extends_documentation_fragment" in DOCUMENTATION' - ) - else: - print(f'{path}: module documents itself as part of action group {action_group!r}, but it should not be') - - for action_group, action_group_data in ACTION_GROUPS.items(): - action_group_content = action_groups.get(action_group) or [] - for module_name in action_group_content: - print( - f'{meta_runtime}: module {module_name} mentioned in {action_group!r} action group' - f' does not exist or does not match pattern defined in {self_path}' - ) - - -if __name__ == '__main__': - main() diff --git a/tests/sanity/extra/extra-docs.json b/tests/sanity/extra/extra-docs.json deleted file mode 100644 index 9a28d17..0000000 --- a/tests/sanity/extra/extra-docs.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "include_symlinks": false, - "prefixes": [ - "docs/docsite/", - "plugins/", - "roles/" - ], - "output": "path-line-column-message", - "requirements": [ - "ansible-core", - "antsibull-docs" - ] -} diff --git a/tests/sanity/extra/extra-docs.json.license b/tests/sanity/extra/extra-docs.json.license deleted file mode 100644 index edff8c7..0000000 --- a/tests/sanity/extra/extra-docs.json.license +++ /dev/null @@ -1,3 +0,0 @@ -GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) -SPDX-License-Identifier: GPL-3.0-or-later -SPDX-FileCopyrightText: Ansible Project diff --git a/tests/sanity/extra/extra-docs.py b/tests/sanity/extra/extra-docs.py deleted file mode 100755 index 251e6d7..0000000 --- a/tests/sanity/extra/extra-docs.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) Ansible Project -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) -# SPDX-License-Identifier: GPL-3.0-or-later -"""Check extra collection docs with antsibull-docs.""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -import os -import sys -import subprocess - - -def main(): - """Main entry point.""" - env = os.environ.copy() - suffix = ':{env}'.format(env=env["ANSIBLE_COLLECTIONS_PATH"]) if 'ANSIBLE_COLLECTIONS_PATH' in env else '' - env['ANSIBLE_COLLECTIONS_PATH'] = '{root}{suffix}'.format(root=os.path.dirname(os.path.dirname(os.path.dirname(os.getcwd()))), suffix=suffix) - p = subprocess.run( - ['antsibull-docs', 'lint-collection-docs', '--plugin-docs', '--skip-rstcheck', '.'], - env=env, - check=False, - ) - if p.returncode not in (0, 3): - print('{0}:0:0: unexpected return code {1}'.format(sys.argv[0], p.returncode)) - - -if __name__ == '__main__': - main() diff --git a/tests/sanity/extra/licenses.json b/tests/sanity/extra/licenses.json deleted file mode 100644 index 50e47ca..0000000 --- a/tests/sanity/extra/licenses.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "include_symlinks": false, - "output": "path-message" -} diff --git a/tests/sanity/extra/licenses.json.license b/tests/sanity/extra/licenses.json.license deleted file mode 100644 index edff8c7..0000000 --- a/tests/sanity/extra/licenses.json.license +++ /dev/null @@ -1,3 +0,0 @@ -GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) -SPDX-License-Identifier: GPL-3.0-or-later -SPDX-FileCopyrightText: Ansible Project diff --git a/tests/sanity/extra/licenses.py b/tests/sanity/extra/licenses.py deleted file mode 100755 index 80eb795..0000000 --- a/tests/sanity/extra/licenses.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2022, Felix Fontein -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) -# SPDX-License-Identifier: GPL-3.0-or-later -"""Prevent files without a correct license identifier from being added to the source tree.""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -import os -import glob -import sys - - -def format_license_list(licenses): - if not licenses: - return '(empty)' - return ', '.join(['"%s"' % license for license in licenses]) - - -def find_licenses(filename, relax=False): - spdx_license_identifiers = [] - other_license_identifiers = [] - has_copyright = False - try: - with open(filename, 'r', encoding='utf-8') as f: - for line in f: - line = line.rstrip() - if 'Copyright ' in line: - has_copyright = True - if 'Copyright: ' in line: - print('%s: found copyright line with "Copyright:". Please remove the colon.' % (filename, )) - if 'SPDX-FileCopyrightText: ' in line: - has_copyright = True - idx = line.find('SPDX-License-Identifier: ') - if idx >= 0: - lic_id = line[idx + len('SPDX-License-Identifier: '):] - spdx_license_identifiers.extend(lic_id.split(' OR ')) - if 'GNU General Public License' in line: - if 'v3.0+' in line: - other_license_identifiers.append('GPL-3.0-or-later') - if 'version 3 or later' in line: - other_license_identifiers.append('GPL-3.0-or-later') - if 'Simplified BSD License' in line: - other_license_identifiers.append('BSD-2-Clause') - if 'Apache License 2.0' in line: - other_license_identifiers.append('Apache-2.0') - if 'PSF License' in line or 'Python-2.0' in line: - other_license_identifiers.append('PSF-2.0') - if 'MIT License' in line: - other_license_identifiers.append('MIT') - except Exception as exc: - print('%s: error while processing file: %s' % (filename, exc)) - if len(set(spdx_license_identifiers)) < len(spdx_license_identifiers): - print('%s: found identical SPDX-License-Identifier values' % (filename, )) - if other_license_identifiers and set(other_license_identifiers) != set(spdx_license_identifiers): - print('%s: SPDX-License-Identifier yielded the license list %s, while manual guessing yielded the license list %s' % ( - filename, format_license_list(spdx_license_identifiers), format_license_list(other_license_identifiers))) - if not has_copyright and not relax: - print('%s: found no copyright notice' % (filename, )) - return sorted(spdx_license_identifiers) - - -def main(): - """Main entry point.""" - paths = sys.argv[1:] or sys.stdin.read().splitlines() - - # The following paths are allowed to have no license identifier - no_comments_allowed = [ - 'changelogs/fragments/*.yml', - 'changelogs/fragments/*.yaml', - ] - - # These files are completely ignored - ignore_paths = [ - '.ansible-test-timeout.json', - '.reuse/dep5', - 'LICENSES/*.txt', - 'COPYING', - ] - - no_comments_allowed = [fn for pattern in no_comments_allowed for fn in glob.glob(pattern)] - ignore_paths = [fn for pattern in ignore_paths for fn in glob.glob(pattern)] - - valid_licenses = [license_file[len('LICENSES/'):-len('.txt')] for license_file in glob.glob('LICENSES/*.txt')] - - for path in paths: - if path.startswith('./'): - path = path[2:] - if path in ignore_paths or path.startswith('tests/output/'): - continue - if os.stat(path).st_size == 0: - continue - if not path.endswith('.license') and os.path.exists(path + '.license'): - path = path + '.license' - valid_licenses_for_path = valid_licenses - if path.startswith('plugins/') and not path.startswith(('plugins/modules/', 'plugins/module_utils/')): - valid_licenses_for_path = [license for license in valid_licenses if license == 'GPL-3.0-or-later'] - licenses = find_licenses(path, relax=path in no_comments_allowed) - if not licenses: - if path not in no_comments_allowed: - print('%s: must have at least one license' % (path, )) - else: - for license in licenses: - if license not in valid_licenses_for_path: - print('%s: found not allowed license "%s", must be one of %s' % ( - path, license, format_license_list(valid_licenses_for_path))) - - -if __name__ == '__main__': - main() diff --git a/tests/sanity/extra/licenses.py.license b/tests/sanity/extra/licenses.py.license deleted file mode 100644 index 6c4958f..0000000 --- a/tests/sanity/extra/licenses.py.license +++ /dev/null @@ -1,3 +0,0 @@ -GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) -SPDX-License-Identifier: GPL-3.0-or-later -SPDX-FileCopyrightText: 2022, Felix Fontein diff --git a/tests/sanity/extra/no-unwanted-files.json b/tests/sanity/extra/no-unwanted-files.json deleted file mode 100644 index c789a7f..0000000 --- a/tests/sanity/extra/no-unwanted-files.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "include_symlinks": true, - "prefixes": [ - "plugins/" - ], - "output": "path-message" -} diff --git a/tests/sanity/extra/no-unwanted-files.json.license b/tests/sanity/extra/no-unwanted-files.json.license deleted file mode 100644 index edff8c7..0000000 --- a/tests/sanity/extra/no-unwanted-files.json.license +++ /dev/null @@ -1,3 +0,0 @@ -GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) -SPDX-License-Identifier: GPL-3.0-or-later -SPDX-FileCopyrightText: Ansible Project diff --git a/tests/sanity/extra/no-unwanted-files.py b/tests/sanity/extra/no-unwanted-files.py deleted file mode 100755 index b39df83..0000000 --- a/tests/sanity/extra/no-unwanted-files.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) Ansible Project -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) -# SPDX-License-Identifier: GPL-3.0-or-later -"""Prevent unwanted files from being added to the source tree.""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -import os -import os.path -import sys - - -def main(): - """Main entry point.""" - paths = sys.argv[1:] or sys.stdin.read().splitlines() - - allowed_extensions = ( - '.cs', - '.ps1', - '.psm1', - '.py', - ) - - skip_paths = set([ - ]) - - skip_directories = ( - ) - - yaml_directories = ( - 'plugins/test/', - 'plugins/filter/', - ) - - for path in paths: - if path in skip_paths: - continue - - if any(path.startswith(skip_directory) for skip_directory in skip_directories): - continue - - if os.path.islink(path): - print('%s: is a symbolic link' % (path, )) - elif not os.path.isfile(path): - print('%s: is not a regular file' % (path, )) - - ext = os.path.splitext(path)[1] - - if ext in ('.yml', ) and any(path.startswith(yaml_directory) for yaml_directory in yaml_directories): - continue - - if ext not in allowed_extensions: - print('%s: extension must be one of: %s' % (path, ', '.join(allowed_extensions))) - - -if __name__ == '__main__': - main() diff --git a/tests/sanity/extra/update-docs.json b/tests/sanity/extra/update-docs.json deleted file mode 100644 index 3a63af8..0000000 --- a/tests/sanity/extra/update-docs.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "include_symlinks": false, - "prefixes": [ - "docs/docsite/rst/api-guide.rst", - "plugins/modules/" - ], - "output": "path-line-column-message", - "requirements": [ - "ansible-core" - ] -} diff --git a/tests/sanity/extra/update-docs.json.license b/tests/sanity/extra/update-docs.json.license deleted file mode 100644 index edff8c7..0000000 --- a/tests/sanity/extra/update-docs.json.license +++ /dev/null @@ -1,3 +0,0 @@ -GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) -SPDX-License-Identifier: GPL-3.0-or-later -SPDX-FileCopyrightText: Ansible Project diff --git a/tests/sanity/extra/update-docs.py b/tests/sanity/extra/update-docs.py deleted file mode 100644 index 6015512..0000000 --- a/tests/sanity/extra/update-docs.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) Ansible Project -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) -# SPDX-License-Identifier: GPL-3.0-or-later -"""Check whether update-docs.py modifies something.""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -import sys -import subprocess - - -def main(): - """Main entry point.""" - p = subprocess.run([sys.executable, 'update-docs.py'], check=False) - if p.returncode not in (0, 1): - print('{0}:0:0: unexpected return code {1}'.format(sys.argv[0], p.returncode)) - - -if __name__ == '__main__': - main() diff --git a/tests/sanity/ignore-2.10.txt b/tests/sanity/ignore-2.10.txt index ece5b29..50c92fb 100644 --- a/tests/sanity/ignore-2.10.txt +++ b/tests/sanity/ignore-2.10.txt @@ -1,9 +1,9 @@ docs/docsite/rst/api-guide.rst rstcheck docs/docsite/rst/quoting.rst rstcheck docs/docsite/rst/ssh-guide.rst rstcheck -update-docs.py compile-2.6 -update-docs.py compile-2.7 -update-docs.py compile-3.5 -update-docs.py future-import-boilerplate -update-docs.py metaclass-boilerplate -update-docs.py shebang +tests/update-docs.py compile-2.6 +tests/update-docs.py compile-2.7 +tests/update-docs.py compile-3.5 +tests/update-docs.py future-import-boilerplate +tests/update-docs.py metaclass-boilerplate +tests/update-docs.py shebang diff --git a/tests/sanity/ignore-2.11.txt b/tests/sanity/ignore-2.11.txt index 876765a..1c7f5da 100644 --- a/tests/sanity/ignore-2.11.txt +++ b/tests/sanity/ignore-2.11.txt @@ -1,6 +1,6 @@ -update-docs.py compile-2.6 -update-docs.py compile-2.7 -update-docs.py compile-3.5 -update-docs.py future-import-boilerplate -update-docs.py metaclass-boilerplate -update-docs.py shebang +tests/update-docs.py compile-2.6 +tests/update-docs.py compile-2.7 +tests/update-docs.py compile-3.5 +tests/update-docs.py future-import-boilerplate +tests/update-docs.py metaclass-boilerplate +tests/update-docs.py shebang diff --git a/tests/sanity/ignore-2.12.txt b/tests/sanity/ignore-2.12.txt index ce635c3..65e5bca 100644 --- a/tests/sanity/ignore-2.12.txt +++ b/tests/sanity/ignore-2.12.txt @@ -1 +1 @@ -update-docs.py shebang +tests/update-docs.py shebang diff --git a/tests/sanity/ignore-2.13.txt b/tests/sanity/ignore-2.13.txt index ce635c3..65e5bca 100644 --- a/tests/sanity/ignore-2.13.txt +++ b/tests/sanity/ignore-2.13.txt @@ -1 +1 @@ -update-docs.py shebang +tests/update-docs.py shebang diff --git a/tests/sanity/ignore-2.14.txt b/tests/sanity/ignore-2.14.txt index ce635c3..65e5bca 100644 --- a/tests/sanity/ignore-2.14.txt +++ b/tests/sanity/ignore-2.14.txt @@ -1 +1 @@ -update-docs.py shebang +tests/update-docs.py shebang diff --git a/tests/sanity/ignore-2.15.txt b/tests/sanity/ignore-2.15.txt index ce635c3..65e5bca 100644 --- a/tests/sanity/ignore-2.15.txt +++ b/tests/sanity/ignore-2.15.txt @@ -1 +1 @@ -update-docs.py shebang +tests/update-docs.py shebang diff --git a/tests/sanity/ignore-2.16.txt b/tests/sanity/ignore-2.16.txt index ce635c3..65e5bca 100644 --- a/tests/sanity/ignore-2.16.txt +++ b/tests/sanity/ignore-2.16.txt @@ -1 +1 @@ -update-docs.py shebang +tests/update-docs.py shebang diff --git a/tests/sanity/ignore-2.17.txt b/tests/sanity/ignore-2.17.txt index ce635c3..65e5bca 100644 --- a/tests/sanity/ignore-2.17.txt +++ b/tests/sanity/ignore-2.17.txt @@ -1 +1 @@ -update-docs.py shebang +tests/update-docs.py shebang diff --git a/tests/sanity/ignore-2.18.txt b/tests/sanity/ignore-2.18.txt index ce635c3..65e5bca 100644 --- a/tests/sanity/ignore-2.18.txt +++ b/tests/sanity/ignore-2.18.txt @@ -1 +1 @@ -update-docs.py shebang +tests/update-docs.py shebang diff --git a/tests/sanity/ignore-2.19.txt b/tests/sanity/ignore-2.19.txt index ce635c3..65e5bca 100644 --- a/tests/sanity/ignore-2.19.txt +++ b/tests/sanity/ignore-2.19.txt @@ -1 +1 @@ -update-docs.py shebang +tests/update-docs.py shebang diff --git a/tests/sanity/ignore-2.9.txt b/tests/sanity/ignore-2.9.txt index ece5b29..50c92fb 100644 --- a/tests/sanity/ignore-2.9.txt +++ b/tests/sanity/ignore-2.9.txt @@ -1,9 +1,9 @@ docs/docsite/rst/api-guide.rst rstcheck docs/docsite/rst/quoting.rst rstcheck docs/docsite/rst/ssh-guide.rst rstcheck -update-docs.py compile-2.6 -update-docs.py compile-2.7 -update-docs.py compile-3.5 -update-docs.py future-import-boilerplate -update-docs.py metaclass-boilerplate -update-docs.py shebang +tests/update-docs.py compile-2.6 +tests/update-docs.py compile-2.7 +tests/update-docs.py compile-3.5 +tests/update-docs.py future-import-boilerplate +tests/update-docs.py metaclass-boilerplate +tests/update-docs.py shebang diff --git a/tests/unit/requirements.yml b/tests/unit/requirements.yml index 6a22736..107fe12 100644 --- a/tests/unit/requirements.yml +++ b/tests/unit/requirements.yml @@ -4,4 +4,4 @@ # SPDX-License-Identifier: GPL-3.0-or-later collections: -- ansible.netcommon + - community.internal_test_tools diff --git a/update-docs.py b/tests/update-docs.py old mode 100755 new mode 100644 similarity index 62% rename from update-docs.py rename to tests/update-docs.py index 4f164c8..5f28f8e --- a/update-docs.py +++ b/tests/update-docs.py @@ -11,9 +11,6 @@ Updates DOCUMENTATION of modules using module_utils._api_data with the correct l import sys -# Ensure that we can import things from ansible_collections -sys.path.append('../../..') - from ansible_collections.community.routeros.plugins.module_utils._api_data import ( PATHS, join_path, @@ -26,24 +23,34 @@ MODULES = [ ] -def update_file(file, begin_line, end_line, choice_line, path_choices): +def update_file(file: str, begin_line: str, end_line: str, choice_line: str, path_choices: list[str]) -> bool: with open(file, 'r', encoding='utf-8') as f: lines = f.read().splitlines() begin_index = lines.index(begin_line) end_index = lines.index(end_line, begin_index + 1) new_lines = lines[:begin_index + 1] + [choice_line.format(choice=choice) for choice in path_choices] + lines[end_index:] - if lines != new_lines: - print(f'{file} has been updated') - with open(file, 'w', encoding='utf-8') as f: - f.write('\n'.join(new_lines) + '\n') + if lines == new_lines: + return False + print(f'{file} has been updated') + with open(file, 'w', encoding='utf-8') as f: + f.write('\n'.join(new_lines) + '\n') + return True -def main(): +def main(args: list[str]) -> int: path_choices = sorted([join_path(path) for path, path_info in PATHS.items() if path_info.fully_understood]) + changes = False for file in MODULES: - update_file(file, ' # BEGIN PATH LIST', ' # END PATH LIST', ' - {choice}', path_choices) + changes |= update_file(file, ' # BEGIN PATH LIST', ' # END PATH LIST', ' - {choice}', path_choices) + + lint = "--lint" in args + if not lint or not changes: + return 0 + + print("Run 'nox -Re update-docs'!") + return 1 if __name__ == '__main__': - main() + sys.exit(main(sys.argv[1:]))