diff -pruN 4.0-2/.cirrus.yml 4.5.3ubuntu2/.cirrus.yml --- 4.0-2/.cirrus.yml 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/.cirrus.yml 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,249 @@ +--- + +# Main environment vars to set for all tasks +env: + + FEDORA_NAME: "fedora-38-beta" + FEDORA_PRIOR_NAME: "fedora-37" + + DEBIAN_NAME: "debian-11" + + UBUNTU_NAME: "ubuntu-22.04" + UBUNTU_PRIOR_NAME: "ubuntu-20.04" + UBUNTU_PRIOR2_NAME: "ubuntu-18.04" + + CENTOS_9_NAME: "centos-stream-9" + CENTOS_8_NAME: "centos-stream-8" + + CENTOS_PROJECT: "centos-cloud" + DEBIAN_PROJECT: "debian-cloud" + FEDORA_PROJECT: "fedora-cloud" + SOS_PROJECT: "sos-devel-jobs" + UBUNTU_PROJECT: "ubuntu-os-cloud" + + # Images exist on GCP already + CENTOS_9_IMAGE_NAME: "centos-stream-9-v20221102" + CENTOS_8_IMAGE_NAME: "centos-stream-8-v20230306" + DEBIAN_IMAGE_NAME: "debian-11-bullseye-v20230306" + FEDORA_IMAGE_NAME: "fedora-cloud-base-gcp-38-beta-1-3-x86-64" + FEDORA_PRIOR_IMAGE_NAME: "fedora-cloud-base-gcp-37-1-7-x86-64" + UBUNTU_IMAGE_NAME: "ubuntu-2204-jammy-v20230302" + UBUNTU_PRIOR_IMAGE_NAME: "ubuntu-2004-focal-v20230302" + UBUNTU_PRIOR2_IMAGE_NAME: "ubuntu-1804-bionic-v20230324" + UBUNTU_SNAP_IMAGE_NAME: "ubuntu-2204-jammy-v20230302" + + # Curl-command prefix for downloading task artifacts, simply add the + # the url-encoded task name, artifact name, and path as a suffix. + # This approach originally seen in the podman project. + ARTCURL: >- + curl --fail --location -O + --url https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID} + +# Default task timeout +timeout_in: 30m + +# enable auto cancelling concurrent builds on main when multiple PRs are +# merged at once +auto_cancellation: true + +gcp_credentials: ENCRYPTED[!77d4c8251094346c41db63cb05eba2ff98eaff04e58c5d0e2a8e2c6f159f7d601b3fe9a2a4fce1666297e371f2fc8752!] + +# Run a simple lint on the community cluster +flake8_task: + alias: "flake8_test" + name: "Flake8 linting test" + container: + image: alpine/flake8:latest + flake_script: flake8 sos + +# Run a check on newer upstream python versions to check for possible +# breaks/changes in common modules. This is not meant to check any of the actual +# collections or archive integrity. +py_break_task: + alias: "py_break" + name: "Breakage test python-$PY_VERSION" + container: + image: "python:${PY_VERSION}" + matrix: + - env: + PY_VERSION: "latest" + - env: + PY_VERSION: "3.9" + # This image has 2 py environments. Install to the one sos uses. + setup_script: pip3 install -t /usr/lib/python3/dist-packages -r requirements.txt + main_script: ./bin/sos report --batch + +# Make sure a user can manually build an rpm from the checkout +rpm_build_task: + alias: "rpm_build" + name: "rpm Build From Checkout - ${BUILD_NAME}" + gce_instance: &standardvm + image_project: "${PROJECT}" + image_name: "${VM_IMAGE_NAME}" + type: e2-medium + matrix: + - env: ¢os9 + PROJECT: ${CENTOS_PROJECT} + BUILD_NAME: ${CENTOS_9_NAME} + VM_IMAGE_NAME: ${CENTOS_9_IMAGE_NAME} + - env: ¢os8 + PROJECT: ${CENTOS_PROJECT} + BUILD_NAME: ${CENTOS_8_NAME} + VM_IMAGE_NAME: ${CENTOS_8_IMAGE_NAME} + - env: &fedora + PROJECT: ${FEDORA_PROJECT} + BUILD_NAME: ${FEDORA_NAME} + VM_IMAGE_NAME: ${FEDORA_IMAGE_NAME} + - env: &fedoraprior + PROJECT: ${FEDORA_PROJECT} + BUILD_NAME: ${FEDORA_PRIOR_NAME} + VM_IMAGE_NAME: ${FEDORA_PRIOR_IMAGE_NAME} + setup_script: | + dnf clean all + dnf -y install rpm-build rpmdevtools gettext python3-devel + main_script: | + mkdir -p /rpmbuild/{BUILD,BUILDROOT,RPMS,SRPMS,SOURCES} + python3 setup.py sdist + cp dist/sos*.tar.gz /rpmbuild/SOURCES/ + rpmbuild -bs sos.spec + rpmbuild -bb sos.spec + # Retrieving the built rpm in later tasks requires knowing the exact name + # of the file. To avoid having to juggle version numbers here, rename it + prep_artifacts_script: mv /rpmbuild/RPMS/noarch/sos-*.rpm ./sos_${BUILD_NAME}.rpm + packages_artifacts: + path: ./sos_${BUILD_NAME}.rpm + type: application/octet-stream + +# Make sure a user can manually build a snap from the checkout +snap_build_task: + alias: "snap_build" + name: "snap Build From Checkout" + gce_instance: + image_project: "${UBUNTU_PROJECT}" + image_name: "${UBUNTU_SNAP_IMAGE_NAME}" + type: e2-medium + setup_script: | + apt update + apt -y install snapd + systemctl start snapd + sed -i -e 's/adopt-info.*/version: test/g' -e '/set version/d' snap/snapcraft.yaml + snap install snapcraft --classic + main_script: | + snapcraft --destructive-mode + packages_artifacts: + path: "*.snap" + on_failure: + fail_script: | + ls -d /root/.cache/snapcraft/log 2> /dev/null | xargs tar cf snap-build-fail-logs.tar + log_artifacts: + path: "snap-build-fail-logs.tar" + +# Run the stage one (no mocking) tests across all distros on GCP +report_stageone_task: + alias: "stageone_report" + name: "Report Stage One - $BUILD_NAME" + depends_on: + - rpm_build + - snap_build + gce_instance: *standardvm + matrix: + - env: *centos9 + - env: *centos8 + - env: *fedora + - env: *fedoraprior + - env: &ubuntu + PROJECT: ${UBUNTU_PROJECT} + BUILD_NAME: ${UBUNTU_NAME} + VM_IMAGE_NAME: ${UBUNTU_IMAGE_NAME} + - env: &ubuntuprior + PROJECT: ${UBUNTU_PROJECT} + BUILD_NAME: ${UBUNTU_PRIOR_NAME} + VM_IMAGE_NAME: ${UBUNTU_PRIOR_IMAGE_NAME} + - env: &ubuntuprior2 + PROJECT: ${UBUNTU_PROJECT} + BUILD_NAME: ${UBUNTU_PRIOR2_NAME} + VM_IMAGE_NAME: ${UBUNTU_PRIOR2_IMAGE_NAME} + setup_script: &setup | + if [ $(command -v apt) ]; then + echo "$ARTCURL/snap%20Build%20From%20Checkout/packages/sosreport_test_amd64.snap" + $ARTCURL/snap%20Build%20From%20Checkout/packages/sosreport_test_amd64.snap + apt -y purge sosreport + apt update --allow-releaseinfo-change + apt -y install python3-pip snapd + systemctl start snapd + snap install ./sosreport_test_amd64.snap --classic --dangerous + snap alias sosreport.sos sos + fi + if [ $(command -v dnf) ]; then + echo "$ARTCURL/rpm%20Build%20From%20Checkout%20-%20${BUILD_NAME}/packages/sos_${BUILD_NAME}.rpm" + $ARTCURL/rpm%20Build%20From%20Checkout%20-%20${BUILD_NAME}/packages/sos_${BUILD_NAME}.rpm + dnf -y remove sos + dnf -y install python3-pip ethtool + dnf -y install ./sos_${BUILD_NAME}.rpm + fi + pip3 install avocado-framework==94.0 + # run the unittests separately as they require a different PYTHONPATH in + # order for the imports to work properly under avocado + unittest_script: PYTHONPATH=. avocado run tests/unittests/ + main_script: PYTHONPATH=tests/ avocado run -p TESTLOCAL=true --test-runner=runner -t stageone tests/{cleaner,collect,report,vendor}_tests + on_failure: + fail_script: &faillogs | + ls -d /var/tmp/avocado* /root/avocado* 2> /dev/null | xargs tar cf sos-fail-logs.tar + log_artifacts: &logs + path: "sos-fail-logs.tar" + +# IFF the stage one tests all pass, then run stage two for latest distros +report_stagetwo_task: + alias: "stagetwo_report" + name: "Report Stage Two - $BUILD_NAME" + depends_on: stageone_report + timeout_in: 45m + gce_instance: *standardvm + matrix: + - env: *centos9 + - env: *centos8 + - env: *fedora + - env: *ubuntu + setup_script: *setup + install_pexpect_script: | + if [ $(command -v apt) ]; then + apt -y install python3-pexpect + fi + if [ $(command -v dnf) ]; then + dnf -y install python3-pexpect + fi + main_script: PYTHONPATH=tests/ avocado run -p TESTLOCAL=true --test-runner=runner -t stagetwo tests/{cleaner,collect,report,vendor}_tests + on_failure: + fail_script: *faillogs + log_artifacts: *logs + +report_foreman_task: + skip: "!changesInclude('.cirrus.yml', '**/{__init__,apache,foreman,foreman_tests,candlepin,pulp,pulpcore}.py')" + timeout_in: 45m + alias: "foreman_integration" + name: "Integration Test - Foreman ${FOREMAN_VER} - ${BUILD_NAME}" + depends_on: stageone_report + gce_instance: &bigvm + <<: *standardvm + type: e2-standard-2 + matrix: + - env: + <<: *centos8 + FOREMAN_VER: "2.5" + - env: + <<: *centos8 + FOREMAN_VER: "3.1" + - env: + <<: *centos8 + FOREMAN_VER: "3.4" + - env: + PROJECT: ${DEBIAN_PROJECT} + VM_IMAGE_NAME: ${DEBIAN_IMAGE_NAME} + BUILD_NAME: ${DEBIAN_NAME} + FOREMAN_VER: "3.4" + setup_script: *setup + foreman_setup_script: ./tests/test_data/foreman_setup.sh + main_script: PYTHONPATH=tests/ avocado run -p TESTLOCAL=true --test-runner=runner -t foreman tests/product_tests/foreman/ + on_failure: + fail_script: *faillogs + log_artifacts: *logs diff -pruN 4.0-2/.editorconfig 4.5.3ubuntu2/.editorconfig --- 4.0-2/.editorconfig 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/.editorconfig 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,9 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true + +[*.py] +indent_style = space +indent_size = 4 diff -pruN 4.0-2/.github/PULL_REQUEST_TEMPLATE.md 4.5.3ubuntu2/.github/PULL_REQUEST_TEMPLATE.md --- 4.0-2/.github/PULL_REQUEST_TEMPLATE.md 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/.github/PULL_REQUEST_TEMPLATE.md 2023-04-28 17:16:21.000000000 +0000 @@ -5,5 +5,4 @@ Please place an 'X' inside each '[]' to - [ ] Is the subject and message clear and concise? - [ ] Does the subject start with **[plugin_name]** if submitting a plugin patch or a **[section_name]** if part of the core sosreport code? - [ ] Does the commit contain a **Signed-off-by: First Lastname **? -- [ ] If this commit closes an existing issue, is the line `Closes: #ISSUENUMBER` included in an independent line? -- [ ] If this commit resolves an existing pull request, is the line `Resolves: #PRNUMBER` included in an independent line? +- [ ] Are any related Issues or existing PRs [properly referenced](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) via a Closes (Issue) or Resolved (PR) line? \ No newline at end of file diff -pruN 4.0-2/.github/codeql/codeql-config.yaml 4.5.3ubuntu2/.github/codeql/codeql-config.yaml --- 4.0-2/.github/codeql/codeql-config.yaml 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/.github/codeql/codeql-config.yaml 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,4 @@ +name: "SoS CodeQL Config" + +paths: + - sos diff -pruN 4.0-2/.github/workflows/codeql.yaml 4.5.3ubuntu2/.github/workflows/codeql.yaml --- 4.0-2/.github/workflows/codeql.yaml 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/.github/workflows/codeql.yaml 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,42 @@ +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: "49 12 * * 6" + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ python ] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + config-file: .github/codeql/codeql-config.yaml + languages: ${{ matrix.language }} + queries: +security-and-quality + + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{ matrix.language }}" diff -pruN 4.0-2/.github/workflows/snap.yaml 4.5.3ubuntu2/.github/workflows/snap.yaml --- 4.0-2/.github/workflows/snap.yaml 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/.github/workflows/snap.yaml 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,32 @@ +name: snap +on: + push: + branches: + - main + +jobs: + build: + runs-on: ubuntu-latest + concurrency: + group: snap-build + cancel-in-progress: true + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: snapcore/action-build@v1 + id: build-snap + # Make sure the snap is installable + - run: | + sudo apt -y remove sosreport + sudo snap install --classic --dangerous ${{ steps.build-snap.outputs.snap }} + sudo snap alias sosreport.sos sos + # Do some testing with the snap + - run: | + sudo sos help + - uses: snapcore/action-publish@v1 + env: + SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }} + with: + snap: ${{ steps.build-snap.outputs.snap }} + release: "latest/edge" diff -pruN 4.0-2/.gitignore 4.5.3ubuntu2/.gitignore --- 4.0-2/.gitignore 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/.gitignore 2023-04-28 17:16:21.000000000 +0000 @@ -16,16 +16,8 @@ venv MANIFEST build/ dist/ +*sos.egg* docs/_build # Pycharm -.idea/**/workspace.xml -.idea/**/tasks.xml -.idea/dictionaries -.idea/**/dataSources/ -.idea/**/dataSources.ids -.idea/**/dataSources.xml -.idea/**/dataSources.local.xml -.idea/**/sqlDataSources.xml -.idea/**/dynamic.xml -.idea/**/uiDesigner.xml +.idea/ diff -pruN 4.0-2/.packit.yaml 4.5.3ubuntu2/.packit.yaml --- 4.0-2/.packit.yaml 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/.packit.yaml 2023-04-28 17:16:21.000000000 +0000 @@ -1,12 +1,25 @@ +upstream_project_url: https://github.com/sosreport/sos +specfile_path: sos.spec downstream_package_name: sos +upstream_package_name: sos + +files_to_sync: + - sos.spec + - .packit.yaml + +srpm_build_deps: + - python3-devel + - gettext + jobs: -- job: copr_build - metadata: + - job: copr_build + trigger: pull_request targets: - - fedora-rawhide-x86_64 - trigger: pull_request -specfile_path: sos.spec -synced_files: -- sos.spec -- .packit.yaml -upstream_package_name: sos + - fedora-development-x86_64 + - fedora-development-aarch64 + - fedora-development-ppc64le + - fedora-development-s390x + +notifications: + pull_request: + successful_build: true diff -pruN 4.0-2/.travis.yml 4.5.3ubuntu2/.travis.yml --- 4.0-2/.travis.yml 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/.travis.yml 1970-01-01 00:00:00.000000000 +0000 @@ -1,65 +0,0 @@ -jobs: - include: - - name: "20.04 flake8 and native run (py3.8)" - os: linux - dist: focal - language: shell - install: sudo apt-get update; sudo apt install flake8 python3-pexpect; - script: - - "flake8 sos tests bin/*" - - "sudo ./tests/simple.sh" - - name: "18.04 pycodestyle and native run (py3.6)" - os: linux - dist: bionic - language: shell - install: sudo apt-get update; sudo apt install python3-pexpect; - script: "sudo ./tests/simple.sh" - - name: "18.04 native run for arm64" - os: linux - dist: bionic - arch: arm64 - language: shell - script: "sudo ./tests/simple.sh" - - name: "18.04 native s390x" - os: linux - dist: bionic - arch: s390x - language: shell - script: "sudo ./tests/simple.sh" - - name: "nosetests and travis Python 3.6" - os: linux - dist: bionic - language: python - python: "3.6" - install: pip install -r requirements.txt; python3 setup.py install; - script: - - "nosetests -v --with-cover --cover-package=sos --cover-html" - - "sudo ./tests/simple.sh ~/virtualenv/python$TRAVIS_PYTHON_VERSION/bin/python" - - name: "nosetests and travis Python 3.7" - os: linux - dist: bionic - language: python - python: "3.7" - install: pip install -r requirements.txt; python3 setup.py install; - script: - - "nosetests -v --with-cover --cover-package=sos --cover-html" - - "sudo ./tests/simple.sh ~/virtualenv/python$TRAVIS_PYTHON_VERSION/bin/python" - - name: "nosetests and travis Python 3.8" - os: linux - dist: bionic - language: python - python: "3.8" - install: pip install -r requirements.txt; python3 setup.py install; - script: - - "nosetests -v --with-cover --cover-package=sos --cover-html" - - "sudo ./tests/simple.sh ~/virtualenv/python$TRAVIS_PYTHON_VERSION/bin/python" - -notifications: - email: - sos-devel@redhat.com - irc: - channels: - - "us.freenode.net#sosreport" - on_success: change -git: - depth: 20 diff -pruN 4.0-2/MANIFEST.in 4.5.3ubuntu2/MANIFEST.in --- 4.0-2/MANIFEST.in 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/MANIFEST.in 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1 @@ +include po/*.po diff -pruN 4.0-2/README.md 4.5.3ubuntu2/README.md --- 4.0-2/README.md 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/README.md 2023-04-28 17:16:21.000000000 +0000 @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.org/sosreport/sos.svg?branch=master)](https://travis-ci.org/sosreport/sos) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/sosreport/sos.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/sosreport/sos/context:python) +[![Build Status](https://api.cirrus-ci.com/github/sosreport/sos.svg?branch=main)](https://cirrus-ci.com/github/sosreport/sos) [![Documentation Status](https://readthedocs.org/projects/sos/badge/?version=main)](https://sos.readthedocs.io/en/main/?badge=main) # SoS @@ -12,31 +12,62 @@ This project is hosted at: For the latest version, to contribute, and for more information, please visit the project pages or join the mailing list. -To clone the current master (development) branch run: +To clone the current main (development) branch run: ``` git clone git://github.com/sosreport/sos.git ``` + ## Reporting bugs Please report bugs via the mailing list or by opening an issue in the [GitHub Issue Tracker][5] +## Chat + +The SoS project has rooms in Matrix and in Libera.Chat. + +Matrix Room: #sosreport:matrix.org + +Libera.Chat: #sos + +These rooms are bridged, so joining either is sufficient as messages from either will +appear in both. + +The Freenode #sos room **is no longer used by this project**. + ## Mailing list -The [sos-devel][4] is the mailing list for any sos-related questions and +The [sos-devel][4] list is the mailing list for any sos-related questions and discussion. Patch submissions and reviews are welcome too. ## Patches and pull requests Patches can be submitted via the mailing list or as GitHub pull requests. If -using GitHub please make sure your branch applies to the current master as a +using GitHub please make sure your branch applies to the current main branch as a 'fast forward' merge (i.e. without creating a merge commit). Use the `git -rebase` command to update your branch to the current master if necessary. +rebase` command to update your branch to the current main if necessary. Please refer to the [contributor guidelines][0] for guidance on formatting patches and commit messages. +Before sending a [pull request][0], it is advisable to check your contribution +against the `flake8` linter, the unit tests, and the stage one avocado test suite: + +``` +# from within the git checkout +$ flake8 sos +$ nosetests -v tests/unittests/ + +# as root +# PYTHONPATH=tests/ avocado run --test-runner=runner -t stageone tests/{cleaner,collect,report,vendor}_tests +``` + +Note that the avocado test suite will generate and remove several reports over its +execution, but no changes will be made to your local system. + +All contributions must pass the entire test suite before being accepted. + ## Documentation User and API [documentation][6] is automatically generated using [Sphinx][7] @@ -54,12 +85,15 @@ and run python3 setup.py build_sphinx -a ``` -Please run `./tests/simple.sh` before sending a [pull request][0], or run the -test suite manually using the `nosetests` command (ideally for the -set of Python versions currently supported by `sos` upstream). ### Wiki +For more in-depth information on the project's features and functionality, please +see [the GitHub wiki][9]. + +If you are interested in contributing an entirely new plugin, or extending sos to +support your distribution of choice, please see these wiki pages: + * [How to write a plugin][1] * [How to write a policy][2] * [Plugin options][3] @@ -74,12 +108,17 @@ pull requests. You can simply run from the git checkout now: ``` -$ sudo ./bin/sos report -a +$ sudo ./bin/sos report ``` The command `sosreport` is still available, as a legacy redirector, and can be used like this: ``` -$ sudo ./bin/sosreport -a +$ sudo ./bin/sosreport +``` + +To see a list of all available plugins and plugin options, run +``` +$ sudo ./bin/sos report -l ``` @@ -94,20 +133,20 @@ To install locally (as root): Fedora/RHEL users install via yum: ``` -yum install sos +# yum install sos ``` Debian users install via apt: ``` -apt install sosreport +# apt install sosreport ``` Ubuntu (14.04 LTS and above) users install via apt: ``` -sudo apt install sosreport +# sudo apt install sosreport ``` [0]: https://github.com/sosreport/sos/wiki/Contribution-Guidelines @@ -116,6 +155,7 @@ sudo apt install sosreport [3]: https://github.com/sosreport/sos/wiki/Plugin-options [4]: https://www.redhat.com/mailman/listinfo/sos-devel [5]: https://github.com/sosreport/sos/issues?state=open - [6]: https://sos.readthedocs.org/en/latest/index.html + [6]: https://sos.readthedocs.org/ [7]: https://www.sphinx-doc.org/ [8]: https://www.readthedocs.org/ + [9]: https://github.com/sosreport/sos/wiki diff -pruN 4.0-2/bin/sos 4.5.3ubuntu2/bin/sos --- 4.0-2/bin/sos 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/bin/sos 2023-04-28 17:16:21.000000000 +0000 @@ -20,5 +20,6 @@ except KeyboardInterrupt: if __name__ == '__main__': sos = SoS(sys.argv[1:]) sos.execute() + os._exit(0) # vim:ts=4 et sw=4 diff -pruN 4.0-2/debian/changelog 4.5.3ubuntu2/debian/changelog --- 4.0-2/debian/changelog 2021-01-27 14:29:24.000000000 +0000 +++ 4.5.3ubuntu2/debian/changelog 2023-05-05 12:37:54.000000000 +0000 @@ -1,15 +1,326 @@ -sosreport (4.0-2) unstable; urgency=medium +sosreport (4.5.3ubuntu2) mantic; urgency=medium - * d/p/0003-systemd-prefer-resolvectl-over-systemd-resolve.patch: - - resolvectl has been introduced in systemd v239. This change - prefers resolvectl but is backward compatible with - systemd-resolve. (closes: #979264) - - * d/control: Recommends lsof, mount and e2fsprogs: - - They aren't marked as Essential in Debian, but they are - for sos to collect generic useful data. (closes: #887181) + * d/control: + - Add 'python3-magic' to runtime depends. - -- Eric Desrochers Wed, 27 Jan 2021 09:29:24 -0500 + -- Nikhil Kshirsagar Fri, 05 May 2023 12:37:54 +0000 + +sosreport (4.5.3ubuntu1) mantic; urgency=medium + + * New 4.5.3 upstream. (LP: #2018270) + + * For more details, full release note is available here: + - https://github.com/sosreport/sos/releases/tag/4.5.3 + + * d/control: + - Add 'python3-magic' as part of the build depends. + - Add 'python3-pexpect' as part of the build depends. + + * Remaining patches: + - d/p/0001-debian-change-tmp-dir-location.patch + + -- Nikhil Kshirsagar Tue, 02 May 2023 05:05:50 +0000 + +sosreport (4.5-1ubuntu0) lunar; urgency=medium + + * New 4.5.1 upstream. (LP: #2009338) + + * For more details, full release note is available here: + - https://github.com/sosreport/sos/releases/tag/4.5.0 + - https://github.com/sosreport/sos/releases/tag/4.5.1 + + * d/control: + - Add 'python3-magic' as part of the build depends. + + * Remaining patches: + - d/p/0001-debian-change-tmp-dir-location.patch + + -- Nikhil Kshirsagar Mon, 06 Mar 2023 06:10:48 +0000 + +sosreport (4.4-1ubuntu2) lunar; urgency=medium + + * Adjust version number to fix the upgrade path + (jammy/kinetic/lunar) (LP: #1995234) + + -- Mauricio Faria de Oliveira Tue, 08 Nov 2022 10:14:00 -0300 + +sosreport (4.4-1ubuntu0) kinetic; urgency=medium + + * New 4.4 upstream. (LP: #1986611) + + * For more details, full release note is available here: + - https://github.com/sosreport/sos/releases/tag/4.4 + + * Former patches, now fixed: + - d/p/0002-fix-setup-py.patch + - d/p/0003-mention-sos-help-in-sos-manpage.patch + + * d/control: + - Add 'python3-magic' as part of the build depends. + + * Remaining patches: + - d/p/0001-debian-change-tmp-dir-location.patch + + -- Nikhil Kshirsagar Wed, 17 Aug 2022 08:38:30 +0000 + +sosreport (4.3-1ubuntu2) jammy; urgency=medium + + * d/p/0003-mention-sos-help-in-sos-manpage.patch: + Fix sos-help manpage. + + -- Eric Desrochers Wed, 16 Feb 2022 13:05:13 -0500 + +sosreport (4.3-1ubuntu1) jammy; urgency=medium + + * New 4.3 upstream. (LP: #1960996) + + * For more details, full release note is available here: + - https://github.com/sosreport/sos/releases/tag/4.3 + + * New patches: + - d/p/0002-fix-setup-py.patch: + Add python sos.help module, it was miss in + upstream release. + + * Former patches, now fixed: + - d/p/0002-report-implement_estimate-only.patch + - d/p/0003-ceph-add-support-for-containerized-ceph-setup.patch + - d/p/0004-ceph-split-plugin-by-components.patch + - d/p/0005-openvswitch-get-userspace-datapath-implementations.patch + - d/p/0006-report-check-for-symlink-before-rmtree.patch + + * Remaining patches: + - d/p/0001-debian-change-tmp-dir-location.patch: + + -- Eric Desrochers Tue, 15 Feb 2022 23:10:27 -0500 + +sosreport (4.2-1ubuntu2) jammy; urgency=medium + + * d/p/0006-report-check-for-symlink-before-rmtree.patch: + - Fixing --estimate-only option by checking if the dirs + are also symlink before performing rmtree() method so + that unlink() method can be used instead. + + -- Eric Desrochers Tue, 19 Oct 2021 14:20:44 -0400 + +sosreport (4.2-1ubuntu1) jammy; urgency=medium + + * New 4.2 upstream release. (LP: #1941745) + - This release contains numerous improvements + and bug fixes to several components within sos, + including an overhaul to the project's test suite + and infrastructure. + + * For more details, full release note is available here: + - https://github.com/sosreport/sos/releases/tag/4.2 + + * New patches: + - d/p/0002-report-implement_estimate-only.patch + - d/p/0003-ceph-add-support-for-containerized-ceph-setup.patch + - d/p/0004-ceph-split-plugin-by-components.patch + - d/p/0005-openvswitch-get-userspace-datapath-implementations.patch + + * Former patches, now fixed: + - d/p/0002-clean-prevent-parsing-ubuntu-user.patch + - d/p/0003-ubuntu-policy-fix-upload.patch + - d/p/0004-chrony-configuration-can-now-be-fragmented.patch + - d/p/0005-global-drop-plugin-version.patch + - d/p/0006-networking-check-presence-of-devlink.patch + - d/p/0007-sosnode-avoid-checksum-cleanup-if-no-archive.patch + + * Remaining patches: + - d/p/0001-debian-change-tmp-dir-location.patch + + * d/control: + - Add 'python3-coverage' as part of the build depends. + + * d/rules: + - Fix misplaced and duplicated sos.conf file in /usr/config. + + -- Eric Desrochers Mon, 18 Oct 2021 09:24:22 -0400 + +sosreport (4.1-1ubuntu3) impish; urgency=medium + + * No-change rebuild to build packages with zstd compression. + + -- Matthias Klose Thu, 07 Oct 2021 12:24:39 +0200 + +sosreport (4.1-1ubuntu2) impish; urgency=medium + + * d/p/series: + - Re-order patches in numerical order. + + * d/p/0003-ubuntu-policy-fix-upload.patch: + - Fix sos archive upload to UA Canonical server + (LP: #1923209) + + * d/p/0004-chrony-configuration-can-now-be-fragmented.patch: + - Chrony 4.0, first introduced in Hirsute, support + fragmented configuration. + + * d/p/0005-global-drop-plugin-version.patch: + - Removal of plugins versionning features as it generate + unhelpful noise. (LP: #1922925) + + * d/p/0006-networking-check-presence-of-devlink.patch: + - On certain kernel configuration, devlink cmds may + trigger the module to load automatically. This will + also prevent simple.sh, part of the autopkgtest, in + Bionic to fail due to devlink kernel conf in 4.15. + (LP: #1923661) + + * d/p/0007-sosnode-avoid-checksum-cleanup-if-no-archive.patch: + - Fixes an exception propagation from `cleanup()` + where an attempt to look for and remove a checksum + file was made when an archive was not generated. + (LP: #1923641) + + -- Eric Desrochers Mon, 26 Apr 2021 15:34:42 -0400 + +sosreport (4.1-1ubuntu1) hirsute; urgency=medium + + * New 4.1 upstream minor release. + - https://github.com/sosreport/sos/releases/tag/4.1 + + * d/test/simple.sh: + - Update the script from upstream + - Modify the script to use /tmp as a target, instead + of /var/tmp. + + * d/test/control: + - Adding isolation-machine as simple.sh wants to + interact with the kernel. + + * Former patches: + - d/p/0002-fix-dict-order-py38-incompatibility.patch + - d/p/0003-sosclean-fix-handling-of-filepath-with-archive-name.patch + - d/p/0004-sosclean-fix-tarball-skipping-regex.patch + - d/p/0005-ceph-collect-balancer-and-pg-autoscale-status.patch + - d/p/0006-rabbitmq-add-info-on-maybe-stuck-processes.patch + - d/p/0007-rabbitmq-add-10sec-timeout-to-call-to-maybestuck.patch + - d/p/0008-networking-include-ip-neigh-and-rule-info.patch + - d/p/0009-conntrack-add-conntrack-info.patch + - d/p/0010-conntrack-gather-per-namespace-data.patch + - d/p/0011-ceph-include-time-sync-status-for-ceph-mon.patch + - d/p/0012-apt-move-unattended-upgrades-log-collection.patch + - d/p/0013-bcache-add-a-new-plugin-for-bcache.patch + - d/p/0014-k8s-add-cdk-master-auth-webhook-to-journal.patch + - d/p/0015-k8s-fix-cdk-related-file-paths.patch + - d/p/0016-systemd-prefer-resolvectl-over-systemd-resolve.patch + - d/p/0017-ovn-extend-information.patch + - d/p/0018-ua-prefer-new-ua-cmd-over-the-deprecated-one.patch + - d/p/0019-ovn-fix-sbctl-cmd-execution.patch + + * Remaining patches: + - d/p/0001-debian-change-tmp-dir-location.patch + + * New patches: + - d/p/0002-clean-prevent-parsing-ubuntu-user.patch + + -- Eric Desrochers Mon, 22 Mar 2021 12:19:59 +0000 + +sosreport (4.0-1ubuntu9) hirsute; urgency=medium + + [Edward Hope-Morley] + * d/p/0019-ovn-fix-sbctl-cmd-execution.patch (LP: #1916635) + + -- Eric Desrochers Wed, 24 Feb 2021 10:40:56 -0500 + +sosreport (4.0-1ubuntu8) hirsute; urgency=medium + + * d/p/0018-ua-prefer-new-ua-cmd-over-the-deprecated-one.patch + + -- Eric Desrochers Tue, 23 Feb 2021 08:49:59 -0500 + +sosreport (4.0-1ubuntu7) hirsute; urgency=medium + + [Edward Hope-Morley] + * d/p/0017-ovn-extend-information.patch (LP: #1915072) + - Extend ovn informations + + -- Eric Desrochers Thu, 11 Feb 2021 11:06:31 -0500 + +sosreport (4.0-1ubuntu6) hirsute; urgency=medium + + [Eric Desrochers] + * d/p/0012-apt-move-unattended-upgrades-log-collection.patch + (LP: #1906302) + + [Ponnuvel Palaniyappan] + * d/p/0013-bcache-add-a-new-plugin-for-bcache.patch + (LP: #1913284) + + [Felipe Reyes] + * d/p/0014-k8s-add-cdk-master-auth-webhook-to-journal.patch + * d/p/0015-k8s-fix-cdk-related-file-paths.patch + (LP: #1913583) + + [Michael Biebl] + * d/p/0016-systemd-prefer-resolvectl-over-systemd-resolve.patch + (LP: #1913581) + + -- Eric Desrochers Fri, 29 Jan 2021 12:20:50 -0500 + +sosreport (4.0-1ubuntu5) hirsute; urgency=medium + + [Ponnuvel Palaniyappan] + * d/p/0011-ceph-include-time-sync-status-for-ceph-mon.patch: + - Ceph mons might get into time sync problems if ntp/chrony + isn't installed or configured correctly. Since Luminous + release, upstream support 'time-sync-status' to detect this + more easily. (LP: #1910264) + + -- Eric Desrochers Tue, 05 Jan 2021 11:18:38 -0500 + +sosreport (4.0-1ubuntu4) hirsute; urgency=medium + + [Hemanth Nakkina] + * d/p/0009-conntrack-add-conntrack-info.patch: rename the + conntrackd plugin to conntrack; add conntrack commands. + (LP: #1898077) + + [Mauricio Oliveira] + * d/p/0010-conntrack-gather-per-namespace-data.patch: add + conntrack commands for network namespaces. + + -- Mauricio Faria de Oliveira Mon, 23 Nov 2020 15:10:12 -0300 + +sosreport (4.0-1ubuntu3) hirsute; urgency=medium + + [Edward Hope-Morley] + * d/p/0008-networking-include-ip-neigh-and-rule-info.patch: + - Include ns ip neigh and ip rule info. (LP: #1901555) + + -- Eric Desrochers Mon, 26 Oct 2020 11:04:41 -0400 + +sosreport (4.0-1ubuntu2) groovy; urgency=medium + + [Nicolas Bock] + * d/p/0007-rabbitmq-add-10sec-timeout-to-call-to-maybestuck.patch: + - Add 10 second timeout to call to `maybe_stuck()`. + + -- Eric Desrochers Mon, 21 Sep 2020 10:47:11 -0400 + +sosreport (4.0-1ubuntu1) groovy; urgency=medium + + [Eric Desrochers] + * d/p/0003-sosclean-fix-handling-of-filepath-with-archive-name.patch: + - Fixes the splitting of filepaths within the archive, + when the archive name is included in the filename + inside the archive. (LP: #1896222) + + * d/p/0004-sosclean-fix-tarball-skipping-regex.patch: + - Fix tarball skipping regex + + [Dan Hill] + * d/p/0005-ceph-collect-balancer-and-pg-autoscale-status.patch: + - Collect balancer and pg-autoscale status (LP: #1893109) + + [Nicolas Bock] + * d/p/0006-rabbitmq-add-info-on-maybe-stuck-processes.patch: + - Add information on maybe_stuck() processes for RMQ. (LP: #1890846) + + -- Eric Desrochers Fri, 18 Sep 2020 09:23:04 -0400 sosreport (4.0-1) unstable; urgency=medium diff -pruN 4.0-2/debian/control 4.5.3ubuntu2/debian/control --- 4.0-2/debian/control 2021-01-27 14:29:24.000000000 +0000 +++ 4.5.3ubuntu2/debian/control 2023-05-05 12:37:36.000000000 +0000 @@ -1,24 +1,26 @@ Source: sosreport -Maintainer: Eric Desrochers +Maintainer: Nikhil Kshirsagar Section: admin Priority: optional -Standards-Version: 4.5.0 +Standards-Version: 4.5.1 Build-Depends: debhelper-compat (= 13), dh-python, gettext, python3-all, + python3-coverage, python3-nose, python3-setuptools, python3-sphinx, + python3-magic, + python3-pexpect, Homepage: https://github.com/sosreport/sos Vcs-Browser: https://salsa.debian.org/sosreport-team/sosreport Vcs-Git: https://salsa.debian.org/sosreport-team/sosreport.git Package: sosreport Architecture: any -Depends: ${python3:Depends}, ${misc:Depends}, python3-pexpect -Recommends: e2fsprogs, lsof, mount +Depends: ${python3:Depends}, ${misc:Depends}, python3-pexpect, python3-magic Description: Set of tools to gather troubleshooting data from a system Sos is a set of tools that gathers information about system hardware and configuration. The information can then be used for diff -pruN 4.0-2/debian/patches/0001-debian-change-tmp-dir-location.patch 4.5.3ubuntu2/debian/patches/0001-debian-change-tmp-dir-location.patch --- 4.0-2/debian/patches/0001-debian-change-tmp-dir-location.patch 2020-08-19 22:49:24.000000000 +0000 +++ 4.5.3ubuntu2/debian/patches/0001-debian-change-tmp-dir-location.patch 2022-02-16 04:06:06.000000000 +0000 @@ -1,6 +1,6 @@ Description: Change default location to /tmp ---- sos-4.0.orig/sos.conf -+++ sos-4.0/sos.conf +--- a/sos.conf ++++ b/sos.conf @@ -7,6 +7,7 @@ #verify = yes #batch = yes diff -pruN 4.0-2/debian/patches/0002-fix-dict-order-py38-incompatibility.patch 4.5.3ubuntu2/debian/patches/0002-fix-dict-order-py38-incompatibility.patch --- 4.0-2/debian/patches/0002-fix-dict-order-py38-incompatibility.patch 2020-08-19 22:49:24.000000000 +0000 +++ 4.5.3ubuntu2/debian/patches/0002-fix-dict-order-py38-incompatibility.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,16 +0,0 @@ -Description: Fix dict order incompability - Fix dictionary keys changed during iteration. -Author: Eric Desrochers -Origin: upstream, https://github.com/sosreport/sos/commit/1d7bab6c7 -Bug: https://github.com/sosreport/sos/issues/2206 ---- sos-4.0.orig/sos/options.py -+++ sos-4.0/sos/options.py -@@ -186,7 +186,7 @@ class SoSOptions(): - if 'verbose' in odict.keys(): - odict['verbosity'] = int(odict.pop('verbose')) - # convert options names -- for key in odict.keys(): -+ for key in list(odict): - if '-' in key: - odict[key.replace('-', '_')] = odict.pop(key) - # set the values according to the config file diff -pruN 4.0-2/debian/patches/0003-systemd-prefer-resolvectl-over-systemd-resolve.patch 4.5.3ubuntu2/debian/patches/0003-systemd-prefer-resolvectl-over-systemd-resolve.patch --- 4.0-2/debian/patches/0003-systemd-prefer-resolvectl-over-systemd-resolve.patch 2021-01-27 14:29:24.000000000 +0000 +++ 4.5.3ubuntu2/debian/patches/0003-systemd-prefer-resolvectl-over-systemd-resolve.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,37 +0,0 @@ -Description: Prefer resolvectl over systemd-resolve - The latter is a deprecated compat symlink. -Author: Michael Biebl -Origin: upstream, https://github.com/sosreport/sos/commit/110757df526d79b2b257077bef78d2553f43e4a7 -Bug: https://github.com/sosreport/sos/pull/2385 -Bug-Debian: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=979264 ---- a/sos/report/plugins/systemd.py -+++ b/sos/report/plugins/systemd.py -@@ -10,6 +10,7 @@ - - from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, - UbuntuPlugin, CosPlugin, SoSPredicate) -+from sos.utilities import is_executable - - - class Systemd(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin, CosPlugin): -@@ -47,11 +48,17 @@ - "timedatectl" - ]) - -- # systemd-resolve command starts systemd-resolved service if that -+ # resolvectl command starts systemd-resolved service if that - # is not running, so gate the commands by this predicate -+ if is_executable('resolvectl'): -+ resolvectl_status = 'resolvectl status' -+ resolvectl_statistics = 'resolvectl statistics' -+ else: -+ resolvectl_status = 'systemd-resolve --status' -+ resolvectl_statistics = 'systemd-resolve --statistics' - self.add_cmd_output([ -- "systemd-resolve --status", -- "systemd-resolve --statistics", -+ resolvectl_status, -+ resolvectl_statistics, - ], pred=SoSPredicate(self, services=["systemd-resolved"])) - - self.add_cmd_output("systemd-analyze plot", diff -pruN 4.0-2/debian/patches/series 4.5.3ubuntu2/debian/patches/series --- 4.0-2/debian/patches/series 2021-01-27 14:26:38.000000000 +0000 +++ 4.5.3ubuntu2/debian/patches/series 2022-08-17 08:38:08.000000000 +0000 @@ -1,3 +1 @@ 0001-debian-change-tmp-dir-location.patch -0002-fix-dict-order-py38-incompatibility.patch -0003-systemd-prefer-resolvectl-over-systemd-resolve.patch diff -pruN 4.0-2/debian/rules 4.5.3ubuntu2/debian/rules --- 4.0-2/debian/rules 2020-08-19 22:49:17.000000000 +0000 +++ 4.5.3ubuntu2/debian/rules 2022-02-16 04:06:06.000000000 +0000 @@ -4,3 +4,12 @@ export PYBUILD_NAME=sosreport %: dh $@ --with python3 --buildsystem=pybuild + +override_dh_install: + # Move config file to the right location. + mv debian/sosreport/usr/config/sos.conf debian/sosreport/etc/sos/sos.conf + # Remove unnecessary unused dir. + rm -rf debian/sosreport/usr/config + +override_dh_auto_test: + nosetests3 -v --with-cover --cover-package=sos tests/unittests diff -pruN 4.0-2/debian/tests/control 4.5.3ubuntu2/debian/tests/control --- 4.0-2/debian/tests/control 2020-08-19 22:49:24.000000000 +0000 +++ 4.5.3ubuntu2/debian/tests/control 2022-02-16 04:06:06.000000000 +0000 @@ -1,3 +1,3 @@ Tests: simple.sh Depends: sosreport -Restrictions: needs-root, allow-stderr +Restrictions: needs-root, allow-stderr, isolation-machine diff -pruN 4.0-2/debian/tests/simple.sh 4.5.3ubuntu2/debian/tests/simple.sh --- 4.0-2/debian/tests/simple.sh 2020-08-19 22:49:24.000000000 +0000 +++ 4.5.3ubuntu2/debian/tests/simple.sh 2022-02-16 04:06:06.000000000 +0000 @@ -1,3 +1,4 @@ +#!/bin/bash # This file is part of the sos project: https://github.com/sosreport/sos # # This copyrighted material is made available to anyone wishing to use, @@ -5,7 +6,6 @@ # version 2 of the GNU General Public License. # # See the LICENSE file in the source distribution for further information. -#/bin/bash # A quick port of the travis tests to bash, requires root # TODO # * look into using a framework.. @@ -14,12 +14,13 @@ # * make it better validate archives and contents PYTHON=${1:-/usr/bin/python3} -SOSPATH=${2:-./bin/sos report} +SOSPATH=${2:-./bin/sos report --batch --tmp-dir=/tmp } NUMOFFAILURES=0 -summary="Summary\n" +summary="\nSummary\n" +FAIL_LIST="" -run_expecting_sucess () { +run_expecting_success () { #$1 - is command options #$2 - kind of check to do, so far only extract FAIL=false @@ -35,7 +36,7 @@ run_expecting_sucess () { echo "### Success" else echo "!!! FAILED !!!" - FAIL=true + add_failure "$1 failed during execution" fi end=`date +%s` @@ -43,8 +44,7 @@ run_expecting_sucess () { echo "#### Sos Total time (seconds):" $runtime if [ -s /dev/shm/stderr ]; then - FAIL=true - echo "!!! FAILED !!!" + add_failure "test generated stderr output, see above" echo "### start stderr" cat /dev/shm/stderr echo "### end stderr" @@ -56,21 +56,19 @@ run_expecting_sucess () { if [ "extract" = "$2" ]; then echo "### start extraction" - rm -f /tmp/sosreport*md5 + rm -f /tmp/sosreport*sha256 mkdir /tmp/sosreport_test/ tar xfa /tmp/sosreport*.tar* -C /tmp/sosreport_test --strip-components=1 if [ -s /tmp/sosreport_test/sos_logs/*errors.txt ]; then FAIL=true echo "!!! FAILED !!!" + add_failure "Test $1 generated errors" echo "#### *errors.txt output" ls -alh /tmp/sosreport_test/sos_logs/ cat /tmp/sosreport_test/sos_logs/*errors.txt fi echo "### stop extraction" fi - - size="$(grep Size /dev/shm/stdout)" - summary="${summary} \n failures ${FAIL} \t time ${runtime} \t ${size} \t ${1} " echo "######### DONE WITH $1 #########" @@ -82,7 +80,160 @@ run_expecting_sucess () { fi } -# If /etc/sos/sos.conf doesn't exist let's just make it.. +update_summary () { + size="$(grep Size /dev/shm/stdout)" + size="$(echo "${size:-"Size 0.00MiB"}")" + summary="${summary} \n failures ${FAIL} \t time ${runtime} \t ${size} \t ${1} " +} + +update_failures () { + if $FAIL; then + NUMOFFAILURES=$(($NUMOFFAILURES + 1)) + fi +} + +add_failure () { + FAIL=true + echo "!!! TEST FAILED: $1 !!!" + FAIL_LIST="${FAIL_LIST}\n \t ${FUNCNAME[1]}: \t\t ${1}" +} + +# Test a no frills run with verbosity and make sure the expected items exist +test_normal_report () { + cmd="-vvv" + # get a list of initial kmods loaded + kmods=( $(lsmod | cut -f1 -d ' ' | sort) ) + run_expecting_success "$cmd" extract + if [ $? -eq 0 ]; then + if [ ! -f /tmp/sosreport_test/sos_reports/sos.html ]; then + add_failure "did not generate html reports" + fi + if [ ! -f /tmp/sosreport_test/sos_reports/manifest.json ]; then + add_failure "did not generate manifest.json" + fi + if [ ! -f /tmp/sosreport_test/free ]; then + add_failure "did not create free symlink in archive root" + fi + if [ ! "$(grep "DEBUG" /tmp/sosreport_test/sos_logs/sos.log)" ]; then + add_failure "did not find debug logging when using -vvv" + fi + # new list, see if we added any + new_kmods=( $(lsmod | cut -f1 -d ' ' | sort) ) + if [ "$(printf '%s\n' "${kmods[@]}" "${new_kmods[@]}" | sort | uniq -u)" ]; then + add_failure "new kernel modules loaded during execution" + echo "$(printf '%s\n' "${kmods[@]}" "${new_kmods[@]}" | sort | uniq -u)" + fi + update_failures + update_summary "$cmd" + fi +} + +# Test for correctly skipping html generation, and label setting +test_noreport_label_only () { + cmd="--no-report --label TEST -o hardware" + run_expecting_success "$cmd" extract + if [ $? -eq 0 ]; then + if [ -f /tmp/sosreport_test/sos_reports/sos.html ]; then + add_failure "html report generated when --no-report used" + fi + if [ ! $(grep /tmp/sosreport-*TEST* /dev/shm/stdout) ]; then + add_failure "no label set on archive" + fi + count=$(find /tmp/sosreport_test/sos_commands/* -type d | wc -l) + if [[ "$count" -gt 1 ]]; then + add_failure "more than one plugin ran when using -o hardware" + fi + update_failures + fi + update_summary "$cmd" +} + +# test using mask +test_mask () { + cmd="--mask" + run_expecting_success "$cmd" extract + if [ $? -eq 0 ]; then + if [ ! $(grep host0 /tmp/sosreport_test/hostname) ]; then + add_failure "hostname not obfuscated with --mask" + fi + # we don't yet support binary obfuscation, so skip binary matches + if [ "$(grep -rI `hostname` /tmp/sosreport_test/*)" ]; then + add_failure "hostname not obfuscated in all places" + echo "$(grep -rI `hostname` /tmp/sosreport_test/*)" + fi + # only tests first interface + mac_addr=$(cat /sys/class/net/$(ip route show default | awk '/default/ {print $5}')/address) + if [ "$(grep -rI $mac_addr /tmp/sosreport_test/*)" ]; then + add_failure "MAC address not obfuscated in all places" + echo "$(grep -rI $mac_addr /tmp/sosreport_test/*)" + fi + # only tests first interface + ip_addr=$(ip route show default | awk '/default/ {print $3}') + if [ "$(grep -rI $ip_addr /tmp/sosreport_test/*)" ]; then + add_failure "IP address not obfuscated in all places" + echo "$(grep -rI $ip_addr /tmp/sosreport/_test/*)" + fi + update_failures + fi + update_summary "$cmd" +} + +# test log-size, env vars, and compression type +test_logsize_env_gzip () { + cmd="--log-size 0 --no-env-vars -z gzip" + run_expecting_success "$cmd" extract + if [ $? -eq 0 ]; then + if [ -f /tmp/sosreport_test/environment ]; then + add_failure "env vars captured when using --no-env-vars" + fi + if [ ! $(grep /tmp/sosreport*.gz /dev/shm/stdout) ]; then + add_failure "archive was not gzip compressed using -z gzip" + fi + update_failures + fi + update_summary "$cmd" +} + +# test plugin enablement, plugopts and at the same time ensure our list option parsing is working +test_enable_opts_postproc () { + cmd="-e opencl -v -k kernel.with-timer,libraries.ldconfigv --no-postproc" + run_expecting_success "$cmd" extract + if [ $? -eq 0 ]; then + if [ ! "$(grep "opencl" /dev/shm/stdout)" ]; then + add_failure "force enabled plugin opencl did not run" + fi + if [ ! -f /tmp/sosreport_test/proc/timer* ]; then + add_failure "/proc/timer* not captured when using -k kernel.with-timer" + fi + if [ ! -f /tmp/sosreport_test/sos_commands/libraries/ldconfig_-v* ]; then + add_failure "ldconfig -v not captured when using -k libraries.ldconfigv" + fi + if [ "$(grep "substituting" /tmp/sosreport_test/sos_logs/sos.log)" ]; then + add_failure "post-processing ran while using --no-post-proc" + fi + + update_failures + update_summary "$cmd" + fi +} + +# test if --build and --threads work properly +test_build_threads () { + cmd="--build -t1 -o host,kernel,filesys,hardware,date,logs" + run_expecting_success "$cmd" + if [ $? -eq 0 ]; then + if [ ! "$(grep "Your sosreport build tree" /dev/shm/stdout)" ]; then + add_failure "did not save the build tree" + fi + if [ $(grep "Finishing plugins" /dev/shm/stdout) ]; then + add_failure "did not limit threads when using --threads 1" + fi + update_failures + update_summary "$cmd" + fi +} + +# If /etc/sos/sos.conf doesn't exist let's just make it if [ -f /etc/sos/sos.conf ]; then echo "/etc/sos/sos.conf already exists" else @@ -91,29 +242,27 @@ else touch /etc/sos/sos.conf fi + # Runs not generating sosreports -run_expecting_sucess " -l" -run_expecting_sucess " --list-presets" -run_expecting_sucess " --list-profiles" - -# Test generating sosreports, 3 (new) options at a time -# Trying to do --batch (1 label/archive/report/verbosity change) (other changes) -run_expecting_sucess " --batch --build --no-env-vars " # Only --build test -run_expecting_sucess " --batch --no-report -o hardware " extract -run_expecting_sucess " --batch --label TEST -a -c never" extract -run_expecting_sucess " --batch --debug --log-size 0 -c always" extract -run_expecting_sucess " --batch -z xz --log-size 1" extract -run_expecting_sucess " --batch -z gzip" extract -run_expecting_sucess " --batch -t 1 -n hardware" extract -run_expecting_sucess " --batch --quiet -e opencl -k kernel.with-timer" extract -run_expecting_sucess " --batch --case-id 10101 --all-logs --since=$(date -d "yesterday 13:00" '+%Y%m%d') " extract -run_expecting_sucess " --batch --verbose --no-postproc" extract -run_expecting_sucess " --batch --mask" extract +run_expecting_success " -l"; update_summary "List plugins" +run_expecting_success " --list-presets"; update_summary "List presets" +run_expecting_success " --list-profiles"; update_summary "List profiles" + +# Runs generating sosreports +# TODO: +# - find a way to test if --since is working +test_build_threads +test_normal_report +test_enable_opts_postproc +test_noreport_label_only +test_logsize_env_gzip +test_mask -echo $summary +echo -e $summary if [ $NUMOFFAILURES -gt 0 ]; then - echo "FAILED $NUMOFFAILURES" + echo -e "\nTests Failed: $NUMOFFAILURES\nFailures within each test:" + echo -e $FAIL_LIST exit 1 else echo "Everything worked!" diff -pruN 4.0-2/docs/archive.rst 4.5.3ubuntu2/docs/archive.rst --- 4.0-2/docs/archive.rst 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/docs/archive.rst 2023-04-28 17:16:21.000000000 +0000 @@ -6,3 +6,11 @@ :members: :undoc-members: :show-inheritance: + +.. autoclass:: sos.archive.Archive + :members: + :undoc-members: + +.. autoclass:: sos.archive.FileCacheArchive + :members: + :undoc-members: diff -pruN 4.0-2/docs/clusters.rst 4.5.3ubuntu2/docs/clusters.rst --- 4.0-2/docs/clusters.rst 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/docs/clusters.rst 2023-04-28 17:16:21.000000000 +0000 @@ -1,7 +1,7 @@ -``sos.collector.clusters`` --- Cluster Profiles -================================================ +``sos.collector.clusters`` --- Cluster Interface +================================================= .. automodule:: sos.collector.clusters - :noindex: :members: + :undoc-members: :show-inheritance: diff -pruN 4.0-2/docs/conf.py 4.5.3ubuntu2/docs/conf.py --- 4.0-2/docs/conf.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/docs/conf.py 2023-04-28 17:16:21.000000000 +0000 @@ -59,9 +59,9 @@ copyright = '2014, Bryn Reeves' # built documents. # # The short X.Y version. -version = '4.0' +version = '4.5.3' # The full version, including alpha/beta/rc tags. -release = '4.0' +release = '4.5.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff -pruN 4.0-2/docs/index.rst 4.5.3ubuntu2/docs/index.rst --- 4.0-2/docs/index.rst 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/docs/index.rst 2023-04-28 17:16:21.000000000 +0000 @@ -13,7 +13,7 @@ https://github.com/sosreport/sos For the latest version, to contribute, and for more information, please visit the project pages or join the mailing list. -To clone the current master (development) branch run: +To clone the current main (development) branch run: .. code:: @@ -32,7 +32,7 @@ Mailing list Patches and pull requests ^^^^^^^^^^^^^^^^^^^^^^^^^ -Patches can be submitted via the mailing list or as GitHub pull requests. If using GitHub please make sure your branch applies to the current master as a 'fast forward' merge (i.e. without creating a merge commit). Use the git rebase command to update your branch to the current master if necessary. +Patches can be submitted via the mailing list or as GitHub pull requests. If using GitHub please make sure your branch applies to the current main branch as a 'fast forward' merge (i.e. without creating a merge commit). Use the git rebase command to update your branch to the current main branch if necessary. Documentation ============= @@ -58,9 +58,7 @@ Manual Installation .. code:: - to install locally (as root) ==> make install - to build an rpm ==> make rpm - to build a deb ==> make deb + python3 setup.py install Pre-built Packaging ^^^^^^^^^^^^^^^^^^^ @@ -80,16 +78,6 @@ Ubuntu (14.04 LTS and above) users insta API === -Plugin Reference -^^^^^^^^^^^^^^^^ - -.. toctree:: - :maxdepth: 2 - - plugins - clusters - parsers - Core Reference ^^^^^^^^^^^^^^ @@ -97,6 +85,8 @@ Core Reference :maxdepth: 4 archive + clusters + parsers policies plugins reporting diff -pruN 4.0-2/docs/parsers.rst 4.5.3ubuntu2/docs/parsers.rst --- 4.0-2/docs/parsers.rst 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/docs/parsers.rst 2023-04-28 17:16:21.000000000 +0000 @@ -1,7 +1,7 @@ -``sos.cleaner.parsers`` --- Cleaning Parser Definition -======================================================= +``sos.cleaner.parsers`` --- Parser Interface +============================================= .. automodule:: sos.cleaner.parsers - :noindex: :members: + :undoc-members: :show-inheritance: diff -pruN 4.0-2/docs/plugins.rst 4.5.3ubuntu2/docs/plugins.rst --- 4.0-2/docs/plugins.rst 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/docs/plugins.rst 2023-04-28 17:16:21.000000000 +0000 @@ -2,6 +2,6 @@ =========================================== .. automodule:: sos.report.plugins - :noindex: :members: + :undoc-members: :show-inheritance: diff -pruN 4.0-2/docs/policies.rst 4.5.3ubuntu2/docs/policies.rst --- 4.0-2/docs/policies.rst 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/docs/policies.rst 2023-04-28 17:16:21.000000000 +0000 @@ -2,6 +2,6 @@ ===================================== .. automodule:: sos.policies - :noindex: :members: + :undoc-members: :show-inheritance: diff -pruN 4.0-2/docs/reporting.rst 4.5.3ubuntu2/docs/reporting.rst --- 4.0-2/docs/reporting.rst 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/docs/reporting.rst 2023-04-28 17:16:21.000000000 +0000 @@ -2,7 +2,6 @@ ================================================ .. automodule:: sos.report.reporting - :noindex: :members: :undoc-members: :show-inheritance: diff -pruN 4.0-2/docs/utilities.rst 4.5.3ubuntu2/docs/utilities.rst --- 4.0-2/docs/utilities.rst 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/docs/utilities.rst 2023-04-28 17:16:21.000000000 +0000 @@ -2,7 +2,6 @@ ======================================== .. automodule:: sos.utilities - :noindex: :members: :undoc-members: :show-inheritance: diff -pruN 4.0-2/man/en/sos-clean.1 4.5.3ubuntu2/man/en/sos-clean.1 --- 4.0-2/man/en/sos-clean.1 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/man/en/sos-clean.1 2023-04-28 17:16:21.000000000 +0000 @@ -4,9 +4,14 @@ sos clean - Obfuscate sensitive data fro .SH SYNOPSIS .B sos clean TARGET [options] [\-\-domains] - [\-\-map] + [\-\-disable-parsers] + [\-\-keywords] + [\-\-keyword-file] + [\-\-map-file] [\-\-jobs] [\-\-no-update] + [\-\-keep-binary-files] + [\-\-archive-type] .SH DESCRIPTION \fBsos clean\fR or \fBsos mask\fR is an sos subcommand used to obfuscate sensitive information from @@ -47,6 +52,17 @@ match a domain given via this option wil For example, if \fB\-\-domains redhat.com\fR is specified, then 'redhat.com' will be obfuscated, as will 'www.redhat.com' and subdomains such as 'foo.redhat.com'. .TP +.B \-\-disable-parsers PARSERS +Provide a comma-delimited list of parsers to disable when cleaning an archive. By +default all parsers are enabled. + +Note that using this option is very likely to leave sensitive information in place in +the target archive, so only use this option when absolutely necessary or you have complete +trust in the party/parties that may handle the generated report. + +Valid values for this option are currently: \fBhostname\fR, \fBip\fR, \fBipv6\fR, +\fBmac\fR, \fBkeyword\fR, and \fBusername\fR. +.TP .B \-\-keywords KEYWORDS Provide a comma-delimited list of keywords to scrub in addition to the default parsers. @@ -54,7 +70,11 @@ Keywords provided by this option will be integer based on the keyword's index in the parser. Note that keywords will be replaced as both standalone words and in substring matches. .TP -.B \-\-map FILE +.B \-\-keyword-file FILE +Provide a file that contains a list of keywords that should be obfuscated. Each word must +be specified on a newline within the file. +.TP +.B \-\-map-file FILE Provide a location to a valid mapping file to use as a reference for existing obfuscation pairs. If one is found, the contents are loaded before parsing is started. This allows consistency between runs of this command for obfuscated pairs. By default, sos will write the generated private map file @@ -71,6 +91,48 @@ Default: 4 .TP .B \-\-no-update Do not write the mapping file contents to /etc/sos/cleaner/default_mapping +.TP +.B \-\-keep-binary-files +Keep unprocessable binary files in the archive, rather than removing them. + +Note that binary files cannot be obfuscated, and thus keeping them in the archive +may result in otherwise sensitive information being included in the final archive. +Users should review any archive that keeps binary files in place before sending to +a third party. + +Default: False (remove encountered binary files) +.TP +.B \-\-archive-type TYPE +Specify the type of archive that TARGET was generated as. +When sos inspects a TARGET archive, it tries to identify what type of archive it is. +For example, it may be a report generated by \fBsos report\fR, or a collection of those +reports generated by \fBsos collect\fR, which require separate approaches. + +This option may be useful if a given TARGET archive is known to be of a specific type, +but due to unknown reasons or some malformed/missing information in the archive directly, +that is not properly identified by sos. + +The following are accepted values for this option: + + \fBauto\fR Automatically detect the archive type + \fBreport\fR An archive generated by \fBsos report\fR + \fBcollect\fR An archive generated by \fBsos collect\fR + \fBinsights\fR An archive generated by the \fBinsights-client\fR package + +The following may also be used, however note that these do not attempt to pre-load +any information from the archives into the parsers. This means that, among other limitations, +items like host and domain names may not be obfuscated unless an obfuscated mapping already exists +on the system from a previous execution. + + \fBdata-dir\fR A plain directory on the filesystem. + \fBtarball\fR A generic tar archive not associated with any known tool + +.SH SEE ALSO +.BR sos (1) +.BR sos-report (1) +.BR sos-collect (1) +.BR sos.conf (5) + .SH MAINTAINER .nf Jake Hunsaker diff -pruN 4.0-2/man/en/sos-collect.1 4.5.3ubuntu2/man/en/sos-collect.1 --- 4.0-2/man/en/sos-collect.1 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/man/en/sos-collect.1 2023-04-28 17:16:21.000000000 +0000 @@ -1,4 +1,4 @@ -.TH sos collect 1 "April 2020" +.TH SOS COLLECT 1 "April 2020" .SH NAME sos collect \- Collect sosreports from multiple (cluster) nodes @@ -11,6 +11,7 @@ sos collect \- Collect sosreports from m [\-\-chroot CHROOT] [\-\-case\-id CASE_ID] [\-\-cluster\-type CLUSTER_TYPE] + [\-\-container\-runtime RUNTIME] [\-e ENABLE_PLUGINS] [--encrypt-key KEY]\fR [--encrypt-pass PASS]\fR @@ -20,21 +21,29 @@ sos collect \- Collect sosreports from m [\-\-nopasswd-sudo] [\-k PLUGIN_OPTION] [\-\-label LABEL] + [\-\-log-size SIZE] [\-n SKIP_PLUGINS] [\-\-nodes NODES] [\-\-no\-pkg\-check] [\-\-no\-local] - [\-\-master MASTER] + [\-\-primary PRIMARY] + [\-\-image IMAGE] + [\-\-force-pull-image TOGGLE, --pull TOGGLE] + [\-\-registry-user USER] + [\-\-registry-password PASSWORD] + [\-\-registry-authfile FILE] [\-o ONLY_PLUGINS] [\-p SSH_PORT] [\-\-password] [\-\-password\-per\-node] [\-\-preset PRESET] + [\-\-skip-commands COMMANDS] + [\-\-skip-files FILES] [\-s|\-\-sysroot SYSROOT] [\-\-ssh\-user SSH_USER] - [\-\-sos-cmd SOS_CMD] [\-t|\-\-threads THREADS] [\-\-timeout TIMEOUT] + [\-\-transport TRANSPORT] [\-\-tmp\-dir TMP_DIR] [\-v|\-\-verbose] [\-\-verify] @@ -46,7 +55,7 @@ collect is an sos subcommand to collect them in a single useful tar archive. sos collect can be run either on a workstation that has SSH key authentication setup -for the nodes in a given cluster, or from a "master" node in a cluster that has SSH +for the nodes in a given cluster, or from a "primary" node in a cluster that has SSH keys configured for the other nodes. Some sosreport options are supported by sos-collect and are passed directly to @@ -91,7 +100,7 @@ Sosreport option. Specifies a case numbe \fB\-\-cluster\-type\fR CLUSTER_TYPE When run by itself, sos collect will attempt to identify the type of cluster at play. This is done by checking package or configuration information against the localhost, or -the master node if \fB"--master"\fR is supplied. +the primary node if \fB"--primary"\fR is supplied. Setting \fB--cluster-type\fR skips this step and forcibly sets a particular profile. @@ -104,6 +113,11 @@ Example: \fBsos collect --cluster-type=k to be run, and thus set sosreport options and attempt to determine a list of nodes using that profile. .TP +\fB\-\-container\-runtime\fR RUNTIME +\fB sos report\fR option. Using this with \fBcollect\fR will pass this option thru +to nodes with sos version 4.3 or later. This option controls the default container +runtime plugins will use for collections. See \fBman sos-report\fR. +.TP \fB\-e\fR ENABLE_PLUGINS, \fB\-\-enable\-plugins\fR ENABLE_PLUGINS Sosreport option. Use this to enable a plugin that would otherwise not be run. @@ -144,10 +158,10 @@ rather than key-pair encryption. \fB\-\-group\fR GROUP Specify an existing host group definition to use. -Host groups are pre-defined settings for the cluster-type, master, and nodes options +Host groups are pre-defined settings for the cluster-type, primary node, and nodes options saved in JSON-formatted files under /var/lib/sos collect/. -If cluster_type and/or master are set in the group, sos collect behaves as if +If cluster_type and/or primary are set in the group, sos collect behaves as if these values were specified on the command-line. If nodes is defined, sos collect \fBextends\fR the \fB\-\-nodes\fR option, if set, @@ -162,8 +176,8 @@ to none. \fB\-\-save\-group\fR GROUP Save the results of this run of sos collect to a host group definition. -sos-colllector will write a JSON-formatted file with name GROUP to /var/lib/sos collect/ -with the settings for cluster-type, master, and the node list as discovered by cluster enumeration. +sos-collector will write a JSON-formatted file with name GROUP to /var/lib/sos collect/ +with the settings for cluster-type, primary, and the node list as discovered by cluster enumeration. Note that this means regexes are not directly saved to host groups, but the results of matching against those regexes are. .TP @@ -196,6 +210,18 @@ both the sos collect archive and the sos If a cluster sets a default label, the user-provided label will be appended to that cluster default. .TP +\fB \--log-size\fR SIZE +Places a limit on the size of collected logs and output in MiB. Note that this +causes sos to capture the last X amount of the file or command output collected. + +By default, this is set to 25 MiB and applies to all files and command output collected +with the exception of journal collections, which are limited to 100 MiB. + +Setting this value to 0 removes all size limitations, and any files or commands +collected will be collected in their entirety, which may drastically increase the +size of the final sos report tarball and the memory usage of sos during collection +of commands, such as very large journals that may be several GiB in size. +.TP \fB\-n\fR SKIP_PLUGINS, \fB\-\-skip\-plugins\fR SKIP_PLUGINS Sosreport option. Disable (skip) a particular plugin that would otherwise run. This is useful if a particular plugin is prone to hanging for one reason or another. @@ -214,22 +240,53 @@ names/addresses and regex strings may be Do not perform package checks. Most cluster profiles check against installed packages to determine if the cluster profile should be applied or not. -Use this with \fB\-\-cluster-type\fR if there are rpm or apt issues on the master/local node. +Use this with \fB\-\-cluster-type\fR if there are rpm or apt issues on the primary/local node. .TP \fB\-\-no\-local\fR Do not collect a sosreport from the local system. -If \fB--master\fR is not supplied, it is assumed that the host running sosreport is part of +If \fB--primary\fR is not supplied, it is assumed that the host running sosreport is part of the cluster that is to be collected. Use this option to skip collection of a local sosreport. -This option is NOT needed if \fB--master\fR is provided. +This option is NOT needed if \fB--primary\fR is provided. .TP -\fB\-\-master\fR MASTER -Specify a master node for the cluster. +\fB\-\-primary\fR PRIMARY +Specify a primary node IP address or hostname for the cluster. -If provided, then sos collect will check the master node, not localhost, for determining +If provided, then sos collect will check the primary node, not localhost, for determining the type of cluster in use. .TP +\fB\-\-image IMAGE\fR +Specify an image to use for the temporary container created for collections on +containerized host, if you do not want to use the default image specified by the +host's policy. Note that this should include the registry. +.TP +\fB\-\-force-pull-image TOGGLE, \-\-pull TOGGLE\fR +When collecting an sos report from a containerized host, force the host to always +pull the specified image, even if that image already exists on the host. +This is useful to ensure that the latest version of that image is always in use. +Disabling this option will use whatever version of the image is present on the node, +and only attempt a pull if there is no copy of the image present at all. + +Enable with true/on/yes or disable with false/off/no + +Default: true +.TP +\fB\-\-registry-user USER\fR +Specify the username to authenticate to the registry with in order to pull the container +image +.TP +\fB\-\-registry-password PASSWORD\fR +Specify the password to authenticate to the registry with in order to pull the container +image. If no password is required, leave this blank. +.TP +\fB\-\-registry-authfile FILE\fR +Specify the filename to use for providing authentication credentials to the registry +to pull the container image. + +Note that this file must exist on the node(s) performing the pull operations, not the +node from which \fBsos collect\fR was run. +.TP \fB\-o\fR ONLY_PLUGINS, \fB\-\-only\-plugins\fR ONLY_PLUGINS Sosreport option. Run ONLY the plugins listed. @@ -261,6 +318,18 @@ defined, or has a version of sos prior t \fB\-p\fR SSH_PORT, \fB\-\-ssh\-port\fR SSH_PORT Specify SSH port for all nodes. Use this if SSH runs on any port other than 22. .TP +\fB\-\-skip-commands\fR COMMANDS +A comma delimited list of commands to skip execution of, but still allowing the +rest of the plugin that calls the command to run. This will generally need to +be some form of UNIX shell-style wildcard matching. For example, using a value +of \fBhostname\fR will skip only that single command, while using \fBhostname*\fR +will skip all commands with names that begin with the string "hostname". +.TP +\fB\-\-skip-files\fR FILES +A comma delimited list of files or filepath wildcard matches to skip collection +of. Values may either be exact filepaths or paths using UNIX shell-style wildcards, +for example \fB/etc/sos/*\fR. +.TP \fB\-\-ssh\-user\fR SSH_USER Specify an SSH user for sos collect to connect to nodes with. Default is root. @@ -269,15 +338,6 @@ sos collect will prompt for a sudo passw \fB\-s\fR SYSROOT, \fB\-\-sysroot\fR SYSROOT Sosreport option. Specify an alternate root file system path. .TP -\fB\-\-sos-cmd\fR SOS_CMD -Define all options that sosreport should be run with on the nodes. This will -override any other commandline options as well as any options specified by a -cluster profile. - -The sosreport command will execute as 'sosreport --batch SOS_CMD'. The BATCH -option cannot be removed from the sosreport command as it is required to run -sosreport non-interactively for sos collect to function. -.TP \fB\-t\fR THREADS \fB\-\-threads\fR THREADS Report option. Specify the number of collection threads to run. @@ -294,6 +354,21 @@ runtime of sos collect via timeout*(numb Default is 180 seconds. .TP +\fB\-\-transport\fR TRANSPORT +Specify the type of remote transport to use to manage connections to remote nodes. + +\fBsos collect\fR uses locally installed binaries to connect to and interact with remote +nodes, instead of directly establishing those connections. By default, OpenSSH's ControlPersist +feature is preferred, however certain cluster types may have preferences of their own for how +remote sessions should be established. + +The types of transports supported are currently as follows: + + \fBauto\fR Allow the cluster type to determine the transport used + \fBcontrol_persist\fR Use OpenSSH's ControlPersist feature. This is the default behavior + \fBoc\fR Use a \fBlocally\fR configured \fBoc\fR binary to deploy collection pods on OCP nodes + +.TP \fB\-\-tmp\-dir\fR TMP_DIR Specify a temporary directory to save sos archives to. By default one will be created in /tmp and then removed after sos collect has finished running. @@ -303,7 +378,7 @@ This is NOT the same as specifying a tem \fB\-v\fR \fB\-\-verbose\fR Print debug information to screen. .TP -\fB\-\-verfiy\fR +\fB\-\-verify\fR Sosreport option. Passes the "--verify" option to sosreport on the nodes which causes sosreport to validate plugin-specific data during collection. @@ -316,6 +391,8 @@ Sosreport option. Override the default c .SH SEE ALSO .BR sos (1) .BR sos-report (1) +.BR sos-clean (1) +.BR sos.conf (5) .SH MAINTAINER Jake Hunsaker diff -pruN 4.0-2/man/en/sos-collector.1 4.5.3ubuntu2/man/en/sos-collector.1 --- 4.0-2/man/en/sos-collector.1 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/man/en/sos-collector.1 2023-04-28 17:16:21.000000000 +0000 @@ -1,4 +1,4 @@ -.TH sos collect 1 "April 2020" +.TH SOS COLLECT 1 "April 2020" .SH NAME sos collect \- Collect sosreports from multiple (cluster) nodes @@ -11,6 +11,7 @@ sos collect \- Collect sosreports from m [\-\-chroot CHROOT] [\-\-case\-id CASE_ID] [\-\-cluster\-type CLUSTER_TYPE] + [\-\-container\-runtime RUNTIME] [\-e ENABLE_PLUGINS] [--encrypt-key KEY]\fR [--encrypt-pass PASS]\fR @@ -20,21 +21,29 @@ sos collect \- Collect sosreports from m [\-\-nopasswd-sudo] [\-k PLUGIN_OPTION] [\-\-label LABEL] + [\-\-log-size SIZE] [\-n SKIP_PLUGINS] [\-\-nodes NODES] [\-\-no\-pkg\-check] [\-\-no\-local] - [\-\-master MASTER] + [\-\-primary PRIMARY] + [\-\-image IMAGE] + [\-\-force-pull-image TOGGLE, --pull TOGGLE] + [\-\-registry-user USER] + [\-\-registry-password PASSWORD] + [\-\-registry-authfile FILE] [\-o ONLY_PLUGINS] [\-p SSH_PORT] [\-\-password] [\-\-password\-per\-node] [\-\-preset PRESET] + [\-\-skip-commands COMMANDS] + [\-\-skip-files FILES] [\-s|\-\-sysroot SYSROOT] [\-\-ssh\-user SSH_USER] - [\-\-sos-cmd SOS_CMD] [\-t|\-\-threads THREADS] [\-\-timeout TIMEOUT] + [\-\-transport TRANSPORT] [\-\-tmp\-dir TMP_DIR] [\-v|\-\-verbose] [\-\-verify] @@ -46,7 +55,7 @@ collect is an sos subcommand to collect them in a single useful tar archive. sos collect can be run either on a workstation that has SSH key authentication setup -for the nodes in a given cluster, or from a "master" node in a cluster that has SSH +for the nodes in a given cluster, or from a "primary" node in a cluster that has SSH keys configured for the other nodes. Some sosreport options are supported by sos-collect and are passed directly to @@ -91,7 +100,7 @@ Sosreport option. Specifies a case numbe \fB\-\-cluster\-type\fR CLUSTER_TYPE When run by itself, sos collect will attempt to identify the type of cluster at play. This is done by checking package or configuration information against the localhost, or -the master node if \fB"--master"\fR is supplied. +the primary node if \fB"--primary"\fR is supplied. Setting \fB--cluster-type\fR skips this step and forcibly sets a particular profile. @@ -104,6 +113,11 @@ Example: \fBsos collect --cluster-type=k to be run, and thus set sosreport options and attempt to determine a list of nodes using that profile. .TP +\fB\-\-container\-runtime\fR RUNTIME +\fB sos report\fR option. Using this with \fBcollect\fR will pass this option thru +to nodes with sos version 4.3 or later. This option controls the default container +runtime plugins will use for collections. See \fBman sos-report\fR. +.TP \fB\-e\fR ENABLE_PLUGINS, \fB\-\-enable\-plugins\fR ENABLE_PLUGINS Sosreport option. Use this to enable a plugin that would otherwise not be run. @@ -144,10 +158,10 @@ rather than key-pair encryption. \fB\-\-group\fR GROUP Specify an existing host group definition to use. -Host groups are pre-defined settings for the cluster-type, master, and nodes options +Host groups are pre-defined settings for the cluster-type, primary node, and nodes options saved in JSON-formatted files under /var/lib/sos collect/. -If cluster_type and/or master are set in the group, sos collect behaves as if +If cluster_type and/or primary are set in the group, sos collect behaves as if these values were specified on the command-line. If nodes is defined, sos collect \fBextends\fR the \fB\-\-nodes\fR option, if set, @@ -162,8 +176,8 @@ to none. \fB\-\-save\-group\fR GROUP Save the results of this run of sos collect to a host group definition. -sos-colllector will write a JSON-formatted file with name GROUP to /var/lib/sos collect/ -with the settings for cluster-type, master, and the node list as discovered by cluster enumeration. +sos-collector will write a JSON-formatted file with name GROUP to /var/lib/sos collect/ +with the settings for cluster-type, primary, and the node list as discovered by cluster enumeration. Note that this means regexes are not directly saved to host groups, but the results of matching against those regexes are. .TP @@ -196,6 +210,18 @@ both the sos collect archive and the sos If a cluster sets a default label, the user-provided label will be appended to that cluster default. .TP +\fB \--log-size\fR SIZE +Places a limit on the size of collected logs and output in MiB. Note that this +causes sos to capture the last X amount of the file or command output collected. + +By default, this is set to 25 MiB and applies to all files and command output collected +with the exception of journal collections, which are limited to 100 MiB. + +Setting this value to 0 removes all size limitations, and any files or commands +collected will be collected in their entirety, which may drastically increase the +size of the final sos report tarball and the memory usage of sos during collection +of commands, such as very large journals that may be several GiB in size. +.TP \fB\-n\fR SKIP_PLUGINS, \fB\-\-skip\-plugins\fR SKIP_PLUGINS Sosreport option. Disable (skip) a particular plugin that would otherwise run. This is useful if a particular plugin is prone to hanging for one reason or another. @@ -214,22 +240,53 @@ names/addresses and regex strings may be Do not perform package checks. Most cluster profiles check against installed packages to determine if the cluster profile should be applied or not. -Use this with \fB\-\-cluster-type\fR if there are rpm or apt issues on the master/local node. +Use this with \fB\-\-cluster-type\fR if there are rpm or apt issues on the primary/local node. .TP \fB\-\-no\-local\fR Do not collect a sosreport from the local system. -If \fB--master\fR is not supplied, it is assumed that the host running sosreport is part of +If \fB--primary\fR is not supplied, it is assumed that the host running sosreport is part of the cluster that is to be collected. Use this option to skip collection of a local sosreport. -This option is NOT needed if \fB--master\fR is provided. +This option is NOT needed if \fB--primary\fR is provided. .TP -\fB\-\-master\fR MASTER -Specify a master node for the cluster. +\fB\-\-primary\fR PRIMARY +Specify a primary node IP address or hostname for the cluster. -If provided, then sos collect will check the master node, not localhost, for determining +If provided, then sos collect will check the primary node, not localhost, for determining the type of cluster in use. .TP +\fB\-\-image IMAGE\fR +Specify an image to use for the temporary container created for collections on +containerized host, if you do not want to use the default image specified by the +host's policy. Note that this should include the registry. +.TP +\fB\-\-force-pull-image TOGGLE, \-\-pull TOGGLE\fR +When collecting an sos report from a containerized host, force the host to always +pull the specified image, even if that image already exists on the host. +This is useful to ensure that the latest version of that image is always in use. +Disabling this option will use whatever version of the image is present on the node, +and only attempt a pull if there is no copy of the image present at all. + +Enable with true/on/yes or disable with false/off/no + +Default: true +.TP +\fB\-\-registry-user USER\fR +Specify the username to authenticate to the registry with in order to pull the container +image +.TP +\fB\-\-registry-password PASSWORD\fR +Specify the password to authenticate to the registry with in order to pull the container +image. If no password is required, leave this blank. +.TP +\fB\-\-registry-authfile FILE\fR +Specify the filename to use for providing authentication credentials to the registry +to pull the container image. + +Note that this file must exist on the node(s) performing the pull operations, not the +node from which \fBsos collect\fR was run. +.TP \fB\-o\fR ONLY_PLUGINS, \fB\-\-only\-plugins\fR ONLY_PLUGINS Sosreport option. Run ONLY the plugins listed. @@ -261,6 +318,18 @@ defined, or has a version of sos prior t \fB\-p\fR SSH_PORT, \fB\-\-ssh\-port\fR SSH_PORT Specify SSH port for all nodes. Use this if SSH runs on any port other than 22. .TP +\fB\-\-skip-commands\fR COMMANDS +A comma delimited list of commands to skip execution of, but still allowing the +rest of the plugin that calls the command to run. This will generally need to +be some form of UNIX shell-style wildcard matching. For example, using a value +of \fBhostname\fR will skip only that single command, while using \fBhostname*\fR +will skip all commands with names that begin with the string "hostname". +.TP +\fB\-\-skip-files\fR FILES +A comma delimited list of files or filepath wildcard matches to skip collection +of. Values may either be exact filepaths or paths using UNIX shell-style wildcards, +for example \fB/etc/sos/*\fR. +.TP \fB\-\-ssh\-user\fR SSH_USER Specify an SSH user for sos collect to connect to nodes with. Default is root. @@ -269,15 +338,6 @@ sos collect will prompt for a sudo passw \fB\-s\fR SYSROOT, \fB\-\-sysroot\fR SYSROOT Sosreport option. Specify an alternate root file system path. .TP -\fB\-\-sos-cmd\fR SOS_CMD -Define all options that sosreport should be run with on the nodes. This will -override any other commandline options as well as any options specified by a -cluster profile. - -The sosreport command will execute as 'sosreport --batch SOS_CMD'. The BATCH -option cannot be removed from the sosreport command as it is required to run -sosreport non-interactively for sos collect to function. -.TP \fB\-t\fR THREADS \fB\-\-threads\fR THREADS Report option. Specify the number of collection threads to run. @@ -294,6 +354,21 @@ runtime of sos collect via timeout*(numb Default is 180 seconds. .TP +\fB\-\-transport\fR TRANSPORT +Specify the type of remote transport to use to manage connections to remote nodes. + +\fBsos collect\fR uses locally installed binaries to connect to and interact with remote +nodes, instead of directly establishing those connections. By default, OpenSSH's ControlPersist +feature is preferred, however certain cluster types may have preferences of their own for how +remote sessions should be established. + +The types of transports supported are currently as follows: + + \fBauto\fR Allow the cluster type to determine the transport used + \fBcontrol_persist\fR Use OpenSSH's ControlPersist feature. This is the default behavior + \fBoc\fR Use a \fBlocally\fR configured \fBoc\fR binary to deploy collection pods on OCP nodes + +.TP \fB\-\-tmp\-dir\fR TMP_DIR Specify a temporary directory to save sos archives to. By default one will be created in /tmp and then removed after sos collect has finished running. @@ -303,7 +378,7 @@ This is NOT the same as specifying a tem \fB\-v\fR \fB\-\-verbose\fR Print debug information to screen. .TP -\fB\-\-verfiy\fR +\fB\-\-verify\fR Sosreport option. Passes the "--verify" option to sosreport on the nodes which causes sosreport to validate plugin-specific data during collection. @@ -316,6 +391,8 @@ Sosreport option. Override the default c .SH SEE ALSO .BR sos (1) .BR sos-report (1) +.BR sos-clean (1) +.BR sos.conf (5) .SH MAINTAINER Jake Hunsaker diff -pruN 4.0-2/man/en/sos-help.1 4.5.3ubuntu2/man/en/sos-help.1 --- 4.0-2/man/en/sos-help.1 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/man/en/sos-help.1 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,41 @@ +.TH SOS HELP 1 "Fri Nov 05 2021" +.SH NAME +sos help - get detailed help information on sos commands and components +.SH SYNOPSIS +.B sos help TOPIC + +.SH DESCRIPTION +\fBsos help\fR is used to retrieve more detailed information on the various SoS +commands and components than is directly available in either other manpages or +--help output. + +This information could for example be investigating a specific plugin to learn more +about its purpose, use case, collections, available plugin options, edge cases, and +more. +.LP +Most aspects of SoS' operation can be investigated this way - the top level functions +such as \fB report, clean,\fR and \fBcollect\fR, as well as constructs that allow those +functions to work; e.g. \fBtransports\fR within \fBsos collect\fR that define how that +function connects to remote nodes. + +.SH REQUIRED ARGUMENTS +.B TOPIC +.TP +The section or topic to retrieve detailed help information for. TOPIC takes the general +form of \fBcommand.component.entity\fR, with \fBcomponent\fR and \fBentity\fR +being optional. +.LP +Top-level \fBcommand\fR help sections will often direct users to \fBcomponent\fR sections +which in turn may point to further \fBentity\fR subsections. + +Some of the more useful or interesting sections are listed below: + + \fBTopic\fR \fBDescription\fR + + \fBreport\fR The \fBsos report\fR command + \fBreport.plugins\fR Information on what report plugins are + \fBreport.plugins.$plugin\fR Information on a specific plugin + \fBclean\fR or \fBmask\fR The \fBsos clean|mask\fR command + \fBcollect\fR The \fBsos collect\fR command + \fBcollect.clusters\fR How \fBcollect\fR enumerates nodes in a cluster + \fBpolicies\fR How SoS behaves on different distributions diff -pruN 4.0-2/man/en/sos-mask.1 4.5.3ubuntu2/man/en/sos-mask.1 --- 4.0-2/man/en/sos-mask.1 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/man/en/sos-mask.1 2023-04-28 17:16:21.000000000 +0000 @@ -4,9 +4,14 @@ sos clean - Obfuscate sensitive data fro .SH SYNOPSIS .B sos clean TARGET [options] [\-\-domains] - [\-\-map] + [\-\-disable-parsers] + [\-\-keywords] + [\-\-keyword-file] + [\-\-map-file] [\-\-jobs] [\-\-no-update] + [\-\-keep-binary-files] + [\-\-archive-type] .SH DESCRIPTION \fBsos clean\fR or \fBsos mask\fR is an sos subcommand used to obfuscate sensitive information from @@ -47,6 +52,17 @@ match a domain given via this option wil For example, if \fB\-\-domains redhat.com\fR is specified, then 'redhat.com' will be obfuscated, as will 'www.redhat.com' and subdomains such as 'foo.redhat.com'. .TP +.B \-\-disable-parsers PARSERS +Provide a comma-delimited list of parsers to disable when cleaning an archive. By +default all parsers are enabled. + +Note that using this option is very likely to leave sensitive information in place in +the target archive, so only use this option when absolutely necessary or you have complete +trust in the party/parties that may handle the generated report. + +Valid values for this option are currently: \fBhostname\fR, \fBip\fR, \fBipv6\fR, +\fBmac\fR, \fBkeyword\fR, and \fBusername\fR. +.TP .B \-\-keywords KEYWORDS Provide a comma-delimited list of keywords to scrub in addition to the default parsers. @@ -54,7 +70,11 @@ Keywords provided by this option will be integer based on the keyword's index in the parser. Note that keywords will be replaced as both standalone words and in substring matches. .TP -.B \-\-map FILE +.B \-\-keyword-file FILE +Provide a file that contains a list of keywords that should be obfuscated. Each word must +be specified on a newline within the file. +.TP +.B \-\-map-file FILE Provide a location to a valid mapping file to use as a reference for existing obfuscation pairs. If one is found, the contents are loaded before parsing is started. This allows consistency between runs of this command for obfuscated pairs. By default, sos will write the generated private map file @@ -71,6 +91,48 @@ Default: 4 .TP .B \-\-no-update Do not write the mapping file contents to /etc/sos/cleaner/default_mapping +.TP +.B \-\-keep-binary-files +Keep unprocessable binary files in the archive, rather than removing them. + +Note that binary files cannot be obfuscated, and thus keeping them in the archive +may result in otherwise sensitive information being included in the final archive. +Users should review any archive that keeps binary files in place before sending to +a third party. + +Default: False (remove encountered binary files) +.TP +.B \-\-archive-type TYPE +Specify the type of archive that TARGET was generated as. +When sos inspects a TARGET archive, it tries to identify what type of archive it is. +For example, it may be a report generated by \fBsos report\fR, or a collection of those +reports generated by \fBsos collect\fR, which require separate approaches. + +This option may be useful if a given TARGET archive is known to be of a specific type, +but due to unknown reasons or some malformed/missing information in the archive directly, +that is not properly identified by sos. + +The following are accepted values for this option: + + \fBauto\fR Automatically detect the archive type + \fBreport\fR An archive generated by \fBsos report\fR + \fBcollect\fR An archive generated by \fBsos collect\fR + \fBinsights\fR An archive generated by the \fBinsights-client\fR package + +The following may also be used, however note that these do not attempt to pre-load +any information from the archives into the parsers. This means that, among other limitations, +items like host and domain names may not be obfuscated unless an obfuscated mapping already exists +on the system from a previous execution. + + \fBdata-dir\fR A plain directory on the filesystem. + \fBtarball\fR A generic tar archive not associated with any known tool + +.SH SEE ALSO +.BR sos (1) +.BR sos-report (1) +.BR sos-collect (1) +.BR sos.conf (5) + .SH MAINTAINER .nf Jake Hunsaker diff -pruN 4.0-2/man/en/sos-report.1 4.5.3ubuntu2/man/en/sos-report.1 --- 4.0-2/man/en/sos-report.1 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/man/en/sos-report.1 2023-04-28 17:16:21.000000000 +0000 @@ -1,8 +1,8 @@ -.TH SOSREPORT 1 "Mon Mar 25 2013" +.TH SOS REPORT 1 "Mon Mar 25 2013" .SH NAME -sosreport \- Collect and package diagnostic and support data +sos report \- Collect and package diagnostic and support data .SH SYNOPSIS -.B sosreport +.B sos report [-l|--list-plugins]\fR [-n|--skip-plugins plugin-names]\fR [-e|--enable-plugins plugin-names]\fR @@ -14,9 +14,12 @@ sosreport \- Collect and package diagnos [--preset preset] [--add-preset add_preset]\fR [--del-preset del_preset] [--desc description]\fR [--batch] [--build] [--debug] [--dry-run]\fR - [--label label] [--case-id id] [--ticket-number nr]\fR + [--estimate-only] [--label label] [--case-id id]\fR [--threads threads]\fR [--plugin-timeout TIMEOUT]\fR + [--cmd-timeout TIMEOUT]\fR + [--namespaces NAMESPACES]\fR + [--container-runtime RUNTIME]\fR [-s|--sysroot SYSROOT]\fR [-c|--chroot {auto|always|never}\fR [--tmp-dir directory]\fR @@ -24,23 +27,31 @@ sosreport \- Collect and package diagnos [--list-profiles]\fR [--verify]\fR [--log-size]\fR + [--journal-size]\fR [--all-logs]\fR [--since YYYYMMDD[HHMMSS]]\fR + [--skip-commands commands]\fR + [--skip-files files]\fR [--allow-system-changes]\fR + [--low-priority]\fR [-z|--compression-type method]\fR + [--encrypt]\fR [--encrypt-key KEY]\fR [--encrypt-pass PASS]\fR [--upload] [--upload-url url] [--upload-user user]\fR [--upload-directory dir] [--upload-pass pass]\fR + [--upload-no-ssl-verify] [--upload-method]\fR + [--upload-protocol protocol]\fR [--experimental]\fR [-h|--help]\fR .SH DESCRIPTION -\fBsosreport\fR generates an archive of configuration and diagnostic -information from the running system. The archive may be stored locally -or centrally for recording or tracking purposes or may be sent to -technical support representatives, developers or system administrators -to assist with technical fault-finding and debugging. +\fBreport\fR is an sos subcommand that generates an archive of +configuration and diagnostic information from the running system. +The archive may be stored locally or centrally for recording or +tracking purposes or may be sent to technical support representatives, +developers or system administrators to assist with technical +fault-finding and debugging. .LP Sos is modular in design and is able to collect data from a wide range of subsystems and packages that may be installed. An @@ -57,8 +68,13 @@ Disable the specified plugin(s). Multipl by repeating the option or as a comma-separated list. .TP .B \-e, --enable-plugins PLUGNAME[,PLUGNAME] -Enable the specified plugin(s). Multiple plug-ins may be specified -by repeating the option or as a comma-separated list. +Enable the specified plugin(s) that would otherwise be disabled. Multiple plugins +may be specified by repeating the option or as a comma-separated list. + +Note that if using \fB-p, --profile\fR this option will \fBnot\fR enable further +plugins. Use \fB-o, --only-plugins\fR to extend the list of plugins enabled by +profiles. + .TP .B \-o, --only-plugins PLUGNAME[,PLUGNAME] Enable the specified plugin(s) only (all other plugins should be @@ -108,8 +124,8 @@ User defined presets are saved under /va .B \--add-preset ADD_PRESET [options] Add a preset with name ADD_PRESET that enables [options] when called. -For example, 'sosreport --add-preset mypreset --log-size=50 -n logs' will enable -a user to run 'sosreport --preset mypreset' that sets the maximum log size to +For example, 'sos report --add-preset mypreset --log-size=50 -n logs' will enable +a user to run 'sos report --preset mypreset' that sets the maximum log size to 50 and disables the logs plugin. Note: to set a description for the preset that is displayed with \fB--list-presets\fR, @@ -149,13 +165,16 @@ compressed report. .B \--list-profiles Display a list of available profiles and the plugins that they enable. .TP -.B \-p, \--profile NAME +.B \-p, \--profile, \--profiles NAME Only run plugins that correspond to the given profile. Multiple profiles may be specified as a comma-separated list; the set of plugins executed -is the union of each of the profile's plugin sets. Currently defined -profiles include: boot, cluster, desktop, debug, hardware, identity, -network, openstack, packagemanager, security, services, storage, -sysmgmt, system, performance, virt, and webserver. +is the union of each of the profile's plugin sets. + +Note that if there are specific plugins outside of the profile(s) passed to this +option that you would also want to enable, use \fB-o, --only-plugins\fR to add those +plugins to the list. + +See \fBsos report --list-profiles\fR for a list of currently supported profiles. .TP .B \--verify Instructs plugins to perform plugin-specific verification during data @@ -164,9 +183,26 @@ testing or other plugin defined behaviou the time taken to generate a report to be considerably longer. .TP .B \--log-size -Places a global limit on the size (in MiB) of any collected set of logs. The -limit is applied separately for each set of logs collected by any -plugin. +Places a limit on the size of collected logs and output in MiB. Note that this +causes sos to capture the last X amount of the file or command output collected. + +By default, this is set to 25 MiB and applies to all files and command output collected +with the exception of journal collections, which are limited by the \fB--journal-size\fR +option instead. + +Setting this value to 0 removes all size limitations, and any files or commands +collected will be collected in their entirety, which may drastically increase the +size of the final sos report tarball and the memory usage of sos during collection +of commands. + +.TP +.B \--journal-size +Places a limit on the size of journals collected in MiB. Note that this causes sos +to capture the last X amount of the journal. + +By default, this is set to 100 MiB. Setting this value to 0 removes all size limitations, +as does the use of the \fB--all-logs\fR option. This may drastically increase the size +of the final sos report tarball. .TP .B \--all-logs Tell plugins to collect all possible log data ignoring any size limits @@ -180,12 +216,49 @@ compression-type file extension for exam This also affects \--all-logs. The date string will be padded with zeros if HHMMSS is not specified. .TP +.B \--skip-commands COMMANDS +A comma delimited list of commands to skip execution of, but still allowing the +rest of the plugin that calls the command to run. This will generally need to +be some form of UNIX shell-style wildcard matching. For example, using a value +of \fBhostname\fR will skip only that single command, while using \fBhostname*\fR +will skip all commands with names that begin with the string "hostname". +.TP +.B \--skip-files FILES +A comma delimited list of files or filepath wildcard matches to skip collection +of. Values may either be exact filepaths or paths using UNIX shell-style wildcards, +for example \fB/etc/sos/*\fR. +.TP .B \--allow-system-changes Run commands even if they can change the system (e.g. load kernel modules). .TP +.B \--low-priority +Set sos to execute as a low priority process so that is does not interfere with +other processes running on the system. Specific distributions may set their own +constraints, but by default this involves setting process niceness to 19 and, if +available, setting an idle IO class via ionice. .B \-z, \--compression-type METHOD Override the default compression type specified by the active policy. .TP +.B \-\-encrypt +Encrypt the resulting archive, and determine the method by which that encryption +is done by either a user prompt or environment variables. + +When run with \fB--batch\fR, using this option will cause sos to look for either the +\fBSOSENCRYPTKEY\fR or \fBSOSENCRYPTPASS\fR environment variables. If set, this will +implicitly enable the \fB--encrypt-key\fR or \fB--encrypt-pass\fR options, respectively, +to the values set by the environment variable. This enables the use of these options +without directly setting those options in a config file or command line string. Note that +use of an encryption key has precedence over a passphrase. + +Otherwise, using this option will cause sos to prompt the user to choose the method +of encryption to use. Choices will be [P]assphrase, [K]ey, [E]nv vars, or [N]o encryption. +If passphrase or key the user will then be prompted for the respective value, env vars will +cause sos to source the information in the manner stated above, and choosing no encryption +will disable encryption. + +See the sections on \fB--encrypt-key\fR and \fB--encrypt-pass\fR below for more +information. +.TP .B \--encrypt-key KEY Encrypts the resulting archive that sosreport produces using GPG. KEY must be an existing key in the user's keyring as GPG does not allow for keyfiles. @@ -229,9 +302,10 @@ Specify the number of threads sosreport .TP .B \--plugin-timeout TIMEOUT Specify a timeout in seconds to allow each plugin to run for. A value of 0 -means no timeout will be set. +means no timeout will be set. A value of -1 is used to indicate the default +timeout of 300 seconds. -Note that this options sets the timeout for all plugins. If you want to set +Note that this option sets the timeout for all plugins. If you want to set a timeout for a specific plugin, use the 'timeout' plugin option available to all plugins - e.g. '-k logs.timeout=600'. @@ -239,15 +313,51 @@ The plugin-specific timeout option will \'--plugin-timeout=60 -k logs.timeout=600\' will set a timeout of 600 seconds for the logs plugin and 60 seconds for all other enabled plugins. .TP +.B \--cmd-timeout TIMEOUT +Specify a timeout limit in seconds for a command execution. Same defaults logic +from --plugin-timeout applies here. + +This option sets the command timeout for all plugins. If you want to set a cmd +timeout for a specific plugin, use the 'cmd-timeout' plugin option available to +all plugins - e.g. '-k logs.cmd-timeout=600'. + +Again, the same plugin/global precedence logic as for --plugin-timeout applies +here. + +Note that setting --cmd-timeout (or -k logs.cmd-timeout) high should be followed +by increasing the --plugin-timeout equivalent, otherwise the plugin can easily +timeout on slow commands execution. +.TP +.B \--namespaces NAMESPACES +For plugins that iterate collections over namespaces that exist on the system, +for example the networking plugin collecting `ip` command output for each network +namespace, use this option to limit the number of namespaces that will be collected. + +Use '0' (default) for no limit - all namespaces will be used for collections. + +Note that specific plugins may provide a similar `namespaces` plugin option. If +the plugin option is used, it will override this option. +.TP +.B \--container-runtime RUNTIME +Force the use of the specified RUNTIME as the default runtime that plugins will +use to collect data from and about containers and container images. By default, +the setting of \fBauto\fR results in the local policy determining what runtime +will be the default runtime (in configurations where multiple runtimes are installed +and active). + +If no container runtimes are active, this option is ignored. If there are runtimes +active, but not one with a name matching RUNTIME, sos will abort. + +Setting this to \fBnone\fR, \fBoff\fR, or \fBdisabled\fR will cause plugins to +\fBNOT\fR leverage any active runtimes for collections. Note that if disabled, plugins +specifically for runtimes (e.g. the podman or docker plugins) will still collect +general data about the runtime, but will not inspect existing containers or images. + +Default: 'auto' (policy determined) +.TP .B \--case-id NUMBER Specify a case identifier to associate with the archive. Identifiers may include alphanumeric characters, commas and periods ('.'). -Synonymous with \--ticket-number. -.TP -.B \--ticket-number NUMBER -Specify a ticket number or other identifier to associate with the archive. -Identifiers may include alphanumeric characters, commas and periods ('.'). -Synonymous with \--case-id. .TP .B \--build Do not archive copied data. Causes sosreport to leave an uncompressed @@ -263,6 +373,21 @@ output, or string data from the system. to understand the actions that sos would have taken without the dry run option. .TP +.B \--estimate-only +Estimate disk space requirements when running sos report. This can be valuable +to prevent sosreport working dir to consume all free disk space. No plugin data +is available at the end. + +Plugins will be collected sequentially, size of collected files and commands outputs +will be calculated and the plugin files will be immediatelly deleted prior execution +of the next plugin. This still can consume whole free disk space, though. + +Please note, size estimations may not be accurate for highly utilized systems due to +changes between an estimate and a real execution. Also some difference between +estimation (using `stat` command) and other commands used (i.e. `du`). + +A rule of thumb is to reserve at least double the estimation. +.TP .B \--upload If specified, attempt to upload the resulting archive to a vendor defined location. @@ -299,6 +424,12 @@ If a vendor does not provide a default u If this option is unused and upload is request, and a vendor default is not set, you will be prompted for one. If --batch is used and this option is omitted, no username will be collected and thus uploads will fail if no vendor default is set. + +You also have the option of providing this value via the SOSUPLOADUSER environment +variable. If this variable is set, then no username prompt will occur and --batch +may be used provided all other required values (case number, upload password) +are provided. + .TP .B \-\-upload-pass PASS Specify the password to use for authentication with the destination server. @@ -312,20 +443,58 @@ Note that this will result in the plaint be collected by sos and be in the archive. If a password must be provided by you for uploading, it is strongly recommended to not use --batch and enter the password when prompted rather than using this option. + +You also have the option of providing this value via the SOSUPLOADPASSWORD environment +variable. If this variable is set, then no password prompt will occur and --batch may +be used provided all other required values (case number, upload user) are provided. + .TP .B \--upload-directory DIR Specify a directory to upload to, if one is not specified by a vendor default location or if your destination server does not allow writes to '/'. .TP +.B \--upload-method METHOD +Specify the HTTP method to use for uploading to the provided --upload-url. Valid +values are 'auto' (default), 'put', or 'post'. The use of 'auto' will default to +the method required by the policy-default upload location, if one exists. + +This option has no effect on upload protocols other than HTTPS. +.TP +.B \--upload-no-ssl-verify +Disable SSL verification for HTTPS uploads. This may be used to allow uploading +to locations that have self-signed certificates, or certificates that are otherwise +untrusted by the local system. + +Default behavior is to perform SSL verification against all upload locations. +.TP +.B \--upload-protocol PROTO +Manually specify the protocol to use for uploading to the target \fBupload-url\fR. + +Normally this is determined via the upload address, assuming that the protocol is part +of the address provided, e.g. 'https://example.com'. By using this option, sos will skip +the protocol check and use the method defined for the specified PROTO. + +For RHEL systems, setting this option to \fBsftp\fR will skip the initial attempt to +upload to the Red Hat Customer Portal, and only attempt an upload to Red Hat's SFTP server, +which is typically used as a fallback target. + +Valid values for PROTO are: 'auto' (default), 'https', 'ftp', 'sftp'. +.TP .B \--experimental Enable plugins marked as experimental. Experimental plugins may not have been tested for this port or may still be under active development. .TP .B \--help Display usage message. +.SH SEE ALSO +.BR sos (1) +.BR sos-clean (1) +.BR sos-collect (1) +.BR sos.conf (5) + .SH MAINTAINER .nf -Bryn M. Reeves +Jake Hunsaker .fi .SH AUTHORS & CONTRIBUTORS See \fBAUTHORS\fR file in the package documentation. diff -pruN 4.0-2/man/en/sos.1 4.5.3ubuntu2/man/en/sos.1 --- 4.0-2/man/en/sos.1 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/man/en/sos.1 2023-04-28 17:16:21.000000000 +0000 @@ -37,7 +37,7 @@ May also be invoked via the alias \fBrep .B collect Collect is used to capture reports on multiple systems simultaneously. These systems can either be defined by the user at the command line and/or defined by -clustering software that exists either on the local system or on a "master" system +clustering software that exists either on the local system or on a "primary" system that is able to inform about other nodes in the cluster. When running collect, sos report will be run on the remote nodes, and then the @@ -50,7 +50,7 @@ May also be invoked via the alias \fBsos \fBsos-collector\fR. .TP -.B clean|mask +.B clean|cleaner|mask This subcommand takes input of either 1) an sosreport tarball, 2) a collection of sosreport tarballs such as from \fBcollect\fR, or 3) the unpackaged directory of an sosreport and obfuscates potentially sensitive system information @@ -63,9 +63,18 @@ between matched data items. See \fB sos clean --help\fR and \fBman sos-clean\fR for more information. -May be invoked via either \fBsos clean\fR, \fBsos mask\fR, or via the \fB--clean\fR or \fB --mask\fR options +May be invoked via either \fBsos clean\fR, \fBsos cleaner\fR, \fBsos mask\fR, +or via the \fB--clean\fR, \fB--cleaner\fR or \fB --mask\fR options for \fBreport\fR and \fBcollect\fR. +.TP +.B help +This subcommand is used to retrieve more detailed information on the various SoS +commands and components than is directly available in either other manpages or +--help output. + +See \fB sos help --help\fR and \fB man sos-help\fR for more information. + .SH GLOBAL OPTIONS sos components provide their own set of options, however the following are available to be set across all components. @@ -73,6 +82,26 @@ to be set across all components. .B \-\-batch Do not prompt interactively, user will not be prompted for any data .TP +.B \-\-encrypt +Encrypt the resulting archive, and determine the method by which that encryption +is done by either a user prompt or environment variables. + +When run with \fB--batch\fR, using this option will cause sos to look for either the +\fBSOSENCRYPTKEY\fR or \fBSOSENCRYPTPASS\fR environment variables. If set, this will +implicitly enable the \fB--encrypt-key\fR or \fB--encrypt-pass\fR options, respectively, +to the values set by the environment variable. This enables the use of these options +without directly setting those options in a config file or command line string. Note that +use of an encryption key has precedence over a passphrase. + +Otherwise, using this option will cause sos to prompt the user to choose the method +of encryption to use. Choices will be [P]assphrase, [K]ey, [E]nv vars, or [N]o encryption. +If passphrase or key the user will then be prompted for the respective value, env vars will +cause sos to source the information in the manner stated above, and choosing no encryption +will disable encryption. + +See the sections on \fB--encrypt-key\fR and \fB--encrypt-pass\fR below for more +information. +.TP .B \--encrypt-key KEY Encrypts the resulting archive that sosreport produces using GPG. KEY must be an existing key in the user's keyring as GPG does not allow for keyfiles. @@ -115,6 +144,14 @@ Specify the number of threads sosreport .B \-v, \--verbose Increase logging verbosity. May be specified multiple times to enable additional debugging messages. + +The following table summarizes the effects of different verbosity levels: + + 1 (-v) : Enable debug messages for sos.log. Show individual plugins starting. + 2 (-vv) : Also print debug messages to console. + 3 (-vvv) : Enable debug messages for archive file operations. Note this will dramatically + increase the amount of logging. + .TP .B \-q, \--quiet Only log fatal errors to stderr. @@ -124,6 +161,8 @@ Compression type to use when compression .TP .B \--help Display usage message. +.SH SEE ALSO +.BR sos.conf (5) .SH MAINTAINER .nf Jake Hunsaker diff -pruN 4.0-2/man/en/sos.conf.5 4.5.3ubuntu2/man/en/sos.conf.5 --- 4.0-2/man/en/sos.conf.5 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/man/en/sos.conf.5 2023-04-28 17:16:21.000000000 +0000 @@ -54,7 +54,7 @@ Expected content of an extras file is as \fBgroups.d\fP This directory is used to store host group configuration files for \fBsos collect\fP. -These files can specify any/all of the \fBmaster\fP, \fBnodes\fP, and \fBcluster-type\fP +These files can specify any/all of the \fBprimary\fP, \fBnodes\fP, and \fBcluster-type\fP options. Users may create their own private host groups in $HOME/.config/sos/groups.d/. If diff -pruN 4.0-2/man/en/sosreport.1 4.5.3ubuntu2/man/en/sosreport.1 --- 4.0-2/man/en/sosreport.1 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/man/en/sosreport.1 2023-04-28 17:16:21.000000000 +0000 @@ -1,8 +1,8 @@ -.TH SOSREPORT 1 "Mon Mar 25 2013" +.TH SOS REPORT 1 "Mon Mar 25 2013" .SH NAME -sosreport \- Collect and package diagnostic and support data +sos report \- Collect and package diagnostic and support data .SH SYNOPSIS -.B sosreport +.B sos report [-l|--list-plugins]\fR [-n|--skip-plugins plugin-names]\fR [-e|--enable-plugins plugin-names]\fR @@ -14,9 +14,12 @@ sosreport \- Collect and package diagnos [--preset preset] [--add-preset add_preset]\fR [--del-preset del_preset] [--desc description]\fR [--batch] [--build] [--debug] [--dry-run]\fR - [--label label] [--case-id id] [--ticket-number nr]\fR + [--estimate-only] [--label label] [--case-id id]\fR [--threads threads]\fR [--plugin-timeout TIMEOUT]\fR + [--cmd-timeout TIMEOUT]\fR + [--namespaces NAMESPACES]\fR + [--container-runtime RUNTIME]\fR [-s|--sysroot SYSROOT]\fR [-c|--chroot {auto|always|never}\fR [--tmp-dir directory]\fR @@ -24,23 +27,31 @@ sosreport \- Collect and package diagnos [--list-profiles]\fR [--verify]\fR [--log-size]\fR + [--journal-size]\fR [--all-logs]\fR [--since YYYYMMDD[HHMMSS]]\fR + [--skip-commands commands]\fR + [--skip-files files]\fR [--allow-system-changes]\fR + [--low-priority]\fR [-z|--compression-type method]\fR + [--encrypt]\fR [--encrypt-key KEY]\fR [--encrypt-pass PASS]\fR [--upload] [--upload-url url] [--upload-user user]\fR [--upload-directory dir] [--upload-pass pass]\fR + [--upload-no-ssl-verify] [--upload-method]\fR + [--upload-protocol protocol]\fR [--experimental]\fR [-h|--help]\fR .SH DESCRIPTION -\fBsosreport\fR generates an archive of configuration and diagnostic -information from the running system. The archive may be stored locally -or centrally for recording or tracking purposes or may be sent to -technical support representatives, developers or system administrators -to assist with technical fault-finding and debugging. +\fBreport\fR is an sos subcommand that generates an archive of +configuration and diagnostic information from the running system. +The archive may be stored locally or centrally for recording or +tracking purposes or may be sent to technical support representatives, +developers or system administrators to assist with technical +fault-finding and debugging. .LP Sos is modular in design and is able to collect data from a wide range of subsystems and packages that may be installed. An @@ -57,8 +68,13 @@ Disable the specified plugin(s). Multipl by repeating the option or as a comma-separated list. .TP .B \-e, --enable-plugins PLUGNAME[,PLUGNAME] -Enable the specified plugin(s). Multiple plug-ins may be specified -by repeating the option or as a comma-separated list. +Enable the specified plugin(s) that would otherwise be disabled. Multiple plugins +may be specified by repeating the option or as a comma-separated list. + +Note that if using \fB-p, --profile\fR this option will \fBnot\fR enable further +plugins. Use \fB-o, --only-plugins\fR to extend the list of plugins enabled by +profiles. + .TP .B \-o, --only-plugins PLUGNAME[,PLUGNAME] Enable the specified plugin(s) only (all other plugins should be @@ -108,8 +124,8 @@ User defined presets are saved under /va .B \--add-preset ADD_PRESET [options] Add a preset with name ADD_PRESET that enables [options] when called. -For example, 'sosreport --add-preset mypreset --log-size=50 -n logs' will enable -a user to run 'sosreport --preset mypreset' that sets the maximum log size to +For example, 'sos report --add-preset mypreset --log-size=50 -n logs' will enable +a user to run 'sos report --preset mypreset' that sets the maximum log size to 50 and disables the logs plugin. Note: to set a description for the preset that is displayed with \fB--list-presets\fR, @@ -149,13 +165,16 @@ compressed report. .B \--list-profiles Display a list of available profiles and the plugins that they enable. .TP -.B \-p, \--profile NAME +.B \-p, \--profile, \--profiles NAME Only run plugins that correspond to the given profile. Multiple profiles may be specified as a comma-separated list; the set of plugins executed -is the union of each of the profile's plugin sets. Currently defined -profiles include: boot, cluster, desktop, debug, hardware, identity, -network, openstack, packagemanager, security, services, storage, -sysmgmt, system, performance, virt, and webserver. +is the union of each of the profile's plugin sets. + +Note that if there are specific plugins outside of the profile(s) passed to this +option that you would also want to enable, use \fB-o, --only-plugins\fR to add those +plugins to the list. + +See \fBsos report --list-profiles\fR for a list of currently supported profiles. .TP .B \--verify Instructs plugins to perform plugin-specific verification during data @@ -164,9 +183,26 @@ testing or other plugin defined behaviou the time taken to generate a report to be considerably longer. .TP .B \--log-size -Places a global limit on the size (in MiB) of any collected set of logs. The -limit is applied separately for each set of logs collected by any -plugin. +Places a limit on the size of collected logs and output in MiB. Note that this +causes sos to capture the last X amount of the file or command output collected. + +By default, this is set to 25 MiB and applies to all files and command output collected +with the exception of journal collections, which are limited by the \fB--journal-size\fR +option instead. + +Setting this value to 0 removes all size limitations, and any files or commands +collected will be collected in their entirety, which may drastically increase the +size of the final sos report tarball and the memory usage of sos during collection +of commands. + +.TP +.B \--journal-size +Places a limit on the size of journals collected in MiB. Note that this causes sos +to capture the last X amount of the journal. + +By default, this is set to 100 MiB. Setting this value to 0 removes all size limitations, +as does the use of the \fB--all-logs\fR option. This may drastically increase the size +of the final sos report tarball. .TP .B \--all-logs Tell plugins to collect all possible log data ignoring any size limits @@ -180,12 +216,49 @@ compression-type file extension for exam This also affects \--all-logs. The date string will be padded with zeros if HHMMSS is not specified. .TP +.B \--skip-commands COMMANDS +A comma delimited list of commands to skip execution of, but still allowing the +rest of the plugin that calls the command to run. This will generally need to +be some form of UNIX shell-style wildcard matching. For example, using a value +of \fBhostname\fR will skip only that single command, while using \fBhostname*\fR +will skip all commands with names that begin with the string "hostname". +.TP +.B \--skip-files FILES +A comma delimited list of files or filepath wildcard matches to skip collection +of. Values may either be exact filepaths or paths using UNIX shell-style wildcards, +for example \fB/etc/sos/*\fR. +.TP .B \--allow-system-changes Run commands even if they can change the system (e.g. load kernel modules). .TP +.B \--low-priority +Set sos to execute as a low priority process so that is does not interfere with +other processes running on the system. Specific distributions may set their own +constraints, but by default this involves setting process niceness to 19 and, if +available, setting an idle IO class via ionice. .B \-z, \--compression-type METHOD Override the default compression type specified by the active policy. .TP +.B \-\-encrypt +Encrypt the resulting archive, and determine the method by which that encryption +is done by either a user prompt or environment variables. + +When run with \fB--batch\fR, using this option will cause sos to look for either the +\fBSOSENCRYPTKEY\fR or \fBSOSENCRYPTPASS\fR environment variables. If set, this will +implicitly enable the \fB--encrypt-key\fR or \fB--encrypt-pass\fR options, respectively, +to the values set by the environment variable. This enables the use of these options +without directly setting those options in a config file or command line string. Note that +use of an encryption key has precedence over a passphrase. + +Otherwise, using this option will cause sos to prompt the user to choose the method +of encryption to use. Choices will be [P]assphrase, [K]ey, [E]nv vars, or [N]o encryption. +If passphrase or key the user will then be prompted for the respective value, env vars will +cause sos to source the information in the manner stated above, and choosing no encryption +will disable encryption. + +See the sections on \fB--encrypt-key\fR and \fB--encrypt-pass\fR below for more +information. +.TP .B \--encrypt-key KEY Encrypts the resulting archive that sosreport produces using GPG. KEY must be an existing key in the user's keyring as GPG does not allow for keyfiles. @@ -229,9 +302,10 @@ Specify the number of threads sosreport .TP .B \--plugin-timeout TIMEOUT Specify a timeout in seconds to allow each plugin to run for. A value of 0 -means no timeout will be set. +means no timeout will be set. A value of -1 is used to indicate the default +timeout of 300 seconds. -Note that this options sets the timeout for all plugins. If you want to set +Note that this option sets the timeout for all plugins. If you want to set a timeout for a specific plugin, use the 'timeout' plugin option available to all plugins - e.g. '-k logs.timeout=600'. @@ -239,15 +313,51 @@ The plugin-specific timeout option will \'--plugin-timeout=60 -k logs.timeout=600\' will set a timeout of 600 seconds for the logs plugin and 60 seconds for all other enabled plugins. .TP +.B \--cmd-timeout TIMEOUT +Specify a timeout limit in seconds for a command execution. Same defaults logic +from --plugin-timeout applies here. + +This option sets the command timeout for all plugins. If you want to set a cmd +timeout for a specific plugin, use the 'cmd-timeout' plugin option available to +all plugins - e.g. '-k logs.cmd-timeout=600'. + +Again, the same plugin/global precedence logic as for --plugin-timeout applies +here. + +Note that setting --cmd-timeout (or -k logs.cmd-timeout) high should be followed +by increasing the --plugin-timeout equivalent, otherwise the plugin can easily +timeout on slow commands execution. +.TP +.B \--namespaces NAMESPACES +For plugins that iterate collections over namespaces that exist on the system, +for example the networking plugin collecting `ip` command output for each network +namespace, use this option to limit the number of namespaces that will be collected. + +Use '0' (default) for no limit - all namespaces will be used for collections. + +Note that specific plugins may provide a similar `namespaces` plugin option. If +the plugin option is used, it will override this option. +.TP +.B \--container-runtime RUNTIME +Force the use of the specified RUNTIME as the default runtime that plugins will +use to collect data from and about containers and container images. By default, +the setting of \fBauto\fR results in the local policy determining what runtime +will be the default runtime (in configurations where multiple runtimes are installed +and active). + +If no container runtimes are active, this option is ignored. If there are runtimes +active, but not one with a name matching RUNTIME, sos will abort. + +Setting this to \fBnone\fR, \fBoff\fR, or \fBdisabled\fR will cause plugins to +\fBNOT\fR leverage any active runtimes for collections. Note that if disabled, plugins +specifically for runtimes (e.g. the podman or docker plugins) will still collect +general data about the runtime, but will not inspect existing containers or images. + +Default: 'auto' (policy determined) +.TP .B \--case-id NUMBER Specify a case identifier to associate with the archive. Identifiers may include alphanumeric characters, commas and periods ('.'). -Synonymous with \--ticket-number. -.TP -.B \--ticket-number NUMBER -Specify a ticket number or other identifier to associate with the archive. -Identifiers may include alphanumeric characters, commas and periods ('.'). -Synonymous with \--case-id. .TP .B \--build Do not archive copied data. Causes sosreport to leave an uncompressed @@ -263,6 +373,21 @@ output, or string data from the system. to understand the actions that sos would have taken without the dry run option. .TP +.B \--estimate-only +Estimate disk space requirements when running sos report. This can be valuable +to prevent sosreport working dir to consume all free disk space. No plugin data +is available at the end. + +Plugins will be collected sequentially, size of collected files and commands outputs +will be calculated and the plugin files will be immediatelly deleted prior execution +of the next plugin. This still can consume whole free disk space, though. + +Please note, size estimations may not be accurate for highly utilized systems due to +changes between an estimate and a real execution. Also some difference between +estimation (using `stat` command) and other commands used (i.e. `du`). + +A rule of thumb is to reserve at least double the estimation. +.TP .B \--upload If specified, attempt to upload the resulting archive to a vendor defined location. @@ -299,6 +424,12 @@ If a vendor does not provide a default u If this option is unused and upload is request, and a vendor default is not set, you will be prompted for one. If --batch is used and this option is omitted, no username will be collected and thus uploads will fail if no vendor default is set. + +You also have the option of providing this value via the SOSUPLOADUSER environment +variable. If this variable is set, then no username prompt will occur and --batch +may be used provided all other required values (case number, upload password) +are provided. + .TP .B \-\-upload-pass PASS Specify the password to use for authentication with the destination server. @@ -312,20 +443,58 @@ Note that this will result in the plaint be collected by sos and be in the archive. If a password must be provided by you for uploading, it is strongly recommended to not use --batch and enter the password when prompted rather than using this option. + +You also have the option of providing this value via the SOSUPLOADPASSWORD environment +variable. If this variable is set, then no password prompt will occur and --batch may +be used provided all other required values (case number, upload user) are provided. + .TP .B \--upload-directory DIR Specify a directory to upload to, if one is not specified by a vendor default location or if your destination server does not allow writes to '/'. .TP +.B \--upload-method METHOD +Specify the HTTP method to use for uploading to the provided --upload-url. Valid +values are 'auto' (default), 'put', or 'post'. The use of 'auto' will default to +the method required by the policy-default upload location, if one exists. + +This option has no effect on upload protocols other than HTTPS. +.TP +.B \--upload-no-ssl-verify +Disable SSL verification for HTTPS uploads. This may be used to allow uploading +to locations that have self-signed certificates, or certificates that are otherwise +untrusted by the local system. + +Default behavior is to perform SSL verification against all upload locations. +.TP +.B \--upload-protocol PROTO +Manually specify the protocol to use for uploading to the target \fBupload-url\fR. + +Normally this is determined via the upload address, assuming that the protocol is part +of the address provided, e.g. 'https://example.com'. By using this option, sos will skip +the protocol check and use the method defined for the specified PROTO. + +For RHEL systems, setting this option to \fBsftp\fR will skip the initial attempt to +upload to the Red Hat Customer Portal, and only attempt an upload to Red Hat's SFTP server, +which is typically used as a fallback target. + +Valid values for PROTO are: 'auto' (default), 'https', 'ftp', 'sftp'. +.TP .B \--experimental Enable plugins marked as experimental. Experimental plugins may not have been tested for this port or may still be under active development. .TP .B \--help Display usage message. +.SH SEE ALSO +.BR sos (1) +.BR sos-clean (1) +.BR sos-collect (1) +.BR sos.conf (5) + .SH MAINTAINER .nf -Bryn M. Reeves +Jake Hunsaker .fi .SH AUTHORS & CONTRIBUTORS See \fBAUTHORS\fR file in the package documentation. diff -pruN 4.0-2/plugins_overview.py 4.5.3ubuntu2/plugins_overview.py --- 4.0-2/plugins_overview.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/plugins_overview.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,119 @@ +# this script generates for each plugin: +# - its name +# - URL to upstream code +# - list of distros +# - list of profiles +# - list of packages that enable the plugin (no other enabling pieces) +# - list of paths it collects (add_copy_spec) +# - list of paths it forbits to collect (add_forbidden_path) +# - list of commands it calls (add_cmd_output) +# +# Output of the script: +# - a JSON object with plugins in keys +# - or CSV format in case "csv" cmdline is provided +# +# TODO: +# - improve parsing that will be never ideal :) +# - add other methods: +# - add_blockdev_cmd +# - add_string_as_file +# - ?? + +import os +import re +import json +import sys + +PLUGDIR = 'sos/report/plugins' + +plugs_data = {} # the map of all plugins data to collect +plugcontent = '' # content of plugin file just being processed + +# method to parse an item of a_s_c/a_c_o/.. methods +# we work on an assumption the item is a string quoted by \" or optionally +# by \'. If we detect at least 2 such chars in the item, take what is between those. +def add_valid_item(dest, item): + for qoutemark in "\"\'": + split = item.split(qoutemark) + if len(split) > 2: + dest.append(split[1]) + return + +# method to find in `plugcontent` all items of given method (a_c_s/a_c_o/..) split +# by comma; add each valid item to the `dest` list +def add_all_items(method, dest, wrapopen='\(', wrapclose='\)'): + regexp = "%s%s(.*?)%s" % (method, wrapopen, wrapclose) + for match in re.findall(regexp, plugcontent, flags=re.MULTILINE|re.DOTALL): + # tuple of distros ended by either (class|from|import) + if isinstance(match,tuple): + for item in list(match): + if item not in ['class', 'from', 'import']: + for it in item.split(','): + # dirty hack to remove spaces and "Plugin" + if "Plugin" not in it: + continue + it = it.strip(' ()')[0:-6] + if len(it): + dest.append(it) + # list of specs separated by comma .. + elif match.startswith('[') or match.startswith('('): + for item in match.split(','): + add_valid_item(dest, item) + # .. or a singleton spec + else: + add_valid_item(dest, match) + +# main body: traverse report's plugins directory and for each plugin, grep for +# add_copy_spec / add_forbidden_path / add_cmd_output there +for plugfile in sorted(os.listdir(PLUGDIR)): + # ignore non-py files and __init__.py + if not plugfile.endswith('.py') or plugfile == '__init__.py': + continue + plugname = plugfile[:-3] +# if plugname != 'bcache': +# continue + plugs_data[plugname] = { + 'sourcecode': 'https://github.com/sosreport/sos/blob/main/sos/report/plugins/%s.py' % plugname, + 'distros': [], + 'profiles': [], + 'packages': [], + 'copyspecs': [], + 'forbidden': [], + 'commands': [], + 'service_status': [], + 'journals': [], + 'env': [], + } + plugcontent = open(os.path.join(PLUGDIR, plugfile)).read().replace('\n','') + add_all_items("from sos.report.plugins import ", plugs_data[plugname]['distros'], wrapopen='', wrapclose='(class|from|import)') + add_all_items("profiles = ", plugs_data[plugname]['profiles'], wrapopen='') + add_all_items("packages = ", plugs_data[plugname]['packages'], wrapopen='') + add_all_items("add_copy_spec", plugs_data[plugname]['copyspecs']) + add_all_items("add_forbidden_path", plugs_data[plugname]['forbidden']) + add_all_items("add_cmd_output", plugs_data[plugname]['commands']) + add_all_items("collect_cmd_output", plugs_data[plugname]['commands']) + add_all_items("add_service_status", plugs_data[plugname]['service_status']) + add_all_items("add_journal", plugs_data[plugname]['journals']) + add_all_items("add_env_var", plugs_data[plugname]['env']) + +# print output; if "csv" is cmdline argument, print in CSV format, else JSON +if (len(sys.argv) > 1) and (sys.argv[1] == "csv"): + print("plugin;url;distros;profiles;packages;copyspecs;forbidden;commands;service_status;journals;env_vars") + for plugname in plugs_data.keys(): + plugin = plugs_data[plugname] + # determine max number of lines - usually "max(len(copyspec),len(commands))" + # ignore 'sourcecode' key as it + maxline = 1 + plugkeys = list(plugin.keys()) + plugkeys.remove('sourcecode') + for key in plugkeys: + maxline = max(maxline, len(plugin[key])) + for line in range(maxline): + out = ";" if line>0 else ("%s;%s" % (plugname, plugin['sourcecode'])) + for key in plugkeys: + out += ";" + if line\n" +"Language-Team: Georgian <(nothing)>\n" +"Language: ka\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=1; plural=0;\n" "Generated-By: pygettext.py 1.5\n" +"X-Generator: Poedit 3.1.1\n" #: ../sos/sosreport.py:745 #, python-format msgid "sosreport (version %s)" -msgstr "" +msgstr "sosreport (ვერსირ%s)" #: ../sos/sosreport.py:977 #, python-format msgid "plugin %s does not validate, skipping" -msgstr "" +msgstr "დáƒáƒ›áƒáƒ¢áƒ”ბრ%s áƒáƒ áƒáƒ¡áƒ¬áƒáƒ áƒ˜ დრგáƒáƒ›áƒáƒ¢áƒáƒ•áƒ”ბულიáƒ" #: ../sos/sosreport.py:979 msgid "does not validate" -msgstr "" +msgstr "áƒáƒ áƒáƒ¡áƒ¬áƒáƒ áƒ˜áƒ" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" -msgstr "" +msgid "plugin %s requires root permissions to execute, skipping" +msgstr "დáƒáƒ›áƒáƒ¢áƒ”ბáƒáƒ¡ %s გáƒáƒ¡áƒáƒ¨áƒ•áƒ”ბáƒáƒ“ root-ის წვდáƒáƒ›áƒ”ბი ესáƒáƒ­áƒ˜áƒ áƒáƒ”ბáƒ. გáƒáƒ›áƒáƒ¢áƒáƒ•áƒ”ბულიáƒ" #: ../sos/sosreport.py:985 msgid "requires root" -msgstr "" +msgstr "სáƒáƒ­áƒ˜áƒ áƒáƒ”ბს root მáƒáƒ›áƒ®áƒ›áƒáƒ áƒ”ბელს" #: ../sos/sosreport.py:993 msgid "excluded" -msgstr "" +msgstr "გáƒáƒ›áƒáƒ áƒ˜áƒªáƒ®áƒ£áƒšáƒ˜áƒ" #: ../sos/sosreport.py:997 msgid "skipped" -msgstr "" +msgstr "გáƒáƒ›áƒáƒ¢áƒáƒ•áƒ”ბული" #: ../sos/sosreport.py:1001 msgid "inactive" -msgstr "" +msgstr "áƒáƒ áƒáƒáƒ¥áƒ¢áƒ˜áƒ£áƒ áƒ˜" #: ../sos/sosreport.py:1005 msgid "optional" -msgstr "" +msgstr "áƒáƒ áƒáƒ¡áƒáƒ•áƒáƒšáƒ“ებულáƒ" #: ../sos/sosreport.py:1015 msgid "not specified" -msgstr "" +msgstr "მითითებული áƒáƒ áƒáƒ" #: ../sos/sosreport.py:1023 #, python-format msgid "plugin %s does not install, skipping: %s" -msgstr "" +msgstr "დáƒáƒ›áƒáƒ¢áƒ”ბრ%s დáƒáƒ§áƒ”ნებული áƒáƒ áƒáƒ დრგáƒáƒ›áƒáƒ¢áƒáƒ•áƒ”ბულიáƒ: %s" #: ../sos/sosreport.py:1027 #, python-format msgid "Unknown or inactive profile(s) provided: %s" -msgstr "" +msgstr "მითითებულირუცნáƒáƒ‘ი áƒáƒœ áƒáƒ áƒáƒáƒ¥áƒ¢áƒ˜áƒ£áƒ áƒ˜ პრáƒáƒ¤áƒ˜áƒšáƒ˜: %s" #: ../sos/sosreport.py:1120 msgid "no valid plugins found" -msgstr "" +msgstr "áƒáƒ áƒªáƒ”რთი დáƒáƒ›áƒáƒ¢áƒ”ბრსწáƒáƒ áƒ˜ áƒáƒ áƒáƒ" #: ../sos/sosreport.py:1124 msgid "The following plugins are currently enabled:" -msgstr "" +msgstr "ჩáƒáƒ áƒ—ული დáƒáƒ›áƒáƒ¢áƒ”ბები:" #: ../sos/sosreport.py:1130 msgid "No plugin enabled." -msgstr "" +msgstr "ჩáƒáƒ áƒ—ული დáƒáƒ›áƒáƒ¢áƒ”ბების გáƒáƒ áƒ”შე." #: ../sos/sosreport.py:1134 msgid "The following plugins are currently disabled:" -msgstr "" +msgstr "áƒáƒ›áƒŸáƒáƒ›áƒáƒ“ გáƒáƒ›áƒáƒ áƒ—ული დáƒáƒ›áƒáƒ¢áƒ”ბები:" #: ../sos/sosreport.py:1145 msgid "The following plugin options are available:" -msgstr "" +msgstr "დáƒáƒ›áƒáƒ¢áƒ”ბების ხელმისáƒáƒ¬áƒ•áƒ“áƒáƒ›áƒ˜ პáƒáƒ áƒáƒ›áƒ”ტრები:" #: ../sos/sosreport.py:1160 msgid "No plugin options available." -msgstr "" +msgstr "დáƒáƒ›áƒáƒ¢áƒ”ბების პáƒáƒ áƒáƒ›áƒ”ტრების გáƒáƒ áƒ”შე." #: ../sos/sosreport.py:1172 msgid "no valid profiles found" -msgstr "" +msgstr "სწáƒáƒ áƒ˜ პრáƒáƒ¤áƒ˜áƒšáƒ”ბი ნáƒáƒžáƒáƒ•áƒœáƒ˜ áƒáƒ áƒáƒ" #: ../sos/sosreport.py:1174 msgid "The following profiles are available:" -msgstr "" +msgstr "ხელმისáƒáƒ¬áƒ•áƒ“áƒáƒ›áƒ˜áƒ პრáƒáƒ¤áƒ˜áƒšáƒ”ბი:" #: ../sos/sosreport.py:1197 msgid "Press ENTER to continue, or CTRL-C to quit.\n" -msgstr "" +msgstr "ENTER გáƒáƒ¡áƒáƒ’რძელებლáƒáƒ“, CTRL-C შესáƒáƒ¬áƒ§áƒ•áƒ”ტáƒáƒ“.\n" #: ../sos/sosreport.py:1216 msgid " Setting up archive ..." -msgstr "" +msgstr " áƒáƒ áƒ¥áƒ˜áƒ•áƒ˜áƒ¡ მáƒáƒ áƒ’ებრ..." #: ../sos/sosreport.py:1250 msgid " Setting up plugins ..." -msgstr "" +msgstr " დáƒáƒ›áƒáƒ¢áƒ”ბების მáƒáƒ áƒ’ებრ..." #: ../sos/sosreport.py:1282 msgid " Running plugins. Please wait ..." -msgstr "" +msgstr " მიმდინáƒáƒ áƒ”áƒáƒ‘ს დáƒáƒ›áƒáƒ¢áƒ”ბების გáƒáƒ¨áƒ•áƒ”ბáƒ. მáƒáƒ˜áƒ—მინეთ ..." #: ../sos/sosreport.py:1490 msgid "Creating compressed archive..." -msgstr "" +msgstr "შეკუმშული áƒáƒ áƒ¥áƒ˜áƒ•áƒ˜áƒ¡ შექმნრ..." #: ../sos/sosreport.py:1498 #, python-format msgid " %s while finalizing archive" -msgstr "" +msgstr " %s áƒáƒ áƒ¥áƒ˜áƒ•áƒ˜áƒ¡ დáƒáƒ¡áƒ áƒ£áƒšáƒ”ბისáƒáƒ¡" #: ../sos/sosreport.py:1517 #, python-format msgid "Error moving directory: %s" -msgstr "" +msgstr "სáƒáƒ¥áƒáƒ¦áƒáƒšáƒ“ის გáƒáƒ“áƒáƒ¢áƒáƒœáƒ˜áƒ¡ შეცდáƒáƒ›áƒ: %s" #: ../sos/sosreport.py:1540 #, python-format msgid "Error moving archive file: %s" -msgstr "" +msgstr "áƒáƒ áƒ¥áƒ˜áƒ•áƒ˜áƒ¡ ფáƒáƒ˜áƒšáƒ˜áƒ¡ გáƒáƒ“áƒáƒ¢áƒáƒœáƒ˜áƒ¡ შეცდáƒáƒ›áƒ: %s" #: ../sos/sosreport.py:1558 #, python-format msgid "Error moving checksum file: %s" -msgstr "" +msgstr "სáƒáƒ™áƒáƒœáƒ¢áƒ áƒáƒšáƒ ჯáƒáƒ›áƒ˜áƒ¡ ფáƒáƒ˜áƒšáƒ˜áƒ¡ გáƒáƒ“áƒáƒ¢áƒáƒœáƒ˜áƒ¡ შეცდáƒáƒ›áƒ: %s" #: ../sos/sosreport.py:1574 msgid "no valid plugins were enabled" -msgstr "" +msgstr "სწáƒáƒ áƒ˜ ჩáƒáƒ áƒ—ული დáƒáƒ›áƒáƒ¢áƒ”ბების გáƒáƒ áƒ”შე" diff -pruN 4.0-2/po/kn.po 4.5.3ubuntu2/po/kn.po --- 4.0-2/po/kn.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/kn.po 2023-04-28 17:16:21.000000000 +0000 @@ -37,7 +37,7 @@ msgstr "ಪà³à²²à²—à³â€Œà²‡à²¨à³ %s à #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "ಪà³à²²à²—à³â€Œà²‡à²¨à³ %s ಅನà³à²¸à³à²¥à²¾à²ªà²¨à³†à²—ೊಂಡಿಲà³à²², ಉಪೇಕà³à²·à²¿à²¸à²²à²¾à²—à³à²¤à³à²¤à²¿à²¦à³†" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/ko.po 4.5.3ubuntu2/po/ko.po --- 4.0-2/po/ko.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/ko.po 2023-04-28 17:16:21.000000000 +0000 @@ -36,7 +36,7 @@ msgstr "%s 플러그ì¸ì´ 유효하지 #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "%s 플러그ì¸ì´ 설치ë˜ì§€ ì•Šì•„ ìƒëžµí•©ë‹ˆë‹¤." #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/ku.po 4.5.3ubuntu2/po/ku.po --- 4.0-2/po/ku.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/ku.po 2023-04-28 17:16:21.000000000 +0000 @@ -32,7 +32,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/lo.po 4.5.3ubuntu2/po/lo.po --- 4.0-2/po/lo.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/lo.po 2023-04-28 17:16:21.000000000 +0000 @@ -32,7 +32,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/lt.po 4.5.3ubuntu2/po/lt.po --- 4.0-2/po/lt.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/lt.po 2023-04-28 17:16:21.000000000 +0000 @@ -34,7 +34,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/lv.po 4.5.3ubuntu2/po/lv.po --- 4.0-2/po/lv.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/lv.po 2023-04-28 17:16:21.000000000 +0000 @@ -34,7 +34,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/mk.po 4.5.3ubuntu2/po/mk.po --- 4.0-2/po/mk.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/mk.po 2023-04-28 17:16:21.000000000 +0000 @@ -32,7 +32,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/ml.po 4.5.3ubuntu2/po/ml.po --- 4.0-2/po/ml.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/ml.po 2023-04-28 17:16:21.000000000 +0000 @@ -36,7 +36,7 @@ msgstr "%s à´Žà´¨àµà´¨à´¤àµ ശരിà #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "%s à´Žà´¨àµà´¨ à´ªàµà´³à´—àµà´—à´¿à´¨àµâ€ ഇനàµâ€à´¸àµà´±àµà´±àµ‹à´³àµâ€ ചെയàµà´¯àµà´µà´¾à´¨àµâ€ സാധàµà´¯à´®à´²àµà´², ഉപേകàµà´·à´¿à´•àµà´•àµà´¨àµà´¨àµ" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/mr.po 4.5.3ubuntu2/po/mr.po --- 4.0-2/po/mr.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/mr.po 2023-04-28 17:16:21.000000000 +0000 @@ -37,7 +37,7 @@ msgstr "पà¥à¤²à¤—इन %s तपाà #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "पà¥à¤²à¤—इन %s चे पà¥à¤°à¤¤à¤¿à¤·à¥à¤ à¤¾à¤ªà¤¾à¤¨ अशकà¥à¤¯, वगळत आहे" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/ms.po 4.5.3ubuntu2/po/ms.po --- 4.0-2/po/ms.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/ms.po 2023-04-28 17:16:21.000000000 +0000 @@ -32,7 +32,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/my.po 4.5.3ubuntu2/po/my.po --- 4.0-2/po/my.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/my.po 2023-04-28 17:16:21.000000000 +0000 @@ -32,7 +32,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/nb.po 4.5.3ubuntu2/po/nb.po --- 4.0-2/po/nb.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/nb.po 2023-04-28 17:16:21.000000000 +0000 @@ -34,7 +34,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/nds.po 4.5.3ubuntu2/po/nds.po --- 4.0-2/po/nds.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/nds.po 2023-04-28 17:16:21.000000000 +0000 @@ -38,7 +38,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/nl.po 4.5.3ubuntu2/po/nl.po --- 4.0-2/po/nl.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/nl.po 2023-04-28 17:16:21.000000000 +0000 @@ -35,7 +35,7 @@ msgstr "plug-in %s valideerde niet, word #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "plug-in %s laat zich niet installeren, wordt overgeslagen" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/nn.po 4.5.3ubuntu2/po/nn.po --- 4.0-2/po/nn.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/nn.po 2023-04-28 17:16:21.000000000 +0000 @@ -33,7 +33,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/nso.po 4.5.3ubuntu2/po/nso.po --- 4.0-2/po/nso.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/nso.po 2023-04-28 17:16:21.000000000 +0000 @@ -32,7 +32,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/or.po 4.5.3ubuntu2/po/or.po --- 4.0-2/po/or.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/or.po 2023-04-28 17:16:21.000000000 +0000 @@ -37,7 +37,7 @@ msgstr "ପà­à¬²à¬—ଇନ %s କୠବ #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "ପà­à¬²à¬—ଇନ %s ସà­à¬¥à¬¾à¬ªà¬¨ କରେନାହିà¬, à¬à¬¡à¬¼à¬¾à¬‡ ଦେଉଛି" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/pa.po 4.5.3ubuntu2/po/pa.po --- 4.0-2/po/pa.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/pa.po 2023-04-28 17:16:21.000000000 +0000 @@ -37,7 +37,7 @@ msgstr "ਪਲੱਗਇਨ %s ਪà©à¨°à #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "ਪਲੱਗਇਨ %s ਇੰਸਟਾਲ ਨਹੀਂ ਹੋਇਆ, ਛੱਡ ਰਿਹਾ ਹੈ" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/pl.po 4.5.3ubuntu2/po/pl.po --- 4.0-2/po/pl.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/pl.po 2023-04-28 17:16:21.000000000 +0000 @@ -34,7 +34,7 @@ msgstr "nieprawidÅ‚owa" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "wtyczka %s do wykonania wymaga uprawnieÅ„ roota, pomijanie" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/pt.po 4.5.3ubuntu2/po/pt.po --- 4.0-2/po/pt.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/pt.po 2023-04-28 17:16:21.000000000 +0000 @@ -36,7 +36,7 @@ msgstr "plugin %s não valida. A passar #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "plugin %s não instala. A passar ao seguinte" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/pt_BR.po 4.5.3ubuntu2/po/pt_BR.po --- 4.0-2/po/pt_BR.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/pt_BR.po 2023-04-28 17:16:21.000000000 +0000 @@ -33,7 +33,7 @@ msgstr "o plugin %s não validou, ignora #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "o plugin %s não instala, ignorando" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/ro.po 4.5.3ubuntu2/po/ro.po --- 4.0-2/po/ro.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/ro.po 2023-04-28 17:16:21.000000000 +0000 @@ -32,7 +32,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/ru.po 4.5.3ubuntu2/po/ru.po --- 4.0-2/po/ru.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/ru.po 2023-04-28 17:16:21.000000000 +0000 @@ -38,7 +38,7 @@ msgstr "модуль %s не прошёР#: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "модуль %s не уÑтанавливаетÑÑ. ПропуÑкаетÑÑ." #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/si.po 4.5.3ubuntu2/po/si.po --- 4.0-2/po/si.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/si.po 2023-04-28 17:16:21.000000000 +0000 @@ -37,7 +37,7 @@ msgstr "%s ප්ලගීනය සතà #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "%s ප්ලගීනය ස්ථà·à¶´à¶±à¶º වන්නේ නà·à¶­, මගහරි" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/sk.po 4.5.3ubuntu2/po/sk.po --- 4.0-2/po/sk.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/sk.po 2023-04-28 17:16:21.000000000 +0000 @@ -35,7 +35,7 @@ msgstr "nie je možné overiÅ¥ modul %s, #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "nie je možné nainÅ¡talovaÅ¥ modul %s, preskakujem" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/sl.po 4.5.3ubuntu2/po/sl.po --- 4.0-2/po/sl.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/sl.po 2023-04-28 17:16:21.000000000 +0000 @@ -34,7 +34,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/sos.pot 4.5.3ubuntu2/po/sos.pot --- 4.0-2/po/sos.pot 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/sos.pot 2023-04-28 17:16:21.000000000 +0000 @@ -33,7 +33,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/sq.po 4.5.3ubuntu2/po/sq.po --- 4.0-2/po/sq.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/sq.po 2023-04-28 17:16:21.000000000 +0000 @@ -32,7 +32,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/sr.po 4.5.3ubuntu2/po/sr.po --- 4.0-2/po/sr.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/sr.po 2023-04-28 17:16:21.000000000 +0000 @@ -35,7 +35,7 @@ msgstr "додатак %s Ñе није #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "додатак %s Ñе није инÑталирао, преÑкачем" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/sr@latin.po 4.5.3ubuntu2/po/sr@latin.po --- 4.0-2/po/sr@latin.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/sr@latin.po 2023-04-28 17:16:21.000000000 +0000 @@ -35,7 +35,7 @@ msgstr "dodatak %s se nije overio, presk #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "dodatak %s se nije instalirao, preskaÄem" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/sv.po 4.5.3ubuntu2/po/sv.po --- 4.0-2/po/sv.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/sv.po 2023-04-28 17:16:21.000000000 +0000 @@ -35,7 +35,7 @@ msgstr "insticksmodul %s validerar inte, #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "insticksmodul %s installerar inte, hoppar över" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/ta.po 4.5.3ubuntu2/po/ta.po --- 4.0-2/po/ta.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/ta.po 2023-04-28 17:16:21.000000000 +0000 @@ -37,7 +37,7 @@ msgstr "கூடà¯à®¤à®²à¯ இணைà #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "கூடà¯à®¤à®²à¯ இணைபà¯à®ªà¯ %s நிறà¯à®µà®ªà¯à®ªà®Ÿà®µà®¿à®²à¯à®²à¯ˆ, தவிரà¯à®•à¯à®•à®ªà¯à®ªà®Ÿà¯à®•à®¿à®±à®¤à¯" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/te.po 4.5.3ubuntu2/po/te.po --- 4.0-2/po/te.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/te.po 2023-04-28 17:16:21.000000000 +0000 @@ -38,7 +38,7 @@ msgstr "à°ªà±à°²à°—à±â€Œà°¯à°¿à°¨à± % #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "à°ªà±à°²à°—ినౠ%s సంసà±à°¥à°¾à°ªà°¿à°‚చబడలేదà±, వదిలివేయà±à°šà±à°¨à±à°¨à°¦à°¿" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/th.po 4.5.3ubuntu2/po/th.po --- 4.0-2/po/th.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/th.po 2023-04-28 17:16:21.000000000 +0000 @@ -35,7 +35,7 @@ msgstr "ส่วนขยาย %s ไà #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "ส่วนขยาย %s ติดตั้งไม่ได้ จะข้ามไป" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/tr.po 4.5.3ubuntu2/po/tr.po --- 4.0-2/po/tr.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/tr.po 2023-04-28 17:16:21.000000000 +0000 @@ -37,7 +37,7 @@ msgstr "%s eklentisi doÄŸrulanamadı, at #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "%s eklentisi kurulamıyor, atlanıyor" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/uk.po 4.5.3ubuntu2/po/uk.po --- 4.0-2/po/uk.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/uk.po 2023-04-28 17:16:21.000000000 +0000 @@ -36,7 +36,7 @@ msgstr "модуль %s не пройшР#: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "модуль %s не вÑтановлюєтьÑÑ. ПропуÑкаєтьÑÑ." #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/ur.po 4.5.3ubuntu2/po/ur.po --- 4.0-2/po/ur.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/ur.po 2023-04-28 17:16:21.000000000 +0000 @@ -32,7 +32,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/vi.po 4.5.3ubuntu2/po/vi.po --- 4.0-2/po/vi.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/vi.po 2023-04-28 17:16:21.000000000 +0000 @@ -33,7 +33,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/zh_CN.po 4.5.3ubuntu2/po/zh_CN.po --- 4.0-2/po/zh_CN.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/zh_CN.po 2023-04-28 17:16:21.000000000 +0000 @@ -37,7 +37,7 @@ msgstr "æ’件 %s 无效，跳过" #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "未安装æ’件 %s，跳过" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/zh_TW.po 4.5.3ubuntu2/po/zh_TW.po --- 4.0-2/po/zh_TW.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/zh_TW.po 2023-04-28 17:16:21.000000000 +0000 @@ -35,7 +35,7 @@ msgstr "å¤–æŽ›ç¨‹å¼ %s 無法驗證，å #: ../sos/sosreport.py:983 #, fuzzy, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "未安è£å¤–æŽ›ç¨‹å¼ %s，故而跳éŽ" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/po/zu.po 4.5.3ubuntu2/po/zu.po --- 4.0-2/po/zu.po 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/po/zu.po 2023-04-28 17:16:21.000000000 +0000 @@ -32,7 +32,7 @@ msgstr "" #: ../sos/sosreport.py:983 #, python-format -msgid "plugin %s requires root permissionsto execute, skipping" +msgid "plugin %s requires root permissions to execute, skipping" msgstr "" #: ../sos/sosreport.py:985 diff -pruN 4.0-2/pylintrc 4.5.3ubuntu2/pylintrc --- 4.0-2/pylintrc 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/pylintrc 1970-01-01 00:00:00.000000000 +0000 @@ -1,354 +0,0 @@ -# lint Python modules using external checkers. -# -# This is the main checker controling the other ones and the reports -# generation. It is itself both a raw checker and an astng checker in order -# to: -# * handle message activation / deactivation at the module level -# * handle some basic but necessary stats'data (number of classes, methods...) -# -# This checker also defines the following reports: -# * R0001: Total errors / warnings -# * R0002: % errors / warnings by module -# * R0003: Messages -# * R0004: Global evaluation -[MASTER] - -# Profiled execution. -profile=no - -# Add to the black list. It should be a base name, not a -# path. You may set this option multiple times. -ignore=CVS - -# Pickle collected data for later comparisons. -persistent=yes - -# Set the cache size for astng objects. -cache-size=500 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - - -[REPORTS] - -# Tells wether to display a full report or only the messages -reports=yes - -# Use HTML as output format instead of text -html=no - -# Use a parseable text output format, so your favorite text editor will be able -# to jump to the line corresponding to a message. -parseable=yes - -# Colorizes text output using ansi escape codes -color=no - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no - -# Python expression which should return a note less than 10 (10 is the highest -# note).You have access to the variables errors warning, statement which -# respectivly contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (R0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Add a comment according to your evaluation note. This is used by the global -# evaluation report (R0004). -comment=no - -# Include message's id in output -include-ids=yes - - -# checks for -# * unused variables / imports -# * undefined variables -# * redefinition of variable from builtins or from an outer scope -# * use of variable before assigment -# -[VARIABLES] - -# Enable / disable this checker -enable-variables=yes - -# Tells wether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching names used for dummy variables (i.e. not used). -dummy-variables-rgx=_|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins=_ - - -# try to find bugs in the code using type inference -# -[TYPECHECK] - -# Enable / disable this checker -enable-typecheck=yes - -# Tells wether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# When zope mode is activated, consider the acquired-members option to ignore -# access to some undefined attributes. -zope=no - -# List of members which are usually get through zope's acquisition mecanism and -# so shouldn't trigger E0201 when accessed (need zope=yes to be considered. -acquired-members=REQUEST,acl_users,aq_parent - - -# checks for : -# * doc strings -# * modules / classes / functions / methods / arguments / variables name -# * number of arguments, local variables, branchs, returns and statements in -# functions, methods -# * required module attributes -# * dangerous default values as arguments -# * redefinition of function / method / class -# * uses of the global statement -# -# This checker also defines the following reports: -# * R0101: Statistics by type -[BASIC] - -# Enable / disable this checker -enable-basic=yes - -#disable-msg=C0121 - -# Required attributes for module, separated by a comma -required-attributes= - -# Regular expression which should only match functions or classes name which do -# not require a docstring -no-docstring-rgx=__.*__ - -# Regular expression which should only match correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression which should only match correct module level names -const-rgx=(([A-Z_][A-Z1-9_]*)|(__.*__))$ - -# Regular expression which should only match correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression which should only match correct function names -function-rgx=[a-z_][A-Za-z0-9_]{2,30}$ - -# Regular expression which should only match correct method names -method-rgx=[a-z_][A-Za-z0-9_]{2,30}$ - -# Regular expression which should only match correct instance attribute names -attr-rgx=[a-z_][A-Za-z0-9_]{2,30}$ - -# Regular expression which should only match correct argument names -argument-rgx=[a-z_][A-Za-z0-9_]{2,30}$ - -# Regular expression which should only match correct variable names -variable-rgx=[a-z_][A-Za-z0-9_]{0,30}$ - -# Regular expression which should only match correct list comprehension / -# generator expression variable names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,apply,input - - -# checks for sign of poor/misdesign: -# * number of methods, attributes, local variables... -# * size, complexity of functions, methods -# -[DESIGN] - -# Enable / disable this checker -enable-design=yes - -# Maximum number of arguments for function / method -max-args=5 - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of branch for function / method body -max-branchs=12 - -# Maximum number of statements in function / method body -max-statements=50 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - - -# checks for : -# * methods without self as first argument -# * overriden methods signature -# * access only to existant members via self -# * attributes not defined in the __init__ method -# * supported interfaces implementation -# * unreachable code -# -[CLASSES] - -# Enable / disable this checker -enable-classes=yes - -# List of interface methods to ignore, separated by a comma. This is used for -# instance to not check methods defines in Zope's Interface base class. -ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - - -# checks for -# * external modules dependencies -# * relative / wildcard imports -# * cyclic imports -# * uses of deprecated modules -# -# This checker also defines the following reports: -# * R0401: External dependencies -# * R0402: Modules dependencies graph -[IMPORTS] - -# Enable / disable this checker -enable-imports=no - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,string,TERMIOS,Bastion,rexec - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report R0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report R0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report R0402 must -# not be disabled) -int-import-graph= - - -# checks for usage of new style capabilities on old style classes and -# other new/old styles conflicts problems -# * use of property, __slots__, super -# * "super" usage -# * raising a new style class as exception -# -[NEWSTYLE] - -# Enable / disable this checker -enable-newstyle=yes - - -# checks for -# * excepts without exception filter -# * string exceptions -# -[EXCEPTIONS] - -# Enable / disable this checker -enable-exceptions=yes - - -# checks for : -# * unauthorized constructions -# * strict indentation -# * line length -# * use of <> instead of != -# -[FORMAT] - -# Enable / disable this checker -enable-format=yes - -# Maximum number of characters on a single line. -max-line-length=132 - -# Maximum number of lines in a module -max-module-lines=1000 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - - -# checks for similarities and duplicated code. This computation may be -# memory / CPU intensive, so you should disable it if you experiments some -# problems. -# -# This checker also defines the following reports: -# * R0801: Duplication -[SIMILARITIES] - -# Enable / disable this checker -enable-similarities=yes - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - - -# checks for: -# * warning notes in the code like FIXME, XXX -# * PEP 263: source code with non ascii character but no encoding declaration -# -[MISCELLANEOUS] - -# Enable / disable this checker -enable-miscellaneous=yes - -# List of note tags to take in consideration, separated by a comma. Default to -# FIXME, XXX, TODO -notes=FIXME,XXX,TODO - - -# does not check anything but gives some raw metrics : -# * total number of lines -# * total number of code lines -# * total number of docstring lines -# * total number of comments lines -# * total number of empty lines -# -# This checker also defines the following reports: -# * R0701: Raw metrics -[METRICS] - -# Enable / disable this checker -enable-metrics=no diff -pruN 4.0-2/requirements.txt 4.5.3ubuntu2/requirements.txt --- 4.0-2/requirements.txt 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/requirements.txt 2023-04-28 17:16:21.000000000 +0000 @@ -1,5 +1,7 @@ pycodestyle>=2.4.0 -nose>=1.3.7 coverage>=4.0.3 Sphinx>=1.3.5 pexpect>=4.0.0 +pyyaml +setuptools + diff -pruN 4.0-2/setup.py 4.5.3ubuntu2/setup.py --- 4.0-2/setup.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/setup.py 2023-04-28 17:16:21.000000000 +0000 @@ -1,84 +1,16 @@ #!/usr/bin/env python -from distutils.core import setup -from distutils.command.build import build -from distutils.command.install_data import install_data -from distutils.dep_util import newer -from distutils.log import error - -import glob -import os -import re -import subprocess -import sys - +from setuptools import setup, find_packages from sos import __version__ as VERSION -PO_DIR = 'po' -MO_DIR = os.path.join('build', 'mo') - -class BuildData(build): - def run(self): - build.run(self) - for po in glob.glob(os.path.join(PO_DIR, '*.po')): - lang = os.path.basename(po[:-3]) - mo = os.path.join(MO_DIR, lang, 'sos.mo') - - directory = os.path.dirname(mo) - if not os.path.exists(directory): - os.makedirs(directory) - - if newer(po, mo): - try: - rc = subprocess.call(['msgfmt', '-o', mo, po]) - if rc != 0: - raise Warning("msgfmt returned %d" % (rc,)) - except Exception as e: - error("Failed gettext.") - sys.exit(1) - -class InstallData(install_data): - def run(self): - self.data_files.extend(self._find_mo_files()) - install_data.run(self) - - def _find_mo_files(self): - data_files = [] - for mo in glob.glob(os.path.join(MO_DIR, '*', 'sos.mo')): - lang = os.path.basename(os.path.dirname(mo)) - dest = os.path.join('share', 'locale', lang, 'LC_MESSAGES') - data_files.append((dest, [mo])) - return data_files - - # Workaround https://bugs.python.org/issue644744 - def copy_file (self, filename, dirname): - (out, _) = install_data.copy_file(self, filename, dirname) - # match for man pages - if re.search(r'/man/man\d/.+\.\d$', out): - return (out+".gz", _) - return (out, _) - -cmdclass = {'build': BuildData, 'install_data': InstallData} -command_options = {} -try: - from sphinx.setup_command import BuildDoc - cmdclass['build_sphinx'] = BuildDoc - command_options={ - 'build_sphinx': { - 'project': ('setup.py', 'sos'), - 'version': ('setup.py', VERSION), - 'source_dir': ('setup.py', 'docs') - } - } -except Exception: - print("Unable to build sphinx docs - module not present. Install sphinx " - "to enable documentation generation") setup( name='sos', version=VERSION, - description=("""A set of tools to gather troubleshooting""" - """ information from a system."""), + install_requires=['pexpect', 'pyyaml'], + description=( + 'A set of tools to gather troubleshooting information from a system' + ), author='Bryn M. Reeves', author_email='bmr@redhat.com', maintainer='Jake Hunsaker', @@ -90,20 +22,13 @@ setup( ('share/man/man1', ['man/en/sosreport.1', 'man/en/sos-report.1', 'man/en/sos.1', 'man/en/sos-collect.1', 'man/en/sos-collector.1', 'man/en/sos-clean.1', - 'man/en/sos-mask.1']), + 'man/en/sos-mask.1', 'man/en/sos-help.1']), ('share/man/man5', ['man/en/sos.conf.5']), ('share/licenses/sos', ['LICENSE']), - ('share/doc/sos', ['AUTHORS', 'README.md']) + ('share/doc/sos', ['AUTHORS', 'README.md']), + ('config', ['sos.conf', 'tmpfiles/tmpfilesd-sos-rh.conf']) ], - packages=[ - 'sos', 'sos.policies', 'sos.report', 'sos.report.plugins', - 'sos.collector', 'sos.collector.clusters', 'sos.cleaner', - 'sos.cleaner.mappings', 'sos.cleaner.parsers' - ], - cmdclass=cmdclass, - command_options=command_options, - requires=['pexpect'] - ) - + packages=find_packages(include=['sos', 'sos.*']) +) # vim: set et ts=4 sw=4 : diff -pruN 4.0-2/snap/snapcraft.yaml 4.5.3ubuntu2/snap/snapcraft.yaml --- 4.0-2/snap/snapcraft.yaml 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/snap/snapcraft.yaml 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,42 @@ +name: sosreport +summary: Sos is an extensible, portable, support data collection tool +description: | + Sos is an extensible, portable, support data collection tool + primarily aimed at Linux distributions and other UNIX-like operating + systems. +grade: stable +base: core22 +confinement: classic +adopt-info: sos +license: GPL-2.0-or-later +environment: + PYTHONPATH: ${SNAP}/lib/python3.10/site-packages:${SNAP}/usr/lib/python3/dist-packages:${PYTHONPATH} + +parts: + sos: + plugin: python + source: . + override-pull: | + craftctl default + craftctl set version="$(git describe --tags --always)" + build-packages: + - git + - python3 + - snapcraft + - gettext + stage-packages: + - python3-venv + python-packages: + - pip + - setuptools + - wheel + - python_magic + - packaging + +apps: + sos: + command: bin/sos + sosreport: + command: bin/sos report + sos-collector: + command: bin/sos collector diff -pruN 4.0-2/sos/__init__.py 4.5.3ubuntu2/sos/__init__.py --- 4.0-2/sos/__init__.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -14,7 +14,7 @@ This module houses the i18n setup and message function. The default is to use gettext to internationalize messages. """ -__version__ = "4.0" +__version__ = "4.5.3" import os import sys @@ -59,9 +59,11 @@ class SoS(): # if no aliases are desired, pass an empty list import sos.report import sos.cleaner + import sos.help self._components = { 'report': (sos.report.SoSReport, ['rep']), - 'clean': (sos.cleaner.SoSCleaner, ['cleaner', 'mask']) + 'clean': (sos.cleaner.SoSCleaner, ['cleaner', 'mask']), + 'help': (sos.help.SoSHelper, []) } # some distros do not want pexpect as a default dep, so try to load # collector here, and if it fails add an entry that implies it is at @@ -110,14 +112,15 @@ class SoS(): for comp in self._components: _com_subparser = self.subparsers.add_parser( comp, - aliases=self._components[comp][1] + aliases=self._components[comp][1], + prog="sos %s" % comp ) _com_subparser.usage = "sos %s [options]" % comp _com_subparser.register('action', 'extend', SosListOption) self._add_common_options(_com_subparser) self._components[comp][0].add_parser_options(parser=_com_subparser) _com_subparser.set_defaults(component=comp) - self.args, _unknown = self.parser.parse_known_args(self.cmdline) + self.args = self.parser.parse_args(self.cmdline) self._init_component() def _add_common_options(self, parser): @@ -156,6 +159,11 @@ class SoS(): # Group to make tarball encryption (via GPG/password) exclusive encrypt_grp = global_grp.add_mutually_exclusive_group() + encrypt_grp.add_argument("--encrypt", default=False, + action="store_true", + help=("Encrypt the archive, either prompting " + "for a password/key or referencing " + "an environment variable")) encrypt_grp.add_argument("--encrypt-key", help="Encrypt the archive using a GPG " "key-pair") diff -pruN 4.0-2/sos/archive.py 4.5.3ubuntu2/sos/archive.py --- 4.0-2/sos/archive.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/archive.py 2023-04-28 17:16:21.000000000 +0000 @@ -16,10 +16,12 @@ import logging import codecs import errno import stat +import re from datetime import datetime from threading import Lock -from sos.utilities import sos_get_command_output, is_executable +from importlib.util import find_spec +from sos.utilities import sos_get_command_output try: import selinux @@ -134,6 +136,9 @@ class FileCacheArchive(Archive): def __init__(self, name, tmpdir, policy, threads, enc_opts, sysroot, manifest=None): self._name = name + # truncate the name just relative to the tmpdir in case of full path + if os.path.commonprefix([self._name, tmpdir]) == tmpdir: + self._name = os.path.relpath(name, tmpdir) self._tmp_dir = tmpdir self._policy = policy self._threads = threads @@ -152,7 +157,7 @@ class FileCacheArchive(Archive): return (os.path.join(self._archive_root, name)) def join_sysroot(self, path): - if path.startswith(self.sysroot): + if not self.sysroot or path.startswith(self.sysroot): return path if path[0] == os.sep: path = path[1:] @@ -250,7 +255,7 @@ class FileCacheArchive(Archive): return dest - def _check_path(self, src, path_type, dest=None, force=False): + def check_path(self, src, path_type, dest=None, force=False): """Check a new destination path in the archive. Since it is possible for multiple plugins to collect the same @@ -325,12 +330,26 @@ class FileCacheArchive(Archive): return None return dest + def _copy_attributes(self, src, dest): + # copy file attributes, skip SELinux xattrs for /sys and /proc + try: + stat = os.stat(src) + if src.startswith("/sys/") or src.startswith("/proc/"): + shutil.copymode(src, dest) + os.utime(dest, ns=(stat.st_atime_ns, stat.st_mtime_ns)) + else: + shutil.copystat(src, dest) + os.chown(dest, stat.st_uid, stat.st_gid) + except Exception as e: + self.log_debug("caught '%s' setting attributes of '%s'" + % (e, dest)) + def add_file(self, src, dest=None): with self._path_lock: if not dest: dest = src - dest = self._check_path(dest, P_FILE) + dest = self.check_path(dest, P_FILE) if not dest: return @@ -341,26 +360,13 @@ class FileCacheArchive(Archive): try: shutil.copy(src, dest) except OSError as e: - self.log_info("File not collected: '%s'" % e) - except IOError as e: # Filter out IO errors on virtual file systems. if src.startswith("/sys/") or src.startswith("/proc/"): pass else: - self.log_info("caught '%s' copying '%s'" % (e, src)) + self.log_info("File %s not collected: '%s'" % (src, e)) - # copy file attributes, skip SELinux xattrs for /sys and /proc - try: - stat = os.stat(src) - if src.startswith("/sys/") or src.startswith("/proc/"): - shutil.copymode(src, dest) - os.utime(dest, ns=(stat.st_atime_ns, stat.st_mtime_ns)) - else: - shutil.copystat(src, dest) - os.chown(dest, stat.st_uid, stat.st_gid) - except Exception as e: - self.log_debug("caught '%s' setting attributes of '%s'" - % (e, dest)) + self._copy_attributes(src, dest) file_name = "'%s'" % src else: # Open file case: first rewind the file to obtain @@ -382,24 +388,20 @@ class FileCacheArchive(Archive): # over any exixting content in the archive, since it is used by # the Plugin postprocessing hooks to perform regex substitution # on file content. - dest = self._check_path(dest, P_FILE, force=True) + dest = self.check_path(dest, P_FILE, force=True) f = codecs.open(dest, mode, encoding='utf-8') if isinstance(content, bytes): content = content.decode('utf8', 'ignore') f.write(content) if os.path.exists(src): - try: - shutil.copystat(src, dest) - except OSError as e: - self.log_error("Unable to add '%s' to archive: %s" % - (dest, e)) + self._copy_attributes(src, dest) self.log_debug("added string at '%s' to FileCacheArchive '%s'" % (src, self._archive_root)) def add_binary(self, content, dest): with self._path_lock: - dest = self._check_path(dest, P_FILE) + dest = self.check_path(dest, P_FILE) if not dest: return @@ -411,7 +413,7 @@ class FileCacheArchive(Archive): def add_link(self, source, link_name): self.log_debug("adding symlink at '%s' -> '%s'" % (link_name, source)) with self._path_lock: - dest = self._check_path(link_name, P_LINK) + dest = self.check_path(link_name, P_LINK) if not dest: return @@ -478,7 +480,6 @@ class FileCacheArchive(Archive): else: self.log_debug("No link follow up: source=%s link_name=%s" % (source, link_name)) - self.log_debug("leaving add_link()") def add_dir(self, path): """Create a directory in the archive. @@ -487,10 +488,10 @@ class FileCacheArchive(Archive): """ # Establish path structure with self._path_lock: - self._check_path(path, P_DIR) + self.check_path(path, P_DIR) def add_node(self, path, mode, device): - dest = self._check_path(path, P_NODE) + dest = self.check_path(path, P_NODE) if not dest: return @@ -503,7 +504,7 @@ class FileCacheArchive(Archive): self.log_info("add_node: %s - mknod '%s'" % (msg, dest)) return raise e - shutil.copystat(path, dest) + self._copy_attributes(path, dest) def name_max(self): if 'PC_NAME_MAX' in os.pathconf_names: @@ -561,17 +562,16 @@ class FileCacheArchive(Archive): def finalize(self, method): self.log_info("finalizing archive '%s' using method '%s'" % (self._archive_root, method)) - self._build_archive() + try: + res = self._build_archive(method) + except Exception as err: + self.log_error("An error occurred compressing the archive: %s" + % err) + return self.name() + self.cleanup() self.log_info("built archive at '%s' (size=%d)" % (self._archive_name, os.stat(self._archive_name).st_size)) - self.method = method - try: - res = self._compress() - except Exception as e: - exp_msg = "An error occurred compressing the archive: " - self.log_error("%s %s" % (exp_msg, e)) - return self.name() if self.enc_opts['encrypt']: try: @@ -638,7 +638,9 @@ class TarFileArchive(FileCacheArchive): super(TarFileArchive, self).__init__(name, tmpdir, policy, threads, enc_opts, sysroot, manifest) self._suffix = "tar" - self._archive_name = os.path.join(tmpdir, self.name()) + self._archive_name = os.path.join( + tmpdir, self.name() # lgtm [py/init-calls-subclass] + ) def set_tarinfo_from_stat(self, tar_info, fstat, mode=None): tar_info.mtime = fstat.st_mtime @@ -654,16 +656,19 @@ class TarFileArchive(FileCacheArchive): # this can be used to set permissions if using the # tarfile.add() interface to add directory trees. def copy_permissions_filter(self, tarinfo): - orig_path = tarinfo.name[len(os.path.split(self._name)[-1]):] + orig_path = tarinfo.name[len(os.path.split(self._archive_root)[-1]):] if not orig_path: orig_path = self._archive_root + skips = ['/version.txt$', '/sos_logs(/.*)?', '/sos_reports(/.*)?'] + if any(re.match(skip, orig_path) for skip in skips): + return None try: fstat = os.stat(orig_path) except OSError: return tarinfo if self._with_selinux_context: context = self.get_selinux_context(orig_path) - if(context): + if context: tarinfo.pax_headers['RHT.security.selinux'] = context self.set_tarinfo_from_stat(tarinfo, fstat) return tarinfo @@ -676,51 +681,42 @@ class TarFileArchive(FileCacheArchive): return None def name(self): - return "%s.%s" % (self._name, self._suffix) + return "%s.%s" % (self._archive_root, self._suffix) def name_max(self): # GNU Tar format supports unlimited file name length. Just return # the limit of the underlying FileCacheArchive. return super(TarFileArchive, self).name_max() - def _build_archive(self): - tar = tarfile.open(self._archive_name, mode="w") + def _build_archive(self, method): + if method == 'auto': + method = 'xz' if find_spec('lzma') is not None else 'gzip' + _comp_mode = method.strip('ip') + self._archive_name = self._archive_name + ".%s" % _comp_mode + # tarfile does not currently have a consistent way to define comnpress + # level for both xz and gzip ('preset' for xz, 'compresslevel' for gz) + if method == 'gzip': + kwargs = {'compresslevel': 6} + else: + kwargs = {'preset': 3} + tar = tarfile.open(self._archive_name, mode="w:%s" % _comp_mode, + **kwargs) + # add commonly reviewed files first, so that they can be more easily + # read from memory without needing to extract the whole archive + for _content in ['version.txt', 'sos_reports', 'sos_logs']: + if not os.path.exists(os.path.join(self._archive_root, _content)): + continue + tar.add( + os.path.join(self._archive_root, _content), + arcname=f"{self._name}/{_content}" + ) # we need to pass the absolute path to the archive root but we # want the names used in the archive to be relative. - tar.add(self._archive_root, arcname=os.path.split(self._name)[1], + tar.add(self._archive_root, arcname=self._name, filter=self.copy_permissions_filter) tar.close() + self._suffix += ".%s" % _comp_mode + return self.name() - def _compress(self): - methods = [] - # Make sure that valid compression commands exist. - for method in ['xz', 'gzip']: - if is_executable(method): - methods.append(method) - else: - self.log_info("\"%s\" compression method unavailable" % method) - if self.method in methods: - methods = [self.method] - - exp_msg = "No compression utilities found." - last_error = Exception(exp_msg) - for cmd in methods: - suffix = "." + cmd.replace('ip', '') - cmd = self._policy.get_cmd_for_compress_method(cmd, self._threads) - try: - exec_cmd = "%s %s" % (cmd, self.name()) - r = sos_get_command_output(exec_cmd, stderr=True, timeout=0) - - if r['status']: - self.log_error(r['output']) - raise Exception("%s exited with %s" % (exec_cmd, - r['status'])) - - self._suffix += suffix - return self.name() - - except Exception as e: - last_error = e - raise last_error # vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/cleaner/__init__.py 4.5.3ubuntu2/sos/cleaner/__init__.py --- 4.0-2/sos/cleaner/__init__.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/cleaner/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -12,9 +12,7 @@ import hashlib import json import logging import os -import re import shutil -import tarfile import tempfile from concurrent.futures import ThreadPoolExecutor @@ -26,26 +24,70 @@ from sos.cleaner.parsers.ip_parser impor from sos.cleaner.parsers.mac_parser import SoSMacParser from sos.cleaner.parsers.hostname_parser import SoSHostnameParser from sos.cleaner.parsers.keyword_parser import SoSKeywordParser -from sos.cleaner.obfuscation_archive import SoSObfuscationArchive +from sos.cleaner.parsers.username_parser import SoSUsernameParser +from sos.cleaner.parsers.ipv6_parser import SoSIPv6Parser +from sos.cleaner.archives.sos import (SoSReportArchive, SoSReportDirectory, + SoSCollectorArchive, + SoSCollectorDirectory) +from sos.cleaner.archives.generic import DataDirArchive, TarballArchive +from sos.cleaner.archives.insights import InsightsArchive from sos.utilities import get_human_readable from textwrap import fill class SoSCleaner(SoSComponent): - """Take an sos report, or collection of sos reports, and scrub them of - potentially sensitive data such as IP addresses, hostnames, MAC addresses, - etc.. that are not obfuscated by individual plugins + """ + This function is designed to obfuscate potentially sensitive information + from an sos report archive in a consistent and reproducible manner. + + It may either be invoked during the creation of a report by using the + --clean option in the report command, or may be used on an already existing + archive by way of 'sos clean'. + + The target of obfuscation are items such as IP addresses, MAC addresses, + hostnames, usernames, and also keywords provided by users via the + --keywords and/or --keyword-file options. + + For every collection made in a report the collection is parsed for such + items, and when items are found SoS will generate an obfuscated replacement + for it, and in all places that item is found replace the text with the + obfuscated replacement mapped to it. These mappings are saved locally so + that future iterations will maintain the same consistent obfuscation + pairing. + + In the case of IP addresses, support is for IPv4 and IPv6 - effort is made + to keep network topology intact so that later analysis is as accurate and + easily understandable as possible. If an IP address is encountered that we + cannot determine the netmask for, a random IP address is used instead. + + For IPv6, note that IPv4-mapped addresses, e.g. ::ffff:10.11.12.13, are + NOT supported currently, and will remain unobfuscated. + + For hostnames, domains are obfuscated as whole units, leaving the TLD in + place. + + For instance, 'example.com' may be obfuscated to 'obfuscateddomain0.com' + and 'foo.example.com' may end up being 'obfuscateddomain1.com'. + + Users will be notified of a 'mapping' file that records all items and the + obfuscated counterpart mapped to them for ease of reference later on. This + file should be kept private. """ desc = "Obfuscate sensitive networking information in a report" arg_defaults = { + 'archive_type': 'auto', 'domains': [], + 'disable_parsers': [], 'jobs': 4, 'keywords': [], + 'keyword_file': None, 'map_file': '/etc/sos/cleaner/default_mapping', 'no_update': False, - 'target': '' + 'keep_binary_files': False, + 'target': '', + 'usernames': [] } def __init__(self, parser=None, args=None, cmdline=None, in_place=False, @@ -66,13 +108,16 @@ class SoSCleaner(SoSComponent): self.from_cmdline = False if not hasattr(self.opts, 'jobs'): self.opts.jobs = 4 + self.opts.archive_type = 'auto' self.soslog = logging.getLogger('sos') self.ui_log = logging.getLogger('sos_ui') # create the tmp subdir here to avoid a potential race condition # when obfuscating a SoSCollector run during archive extraction os.makedirs(os.path.join(self.tmpdir, 'cleaner'), exist_ok=True) - self.validate_map_file() + self.validate_parser_values() + + self.cleaner_mapping = self.load_map_file() os.umask(0o77) self.in_place = in_place self.hash_name = self.policy.get_preferred_hash_name() @@ -80,12 +125,39 @@ class SoSCleaner(SoSComponent): self.cleaner_md = self.manifest.components.add_section('cleaner') self.parsers = [ - SoSHostnameParser(self.opts.map_file, self.opts.domains), - SoSIPParser(self.opts.map_file), - SoSMacParser(self.opts.map_file), - SoSKeywordParser(self.opts.map_file, self.opts.keywords) + SoSHostnameParser(self.cleaner_mapping, self.opts.domains), + SoSIPParser(self.cleaner_mapping), + SoSIPv6Parser(self.cleaner_mapping), + SoSMacParser(self.cleaner_mapping), + SoSKeywordParser(self.cleaner_mapping, self.opts.keywords, + self.opts.keyword_file), + SoSUsernameParser(self.cleaner_mapping, self.opts.usernames) ] + for _parser in self.opts.disable_parsers: + for _loaded in self.parsers: + _loaded_name = _loaded.name.lower().split('parser')[0].strip() + if _parser.lower().strip() == _loaded_name: + self.log_info("Disabling parser: %s" % _loaded_name) + self.ui_log.warning( + "Disabling the '%s' parser. Be aware that this may " + "leave sensitive plain-text data in the archive." + % _parser + ) + self.parsers.remove(_loaded) + + self.archive_types = [ + SoSReportDirectory, + SoSReportArchive, + SoSCollectorDirectory, + SoSCollectorArchive, + InsightsArchive, + # make sure these two are always last as they are fallbacks + DataDirArchive, + TarballArchive + ] + self.nested_archive = None + self.log_info("Cleaner initialized. From cmdline: %s" % self.from_cmdline) @@ -108,12 +180,18 @@ class SoSCleaner(SoSComponent): _fmt = _fmt + fill(line, width, replace_whitespace=False) + '\n' return _fmt - def validate_map_file(self): + @classmethod + def display_help(cls, section): + section.set_title("SoS Cleaner Detailed Help") + section.add_text(cls.__doc__) + + def load_map_file(self): """Verifies that the map file exists and has usable content. If the provided map file does not exist, or it is empty, we will print a warning and continue on with cleaning building a fresh map """ + _conf = {} default_map = '/etc/sos/cleaner/default_mapping' if os.path.isdir(self.opts.map_file): raise Exception("Requested map file %s is a directory" @@ -123,6 +201,17 @@ class SoSCleaner(SoSComponent): self.log_error( "ERROR: map file %s does not exist, will not load any " "obfuscation matches" % self.opts.map_file) + else: + with open(self.opts.map_file, 'r') as mf: + try: + _conf = json.load(mf) + except json.JSONDecodeError: + self.log_error("ERROR: Unable to parse map file, json is " + "malformed. Will not load any mappings.") + except Exception as err: + self.log_error("ERROR: Could not load '%s': %s" + % (self.opts.map_file, err)) + return _conf def print_disclaimer(self): """When we are directly running `sos clean`, rather than hooking into @@ -150,6 +239,8 @@ third party. except KeyboardInterrupt: self.ui_log.info("\nExiting on user cancel") self._exit(130) + except Exception as e: + self._exit(1, e) @classmethod def add_parser_options(cls, parser): @@ -160,21 +251,41 @@ third party. ) clean_grp.add_argument('target', metavar='TARGET', help='The directory or archive to obfuscate') + clean_grp.add_argument('--archive-type', default='auto', + choices=['auto', 'report', 'collect', + 'insights', 'data-dir', 'tarball'], + help=('Specify what kind of archive the target ' + 'was generated as')) clean_grp.add_argument('--domains', action='extend', default=[], help='List of domain names to obfuscate') + clean_grp.add_argument('--disable-parsers', action='extend', + default=[], dest='disable_parsers', + help=('Disable specific parsers, so that those ' + 'elements are not obfuscated')) clean_grp.add_argument('-j', '--jobs', default=4, type=int, help='Number of concurrent archives to clean') clean_grp.add_argument('--keywords', action='extend', default=[], dest='keywords', help='List of keywords to obfuscate') - clean_grp.add_argument('--map', dest='map_file', + clean_grp.add_argument('--keyword-file', default=None, + dest='keyword_file', + help='Provide a file a keywords to obfuscate') + clean_grp.add_argument('--map-file', dest='map_file', default='/etc/sos/cleaner/default_mapping', help=('Provide a previously generated mapping ' 'file for obfuscation')) clean_grp.add_argument('--no-update', dest='no_update', default=False, action='store_true', - help='Do not update the --map file with new ' + help='Do not update the --map-file with new ' 'mappings from this run') + clean_grp.add_argument('--keep-binary-files', default=False, + action='store_true', + dest='keep_binary_files', + help='Keep unprocessable binary files in the ' + 'archive instead of removing them') + clean_grp.add_argument('--usernames', dest='usernames', default=[], + action='extend', + help='List of usernames to obfuscate') def set_target_path(self, path): """For use by report and collect to set the TARGET option appropriately @@ -189,57 +300,40 @@ third party. In the event the target path is not an archive, abort. """ - if not tarfile.is_tarfile(self.opts.target): - self.ui_log.error( - "Invalid target: must be directory or tar archive" - ) - self._exit(1) - - archive = tarfile.open(self.opts.target) - self.arc_name = self.opts.target.split('/')[-1].split('.')[:-2][0] - - try: - archive.getmember(os.path.join(self.arc_name, 'sos_logs')) - except Exception: - # this is not an sos archive - self.ui_log.error("Invalid target: not an sos archive") - self._exit(1) - - # see if there are archives within this archive - nested_archives = [] - for _file in archive.getmembers(): - if (re.match('sosreport-.*.tar', _file.name.split('/')[-1]) and not - _file.name.endswith('.md5')): - nested_archives.append(_file.name.split('/')[-1]) - - if nested_archives: - self.log_info("Found nested archive(s), extracting top level") - nested_path = self.extract_archive(archive) - for arc_file in os.listdir(nested_path): - if re.match('sosreport.*.tar.*', arc_file): - self.report_paths.append(os.path.join(nested_path, - arc_file)) - # add the toplevel extracted archive - self.report_paths.append(nested_path) + _arc = None + if self.opts.archive_type != 'auto': + check_type = self.opts.archive_type.replace('-', '_') + for archive in self.archive_types: + if archive.type_name == check_type: + _arc = archive(self.opts.target, self.tmpdir) else: - self.report_paths.append(self.opts.target) - - archive.close() - - def extract_archive(self, archive): - """Extract an archive into our tmpdir so that we may inspect it or - iterate through its contents for obfuscation - - Positional arguments: - - :param archive: An open TarFile object for the archive - - """ - if not isinstance(archive, tarfile.TarFile): - archive = tarfile.open(archive) - path = os.path.join(self.tmpdir, 'cleaner') - archive.extractall(path) - return os.path.join(path, archive.name.split('/')[-1].split('.tar')[0]) + for arc in self.archive_types: + if arc.check_is_type(self.opts.target): + _arc = arc(self.opts.target, self.tmpdir) + break + if not _arc: + return + self.report_paths.append(_arc) + if _arc.is_nested: + self.report_paths.extend(_arc.get_nested_archives()) + # We need to preserve the top level archive until all + # nested archives are processed + self.report_paths.remove(_arc) + self.nested_archive = _arc + if self.nested_archive: + self.nested_archive.ui_name = self.nested_archive.description + + def validate_parser_values(self): + """Check any values passed to the parsers via the commandline, e.g. + the --domains option, to ensure that they are valid for the parser in + question. + """ + for _dom in self.opts.domains: + if len(_dom.split('.')) < 2: + raise Exception( + f"Invalid value '{_dom}' given: --domains values must be " + "actual domains" + ) def execute(self): """SoSCleaner will begin by inspecting the TARGET option to determine @@ -252,6 +346,7 @@ third party. be unpacked, cleaned, and repacked and the final top-level archive will then be repacked as well. """ + self.arc_name = self.opts.target.split('/')[-1].split('.tar')[0] if self.from_cmdline: self.print_disclaimer() self.report_paths = [] @@ -259,29 +354,17 @@ third party. self.ui_log.error("Invalid target: no such file or directory %s" % self.opts.target) self._exit(1) - if os.path.isdir(self.opts.target): - self.arc_name = self.opts.target.split('/')[-1] - for _file in os.listdir(self.opts.target): - if _file == 'sos_logs': - self.report_paths.append(self.opts.target) - if re.match('sosreport.*.tar.*[^md5]', _file): - self.report_paths.append(os.path.join(self.opts.target, - _file)) - if not self.report_paths: - self.ui_log.error("Invalid target: not an sos directory") - self._exit(1) - else: - self.inspect_target_archive() - # remove any lingering md5 files - self.report_paths = [p for p in self.report_paths if '.md5' not in p] + self.inspect_target_archive() if not self.report_paths: - self.ui_log.error("No valid sos archives or directories found\n") + self.ui_log.error("No valid archives or directories found\n") self._exit(1) # we have at least one valid target to obfuscate self.completed_reports = [] + self.preload_all_archives_into_maps() + self.generate_parser_item_regexes() self.obfuscate_report_paths() if not self.completed_reports: @@ -304,33 +387,7 @@ third party. final_path = None if len(self.completed_reports) > 1: - # we have an archive of archives, so repack the obfuscated tarball - arc_name = self.arc_name + '-obfuscated' - self.setup_archive(name=arc_name) - for arc in self.completed_reports: - if arc.is_tarfile: - arc_dest = self.obfuscate_string( - arc.final_archive_path.split('/')[-1] - ) - self.archive.add_file(arc.final_archive_path, - dest=arc_dest) - checksum = self.get_new_checksum(arc.final_archive_path) - if checksum is not None: - dname = self.obfuscate_string( - "checksums/%s.%s" % (arc_dest, self.hash_name) - ) - self.archive.add_string(checksum, dest=dname) - else: - for dirname, dirs, files in os.walk(arc.archive_path): - for filename in files: - if filename.startswith('sosreport'): - continue - fname = os.path.join(dirname, filename) - dnm = self.obfuscate_string( - fname.split(arc.archive_name)[-1].lstrip('/') - ) - self.archive.add_file(fname, dest=dnm) - arc_path = self.archive.finalize(self.opts.compression_type) + arc_path = self.rebuild_nested_archive() else: arc = self.completed_reports[0] arc_path = arc.final_archive_path @@ -341,28 +398,56 @@ third party. ) with open(os.path.join(self.sys_tmp, chksum_name), 'w') as cf: cf.write(checksum) + self.write_cleaner_log() - self.write_cleaner_log() - - final_path = self.obfuscate_string( - os.path.join(self.sys_tmp, arc_path.split('/')[-1]) + final_path = os.path.join( + self.sys_tmp, + self.obfuscate_string(arc_path.split('/')[-1]) ) shutil.move(arc_path, final_path) arcstat = os.stat(final_path) - # logging will have been shutdown at this point - print("A mapping of obfuscated elements is available at\n\t%s" - % map_path) - - print("\nThe obfuscated archive is available at\n\t%s\n" % final_path) - print("\tSize\t%s" % get_human_readable(arcstat.st_size)) - print("\tOwner\t%s\n" % getpwuid(arcstat.st_uid).pw_name) + # while these messages won't be included in the log file in the archive + # some facilities, such as our avocado test suite, will sometimes not + # capture print() output, so leverage the ui_log to print to console + self.ui_log.info( + f"A mapping of obfuscated elements is available at\n\t{map_path}" + ) + self.ui_log.info( + f"\nThe obfuscated archive is available at\n\t{final_path}\n" + ) - print("Please send the obfuscated archive to your support " - "representative and keep the mapping file private") + self.ui_log.info(f"\tSize\t{get_human_readable(arcstat.st_size)}") + self.ui_log.info(f"\tOwner\t{getpwuid(arcstat.st_uid).pw_name}\n") + self.ui_log.info("Please send the obfuscated archive to your support " + "representative and keep the mapping file private") self.cleanup() + def rebuild_nested_archive(self): + """Handles repacking the nested tarball, now containing only obfuscated + copies of the reports, log files, manifest, etc... + """ + # we have an archive of archives, so repack the obfuscated tarball + arc_name = self.arc_name + '-obfuscated' + self.setup_archive(name=arc_name) + for archive in self.completed_reports: + arc_dest = archive.final_archive_path.split('/')[-1] + checksum = self.get_new_checksum(archive.final_archive_path) + if checksum is not None: + dname = "checksums/%s.%s" % (arc_dest, self.hash_name) + self.archive.add_string(checksum, dest=dname) + for dirn, dirs, files in os.walk(self.nested_archive.extracted_path): + for filename in files: + fname = os.path.join(dirn, filename) + dname = fname.split(self.nested_archive.extracted_path)[-1] + dname = dname.lstrip('/') + self.archive.add_file(fname, dest=dname) + # remove it now so we don't balloon our fs space needs + os.remove(fname) + self.write_cleaner_log(archive=True) + return self.archive.finalize(self.opts.compression_type) + def compile_mapping_dict(self): """Build a dict that contains each parser's map as a key, with the contents as that key's value. This will then be written to disk in the @@ -372,7 +457,7 @@ third party. _map = {} for parser in self.parsers: _map[parser.map_file_key] = {} - _map[parser.map_file_key].update(parser.mapping.dataset) + _map[parser.map_file_key].update(parser.get_map_contents()) return _map @@ -386,8 +471,9 @@ third party. def write_map_for_archive(self, _map): try: - map_path = self.obfuscate_string( - os.path.join(self.sys_tmp, "%s-private_map" % self.arc_name) + map_path = os.path.join( + self.sys_tmp, + self.obfuscate_string("%s-private_map" % self.arc_name) ) return self.write_map_to_file(_map, map_path) except Exception as err: @@ -399,14 +485,19 @@ third party. able to provide the same consistent mapping """ if self.opts.map_file and not self.opts.no_update: + cleaner_dir = os.path.dirname(self.opts.map_file) + """ Attempt to create the directory /etc/sos/cleaner + just in case it didn't exist previously + """ try: + os.makedirs(cleaner_dir, exist_ok=True) self.write_map_to_file(_map, self.opts.map_file) self.log_debug("Wrote mapping to %s" % self.opts.map_file) except Exception as err: self.log_error("Could not update mapping config file: %s" % err) - def write_cleaner_log(self): + def write_cleaner_log(self, archive=False): """When invoked via the command line, the logging from SoSCleaner will not be added to the archive(s) it processes, so we need to write it separately to disk @@ -419,6 +510,10 @@ third party. for line in self.sos_log_file.readlines(): logfile.write(line) + if archive: + self.obfuscate_file(log_name) + self.archive.add_file(log_name, dest="sos_logs/cleaner.log") + def get_new_checksum(self, archive_path): """Calculate a new checksum for the obfuscated archive, as the previous checksum will no longer be valid @@ -446,19 +541,93 @@ third party. be obfuscated concurrently. """ try: - if len(self.report_paths) > 1: - msg = ("Found %s total reports to obfuscate, processing up to " - "%s concurrently\n" - % (len(self.report_paths), self.opts.jobs)) - self.ui_log.info(msg) + msg = ( + "Found %s total reports to obfuscate, processing up to %s " + "concurrently\n" % (len(self.report_paths), self.opts.jobs) + ) + self.ui_log.info(msg) + if self.opts.keep_binary_files: + self.ui_log.warning( + "WARNING: binary files that potentially contain sensitive " + "information will NOT be removed from the final archive\n" + ) pool = ThreadPoolExecutor(self.opts.jobs) pool.map(self.obfuscate_report, self.report_paths, chunksize=1) pool.shutdown(wait=True) + # finally, obfuscate the nested archive if one exists + if self.nested_archive: + self._replace_obfuscated_archives() + self.obfuscate_report(self.nested_archive) except KeyboardInterrupt: self.ui_log.info("Exiting on user cancel") os._exit(130) - def obfuscate_report(self, report): + def _replace_obfuscated_archives(self): + """When we have a nested archive, we need to rebuild the original + archive, which entails replacing the existing archives with their + obfuscated counterparts + """ + for archive in self.completed_reports: + os.remove(archive.archive_path) + dest = self.nested_archive.extracted_path + tarball = archive.final_archive_path.split('/')[-1] + dest_name = os.path.join(dest, tarball) + shutil.move(archive.final_archive_path, dest) + archive.final_archive_path = dest_name + + def generate_parser_item_regexes(self): + """For the parsers that use prebuilt lists of items, generate those + regexes now since all the parsers should be preloaded by the archive(s) + as well as being handed cmdline options and mapping file configuration. + """ + for parser in self.parsers: + parser.generate_item_regexes() + + def preload_all_archives_into_maps(self): + """Before doing the actual obfuscation, if we have multiple archives + to obfuscate then we need to preload each of them into the mappings + to ensure that node1 is obfuscated in node2 as well as node2 being + obfuscated in node1's archive. + """ + self.log_info("Pre-loading all archives into obfuscation maps") + for _arc in self.report_paths: + for _parser in self.parsers: + try: + pfile = _arc.prep_files[_parser.name.lower().split()[0]] + if not pfile: + continue + except (IndexError, KeyError): + continue + if isinstance(pfile, str): + pfile = [pfile] + for parse_file in pfile: + self.log_debug("Attempting to load %s" % parse_file) + try: + content = _arc.get_file_content(parse_file) + if not content: + continue + if isinstance(_parser, SoSUsernameParser): + _parser.load_usernames_into_map(content) + elif isinstance(_parser, SoSHostnameParser): + if 'hostname' in parse_file: + _parser.load_hostname_into_map( + content.splitlines()[0] + ) + elif 'etc/hosts' in parse_file: + _parser.load_hostname_from_etc_hosts( + content + ) + else: + for line in content.splitlines(): + self.obfuscate_line(line) + except Exception as err: + self.log_info( + "Could not prepare %s from %s (archive: %s): %s" + % (_parser.name, parse_file, _arc.archive_name, + err) + ) + + def obfuscate_report(self, archive): """Individually handle each archive or directory we've discovered by running through each file therein. @@ -467,25 +636,22 @@ third party. :param report str: Filepath to the directory or archive """ try: - if not os.access(report, os.W_OK): - msg = "Insufficient permissions on %s" % report - self.log_info(msg) - self.ui_log.error(msg) - return - - archive = SoSObfuscationArchive(report, self.tmpdir) arc_md = self.cleaner_md.add_section(archive.archive_name) start_time = datetime.now() arc_md.add_field('start_time', start_time) - archive.extract() - self.prep_maps_from_archive(archive) + # don't double extract nested archives + if not archive.is_extracted: + archive.extract() archive.report_msg("Beginning obfuscation...") - file_list = archive.get_file_list() - for fname in file_list: - short_name = fname.split(archive.archive_name)[1].lstrip('/') + for fname in archive.get_file_list(): + short_name = fname.split(archive.archive_name + '/')[1] if archive.should_skip_file(short_name): continue + if (not self.opts.keep_binary_files and + archive.should_remove_file(short_name)): + archive.remove_file(short_name) + continue try: count = self.obfuscate_file(fname, short_name, archive.archive_name) @@ -495,64 +661,50 @@ third party. self.log_debug("Unable to parse file %s: %s" % (short_name, err)) + try: + self.obfuscate_directory_names(archive) + except Exception as err: + self.log_info("Failed to obfuscate directories: %s" % err, + caller=archive.archive_name) + + try: + self.obfuscate_symlinks(archive) + except Exception as err: + self.log_info("Failed to obfuscate symlinks: %s" % err, + caller=archive.archive_name) + # if the archive was already a tarball, repack it - method = archive.get_compression() - if method: - archive.report_msg("Re-compressing...") - try: - archive.rename_top_dir( - self.obfuscate_string(archive.archive_name) - ) - cmd = self.policy.get_cmd_for_compress_method( - method, - self.opts.threads - ) - archive.compress(cmd) - except Exception as err: - self.log_debug("Archive %s failed to compress: %s" - % (archive.archive_name, err)) - archive.report_msg("Failed to re-compress archive: %s" - % err) - return + if not archive.is_nested: + method = archive.get_compression() + if method: + archive.report_msg("Re-compressing...") + try: + archive.rename_top_dir( + self.obfuscate_string(archive.archive_name) + ) + archive.compress(method) + except Exception as err: + self.log_debug("Archive %s failed to compress: %s" + % (archive.archive_name, err)) + archive.report_msg("Failed to re-compress archive: %s" + % err) + return + self.completed_reports.append(archive) end_time = datetime.now() arc_md.add_field('end_time', end_time) arc_md.add_field('run_time', end_time - start_time) arc_md.add_field('files_obfuscated', len(archive.file_sub_list)) arc_md.add_field('total_substitutions', archive.total_sub_count) - self.completed_reports.append(archive) - archive.report_msg("Obfuscation completed") + rmsg = '' + if archive.removed_file_count: + rmsg = " [removed %s unprocessable files]" + rmsg = rmsg % archive.removed_file_count + archive.report_msg("Obfuscation completed%s" % rmsg) except Exception as err: self.ui_log.info("Exception while processing %s: %s" - % (report, err)) - - def prep_maps_from_archive(self, archive): - """Open specific files from an archive and try to load those values - into our mappings before iterating through the entire archive. - - Positional arguments: - - :param archive SoSObfuscationArchive: An open archive object - """ - for parser in self.parsers: - if not parser.prep_map_file: - continue - prep_file = archive.get_file_path(parser.prep_map_file) - if not prep_file: - self.log_debug("Could not prepare %s: %s does not exist" - % (parser.name, parser.prep_map_file), - caller=archive.archive_name) - continue - # this is a bit clunky, but we need to load this particular - # parser in a different way due to how hostnames are validated for - # obfuscation - if isinstance(parser, SoSHostnameParser): - with open(prep_file, 'r') as host_file: - hostname = host_file.readline().strip() - parser.load_hostname_into_map(hostname) - self.obfuscate_file(prep_file, parser.prep_map_file, - archive.archive_name) + % (archive.archive_name, err)) def obfuscate_file(self, filename, short_name=None, arc_name=None): """Obfuscate and individual file, line by line. @@ -571,41 +723,124 @@ third party. if not filename: # the requested file doesn't exist in the archive return - self.log_debug("Obfuscating %s" % short_name or filename, - caller=arc_name) subs = 0 - tfile = tempfile.NamedTemporaryFile(mode='w', dir=self.tmpdir) - with open(filename, 'r') as fname: - for line in fname: - if not line.strip(): - continue - try: - line, count = self.obfuscate_line(line, short_name) - subs += count - tfile.write(line) - except Exception as err: - self.log_debug("Unable to obfuscate %s: %s" - % (short_name, err), caller=arc_name) - tfile.seek(0) - if subs: - shutil.copy(tfile.name, filename) - tfile.close() - _ob_filename = self.obfuscate_string(short_name) + if not short_name: + short_name = filename.split('/')[-1] + if not os.path.islink(filename): + # don't run the obfuscation on the link, but on the actual file + # at some other point. + self.log_debug("Obfuscating %s" % short_name or filename, + caller=arc_name) + tfile = tempfile.NamedTemporaryFile(mode='w', dir=self.tmpdir) + _parsers = [ + _p for _p in self.parsers if not + any( + _skip.match(short_name) for _skip in _p.skip_patterns + ) + ] + with open(filename, 'r', errors='replace') as fname: + for line in fname: + try: + line, count = self.obfuscate_line(line, _parsers) + subs += count + tfile.write(line) + except Exception as err: + self.log_debug("Unable to obfuscate %s: %s" + % (short_name, err), caller=arc_name) + tfile.seek(0) + if subs: + shutil.copy(tfile.name, filename) + tfile.close() + + _ob_short_name = self.obfuscate_string(short_name.split('/')[-1]) + _ob_filename = short_name.replace(short_name.split('/')[-1], + _ob_short_name) + if _ob_filename != short_name: arc_path = filename.split(short_name)[0] _ob_path = os.path.join(arc_path, _ob_filename) - os.rename(filename, _ob_path) + # ensure that any plugin subdirs that contain obfuscated strings + # get created with obfuscated counterparts + if not os.path.islink(filename): + os.rename(filename, _ob_path) + else: + # generate the obfuscated name of the link target + _target_ob = self.obfuscate_string(os.readlink(filename)) + # remove the unobfuscated original symlink first, in case the + # symlink name hasn't changed but the target has + os.remove(filename) + # create the newly obfuscated symlink, pointing to the + # obfuscated target name, which may not exist just yet, but + # when the actual file is obfuscated, will be created + os.symlink(_target_ob, _ob_path) + return subs + def obfuscate_symlinks(self, archive): + """Iterate over symlinks in the archive and obfuscate their names. + The content of the link target will have already been cleaned, and this + second pass over just the names of the links is to ensure we avoid a + possible race condition dependent on the order in which the link or the + target get obfuscated. + + :param archive: The archive being obfuscated + :type archive: ``SoSObfuscationArchive`` + """ + self.log_info("Obfuscating symlink names", caller=archive.archive_name) + for symlink in archive.get_symlinks(): + try: + # relative name of the symlink in the archive + _sym = symlink.split(archive.extracted_path)[1].lstrip('/') + self.log_debug("Obfuscating symlink %s" % _sym, + caller=archive.archive_name) + # current target of symlink, again relative to the archive + _target = os.readlink(symlink) + # get the potentially renamed symlink name, this time the full + # path as it exists on disk + _ob_sym_name = os.path.join(archive.extracted_path, + self.obfuscate_string(_sym)) + # get the potentially renamed relative target filename + _ob_target = self.obfuscate_string(_target) + + # if either the symlink name or the target name has changed, + # recreate the symlink + if (_ob_sym_name != symlink) or (_ob_target != _target): + os.remove(symlink) + os.symlink(_ob_target, _ob_sym_name) + except Exception as err: + self.log_info("Error obfuscating symlink '%s': %s" + % (symlink, err)) + + def obfuscate_directory_names(self, archive): + """For all directories that exist within the archive, obfuscate the + directory name if it contains sensitive strings found during execution + """ + self.log_info("Obfuscating directory names in archive %s" + % archive.archive_name) + for dirpath in sorted(archive.get_directory_list(), reverse=True): + for _name in os.listdir(dirpath): + _dirname = os.path.join(dirpath, _name) + _arc_dir = _dirname.split(archive.extracted_path)[-1] + if os.path.isdir(_dirname): + _ob_dirname = self.obfuscate_string(_name) + if _ob_dirname != _name: + _ob_arc_dir = _arc_dir.rstrip(_name) + _ob_arc_dir = os.path.join( + archive.extracted_path, + _ob_arc_dir.lstrip('/'), + _ob_dirname + ) + os.rename(_dirname, _ob_arc_dir) + def obfuscate_string(self, string_data): for parser in self.parsers: try: string_data = parser.parse_string_for_keys(string_data) - except Exception: - pass + except Exception as err: + self.log_info("Error obfuscating string data: %s" % err) return string_data - def obfuscate_line(self, line, filename): + def obfuscate_line(self, line, parsers=None): """Run a line through each of the obfuscation parsers, keeping a cumulative total of substitutions done on that particular line. @@ -613,16 +848,19 @@ third party. :param line str: The raw line as read from the file being processed - :param filename str: Filename the line was read from + :param parsers: A list of parser objects to obfuscate + with. If None, use all. Returns the fully obfuscated line and the number of substitutions made """ + # don't iterate over blank lines, but still write them to the tempfile + # to maintain the same structure when we write a scrubbed file back count = 0 - for parser in self.parsers: - if filename and any([ - re.match(_s, filename) for _s in parser.skip_files - ]): - continue + if not line.strip(): + return line, count + if parsers is None: + parsers = self.parsers + for parser in parsers: try: line, _count = parser.parse_line(line) count += _count @@ -637,3 +875,5 @@ third party. for parser in self.parsers: _sec = parse_sec.add_section(parser.name.replace(' ', '_').lower()) _sec.add_field('entries', len(parser.mapping.dataset.keys())) + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/cleaner/archives/__init__.py 4.5.3ubuntu2/sos/cleaner/archives/__init__.py --- 4.0-2/sos/cleaner/archives/__init__.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/cleaner/archives/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,391 @@ +# Copyright 2020 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import logging +import os +import shutil +import stat +import tarfile +import re + +from concurrent.futures import ProcessPoolExecutor +from sos.utilities import file_is_binary + + +# python older than 3.8 will hit a pickling error when we go to spawn a new +# process for extraction if this method is a part of the SoSObfuscationArchive +# class. So, the simplest solution is to remove it from the class. +def extract_archive(archive_path, tmpdir): + archive = tarfile.open(archive_path) + path = os.path.join(tmpdir, 'cleaner') + archive.extractall(path) + archive.close() + return os.path.join(path, archive.name.split('/')[-1].split('.tar')[0]) + + +class SoSObfuscationArchive(): + """A representation of an extracted archive or an sos archive build + directory which is used by SoSCleaner. + + Each archive that needs to be obfuscated is loaded into an instance of this + class. All report-level operations should be contained within this class. + """ + + file_sub_list = [] + total_sub_count = 0 + removed_file_count = 0 + type_name = 'undetermined' + description = 'undetermined' + is_nested = False + skip_files = [] + prep_files = {} + + def __init__(self, archive_path, tmpdir): + self.archive_path = archive_path + self.final_archive_path = self.archive_path + self.tmpdir = tmpdir + self.archive_name = self.archive_path.split('/')[-1].split('.tar')[0] + self.ui_name = self.archive_name + self.soslog = logging.getLogger('sos') + self.ui_log = logging.getLogger('sos_ui') + self.skip_list = self._load_skip_list() + self.is_extracted = False + self._load_self() + self.archive_root = '' + self.log_info( + "Loaded %s as type %s" + % (self.archive_path, self.description) + ) + + @classmethod + def check_is_type(cls, arc_path): + """Check if the archive is a well-known type we directly support""" + return False + + def _load_self(self): + if self.is_tarfile: + self.tarobj = tarfile.open(self.archive_path) + + def get_nested_archives(self): + """Return a list of ObfuscationArchives that represent additional + archives found within the target archive. For example, an archive from + `sos collect` will return a list of ``SoSReportArchive`` objects. + + This should be overridden by individual types of ObfuscationArchive's + """ + return [] + + def get_archive_root(self): + """Set the root path for the archive that should be prepended to any + filenames given to methods in this class. + """ + if self.is_tarfile: + toplevel = self.tarobj.firstmember + if toplevel.isdir(): + return toplevel.name + else: + return os.sep + return os.path.abspath(self.archive_path) + + def report_msg(self, msg): + """Helper to easily format ui messages on a per-report basis""" + self.ui_log.info("{:<50} {}".format(self.ui_name + ' :', msg)) + + def _fmt_log_msg(self, msg): + return "[cleaner:%s] %s" % (self.archive_name, msg) + + def log_debug(self, msg): + self.soslog.debug(self._fmt_log_msg(msg)) + + def log_info(self, msg): + self.soslog.info(self._fmt_log_msg(msg)) + + def _load_skip_list(self): + """Provide a list of files and file regexes to skip obfuscation on + + Returns: list of files and file regexes + """ + return [ + 'proc/kallsyms', + 'sosreport-', + 'sys/firmware', + 'sys/fs', + 'sys/kernel/debug', + 'sys/module' + ] + + @property + def is_tarfile(self): + try: + return tarfile.is_tarfile(self.archive_path) + except Exception: + return False + + def remove_file(self, fname): + """Remove a file from the archive. This is used when cleaner encounters + a binary file, which we cannot reliably obfuscate. + """ + full_fname = self.get_file_path(fname) + # don't call a blank remove() here + if full_fname: + self.log_info("Removing binary file '%s' from archive" % fname) + os.remove(full_fname) + self.removed_file_count += 1 + + def format_file_name(self, fname): + """Based on the type of archive we're dealing with, do whatever that + archive requires to a provided **relative** filepath to be able to + access it within the archive + """ + if not self.is_extracted: + if not self.archive_root: + self.archive_root = self.get_archive_root() + return os.path.join(self.archive_root, fname) + else: + return os.path.join(self.extracted_path, fname) + + def get_file_content(self, fname): + """Return the content from the specified fname. Particularly useful for + tarball-type archives so we can retrieve prep file contents prior to + extracting the entire archive + """ + if self.is_extracted is False and self.is_tarfile: + filename = self.format_file_name(fname) + try: + return self.tarobj.extractfile(filename).read().decode('utf-8') + except KeyError: + self.log_debug( + "Unable to retrieve %s: no such file in archive" % fname + ) + return '' + else: + with open(self.format_file_name(fname), 'r') as to_read: + return to_read.read() + + def extract(self, quiet=False): + if self.is_tarfile: + if not quiet: + self.report_msg("Extracting...") + self.extracted_path = self.extract_self() + self.is_extracted = True + else: + self.extracted_path = self.archive_path + # if we're running as non-root (e.g. collector), then we can have a + # situation where a particular path has insufficient permissions for + # us to rewrite the contents and/or add it to the ending tarfile. + # Unfortunately our only choice here is to change the permissions + # that were preserved during report collection + if os.getuid() != 0: + self.log_debug('Verifying permissions of archive contents') + for dirname, dirs, files in os.walk(self.extracted_path): + try: + for _dir in dirs: + _dirname = os.path.join(dirname, _dir) + _dir_perms = os.stat(_dirname).st_mode + os.chmod(_dirname, _dir_perms | stat.S_IRWXU) + for filename in files: + fname = os.path.join(dirname, filename) + # protect against symlink race conditions + if not os.path.exists(fname) or os.path.islink(fname): + continue + if (not os.access(fname, os.R_OK) or not + os.access(fname, os.W_OK)): + self.log_debug( + "Adding owner rw permissions to %s" + % fname.split(self.archive_path)[-1] + ) + os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR) + except Exception as err: + self.log_debug("Error while trying to set perms: %s" % err) + self.log_debug("Extracted path is %s" % self.extracted_path) + + def rename_top_dir(self, new_name): + """Rename the top-level directory to new_name, which should be an + obfuscated string that scrubs the hostname from the top-level dir + which would be named after the unobfuscated sos report + """ + _path = self.extracted_path.replace(self.archive_name, new_name) + self.archive_name = new_name + os.rename(self.extracted_path, _path) + self.extracted_path = _path + + def get_compression(self): + """Return the compression type used by the archive, if any. This is + then used by SoSCleaner to generate a policy-derived compression + command to repack the archive + """ + if self.is_tarfile: + if self.archive_path.endswith('xz'): + return 'xz' + return 'gz' + return None + + def build_tar_file(self, method): + """Pack the extracted archive as a tarfile to then be re-compressed + """ + mode = 'w' + tarpath = self.extracted_path + '-obfuscated.tar' + compr_args = {} + if method: + mode += ":%s" % method + tarpath += ".%s" % method + if method == 'xz': + compr_args = {'preset': 3} + else: + compr_args = {'compresslevel': 6} + self.log_debug("Building tar file %s" % tarpath) + tar = tarfile.open(tarpath, mode=mode, **compr_args) + tar.add(self.extracted_path, + arcname=os.path.split(self.archive_name)[1]) + tar.close() + return tarpath + + def compress(self, method): + """Execute the compression command, and set the appropriate final + archive path for later reference by SoSCleaner on a per-archive basis + """ + try: + self.final_archive_path = self.build_tar_file(method) + except Exception as err: + self.log_debug("Exception while re-compressing archive: %s" % err) + raise + self.log_debug("Compressed to %s" % self.final_archive_path) + try: + self.remove_extracted_path() + except Exception as err: + self.log_debug("Failed to remove extraction directory: %s" % err) + self.report_msg('Failed to remove temporary extraction directory') + + def remove_extracted_path(self): + """After the tarball has been re-compressed, remove the extracted path + so that we don't take up that duplicate space any longer during + execution + """ + def force_delete_file(action, name, exc): + os.chmod(name, stat.S_IWUSR) + if os.path.isfile(name): + os.remove(name) + else: + shutil.rmtree(name) + self.log_debug("Removing %s" % self.extracted_path) + shutil.rmtree(self.extracted_path, onerror=force_delete_file) + + def extract_self(self): + """Extract an archive into our tmpdir so that we may inspect it or + iterate through its contents for obfuscation + """ + + with ProcessPoolExecutor(1) as _pool: + _path_future = _pool.submit(extract_archive, + self.archive_path, self.tmpdir) + path = _path_future.result() + return path + + def get_symlinks(self): + """Iterator for a list of symlinks in the archive""" + for dirname, dirs, files in os.walk(self.extracted_path): + for _dir in dirs: + _dirpath = os.path.join(dirname, _dir) + if os.path.islink(_dirpath): + yield _dirpath + for filename in files: + _fname = os.path.join(dirname, filename) + if os.path.islink(_fname): + yield _fname + + def get_file_list(self): + """Iterator for a list of files in the archive, to allow clean to + iterate over. + + Will not include symlinks, as those are handled separately + """ + for dirname, dirs, files in os.walk(self.extracted_path): + for filename in files: + _fname = os.path.join(dirname, filename.lstrip('/')) + if os.path.islink(_fname): + continue + else: + yield _fname + + def get_directory_list(self): + """Return a list of all directories within the archive""" + dir_list = [] + for dirname, dirs, files in os.walk(self.extracted_path): + dir_list.append(dirname) + return dir_list + + def update_sub_count(self, fname, count): + """Called when a file has finished being parsed and used to track + total substitutions made and number of files that had changes made + """ + self.file_sub_list.append(fname) + self.total_sub_count += count + + def get_file_path(self, fname): + """Return the filepath of a specific file within the archive so that + it may be selectively inspected if it exists + """ + _path = os.path.join(self.extracted_path, fname.lstrip('/')) + return _path if os.path.exists(_path) else '' + + def should_skip_file(self, filename): + """Checks the provided filename against a list of filepaths to not + perform obfuscation on, as defined in self.skip_list + + Positional arguments: + + :param filename str: Filename relative to the extracted + archive root + """ + + if (not os.path.isfile(self.get_file_path(filename)) and not + os.path.islink(self.get_file_path(filename))): + return True + + for _skip in self.skip_list: + if filename.startswith(_skip) or re.match(_skip, filename): + return True + return False + + def should_remove_file(self, fname): + """Determine if the file should be removed or not, due to an inability + to reliably obfuscate that file based on the filename. + + :param fname: Filename relative to the extracted archive root + :type fname: ``str`` + + :returns: ``True`` if the file cannot be reliably obfuscated + :rtype: ``bool`` + """ + obvious_removes = [ + r'.*\.gz$', # TODO: support flat gz/xz extraction + r'.*\.xz$', + r'.*\.bzip2$', + r'.*\.tar\..*', # TODO: support archive unpacking + r'.*\.txz$', + r'.*\.tgz$', + r'.*\.bin$', + r'.*\.journal$', + r'.*\~$' + ] + + # if the filename matches, it is obvious we can remove them without + # doing the read test + for _arc_reg in obvious_removes: + if re.match(_arc_reg, fname): + return True + + _full_path = self.get_file_path(fname) + if os.path.isfile(_full_path): + return file_is_binary(_full_path) + # don't fail on dir-level symlinks + return False + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/cleaner/archives/generic.py 4.5.3ubuntu2/sos/cleaner/archives/generic.py --- 4.0-2/sos/cleaner/archives/generic.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/cleaner/archives/generic.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,52 @@ +# Copyright 2020 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + + +from sos.cleaner.archives import SoSObfuscationArchive + +import os +import tarfile + + +class DataDirArchive(SoSObfuscationArchive): + """A plain directory on the filesystem that is not directly associated with + any known or supported collection utility + """ + + type_name = 'data_dir' + description = 'unassociated directory' + + @classmethod + def check_is_type(cls, arc_path): + return os.path.isdir(arc_path) + + def set_archive_root(self): + return os.path.abspath(self.archive_path) + + +class TarballArchive(SoSObfuscationArchive): + """A generic tar archive that is not associated with any known or supported + collection utility + """ + + type_name = 'tarball' + description = 'unassociated tarball' + + @classmethod + def check_is_type(cls, arc_path): + try: + return tarfile.is_tarfile(arc_path) + except Exception: + return False + + def set_archive_root(self): + if self.tarobj.firstmember.isdir(): + return self.tarobj.firstmember.name + return '' diff -pruN 4.0-2/sos/cleaner/archives/insights.py 4.5.3ubuntu2/sos/cleaner/archives/insights.py --- 4.0-2/sos/cleaner/archives/insights.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/cleaner/archives/insights.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,42 @@ +# Copyright 2021 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + + +from sos.cleaner.archives import SoSObfuscationArchive + +import tarfile + + +class InsightsArchive(SoSObfuscationArchive): + """This class represents archives generated by the insights-client utility + for RHEL systems. + """ + + type_name = 'insights' + description = 'insights-client archive' + + prep_files = { + 'hostname': 'data/insights_commands/hostname_-f', + 'ip': 'data/insights_commands/ip_addr', + 'mac': 'data/insights_commands/ip_addr' + } + + @classmethod + def check_is_type(cls, arc_path): + try: + return tarfile.is_tarfile(arc_path) and 'insights-' in arc_path + except Exception: + return False + + def get_archive_root(self): + top = self.archive_path.split('/')[-1].split('.tar')[0] + if self.tarobj.firstmember.name == '.': + top = './' + top + return top diff -pruN 4.0-2/sos/cleaner/archives/sos.py 4.5.3ubuntu2/sos/cleaner/archives/sos.py --- 4.0-2/sos/cleaner/archives/sos.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/cleaner/archives/sos.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,111 @@ +# Copyright 2021 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + + +from sos.cleaner.archives import SoSObfuscationArchive + +import os +import tarfile + + +class SoSReportArchive(SoSObfuscationArchive): + """This is the class representing an sos report, or in other words the + type the archive the SoS project natively generates + """ + + type_name = 'report' + description = 'sos report archive' + prep_files = { + 'hostname': [ + 'sos_commands/host/hostname', + 'etc/hosts' + ], + 'ip': 'sos_commands/networking/ip_-o_addr', + 'mac': 'sos_commands/networking/ip_-d_address', + 'username': [ + 'sos_commands/login/lastlog_-u_1000-60000', + 'sos_commands/login/lastlog_-u_60001-65536', + 'sos_commands/login/lastlog_-u_65537-4294967295', + # AD users will be reported here, but favor the lastlog files since + # those will include local users who have not logged in + 'sos_commands/login/last', + 'etc/cron.allow', + 'etc/cron.deny' + ] + } + + @classmethod + def check_is_type(cls, arc_path): + try: + return tarfile.is_tarfile(arc_path) and 'sosreport-' in arc_path + except Exception: + return False + + +class SoSReportDirectory(SoSReportArchive): + """This is the archive class representing a build directory, or in other + words what `sos report --clean` will end up using for in-line obfuscation + """ + + type_name = 'report_dir' + description = 'sos report directory' + + @classmethod + def check_is_type(cls, arc_path): + if os.path.isdir(arc_path): + return 'sos_logs' in os.listdir(arc_path) + return False + + +class SoSCollectorArchive(SoSObfuscationArchive): + """Archive class representing the tarball created by ``sos collect``. It + will not provide prep files on its own, however it will provide a list + of SoSReportArchive's which will then be used to prep the parsers + """ + + type_name = 'collect' + description = 'sos collect tarball' + is_nested = True + + @classmethod + def check_is_type(cls, arc_path): + try: + return (tarfile.is_tarfile(arc_path) and 'sos-collect' in arc_path) + except Exception: + return False + + def get_nested_archives(self): + self.extract(quiet=True) + _path = self.extracted_path + archives = [] + for fname in os.listdir(_path): + arc_name = os.path.join(_path, fname) + if 'sosreport-' in fname and tarfile.is_tarfile(arc_name): + archives.append(SoSReportArchive(arc_name, self.tmpdir)) + return archives + + +class SoSCollectorDirectory(SoSCollectorArchive): + """The archive class representing the temp directory used by ``sos + collect`` when ``--clean`` is used during runtime. + """ + + type_name = 'collect_dir' + description = 'sos collect directory' + + @classmethod + def check_is_type(cls, arc_path): + if os.path.isdir(arc_path): + for fname in os.listdir(arc_path): + if 'sos-collector-' in fname: + return True + return False + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/cleaner/mappings/__init__.py 4.5.3ubuntu2/sos/cleaner/mappings/__init__.py --- 4.0-2/sos/cleaner/mappings/__init__.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/cleaner/mappings/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -20,24 +20,28 @@ class SoSMap(): corresponding SoSMap() object, to allow for easy retrieval of obfuscated items. """ - + # used for regex skips in parser.parse_line() ignore_matches = [] + # used for filename obfuscations in parser.parse_string_for_keys() + skip_keys = [] + compile_regexes = True def __init__(self): self.dataset = {} + self._regexes_made = set() + self.compiled_regexes = [] self.lock = Lock() def ignore_item(self, item): """Some items need to be completely ignored, for example link-local or loopback addresses should not be obfuscated """ + if not item or item in self.skip_keys or item in self.dataset.values(): + return True for skip in self.ignore_matches: - if re.match(skip, item): + if re.match(skip, item, re.I): return True - def item_in_dataset_values(self, item): - return item in self.dataset.values() - def add(self, item): """Add a particular item to the map, generating an obfuscated pair for it. @@ -46,10 +50,52 @@ class SoSMap(): :param item: The plaintext object to obfuscate """ + if self.ignore_item(item): + return item with self.lock: self.dataset[item] = self.sanitize_item(item) + if self.compile_regexes: + self.add_regex_item(item) return self.dataset[item] + def add_regex_item(self, item): + """Add an item to the regexes dict and then re-sort the list that the + parsers will use during parse_line() + + :param item: The unobfuscated item to generate a regex for + :type item: ``str`` + """ + if self.ignore_item(item): + return + if item not in self._regexes_made: + # save the item in a set to avoid clobbering existing regexes, + # as searching this set is significantly faster than searching + # through the actual compiled_regexes list, especially for very + # large collections of entries + self._regexes_made.add(item) + # add the item, Pattern tuple directly to the compiled_regexes list + # and then sort the existing list, rather than rebuild the list + # from scratch every time we add something like we would do if we + # tracked/saved the item and the Pattern() object in a dict or in + # the set above + self.compiled_regexes.append((item, self.get_regex_result(item))) + self.compiled_regexes.sort(key=lambda x: len(x[0]), reverse=True) + + def get_regex_result(self, item): + """Generate the object/value that is used by the parser when iterating + over pre-generated regexes during parse_line(). For most parsers this + will simply be a ``re.Pattern()`` object, but for more complex parsers + this can be overridden to provide a different object, e.g. a tuple, + for that parer's specific iteration needs. + + :param item: The unobfuscated string to generate the regex for + :type item: ``str`` + + :returns: A compiled regex pattern for the item + :rtype: ``re.Pattern`` + """ + return re.compile(re.escape(item), re.I) + def sanitize_item(self, item): """Perform the obfuscation relevant to the item being added to the map. @@ -65,7 +111,7 @@ class SoSMap(): """Retrieve an item's obfuscated counterpart from the map. If the item does not yet exist in the map, add it by generating one on the fly """ - if self.ignore_item(item) or self.item_in_dataset_values(item): + if self.ignore_item(item): return item if item not in self.dataset: return self.add(item) diff -pruN 4.0-2/sos/cleaner/mappings/hostname_map.py 4.5.3ubuntu2/sos/cleaner/mappings/hostname_map.py --- 4.0-2/sos/cleaner/mappings/hostname_map.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/cleaner/mappings/hostname_map.py 2023-04-28 17:16:21.000000000 +0000 @@ -35,70 +35,204 @@ class SoSHostnameMap(SoSMap): '^com..*' ] + skip_keys = [ + 'www', + 'api' + ] + + strip_exts = ('.yaml', '.yml', '.crt', '.key', '.pem', '.log', '.repo', + '.rules') + host_count = 0 domain_count = 0 _domains = {} hosts = {} - def __init__(self, opt_domains): - super(SoSHostnameMap, self).__init__() - self.load_domains_from_options(opt_domains) + def load_domains_from_map(self): + """Because we use 'intermediary' dicts for host names and domain names + in this parser, we need to re-inject entries from the map_file into + these dicts and not just the underlying 'dataset' dict + """ + for domain, ob_pair in self.dataset.items(): + if len(domain.split('.')) == 1: + self.hosts[domain.split('.')[0]] = self.dataset[domain] + else: + if ob_pair.startswith('obfuscateddomain'): + # directly exact domain matches + self._domains[domain] = ob_pair.split('.')[0] + continue + # strip the host name and trailing top-level domain so that + # we in inject the domain properly for later string matching + + # note: this is artificially complex due to our stance on + # preserving TLDs. If in the future the project decides to + # obfuscate TLDs as well somehow, then this will all become + # much simpler + _domain_to_inject = '.'.join(domain.split('.')[1:-1]) + if not _domain_to_inject: + continue + for existing_domain in self.dataset.keys(): + _existing = '.'.join(existing_domain.split('.')[:-1]) + if _existing == _domain_to_inject: + _ob_domain = '.'.join( + self.dataset[existing_domain].split('.')[:-1] + ) + self._domains[_domain_to_inject] = _ob_domain + self.set_initial_counts() def load_domains_from_options(self, domains): for domain in domains: self.sanitize_domain(domain.split('.')) + def get_regex_result(self, item): + """Override the base get_regex_result() to provide a regex that, if + this is an FQDN or a straight domain, will include an underscore + formatted regex as well. + """ + if '.' in item: + item = item.replace('.', '(\\.|_)') + return re.compile(item, re.I) + + def set_initial_counts(self): + """Set the initial counter for host and domain obfuscation numbers + based on what is already present in the mapping. + """ + # hostnames/short names + try: + h = sorted(self.hosts.values(), reverse=True)[0].split('host')[1] + self.host_count = int(h) + 1 + except IndexError: + # no hosts loaded yet + pass + + # domain names + try: + d = sorted(self._domains.values(), reverse=True)[0].split('domain') + self.domain_count = int(d[1].split('.')[0]) + 1 + except IndexError: + # no domains loaded yet + pass + def domain_name_in_loaded_domains(self, domain): """Check if a potential domain is in one of the domains we've loaded and should be obfuscated """ + if domain in self._domains: + return True host = domain.split('.') + no_tld = '.'.join(domain.split('.')[0:-1]) if len(host) == 1: # don't block on host's shortname + return host[0] in self.hosts + elif any([no_tld.endswith(_d) for _d in self._domains]): return True - if len(host) < 2: - return False - else: - domain = host[0:-1] - for known_domain in self._domains: - if known_domain in domain: - return True + return False def get(self, item): - if item.startswith(('.', '_')): - item = item.lstrip('._') - item = item.strip() - if not self.domain_name_in_loaded_domains(item): - return item - return super(SoSHostnameMap, self).get(item) + prefix = '' + suffix = '' + final = None + # The regex pattern match may include a leading and/or trailing '_' + # character due to the need to use word boundary matching, so we need + # to strip these from the string during processing, but still keep them + # in the returned string to not mangle the string replacement in the + # context of the file or filename + while item.startswith(('.', '_')): + prefix += item[0] + item = item[1:] + while item.endswith(('.', '_')): + suffix += item[-1] + item = item[0:-1] + if item in self.dataset: + return self.dataset[item] + if not self.domain_name_in_loaded_domains(item.lower()): + # no match => return the original string with optional + # leading/trailing '.' or '_' characters + return ''.join([prefix, item, suffix]) + if item.endswith(self.strip_exts): + ext = '.' + item.split('.')[-1] + item = item.replace(ext, '') + suffix += ext + if item not in self.dataset.keys(): + # try to account for use of '-' in names that include hostnames + # and don't create new mappings for each of these + for _existing in sorted(self.dataset.keys(), reverse=True, + key=lambda x: len(x)): + _host_substr = False + _test = item.split(_existing) + _h = _existing.split('.') + # avoid considering a full FQDN match as a new match off of + # the hostname of an existing match + if _h[0] and _h[0] in self.hosts.keys(): + _host_substr = True + if len(_test) == 1 or not _test[0]: + # does not match existing obfuscation + continue + elif not _host_substr and (_test[0].endswith('.') or + item.endswith(_existing)): + # new hostname in known domain + final = super(SoSHostnameMap, self).get(item) + break + elif item.split(_test[0]): + # string that includes existing FQDN obfuscation substring + # so, only obfuscate the FQDN part + try: + itm = item.split(_test[0])[1] + final = _test[0] + super(SoSHostnameMap, self).get(itm) + break + except Exception: + # fallback to still obfuscating the entire item + pass + + if not final: + final = super(SoSHostnameMap, self).get(item) + return prefix + final + suffix def sanitize_item(self, item): host = item.split('.') if len(host) == 1: # we have a shortname for a host - return self.sanitize_short_name(host[0]) + return self.sanitize_short_name(host[0].lower()) if len(host) == 2: # we have just a domain name, e.g. example.com - return self.sanitize_domain(host) + dname = self.sanitize_domain(host) + if all([h.isupper() for h in host]): + dname = dname.upper() + return dname if len(host) > 2: # we have an FQDN, e.g. foo.example.com hostname = host[0] domain = host[1:] # obfuscate the short name - ob_hostname = self.sanitize_short_name(hostname) + if len(hostname) > 2: + ob_hostname = self.sanitize_short_name(hostname.lower()) + else: + # by best practice it appears the host part of the fqdn was cut + # off due to some form of truncating, as such don't obfuscate + # short strings that are likely to throw off obfuscation of + # unrelated bits and paths + ob_hostname = 'unknown' ob_domain = self.sanitize_domain(domain) - return '.'.join([ob_hostname, ob_domain]) + self.dataset[item] = ob_domain + _fqdn = '.'.join([ob_hostname, ob_domain]) + if all([h.isupper() for h in host]): + _fqdn = _fqdn.upper() + return _fqdn def sanitize_short_name(self, hostname): """Obfuscate the short name of the host with an incremented counter based on the total number of obfuscated host names """ - if hostname not in self.hosts: + if not hostname or hostname in self.skip_keys: + return hostname + if hostname not in self.dataset: ob_host = "host%s" % self.host_count self.hosts[hostname] = ob_host self.host_count += 1 - return self.hosts[hostname] + self.dataset[hostname] = ob_host + self.add_regex_item(hostname) + return self.dataset[hostname] def sanitize_domain(self, domain): """Obfuscate the domainname, broken out into subdomains. Top-level @@ -108,10 +242,11 @@ class SoSHostnameMap(SoSMap): # don't obfuscate vendor domains if re.match(_skip, '.'.join(domain)): return '.'.join(domain) - top_domain = domain[-1] - dname = '.'.join(domain[0:-1]) + top_domain = domain[-1].lower() + dname = '.'.join(domain[0:-1]).lower() ob_domain = self._new_obfuscated_domain(dname) ob_domain = '.'.join([ob_domain, top_domain]) + self.dataset['.'.join(domain)] = ob_domain return ob_domain def _new_obfuscated_domain(self, dname): diff -pruN 4.0-2/sos/cleaner/mappings/ip_map.py 4.5.3ubuntu2/sos/cleaner/mappings/ip_map.py --- 4.0-2/sos/cleaner/mappings/ip_map.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/cleaner/mappings/ip_map.py 2023-04-28 17:16:21.000000000 +0000 @@ -44,6 +44,7 @@ class SoSIPMap(SoSMap): _networks = {} network_first_octet = 100 skip_network_octets = ['127', '169', '172', '192'] + compile_regexes = False def ip_in_dataset(self, ipaddr): """There are multiple ways in which an ip address could be handed to us @@ -121,13 +122,12 @@ class SoSIPMap(SoSMap): # network and if it has, replace the default /32 netmask that # ipaddress applies to no CIDR-notated addresses self.set_ip_cidr_from_existing_subnet(addr) - return self.sanitize_ipaddr(addr) else: # we have a CIDR notation, so generate an obfuscated network # address and then generate an IP address within that network's # range self.sanitize_network(network) - return self.sanitize_ipaddr(addr) + return self.sanitize_ipaddr(addr) def sanitize_network(self, network): """Obfuscate the network address provided, and if there are host bits diff -pruN 4.0-2/sos/cleaner/mappings/ipv6_map.py 4.5.3ubuntu2/sos/cleaner/mappings/ipv6_map.py --- 4.0-2/sos/cleaner/mappings/ipv6_map.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/cleaner/mappings/ipv6_map.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,279 @@ +# Copyright 2022 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import ipaddress + +from random import getrandbits +from sos.cleaner.mappings import SoSMap + + +def generate_hextets(hextets): + """Generate a random set of hextets, based on the length of the source + hextet. If any hextets are compressed, keep that compression. + + E.G. '::1234:bcd' will generate a leading empty '' hextet, followed by two + 4-character hextets. + + :param hextets: The extracted hextets from a source address + :type hextets: ``list`` + + :returns: A set of randomized hextets for use in an obfuscated + address + :rtype: ``list`` + """ + return [random_hex(4) if h else '' for h in hextets] + + +def random_hex(length): + """Generate a string of size length of random hex characters. + + :param length: The number of characters to generate + :type length: ``int`` + + :returns: A string of ``length`` hex characters + :rtype: ``str`` + """ + return f"{getrandbits(4*length):0{length}x}" + + +class SoSIPv6Map(SoSMap): + """Mapping for IPv6 addresses and networks. + + Much like the IP map handles IPv4 addresses, this map is designed to take + IPv6 strings and obfuscate them consistently to maintain network topology. + To do this, addresses will be manipulated by the ipaddress library. + + If an IPv6 address is encountered without a netmask, it is assumed to be a + /64 address. + """ + + networks = {} + + ignore_matches = [ + r'^::1/.*', + r'::/0', + r'fd53:.*', + r'^53..:' + ] + + first_hexes = ['534f'] + + compile_regexes = False + version = 1 + + def conf_update(self, config): + """Override the base conf_update() so that we can load the existing + networks into ObfuscatedIPv6Network() objects for the current run. + """ + if 'networks' not in config: + return + for network in config['networks']: + _orig = ipaddress.ip_network(network) + _obfuscated = config['networks'][network]['obfuscated'] + _net = self._get_network(_orig, _obfuscated) + self.dataset[_net.original_address] = _net.obfuscated_address + for host in config['networks'][network]['hosts']: + _ob_host = config['networks'][network]['hosts'][host] + _net.add_obfuscated_host_address(host, _ob_host) + self.dataset[host] = _ob_host + + def sanitize_item(self, ipaddr): + _prefix = ipaddr.split('/')[-1] if '/' in ipaddr else '' + _ipaddr = ipaddr + if not _prefix: + # assume a /64 default per protocol + _ipaddr += "/64" + try: + _addr = ipaddress.ip_network(_ipaddr) + # ipaddr was an actual network per protocol + _net = self._get_network(_addr) + _ipaddr = _net.obfuscated_address + except ValueError: + # A ValueError is raised from the ipaddress module when passing + # an address such as 2620:52:0:2d80::4fe/64, which has host bits + # '::4fe' set - the /64 is generally interpreted only for network + # addresses. We use this behavior to properly obfuscate the network + # before obfuscating a host address within that network + _addr = ipaddress.ip_network(_ipaddr, strict=False) + _net = self._get_network(_addr) + if _net.network_addr not in self.dataset: + self.dataset[_net.original_address] = _net.obfuscated_address + # then, get the address within the network + _hostaddr = ipaddress.ip_address(_ipaddr.split('/')[0]) + _ipaddr = _net.obfuscate_host_address(_hostaddr) + + if _prefix and '/' not in _ipaddr: + return f"{_ipaddr}/{_prefix}" + return _ipaddr + + def _get_network(self, address, obfuscated=''): + """Attempt to find an existing ObfuscatedIPv6Network object from which + to either find an existing obfuscated match, or create a new one. If + no such object already exists, create it. + """ + _addr = address.compressed + if _addr not in self.networks: + self.networks[_addr] = ObfuscatedIPv6Network(address, obfuscated, + self.first_hexes) + return self.networks[_addr] + + +class ObfuscatedIPv6Network(): + """An abstraction class that represents a network that is (to be) handled + by sos. + + Each distinct IPv6 network that we encounter will have a representative + instance of this class, from which new obfuscated subnets and host + addresses will be generated. + + This class should be built from an ``ipaddress.IPv6Network`` object. If + an obfuscation string is not passed, one will be created during init. + """ + + def __init__(self, addr, obfuscation='', used_hexes=None): + """Basic setup for the obfuscated network. Minor validation on the addr + used to create the instance, as well as on an optional ``obfuscation`` + which if set, will serve as the obfuscated_network address. + + :param addr: The *un*obfuscated network to be handled + :type addr: ``ipaddress.IPv6Network`` + + :param obfuscation: An optional pre-determined string representation of + the obfuscated network address + :type obfuscation: ``str`` + + :param used_hexes: A list of already used hexes for the first hextet + of a potential global address obfuscation + :type used_hexes: ``list`` + """ + if not isinstance(addr, ipaddress.IPv6Network): + raise Exception('Invalid network: not an IPv6Network object') + self.addr = addr + self.prefix = addr.prefixlen + self.network_addr = addr.network_address.compressed + self.hosts = {} + if used_hexes is None: + self.first_hexes = ['534f'] + else: + self.first_hexes = used_hexes + if not obfuscation: + self._obfuscated_network = self._obfuscate_network_address() + else: + if not isinstance(obfuscation, str): + raise TypeError(f"Pre-determined obfuscated network address " + f"must be str, not {type(obfuscation)}") + self._obfuscated_network = obfuscation.split('/')[0] + + @property + def obfuscated_address(self): + return f"{self._obfuscated_network}/{self.prefix}" + + @property + def original_address(self): + return self.addr.compressed + + def _obfuscate_network_address(self): + """Generate the obfuscated pair for the network address. This is + determined based on the netmask of the network this class was built + on top of. + """ + if self.addr.is_global: + return self._obfuscate_global_address() + elif self.addr.is_link_local: + # link-local addresses are always fe80::/64. This is not sensitive + # in itself, and retaining the information that an address is a + # link-local address is important for problem analysis, so don't + # obfuscate this network information. + return self.network_addr + elif self.addr.is_private: + return self._obfuscate_private_address() + return self.network_addr + + def _obfuscate_global_address(self): + """Global unicast addresses have a 48-bit global routing prefix and a + 16-bit subnet. We set the global routing prefix to a static + sos-specific identifier that could never be seen in the wild, + '534f:' + + We then randomize the subnet hextet. + """ + _hextets = self.network_addr.split(':')[1:] + _ob_hex = ['534f'] + if all(not c for c in _hextets): + # we have only a single defined hextet, e.g. ff00::/64, so we need + # to not use the standard first-hex identifier or we'll overlap + # every similar address obfuscation. + # Set the leading bits to 53, but increment upwards from there for + # when we exceed 256 networks obfuscated in this manner. + _start = 53 + (len(self.first_hexes) // 256) + _ob_hex = f"{_start}{random_hex(2)}" + while _ob_hex in self.first_hexes: + # prevent duplicates + _ob_hex = f"{_start}{random_hex(2)}" + self.first_hexes.append(_ob_hex) + _ob_hex = [_ob_hex] + _ob_hex.extend(generate_hextets(_hextets)) + return ':'.join(_ob_hex) + + def _obfuscate_private_address(self): + """The first 8 bits will always be 'fd', the next 40 bits are meant + to be a global ID, followed by 16 bits for the subnet. To keep things + relatively simply we maintain the first hextet as 'fd53', and then + randomize any remaining hextets + """ + _hextets = self.network_addr.split(':')[1:] + _ob_hex = ['fd53'] + _ob_hex.extend(generate_hextets(_hextets)) + return ':'.join(_ob_hex) + + def obfuscate_host_address(self, addr): + """Given an unobfuscated address, generate an obfuscated match for it, + and save it to this network for tracking during the execution of clean. + + Note: another way to do this would be to convert the obfuscated network + to bytes, and add a random amount to that based on the number of + addresses that the network can support and from that new bytes count + craft a new IPv6 address. This has the advantage of absolutely + guaranteeing the new address is within the network space (whereas the + method employed below could *theoretically* generate an overlapping + address), but would in turn remove any ability to compress obfuscated + addresses to match the general format/syntax of the address it is + replacing. For the moment, it is assumed that being able to maintain a + quick mental note of "unobfuscated device ff00::1 is obfuscated device + 53ad::a1b2" is more desireable than "ff00::1 is now obfuscated as + 53ad::1234:abcd:9876:a1b2:". + + :param addr: The unobfuscated IPv6 address + :type addr: ``ipaddress.IPv6Address`` + + :returns: An obfuscated address within this network + :rtype: ``str`` + """ + def _generate_address(): + return ''.join([ + self._obfuscated_network, + ':'.join(generate_hextets(_host.split(':'))) + ]) + + if addr.compressed not in self.hosts: + # separate host from the address by removing its network prefix + _n = self.network_addr.rstrip(':') + _host = addr.compressed[len(_n):].lstrip(':') + _ob_host = _generate_address() + while _ob_host in self.hosts.values(): + _ob_host = _generate_address() + self.add_obfuscated_host_address(addr.compressed, _ob_host) + return self.hosts[addr.compressed] + + def add_obfuscated_host_address(self, host, obfuscated): + """Adds an obfuscated pair to the class for tracking and ongoing + consistency in obfuscation. + """ + self.hosts[host] = obfuscated diff -pruN 4.0-2/sos/cleaner/mappings/mac_map.py 4.5.3ubuntu2/sos/cleaner/mappings/mac_map.py --- 4.0-2/sos/cleaner/mappings/mac_map.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/cleaner/mappings/mac_map.py 2023-04-28 17:16:21.000000000 +0000 @@ -48,6 +48,7 @@ class SoSMacMap(SoSMap): mac_template = '53:4f:53:%s:%s:%s' mac6_template = '53:4f:53:ff:fe:%s:%s:%s' mac6_quad_template = '534f:53ff:fe%s:%s%s' + compile_regexes = False def add(self, item): item = item.replace('-', ':').lower().strip('=.,').strip() diff -pruN 4.0-2/sos/cleaner/mappings/username_map.py 4.5.3ubuntu2/sos/cleaner/mappings/username_map.py --- 4.0-2/sos/cleaner/mappings/username_map.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/cleaner/mappings/username_map.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,37 @@ +# Copyright 2020 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.cleaner.mappings import SoSMap + + +class SoSUsernameMap(SoSMap): + """Mapping to store usernames matched from ``lastlog`` output. + + Usernames are obfuscated as ``obfuscateduserX`` where ``X`` is a counter + that gets incremented for every new username found. + + Note that this specifically obfuscates user_names_ and not UIDs. + """ + + name_count = 0 + + def load_names_from_options(self, opt_names): + for name in opt_names: + if name and name not in self.dataset.keys(): + self.add(name) + + def sanitize_item(self, username): + """Obfuscate a new username not currently found in the map + """ + ob_name = "obfuscateduser%s" % self.name_count + self.name_count += 1 + if ob_name in self.dataset.values(): + return self.sanitize_item(username.lower()) + return ob_name diff -pruN 4.0-2/sos/cleaner/obfuscation_archive.py 4.5.3ubuntu2/sos/cleaner/obfuscation_archive.py --- 4.0-2/sos/cleaner/obfuscation_archive.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/cleaner/obfuscation_archive.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,239 +0,0 @@ -# Copyright 2020 Red Hat, Inc. Jake Hunsaker - -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information. - -import logging -import os -import shutil -import stat -import tarfile -import re - -from sos.utilities import sos_get_command_output - - -class SoSObfuscationArchive(): - """A representation of an extracted archive or an sos archive build - directory which is used by SoSCleaner. - - Each archive that needs to be obfuscated is loaded into an instance of this - class. All report-level operations should be contained within this class. - """ - - file_sub_list = [] - total_sub_count = 0 - - def __init__(self, archive_path, tmpdir): - self.archive_path = archive_path - self.final_archive_path = self.archive_path - self.tmpdir = tmpdir - self.archive_name = self.archive_path.split('/')[-1].split('.tar')[0] - self.ui_name = self.archive_name - self.soslog = logging.getLogger('sos') - self.ui_log = logging.getLogger('sos_ui') - self.skip_list = self._load_skip_list() - self.log_info("Loaded %s as an archive" % self.archive_path) - - def report_msg(self, msg): - """Helper to easily format ui messages on a per-report basis""" - self.ui_log.info("{:<50} {}".format(self.ui_name + ' :', msg)) - - def _fmt_log_msg(self, msg): - return "[cleaner:%s] %s" % (self.archive_name, msg) - - def log_debug(self, msg): - self.soslog.debug(self._fmt_log_msg(msg)) - - def log_info(self, msg): - self.soslog.info(self._fmt_log_msg(msg)) - - def _load_skip_list(self): - """Provide a list of files and file regexes to skip obfuscation on - - Returns: list of files and file regexes - """ - return [ - 'installed-debs', - 'installed-rpms', - 'sos_commands/dpkg', - 'sos_commands/python/pip_list', - 'sos_commands/rpm', - 'sos_commands/yum/.*list.*', - 'sos_commands/snappy/snap_list_--all', - 'sos_commands/snappy/snap_--version', - 'sos_commands/vulkan/vulkaninfo', - 'sys/firmware', - 'sys/fs', - 'sys/kernel/debug', - 'sys/module', - 'var/log/.*dnf.*', - '.*.tar.*', # TODO: support archive unpacking - '.*.gz' - ] - - @property - def is_tarfile(self): - try: - return tarfile.is_tarfile(self.archive_path) - except Exception: - return False - - def extract(self): - if self.is_tarfile: - self.report_msg("Extracting...") - self.extracted_path = self.extract_self() - else: - self.extracted_path = self.archive_path - # if we're running as non-root (e.g. collector), then we can have a - # situation where a particular path has insufficient permissions for - # us to rewrite the contents and/or add it to the ending tarfile. - # Unfortunately our only choice here is to change the permissions - # that were preserved during report collection - if os.getuid() != 0: - self.log_debug('Verifying permissions of archive contents') - for dirname, dirs, files in os.walk(self.extracted_path): - try: - for _dir in dirs: - _dirname = os.path.join(dirname, _dir) - _dir_perms = os.stat(_dirname).st_mode - os.chmod(_dirname, _dir_perms | stat.S_IRWXU) - for filename in files: - fname = os.path.join(dirname, filename) - # protect against symlink race conditions - if not os.path.exists(fname) or os.path.islink(fname): - continue - if (not os.access(fname, os.R_OK) or not - os.access(fname, os.W_OK)): - self.log_debug( - "Adding owner rw permissions to %s" - % fname.split(self.archive_path)[-1] - ) - os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR) - except Exception as err: - self.log_debug("Error while trying to set perms: %s" % err) - self.log_debug("Extracted path is %s" % self.extracted_path) - - def rename_top_dir(self, new_name): - """Rename the top-level directory to new_name, which should be an - obfuscated string that scrubs the hostname from the top-level dir - which would be named after the unobfuscated sos report - """ - _path = self.extracted_path.replace(self.archive_name, new_name) - self.archive_name = new_name - os.rename(self.extracted_path, _path) - self.extracted_path = _path - - def get_compression(self): - """Return the compression type used by the archive, if any. This is - then used by SoSCleaner to generate a policy-derived compression - command to repack the archive - """ - if self.is_tarfile: - if self.archive_path.endswith('xz'): - return 'xz' - return 'gzip' - return None - - def build_tar_file(self): - """Pack the extracted archive as a tarfile to then be re-compressed - """ - self.tarpath = self.extracted_path + '-obfuscated.tar' - self.log_debug("Building tar file %s" % self.tarpath) - tar = tarfile.open(self.tarpath, mode="w") - tar.add(self.extracted_path, - arcname=os.path.split(self.archive_name)[1]) - tar.close() - - def compress(self, cmd): - """Execute the compression command, and set the appropriate final - archive path for later reference by SoSCleaner on a per-archive basis - """ - self.build_tar_file() - exec_cmd = "%s %s" % (cmd, self.tarpath) - res = sos_get_command_output(exec_cmd, timeout=0, stderr=True) - if res['status'] == 0: - self.final_archive_path = self.tarpath + '.' + exec_cmd[0:2] - self.log_debug("Compressed to %s" % self.final_archive_path) - try: - self.remove_extracted_path() - except Exception as err: - self.log_debug("Failed to remove extraction directory: %s" - % err) - self.report_msg('Failed to remove temporary extraction ' - 'directory') - else: - err = res['output'].split(':')[-1] - self.log_debug("Exception while compressing archive: %s" % err) - raise Exception(err) - - def remove_extracted_path(self): - """After the tarball has been re-compressed, remove the extracted path - so that we don't take up that duplicate space any longer during - execution - """ - def force_delete_file(action, name, exc): - os.chmod(name, stat.S_IWUSR) - if os.path.isfile(name): - os.remove(name) - else: - shutil.rmtree(name) - self.log_debug("Removing %s" % self.extracted_path) - shutil.rmtree(self.extracted_path, onerror=force_delete_file) - - def extract_self(self): - """Extract an archive into our tmpdir so that we may inspect it or - iterate through its contents for obfuscation - """ - archive = tarfile.open(self.archive_path) - path = os.path.join(self.tmpdir, 'cleaner') - archive.extractall(path) - archive.close() - return os.path.join(path, archive.name.split('/')[-1].split('.tar')[0]) - - def get_file_list(self): - """Return a list of all files within the archive""" - self.file_list = [] - for dirname, dirs, files in os.walk(self.extracted_path): - for filename in files: - self.file_list.append(os.path.join(dirname, filename)) - return self.file_list - - def update_sub_count(self, fname, count): - """Called when a file has finished being parsed and used to track - total substitutions made and number of files that had changes made - """ - self.file_sub_list.append(fname) - self.total_sub_count += count - - def get_file_path(self, fname): - """Return the filepath of a specific file within the archive so that - it may be selectively inspected if it exists - """ - _path = os.path.join(self.extracted_path, fname.lstrip('/')) - return _path if os.path.exists(_path) else '' - - def should_skip_file(self, filename): - """Checks the provided filename against a list of filepaths to not - perform obfuscation on, as defined in self.skip_list - - Positional arguments: - - :param filename str: Filename relative to the extracted - archive root - """ - if filename in self.file_sub_list: - return True - - if not os.path.isfile(self.get_file_path(filename)): - return True - - for _skip in self.skip_list: - if filename.startswith(_skip) or re.match(_skip, filename): - return True - return False diff -pruN 4.0-2/sos/cleaner/parsers/__init__.py 4.5.3ubuntu2/sos/cleaner/parsers/__init__.py --- 4.0-2/sos/cleaner/parsers/__init__.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/cleaner/parsers/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -8,7 +8,6 @@ # # See the LICENSE file in the source distribution for further information. -import json import re @@ -38,11 +37,6 @@ class SoSCleanerParser(): :cvar map_file_key: The key in the ``map_file`` to read when loading previous obfuscation matches :vartype map_file_key: ``str`` - - - :cvar prep_map_file: File to read from an archive to pre-seed the map with - matches. E.G. ip_addr for loading IP addresses - :vartype prep_map_fie: ``str`` """ name = 'Undefined Parser' @@ -50,25 +44,75 @@ class SoSCleanerParser(): skip_line_patterns = [] skip_files = [] map_file_key = 'unset' - prep_map_file = 'unset' + compile_regexes = True + + def __init__(self, config={}): + if self.map_file_key in config: + self.mapping.conf_update(config[self.map_file_key]) + self._generate_skip_regexes() + + def _generate_skip_regexes(self): + """Generate the regexes for the parser's configured `skip_files`, + so that we don't regenerate them on every file being examined for if + the parser should skip a given file. + """ + self.skip_patterns = [] + for p in self.skip_files: + self.skip_patterns.append(re.compile(p)) + + def generate_item_regexes(self): + """Generate regexes for items the parser will be searching for + repeatedly without needing to generate them for every file and/or line + we process - def __init__(self, conf_file=None): - # attempt to load previous run data into the mapping for the parser - if conf_file: - try: - with open(conf_file, 'r') as map_file: - _default_mappings = json.load(map_file) - if self.map_file_key in _default_mappings: - self.mapping.conf_update( - _default_mappings[self.map_file_key] - ) - except IOError: - pass + Not used by all parsers. + """ + if not self.compile_regexes: + return + for obitem in self.mapping.dataset: + self.mapping.add_regex_item(obitem) def parse_line(self, line): """This will be called for every line in every file we process, so that every parser has a chance to scrub everything. + This will first try to identify needed obfuscations for items we have + already encountered (if the parser uses compiled regexes that is) and + make those substitutions early on. After which, we will then parse the + line again looking for new matches. + """ + count = 0 + for skip_pattern in self.skip_line_patterns: + if re.match(skip_pattern, line, re.I): + return line, count + if self.compile_regexes: + line, _rcount = self._parse_line_with_compiled_regexes(line) + count += _rcount + line, _count = self._parse_line(line) + count += _count + return line, count + + def _parse_line_with_compiled_regexes(self, line): + """Check the provided line against known items we have encountered + before and have pre-generated regex Pattern() objects for. + + :param line: The line to parse for possible matches for obfuscation + :type line: ``str`` + + :returns: The obfuscated line and the number of changes made + :rtype: ``str``, ``int`` + """ + count = 0 + for item, reg in self.mapping.compiled_regexes: + if reg.search(line): + line, _count = reg.subn(self.mapping.get(item.lower()), line) + count += _count + return line, count + + def _parse_line(self, line): + """Check the provided line against the parser regex patterns to try + and discover _new_ items to obfuscate + :param line: The line to parse for possible matches for obfuscation :type line: ``str`` @@ -76,16 +120,18 @@ class SoSCleanerParser(): :rtype: ``tuple``, ``(str, int))`` """ count = 0 - for skip_pattern in self.skip_line_patterns: - if re.match(skip_pattern, line, re.I): - return line, count for pattern in self.regex_patterns: matches = [m[0] for m in re.findall(pattern, line, re.I)] if matches: + matches.sort(reverse=True, key=len) count += len(matches) for match in matches: - new_match = self.mapping.get(match.strip()) - line = line.replace(match.strip(), new_match) + match = match.strip() + if match in self.mapping.dataset.values(): + continue + new_match = self.mapping.get(match) + if new_match != match: + line = line.replace(match, new_match) return line, count def parse_string_for_keys(self, string_data): @@ -102,9 +148,17 @@ class SoSCleanerParser(): :returns: The obfuscated line :rtype: ``str`` """ - for key, val in self.mapping.dataset.items(): - if key in string_data: - return string_data.replace(key, val) + if self.compile_regexes: + for item, reg in self.mapping.compiled_regexes: + if reg.search(string_data): + string_data = reg.sub(self.mapping.get(item), string_data) + else: + for k, ob in sorted(self.mapping.dataset.items(), reverse=True, + key=lambda x: len(x[0])): + if k in self.mapping.skip_keys: + continue + if k in string_data: + string_data = string_data.replace(k, ob) return string_data def get_map_contents(self): diff -pruN 4.0-2/sos/cleaner/parsers/hostname_parser.py 4.5.3ubuntu2/sos/cleaner/parsers/hostname_parser.py --- 4.0-2/sos/cleaner/parsers/hostname_parser.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/cleaner/parsers/hostname_parser.py 2023-04-28 17:16:21.000000000 +0000 @@ -8,6 +8,7 @@ # # See the LICENSE file in the source distribution for further information. +import re from sos.cleaner.parsers import SoSCleanerParser from sos.cleaner.mappings.hostname_map import SoSHostnameMap @@ -16,15 +17,48 @@ class SoSHostnameParser(SoSCleanerParser name = 'Hostname Parser' map_file_key = 'hostname_map' - prep_map_file = 'sos_commands/host/hostname' regex_patterns = [ - r'(((\b|_)[a-zA-Z0-9-\.]{1,200}\.[a-zA-Z]{1,63}\b))' + r'(((\b|_)[a-zA-Z0-9-\.]{1,200}\.[a-zA-Z]{1,63}(\b|_)))' ] - def __init__(self, conf_file=None, opt_domains=None): - self.mapping = SoSHostnameMap(opt_domains) + def __init__(self, config, opt_domains=None): + self.mapping = SoSHostnameMap() + super(SoSHostnameParser, self).__init__(config) + self.mapping.load_domains_from_map() + self.mapping.load_domains_from_options(opt_domains) self.short_names = [] - super(SoSHostnameParser, self).__init__(conf_file) + self.load_short_names_from_mapping() + self.mapping.set_initial_counts() + + def parse_line(self, line): + """This will be called for every line in every file we process, so that + every parser has a chance to scrub everything. + + We are overriding parent method since we need to swap ordering of + _parse_line_with_compiled_regexes and _parse_line calls. + """ + count = 0 + for skip_pattern in self.skip_line_patterns: + if re.match(skip_pattern, line, re.I): + return line, count + line, _count = self._parse_line(line) + count += _count + if self.compile_regexes: + line, _rcount = self._parse_line_with_compiled_regexes(line) + count += _rcount + return line, count + + def load_short_names_from_mapping(self): + """When we load the mapping file into the hostname map, we have to do + some dancing to get those loaded properly into the "intermediate" dicts + that the map uses to hold hosts and domains. Similarly, we need to also + extract shortnames known to the map here. + """ + for hname in self.mapping.dataset.keys(): + if len(hname.split('.')) == 1: + # we have a short name only with no domain + if hname not in self.short_names: + self.short_names.append(hname) def load_hostname_into_map(self, hostname_string): """Force add the domainname found in /sos_commands/host/hostname into @@ -46,14 +80,22 @@ class SoSHostnameParser(SoSCleanerParser self.mapping.add(high_domain) self.mapping.add(hostname_string) - def parse_line(self, line): - """Override the default parse_line() method to also check for the - shortname of the host derived from the hostname. + def load_hostname_from_etc_hosts(self, content): + """Parse an archive's copy of /etc/hosts, which requires handling that + is separate from the output of the `hostname` command. Just like + load_hostname_into_map(), this has to be done explicitly and we + cannot rely upon the more generic methods to do this reliably. """ - count = 0 - line, count = super(SoSHostnameParser, self).parse_line(line) - for short_name in self.short_names: - if short_name in line: - count += 1 - line = line.replace(short_name, self.mapping.get(short_name)) - return line, count + lines = content.splitlines() + for line in lines: + if line.startswith('#') or 'localhost' in line: + continue + hostln = line.split()[1:] + for host in hostln: + if len(host.split('.')) == 1: + # only generate a mapping for fqdns but still record the + # short name here for later obfuscation with parse_line() + self.short_names.append(host) + self.mapping.add_regex_item(host) + else: + self.mapping.add(host) diff -pruN 4.0-2/sos/cleaner/parsers/ip_parser.py 4.5.3ubuntu2/sos/cleaner/parsers/ip_parser.py --- 4.0-2/sos/cleaner/parsers/ip_parser.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/cleaner/parsers/ip_parser.py 2023-04-28 17:16:21.000000000 +0000 @@ -24,9 +24,26 @@ class SoSIPParser(SoSCleanerParser): # don't match package versions recorded in journals r'.*dnf\[.*\]:' ] + + skip_files = [ + # skip these as version numbers will frequently look like IP addresses + # when using regex matching + 'installed-debs', + 'installed-rpms', + 'sos_commands/dpkg', + 'sos_commands/python/pip_list', + 'sos_commands/rpm', + 'sos_commands/yum/.*list.*', + 'sos_commands/snappy/snap_list_--all', + 'sos_commands/vulkan/vulkaninfo', + 'var/log/.*dnf.*', + 'var/log/.*packag.*', # get 'packages' and 'packaging' logs + '.*(version|release)(\\.txt)?$', # obvious version files + ] + map_file_key = 'ip_map' - prep_map_file = 'sos_commands/networking/ip_-o_addr' + compile_regexes = False - def __init__(self, conf_file=None): + def __init__(self, config): self.mapping = SoSIPMap() - super(SoSIPParser, self).__init__(conf_file) + super(SoSIPParser, self).__init__(config) diff -pruN 4.0-2/sos/cleaner/parsers/ipv6_parser.py 4.5.3ubuntu2/sos/cleaner/parsers/ipv6_parser.py --- 4.0-2/sos/cleaner/parsers/ipv6_parser.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/cleaner/parsers/ipv6_parser.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,60 @@ +# Copyright 2022 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.cleaner.parsers import SoSCleanerParser +from sos.cleaner.mappings.ipv6_map import SoSIPv6Map + + +class SoSIPv6Parser(SoSCleanerParser): + """Parser for handling IPv6 networks and addresses""" + + name = 'IPv6 Parser' + map_file_key = 'ipv6_map' + regex_patterns = [ + # Attention: note that this is a single long regex, not several entries + # This is initially based off of two regexes from the Java library + # for validating an IPv6 string. However, this is modified to begin and + # end with a negative lookbehind to ensure that a substring of 'ed::' + # is not extracted from a log message such as 'SomeFuncUsed::ADiffFunc' + # that come components may log with. Further, we optionally try to grab + # a trailing prefix for the network bits. + r"(? + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.cleaner.parsers import SoSCleanerParser +from sos.cleaner.mappings.username_map import SoSUsernameMap + + +class SoSUsernameParser(SoSCleanerParser): + """Parser for obfuscating usernames within an sosreport archive. + + Note that this parser does not rely on regex matching directly, like most + other parsers do. Instead, usernames are discovered via scraping the + collected output of lastlog. As such, we do not discover new usernames + later on, and only usernames present in lastlog output will be obfuscated, + and those passed via the --usernames option if one is provided. + """ + + name = 'Username Parser' + map_file_key = 'username_map' + regex_patterns = [] + skip_list = [ + 'core', + 'nobody', + 'nfsnobody', + 'shutdown', + 'stack', + 'reboot', + 'root', + 'ubuntu', + 'username', + 'wtmp' + ] + + def __init__(self, config, opt_names=None): + self.mapping = SoSUsernameMap() + super(SoSUsernameParser, self).__init__(config) + self.mapping.load_names_from_options(opt_names) + + def load_usernames_into_map(self, content): + """Since we don't get the list of usernames from a straight regex for + this parser, we need to override the initial parser prepping here. + """ + users = set() + for line in content.splitlines(): + try: + user = line.split()[0] + except Exception: + continue + if not user or user.lower() in self.skip_list: + continue + users.add(user.lower()) + for each in sorted(users, key=len, reverse=True): + self.mapping.get(each) + if '\\' in each: + self.mapping.get(each.split('\\')[-1]) + + def _parse_line(self, line): + return line, 0 diff -pruN 4.0-2/sos/collector/__init__.py 4.5.3ubuntu2/sos/collector/__init__.py --- 4.0-2/sos/collector/__init__.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -17,7 +17,6 @@ import re import string import socket import shutil -import subprocess import sys from datetime import datetime @@ -28,21 +27,43 @@ from pipes import quote from textwrap import fill from sos.cleaner import SoSCleaner from sos.collector.sosnode import SosNode -from sos.collector.exceptions import ControlPersistUnsupportedException -from sos.options import ClusterOption +from sos.options import ClusterOption, str_to_bool from sos.component import SoSComponent +from sos.utilities import bold from sos import __version__ COLLECTOR_CONFIG_DIR = '/etc/sos/groups.d' class SoSCollector(SoSComponent): - """Collector is the formerly standalone sos-collector project, brought into - sos natively in 4.0 + """ + sos collect, or SoS Collector, is the formerly standalone sos-collector + project, brought into sos natively in 4.0 and later. + + It is meant to collect sos reports from an arbitrary number of remote + nodes, as well as the localhost, at the same time. These nodes may be + either user defined, defined by some clustering software, or both. + + For cluster defined lists of nodes, cluster profiles exist that not only + define how these node lists are generated but may also influence the + sos report command run on nodes depending upon their role within the + cluster. + + Nodes are connected to via a 'transport' which defaults to the use of + OpenSSH's Control Persist feature. Other transport types are available, and + may be specifically linked to use with a certain cluster profile (or, at + minimum, a node within a certain cluster type even if that profile is not + used). + + sos collect may be run from either a node within the cluster that is + capable of enumerating/discovering the other cluster nodes, or may be run + from a user's workstation and instructed to first connect to such a node + via the --primary option. If run in the latter manner, users will likely + want to use the --no-local option, as by default sos collect will also + collect an sos report locally. - It is meant to collect reports from an arbitrary number of remote nodes, - as well as the localhost, at the same time. These nodes may be either user - defined, defined by some clustering software, or both. + Users should expect this command to result in a tarball containing one or + more sos report archives on the system that sos collect was executed on. """ desc = 'Collect an sos report from multiple nodes simultaneously' @@ -57,19 +78,27 @@ class SoSCollector(SoSComponent): 'clean': False, 'cluster_options': [], 'cluster_type': None, + 'container_runtime': 'auto', 'domains': [], + 'disable_parsers': [], 'enable_plugins': [], 'encrypt_key': '', 'encrypt_pass': '', 'group': None, 'image': '', + 'force_pull_image': True, 'jobs': 4, + 'journal_size': 0, 'keywords': [], + 'keyword_file': None, + 'keep_binary_files': False, 'label': '', 'list_options': False, 'log_size': 0, + 'low_priority': False, 'map_file': '/etc/sos/cleaner/default_mapping', - 'master': '', + 'primary': '', + 'namespaces': None, 'nodes': [], 'no_env_vars': False, 'no_local': False, @@ -79,18 +108,33 @@ class SoSCollector(SoSComponent): 'only_plugins': [], 'password': False, 'password_per_node': False, - 'plugin_options': [], + 'plugopts': [], 'plugin_timeout': None, + 'cmd_timeout': None, 'preset': '', + 'registry_user': None, + 'registry_password': None, + 'registry_authfile': None, 'save_group': '', 'since': '', + 'skip_commands': [], + 'skip_files': [], 'skip_plugins': [], - 'sos_opt_line': '', 'ssh_key': '', 'ssh_port': 22, 'ssh_user': 'root', 'timeout': 600, - 'verify': False + 'transport': 'auto', + 'verify': False, + 'usernames': [], + 'upload': False, + 'upload_url': None, + 'upload_directory': None, + 'upload_user': None, + 'upload_pass': None, + 'upload_method': 'auto', + 'upload_no_ssl_verify': False, + 'upload_protocol': 'auto' } def __init__(self, parser, parsed_args, cmdline_args): @@ -98,7 +142,7 @@ class SoSCollector(SoSComponent): os.umask(0o77) self.client_list = [] self.node_list = [] - self.master = False + self.primary = None self.retrieved = 0 self.cluster = None self.cluster_type = None @@ -135,7 +179,6 @@ class SoSCollector(SoSComponent): try: self.parse_node_strings() self.parse_cluster_options() - self._check_for_control_persist() self.log_debug('Executing %s' % ' '.join(s for s in sys.argv)) self.log_debug("Found cluster profiles: %s" % self.clusters.keys()) @@ -158,15 +201,17 @@ class SoSCollector(SoSComponent): supported_clusters[cluster[0]] = cluster[1](self.commons) return supported_clusters - def _load_modules(self, package, submod): + @classmethod + def _load_modules(cls, package, submod): """Helper to import cluster and host types""" modules = [] for path in package.__path__: if os.path.isdir(path): - modules.extend(self._find_modules_in_path(path, submod)) + modules.extend(cls._find_modules_in_path(path, submod)) return modules - def _find_modules_in_path(self, path, modulename): + @classmethod + def _find_modules_in_path(cls, path, modulename): """Given a path and a module name, find everything that can be imported and then import it @@ -185,9 +230,10 @@ class SoSCollector(SoSComponent): continue fname, ext = os.path.splitext(pyfile) modname = 'sos.collector.%s.%s' % (modulename, fname) - modules.extend(self._import_modules(modname)) + modules.extend(cls._import_modules(modname)) return modules + @classmethod def _import_modules(self, modname): """Import and return all found classes in a module""" mod_short_name = modname.split('.')[2] @@ -251,26 +297,46 @@ class SoSCollector(SoSComponent): sos_grp.add_argument('--chroot', default='', choices=['auto', 'always', 'never'], help="chroot executed commands to SYSROOT") + sos_grp.add_argument("--container-runtime", default="auto", + help="Default container runtime to use for " + "collections. 'auto' for policy control.") sos_grp.add_argument('-e', '--enable-plugins', action="extend", help='Enable specific plugins for sosreport') - sos_grp.add_argument('-k', '--plugin-options', action="extend", + sos_grp.add_argument('--journal-size', type=int, default=0, + help='Limit the size of journals in MiB') + sos_grp.add_argument('-k', '--plugin-option', '--plugopts', + action="extend", dest='plugopts', help='Plugin option as plugname.option=value') sos_grp.add_argument('--log-size', default=0, type=int, - help='Limit the size of individual logs (in MiB)') + help='Limit the size of individual logs ' + '(not journals) in MiB') + sos_grp.add_argument('--low-priority', action='store_true', + default=False, help='Run reports as low priority') sos_grp.add_argument('-n', '--skip-plugins', action="extend", help='Skip these plugins') sos_grp.add_argument('-o', '--only-plugins', action="extend", default=[], help='Run these plugins only') + sos_grp.add_argument('--namespaces', default=None, + help='limit number of namespaces to collect ' + 'output for - 0 means unlimited') sos_grp.add_argument('--no-env-vars', action='store_true', default=False, help='Do not collect env vars in sosreports') sos_grp.add_argument('--plugin-timeout', type=int, default=None, help='Set the global plugin timeout value') + sos_grp.add_argument('--cmd-timeout', type=int, default=None, + help='Set the global command timeout value') sos_grp.add_argument('--since', default=None, help=('Escapes archived files older than date. ' 'This will also affect --all-logs. ' 'Format: YYYYMMDD[HHMMSS]')) + sos_grp.add_argument('--skip-commands', default=[], action='extend', + dest='skip_commands', + help="do not execute these commands") + sos_grp.add_argument('--skip-files', default=[], action='extend', + dest='skip_files', + help="do not collect these files") sos_grp.add_argument('--verify', action="store_true", help='perform pkg verification during collection') @@ -298,6 +364,20 @@ class SoSCollector(SoSComponent): collect_grp.add_argument('--image', help=('Specify the container image to use for' ' containerized hosts.')) + collect_grp.add_argument('--force-pull-image', '--pull', + default=True, choices=(True, False), + type=str_to_bool, + help='Force pull the container image even if ' + 'it already exists on the host') + collect_grp.add_argument('--registry-user', default=None, + help='Username to authenticate to the ' + 'registry with for pulling an image') + collect_grp.add_argument('--registry-password', default=None, + help='Password to authenticate to the ' + 'registry with for pulling an image') + collect_grp.add_argument('--registry-authfile', default=None, + help='Use this authfile to provide registry ' + 'authentication when pulling an image') collect_grp.add_argument('-i', '--ssh-key', help='Specify an ssh key') collect_grp.add_argument('-j', '--jobs', default=4, type=int, help='Number of concurrent nodes to collect') @@ -305,7 +385,10 @@ class SoSCollector(SoSComponent): help='List options available for profiles') collect_grp.add_argument('--label', help='Assign a label to the archives') - collect_grp.add_argument('--master', help='Specify a master node') + collect_grp.add_argument('--primary', '--manager', '--controller', + dest='primary', default='', + help='Specify a primary node for cluster ' + 'enumeration') collect_grp.add_argument('--nopasswd-sudo', action='store_true', help='Use passwordless sudo on nodes') collect_grp.add_argument('--nodes', action="append", @@ -327,82 +410,131 @@ class SoSCollector(SoSComponent): help='Prompt for password for each node') collect_grp.add_argument('--preset', default='', required=False, help='Specify a sos preset to use') - collect_grp.add_argument('--sos-cmd', dest='sos_opt_line', - help=('Manually specify the commandline ' - 'for sos report on nodes')) collect_grp.add_argument('--ssh-user', help='Specify an SSH user. Default root') collect_grp.add_argument('--timeout', type=int, required=False, help='Timeout for sosreport on each node.') + collect_grp.add_argument('--transport', default='auto', type=str, + help='Remote connection transport to use') + collect_grp.add_argument("--upload", action="store_true", + default=False, + help="Upload archive to a policy-default " + "location") + collect_grp.add_argument("--upload-url", default=None, + help="Upload the archive to specified server") + collect_grp.add_argument("--upload-directory", default=None, + help="Specify upload directory for archive") + collect_grp.add_argument("--upload-user", default=None, + help="Username to authenticate with") + collect_grp.add_argument("--upload-pass", default=None, + help="Password to authenticate with") + collect_grp.add_argument("--upload-method", default='auto', + choices=['auto', 'put', 'post'], + help="HTTP method to use for uploading") + collect_grp.add_argument("--upload-no-ssl-verify", default=False, + action='store_true', + help="Disable SSL verification for upload url" + ) + collect_grp.add_argument("--upload-protocol", default='auto', + choices=['auto', 'https', 'ftp', 'sftp'], + help="Manually specify the upload protocol") # Group the cleaner options together cleaner_grp = parser.add_argument_group( 'Cleaner/Masking Options', 'These options control how data obfuscation is performed' ) - cleaner_grp.add_argument('--clean', '--mask', dest='clean', + cleaner_grp.add_argument('--clean', '--cleaner', '--mask', + dest='clean', default=False, action='store_true', - help='Obfuscate sensistive information') + help='Obfuscate sensitive information') + cleaner_grp.add_argument('--keep-binary-files', default=False, + action='store_true', dest='keep_binary_files', + help='Keep unprocessable binary files in the ' + 'archive instead of removing them') cleaner_grp.add_argument('--domains', dest='domains', default=[], action='extend', help='Additional domain names to obfuscate') + cleaner_grp.add_argument('--disable-parsers', action='extend', + default=[], dest='disable_parsers', + help=('Disable specific parsers, so that ' + 'those elements are not obfuscated')) cleaner_grp.add_argument('--keywords', action='extend', default=[], dest='keywords', help='List of keywords to obfuscate') + cleaner_grp.add_argument('--keyword-file', default=None, + dest='keyword_file', + help='Provide a file a keywords to obfuscate') cleaner_grp.add_argument('--no-update', action='store_true', default=False, dest='no_update', help='Do not update the default cleaner map') - cleaner_grp.add_argument('--map', dest='map_file', + cleaner_grp.add_argument('--map-file', dest='map_file', default='/etc/sos/cleaner/default_mapping', help=('Provide a previously generated mapping' ' file for obfuscation')) + cleaner_grp.add_argument('--usernames', dest='usernames', default=[], + action='extend', + help='List of usernames to obfuscate') + + @classmethod + def display_help(cls, section): + section.set_title('SoS Collect Detailed Help') + section.add_text(cls.__doc__) + + hsections = { + 'collect.clusters': 'Information on cluster profiles', + 'collect.clusters.$cluster': 'Specific profile information', + 'collect.transports': 'Information on how connections are made', + 'collect.transports.$transport': 'Specific transport information' + } + section.add_text( + 'The following help sections may be of further interest:\n' + ) + for hsec in hsections: + section.add_text( + "{:>8}{:<40}{:<30}".format(' ', bold(hsec), hsections[hsec]), + newline=False + ) - def _check_for_control_persist(self): - """Checks to see if the local system supported SSH ControlPersist. + def exit(self, msg=None, error=0, force=False): + """Used to terminate and ensure all cleanup is done, setting the exit + code as specified if required. - ControlPersist allows OpenSSH to keep a single open connection to a - remote host rather than building a new session each time. This is the - same feature that Ansible uses in place of paramiko, which we have a - need to drop in sos-collector. - - This check relies on feedback from the ssh binary. The command being - run should always generate stderr output, but depending on what that - output reads we can determine if ControlPersist is supported or not. + :param msg: Log the provided message as an error + :type msg: ``str`` - For our purposes, a host that does not support ControlPersist is not - able to run sos-collector. + :param error: The exit code to use when terminating + :type error: ``int`` - Returns - True if ControlPersist is supported, else raise Exception. + :param force: Use os.exit() to break out of nested threads if needed + :type force: ``bool`` """ - ssh_cmd = ['ssh', '-o', 'ControlPersist'] - cmd = subprocess.Popen(ssh_cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out, err = cmd.communicate() - err = err.decode('utf-8') - if 'Bad configuration option' in err or 'Usage:' in err: - raise ControlPersistUnsupportedException - return True - - def exit(self, msg, error=1): - """Used to safely terminate if sos-collector encounters an error""" - self.log_error(msg) + if self.cluster: + self.cluster.cleanup() + if msg: + self.log_error(msg) try: self.close_all_connections() except Exception: pass - self.cleanup() - sys.exit(error) + if error != 130: + # keep the tempdir around when a user issues a keyboard interrupt + # like we do for report + self.cleanup() + if not force: + sys.exit(error) + else: + os._exit(error) def _parse_options(self): """From commandline options, defaults, etc... build a set of commons to hand to other collector mechanisms """ self.commons = { + 'cmdlineopts': self.opts, 'need_sudo': True if self.opts.ssh_user != 'root' else False, - 'opts': self.opts, 'tmpdir': self.tmpdir, - 'hostlen': len(self.opts.master) or len(self.hostname), + 'hostlen': max(len(self.opts.primary), len(self.hostname)), 'policy': self.policy } @@ -440,7 +572,7 @@ class SoSCollector(SoSComponent): break if not match: self.exit('Unknown cluster option provided: %s.%s' - % (opt.cluster, opt.name)) + % (opt.cluster, opt.name), 1) def _validate_option(self, default, cli): """Checks to make sure that the option given on the CLI is valid. @@ -453,14 +585,14 @@ class SoSCollector(SoSComponent): if not default.opt_type == bool: if not default.opt_type == cli.opt_type: msg = "Invalid option type for %s. Expected %s got %s" - self.exit(msg % (cli.name, default.opt_type, cli.opt_type)) + self.exit(msg % (cli.name, default.opt_type, cli.opt_type), 1) return cli.value else: val = cli.value.lower() if val not in ['true', 'on', 'yes', 'false', 'off', 'no']: msg = ("Invalid value for %s. Accepted values are: 'true', " "'false', 'on', 'off', 'yes', 'no'.") - self.exit(msg % cli.name) + self.exit(msg % cli.name, 1) else: if val in ['true', 'on', 'yes']: return True @@ -473,7 +605,7 @@ class SoSCollector(SoSComponent): def log_warn(self, msg): """Log warn messages to both console and log file""" - self.soslog.warn(msg) + self.soslog.warning(msg) def log_error(self, msg): """Log error messages to both console and log file""" @@ -573,7 +705,7 @@ class SoSCollector(SoSComponent): on the commandline to point to one existing anywhere on the system Host groups define a list of nodes and/or regexes and optionally the - master and cluster-type options. + primary and cluster-type options. """ grp = self.opts.group paths = [ @@ -594,7 +726,7 @@ class SoSCollector(SoSComponent): with open(fname, 'r') as hf: _group = json.load(hf) - for key in ['master', 'cluster_type']: + for key in ['primary', 'cluster_type']: if _group[key]: self.log_debug("Setting option '%s' to '%s' per host group" % (key, _group[key])) @@ -608,12 +740,12 @@ class SoSCollector(SoSComponent): Saves the results of this run of sos-collector to a host group file on the system so it can be used later on. - The host group will save the options master, cluster_type, and nodes + The host group will save the options primary, cluster_type, and nodes as determined by sos-collector prior to execution of sosreports. """ cfg = { 'name': self.opts.save_group, - 'master': self.opts.master, + 'primary': self.opts.primary, 'cluster_type': self.cluster.cluster_type[0], 'nodes': [n for n in self.node_list] } @@ -626,10 +758,11 @@ class SoSCollector(SoSComponent): fname = os.path.join(group_path, cfg['name']) with open(fname, 'w') as hf: json.dump(cfg, hf) - os.chmod(fname, 0o644) + os.chmod(fname, 0o600) return fname def prep(self): + self.policy.set_commons(self.commons) if (not self.opts.password and not self.opts.password_per_node): self.log_debug('password not specified, assuming SSH keys') @@ -637,26 +770,29 @@ class SoSCollector(SoSComponent): 'nodes unless the --password option is provided.\n') self.ui_log.info(self._fmt_msg(msg)) - if ((self.opts.password or (self.opts.password_per_node and - self.opts.master)) - and not self.opts.batch): - self.log_debug('password specified, not using SSH keys') - msg = ('Provide the SSH password for user %s: ' - % self.opts.ssh_user) - self.opts.password = getpass(prompt=msg) - - if ((self.commons['need_sudo'] and not self.opts.nopasswd_sudo) - and not self.opts.batch): - if not self.opts.password and not self.opts.password_per_node: - self.log_debug('non-root user specified, will request ' - 'sudo password') - msg = ('A non-root user has been provided. Provide sudo ' - 'password for %s on remote nodes: ' + try: + if ((self.opts.password or (self.opts.password_per_node and + self.opts.primary)) + and not self.opts.batch): + self.log_debug('password specified, not using SSH keys') + msg = ('Provide the SSH password for user %s: ' % self.opts.ssh_user) - self.opts.sudo_pw = getpass(prompt=msg) - else: - if not self.opts.nopasswd_sudo: - self.opts.sudo_pw = self.opts.password + self.opts.password = getpass(prompt=msg) + + if ((self.commons['need_sudo'] and not self.opts.nopasswd_sudo) + and not self.opts.batch): + if not self.opts.password and not self.opts.password_per_node: + self.log_debug('non-root user specified, will request ' + 'sudo password') + msg = ('A non-root user has been provided. Provide sudo ' + 'password for %s on remote nodes: ' + % self.opts.ssh_user) + self.opts.sudo_pw = getpass(prompt=msg) + else: + if not self.opts.nopasswd_sudo: + self.opts.sudo_pw = self.opts.password + except KeyboardInterrupt: + self.exit("\nExiting on user cancel\n", 130) if self.opts.become_root: if not self.opts.ssh_user == 'root': @@ -679,12 +815,17 @@ class SoSCollector(SoSComponent): try: self._load_group_config() except Exception as err: - self.log_error("Could not load specified group %s: %s" - % (self.opts.group, err)) - self._exit(1) + msg = ("Could not load specified group %s: %s" + % (self.opts.group, err)) + self.exit(msg, 1) + + try: + self.policy.pre_work() + except KeyboardInterrupt: + self.exit("Exiting on user cancel\n", 130) - if self.opts.master: - self.connect_to_master() + if self.opts.primary: + self.connect_to_primary() self.opts.no_local = True else: try: @@ -711,9 +852,9 @@ class SoSCollector(SoSComponent): self.ui_log.info(skip_local_msg) can_run_local = False self.opts.no_local = True - self.master = SosNode('localhost', self.commons, - local_sudo=local_sudo, - load_facts=can_run_local) + self.primary = SosNode('localhost', self.commons, + local_sudo=local_sudo, + load_facts=can_run_local) except Exception as err: self.log_debug("Unable to determine local installation: %s" % err) @@ -721,18 +862,19 @@ class SoSCollector(SoSComponent): '--no-local option if localhost should not be ' 'included.\nAborting...\n', 1) - self.collect_md.add_field('master', self.master.address) + self.collect_md.add_field('primary', self.primary.address) self.collect_md.add_section('nodes') - self.collect_md.nodes.add_section(self.master.address) - self.master.set_node_manifest(getattr(self.collect_md.nodes, - self.master.address)) + self.collect_md.nodes.add_section(self.primary.address) + self.primary.set_node_manifest(getattr(self.collect_md.nodes, + self.primary.address)) if self.opts.cluster_type: if self.opts.cluster_type == 'none': self.cluster = self.clusters['jbon'] else: self.cluster = self.clusters[self.opts.cluster_type] - self.cluster.master = self.master + self.cluster_type = self.opts.cluster_type + self.cluster.primary = self.primary else: self.determine_cluster() @@ -748,7 +890,9 @@ class SoSCollector(SoSComponent): self.cluster_type = 'none' self.collect_md.add_field('cluster_type', self.cluster_type) if self.cluster: - self.master.cluster = self.cluster + self.primary.cluster = self.cluster + if self.opts.transport == 'auto': + self.opts.transport = self.cluster.set_transport_type() self.cluster.setup() if self.cluster.cluster_ssh_key: if not self.opts.ssh_key: @@ -771,71 +915,73 @@ class SoSCollector(SoSComponent): """ self.ui_log.info('') - if not self.node_list and not self.master.connected: + if not self.node_list and not self.primary.connected: self.exit('No nodes were detected, or nodes do not have sos ' - 'installed.\nAborting...') + 'installed.\nAborting...', 1) self.ui_log.info('The following is a list of nodes to collect from:') - if self.master.connected and self.master.hostname is not None: - if not (self.master.local and self.opts.no_local): + if self.primary.connected and self.primary.hostname is not None: + if not ((self.primary.local and self.opts.no_local) + or self.cluster.strict_node_list): self.ui_log.info('\t%-*s' % (self.commons['hostlen'], - self.master.hostname)) + self.primary.hostname)) for node in sorted(self.node_list): self.ui_log.info("\t%-*s" % (self.commons['hostlen'], node)) self.ui_log.info('') + if not self.opts.batch: + try: + input("\nPress ENTER to continue with these nodes, or press " + "CTRL-C to quit\n") + self.ui_log.info("") + except KeyboardInterrupt: + self.exit("Exiting on user cancel", 130) + except Exception as e: + self.exit(repr(e), 1) def configure_sos_cmd(self): """Configures the sosreport command that is run on the nodes""" - self.sos_cmd = 'sosreport --batch ' - if self.opts.sos_opt_line: - filt = ['&', '|', '>', '<', ';'] - if any(f in self.opts.sos_opt_line for f in filt): - self.log_warn('Possible shell script found in provided sos ' - 'command. Ignoring --sos-opt-line entirely.') - self.opts.sos_opt_line = None - else: - self.sos_cmd = '%s %s' % ( - self.sos_cmd, quote(self.opts.sos_opt_line)) - self.log_debug("User specified manual sosreport command. " - "Command set to %s" % self.sos_cmd) - return True + sos_cmd = 'sosreport --batch ' - sos_opts = [] + sos_options = {} if self.opts.case_id: - sos_opts.append('--case-id=%s' % (quote(self.opts.case_id))) + sos_options['case-id'] = quote(self.opts.case_id) if self.opts.alloptions: - sos_opts.append('--alloptions') + sos_options['alloptions'] = '' if self.opts.all_logs: - sos_opts.append('--all-logs') + sos_options['all-logs'] = '' if self.opts.verify: - sos_opts.append('--verify') + sos_options['verify'] = '' if self.opts.log_size: - sos_opts.append(('--log-size=%s' % quote(str(self.opts.log_size)))) + sos_options['log-size'] = quote(str(self.opts.log_size)) if self.opts.sysroot: - sos_opts.append('-s %s' % quote(self.opts.sysroot)) + sos_options['sysroot'] = quote(self.opts.sysroot) if self.opts.chroot: - sos_opts.append('-c %s' % quote(self.opts.chroot)) + sos_options['chroot'] = quote(self.opts.chroot) if self.opts.compression_type != 'auto': - sos_opts.append('-z %s' % (quote(self.opts.compression_type))) - self.sos_cmd = self.sos_cmd + ' '.join(sos_opts) - self.log_debug("Initial sos cmd set to %s" % self.sos_cmd) - self.commons['sos_cmd'] = self.sos_cmd - self.collect_md.add_field('initial_sos_cmd', self.sos_cmd) + sos_options['compression-type'] = quote(self.opts.compression_type) - def connect_to_master(self): - """If run with --master, we will run cluster checks again that + for k, v in sos_options.items(): + sos_cmd += f"--{k} {v} " + sos_cmd = sos_cmd.rstrip() + self.log_debug(f"Initial sos cmd set to {sos_cmd}") + self.commons['sos_cmd'] = 'sosreport --batch ' + self.commons['sos_options'] = sos_options + self.collect_md.add_field('initial_sos_cmd', sos_cmd) + + def connect_to_primary(self): + """If run with --primary, we will run cluster checks again that instead of the localhost. """ try: - self.master = SosNode(self.opts.master, self.commons) + self.primary = SosNode(self.opts.primary, self.commons) self.ui_log.info('Connected to %s, determining cluster type...' - % self.opts.master) + % self.opts.primary) except Exception as e: - self.log_debug('Failed to connect to master: %s' % e) - self.exit('Could not connect to master node. Aborting...', 1) + self.log_debug('Failed to connect to primary node: %s' % e) + self.exit('Could not connect to primary node. Aborting...', 1) def determine_cluster(self): """This sets the cluster type and loads that cluster's cluster. @@ -849,7 +995,7 @@ class SoSCollector(SoSComponent): checks = list(self.clusters.values()) for cluster in self.clusters.values(): checks.remove(cluster) - cluster.master = self.master + cluster.primary = self.primary if cluster.check_enabled(): cname = cluster.__class__.__name__ self.log_debug("Installation matches %s, checking for layered " @@ -860,7 +1006,7 @@ class SoSCollector(SoSComponent): self.log_debug("Layered profile %s found. " "Checking installation" % rname) - remaining.master = self.master + remaining.primary = self.primary if remaining.check_enabled(): self.log_debug("Installation matches both layered " "profile %s and base profile %s, " @@ -884,18 +1030,19 @@ class SoSCollector(SoSComponent): return [] def reduce_node_list(self): - """Reduce duplicate entries of the localhost and/or master node + """Reduce duplicate entries of the localhost and/or primary node if applicable""" if (self.hostname in self.node_list and self.opts.no_local): self.node_list.remove(self.hostname) - for i in self.ip_addrs: - if i in self.node_list: - self.node_list.remove(i) - # remove the master node from the list, since we already have + if not self.cluster.strict_node_list: + for i in self.ip_addrs: + if i in self.node_list: + self.node_list.remove(i) + # remove the primary node from the list, since we already have # an open session to it. - if self.master: + if self.primary is not None and not self.cluster.strict_node_list: for n in self.node_list: - if n == self.master.hostname or n == self.opts.master: + if n == self.primary.hostname or n == self.opts.primary: self.node_list.remove(n) self.node_list = list(set(n for n in self.node_list if n)) self.log_debug('Node list reduced to %s' % self.node_list) @@ -916,11 +1063,11 @@ class SoSCollector(SoSComponent): def get_nodes(self): """ Sets the list of nodes to collect sosreports from """ - if not self.master and not self.cluster: + if not self.primary and not self.cluster: msg = ('Could not determine a cluster type and no list of ' - 'nodes or master node was provided.\nAborting...' + 'nodes or primary node was provided.\nAborting...' ) - self.exit(msg) + self.exit(msg, 1) try: nodes = self.get_nodes_from_cluster() @@ -947,18 +1094,20 @@ class SoSCollector(SoSComponent): self.log_debug("Force adding %s to node list" % node) self.node_list.append(node) - if not self.master: + if not self.primary: host = self.hostname.split('.')[0] # trust the local hostname before the node report from cluster for node in self.node_list: if host == node.split('.')[0]: self.node_list.remove(node) - self.node_list.append(self.hostname) + if not self.cluster.strict_node_list: + self.node_list.append(self.hostname) self.reduce_node_list() try: - self.commons['hostlen'] = len(max(self.node_list, key=len)) + _node_max = len(max(self.node_list, key=len)) + self.commons['hostlen'] = max(_node_max, self.commons['hostlen']) except (TypeError, ValueError): - self.commons['hostlen'] = len(self.opts.master) + pass def _connect_to_node(self, node): """Try to connect to the node, and if we can add to the client list to @@ -977,8 +1126,9 @@ class SoSCollector(SoSComponent): client.set_node_manifest(getattr(self.collect_md.nodes, node[0])) else: - client.close_ssh_session() + client.disconnect() except Exception: + # all exception logging is handled within SoSNode pass def intro(self): @@ -986,12 +1136,11 @@ class SoSCollector(SoSComponent): provided on the command line """ disclaimer = ("""\ -This utility is used to collect sosreports from multiple \ -nodes simultaneously. It uses OpenSSH's ControlPersist feature \ -to connect to nodes and run commands remotely. If your system \ -installation of OpenSSH is older than 5.6, please upgrade. +This utility is used to collect sos reports from multiple \ +nodes simultaneously. Remote connections are made and/or maintained \ +to those nodes via well-known transport protocols such as SSH. -An archive of sosreport tarballs collected from the nodes will be \ +An archive of sos report tarballs collected from the nodes will be \ generated in %s and may be provided to an appropriate support representative. The generated archive may contain data considered sensitive \ @@ -1004,6 +1153,7 @@ this utility or remote systems that it c self.ui_log.info("\nsos-collector (version %s)\n" % __version__) intro_msg = self._fmt_msg(disclaimer % self.tmpdir) self.ui_log.info(intro_msg) + prompt = "\nPress ENTER to continue, or CTRL-C to quit\n" if not self.opts.batch: try: @@ -1011,18 +1161,16 @@ this utility or remote systems that it c self.ui_log.info("") except KeyboardInterrupt: self.exit("Exiting on user cancel", 130) - - if not self.opts.case_id and not self.opts.batch: - msg = 'Please enter the case id you are collecting reports for: ' - self.opts.case_id = input(msg) + except Exception as e: + self.exit(e, 1) def execute(self): if self.opts.list_options: self.list_options() - self.cleanup() - raise SystemExit + self.exit() self.intro() + self.configure_sos_cmd() self.prep() self.display_nodes() @@ -1033,16 +1181,16 @@ this utility or remote systems that it c self.archive.makedirs('sos_logs', 0o755) self.collect() - self.cleanup() + self.exit() def collect(self): """ For each node, start a collection thread and then tar all collected sosreports """ - if self.master.connected: - self.client_list.append(self.master) + if self.primary.connected and not self.cluster.strict_node_list: + self.client_list.append(self.primary) self.ui_log.info("\nConnecting to nodes...") - filters = [self.master.address, self.master.hostname] + filters = [self.primary.address, self.primary.hostname] nodes = [(n, None) for n in self.node_list if n not in filters] if self.opts.password_per_node: @@ -1066,7 +1214,7 @@ this utility or remote systems that it c self.report_num = len(self.client_list) if self.report_num == 0: - self.exit("No nodes connected. Aborting...") + self.exit("No nodes connected. Aborting...", 1) elif self.report_num == 1: if self.client_list[0].address == 'localhost': self.exit( @@ -1074,7 +1222,7 @@ this utility or remote systems that it c "failure to either enumerate or connect to cluster " "nodes. Assuming single collection from localhost is " "not desired.\n" - "Aborting..." + "Aborting...", 1 ) self.ui_log.info("\nBeginning collection of sosreports from %s " @@ -1082,30 +1230,50 @@ this utility or remote systems that it c "concurrently\n" % (self.report_num, self.opts.jobs)) + npool = ThreadPoolExecutor(self.opts.jobs) + npool.map(self._finalize_sos_cmd, self.client_list, chunksize=1) + npool.shutdown(wait=True) + pool = ThreadPoolExecutor(self.opts.jobs) pool.map(self._collect, self.client_list, chunksize=1) pool.shutdown(wait=True) except KeyboardInterrupt: - self.log_error('Exiting on user cancel\n') - os._exit(130) + self.exit("Exiting on user cancel\n", 130, force=True) except Exception as err: - self.log_error('Could not connect to nodes: %s' % err) - os._exit(1) + msg = 'Could not connect to nodes: %s' % err + self.exit(msg, 1, force=True) if hasattr(self.cluster, 'run_extra_cmd'): - self.ui_log.info('Collecting additional data from master node...') + self.ui_log.info('Collecting additional data from primary node...') files = self.cluster._run_extra_cmd() if files: - self.master.collect_extra_cmd(files) + self.primary.collect_extra_cmd(files) msg = '\nSuccessfully captured %s of %s sosreports' self.log_info(msg % (self.retrieved, self.report_num)) self.close_all_connections() if self.retrieved > 0: - self.create_cluster_archive() + arc_name = self.create_cluster_archive() else: msg = 'No sosreports were collected, nothing to archive...' self.exit(msg, 1) + if self.opts.upload and self.policy.get_upload_url(): + try: + self.policy.upload_archive(arc_name) + self.ui_log.info("Uploaded archive successfully") + except Exception as err: + self.ui_log.error("Upload attempt failed: %s" % err) + + def _finalize_sos_cmd(self, client): + """Calls finalize_sos_cmd() on each node so that we have the final + command before we thread out the actual execution of sos + """ + try: + client.finalize_sos_cmd() + except Exception as err: + self.log_error("Could not finalize sos command for %s: %s" + % (client.address, err)) + def _collect(self, client): """Runs sosreport on each node""" try: @@ -1120,10 +1288,11 @@ this utility or remote systems that it c self.log_error("Error running sosreport: %s" % err) def close_all_connections(self): - """Close all ssh sessions for nodes""" + """Close all sessions for nodes""" for client in self.client_list: - self.log_debug('Closing SSH connection to %s' % client.address) - client.close_ssh_session() + if client.connected: + self.log_debug('Closing connection to %s' % client.address) + client.disconnect() def create_cluster_archive(self): """Calls for creation of tar archive then cleans up the temporary @@ -1158,8 +1327,6 @@ this utility or remote systems that it c self.log_info('Creating archive of sosreports...') for fname in arc_paths: dest = fname.split('/')[-1] - if fname.endswith(('.md5',)): - dest = os.path.join('checksums', fname.split('/')[-1]) if do_clean: dest = cleaner.obfuscate_string(dest) name = os.path.join(self.tmpdir, fname) @@ -1169,7 +1336,7 @@ this utility or remote systems that it c checksum = cleaner.get_new_checksum(fname) if checksum: name = os.path.join('checksums', fname.split('/')[-1]) - name += '.md5' + name += '.sha256' self.archive.add_string(checksum, name) self.archive.add_file(self.sos_log_file, dest=os.path.join('sos_logs', 'sos.log')) @@ -1218,6 +1385,7 @@ this utility or remote systems that it c self.ui_log.info('\nThe following archive has been created. ' 'Please provide it to your support team.') self.ui_log.info('\t%s\n' % final_name) + return final_name except Exception as err: msg = ("Could not finalize archive: %s\n\nData may still be " "available uncompressed at %s" % (err, self.archive_path)) diff -pruN 4.0-2/sos/collector/clusters/__init__.py 4.5.3ubuntu2/sos/collector/clusters/__init__.py --- 4.0-2/sos/collector/clusters/__init__.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/clusters/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -11,6 +11,8 @@ import logging from sos.options import ClusterOption +from sos.utilities import bold +from threading import Lock class Cluster(): @@ -39,6 +41,9 @@ class Cluster(): :cvar sos_plugins: Which plugins to forcibly enable for node reports :vartype sos_plugins: ``list`` + :cvar sos_options: Options to pass to report on every node + :vartype sos_options: ``dict`` + :cvar sos_plugin_options: Plugin options to forcibly set for nodes :vartype sos_plugin_options: ``dict`` @@ -52,20 +57,26 @@ class Cluster(): option_list = [] packages = ('',) sos_plugins = [] + sos_options = {} sos_plugin_options = {} sos_preset = '' cluster_name = None + # set this to True if the local host running collect should *not* be + # forcibly added to the node list. This can be helpful in situations where + # the host's fqdn and the name the cluster uses are different + strict_node_list = False def __init__(self, commons): - self.master = None + self.primary = None self.cluster_ssh_key = None self.tmpdir = commons['tmpdir'] - self.opts = commons['opts'] + self.opts = commons['cmdlineopts'] self.cluster_type = [self.__class__.__name__] for cls in self.__class__.__bases__: if cls.__name__ != 'Cluster': self.cluster_type.append(cls.__name__) self.node_list = None + self.lock = Lock() self.soslog = logging.getLogger('sos') self.ui_log = logging.getLogger('sos_ui') self.options = [] @@ -79,6 +90,111 @@ class Cluster(): return cls.cluster_name return cls.__name__.lower() + @classmethod + def display_help(cls, section): + if cls is Cluster: + cls.display_self_help(section) + return + section.set_title("%s Cluster Profile Detailed Help" + % cls.cluster_name) + if cls.__doc__ and cls.__doc__ is not Cluster.__doc__: + section.add_text(cls.__doc__) + # [1] here is the actual cluster profile + elif cls.__mro__[1].__doc__ and cls.__mro__[1] is not Cluster: + section.add_text(cls.__mro__[1].__doc__) + else: + section.add_text( + "\n\tDetailed help not available for this profile\n" + ) + + if cls.packages: + section.add_text( + "Enabled by the following packages: %s" + % ', '.join(p for p in cls.packages), + newline=False + ) + + if cls.sos_preset: + section.add_text( + "Uses the following sos preset: %s" % cls.sos_preset, + newline=False + ) + + if cls.sos_options: + _opts = ', '.join(f'--{k} {v}' for k, v in cls.sos_options.items()) + section.add_text(f"Sets the following sos options: {_opts}") + + if cls.sos_plugins: + section.add_text( + "Enables the following plugins: %s" + % ', '.join(plug for plug in cls.sos_plugins), + newline=False + ) + + if cls.sos_plugin_options: + _opts = cls.sos_plugin_options + opts = ', '.join("%s=%s" % (opt, _opts[opt]) for opt in _opts) + section.add_text( + "Sets the following plugin options: %s" % opts, + newline=False + ) + + if cls.option_list: + optsec = section.add_section("Available cluster options") + optsec.add_text( + "These options may be toggled or changed using '%s'" + % bold("-c %s.$option=$value" % cls.__name__) + ) + optsec.add_text(bold( + "\n{:<4}{:<20}{:<30}{:<20}\n".format( + ' ', "Option Name", "Default", "Description") + ), newline=False + ) + for opt in cls.option_list: + val = opt[1] + if isinstance(val, bool): + if val: + val = 'True/On' + else: + val = 'False/Off' + _ln = "{:<4}{:<20}{:<30}{:<20}".format(' ', opt[0], val, + opt[2]) + optsec.add_text(_ln, newline=False) + + @classmethod + def display_self_help(cls, section): + section.set_title('SoS Collect Cluster Profiles Detailed Help') + section.add_text( + '\nCluster profiles are used to represent different clustering ' + 'technologies or platforms. Profiles define how cluster nodes are ' + 'discovered, and optionally filtered, for default executions of ' + 'collector.' + ) + section.add_text( + 'Cluster profiles are enabled similarly to SoS report plugins; ' + 'usually by package, command, or configuration file presence. ' + 'Clusters may also define default transports for SoS collect.' + ) + + from sos.collector import SoSCollector + import inspect + clusters = SoSCollector._load_modules(inspect.getmodule(cls), + 'clusters') + + section.add_text( + 'The following cluster profiles are locally available:\n' + ) + section.add_text( + "{:>8}{:<40}{:<30}".format(' ', 'Name', 'Description'), + newline=False + ) + for cluster in clusters: + _sec = bold("collect.clusters.%s" % cluster[0]) + section.add_text( + "{:>8}{:<40}{:<30}".format(' ', _sec, cluster[1].cluster_name), + newline=False + ) + def _get_options(self): """Loads the options defined by a cluster and sets the default value""" for opt in self.option_list: @@ -104,7 +220,7 @@ class Cluster(): def log_warn(self, msg): """Used to print warning messages""" - self.soslog.warn(self._fmt_msg(msg)) + self.soslog.warning(self._fmt_msg(msg)) def get_option(self, option): """ @@ -133,12 +249,53 @@ class Cluster(): key rather than prompting the user for one or a password. Note this will only function if collector is being run locally on the - master node. + primary node. """ self.cluster_ssh_key = key - def exec_master_cmd(self, cmd, need_root=False): - """Used to retrieve command output from a (master) node in a cluster + def set_node_options(self, node): + """If there is a need to set specific options on ONLY the non-primary + nodes in a collection, override this method in the cluster profile + and do that here. + + :param node: The non-primary node + :type node: ``SoSNode`` + """ + pass + + def set_transport_type(self): + """The default connection type used by sos collect is to leverage the + local system's SSH installation using ControlPersist, however certain + cluster types may want to use something else. + + Override this in a specific cluster profile to set the ``transport`` + option according to what type of transport should be used. + """ + return 'control_persist' + + def set_primary_options(self, node): + """If there is a need to set specific options in the sos command being + run on the cluster's primary nodes, override this method in the cluster + profile and do that here. + + :param node: The primary node + :type node: ``SoSNode`` + """ + pass + + def check_node_is_primary(self, node): + """In the event there are multiple primaries, or if the collect command + is being run from a system that is technically capable of enumerating + nodes but the cluster profiles needs to specify primary-specific + options for other nodes, override this method in the cluster profile + + :param node: The node for the cluster to check + :type node: ``SoSNode`` + """ + return node.address == self.primary.address + + def exec_primary_cmd(self, cmd, need_root=False): + """Used to retrieve command output from a (primary) node in a cluster :param cmd: The command to run :type cmd: ``str`` @@ -149,9 +306,10 @@ class Cluster(): :returns: The output and status of `cmd` :rtype: ``dict`` """ - res = self.master.run_command(cmd, get_pty=True, need_root=need_root) - if res['stdout']: - res['stdout'] = res['stdout'].replace('Password:', '') + pty = self.primary.local is False + res = self.primary.run_command(cmd, get_pty=pty, need_root=need_root) + if res['output']: + res['output'] = res['output'].replace('Password:', '') return res def setup(self): @@ -176,10 +334,20 @@ class Cluster(): :rtype: ``bool`` """ for pkg in self.packages: - if self.master.is_installed(pkg): + if self.primary.is_installed(pkg): return True return False + def cleanup(self): + """ + This may be overridden by clusters + + Perform any necessary cleanup steps required by the cluster profile. + This helps ensure that sos does make lasting changes to the environment + in which we are running + """ + pass + def get_nodes(self): """ This MUST be overridden by a cluster profile subclassing this class @@ -217,8 +385,8 @@ class Cluster(): def set_node_label(self, node): """This may be overridden by clusters profiles subclassing this class - If there is a distinction between masters and nodes, or types of nodes, - then this can be used to label the sosreport archive differently. + If there is a distinction between primaries and nodes, or types of + nodes, then this can be used to label the sosreport archive differently """ return '' @@ -232,13 +400,14 @@ class Cluster(): """ try: nodes = self.get_nodes() - except Exception as e: - self.log_error('Cluster failed to enumerate nodes: %s' % e) - raise + except Exception as err: + raise Exception(f"Cluster failed to enumerate nodes: {err}") if isinstance(nodes, list): node_list = [n.strip() for n in nodes if n] elif isinstance(nodes, str): node_list = [n.split(',').strip() for n in nodes] + else: + raise Exception(f"Cluster returned unexpected node list: {nodes}") node_list = list(set(node_list)) for node in node_list: if node.startswith(('-', '_', '(', ')', '[', ']', '/', '\\')): diff -pruN 4.0-2/sos/collector/clusters/ceph.py 4.5.3ubuntu2/sos/collector/clusters/ceph.py --- 4.0-2/sos/collector/clusters/ceph.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/clusters/ceph.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,67 @@ +# Copyright (C) 2022 Red Hat Inc., Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import json + +from sos.collector.clusters import Cluster + + +class ceph(Cluster): + """ + This cluster profile is for Ceph Storage clusters, and is primarily + built around Red Hat Ceph Storage 5. Nodes are enumerated via `cephadm`; if + your Ceph deployment uses cephadm but is not RHCS 5, this profile may work + as intended, but it is not currently guaranteed to do so. If you are using + such an environment and this profile does not work for you, please file a + bug report detailing what is failing. + + By default, all nodes in the cluster will be returned for collection. This + may not be desirable, so users are encouraged to use the `labels` option + to specify a colon-delimited set of ceph node labels to restrict the list + of nodes to. + + For example, using `-c ceph.labels=osd:mgr` will return only nodes labeled + with *either* `osd` or `mgr`. + """ + + cluster_name = 'Ceph Storage Cluster' + sos_plugins = [ + 'ceph_common', + ] + sos_options = {'log-size': 50} + packages = ('cephadm',) + option_list = [ + ('labels', '', 'Colon delimited list of labels to select nodes with') + ] + + def get_nodes(self): + self.nodes = [] + ceph_out = self.exec_primary_cmd( + 'cephadm shell -- ceph orch host ls --format json', + need_root=True + ) + + if not ceph_out['status'] == 0: + self.log_error( + f"Could not enumerate nodes via cephadm: {ceph_out['output']}" + ) + return self.nodes + + nodes = json.loads(ceph_out['output'].splitlines()[-1]) + _labels = [lab for lab in self.get_option('labels').split(':') if lab] + for node in nodes: + if _labels and not any(_l in node['labels'] for _l in _labels): + self.log_debug(f"{node} filtered from list due to labels") + continue + self.nodes.append(node['hostname']) + + return self.nodes + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/collector/clusters/jbon.py 4.5.3ubuntu2/sos/collector/clusters/jbon.py --- 4.0-2/sos/collector/clusters/jbon.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/clusters/jbon.py 2023-04-28 17:16:21.000000000 +0000 @@ -12,11 +12,14 @@ from sos.collector.clusters import Clust class jbon(Cluster): - '''Just a Bunch of Nodes - - Used when --cluster-type=none (or jbon), to avoid cluster checks, and just - use the provided --nodes list - ''' + """ + Used when --cluster-type=none (or jbon) to avoid cluster checks, and just + use the provided --nodes list. + + Using this profile will skip any and all operations that a cluster profile + normally performs, and will not set any plugins, plugin options, or presets + for the sos report generated on the nodes provided by --nodes. + """ cluster_name = 'Just a Bunch Of Nodes (no cluster)' packages = None @@ -28,3 +31,5 @@ class jbon(Cluster): # This should never be called, but as insurance explicitly never # allow this to be enabled via the determine_cluster() path return False + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/collector/clusters/juju.py 4.5.3ubuntu2/sos/collector/clusters/juju.py --- 4.0-2/sos/collector/clusters/juju.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/clusters/juju.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,214 @@ +# Copyright (c) 2023 Canonical Ltd., Chi Wai Chan + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import json +import re + +from sos.collector.clusters import Cluster + + +def _parse_option_string(strings=None): + """Parse comma separated string.""" + if not strings: + return [] + return [string.strip() for string in strings.split(",")] + + +def _get_index(model_name): + """Helper function to get Index. + + The reason why we need Index defined in function is because currently + the collector.__init__ will load all the classes in this module + and also Index. This will cause bug because it think Index is + Cluster type. Also We don't want to provide a customized + filter to remove Index class. + """ + + class Index: + """Index structure to help parse juju status output. + + Attributes apps, units and machines are dict which key + is the app/unit/machine name + and the value is list of targets which format are + {model_name}:{machine_id}. + """ + + def __init__(self, model_name): + self.model_name: str = model_name + self.apps = {} + self.units = {} + self.machines = {} + + def add_principals(self, juju_status): + """Adds principal units to index.""" + for app, app_info in juju_status["applications"].items(): + nodes = [] + units = app_info.get("units", {}) + for unit, unit_info in units.items(): + machine = unit_info["machine"] + node = f"{self.model_name}:{machine}" + self.units[unit] = [node] + self.machines[machine] = [node] + nodes.append(node) + + self.apps[app] = nodes + + def add_subordinates(self, juju_status): + """Add subordinates to index. + + Since subordinates does not have units they need to be + manually added. + """ + for app, app_info in juju_status["applications"].items(): + subordinate_to = app_info.get("subordinate-to", []) + for parent in subordinate_to: + self.apps[app].extend(self.apps[parent]) + units = juju_status["applications"][parent]["units"] + for unit, unit_info in units.items(): + node = f"{self.model_name}:{unit_info['machine']}" + for sub_key, sub_value in unit_info.get( + "subordinates", {} + ).items(): + if sub_key.startswith(app + "/"): + self.units[sub_key] = [node] + + def add_machines(self, juju_status): + """Add machines to index. + + If model does not have any applications it needs to be + manually added. + """ + for machine in juju_status["machines"].keys(): + node = f"{self.model_name}:{machine}" + self.machines[machine] = [node] + + return Index(model_name) + + +class juju(Cluster): + """ + The juju cluster profile is intended to be used on juju managed clouds. + It"s assumed that `juju` is installed on the machine where `sos` is called, + and that the juju user has superuser privilege to the current controller. + + By default, the sos reports will be collected from all the applications in + the current model. If necessary, you can filter the nodes by models / + applications / units / machines with cluster options. + + Example: + + sos collect --cluster-type juju -c "juju.models=sos" -c "juju.apps=a,b,c" + + """ + + cmd = "juju" + cluster_name = "Juju Managed Clouds" + option_list = [ + ("apps", "", "Filter node list by apps (comma separated regex)."), + ("units", "", "Filter node list by units (comma separated string)."), + ("models", "", "Filter node list by models (comma separated string)."), + ( + "machines", + "", + "Filter node list by machines (comma separated string).", + ), + ] + + def _cleanup_juju_output(self, output): + """Remove leading characters before {.""" + return re.sub(r"(^[^{]*)(.*)", "\\2", output, 0, re.MULTILINE) + + def _get_model_info(self, model_name): + """Parse juju status output and return target dict. + + Here are couple helper functions to parse the juju principals units, + subordinate units and machines. + """ + juju_status = self._execute_juju_status(model_name) + + index = _get_index(model_name=model_name) + index.add_principals(juju_status) + index.add_subordinates(juju_status) + index.add_machines(juju_status) + + return index + + def _execute_juju_status(self, model_name): + model_option = f"-m {model_name}" if model_name else "" + format_option = "--format json" + status_cmd = f"{self.cmd} status {model_option} {format_option}" + res = self.exec_primary_cmd(status_cmd) + if not res["status"] == 0: + raise Exception(f"'{status_cmd}' returned error: {res['status']}") + juju_json_output = self._cleanup_juju_output((res["output"])) + + juju_status = None + try: + juju_status = json.loads(juju_json_output) + except json.JSONDecodeError: + raise Exception( + "Juju output is not valid json format." + f"Output: {juju_json_output}" + ) + return juju_status + + def _filter_by_pattern(self, key, patterns, model_info): + """Filter with regex match.""" + nodes = set() + for pattern in patterns: + for param, value in getattr(model_info, key).items(): + if re.match(pattern, param): + nodes.update(value or []) + return nodes + + def _filter_by_fixed(self, key, patterns, model_info): + """Filter with fixed match.""" + nodes = set() + for pattern in patterns: + for param, value in getattr(model_info, key).items(): + if pattern == param: + nodes.update(value or []) + return nodes + + def set_transport_type(self): + """Dynamically change transport to 'juju'.""" + return "juju" + + def get_nodes(self): + """Get the machine numbers from `juju status`.""" + models = _parse_option_string(self.get_option("models")) + apps = _parse_option_string(self.get_option("apps")) + units = _parse_option_string(self.get_option("units")) + machines = _parse_option_string(self.get_option("machines")) + filters = {"apps": apps, "units": units, "machines": machines} + + # Return empty nodes if no model and filter provided. + if not any(filters.values()) and not models: + return [] + + if not models: + models = [""] # use current model by default + + nodes = set() + + for model in models: + model_info = self._get_model_info(model) + for key, resource in filters.items(): + # Filter node by different policies + if key == "apps": + _nodes = self._filter_by_pattern(key, resource, model_info) + else: + _nodes = self._filter_by_fixed(key, resource, model_info) + nodes.update(_nodes) + + return list(nodes) + + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/collector/clusters/kubernetes.py 4.5.3ubuntu2/sos/collector/clusters/kubernetes.py --- 4.0-2/sos/collector/clusters/kubernetes.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/clusters/kubernetes.py 2023-04-28 17:16:21.000000000 +0000 @@ -13,7 +13,12 @@ from sos.collector.clusters import Clust class kubernetes(Cluster): - + """ + The kuberentes cluster profile is intended to be used on kubernetes + clusters built from the upstream/source kubernetes (k8s) project. It is + not intended for use with other projects or platforms that are built ontop + of kubernetes. + """ cluster_name = 'Community Kubernetes' packages = ('kubernetes-master',) sos_plugins = ['kubernetes'] @@ -30,11 +35,11 @@ class kubernetes(Cluster): self.cmd += ' get nodes' if self.get_option('label'): self.cmd += ' -l %s ' % quote(self.get_option('label')) - res = self.exec_master_cmd(self.cmd) + res = self.exec_primary_cmd(self.cmd) if res['status'] == 0: nodes = [] roles = [x for x in self.get_option('role').split(',') if x] - for nodeln in res['stdout'].splitlines()[1:]: + for nodeln in res['output'].splitlines()[1:]: node = nodeln.split() if not roles: nodes.append(node[0]) @@ -45,10 +50,4 @@ class kubernetes(Cluster): else: raise Exception('Node enumeration did not return usable output') - -class openshift(kubernetes): - - cluster_name = 'OpenShift Container Platform' - packages = ('atomic-openshift',) - sos_preset = 'ocp' - cmd = 'oc' +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/collector/clusters/ocp.py 4.5.3ubuntu2/sos/collector/clusters/ocp.py --- 4.0-2/sos/collector/clusters/ocp.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/clusters/ocp.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,370 @@ +# Copyright Red Hat 2021, Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import os + +from pipes import quote +from sos.collector.clusters import Cluster +from sos.utilities import is_executable + + +class ocp(Cluster): + """ + This profile is for use with OpenShift Container Platform (v4) clusters + instead of the kubernetes profile. + + This profile will favor using the `oc` transport type, which means it will + leverage a locally installed `oc` binary. This is also how node enumeration + is done. To instead use SSH to connect to the nodes, use the + '--transport=control_persist' option. + + Thus, a functional `oc` binary for the user executing sos collect is + required. Functional meaning that the user can run `oc` commands with + clusterAdmin privileges. + + If this requires the use of a secondary configuration file, specify that + path with the 'kubeconfig' cluster option. This config file will also be + used on a single master node to perform API collections if the `with-api` + option is enabled (default disabled). If no `kubeconfig` option is given, + but `with-api` is enabled, the cluster profile will attempt to use a + well-known default kubeconfig file if it is available on the host. + + Alternatively, provide a clusterAdmin access token either via the 'token' + cluster option or, preferably, the SOSOCPTOKEN environment variable. + + By default, this profile will enumerate only master nodes within the + cluster, and this may be changed by overriding the 'role' cluster option. + To collect from all nodes in the cluster regardless of role, use the form + -c ocp.role=''. + + Filtering nodes by a label applied to that node is also possible via the + label cluster option, though be aware that this is _combined_ with the role + option mentioned above. + + To avoid redundant collections of OCP API information (e.g. 'oc get' + commands), this profile will attempt to enable the API collections on only + a single master node. If the none of the master nodes have a functional + 'oc' binary available, *and* the --no-local option is used, that means that + no API data will be collected. + """ + + cluster_name = 'OpenShift Container Platform v4' + packages = ('openshift-hyperkube', 'openshift-clients') + + api_collect_enabled = False + token = None + project = 'sos-collect-tmp' + oc_cluster_admin = None + _oc_cmd = '' + + option_list = [ + ('label', '', 'Colon delimited list of labels to select nodes with'), + ('role', 'master', 'Colon delimited list of roles to filter on'), + ('kubeconfig', '', 'Path to the kubeconfig file'), + ('token', '', 'Service account token to use for oc authorization'), + ('with-api', False, 'Collect OCP API data from a master node') + ] + + @property + def oc_cmd(self): + if not self._oc_cmd: + self._oc_cmd = 'oc' + if self.primary.host.in_container(): + _oc_path = self.primary.run_command( + 'which oc', chroot=self.primary.host.sysroot + ) + if _oc_path['status'] == 0: + self._oc_cmd = os.path.join( + self.primary.host.sysroot, + _oc_path['output'].strip().lstrip('/') + ) + else: + self.log_warning( + "Unable to to determine PATH for 'oc' command, " + "node enumeration may fail." + ) + self.log_debug("Locating 'oc' failed: %s" + % _oc_path['output']) + if self.get_option('kubeconfig'): + self._oc_cmd += " --config %s" % self.get_option('kubeconfig') + self.log_debug("oc base command set to %s" % self._oc_cmd) + return self._oc_cmd + + def fmt_oc_cmd(self, cmd): + """Format the oc command to optionall include the kubeconfig file if + one is specified + """ + return "%s %s" % (self.oc_cmd, cmd) + + def _attempt_oc_login(self): + """Attempt to login to the API using the oc command using a provided + token + """ + _res = self.exec_primary_cmd( + self.fmt_oc_cmd("login --insecure-skip-tls-verify=True --token=%s" + % self.token) + ) + return _res['status'] == 0 + + def check_enabled(self): + if super(ocp, self).check_enabled(): + return True + self.token = self.get_option('token') or os.getenv('SOSOCPTOKEN', None) + if self.token: + self._attempt_oc_login() + _who = self.fmt_oc_cmd('whoami') + return self.exec_primary_cmd(_who)['status'] == 0 + + def setup(self): + """Create the project that we will be executing in for any nodes' + collection via a container image + """ + if not self.set_transport_type() == 'oc': + return + + out = self.exec_primary_cmd(self.fmt_oc_cmd("auth can-i '*' '*'")) + self.oc_cluster_admin = out['status'] == 0 + if not self.oc_cluster_admin: + self.log_debug("Check for cluster-admin privileges returned false," + " cannot create project in OCP cluster") + raise Exception("Insufficient permissions to create temporary " + "collection project.\nAborting...") + + self.log_info("Creating new temporary project '%s'" % self.project) + ret = self.exec_primary_cmd( + self.fmt_oc_cmd("new-project %s" % self.project) + ) + if ret['status'] == 0: + self._label_sos_project() + return True + + self.log_debug("Failed to create project: %s" % ret['output']) + raise Exception("Failed to create temporary project for collection. " + "\nAborting...") + + def _label_sos_project(self): + """Add pertinent labels to the temporary project we've created so that + our privileged containers can properly run. + """ + labels = [ + "security.openshift.io/scc.podSecurityLabelSync=false", + "pod-security.kubernetes.io/enforce=privileged" + ] + for label in labels: + ret = self.exec_primary_cmd( + self.fmt_oc_cmd( + f"label namespace {self.project} {label} --overwrite" + ) + ) + if not ret['status'] == 0: + raise Exception( + f"Error applying namespace labels: {ret['output']}" + ) + + def cleanup(self): + """Remove the project we created to execute within + """ + if self.project: + ret = self.exec_primary_cmd( + self.fmt_oc_cmd("delete project %s" % self.project) + ) + if not ret['status'] == 0: + self.log_error("Error deleting temporary project: %s" + % ret['output']) + ret = self.exec_primary_cmd( + self.fmt_oc_cmd("wait namespace/%s --for=delete --timeout=30s" + % self.project) + ) + if not ret['status'] == 0: + self.log_error("Error waiting for temporary project to be " + "deleted: %s" % ret['output']) + # don't leave the config on a non-existing project + self.exec_primary_cmd(self.fmt_oc_cmd("project default")) + self.project = None + return True + + def _build_dict(self, nodelist): + """From the output of get_nodes(), construct an easier-to-reference + dict of nodes that will be used in determining labels, primary status, + etc... + + :param nodelist: The split output of `oc get nodes` + :type nodelist: ``list`` + + :returns: A dict of nodes with `get nodes` columns as keys + :rtype: ``dict`` + """ + nodes = {} + if 'NAME' in nodelist[0]: + # get the index of the fields + statline = nodelist.pop(0).split() + idx = {} + for state in ['status', 'roles', 'version', 'os-image']: + try: + idx[state] = statline.index(state.upper()) + except Exception: + pass + for node in nodelist: + _node = node.split() + nodes[_node[0]] = {} + for column in idx: + nodes[_node[0]][column] = _node[idx[column]] + return nodes + + def set_transport_type(self): + if self.opts.transport != 'auto': + return self.opts.transport + if is_executable('oc', sysroot=self.primary.host.sysroot): + return 'oc' + self.log_info("Local installation of 'oc' not found or is not " + "correctly configured. Will use ControlPersist.") + self.ui_log.warn( + "Preferred transport 'oc' not available, will fallback to SSH." + ) + if not self.opts.batch: + input("Press ENTER to continue connecting with SSH, or Ctrl+C to" + "abort.") + return 'control_persist' + + def get_nodes(self): + nodes = [] + self.node_dict = {} + cmd = 'get nodes -o wide' + if self.get_option('label'): + labels = ','.join(self.get_option('label').split(':')) + cmd += " -l %s" % quote(labels) + res = self.exec_primary_cmd(self.fmt_oc_cmd(cmd)) + if res['status'] == 0: + if self.get_option('role') == 'master': + self.log_warn("NOTE: By default, only master nodes are listed." + "\nTo collect from all/more nodes, override the " + "role option with '-c ocp.role=role1:role2'") + roles = [r for r in self.get_option('role').split(':')] + self.node_dict = self._build_dict(res['output'].splitlines()) + for node_name, node in self.node_dict.items(): + if roles: + for role in roles: + if role in node['roles']: + nodes.append(node_name) + break + else: + nodes.append(node_name) + else: + msg = "'oc' command failed" + if 'Missing or incomplete' in res['output']: + msg = ("'oc' failed due to missing kubeconfig on primary node." + " Specify one via '-c ocp.kubeconfig='") + raise Exception(msg) + return nodes + + def set_node_label(self, node): + if node.address not in self.node_dict: + return '' + for label in ['master', 'worker']: + if label in self.node_dict[node.address]['roles']: + return label + return '' + + def check_node_is_primary(self, sosnode): + if sosnode.address not in self.node_dict: + return False + return 'master' in self.node_dict[sosnode.address]['roles'] + + def _toggle_api_opt(self, node, use_api): + """In earlier versions of sos, the openshift plugin option that is + used to toggle the API collections was called `no-oc` rather than + `with-api`. This older plugin option had the inverse logic of the + current `with-api` option. + + Use this to toggle the correct plugin option given the node's sos + version. Note that the use of version 4.2 here is tied to the RHEL + release (the only usecase for this cluster profile) rather than + the upstream version given the backports for that downstream. + + :param node: The node being inspected for API collections + :type node: ``SoSNode`` + + :param use_api: Should this node enable API collections? + :type use_api: ``bool`` + """ + if node.check_sos_version('4.2-16'): + _opt = 'with-api' + _val = 'on' if use_api else 'off' + else: + _opt = 'no-oc' + _val = 'off' if use_api else 'on' + node.plugopts.append("openshift.%s=%s" % (_opt, _val)) + + def set_primary_options(self, node): + + node.enable_plugins.append('openshift') + if not self.get_option('with-api'): + self._toggle_api_opt(node, False) + return + if self.api_collect_enabled: + # a primary has already been enabled for API collection, disable + # it among others + self._toggle_api_opt(node, False) + else: + # running in a container, so reference the /host mount point + master_kube = ( + '/host/etc/kubernetes/static-pod-resources/' + 'kube-apiserver-certs/secrets/node-kubeconfigs/' + 'localhost.kubeconfig' + ) + _optconfig = self.get_option('kubeconfig') + if _optconfig and not _optconfig.startswith('/host'): + _optconfig = '/host/' + _optconfig + _kubeconfig = _optconfig or master_kube + _oc_cmd = 'oc' + if node.host.containerized: + _oc_cmd = '/host/bin/oc' + # when run from a container, the oc command does not inherit + # the default config, so if it's present then pass it here to + # detect a funcitonal oc command. This is sidestepped in sos + # report by being able to chroot the `oc` execution which we + # cannot do remotely + if node.file_exists('/root/.kube/config', need_root=True): + _oc_cmd += ' --kubeconfig /host/root/.kube/config' + can_oc = node.run_command("%s whoami" % _oc_cmd, + use_container=node.host.containerized, + # container is available only to root + # and if rhel, need to run sos as root + # anyways which will run oc as root + need_root=True) + if can_oc['status'] == 0: + # the primary node can already access the API + self._toggle_api_opt(node, True) + self.api_collect_enabled = True + elif self.token: + node.sos_env_vars['SOSOCPTOKEN'] = self.token + self._toggle_api_opt(node, True) + self.api_collect_enabled = True + elif node.file_exists(_kubeconfig): + # if the file exists, then the openshift sos plugin will use it + # if the with-api option is turned on + if not _kubeconfig == master_kube: + node.plugopts.append( + "openshift.kubeconfig=%s" % _kubeconfig + ) + self._toggle_api_opt(node, True) + self.api_collect_enabled = True + if self.api_collect_enabled: + msg = ("API collections will be performed on %s\nNote: API " + "collections may extend runtime by 10s of minutes\n" + % node.address) + self.soslog.info(msg) + self.ui_log.info(msg) + + def set_node_options(self, node): + # don't attempt OC API collections on non-primary nodes + self._toggle_api_opt(node, False) + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/collector/clusters/openstack.py 4.5.3ubuntu2/sos/collector/clusters/openstack.py --- 4.0-2/sos/collector/clusters/openstack.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/clusters/openstack.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,68 @@ +# Copyright Red Hat 2022, Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import yaml + +from sos.collector.clusters import Cluster + +INVENTORY = "/var/lib/mistral/overcloud/tripleo-ansible-inventory.yaml" + + +class rhosp(Cluster): + """ + This cluster profile is for use with Red Hat OpenStack Platform + environments. + + Different types of nodes may be enumerated by toggling the various profile + options such as Controllers and Compute nodes. By default, only Controller + nodes are enumerated. + + Node enumeration is done by inspecting the ansible inventory file used for + deployment of the environment. This is canonically located at + /var/lib/mistral/overcloud/tripleo-ansible-inventory.yaml. Similarly, the + presence of this file on the primary node is what triggers the automatic + enablement of this profile. + + Special consideration should be taken for where `sos collect` is being run + from, in that the hostnames of the enumerated nodes must be resolveable + from that system - not just from the primary node from which those nodes + are discovered. If this is not possible, consider enabling the `use-ip` + cluster option to instead have this profile source the IP addresses of the + nodes in question. + """ + + cluster_name = 'Red Hat OpenStack Platform' + option_list = [ + ('use-ip', False, 'use IP addresses instead of hostnames to connect'), + ('controller', True, 'collect reports from controller nodes'), + ('compute', False, 'collect reports from compute nodes') + ] + + def check_enabled(self): + return self.primary.file_exists(INVENTORY, need_root=True) + + def get_nodes(self): + _nodes = [] + _addr_field = ('external_ip' if self.get_option('use-ip') else + 'ctlplane_hostname') + try: + _inv = yaml.safe_load(self.primary.read_file(INVENTORY)) + except Exception as err: + self.log_info("Error parsing yaml: %s" % err) + raise Exception("Could not parse yaml for node addresses") + try: + for _t in ['Controller', 'Compute']: + # fields are titled in the yaml, but our opts are lowercase + if self.get_option(_t.lower()): + for host in _inv[_t]['hosts'].keys(): + _nodes.append(_inv[_t]['hosts'][host][_addr_field]) + except Exception as err: + self.log_error("Error getting %s host addresses: %s" % (_t, err)) + return _nodes diff -pruN 4.0-2/sos/collector/clusters/ovirt.py 4.5.3ubuntu2/sos/collector/clusters/ovirt.py --- 4.0-2/sos/collector/clusters/ovirt.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/clusters/ovirt.py 2023-04-28 17:16:21.000000000 +0000 @@ -17,6 +17,33 @@ ENGINE_KEY = '/etc/pki/ovirt-engine/keys class ovirt(Cluster): + """ + This cluster profile is for the oVirt/RHV project which provides for a + virtualization cluster built ontop of KVM. + + Nodes enumerated will be hypervisors within the envrionment, not virtual + machines running on those hypervisors. By default, ALL hypervisors within + the environment are returned. This may be influenced by the 'cluster' and + 'datacenter' cluster options, which will limit enumeration to hypervisors + within the specific cluster and/or datacenter. The spm-only cluster option + may also be used to only collect from hypervisors currently holding the + SPM role. + + Optionally, to only collect an archive from manager and the postgresql + database, use the no-hypervisors cluster option. + + By default, a second archive from the manager will be collected that is + just the postgresql plugin configured in such a way that a dump of the + manager's database that can be explored and restored to other systems will + be collected. + + The ovirt profile focuses on the upstream, community ovirt project. + + The rhv profile is for Red Hat customers running RHV (formerly RHEV). + + The rhhi_virt profile is for Red Hat customers running RHV in a + hyper-converged setup and enables gluster collections. + """ cluster_name = 'Community oVirt' packages = ('ovirt-engine',) @@ -32,11 +59,11 @@ class ovirt(Cluster): def _run_db_query(self, query): ''' - Wrapper for running DB queries on the master. Any scrubbing of the + Wrapper for running DB queries on the manager. Any scrubbing of the query should be done _before_ passing the query to this method. ''' cmd = "%s %s" % (self.db_exec, quote(query)) - return self.exec_master_cmd(cmd, need_root=True) + return self.exec_primary_cmd(cmd, need_root=True) def _sql_scrub(self, val): ''' @@ -62,10 +89,10 @@ class ovirt(Cluster): This only runs if we're locally on the RHV-M, *and* if no ssh-keys are called out on the command line, *and* no --password option is given. ''' - if self.master.local: + if self.primary.local: if not any([self.opts.ssh_key, self.opts.password, self.opts.password_per_node]): - if self.master.file_exists(ENGINE_KEY): + if self.primary.file_exists(ENGINE_KEY): self.add_default_ssh_key(ENGINE_KEY) self.log_debug("Found engine SSH key. User command line" " does not specify a key or password, using" @@ -98,7 +125,7 @@ class ovirt(Cluster): return [] res = self._run_db_query(self.dbquery) if res['status'] == 0: - nodes = res['stdout'].splitlines()[2:-1] + nodes = res['output'].splitlines()[2:-1] return [n.split('(')[0].strip() for n in nodes] else: raise Exception('database query failed, return code: %s' @@ -112,9 +139,9 @@ class ovirt(Cluster): def parse_db_conf(self): conf = {} engconf = '/etc/ovirt-engine/engine.conf.d/10-setup-database.conf' - res = self.exec_master_cmd('cat %s' % engconf, need_root=True) + res = self.exec_primary_cmd('cat %s' % engconf, need_root=True) if res['status'] == 0: - config = res['stdout'].splitlines() + config = res['output'].splitlines() for line in config: try: k = str(line.split('=')[0]) @@ -140,12 +167,12 @@ class ovirt(Cluster): cmd = ('PGPASSWORD={} /usr/sbin/sosreport --name=postgresql ' '--batch -o postgresql {}' ).format(self.conf['ENGINE_DB_PASSWORD'], sos_opt) - db_sos = self.exec_master_cmd(cmd, need_root=True) - for line in db_sos['stdout'].splitlines(): + db_sos = self.exec_primary_cmd(cmd, need_root=True) + for line in db_sos['output'].splitlines(): if fnmatch.fnmatch(line, '*sosreport-*tar*'): _pg_dump = line.strip() - self.master.manifest.add_field('postgresql_dump', - _pg_dump.split('/')[-1]) + self.primary.manifest.add_field('postgresql_dump', + _pg_dump.split('/')[-1]) return _pg_dump self.log_error('Failed to gather database dump') return False @@ -158,7 +185,7 @@ class rhv(ovirt): sos_preset = 'rhv' def set_node_label(self, node): - if node.address == self.master.address: + if node.address == self.primary.address: return 'manager' if node.is_installed('ovirt-node-ng-nodectl'): return 'rhvh' @@ -174,11 +201,13 @@ class rhhi_virt(rhv): sos_preset = 'rhv' def check_enabled(self): - return (self.master.is_installed('rhvm') and self._check_for_rhhiv()) + return (self.primary.is_installed('rhvm') and self._check_for_rhhiv()) def _check_for_rhhiv(self): ret = self._run_db_query('SELECT count(server_id) FROM gluster_server') if ret['status'] == 0: # if there are any entries in this table, RHHI-V is in use - return ret['stdout'].splitlines()[2].strip() != '0' + return ret['output'].splitlines()[2].strip() != '0' return False + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/collector/clusters/pacemaker.py 4.5.3ubuntu2/sos/collector/clusters/pacemaker.py --- 4.0-2/sos/collector/clusters/pacemaker.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/clusters/pacemaker.py 2023-04-28 17:16:21.000000000 +0000 @@ -8,7 +8,11 @@ # # See the LICENSE file in the source distribution for further information. +import re + from sos.collector.clusters import Cluster +from sos.utilities import parse_version +from xml.etree import ElementTree class pacemaker(Cluster): @@ -16,42 +20,83 @@ class pacemaker(Cluster): cluster_name = 'Pacemaker High Availability Cluster Manager' sos_plugins = ['pacemaker'] packages = ('pacemaker',) + strict_node_list = True option_list = [ ('online', True, 'Collect nodes listed as online'), - ('offline', True, 'Collect nodes listed as offline') + ('offline', True, 'Collect nodes listed as offline'), + ('only-corosync', False, 'Only use corosync.conf to enumerate nodes') ] def get_nodes(self): - self.res = self.exec_master_cmd('pcs status') - if self.res['status'] != 0: - self.log_error('Cluster status could not be determined. Is the ' - 'cluster running on this node?') - return [] - if 'node names do not match' in self.res['stdout']: - self.log_warn('Warning: node name mismatch reported. Attempts to ' - 'connect to some nodes may fail.\n') - return self.parse_pcs_output() - - def parse_pcs_output(self): - nodes = [] - if self.get_option('online'): - nodes += self.get_online_nodes() - if self.get_option('offline'): - nodes += self.get_offline_nodes() - return nodes - - def get_online_nodes(self): - for line in self.res['stdout'].splitlines(): - if line.startswith('Online:'): - nodes = line.split('[')[1].split(']')[0] - return [n for n in nodes.split(' ') if n] - - def get_offline_nodes(self): - offline = [] - for line in self.res['stdout'].splitlines(): - if line.startswith('Node') and line.endswith('(offline)'): - offline.append(line.split()[1].replace(':', '')) - if line.startswith('OFFLINE:'): - nodes = line.split('[')[1].split(']')[0] - offline.extend([n for n in nodes.split(' ') if n]) - return offline + self.nodes = [] + # try crm_mon first + try: + if not self.get_option('only-corosync'): + try: + self.get_nodes_from_crm() + except Exception as err: + self.log_warn("Falling back to sourcing corosync.conf. " + "Could not parse crm_mon output: %s" % err) + if not self.nodes: + # fallback to corosync.conf, in case the node we're inspecting + # is offline from the cluster + self.get_nodes_from_corosync() + except Exception as err: + self.log_error("Could not determine nodes from cluster: %s" % err) + + _shorts = [n for n in self.nodes if '.' not in n] + if _shorts: + self.log_warn( + "WARNING: Node addresses '%s' may not resolve locally if you " + "are not running on a node in the cluster. Try using option " + "'-c pacemaker.only-corosync' if these connections fail." + % ','.join(_shorts) + ) + return self.nodes + + def get_nodes_from_crm(self): + """ + Try to parse crm_mon output for node list and status. + """ + xmlopt = '--output-as=xml' + # older pacemaker had a different option for xml output + _ver = self.exec_primary_cmd('crm_mon --version') + if _ver['status'] == 0: + cver = _ver['output'].split()[1].split('-')[0] + if not parse_version(cver) > parse_version('2.0.3'): + xmlopt = '--as-xml' + else: + return + _out = self.exec_primary_cmd( + "crm_mon --one-shot --inactive %s" % xmlopt, + need_root=True + ) + if _out['status'] == 0: + self.parse_crm_xml(_out['output']) + + def parse_crm_xml(self, xmlstring): + """ + Parse the xml output string provided by crm_mon + """ + _xml = ElementTree.fromstring(xmlstring) + nodes = _xml.find('nodes') + for node in nodes: + _node = node.attrib + if self.get_option('online') and _node['online'] == 'true': + self.nodes.append(_node['name']) + elif self.get_option('offline') and _node['online'] == 'false': + self.nodes.append(_node['name']) + + def get_nodes_from_corosync(self): + """ + As a fallback measure, read corosync.conf to get the node list. Note + that this prevents us from separating online nodes from offline nodes. + """ + self.log_warn("WARNING: unable to distinguish online nodes from " + "offline nodes when sourcing from corosync.conf") + cc = self.primary.read_file('/etc/corosync/corosync.conf') + nodes = re.findall(r'((\sring0_addr:)(.*))', cc) + for node in nodes: + self.nodes.append(node[-1].strip()) + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/collector/clusters/satellite.py 4.5.3ubuntu2/sos/collector/clusters/satellite.py --- 4.0-2/sos/collector/clusters/satellite.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/clusters/satellite.py 2023-04-28 17:16:21.000000000 +0000 @@ -13,7 +13,14 @@ from sos.collector.clusters import Clust class satellite(Cluster): - """Red Hat Satellite 6""" + """ + This profile is specifically for Red Hat Satellite 6, and not earlier + releases of Satellite. + + While note technically a 'cluster' in the traditional sense, Satellite + does provide for 'capsule' nodes which is what this profile aims to + enumerate beyond the 'primary' Satellite system. + """ cluster_name = 'Red Hat Satellite 6' packages = ('satellite', 'satellite-installer') @@ -25,16 +32,18 @@ class satellite(Cluster): def get_nodes(self): cmd = self._psql_cmd('copy (select name from smart_proxies) to stdout') - res = self.exec_master_cmd(cmd, need_root=True) + res = self.exec_primary_cmd(cmd, need_root=True) if res['status'] == 0: nodes = [ - n.strip() for n in res['stdout'].splitlines() + n.strip() for n in res['output'].splitlines() if 'could not change directory' not in n ] return nodes return [] def set_node_label(self, node): - if node.address == self.master.address: + if node.address == self.primary.address: return 'satellite' return 'capsule' + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/collector/exceptions.py 4.5.3ubuntu2/sos/collector/exceptions.py --- 4.0-2/sos/collector/exceptions.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/exceptions.py 2023-04-28 17:16:21.000000000 +0000 @@ -94,6 +94,35 @@ class UnsupportedHostException(Exception super(UnsupportedHostException, self).__init__(message) +class InvalidTransportException(Exception): + """Raised when a transport is requested but it does not exist or is + not supported locally""" + + def __init__(self, transport=None): + message = ("Connection failed: unknown or unsupported transport %s" + % transport if transport else '') + super(InvalidTransportException, self).__init__(message) + + +class SaltStackMasterUnsupportedException(Exception): + """Raised when SaltStack Master is unsupported locally""" + + def __init__(self): + message = 'Master unsupported by local SaltStack installation' + super(SaltStackMasterUnsupportedException, self).__init__(message) + + +class JujuNotInstalledException(Exception): + """Raised when juju is not installed locally""" + + def __init__(self): + message = ( + 'Juju is not installed, ' + 'please ensure you have installed juju.' + ) + super(JujuNotInstalledException, self).__init__(message) + + __all__ = [ 'AuthPermissionDeniedException', 'CommandTimeoutException', @@ -103,6 +132,9 @@ __all__ = [ 'ControlSocketMissingException', 'InvalidPasswordException', 'PasswordRequestException', + 'SaltStackMasterUnsupportedException', 'TimeoutPasswordAuthException', - 'UnsupportedHostException' + 'UnsupportedHostException', + 'InvalidTransportException', + 'JujuNotInstalledException' ] diff -pruN 4.0-2/sos/collector/sosnode.py 4.5.3ubuntu2/sos/collector/sosnode.py --- 4.0-2/sos/collector/sosnode.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/sosnode.py 2023-04-28 17:16:21.000000000 +0000 @@ -12,22 +12,29 @@ import fnmatch import inspect import logging import os -import pexpect import re -import shutil -from distutils.version import LooseVersion from pipes import quote -from sos.policies import load, InitSystem -from sos.collector.exceptions import (InvalidPasswordException, - TimeoutPasswordAuthException, - PasswordRequestException, - AuthPermissionDeniedException, +from sos.policies import load +from sos.policies.init_systems import InitSystem +from sos.collector.transports.juju import JujuSSH +from sos.collector.transports.control_persist import SSHControlPersist +from sos.collector.transports.local import LocalTransport +from sos.collector.transports.oc import OCTransport +from sos.collector.transports.saltstack import SaltStackMaster +from sos.collector.exceptions import (CommandTimeoutException, ConnectionException, - CommandTimeoutException, - ConnectionTimeoutException, - ControlSocketMissingException, - UnsupportedHostException) + UnsupportedHostException, + InvalidTransportException) +from sos.utilities import parse_version + +TRANSPORTS = { + 'local': LocalTransport, + 'control_persist': SSHControlPersist, + 'oc': OCTransport, + 'saltstack': SaltStackMaster, + 'juju': JujuSSH, +} class SosNode(): @@ -36,14 +43,18 @@ class SosNode(): load_facts=True): self.address = address.strip() self.commons = commons - self.opts = commons['opts'] + self.opts = commons['cmdlineopts'] + self._assign_config_opts() self.tmpdir = commons['tmpdir'] self.hostlen = commons['hostlen'] self.need_sudo = commons['need_sudo'] + self.sos_options = commons['sos_options'] self.local = False self.host = None self.cluster = None self.hostname = None + self.sos_env_vars = {} + self._env_vars = {} self._password = password or self.opts.password if not self.opts.nopasswd_sudo and not self.opts.sudo_pw: self.opts.sudo_pw = self._password @@ -62,34 +73,30 @@ class SosNode(): 'presets': [], 'sos_cmd': commons['sos_cmd'] } - filt = ['localhost', '127.0.0.1'] + self.sos_bin = 'sosreport' self.soslog = logging.getLogger('sos') self.ui_log = logging.getLogger('sos_ui') - self.control_path = ("%s/.sos-collector-%s" - % (self.tmpdir, self.address)) - self.ssh_cmd = self._create_ssh_command() - if self.address not in filt: - try: - self.connected = self._create_ssh_session() - except Exception as err: - self.log_error('Unable to open SSH session: %s' % err) - raise - else: - self.connected = True - self.local = True - self.need_sudo = os.getuid() != 0 + self._transport = self._load_remote_transport(commons) + # Overwrite need_sudo if transports default_user + # is set and is not root. + if self._transport.default_user: + self.need_sudo = self._transport.default_user != 'root' + try: + self._transport.connect(self._password) + except Exception as err: + self.log_error('Unable to open remote session: %s' % err) + raise # load the host policy now, even if we don't want to load further # host information. This is necessary if we're running locally on the - # cluster master but do not want a local report as we still need to do + # cluster primary but do not want a local report as we still need to do # package checks in that instance self.host = self.determine_host_policy() - self.get_hostname() + self.hostname = self._transport.hostname if self.local and self.opts.no_local: load_facts = False if self.connected and load_facts: if not self.host: - self.connected = False - self.close_ssh_session() + self._transport.disconnect() return None if self.local: if self.check_in_container(): @@ -98,15 +105,53 @@ class SosNode(): self.create_sos_container() self._load_sos_info() - def _create_ssh_command(self): - """Build the complete ssh command for this node""" - cmd = "ssh -oControlPath=%s " % self.control_path - cmd += "%s@%s " % (self.opts.ssh_user, self.address) - return cmd + @property + def connected(self): + if self._transport: + return self._transport.connected + # if no transport, we're running locally + return True + + def disconnect(self): + """Wrapper to close the remote session via our transport agent + """ + self._transport.disconnect() + + def _load_remote_transport(self, commons): + """Determine the type of remote transport to load for this node, then + return an instantiated instance of that transport + """ + if self.address in ['localhost', '127.0.0.1']: + self.local = True + return LocalTransport(self.address, commons) + elif self.opts.transport in TRANSPORTS.keys(): + return TRANSPORTS[self.opts.transport](self.address, commons) + elif self.opts.transport != 'auto': + self.log_error( + "Connection failed: unknown or unsupported transport %s" + % self.opts.transport + ) + raise InvalidTransportException(self.opts.transport) + return SSHControlPersist(self.address, commons) def _fmt_msg(self, msg): return '{:<{}} : {}'.format(self._hostname, self.hostlen + 1, msg) + @property + def env_vars(self): + if not self._env_vars: + if self.local: + self._env_vars = os.environ.copy() + else: + ret = self.run_command("env --null") + if ret['status'] == 0: + for ln in ret['output'].split('\x00'): + if not ln: + continue + _val = ln.split('=') + self._env_vars[_val[0]] = _val[1] + return self._env_vars + def set_node_manifest(self, manifest): """Set the manifest section that this node will write to """ @@ -114,6 +159,8 @@ class SosNode(): self.manifest.add_field('hostname', self._hostname) self.manifest.add_field('policy', self.host.distro) self.manifest.add_field('sos_version', self.sos_info['version']) + self.manifest.add_field('final_sos_command', '') + self.manifest.add_field('transport', self._transport.name) def check_in_container(self): """ @@ -131,9 +178,27 @@ class SosNode(): """If the host is containerized, create the container we'll be using """ if self.host.containerized: - res = self.run_command(self.host.create_sos_container(), - need_root=True) - if res['status'] in [0, 125]: # 125 means container exists + cmd = self.host.create_sos_container( + image=self.opts.image, + auth=self.get_container_auth(), + force_pull=self.opts.force_pull_image + ) + res = self.run_command(cmd, need_root=True) + if res['status'] in [0, 125]: + if res['status'] == 125: + if 'unable to retrieve auth token' in res['output']: + self.log_error( + "Could not pull image. Provide either a username " + "and password or authfile" + ) + raise Exception + elif 'unknown: Not found' in res['output']: + self.log_error('Specified image not found on registry') + raise Exception + # 'name exists' with code 125 means the container was + # created successfully, so ignore it. + # initial creations leads to an exited container, restarting it + # here will keep it alive for us to exec through ret = self.run_command(self.host.restart_sos_container(), need_root=True) if ret['status'] == 0: @@ -142,27 +207,34 @@ class SosNode(): return True else: self.log_error("Could not start container after create: %s" - % ret['stdout']) + % ret['output']) raise Exception else: self.log_error("Could not create container on host: %s" - % res['stdout']) + % res['output']) raise Exception - def file_exists(self, fname): + def get_container_auth(self): + """Determine what the auth string should be to pull the image used to + deploy our temporary container + """ + if self.opts.registry_user: + return self.host.runtimes['default'].fmt_registry_credentials( + self.opts.registry_user, + self.opts.registry_password + ) + else: + return self.host.runtimes['default'].fmt_registry_authfile( + self.opts.registry_authfile or self.host.container_authfile + ) + + def file_exists(self, fname, need_root=False): """Checks for the presence of fname on the remote node""" - if not self.local: - try: - res = self.run_command("stat %s" % fname) - return res['status'] == 0 - except Exception: - return False - else: - try: - os.stat(fname) - return True - except Exception: - return False + try: + res = self.run_command("stat %s" % fname, need_root=need_root) + return res['status'] == 0 + except Exception: + return False @property def _hostname(self): @@ -170,18 +242,6 @@ class SosNode(): return self.hostname return self.address - @property - def control_socket_exists(self): - """Check if the SSH control socket exists - - The control socket is automatically removed by the SSH daemon in the - event that the last connection to the node was greater than the timeout - set by the ControlPersist option. This can happen for us if we are - collecting from a large number of nodes, and the timeout expires before - we start collection. - """ - return os.path.exists(self.control_path) - def _sanitize_log_msg(self, msg): """Attempts to obfuscate sensitive information in log messages such as passwords""" @@ -211,12 +271,6 @@ class SosNode(): msg = '[%s:%s] %s' % (self._hostname, caller, msg) self.soslog.debug(msg) - def get_hostname(self): - """Get the node's hostname""" - sout = self.run_command('hostname') - self.hostname = sout['stdout'].strip() - self.log_info('Hostname set to %s' % self.hostname) - def _format_cmd(self, cmd): """If we need to provide a sudo or root password to a command, then here we prefix the command with the correct bits @@ -227,52 +281,62 @@ class SosNode(): return "sudo -S %s" % cmd return cmd - def _fmt_output(self, output=None, rc=0): - """Formats the returned output from a command into a dict""" - if rc == 0: - stdout = output - stderr = '' - else: - stdout = '' - stderr = output - res = {'status': rc, - 'stdout': stdout, - 'stderr': stderr} - return res - def _load_sos_info(self): """Queries the node for information about the installed version of sos """ + ver = None + rel = None if self.host.container_version_command is None: pkg = self.host.package_manager.pkg_version(self.host.sos_pkg_name) if pkg is not None: ver = '.'.join(pkg['version']) - self.sos_info['version'] = ver + if pkg['release']: + rel = pkg['release'] + else: # use the containerized policy's command pkgs = self.run_command(self.host.container_version_command, use_container=True, need_root=True) - ver = pkgs['stdout'].strip().split('-')[1] - if ver: - self.sos_info['version'] = ver - if 'version' in self.sos_info: + if pkgs['status'] == 0: + _, ver, rel = pkgs['output'].strip().split('-') + + if ver: + if len(ver.split('.')) == 2: + # safeguard against maintenance releases throwing off the + # comparison by parse_version + ver += '.0' + try: + ver += '-%s' % rel.split('.')[0] + except Exception as err: + self.log_debug("Unable to fully parse sos release: %s" % err) + + self.sos_info['version'] = ver + + if self.sos_info['version']: self.log_info('sos version is %s' % self.sos_info['version']) else: - self.log_error('sos is not installed on this node') + if not self.address == self.opts.primary: + # in the case where the 'primary' enumerates nodes but is not + # intended for collection (bastions), don't worry about sos not + # being present + self.log_error('sos is not installed on this node') self.connected = False return False - cmd = 'sosreport -l' - sosinfo = self.run_command(cmd, use_container=True) + # sos-4.0 changes the binary + if self.check_sos_version('4.0'): + self.sos_bin = 'sos report' + cmd = "%s -l" % self.sos_bin + sosinfo = self.run_command(cmd, use_container=True, need_root=True) if sosinfo['status'] == 0: - self._load_sos_plugins(sosinfo['stdout']) + self._load_sos_plugins(sosinfo['output']) if self.check_sos_version('3.6'): self._load_sos_presets() def _load_sos_presets(self): - cmd = 'sosreport --list-presets' - res = self.run_command(cmd, use_container=True) + cmd = '%s --list-presets' % self.sos_bin + res = self.run_command(cmd, use_container=True, need_root=True) if res['status'] == 0: - for line in res['stdout'].splitlines(): + for line in res['output'].splitlines(): if line.strip().startswith('name:'): pname = line.split('name:')[1].strip() self.sos_info['presets'].append(pname) @@ -280,11 +344,12 @@ class SosNode(): def _load_sos_plugins(self, sosinfo): ENABLED = 'The following plugins are currently enabled:' DISABLED = 'The following plugins are currently disabled:' + ALL_OPTIONS = 'The following options are available for ALL plugins:' OPTIONS = 'The following plugin options are available:' PROFILES = 'Profiles:' enablereg = ENABLED + '(.*?)' + DISABLED - disreg = DISABLED + '(.*?)' + OPTIONS + disreg = DISABLED + '(.*?)' + ALL_OPTIONS optreg = OPTIONS + '(.*?)' + PROFILES proreg = PROFILES + '(.*?)' + '\n\n' @@ -311,21 +376,7 @@ class SosNode(): """Reads the specified file and returns the contents""" try: self.log_info("Reading file %s" % to_read) - if not self.local: - res = self.run_command("cat %s" % to_read, timeout=5) - if res['status'] == 0: - return res['stdout'] - else: - if 'No such file' in res['stdout']: - self.log_debug("File %s does not exist on node" - % to_read) - else: - self.log_error("Error reading %s: %s" % - (to_read, res['stdout'].split(':')[1:])) - return '' - else: - with open(to_read, 'r') as rfile: - return rfile.read() + return self._transport.read_file(to_read) except Exception as err: self.log_error("Exception while reading %s: %s" % (to_read, err)) return '' @@ -339,7 +390,8 @@ class SosNode(): % self.commons['policy'].distro) return self.commons['policy'] host = load(cache={}, sysroot=self.opts.sysroot, init=InitSystem(), - probe_runtime=False, remote_exec=self.ssh_cmd, + probe_runtime=True, + remote_exec=self._transport.run_command, remote_check=self.read_file('/etc/os-release')) if host: self.log_info("loaded policy %s for host" % host.distro) @@ -351,8 +403,37 @@ class SosNode(): """Checks to see if the sos installation on the node is AT LEAST the given ver. This means that if the installed version is greater than ver, this will still return True + + :param ver: Version number we are trying to verify is installed + :type ver: ``str`` + + :returns: True if installed version is at least ``ver``, else False + :rtype: ``bool`` """ - return LooseVersion(self.sos_info['version']) >= ver + def _format_version(ver): + # format the version we're checking to a standard form of X.Y.Z-R + try: + _fver = ver.split('-')[0] + _rel = '' + if '-' in ver: + _rel = '-' + ver.split('-')[-1].split('.')[0] + if len(_fver.split('.')) == 2: + _fver += '.0' + + return _fver + _rel + except Exception as err: + self.log_debug("Unable to format '%s': %s" % (ver, err)) + return ver + + _ver = _format_version(ver) + + try: + _node_ver = parse_version(self.sos_info['version']) + _test_ver = parse_version(_ver) + return _node_ver >= _test_ver + except Exception as err: + self.log_error("Error checking sos version: %s" % err) + return False def is_installed(self, pkg): """Checks if a given package is installed on the node""" @@ -361,7 +442,7 @@ class SosNode(): return self.host.package_manager.pkg_by_name(pkg) is not None def run_command(self, cmd, timeout=180, get_pty=False, need_root=False, - force_local=False, use_container=False): + use_container=False, env=None): """Runs a given cmd, either via the SSH session or locally Arguments: @@ -372,60 +453,35 @@ class SosNode(): need_root - if a command requires root privileges, setting this to True tells sos-collector to format the command with sudo or su - as appropriate and to input the password - force_local - force a command to run locally. Mainly used for scp. use_container - Run this command in a container *IF* the host is containerized """ - if not self.control_socket_exists and not self.local: - self.log_debug('Control socket does not exist, attempting to ' - 're-create') + if not self.connected and not self.local: + self.log_debug('Node is disconnected, attempting to reconnect') try: - _sock = self._create_ssh_session() - if not _sock: - self.log_debug('Failed to re-create control socket') - raise ControlSocketMissingException + reconnected = self._transport.reconnect(self._password) + if not reconnected: + self.log_debug('Failed to reconnect to node') + raise ConnectionException except Exception as err: - self.log_error('Cannot run command: control socket does not ' - 'exist') - self.log_debug("Error while trying to create new SSH control " - "socket: %s" % err) + self.log_debug("Error while trying to reconnect: %s" % err) raise - if cmd.startswith('sosreport'): - cmd = cmd.replace('sosreport', self.host.sos_bin_path) - need_root = True if use_container and self.host.containerized: cmd = self.host.format_container_command(cmd) if need_root: - get_pty = True cmd = self._format_cmd(cmd) - self.log_debug('Running command %s' % cmd) + if 'atomic' in cmd: get_pty = True - if not self.local and not force_local: - cmd = "%s %s" % (self.ssh_cmd, quote(cmd)) - else: - if get_pty: - cmd = "/bin/bash -c %s" % quote(cmd) - res = pexpect.spawn(cmd, encoding='utf-8') - if need_root: - if self.need_sudo: - res.sendline(self.opts.sudo_pw) - if self.opts.become_root: - res.sendline(self.opts.root_password) - output = res.expect([pexpect.EOF, pexpect.TIMEOUT], - timeout=timeout) - if output == 0: - out = res.before - res.close() - rc = res.exitstatus - return {'status': rc, 'stdout': out} - elif output == 1: - raise CommandTimeoutException(cmd) + + if env: + _cmd_env = self.env_vars + env = _cmd_env.update(env) + return self._transport.run_command(cmd, timeout, need_root, env, + get_pty) def sosreport(self): - """Run a sosreport on the node, then collect it""" - self.sos_cmd = self.finalize_sos_cmd() - self.log_info('Final sos command set to %s' % self.sos_cmd) + """Run an sos report on the node, then collect it""" try: path = self.execute_sos_command() if path: @@ -438,109 +494,6 @@ class SosNode(): pass self.cleanup() - def _create_ssh_session(self): - """ - Using ControlPersist, create the initial connection to the node. - - This will generate an OpenSSH ControlPersist socket within the tmp - directory created or specified for sos-collector to use. - - At most, we will wait 30 seconds for a connection. This involves a 15 - second wait for the initial connection attempt, and a subsequent 15 - second wait for a response when we supply a password. - - Since we connect to nodes in parallel (using the --threads value), this - means that the time between 'Connecting to nodes...' and 'Beginning - collection of sosreports' that users see can be up to an amount of time - equal to 30*(num_nodes/threads) seconds. - - Returns - True if session is successfully opened, else raise Exception - """ - # Don't use self.ssh_cmd here as we need to add a few additional - # parameters to establish the initial connection - self.log_info('Opening SSH session to create control socket') - connected = False - ssh_key = '' - ssh_port = '' - if self.opts.ssh_port != 22: - ssh_port = "-p%s " % self.opts.ssh_port - if self.opts.ssh_key: - ssh_key = "-i%s" % self.opts.ssh_key - cmd = ("ssh %s %s -oControlPersist=600 -oControlMaster=auto " - "-oStrictHostKeyChecking=no -oControlPath=%s %s@%s " - "\"echo Connected\"" % (ssh_key, - ssh_port, - self.control_path, - self.opts.ssh_user, - self.address)) - res = pexpect.spawn(cmd, encoding='utf-8') - - connect_expects = [ - u'Connected', - u'password:', - u'.*Permission denied.*', - u'.* port .*: No route to host', - u'.*Could not resolve hostname.*', - pexpect.TIMEOUT - ] - - index = res.expect(connect_expects, timeout=15) - - if index == 0: - connected = True - elif index == 1: - if self._password: - pass_expects = [ - u'Connected', - u'Permission denied, please try again.', - pexpect.TIMEOUT - ] - res.sendline(self._password) - pass_index = res.expect(pass_expects, timeout=15) - if pass_index == 0: - connected = True - elif pass_index == 1: - # Note that we do not get an exitstatus here, so matching - # this line means an invalid password will be reported for - # both invalid passwords and invalid user names - raise InvalidPasswordException - elif pass_index == 2: - raise TimeoutPasswordAuthException - else: - raise PasswordRequestException - elif index == 2: - raise AuthPermissionDeniedException - elif index == 3: - raise ConnectionException(self.address, self.opts.ssh_port) - elif index == 4: - raise ConnectionException(self.address) - elif index == 5: - raise ConnectionTimeoutException - else: - raise Exception("Unknown error, client returned %s" % res.before) - if connected: - self.log_debug("Successfully created control socket at %s" - % self.control_path) - return True - return False - - def close_ssh_session(self): - """Remove the control socket to effectively terminate the session""" - if self.local: - return True - try: - res = self.run_command("rm -f %s" % self.control_path, - force_local=True) - if res['status'] == 0: - return True - self.log_error("Could not remove ControlPath %s: %s" - % (self.control_path, res['stdout'])) - return False - except Exception as e: - self.log_error('Error closing SSH session: %s' % e) - return False - def _preset_exists(self, preset): """Verifies if the given preset exists on the node""" return preset in self.sos_info['presets'] @@ -587,31 +540,56 @@ class SosNode(): self.cluster = cluster def update_cmd_from_cluster(self): - """This is used to modify the sosreport command run on the nodes. - By default, sosreport is run without any options, using this will + """This is used to modify the sos report command run on the nodes. + By default, sos report is run without any options, using this will allow the profile to specify what plugins to run or not and what options to use. This will NOT override user supplied options. """ if self.cluster.sos_preset: - if not self.opts.preset: - self.opts.preset = self.cluster.sos_preset + if not self.preset: + self.preset = self.cluster.sos_preset else: self.log_info('Cluster specified preset %s but user has also ' 'defined a preset. Using user specification.' % self.cluster.sos_preset) if self.cluster.sos_plugins: for plug in self.cluster.sos_plugins: - if plug not in self.opts.enable_plugins: - self.opts.enable_plugins.append(plug) + if plug not in self.enable_plugins: + self.enable_plugins.append(plug) + + if self.cluster.sos_options: + for opt in self.cluster.sos_options: + # take the user specification over any cluster defaults + if opt not in self.sos_options: + self.sos_options[opt] = self.cluster.sos_options[opt] if self.cluster.sos_plugin_options: for opt in self.cluster.sos_plugin_options: - if not any(opt in o for o in self.opts.plugin_options): + if not any(opt in o for o in self.plugopts): option = '%s=%s' % (opt, self.cluster.sos_plugin_options[opt]) - self.opts.plugin_options.append(option) + self.plugopts.append(option) + + # set primary-only options + if self.cluster.check_node_is_primary(self): + with self.cluster.lock: + self.cluster.set_primary_options(self) + else: + with self.cluster.lock: + self.cluster.set_node_options(self) + + def _assign_config_opts(self): + """From the global opts configuration, assign those values locally + to this node so that they may be acted on individually. + """ + # assign these to new, private copies + self.only_plugins = list(self.opts.only_plugins) + self.skip_plugins = list(self.opts.skip_plugins) + self.enable_plugins = list(self.opts.enable_plugins) + self.plugopts = list(self.opts.plugopts) + self.preset = list(self.opts.preset) def finalize_sos_cmd(self): """Use host facts and compare to the cluster type to modify the sos @@ -619,10 +597,7 @@ class SosNode(): sos_cmd = self.sos_info['sos_cmd'] label = self.determine_sos_label() if label: - self.sos_cmd = '%s %s ' % (sos_cmd, quote(label)) - - if self.opts.sos_opt_line: - return '%s %s' % (sos_cmd, self.opts.sos_opt_line) + sos_cmd = '%s %s ' % (sos_cmd, quote(label)) sos_opts = [] @@ -649,63 +624,108 @@ class SosNode(): if self.opts.since: sos_opts.append('--since=%s' % quote(self.opts.since)) - if self.opts.only_plugins: - plugs = [o for o in self.opts.only_plugins - if self._plugin_exists(o)] - if len(plugs) != len(self.opts.only_plugins): - not_only = list(set(self.opts.only_plugins) - set(plugs)) + if self.check_sos_version('4.1'): + if self.opts.skip_commands: + sos_opts.append( + '--skip-commands=%s' % ( + quote(','.join(self.opts.skip_commands))) + ) + if self.opts.skip_files: + sos_opts.append( + '--skip-files=%s' % (quote(','.join(self.opts.skip_files))) + ) + + if self.check_sos_version('4.2'): + if self.opts.cmd_timeout: + sos_opts.append('--cmd-timeout=%s' + % quote(str(self.opts.cmd_timeout))) + + # handle downstream versions that backported this option + if self.check_sos_version('4.3') or self.check_sos_version('4.2-13'): + if self.opts.container_runtime != 'auto': + sos_opts.append( + "--container-runtime=%s" % self.opts.container_runtime + ) + if self.opts.namespaces: + sos_opts.append( + "--namespaces=%s" % self.opts.namespaces + ) + + if self.check_sos_version('4.5.2'): + if self.opts.journal_size: + sos_opts.append(f"--journal-size={self.opts.journal_size}") + if self.opts.low_priority: + sos_opts.append('--low-priority') + + self.update_cmd_from_cluster() + + sos_cmd = sos_cmd.replace( + 'sosreport', + os.path.join(self.host.sos_bin_path, self.sos_bin) + ) + + for opt in self.sos_options: + _val = self.sos_options[opt] + sos_opts.append(f"--{opt} {_val if _val else ''}") + + if self.plugopts: + opts = [o for o in self.plugopts + if self._plugin_exists(o.split('.')[0]) + and self._plugin_option_exists(o.split('=')[0])] + if opts: + sos_opts.append('-k %s' % quote(','.join(o for o in opts))) + + if self.preset: + if self._preset_exists(self.preset): + sos_opts.append('--preset=%s' % quote(self.preset)) + else: + self.log_debug('Requested to enable preset %s but preset does ' + 'not exist on node' % self.preset) + + if self.only_plugins: + plugs = [o for o in self.only_plugins if self._plugin_exists(o)] + if len(plugs) != len(self.only_plugins): + not_only = list(set(self.only_plugins) - set(plugs)) self.log_debug('Requested plugins %s were requested to be ' 'enabled but do not exist' % not_only) - only = self._fmt_sos_opt_list(self.opts.only_plugins) + only = self._fmt_sos_opt_list(self.only_plugins) if only: sos_opts.append('--only-plugins=%s' % quote(only)) - return "%s %s" % (sos_cmd, ' '.join(sos_opts)) + self.sos_cmd = "%s %s" % (sos_cmd, ' '.join(sos_opts)) + self.log_info('Final sos command set to %s' % self.sos_cmd) + self.manifest.add_field('final_sos_command', self.sos_cmd) + return - if self.opts.skip_plugins: + if self.skip_plugins: # only run skip-plugins for plugins that are enabled - skip = [o for o in self.opts.skip_plugins - if self._check_enabled(o)] - if len(skip) != len(self.opts.skip_plugins): - not_skip = list(set(self.opts.skip_plugins) - set(skip)) + skip = [o for o in self.skip_plugins if self._check_enabled(o)] + if len(skip) != len(self.skip_plugins): + not_skip = list(set(self.skip_plugins) - set(skip)) self.log_debug('Requested to skip plugins %s, but plugins are ' 'already not enabled' % not_skip) skipln = self._fmt_sos_opt_list(skip) if skipln: sos_opts.append('--skip-plugins=%s' % quote(skipln)) - if self.opts.enable_plugins: + if self.enable_plugins: # only run enable for plugins that are disabled - opts = [o for o in self.opts.enable_plugins - if o not in self.opts.skip_plugins + opts = [o for o in self.enable_plugins + if o not in self.skip_plugins and self._check_disabled(o) and self._plugin_exists(o)] - if len(opts) != len(self.opts.enable_plugins): - not_on = list(set(self.opts.enable_plugins) - set(opts)) + if len(opts) != len(self.enable_plugins): + not_on = list(set(self.enable_plugins) - set(opts)) self.log_debug('Requested to enable plugins %s, but plugins ' 'are already enabled or do not exist' % not_on) enable = self._fmt_sos_opt_list(opts) if enable: sos_opts.append('--enable-plugins=%s' % quote(enable)) - if self.opts.plugin_options: - opts = [o for o in self.opts.plugin_options - if self._plugin_exists(o.split('.')[0]) - and self._plugin_option_exists(o.split('=')[0])] - if opts: - sos_opts.append('-k %s' % quote(','.join(o for o in opts))) - - if self.opts.preset: - if self._preset_exists(self.opts.preset): - sos_opts.append('--preset=%s' % quote(self.opts.preset)) - else: - self.log_debug('Requested to enable preset %s but preset does ' - 'not exist on node' % self.opts.preset) - - _sos_cmd = "%s %s" % (sos_cmd, ' '.join(sos_opts)) - self.manifest.add_field('final_sos_command', _sos_cmd) - return _sos_cmd + self.sos_cmd = "%s %s" % (sos_cmd, ' '.join(sos_opts)) + self.log_info('Final sos command set to %s' % self.sos_cmd) + self.manifest.add_field('final_sos_command', self.sos_cmd) def determine_sos_label(self): - """Determine what, if any, label should be added to the sosreport""" + """Determine what, if any, label should be added to the sos report""" label = '' label += self.cluster.get_node_label(self) @@ -716,7 +736,7 @@ class SosNode(): if not label: return None - self.log_debug('Label for sosreport set to %s' % label) + self.log_debug('Label for sos report set to %s' % label) if self.check_sos_version('3.6'): lcmd = '--label' else: @@ -738,70 +758,73 @@ class SosNode(): def determine_sos_error(self, rc, stdout): if rc == -1: - return 'sosreport process received SIGKILL on node' + return 'sos report process received SIGKILL on node' if rc == 1: if 'sudo' in stdout: return 'sudo attempt failed' if rc == 127: - return 'sosreport terminated unexpectedly. Check disk space' + return 'sos report terminated unexpectedly. Check disk space' if len(stdout) > 0: return stdout.split('\n')[0:1] else: return 'sos exited with code %s' % rc def execute_sos_command(self): - """Run sosreport and capture the resulting file path""" - self.ui_msg('Generating sosreport...') + """Run sos report and capture the resulting file path""" + self.ui_msg('Generating sos report...') try: path = False + checksum = False res = self.run_command(self.sos_cmd, timeout=self.opts.timeout, get_pty=True, need_root=True, - use_container=True) + use_container=True, + env=self.sos_env_vars) if res['status'] == 0: - for line in res['stdout'].splitlines(): + for line in res['output'].splitlines(): if fnmatch.fnmatch(line, '*sosreport-*tar*'): path = line.strip() + if line.startswith((" sha256\t", " md5\t")): + checksum = line.split("\t")[1] + elif line.startswith("The checksum is: "): + checksum = line.split()[3] + + if checksum: + self.manifest.add_field('checksum', checksum) + if len(checksum) == 32: + self.manifest.add_field('checksum_type', 'md5') + elif len(checksum) == 64: + self.manifest.add_field('checksum_type', 'sha256') + else: + self.manifest.add_field('checksum_type', 'unknown') + else: + self.manifest.add_field('checksum_type', 'unknown') else: - err = self.determine_sos_error(res['status'], res['stdout']) - self.log_debug("Error running sosreport. rc = %s msg = %s" - % (res['status'], res['stdout'] or - res['stderr'])) + err = self.determine_sos_error(res['status'], res['output']) + self.log_debug("Error running sos report. rc = %s msg = %s" + % (res['status'], res['output'])) raise Exception(err) return path except CommandTimeoutException: self.log_error('Timeout exceeded') raise except Exception as e: - self.log_error('Error running sosreport: %s' % e) + self.log_error('Error running sos report: %s' % e) raise def retrieve_file(self, path): """Copies the specified file from the host to our temp dir""" destdir = self.tmpdir + '/' - dest = destdir + path.split('/')[-1] + dest = os.path.join(destdir, path.split('/')[-1]) try: - if not self.local: - if self.file_exists(path): - self.log_info("Copying remote %s to local %s" % - (path, destdir)) - cmd = "/usr/bin/scp -oControlPath=%s %s@%s:%s %s" % ( - self.control_path, - self.opts.ssh_user, - self.address, - path, - destdir - ) - res = self.run_command(cmd, force_local=True) - return res['status'] == 0 - else: - self.log_debug("Attempting to copy remote file %s, but it " - "does not exist on filesystem" % path) - return False + if self.file_exists(path): + self.log_info("Copying remote %s to local %s" % + (path, destdir)) + return self._transport.retrieve_file(path, dest) else: - self.log_debug("Moving %s to %s" % (path, destdir)) - shutil.copy(path, dest) - return True + self.log_debug("Attempting to copy remote file %s, but it " + "does not exist on filesystem" % path) + return False except Exception as err: self.log_debug("Failed to retrieve %s: %s" % (path, err)) return False @@ -812,7 +835,7 @@ class SosNode(): """ path = ''.join(path.split()) try: - if len(path) <= 2: # ensure we have a non '/' path + if len(path.split('/')) <= 2: # ensure we have a non '/' path self.log_debug("Refusing to remove path %s: appears to be " "incorrect and possibly dangerous" % path) return False @@ -838,23 +861,20 @@ class SosNode(): except Exception: self.log_error('Failed to make archive readable') return False - try: - self.make_archive_readable(self.sos_path + '.md5') - except Exception: - self.log_debug('Failed to make md5 readable') - self.soslog.info('Retrieving sosreport from %s' % self.address) - self.ui_msg('Retrieving sosreport...') - ret = self.retrieve_file(self.sos_path) + self.log_info('Retrieving sos report from %s' % self.address) + self.ui_msg('Retrieving sos report...') + try: + ret = self.retrieve_file(self.sos_path) + except Exception as err: + self.log_error(err) + return False if ret: - self.ui_msg('Successfully collected sosreport') + self.ui_msg('Successfully collected sos report') self.file_list.append(self.sos_path.split('/')[-1]) + return True else: - self.log_error('Failed to retrieve sosreport') - raise SystemExit - self.hash_retrieved = self.retrieve_file(self.sos_path + '.md5') - if self.hash_retrieved: - self.file_list.append(self.sos_path.split('/')[-1] + '.md5') - return True + self.ui_msg('Failed to retrieve sos report') + return False else: # sos sometimes fails but still returns a 0 exit code if self.stderr.read(): @@ -862,31 +882,35 @@ class SosNode(): else: e = [x.strip() for x in self.stdout.readlines() if x.strip][-1] self.soslog.error( - 'Failed to run sosreport on %s: %s' % (self.address, e)) - self.log_error('Failed to run sosreport. %s' % e) + 'Failed to run sos report on %s: %s' % (self.address, e)) + self.log_error('Failed to run sos report. %s' % e) return False def remove_sos_archive(self): """Remove the sosreport archive from the node, since we have collected it and it would be wasted space otherwise""" - if self.sos_path is None: + if self.sos_path is None or self.local: + # local transport moves the archive rather than copies it, so there + # is no archive at the original location to remove return if 'sosreport' not in self.sos_path: - self.log_debug("Node sosreport path %s looks incorrect. Not " + self.log_debug("Node sos report path %s looks incorrect. Not " "attempting to remove path" % self.sos_path) return removed = self.remove_file(self.sos_path) if not removed: - self.log_error('Failed to remove sosreport') + self.log_error('Failed to remove sos report') def cleanup(self): """Remove the sos archive from the node once we have it locally""" self.remove_sos_archive() - if self.hash_retrieved: - self.remove_file(self.sos_path + '.md5') + if self.sos_path: + for ext in ['.sha256', '.md5']: + if self.remove_file(self.sos_path + ext): + break cleanup = self.host.set_cleanup_cmd() if cleanup: - self.run_command(cleanup) + self.run_command(cleanup, need_root=True) def collect_extra_cmd(self, filenames): """Collect the file created by a cluster outside of sos""" @@ -907,7 +931,7 @@ class SosNode(): else: self.log_error("Unable to retrieve file %s" % filename) except Exception as e: - msg = 'Error collecting additional data from master: %s' % e + msg = 'Error collecting additional data from primary: %s' % e self.log_error(msg) def make_archive_readable(self, filepath): @@ -924,3 +948,5 @@ class SosNode(): msg = "Exception while making %s readable. Return code was %s" self.log_error(msg % (filepath, res['status'])) raise Exception + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/collector/transports/__init__.py 4.5.3ubuntu2/sos/collector/transports/__init__.py --- 4.0-2/sos/collector/transports/__init__.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/transports/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,399 @@ +# Copyright Red Hat 2021, Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import inspect +import logging +import pexpect +import re + +from pipes import quote +from sos.collector.exceptions import (ConnectionException, + CommandTimeoutException) +from sos.utilities import bold + + +class RemoteTransport(): + """The base class used for defining supported remote transports to connect + to remote nodes in conjunction with `sos collect`. + + This abstraction is used to manage the backend connections to nodes so that + SoSNode() objects can be leveraged generically to connect to nodes, inspect + those nodes, and run commands on them. + """ + + name = 'undefined' + default_user = None + + def __init__(self, address, commons): + self.address = address + self.opts = commons['cmdlineopts'] + self.tmpdir = commons['tmpdir'] + self.need_sudo = commons['need_sudo'] + self._hostname = None + self.soslog = logging.getLogger('sos') + self.ui_log = logging.getLogger('sos_ui') + + def _sanitize_log_msg(self, msg): + """Attempts to obfuscate sensitive information in log messages such as + passwords""" + reg = r'(?P(pass|key|secret|PASS|KEY|SECRET).*?=)(?P.*?\s)' + return re.sub(reg, r'\g****** ', msg) + + def log_info(self, msg): + """Used to print and log info messages""" + caller = inspect.stack()[1][3] + lmsg = '[%s:%s] %s' % (self.hostname, caller, msg) + self.soslog.info(lmsg) + + def log_error(self, msg): + """Used to print and log error messages""" + caller = inspect.stack()[1][3] + lmsg = '[%s:%s] %s' % (self.hostname, caller, msg) + self.soslog.error(lmsg) + + def log_debug(self, msg): + """Used to print and log debug messages""" + msg = self._sanitize_log_msg(msg) + caller = inspect.stack()[1][3] + msg = '[%s:%s] %s' % (self.hostname, caller, msg) + self.soslog.debug(msg) + + @property + def hostname(self): + if self._hostname and 'localhost' not in self._hostname: + return self._hostname + return self.address + + @property + def connected(self): + """Is the transport __currently__ connected to the node, or otherwise + capable of seamlessly running a command or similar on the node? + """ + return False + + @property + def remote_exec(self): + """This is the command string needed to leverage the remote transport + when executing commands. For example, for an SSH transport this would + be the `ssh ` string prepended to any command so that the + command is executed by the ssh binary. + + This is also referenced by the `remote_exec` parameter for policies + when loading a policy for a remote node + """ + return None + + @classmethod + def display_help(cls, section): + if cls is RemoteTransport: + return cls.display_self_help(section) + section.set_title("%s Transport Detailed Help" + % cls.name.title().replace('_', ' ')) + if cls.__doc__ and cls.__doc__ is not RemoteTransport.__doc__: + section.add_text(cls.__doc__) + else: + section.add_text( + 'Detailed information not available for this transport' + ) + + @classmethod + def display_self_help(cls, section): + section.set_title('SoS Remote Transport Help') + section.add_text( + "\nTransports define how SoS connects to nodes and executes " + "commands on them for the purposes of an %s run. Generally, " + "this means transports define how commands are wrapped locally " + "so that they are executed on the remote node(s) instead." + % bold('sos collect') + ) + + section.add_text( + "Transports are generally selected by the cluster profile loaded " + "for a given execution, however users may explicitly set one " + "using '%s'. Note that not all transports will function for all " + "cluster/node types." + % bold('--transport=$transport_name') + ) + + section.add_text( + 'By default, OpenSSH Control Persist is attempted. Additional ' + 'information for each supported transport is available in the ' + 'following help sections:\n' + ) + + from sos.collector.sosnode import TRANSPORTS + for transport in TRANSPORTS: + _sec = bold("collect.transports.%s" % transport) + _desc = "The '%s' transport" % transport.lower() + section.add_text( + "{:>8}{:<45}{:<30}".format(' ', _sec, _desc), + newline=False + ) + + def connect(self, password): + """Perform the connection steps in order to ensure that we are able to + connect to the node for all future operations. Note that this should + not provide an interactive shell at this time. + """ + if self._connect(password): + if not self._hostname: + self._get_hostname() + return True + return False + + def _connect(self, password): + """Actually perform the connection requirements. Should be overridden + by specific transports that subclass RemoteTransport + """ + raise NotImplementedError("Transport %s does not define connect" + % self.name) + + def reconnect(self, password): + """Attempts to reconnect to the node using the standard connect() + but does not do so indefinitely. This imposes a strict number of retry + attempts before failing out + """ + attempts = 1 + last_err = 'unknown' + while attempts < 5: + self.log_debug("Attempting reconnect (#%s) to node" % attempts) + try: + if self.connect(password): + return True + except Exception as err: + self.log_debug("Attempt #%s exception: %s" % (attempts, err)) + last_err = err + attempts += 1 + self.log_error("Unable to reconnect to node after 5 attempts, " + "aborting.") + raise ConnectionException("last exception from transport: %s" + % last_err) + + def disconnect(self): + """Perform whatever steps are necessary, if any, to terminate any + connection to the node + """ + try: + if self._disconnect(): + self.log_debug("Successfully disconnected from node") + else: + self.log_error("Unable to successfully disconnect, see log for" + " more details") + except Exception as err: + self.log_error("Failed to disconnect: %s" % err) + + def _disconnect(self): + raise NotImplementedError("Transport %s does not define disconnect" + % self.name) + + def run_command(self, cmd, timeout=180, need_root=False, env=None, + get_pty=False): + """Run a command on the node, returning its output and exit code. + This should return the exit code of the command being executed, not the + exit code of whatever mechanism the transport uses to execute that + command + + :param cmd: The command to run + :type cmd: ``str`` + + :param timeout: The maximum time in seconds to allow the cmd to run + :type timeout: ``int`` + + :param get_pty: Does ``cmd`` require a pty? + :type get_pty: ``bool`` + + :param need_root: Does ``cmd`` require root privileges? + :type neeed_root: ``bool`` + + :param env: Specify env vars to be passed to the ``cmd`` + :type env: ``dict`` + + :param get_pty: Does ``cmd`` require execution with a pty? + :type get_pty: ``bool`` + + :returns: Output of ``cmd`` and the exit code + :rtype: ``dict`` with keys ``output`` and ``status`` + """ + self.log_debug('Running command %s' % cmd) + if get_pty: + cmd = "/bin/bash -c %s" % quote(cmd) + # currently we only use/support the use of pexpect for handling the + # execution of these commands, as opposed to directly invoking + # subprocess.Popen() in conjunction with tools like sshpass. + # If that changes in the future, we'll add decision making logic here + # to route to the appropriate handler, but for now we just go straight + # to using pexpect + return self._run_command_with_pexpect(cmd, timeout, need_root, env) + + def _format_cmd_for_exec(self, cmd): + """Format the command in the way needed for the remote transport to + successfully execute it as one would when manually executing it + + :param cmd: The command being executed, as formatted by SoSNode + :type cmd: ``str`` + + + :returns: The command further formatted as needed by this + transport + :rtype: ``str`` + """ + cmd = "%s %s" % (self.remote_exec, quote(cmd)) + cmd = cmd.lstrip() + return cmd + + def _run_command_with_pexpect(self, cmd, timeout, need_root, env): + """Execute the command using pexpect, which allows us to more easily + handle prompts and timeouts compared to directly leveraging the + subprocess.Popen() method. + + :param cmd: The command to execute. This will be automatically + formatted to use the transport. + :type cmd: ``str`` + + :param timeout: The maximum time in seconds to run ``cmd`` + :type timeout: ``int`` + + :param need_root: Does ``cmd`` need to run as root or with sudo? + :type need_root: ``bool`` + + :param env: Any env vars that ``cmd`` should be run with + :type env: ``dict`` + """ + cmd = self._format_cmd_for_exec(cmd) + + # if for any reason env is empty, set it to None as otherwise + # pexpect interprets this to mean "run this command with no env vars of + # any kind" + if not env: + env = None + + try: + result = pexpect.spawn(cmd, encoding='utf-8', env=env) + except pexpect.exceptions.ExceptionPexpect as err: + self.log_debug(err.value) + return {'status': 127, 'output': ''} + + _expects = [pexpect.EOF, pexpect.TIMEOUT] + if need_root and self.opts.ssh_user != 'root': + _expects.extend([ + '\\[sudo\\] password for .*:', + 'Password:' + ]) + + index = result.expect(_expects, timeout=timeout) + + if index in [2, 3]: + self._send_pexpect_password(index, result) + index = result.expect(_expects, timeout=timeout) + + if index == 0: + out = result.before + result.close() + return {'status': result.exitstatus, 'output': out} + elif index == 1: + raise CommandTimeoutException(cmd) + + def _send_pexpect_password(self, index, result): + """Handle password prompts for sudo and su usage for non-root SSH users + + :param index: The index pexpect.spawn returned to match against + either a sudo or su prompt + :type index: ``int`` + + :param result: The spawn running the command + :type result: ``pexpect.spawn`` + """ + if index == 2: + if not self.opts.sudo_pw and not self.opt.nopasswd_sudo: + msg = ("Unable to run command: sudo password " + "required but not provided") + self.log_error(msg) + raise Exception(msg) + result.sendline(self.opts.sudo_pw) + elif index == 3: + if not self.opts.root_password: + msg = ("Unable to run command as root: no root password given") + self.log_error(msg) + raise Exception(msg) + result.sendline(self.opts.root_password) + + def _get_hostname(self): + """Determine the hostname of the node and set that for future reference + and logging + + :returns: The hostname of the system, per the `hostname` command + :rtype: ``str`` + """ + _out = self.run_command('hostname') + if _out['status'] == 0: + self._hostname = _out['output'].strip() + + if not self._hostname: + self._hostname = self.address + self.log_info("Hostname set to %s" % self._hostname) + return self._hostname + + def retrieve_file(self, fname, dest): + """Copy a remote file, fname, to dest on the local node + + :param fname: The name of the file to retrieve + :type fname: ``str`` + + :param dest: Where to save the file to locally + :type dest: ``str`` + + :returns: True if file was successfully copied from remote, or False + :rtype: ``bool`` + """ + attempts = 0 + try: + while attempts < 5: + attempts += 1 + ret = self._retrieve_file(fname, dest) + if ret: + return True + self.log_info("File retrieval attempt %s failed" % attempts) + self.log_info("File retrieval failed after 5 attempts") + return False + except Exception as err: + self.log_error("Exception encountered during retrieval attempt %s " + "for %s: %s" % (attempts, fname, err)) + raise err + + def _retrieve_file(self, fname, dest): + raise NotImplementedError("Transport %s does not support file copying" + % self.name) + + def read_file(self, fname): + """Read the given file fname and return its contents + + :param fname: The name of the file to read + :type fname: ``str`` + + :returns: The content of the file + :rtype: ``str`` + """ + self.log_debug("Reading file %s" % fname) + return self._read_file(fname) + + def _read_file(self, fname): + res = self.run_command("cat %s" % fname, timeout=10) + if res['status'] == 0: + return res['output'] + else: + if 'No such file' in res['output']: + self.log_debug("File %s does not exist on node" + % fname) + else: + self.log_error("Error reading %s: %s" % + (fname, res['output'].split(':')[1:])) + return '' + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/collector/transports/control_persist.py 4.5.3ubuntu2/sos/collector/transports/control_persist.py --- 4.0-2/sos/collector/transports/control_persist.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/transports/control_persist.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,207 @@ +# Copyright Red Hat 2021, Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + + +import os +import pexpect +import subprocess + +from sos.collector.transports import RemoteTransport +from sos.collector.exceptions import (InvalidPasswordException, + TimeoutPasswordAuthException, + PasswordRequestException, + AuthPermissionDeniedException, + ConnectionException, + ConnectionTimeoutException, + ControlSocketMissingException, + ControlPersistUnsupportedException) +from sos.utilities import sos_get_command_output + + +class SSHControlPersist(RemoteTransport): + """ + A transport for collect that leverages OpenSSH's ControlPersist + functionality which uses control sockets to transparently keep a connection + open to the remote host without needing to rebuild the SSH connection for + each and every command executed on the node. + + This transport will by default assume the use of SSH keys, meaning keys + have already been distributed to target nodes. If this is not the case, + users will need to provide a password using the --password or + --password-per-node option, depending on if the password to connect to all + nodes is the same or not. Note that these options prevent the use of the + --batch option, as they require user input. + """ + + name = 'control_persist' + + def _check_for_control_persist(self): + """Checks to see if the local system supported SSH ControlPersist. + + ControlPersist allows OpenSSH to keep a single open connection to a + remote host rather than building a new session each time. This is the + same feature that Ansible uses in place of paramiko, which we have a + need to drop in sos-collector. + + This check relies on feedback from the ssh binary. The command being + run should always generate stderr output, but depending on what that + output reads we can determine if ControlPersist is supported or not. + + For our purposes, a host that does not support ControlPersist is not + able to run sos-collector. + + Returns + True if ControlPersist is supported, else raise Exception. + """ + ssh_cmd = ['ssh', '-o', 'ControlPersist'] + cmd = subprocess.Popen(ssh_cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = cmd.communicate() + err = err.decode('utf-8') + if 'Bad configuration option' in err or 'Usage:' in err: + raise ControlPersistUnsupportedException + return True + + def _connect(self, password=''): + """ + Using ControlPersist, create the initial connection to the node. + + This will generate an OpenSSH ControlPersist socket within the tmp + directory created or specified for sos-collector to use. + + At most, we will wait 30 seconds for a connection. This involves a 15 + second wait for the initial connection attempt, and a subsequent 15 + second wait for a response when we supply a password. + + Since we connect to nodes in parallel (using the --threads value), this + means that the time between 'Connecting to nodes...' and 'Beginning + collection of sosreports' that users see can be up to an amount of time + equal to 30*(num_nodes/threads) seconds. + + Returns + True if session is successfully opened, else raise Exception + """ + try: + self._check_for_control_persist() + except ControlPersistUnsupportedException: + self.log_error("OpenSSH ControlPersist is not locally supported. " + "Please update your OpenSSH installation.") + raise + self.log_info('Opening SSH session to create control socket') + self.control_path = ("%s/.sos-collector-%s" % (self.tmpdir, + self.address)) + self.ssh_cmd = '' + connected = False + ssh_key = '' + ssh_port = '' + if self.opts.ssh_port != 22: + ssh_port = "-p%s " % self.opts.ssh_port + if self.opts.ssh_key: + ssh_key = "-i%s" % self.opts.ssh_key + + cmd = ("ssh %s %s -oControlPersist=600 -oControlMaster=auto " + "-oStrictHostKeyChecking=no -oControlPath=%s %s@%s " + "\"echo Connected\"" % (ssh_key, + ssh_port, + self.control_path, + self.opts.ssh_user, + self.address)) + res = pexpect.spawn(cmd, encoding='utf-8') + + connect_expects = [ + u'Connected', + u'password:', + u'.*Permission denied.*', + u'.* port .*: No route to host', + u'.*Could not resolve hostname.*', + pexpect.TIMEOUT + ] + + index = res.expect(connect_expects, timeout=15) + + if index == 0: + connected = True + elif index == 1: + if password: + pass_expects = [ + u'Connected', + u'Permission denied, please try again.', + pexpect.TIMEOUT + ] + res.sendline(password) + pass_index = res.expect(pass_expects, timeout=15) + if pass_index == 0: + connected = True + elif pass_index == 1: + # Note that we do not get an exitstatus here, so matching + # this line means an invalid password will be reported for + # both invalid passwords and invalid user names + raise InvalidPasswordException + elif pass_index == 2: + raise TimeoutPasswordAuthException + else: + raise PasswordRequestException + elif index == 2: + raise AuthPermissionDeniedException + elif index == 3: + raise ConnectionException(self.address, self.opts.ssh_port) + elif index == 4: + raise ConnectionException(self.address) + elif index == 5: + raise ConnectionTimeoutException + else: + raise Exception("Unknown error, client returned %s" % res.before) + if connected: + if not os.path.exists(self.control_path): + raise ControlSocketMissingException + self.log_debug("Successfully created control socket at %s" + % self.control_path) + return True + return False + + def _disconnect(self): + if os.path.exists(self.control_path): + try: + os.remove(self.control_path) + return True + except Exception as err: + self.log_debug("Could not disconnect properly: %s" % err) + return False + self.log_debug("Control socket not present when attempting to " + "terminate session") + + @property + def connected(self): + """Check if the SSH control socket exists + + The control socket is automatically removed by the SSH daemon in the + event that the last connection to the node was greater than the timeout + set by the ControlPersist option. This can happen for us if we are + collecting from a large number of nodes, and the timeout expires before + we start collection. + """ + return os.path.exists(self.control_path) + + @property + def remote_exec(self): + if not self.ssh_cmd: + self.ssh_cmd = "ssh -oControlPath=%s %s@%s" % ( + self.control_path, self.opts.ssh_user, self.address + ) + return self.ssh_cmd + + def _retrieve_file(self, fname, dest): + cmd = "/usr/bin/scp -oControlPath=%s %s@%s:%s %s" % ( + self.control_path, self.opts.ssh_user, self.address, fname, dest + ) + res = sos_get_command_output(cmd) + return res['status'] == 0 + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/collector/transports/juju.py 4.5.3ubuntu2/sos/collector/transports/juju.py --- 4.0-2/sos/collector/transports/juju.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/transports/juju.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,84 @@ +# Copyright (c) 2023 Canonical Ltd., Chi Wai Chan + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + + +import subprocess + +from sos.collector.exceptions import JujuNotInstalledException +from sos.collector.transports import RemoteTransport +from sos.utilities import sos_get_command_output + + +class JujuSSH(RemoteTransport): + """ + A "transport" that leverages `juju ssh` to perform commands on the remote + hosts. + + This transport is expected to be used in juju managed environment, and the + user should have the necessary credential for accessing the controller. + When using this transport, the --nodes option will be expected to be a + comma separated machine IDs, **not** IP addr, since `juju ssh` identifies + the ssh target by machine ID. + + Examples: + + sos collect --nodes 0,1,2 --no-local --transport juju --batch + + """ + + name = "juju_ssh" + default_user = "ubuntu" + + def _check_juju_installed(self): + cmd = "juju version" + try: + subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError: + self.log_error("Failed to check `juju` version") + raise JujuNotInstalledException + return True + + def _chmod(self, fname): + cmd = f"{self.remote_exec} sudo chmod o+r {fname}" + try: + subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError: + self.log_error(f"Failed to make {fname} world-readable") + raise + return True + + def _connect(self, password=""): + self._connected = self._check_juju_installed() + return self._connected + + def _disconnect(self): + return True + + @property + def connected(self): + return self._connected + + @property + def remote_exec(self): + model, target_option = self.address.split(":") + model_option = f"-m {model}" if model else "" + option = f"{model_option} {target_option}" + return f"juju ssh {option}" + + def _retrieve_file(self, fname, dest): + self._chmod(fname) # juju scp needs the archive to be world-readable + model, unit = self.address.split(":") + model_option = f"-m {model}" if model else "" + cmd = f"juju scp {model_option} -- -r {unit}:{fname} {dest}" + res = sos_get_command_output(cmd) + return res["status"] == 0 + + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/collector/transports/local.py 4.5.3ubuntu2/sos/collector/transports/local.py --- 4.0-2/sos/collector/transports/local.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/transports/local.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,51 @@ +# Copyright Red Hat 2021, Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import os +import shutil + +from sos.collector.transports import RemoteTransport + + +class LocalTransport(RemoteTransport): + """ + A 'transport' to represent a local node. No remote connection is actually + made, and all commands set to be run by this transport are executed locally + without any wrappers. + """ + + name = 'local_node' + + def _connect(self, password): + return True + + def _disconnect(self): + return True + + @property + def connected(self): + return True + + def _retrieve_file(self, fname, dest): + self.log_debug("Moving %s to %s" % (fname, dest)) + shutil.copy(fname, dest) + return True + + def _format_cmd_for_exec(self, cmd): + return cmd + + def _read_file(self, fname): + if os.path.exists(fname): + with open(fname, 'r') as rfile: + return rfile.read() + self.log_debug("No such file: %s" % fname) + return '' + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/collector/transports/oc.py 4.5.3ubuntu2/sos/collector/transports/oc.py --- 4.0-2/sos/collector/transports/oc.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/transports/oc.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,242 @@ +# Copyright Red Hat 2021, Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import json +import tempfile +import os + +from sos.collector.transports import RemoteTransport +from sos.utilities import (is_executable, sos_get_command_output, + SoSTimeoutError) + + +class OCTransport(RemoteTransport): + """ + This transport leverages the execution of commands via a locally + available and configured ``oc`` binary for OCPv4 environments. + + The location of the oc binary MUST be in the $PATH used by the locally + loaded SoS policy. Specifically this means that the binary cannot be in the + running user's home directory, such as ~/.local/bin. + + OCPv4 clusters generally discourage the use of SSH, so this transport may + be used to remove our use of SSH in favor of the environment provided + method of connecting to nodes and executing commands via debug pods. + + The debug pod created will be a privileged pod that mounts the host's + filesystem internally so that sos report collections reflect the host, and + not the container in which it runs. + + This transport will execute within a temporary 'sos-collect-tmp' project + created by the OCP cluster profile. The project will be removed at the end + of execution. + + In the event of failures due to a misbehaving OCP API or oc binary, it is + recommended to fallback to the control_persist transport by manually + setting the --transport option. + """ + + name = 'oc' + project = 'sos-collect-tmp' + + def run_oc(self, cmd, **kwargs): + """Format and run a command with `oc` in the project defined for our + execution + """ + return sos_get_command_output( + "oc -n %s %s" % (self.project, cmd), + **kwargs + ) + + @property + def connected(self): + up = self.run_oc( + "wait --timeout=0s --for=condition=ready pod/%s" % self.pod_name + ) + return up['status'] == 0 + + def get_node_pod_config(self): + """Based on our template for the debug container, add the node-specific + items so that we can deploy one of these on each node we're collecting + from + """ + return { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "%s-sos-collector" % self.address.split('.')[0], + "namespace": self.project + }, + "priorityClassName": "system-cluster-critical", + "spec": { + "volumes": [ + { + "name": "host", + "hostPath": { + "path": "/", + "type": "Directory" + } + }, + { + "name": "run", + "hostPath": { + "path": "/run", + "type": "Directory" + } + }, + { + "name": "varlog", + "hostPath": { + "path": "/var/log", + "type": "Directory" + } + }, + { + "name": "machine-id", + "hostPath": { + "path": "/etc/machine-id", + "type": "File" + } + } + ], + "containers": [ + { + "name": "sos-collector-tmp", + "image": "registry.redhat.io/rhel8/support-tools" + if not self.opts.image else self.opts.image, + "command": [ + "/bin/bash" + ], + "env": [ + { + "name": "HOST", + "value": "/host" + } + ], + "resources": {}, + "volumeMounts": [ + { + "name": "host", + "mountPath": "/host" + }, + { + "name": "run", + "mountPath": "/run" + }, + { + "name": "varlog", + "mountPath": "/var/log" + }, + { + "name": "machine-id", + "mountPath": "/etc/machine-id" + } + ], + "securityContext": { + "privileged": True, + "runAsUser": 0 + }, + "stdin": True, + "stdinOnce": True, + "tty": True + } + ], + "imagePullPolicy": + "Always" if self.opts.force_pull_image else "IfNotPresent", + "restartPolicy": "Never", + "nodeName": self.address, + "hostNetwork": True, + "hostPID": True, + "hostIPC": True + } + } + + def _connect(self, password): + # the oc binary must be _locally_ available for this to work + if not is_executable('oc'): + return False + + # deploy the debug container we'll exec into + podconf = self.get_node_pod_config() + self.pod_name = podconf['metadata']['name'] + fd, self.pod_tmp_conf = tempfile.mkstemp(dir=self.tmpdir) + with open(fd, 'w') as cfile: + json.dump(podconf, cfile) + self.log_debug("Starting sos collector container '%s'" % self.pod_name) + # this specifically does not need to run with a project definition + out = sos_get_command_output( + "oc create -f %s" % self.pod_tmp_conf + ) + if (out['status'] != 0 or "pod/%s created" % self.pod_name not in + out['output']): + self.log_error("Unable to deploy sos collect pod") + self.log_debug("Debug pod deployment failed: %s" % out['output']) + return False + self.log_debug("Pod '%s' successfully deployed, waiting for pod to " + "enter ready state" % self.pod_name) + + # wait for the pod to report as running + try: + up = self.run_oc("wait --for=condition=Ready pod/%s --timeout=30s" + % self.pod_name, + # timeout is for local safety, not oc + timeout=40) + if not up['status'] == 0: + self.log_error("Pod not available after 30 seconds") + return False + except SoSTimeoutError: + self.log_error("Timeout while polling for pod readiness") + return False + except Exception as err: + self.log_error("Error while waiting for pod to be ready: %s" + % err) + return False + + return True + + def _format_cmd_for_exec(self, cmd): + if cmd.startswith('oc'): + return ("oc -n %s exec --request-timeout=0 %s -- chroot /host %s" + % (self.project, self.pod_name, cmd)) + return super(OCTransport, self)._format_cmd_for_exec(cmd) + + def run_command(self, cmd, timeout=180, need_root=False, env=None, + get_pty=False): + # debug pod setup is slow, extend all timeouts to account for this + if timeout: + timeout += 10 + + # since we always execute within a bash shell, force disable get_pty + # to avoid double-quoting + return super(OCTransport, self).run_command(cmd, timeout, need_root, + env, False) + + def _disconnect(self): + if os.path.exists(self.pod_tmp_conf): + os.unlink(self.pod_tmp_conf) + removed = self.run_oc("delete pod %s" % self.pod_name) + if "deleted" not in removed['output']: + self.log_debug("Calling delete on pod '%s' failed: %s" + % (self.pod_name, removed)) + return False + return True + + @property + def remote_exec(self): + return ("oc -n %s exec --request-timeout=0 %s -- /bin/bash -c" + % (self.project, self.pod_name)) + + def _retrieve_file(self, fname, dest): + # check if --retries flag is available for given version of oc + result = self.run_oc("cp --retries", stderr=True) + flags = '' if "unknown flag" in result["output"] else '--retries=5' + cmd = self.run_oc("cp %s %s:%s %s" + % (flags, self.pod_name, fname, dest)) + return cmd['status'] == 0 diff -pruN 4.0-2/sos/collector/transports/saltstack.py 4.5.3ubuntu2/sos/collector/transports/saltstack.py --- 4.0-2/sos/collector/transports/saltstack.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/collector/transports/saltstack.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,136 @@ +# Copyright Red Hat 2022, Trevor Benson + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import contextlib +import json +import os +import shutil +from sos.collector.transports import RemoteTransport +from sos.collector.exceptions import (ConnectionException, + SaltStackMasterUnsupportedException) +from sos.utilities import (is_executable, + sos_get_command_output) + + +class SaltStackMaster(RemoteTransport): + """ + A transport for collect that leverages SaltStack's Master Pub/Sub + functionality to send commands to minions. + + This transport will by default assume the use cmd.shell module to + execute commands on the minions. + """ + + name = 'saltstack' + + def _convert_output_json(self, json_output): + return list(json.loads(json_output).values())[0] + + def run_command( + self, cmd, timeout=180, need_root=False, env=None, get_pty=False): + """ + Run a command on the remote host using SaltStack Master. + If the output is json, convert it to a string. + """ + ret = super(SaltStackMaster, self).run_command( + cmd, timeout, need_root, env, get_pty) + with contextlib.suppress(Exception): + ret['output'] = self._convert_output_json(ret['output']) + return ret + + def _salt_retrieve_file(self, node, fname, dest): + """ + Execute cp.push on the remote host using SaltStack Master + """ + cmd = f"salt {node} cp.push {fname}" + res = sos_get_command_output(cmd) + if res['status'] == 0: + cachedir = f"/var/cache/salt/master/minions/{self.address}/files" + cachedir_file = os.path.join(cachedir, fname.lstrip('/')) + shutil.move(cachedir_file, dest) + return True + return False + + @property + def connected(self): + """Check if the remote host is responding using SaltStack Master.""" + up = self.run_command("echo Connected", timeout=10) + return up['status'] == 0 + + def _check_for_saltstack(self, password=None): + """Checks to see if the local system supported SaltStack Master. + + This check relies on feedback from the salt binary. The command being + run should always generate stderr output, but depending on what that + output reads we can determine if SaltStack Master is supported or not. + + For our purposes, a host that does not support SaltStack Master is not + able to run sos-collector. + + Returns + True if SaltStack Master is supported, else raise Exception + """ + + cmd = 'salt-run manage.status' + res = sos_get_command_output(cmd) + if res['status'] == 0: + return res['status'] == 0 + else: + raise SaltStackMasterUnsupportedException + + def _connect(self, password=None): + """Connect to the remote host using SaltStack Master. + + This method will attempt to connect to the remote host using SaltStack + Master. If the connection fails, an exception will be raised. + + If the connection is successful, the connection will be stored in the + self._connection attribute. + """ + if not is_executable('salt'): + self.log_error("salt command is not executable. ") + return False + + try: + self._check_for_saltstack() + except ConnectionException: + self.log_error("Transport is not locally supported. ") + raise + self.log_info("Transport is locally supported and service running. ") + cmd = "echo Connected" + result = self.run_command(cmd, timeout=180) + return result['status'] == 0 + + def _disconnect(self): + return True + + @property + def remote_exec(self): + """The remote execution command to use for this transport.""" + salt_args = "--out json --static --no-color" + return f"salt {salt_args} {self.address} cmd.shell " + + def _retrieve_file(self, fname, dest): + """Retrieve a file from the remote host using saltstack + + Parameters + fname The path to the file on the remote host + dest The path to the destination directory on the master + + Returns + True if the file was retrieved, else False + """ + return ( + self._salt_retrieve_file(self.address, fname, dest) + if self.connected + else False + ) + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/component.py 4.5.3ubuntu2/sos/component.py --- 4.0-2/sos/component.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/component.py 2023-04-28 17:16:21.000000000 +0000 @@ -14,15 +14,17 @@ import logging import os import tempfile import sys +import time from argparse import SUPPRESS from datetime import datetime +from getpass import getpass from shutil import rmtree from pathlib import Path from sos import __version__ from sos.archive import TarFileArchive from sos.options import SoSOptions -from sos.utilities import TempFileUtil +from sos.utilities import TempFileUtil, shell_out class SoSComponent(): @@ -49,6 +51,7 @@ class SoSComponent(): arg_defaults = {} configure_logging = True load_policy = True + load_probe = True root_required = False _arg_defaults = { @@ -56,6 +59,7 @@ class SoSComponent(): "compression_type": 'auto', "config_file": '/etc/sos/sos.conf', "debug": False, + "encrypt": False, "encrypt_key": None, "encrypt_pass": None, "quiet": False, @@ -81,9 +85,9 @@ class SoSComponent(): except Exception: pass - # update args from component's arg_defaults defintion + # update args from component's arg_defaults definition self._arg_defaults.update(self.arg_defaults) - self.opts = self.load_options() + self.opts = self.load_options() # lgtm [py/init-calls-subclass] if self.configure_logging: tmpdir = self.get_tmpdir_default() @@ -105,12 +109,7 @@ class SoSComponent(): self._setup_logging() if self.load_policy: - try: - import sos.policies - self.policy = sos.policies.load(sysroot=self.opts.sysroot) - except KeyboardInterrupt: - self._exit(0) - self._is_root = self.policy.is_root() + self.load_local_policy() if self.manifest is not None: self.manifest.add_field('version', __version__) @@ -120,16 +119,34 @@ class SoSComponent(): self.manifest.add_field('end_time', '') self.manifest.add_field('run_time', '') self.manifest.add_field('compression', '') + self.manifest.add_field('tmpdir', self.tmpdir) + self.manifest.add_field('tmpdir_fs_type', self.tmpfstype) self.manifest.add_field('policy', self.policy.distro) self.manifest.add_section('components') + def load_local_policy(self): + try: + import sos.policies + self.policy = sos.policies.load(sysroot=self.opts.sysroot, + probe_runtime=self.load_probe) + self.sysroot = self.policy.sysroot + except KeyboardInterrupt: + self._exit(0) + self._is_root = self.policy.is_root() + + def execute(self): + raise NotImplementedError + def get_exit_handler(self): def exit_handler(signum, frame): self.exit_process = True self._exit() return exit_handler - def _exit(self, error=0): + def _exit(self, error=0, msg=None): + if msg: + self.ui_log.error("") + self.ui_log.error(msg) raise SystemExit(error) def get_tmpdir_default(self): @@ -138,13 +155,27 @@ class SoSComponent(): use a standardized env var to redirect to the host's filesystem instead """ if self.opts.tmp_dir: - return self.opts.tmp_dir - - tmpdir = '/var/tmp' + tmpdir = os.path.abspath(self.opts.tmp_dir) + else: + tmpdir = os.getenv('TMPDIR', None) or '/var/tmp' if os.getenv('HOST', None) and os.getenv('container', None): tmpdir = os.path.join(os.getenv('HOST'), tmpdir.lstrip('/')) + # no standard library method exists for this, so call out to stat to + # avoid bringing in a dependency on psutil + self.tmpfstype = shell_out( + "stat --file-system --format=%s %s" % ("%T", tmpdir) + ).strip() + + if self.tmpfstype == 'tmpfs': + # can't log to the ui or sos.log yet since those require a defined + # tmpdir to setup + print("WARNING: tmp-dir is set to a tmpfs filesystem. This may " + "increase memory pressure and cause instability on low " + "memory systems, or when using --all-logs.") + time.sleep(2) + return tmpdir def check_listing_options(self): @@ -187,9 +218,21 @@ class SoSComponent(): # set all values back to their normal default codict = cmdopts.dict(preset_filter=False) for opt, val in codict.items(): - if opt not in cmdopts.arg_defaults.keys(): + if opt not in cmdopts.arg_defaults.keys() or val in [None, [], '']: continue - if val and val != opts.arg_defaults[opt]: + # A plugin that is [enabled|disabled|only] in cmdopts must + # overwrite these three options of itself in opts - reset it first + if opt in ["enable_plugins", "skip_plugins", "only_plugins"]: + for oopt in ["enable_plugins", "skip_plugins", "only_plugins"]: + common = set(val) & set(getattr(opts, oopt)) + # common has all plugins that are in this combination of + # "[-e|-o|-n] plug" of cmdopts & "[-e|-o|-n] plug" of opts + # so remove those plugins from this [-e|-o|-n] opts + if common: + setattr(opts, oopt, [x for x in getattr(opts, oopt) + if x not in common]) + + if val != opts.arg_defaults[opt]: setattr(opts, opt, val) return opts @@ -206,6 +249,14 @@ class SoSComponent(): option.default = None opts.update_from_conf(self.args.config_file, self.args.component) + + # directly check the cmdline options here as they have yet to be loaded + # as SoSOptions, and if we do this check after they are loaded we would + # need to do a second update from cmdline options for overriding config + # file values + if '--clean' in self.cmdline or '--mask' in self.cmdline: + opts.update_from_conf(self.args.config_file, 'clean') + if os.getuid() != 0: userconf = os.path.join(Path.home(), '.config/sos/sos.conf') if os.path.exists(userconf): @@ -229,7 +280,45 @@ class SoSComponent(): print("Failed to finish cleanup: %s\nContents may remain in %s" % (err, self.tmpdir)) + def _set_encrypt_from_env_vars(self): + msg = ('No encryption environment variables set, archive will not be ' + 'encrypted') + if os.environ.get('SOSENCRYPTKEY'): + self.opts.encrypt_key = os.environ.get('SOSENCRYPTKEY') + msg = 'Encryption key set via environment variable' + elif os.environ.get('SOSENCRYPTPASS'): + self.opts.encrypt_pass = os.environ.get('SOSENCRYPTPASS') + msg = 'Encryption passphrase set via environment variable' + self.soslog.info(msg) + self.ui_log.info(msg) + + def _get_encryption_method(self): + if not self.opts.batch: + _enc = None + while _enc not in ('P', 'K', 'E', 'N'): + _enc = input(( + 'Specify encryption method [P]assphrase, [K]ey, [E]nv ' + 'vars, [N]o encryption: ' + )).upper() + if _enc == 'P': + self.opts.encrypt_pass = getpass('Specify encryption ' + 'passphrase: ') + elif _enc == 'K': + self.opts.encrypt_key = input('Specify encryption key: ') + elif _enc == 'E': + self._set_encrypt_from_env_vars() + else: + self.opts.encrypt_key = None + self.opts.encrypt_pass = None + self.soslog.info("User specified --encrypt, but chose no " + "encryption when prompted.") + self.ui_log.warning("Archive will not be encrypted") + else: + self._set_encrypt_from_env_vars() + def setup_archive(self, name=''): + if self.opts.encrypt: + self._get_encryption_method() enc_opts = { 'encrypt': True if (self.opts.encrypt_pass or self.opts.encrypt_key) else False, @@ -243,16 +332,33 @@ class SoSComponent(): auto_archive = self.policy.get_preferred_archive() self.archive = auto_archive(archive_name, self.tmpdir, self.policy, self.opts.threads, - enc_opts, self.opts.sysroot, + enc_opts, self.sysroot, self.manifest) else: self.archive = TarFileArchive(archive_name, self.tmpdir, self.policy, self.opts.threads, - enc_opts, self.opts.sysroot, + enc_opts, self.sysroot, self.manifest) - self.archive.set_debug(True if self.opts.debug else False) + self.archive.set_debug(self.opts.verbosity > 2) + + def add_ui_log_to_stdout(self): + ui_console = logging.StreamHandler(sys.stdout) + ui_console.setFormatter(logging.Formatter('%(message)s')) + ui_console.setLevel(logging.INFO) + self.ui_log.addHandler(ui_console) + + def set_loggers_verbosity(self, verbosity): + if verbosity: + if self.flog: + self.flog.setLevel(logging.DEBUG) + if self.opts.verbosity > 1: + self.console.setLevel(logging.DEBUG) + else: + self.console.setLevel(logging.WARNING) + else: + self.console.setLevel(logging.WARNING) def _setup_logging(self): """Creates the log handler that shall be used by all components and any @@ -262,29 +368,20 @@ class SoSComponent(): # main soslog self.soslog = logging.getLogger('sos') self.soslog.setLevel(logging.DEBUG) - flog = None + self.flog = None if not self.check_listing_options(): self.sos_log_file = self.get_temp_file() - flog = logging.StreamHandler(self.sos_log_file) - flog.setFormatter(logging.Formatter( + self.flog = logging.StreamHandler(self.sos_log_file) + self.flog.setFormatter(logging.Formatter( '%(asctime)s %(levelname)s: %(message)s')) - flog.setLevel(logging.INFO) - self.soslog.addHandler(flog) + self.flog.setLevel(logging.INFO) + self.soslog.addHandler(self.flog) if not self.opts.quiet: - console = logging.StreamHandler(sys.stdout) - console.setFormatter(logging.Formatter('%(message)s')) - if self.opts.verbosity and self.opts.verbosity > 1: - console.setLevel(logging.DEBUG) - if flog: - flog.setLevel(logging.DEBUG) - elif self.opts.verbosity and self.opts.verbosity > 0: - console.setLevel(logging.INFO) - if flog: - flog.setLevel(logging.DEBUG) - else: - console.setLevel(logging.WARNING) - self.soslog.addHandler(console) + self.console = logging.StreamHandler(sys.stdout) + self.console.setFormatter(logging.Formatter('%(message)s')) + self.set_loggers_verbosity(self.opts.verbosity) + self.soslog.addHandler(self.console) # still log ERROR level message to console, but only setup this handler # when --quiet is used, as otherwise we'll double log else: @@ -305,10 +402,7 @@ class SoSComponent(): self.ui_log.addHandler(ui_fhandler) if not self.opts.quiet: - ui_console = logging.StreamHandler(sys.stdout) - ui_console.setFormatter(logging.Formatter('%(message)s')) - ui_console.setLevel(logging.INFO) - self.ui_log.addHandler(ui_console) + self.add_ui_log_to_stdout() def get_temp_file(self): return self.tempfile_util.new() @@ -323,16 +417,32 @@ class SoSMetadata(): metadata """ + def __init__(self): + self._values = {} + + def __iter__(self): + for item in self._values.items(): + yield item[1] + + def __getitem__(self, item): + return self._values[item] + + def __getattr__(self, attr): + try: + return self._values[attr] + except Exception: + raise AttributeError(attr) + def add_field(self, field_name, content): """Add a key, value entry to the current metadata instance """ - setattr(self, field_name, content) + self._values[field_name] = content def add_section(self, section_name): """Adds a new instance of SoSMetadata to the current instance """ - setattr(self, section_name, SoSMetadata()) - return getattr(self, section_name) + self._values[section_name] = SoSMetadata() + return self._values[section_name] def add_list(self, list_name, content=[]): """Add a named list element to the current instance. If content is not @@ -340,7 +450,7 @@ class SoSMetadata(): """ if not isinstance(content, list): raise TypeError('content added must be list') - setattr(self, list_name, content) + self._values[list_name] = content def get_json(self, indent=None): """Convert contents of this SoSMetdata instance, and all other nested @@ -349,7 +459,7 @@ class SoSMetadata(): Used to write manifest.json to the final archives. """ return json.dumps(self, - default=lambda o: getattr(o, '__dict__', str(o)), + default=lambda o: getattr(o, '_values', str(o)), indent=indent) # vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/help/__init__.py 4.5.3ubuntu2/sos/help/__init__.py --- 4.0-2/sos/help/__init__.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/help/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,302 @@ +# Copyright 2020 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import inspect +import importlib +import sys +import os + +from collections import OrderedDict +from sos.component import SoSComponent +from sos.policies import import_policy +from sos.report.plugins import Plugin +from sos.utilities import bold, ImporterHelper +from textwrap import fill + +try: + TERMSIZE = min(os.get_terminal_size().columns, 120) +except Exception: + TERMSIZE = 120 + + +class SoSHelper(SoSComponent): + """Provide better, more in-depth help for specific parts of sos than is + provided in either standard --help output or in manpages. + """ + + desc = 'Detailed help infomation' + configure_logging = False + load_policy = False + load_probe = False + + arg_defaults = { + 'topic': '' + } + + def __init__(self, parser, args, cmdline): + super(SoSHelper, self).__init__(parser, args, cmdline) + self.topic = self.opts.topic + + @classmethod + def add_parser_options(cls, parser): + parser.usage = 'sos help TOPIC [options]' + help_grp = parser.add_argument_group( + 'Help Information Options', + 'These options control what detailed information is displayed' + ) + help_grp.add_argument('topic', metavar='TOPIC', default='', nargs='?', + help=('name of the topic or component to show ' + 'help for')) + + def sanitize_topic_component(self): + _com = self.opts.topic.split('.')[0] + _replace = { + 'clean': 'cleaner', + 'mask': 'cleaner', + 'collect': 'collector' + } + if _com in _replace: + self.opts.topic = self.opts.topic.replace(_com, _replace[_com]) + + def execute(self): + if not self.opts.topic: + self.display_self_help() + sys.exit(0) + + # standardize the command to the module naming pattern + self.sanitize_topic_component() + + try: + klass = self.get_obj_for_topic() + except Exception as err: + print("Could not load help for '%s': %s" % (self.opts.topic, err)) + sys.exit(1) + + if klass: + try: + ht = HelpSection() + klass.display_help(ht) + ht.display() + except Exception as err: + print("Error loading help: %s" % err) + else: + print("No help section found for '%s'" % self.opts.topic) + + def get_obj_for_topic(self): + """Based on the help topic we're after, try to smartly decide which + object we need to manipulate in order to get help information. + """ + static_map = { + 'report': 'SoSReport', + 'report.plugins': 'Plugin', + 'cleaner': 'SoSCleaner', + 'collector': 'SoSCollector', + 'collector.transports': 'RemoteTransport', + 'collector.clusters': 'Cluster', + 'policies': 'Policy' + } + + cls = None + + if self.opts.topic in static_map: + mod = importlib.import_module('sos.' + self.opts.topic) + cls = getattr(mod, static_map[self.opts.topic]) + else: + _help = { + 'report.plugins.': self._get_plugin_variant, + 'policies.': self._get_policy_by_name, + 'collector.transports.': self._get_collect_transport, + 'collector.clusters.': self._get_collect_cluster, + } + for _sec in _help: + if self.opts.topic.startswith(_sec): + cls = _help[_sec]() + break + return cls + + def _get_collect_transport(self): + from sos.collector.sosnode import TRANSPORTS + _transport = self.opts.topic.split('.')[-1] + if _transport in TRANSPORTS: + return TRANSPORTS[_transport] + + def _get_collect_cluster(self): + from sos.collector import SoSCollector + import sos.collector.clusters + clusters = SoSCollector._load_modules(sos.collector.clusters, + 'clusters') + for cluster in clusters: + if cluster[0] == self.opts.topic.split('.')[-1]: + return cluster[1] + + def _get_plugin_variant(self): + mod = importlib.import_module('sos.' + self.opts.topic) + self.load_local_policy() + mems = inspect.getmembers(mod, inspect.isclass) + plugins = [m[1] for m in mems if issubclass(m[1], Plugin)] + for plugin in plugins: + if plugin.__subclasses__(): + cls = self.policy.match_plugin(plugin.__subclasses__()) + return cls + + def _get_policy_by_name(self): + _topic = self.opts.topic.split('.')[-1] + # mimic policy loading to discover all policiy classes without + # needing to manually define each here + import sos.policies.distros + _helper = ImporterHelper(sos.policies.distros) + for mod in _helper.get_modules(): + for policy in import_policy(mod): + _p = policy.__name__.lower().replace('policy', '') + if _p == _topic: + return policy + + def display_self_help(self): + """Displays the help information for this component directly, that is + help for `sos help`. + """ + self_help = HelpSection( + 'Detailed help for sos help', + ('The \'help\' sub-command is used to provide more detailed ' + 'information on different sub-commands available to sos as well ' + 'as different components at play within those sub-commands.') + ) + self_help.add_text( + 'SoS - officially pronounced "ess-oh-ess" - is a diagnostic and ' + 'supportability utility used by several Linux distributions as an ' + 'easy-to-use tool for standardized data collection. The most known' + ' component of which is %s (formerly sosreport) which is used to ' + 'collect troubleshooting information into an archive for review ' + 'by sysadmins or technical support teams.' + % bold('sos report') + ) + + subsect = self_help.add_section('How to search using sos help') + usage = bold('$component.$topic.$subtopic') + subsect.add_text( + 'To get more information on a given topic, use the form \'%s\'.' + % usage + ) + + rep_ex = bold('sos help report.plugins.kernel') + subsect.add_text("For example '%s' will provide more information on " + "the kernel plugin for the report function." % rep_ex) + + avail_help = self_help.add_section('Available Help Sections') + avail_help.add_text( + 'The following help sections are available. Additional help' + ' topics and subtopics may be displayed within their respective ' + 'help section.\n' + ) + + sections = { + 'report': 'Detailed help on the report command', + 'report.plugins': 'Information on the plugin design of sos', + 'report.plugins.$plugin': 'Information on a specific $plugin', + 'clean': 'Detailed help on the clean command', + 'collect': 'Detailed help on the collect command', + 'policies': 'How sos operates on different distributions' + } + + for sect in sections: + avail_help.add_text( + "\t{:<36}{}".format(bold(sect), sections[sect]), + newline=False + ) + + self_help.display() + + +class HelpSection(): + """This class is used to build the output displayed by `sos help` in a + standard fashion that provides easy formatting controls. + """ + + def __init__(self, title='', content='', indent=''): + """ + :param title: The title of the output section, will be prominently + displayed + :type title: ``str`` + + :param content: The text content to be displayed with this section + :type content: ``str`` + + :param indent: If the section should be nested, set this to a multiple + of 4. + :type indent: ``int`` + """ + self.title = title + self.content = content + self.indent = indent + self.sections = OrderedDict() + + def set_title(self, title): + """Set or override the title for this help section + + :param title: The name to set for this help section + :type title: ``str`` + """ + self.title = title + + def add_text(self, content, newline=True): + """Add body text to this section. If content for this section already + exists, append the new ``content`` after a newline. + + :param content: The text to add to the section + :type content: ``str`` + """ + if self.content: + ln = '\n\n' if newline else '\n' + content = ln + content + self.content += content + + def add_section(self, title, content='', indent=''): + """Add a section of text to the help section that will be displayed + when the HelpSection object is printed. + + Sections will be printed *in the order added*. + + This will return a subsection object with which block(s) of text may be + added to the subsection associated with ``title``. + + :param title: The title of the subsection being added + :type title: ``str`` + + :param content: The text the new section should contain + :type content: ``str`` + + :returns: The newly created subsection for ``title`` + :rtype: ``HelpSection`` + """ + self._add_section(title, content, indent) + return self.sections[title] + + def _add_section(self, title, content='', indent=''): + """Internal method used to add a new subsection to this help output + + :param title: The title of the subsection being added + :type title: ``str` + """ + if title in self.sections: + raise Exception('A section with that title already exists') + self.sections[title] = HelpSection(title, content, indent) + + def display(self): + """Print the HelpSection contents, including any subsections, to + console. + """ + print(fill( + bold(self.title), width=TERMSIZE, initial_indent=self.indent + )) + for ln in self.content.splitlines(): + print(fill(ln, width=TERMSIZE, initial_indent=self.indent)) + for section in self.sections: + print('') + self.sections[section].display() diff -pruN 4.0-2/sos/missing.py 4.5.3ubuntu2/sos/missing.py --- 4.0-2/sos/missing.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/missing.py 2023-04-28 17:16:21.000000000 +0000 @@ -14,6 +14,10 @@ class MissingCollect(SoSComponent): load_policy = False configure_logging = False desc = '(unavailable) Collect an sos report from multiple nodes' + missing_msg = ( + 'It appears likely that your distribution separately ships a package ' + 'called sos-collector. Please install it to enable this function' + ) def execute(self): sys.stderr.write( @@ -30,15 +34,34 @@ class MissingCollect(SoSComponent): """ return [] + @classmethod + def add_parser_options(cls, parser): + """Set the --help output for collect to a message that shows that + the functionality is unavailable + """ + msg = "%s %s" % ( + 'WARNING: `collect` is not available with this installation!', + cls.missing_msg + ) + parser.epilog = msg + return parser + class MissingPexpect(MissingCollect): """This is used as a placeholder for when the collect component is locally installed, but cannot be used due to a missing pexpect dependency. """ + missing_msg = ( + 'Please install the python3-pexpect package for your distribution in ' + 'order to enable this function' + ) + def execute(self): sys.stderr.write( "The collect command is unavailable due to a missing dependency " "on python3-pexpect.\n\nPlease install python3-pexpect to enable " "this functionality.\n" ) + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/options.py 4.5.3ubuntu2/sos/options.py --- 4.0-2/sos/options.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/options.py 2023-04-28 17:16:21.000000000 +0000 @@ -18,6 +18,16 @@ def _is_seq(val): return val_type is list or val_type is tuple +def str_to_bool(val): + _val = val.lower() + if _val in ['true', 'on', 'yes']: + return True + elif _val in ['false', 'off', 'no']: + return False + else: + return None + + class SoSOptions(): def _merge_opt(self, opt, src, is_default): @@ -153,15 +163,13 @@ class SoSOptions(): if isinstance(self.arg_defaults[key], list): return [v for v in val.split(',')] if isinstance(self.arg_defaults[key], bool): - _val = val.lower() - if _val in ['true', 'on', 'yes']: - return True - elif _val in ['false', 'off', 'no']: - return False - else: + val = str_to_bool(val) + if val is None: raise Exception( "Value of '%s' in %s must be True or False or analagous" % (key, conf)) + else: + return val if isinstance(self.arg_defaults[key], int): try: return int(val) @@ -186,12 +194,24 @@ class SoSOptions(): if 'verbose' in odict.keys(): odict['verbosity'] = int(odict.pop('verbose')) # convert options names - for key in odict.keys(): + # unify some of them if multiple variants of the + # cmdoption exist + rename_opts = { + 'name': 'label', + 'plugin_option': 'plugopts', + 'profile': 'profiles' + } + for key in list(odict): if '-' in key: odict[key.replace('-', '_')] = odict.pop(key) + if key in rename_opts: + odict[rename_opts[key]] = odict.pop(key) # set the values according to the config file for key, val in odict.items(): - if isinstance(val, str): + # most option values do not tolerate spaces, special + # exception however for --keywords which we do want to + # support phrases, and thus spaces, for + if isinstance(val, str) and key != 'keywords': val = val.replace(' ', '') if key not in self.arg_defaults: # read an option that is not loaded by the current @@ -272,6 +292,8 @@ class SoSOptions(): null_values = ("False", "None", "[]", '""', "''", "0") if not value or value in null_values: return False + if name == 'plugopts' and value: + return True if name in self.arg_defaults: if str(value) == str(self.arg_defaults[name]): return False @@ -282,6 +304,10 @@ class SoSOptions(): """ if name in ("add_preset", "del_preset", "desc", "note"): return False + # Exception list for options that still need to be reported when 0 + if name in ['log_size', 'plugin_timeout', 'cmd_timeout'] \ + and value == 0: + return True return has_value(name, value) def argify(name, value): diff -pruN 4.0-2/sos/policies/__init__.py 4.5.3ubuntu2/sos/policies/__init__.py --- 4.0-2/sos/policies/__init__.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -1,38 +1,27 @@ +import logging import os -import re import platform import time import json -import fnmatch import tempfile import random import string +import sys -from getpass import getpass from pwd import getpwuid -from sos.utilities import (ImporterHelper, - import_module, - is_executable, - shell_out, - sos_get_command_output, - get_human_readable) +from sos.presets import (NO_PRESET, GENERIC_PRESETS, PRESETS_PATH, + PresetDefaults, DESC, NOTE, OPTS) +from sos.policies.package_managers import PackageManager +from sos.utilities import (ImporterHelper, import_module, get_human_readable, + bold) from sos.report.plugins import IndependentPlugin, ExperimentalPlugin from sos.options import SoSOptions from sos import _sos as _ from textwrap import fill -from pipes import quote - -PRESETS_PATH = "/etc/sos/presets.d" - -try: - import requests - REQUESTS_LOADED = True -except ImportError: - REQUESTS_LOADED = False def import_policy(name): - policy_fqname = "sos.policies.%s" % name + policy_fqname = "sos.policies.distros.%s" % name try: return import_module(policy_fqname, Policy) except ImportError: @@ -44,8 +33,8 @@ def load(cache={}, sysroot=None, init=No if 'policy' in cache: return cache.get('policy') - import sos.policies - helper = ImporterHelper(sos.policies) + import sos.policies.distros + helper = ImporterHelper(sos.policies.distros) for module in helper.get_modules(): for policy in import_policy(module): if policy.check(remote=remote_check): @@ -53,697 +42,16 @@ def load(cache={}, sysroot=None, init=No probe_runtime=probe_runtime, remote_exec=remote_exec) + if sys.platform != 'linux': + raise Exception("SoS is not supported on this platform") + if 'policy' not in cache: - cache['policy'] = GenericPolicy() + cache['policy'] = sos.policies.distros.GenericLinuxPolicy() return cache['policy'] -class ContainerRuntime(object): - """Encapsulates a container runtime that provides the ability to plugins to - check runtime status, check for the presence of specific containers, and - to format commands to run in those containers - - :param policy: The loaded policy for the system - :type policy: ``Policy()`` - - :cvar name: The name of the container runtime, e.g. 'podman' - :vartype name: ``str`` - - :cvar containers: A list of containers known to the runtime - :vartype containers: ``list`` - - :cvar images: A list of images known to the runtime - :vartype images: ``list`` - - :cvar binary: The binary command to run for the runtime, must exit within - $PATH - :vartype binary: ``str`` - """ - - name = 'Undefined' - containers = [] - images = [] - volumes = [] - binary = '' - active = False - - def __init__(self, policy=None): - self.policy = policy - self.run_cmd = "%s exec " % self.binary - - def load_container_info(self): - """If this runtime is found to be active, attempt to load information - on the objects existing in the runtime. - """ - self.containers = self.get_containers() - self.images = self.get_images() - self.volumes = self.get_volumes() - - def check_is_active(self): - """Check to see if the container runtime is both present AND active. - - Active in this sense means that the runtime can be used to glean - information about the runtime itself and containers that are running. - - :returns: ``True`` if the runtime is active, else ``False`` - :rtype: ``bool`` - """ - if is_executable(self.binary): - self.active = True - return True - return False - - def get_containers(self, get_all=False): - """Get a list of containers present on the system. - - :param get_all: If set, include stopped containers as well - :type get_all: ``bool`` - """ - containers = [] - _cmd = "%s ps %s" % (self.binary, '-a' if get_all else '') - if self.active: - out = sos_get_command_output(_cmd) - if out['status'] == 0: - for ent in out['output'].splitlines()[1:]: - ent = ent.split() - # takes the form (container_id, container_name) - containers.append((ent[0], ent[-1])) - return containers - - def get_container_by_name(self, name): - """Get the container ID for the container matching the provided - name - - :param name: The name of the container, note this can be a regex - :type name: ``str`` - - :returns: The id of the first container to match `name`, else ``None`` - :rtype: ``str`` - """ - if not self.active or name is None: - return None - for c in self.containers: - if re.match(name, c[1]): - return c[1] - return None - - def get_images(self): - """Get a list of images present on the system - - :returns: A list of 2-tuples containing (image_name, image_id) - :rtype: ``list`` - """ - images = [] - fmt = '{{lower .Repository}}:{{lower .Tag}} {{lower .ID}}' - if self.active: - out = sos_get_command_output("%s images --format '%s'" - % (self.binary, fmt)) - if out['status'] == 0: - for ent in out['output'].splitlines(): - ent = ent.split() - # takes the form (image_name, image_id) - images.append((ent[0], ent[1])) - return images - - def get_volumes(self): - """Get a list of container volumes present on the system - - :returns: A list of volume IDs on the system - :rtype: ``list`` - """ - vols = [] - if self.active: - out = sos_get_command_output("%s volume ls" % self.binary) - if out['status'] == 0: - for ent in out['output'].splitlines()[1:]: - ent = ent.split() - vols.append(ent[-1]) - return vols - - def fmt_container_cmd(self, container, cmd): - """Format a command to run inside a container using the runtime - - :param container: The name or ID of the container in which to run - :type container: ``str`` - - :param cmd: The command to run inside `container` - :type cmd: ``str`` - - :returns: Formatted string to run `cmd` inside `container` - :rtype: ``str`` - """ - return "%s %s %s" % (self.run_cmd, container, quote(cmd)) - - def get_logs_command(self, container): - """Get the command string used to dump container logs from the - runtime - - :param container: The name or ID of the container to get logs for - :type container: ``str`` - - :returns: Formatted runtime command to get logs from `container` - :type: ``str`` - """ - return "%s logs -t %s" % (self.binary, container) - - -class DockerContainerRuntime(ContainerRuntime): - """Runtime class to use for systems running Docker""" - - name = 'docker' - binary = 'docker' - - def check_is_active(self): - # the daemon must be running - if (is_executable('docker') and - self.policy.init_system.is_running('docker')): - self.active = True - return True - return False - - -class PodmanContainerRuntime(ContainerRuntime): - """Runtime class to use for systems running Podman""" - - name = 'podman' - binary = 'podman' - - -class InitSystem(object): - """Encapsulates an init system to provide service-oriented functions to - sos. - - This should be used to query the status of services, such as if they are - enabled or disabled on boot, or if the service is currently running. - - :param init_cmd: The binary used to interact with the init system - :type init_cmd: ``str`` - - :param list_cmd: The list subcmd given to `init_cmd` to list services - :type list_cmd: ``str`` - - :param query_cmd: The query subcmd given to `query_cmd` to query the - status of services - :type query_cmd: ``str`` - - """ - - def __init__(self, init_cmd=None, list_cmd=None, query_cmd=None): - """Initialize a new InitSystem()""" - - self.services = {} - - self.init_cmd = init_cmd - self.list_cmd = "%s %s" % (self.init_cmd, list_cmd) or None - self.query_cmd = "%s %s" % (self.init_cmd, query_cmd) or None - - def is_enabled(self, name): - """Check if given service name is enabled - - :param name: The name of the service - :type name: ``str`` - - :returns: ``True`` if the service is enabled, else ``False`` - :rtype: ``bool`` - """ - if self.services and name in self.services: - return self.services[name]['config'] == 'enabled' - return False - - def is_disabled(self, name): - """Check if a given service name is disabled - :param name: The name of the service - :type name: ``str`` - - :returns: ``True`` if the service is disabled, else ``False`` - :rtype: ``bool`` - """ - if self.services and name in self.services: - return self.services[name]['config'] == 'disabled' - return False - - def is_service(self, name): - """Checks if the given service name exists on the system at all, this - does not check for the service status - - :param name: The name of the service - :type name: ``str`` - - :returns: ``True`` if the service exists, else ``False`` - :rtype: ``bool`` - """ - return name in self.services - - def is_running(self, name): - """Checks if the given service name is in a running state. - - This should be overridden by initsystems that subclass InitSystem - - :param name: The name of the service - :type name: ``str`` - - :returns: ``True`` if the service is running, else ``False`` - :rtype: ``bool`` - """ - # This is going to be primarily used in gating if service related - # commands are going to be run or not. Default to always returning - # True when an actual init system is not specified by policy so that - # we don't inadvertantly restrict sosreports on those systems - return True - - def load_all_services(self): - """This loads all services known to the init system into a dict. - The dict should be keyed by the service name, and contain a dict of the - name and service status - - This must be overridden by anything that subclasses `InitSystem` in - order for service methods to function properly - """ - pass - - def _query_service(self, name): - """Query an individual service""" - if self.query_cmd: - try: - return sos_get_command_output("%s %s" % (self.query_cmd, name)) - except Exception: - return None - return None - - def parse_query(self, output): - """Parses the output returned by the query command to make a - determination of what the state of the service is - - This should be overriden by anything that subclasses InitSystem - - :param output: The raw output from querying the service with the - configured `query_cmd` - :type output: ``str`` - - :returns: A state for the service, e.g. 'active', 'disabled', etc... - :rtype: ``str`` - """ - return output - - def get_service_names(self, regex): - """Get a list of all services discovered on the system that match the - given regex. - - :param regex: The service name regex to match against - :type regex: ``str`` - """ - reg = re.compile(regex, re.I) - return [s for s in self.services.keys() if reg.match(s)] - - def get_service_status(self, name): - """Get the status for the given service name along with the output - of the query command - - :param name: The name of the service - :type name: ``str`` - - :returns: Service status and query_cmd output from the init system - :rtype: ``dict`` with keys `name`, `status`, and `output` - """ - _default = { - 'name': name, - 'status': 'missing', - 'output': '' - } - if name not in self.services: - return _default - if 'status' in self.services[name]: - # service status has been queried before, return existing info - return self.services[name] - svc = self._query_service(name) - if svc is not None: - self.services[name]['status'] = self.parse_query(svc['output']) - self.services[name]['output'] = svc['output'] - return self.services[name] - return _default - - -class SystemdInit(InitSystem): - """InitSystem abstraction for SystemD systems""" - - def __init__(self): - super(SystemdInit, self).__init__( - init_cmd='systemctl', - list_cmd='list-unit-files --type=service', - query_cmd='status' - ) - self.load_all_services() - - def parse_query(self, output): - for line in output.splitlines(): - if line.strip().startswith('Active:'): - return line.split()[1] - return 'unknown' - - def load_all_services(self): - svcs = shell_out(self.list_cmd).splitlines()[1:] - for line in svcs: - try: - name = line.split('.service')[0] - config = line.split()[1] - self.services[name] = { - 'name': name, - 'config': config - } - except IndexError: - pass - - def is_running(self, name): - svc = self.get_service_status(name) - return svc['status'] == 'active' - - -class PackageManager(object): - """Encapsulates a package manager. If you provide a query_command to the - constructor it should print each package on the system in the following - format:: - - package name|package.version - - You may also subclass this class and provide a get_pkg_list method to - build the list of packages and versions. - - :cvar query_command: The command to use for querying packages - :vartype query_command: ``str`` or ``None`` - - :cvar verify_command: The command to use for verifying packages - :vartype verify_command: ``str`` or ``None`` - - :cvar verify_filter: Optional filter to use for controlling package - verification - :vartype verify_filter: ``str or ``None`` - - :cvar files_command: The command to use for getting file lists for packages - :vartype files_command: ``str`` or ``None`` - - :cvar chroot: Perform a chroot when executing `files_command` - :vartype chroot: ``bool`` - - :cvar remote_exec: If package manager is on a remote system (e.g. for - sos collect), prepend this SSH command to run remotely - :vartype remote_exec: ``str`` or ``None`` - """ - - query_command = None - verify_command = None - verify_filter = None - chroot = None - files = None - - def __init__(self, chroot=None, query_command=None, - verify_command=None, verify_filter=None, - files_command=None, remote_exec=None): - self.packages = {} - self.files = [] - - self.query_command = query_command if query_command else None - self.verify_command = verify_command if verify_command else None - self.verify_filter = verify_filter if verify_filter else None - self.files_command = files_command if files_command else None - - # if needed, append the remote command to these so that this returns - # the remote package details, not local - if remote_exec: - for cmd in ['query_command', 'verify_command', 'files_command']: - if getattr(self, cmd) is not None: - _cmd = getattr(self, cmd) - setattr(self, cmd, "%s %s" % (remote_exec, quote(_cmd))) - - if chroot: - self.chroot = chroot - - def all_pkgs_by_name(self, name): - """ - Get a list of packages that match name. - - :param name: The name of the package - :type name: ``str`` - - :returns: List of all packages matching `name` - :rtype: ``list`` - """ - return fnmatch.filter(self.all_pkgs().keys(), name) - - def all_pkgs_by_name_regex(self, regex_name, flags=0): - """ - Get a list of packages that match regex_name. - - :param regex_name: The regex to use for matching package names against - :type regex_name: ``str`` - - :param flags: Flags for the `re` module when matching `regex_name` - - :returns: All packages matching `regex_name` - :rtype: ``list`` - """ - reg = re.compile(regex_name, flags) - return [pkg for pkg in self.all_pkgs().keys() if reg.match(pkg)] - - def pkg_by_name(self, name): - """ - Get a single package that matches name. - - :param name: The name of the package - :type name: ``str`` - - :returns: The first package that matches `name` - :rtype: ``str`` - """ - pkgmatches = self.all_pkgs_by_name(name) - if (len(pkgmatches) != 0): - return self.all_pkgs_by_name(name)[-1] - else: - return None - - def get_pkg_list(self): - """Returns a dictionary of packages in the following - format:: - - {'package_name': {'name': 'package_name', - 'version': 'major.minor.version'}} - - """ - if self.query_command: - cmd = self.query_command - pkg_list = shell_out( - cmd, timeout=0, chroot=self.chroot - ).splitlines() - - for pkg in pkg_list: - if '|' not in pkg: - continue - elif pkg.count("|") == 1: - name, version = pkg.split("|") - release = None - elif pkg.count("|") == 2: - name, version, release = pkg.split("|") - self.packages[name] = { - 'name': name, - 'version': version.split(".") - } - release = release if release else None - self.packages[name]['release'] = release - - return self.packages - - def pkg_version(self, pkg): - """Returns the entry in self.packages for pkg if it exists - - :param pkg: The name of the package - :type pkg: ``str`` - - :returns: Package name and version, if package exists - :rtype: ``dict`` if found, else ``None`` - """ - pkgs = self.all_pkgs() - if pkg in pkgs: - return pkgs[pkg] - return None - - def all_pkgs(self): - """ - Get a list of all packages. - - :returns: All packages, with name and version, installed on the system - :rtype: ``dict`` - """ - if not self.packages: - self.packages = self.get_pkg_list() - return self.packages - - def pkg_nvra(self, pkg): - """Get the name, version, release, and architecture for a package - - :param pkg: The name of the package - :type pkg: ``str`` - - :returns: name, version, release, and arch of the package - :rtype: ``tuple`` - """ - fields = pkg.split("-") - version, release, arch = fields[-3:] - name = "-".join(fields[:-3]) - return (name, version, release, arch) - - def all_files(self): - """ - Get a list of files known by the package manager - - :returns: All files known by the package manager - :rtype: ``list`` - """ - if self.files_command and not self.files: - cmd = self.files_command - files = shell_out(cmd, timeout=0, chroot=self.chroot) - self.files = files.splitlines() - return self.files - - def build_verify_command(self, packages): - """build_verify_command(self, packages) -> str - Generate a command to verify the list of packages given - in ``packages`` using the native package manager's - verification tool. - - The command to be executed is returned as a string that - may be passed to a command execution routine (for e.g. - ``sos_get_command_output()``. - - :param packages: a string, or a list of strings giving - package names to be verified. - :returns: a string containing an executable command - that will perform verification of the given - packages. - :rtype: str or ``NoneType`` - """ - if not self.verify_command: - return None - - # The re.match(pkg) used by all_pkgs_by_name_regex() may return - # an empty list (`[[]]`) when no package matches: avoid building - # an rpm -V command line with the empty string as the package - # list in this case. - by_regex = self.all_pkgs_by_name_regex - verify_list = filter(None, map(by_regex, packages)) - - # No packages after regex match? - if not verify_list: - return None - - verify_packages = "" - for package_list in verify_list: - for package in package_list: - if any([f in package for f in self.verify_filter]): - continue - if len(verify_packages): - verify_packages += " " - verify_packages += package - return self.verify_command + " " + verify_packages - - -#: Constants for on-disk preset fields -DESC = "desc" -NOTE = "note" -OPTS = "args" - - -class PresetDefaults(object): - """Preset command line defaults to allow for quick reference to sets of - commonly used options - - :param name: The name of the new preset - :type name: ``str`` - - :param desc: A description for the new preset - :type desc: ``str`` - - :param note: Note for the new preset - :type note: ``str`` - - :param opts: Options set for the new preset - :type opts: ``SoSOptions`` - """ - #: Preset name, used for selection - name = None - #: Human readable preset description - desc = None - #: Notes on preset behaviour - note = None - #: Options set for this preset - opts = SoSOptions() - - #: ``True`` if this preset if built-in or ``False`` otherwise. - builtin = True - - def __str__(self): - """Return a human readable string representation of this - ``PresetDefaults`` object. - """ - return ("name=%s desc=%s note=%s opts=(%s)" % - (self.name, self.desc, self.note, str(self.opts))) - - def __repr__(self): - """Return a machine readable string representation of this - ``PresetDefaults`` object. - """ - return ("PresetDefaults(name='%s' desc='%s' note='%s' opts=(%s)" % - (self.name, self.desc, self.note, repr(self.opts))) - - def __init__(self, name="", desc="", note=None, opts=SoSOptions()): - """Initialise a new ``PresetDefaults`` object with the specified - arguments. - - :returns: The newly initialised ``PresetDefaults`` - """ - self.name = name - self.desc = desc - self.note = note - self.opts = opts - - def write(self, presets_path): - """Write this preset to disk in JSON notation. - - :param presets_path: the directory where the preset will be written - :type presets_path: ``str`` - """ - if self.builtin: - raise TypeError("Cannot write built-in preset") - - # Make dictionaries of PresetDefaults values - odict = self.opts.dict() - pdict = {self.name: {DESC: self.desc, NOTE: self.note, OPTS: odict}} - - if not os.path.exists(presets_path): - os.makedirs(presets_path, mode=0o755) - - with open(os.path.join(presets_path, self.name), "w") as pfile: - json.dump(pdict, pfile) - - def delete(self, presets_path): - """Delete a preset from disk - - :param presets_path: the directory where the preset is saved - :type presets_path: ``str`` - """ - os.unlink(os.path.join(presets_path, self.name)) - - -NO_PRESET = 'none' -NO_PRESET_DESC = 'Do not load a preset' -NO_PRESET_NOTE = 'Use to disable automatically loaded presets' - -GENERIC_PRESETS = { - NO_PRESET: PresetDefaults(name=NO_PRESET, desc=NO_PRESET_DESC, - note=NO_PRESET_NOTE, opts=SoSOptions()) - } - - -class Policy(object): +class Policy(): """Policies represent distributions that sos supports, and define the way in which sos behaves on those distributions. A policy should define at minimum a way to identify the distribution, and a package manager to allow @@ -761,14 +69,20 @@ class Policy(object): :param probe_runtime: Should the Policy try to load a ContainerRuntime :type probe_runtime: ``bool`` + :param remote_exec: If this policy is loaded for a remote node, use + this to facilitate executing commands via the + SoSTransport in use + :type remote_exec: ``SoSTranport.run_command()`` + :cvar distro: The name of the distribution the Policy represents :vartype distro: ``str`` :cvar vendor: The name of the vendor producing the distribution :vartype vendor: ``str`` - :cvar vendor_url: URL for the vendor's website, or support portal - :vartype vendor_url: ``str`` + :cvar vendor_urls: List of URLs for the vendor's website, or support portal + :vartype vendor_urls: ``list`` of ``tuples`` formatted + ``(``description``, ``url``)`` :cvar vendor_text: Additional text to add to the banner message :vartype vendor_text: ``str`` @@ -786,7 +100,7 @@ from this %(distro)s system. For more information on %(vendor)s visit: - %(vendor_url)s + %(vendor_urls)s The generated archive may contain data considered sensitive and its content \ should be reviewed by the originating organization before being passed to \ @@ -799,7 +113,7 @@ any third party. distro = "Unknown" vendor = "Unknown" - vendor_url = "http://www.example.com/" + vendor_urls = [('Example URL', "http://www.example.com/")] vendor_text = "" PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" default_scl_prefix = "" @@ -807,38 +121,26 @@ any third party. presets = {"": PresetDefaults()} presets_path = PRESETS_PATH _in_container = False - _host_sysroot = '/' - def __init__(self, sysroot=None, probe_runtime=True): + def __init__(self, sysroot=None, probe_runtime=True, remote_exec=None): """Subclasses that choose to override this initializer should call super() to ensure that they get the required platform bits attached. super(SubClass, self).__init__(). Policies that require runtime tests to construct PATH must call self.set_exec_path() after modifying PATH in their own initializer.""" + self.soslog = logging.getLogger('sos') + self.ui_log = logging.getLogger('sos_ui') self._parse_uname() self.case_id = None self.probe_runtime = probe_runtime self.package_manager = PackageManager() - self._valid_subclasses = [] - self.set_exec_path() - self._host_sysroot = sysroot + self.valid_subclasses = [IndependentPlugin] + self.remote_exec = remote_exec + if not self.remote_exec: + self.set_exec_path() + self.sysroot = sysroot self.register_presets(GENERIC_PRESETS) - def get_valid_subclasses(self): - return [IndependentPlugin] + self._valid_subclasses - - def set_valid_subclasses(self, subclasses): - self._valid_subclasses = subclasses - - def del_valid_subclasses(self): - del self._valid_subclasses - - valid_subclasses = property(get_valid_subclasses, - set_valid_subclasses, - del_valid_subclasses, - "list of subclasses that this policy can " - "process") - def check(self, remote=''): """ This function is responsible for determining if the underlying system @@ -853,6 +155,35 @@ any third party. """ return False + @property + def forbidden_paths(self): + """This property is used to determine the list of forbidden paths + set by the policy. Note that this property will construct a + *cumulative* list based on all subclasses of a given policy. + + :returns: All patterns of policy forbidden paths + :rtype: ``list`` + """ + if not hasattr(self, '_forbidden_paths'): + self._forbidden_paths = [] + for cls in self.__class__.__mro__: + if hasattr(cls, 'set_forbidden_paths'): + self._forbidden_paths.extend(cls.set_forbidden_paths()) + return list(set(self._forbidden_paths)) + + @classmethod + def set_forbidden_paths(cls): + """Use this to *append* policy-specifc forbidden paths that apply to + all plugins. Setting this classmethod on an invidual policy will *not* + override subclass-specific paths + """ + return [ + '*.egg', + '*.pyc', + '*.pyo', + '*.swp' + ] + def in_container(self): """Are we running inside a container? @@ -861,14 +192,6 @@ any third party. """ return self._in_container - def host_sysroot(self): - """Get the host's default sysroot - - :returns: Host sysroot - :rtype: ``str`` or ``None`` - """ - return self._host_sysroot - def dist_version(self): """ Return the OS version @@ -945,24 +268,6 @@ any third party. def _get_pkg_name_for_binary(self, binary): return binary - def get_cmd_for_compress_method(self, method, threads): - """Determine the command to use for compressing the archive - - :param method: The compression method/binary to use - :type method: ``str`` - - :param threads: Number of threads compression should use - :type threads: ``int`` - - :returns: Full command to use to compress the archive - :rtype: ``str`` - """ - cmd = method - if cmd.startswith("xz"): - # XZ set compression to -2 and use threads - cmd = "%s -2 -T%d" % (cmd, threads) - return cmd - def get_tmp_dir(self, opt_tmp_dir): if not opt_tmp_dir: return tempfile.gettempdir() @@ -978,9 +283,9 @@ any third party. :param plugin_classes: The classes that the Plugin subclasses :type plugin_classes: ``list`` - :returns: The first subclass that matches one of the Policy's + :returns: The first tagging class that matches one of the Policy's `valid_subclasses` - :rtype: A tagging class for Plugins + :rtype: ``PluginDistroTag`` """ if len(plugin_classes) > 1: for p in plugin_classes: @@ -996,7 +301,7 @@ any third party. Verifies that the plugin_class should execute under this policy :param plugin_class: The tagging class being checked - :type plugin_class: A Plugin() tagging class + :type plugin_class: ``PluginDistroTag`` :returns: ``True`` if the `plugin_class` is allowed by the policy :rtype: ``bool`` @@ -1062,7 +367,50 @@ any third party. def get_preferred_hash_name(self): """Returns the string name of the hashlib-supported checksum algorithm to use""" - return "md5" + return "sha256" + + @classmethod + def display_help(self, section): + section.set_title('SoS Policies') + section.add_text( + 'Policies help govern how SoS operates on across different distri' + 'butions of Linux. They control aspects such as plugin enablement,' + ' $PATH determination, how/which package managers are queried, ' + 'default upload specifications, and more.' + ) + + section.add_text( + "When SoS intializes most functions, for example %s and %s, one " + "of the first operations is to determine the correct policy to " + "load for the local system. Policies will determine the proper " + "package manager to use, any applicable container runtime(s), and " + "init systems so that SoS and report plugins can properly function" + " for collections. Generally speaking a single policy will map to" + " a single distribution; for example there are separate policies " + "for Debian, Ubuntu, RHEL, and Fedora." + % (bold('sos report'), bold('sos collect')) + ) + + section.add_text( + "It is currently not possible for users to directly control which " + "policy is loaded." + ) + + pols = { + 'policies.cos': 'The Google Cloud-Optimized OS distribution', + 'policies.debian': 'The Debian distribution', + 'policies.redhat': ('Red Hat family distributions, not necessarily' + ' including forks'), + 'policies.ubuntu': 'Ubuntu/Canonical distributions' + } + + seealso = section.add_section('See Also') + seealso.add_text( + "For more information on distribution policies, see below\n" + ) + for pol in pols: + seealso.add_text("{:>8}{:<20}{:<30}".format(' ', pol, pols[pol]), + newline=False) def display_results(self, archive, directory, checksum, archivestat=None, map_file=None): @@ -1084,44 +432,41 @@ any third party. file for this run :type map_file: ``str`` """ - # Logging is already shutdown and all terminal output must use the - # print() call. + # Logging is shut down, but there are some edge cases where automation + # does not capture printed output (e.g. avocado CI). Use the ui_log to + # still print to console in this case. # make sure a report exists if not archive and not directory: return False - self._print() - if map_file: - self._print(_("A mapping of obfuscated elements is available at" - "\n\t%s\n" % map_file)) + self.ui_log.info( + _(f"\nA mapping of obfuscated elements is available at" + f"\n\t{map_file}") + ) if archive: - self._print(_("Your sosreport has been generated and saved " - "in:\n\t%s\n") % archive, always=True) - self._print(_(" Size\t%s") % - get_human_readable(archivestat.st_size)) - self._print(_(" Owner\t%s") % - getpwuid(archivestat.st_uid).pw_name) + self.ui_log.info( + _(f"\nYour sosreport has been generated and saved in:" + f"\n\t{archive}\n") + ) + self.ui_log.info( + _(f" Size\t{get_human_readable(archivestat.st_size)}") + ) + self.ui_log.info( + _(f" Owner\t{getpwuid(archivestat.st_uid).pw_name}") + ) else: - self._print(_("Your sosreport build tree has been generated " - "in:\n\t%s\n") % directory, always=True) + self.ui_log.info( + _(f"Your sosreport build tree has been generated in:" + f"\n\t{directory}\n") + ) if checksum: - self._print(" " + self.get_preferred_hash_name() + "\t" + checksum) - self._print() - self._print(_("Please send this file to your support " - "representative.")) - self._print() - - def _print(self, msg=None, always=False): - """A wrapper around print that only prints if we are not running in - quiet mode""" - if always or not self.commons['cmdlineopts'].quiet: - if msg: - print(msg) - else: - print() + self.ui_log.info(f" {self.get_preferred_hash_name()}\t{checksum}") + self.ui_log.info( + _("\nPlease send this file to your support representative.\n") + ) def get_msg(self): """This method is used to prepare the preamble text to display to @@ -1138,7 +483,7 @@ any third party. changes_text = "No changes will be made to system configuration." width = 72 _msg = self.msg % {'distro': self.distro, 'vendor': self.vendor, - 'vendor_url': self.vendor_url, + 'vendor_urls': self._fmt_vendor_urls(), 'vendor_text': self.vendor_text, 'tmpdir': self.commons['tmpdir'], 'changes_text': changes_text} @@ -1147,6 +492,19 @@ any third party. _fmt = _fmt + fill(line, width, replace_whitespace=False) + '\n' return _fmt + def _fmt_vendor_urls(self): + """Formats all items in the ``vendor_urls`` class attr into a usable + string for the banner message. + + :returns: Formatted string of URLS + :rtype: ``str`` + """ + width = max([len(v[0]) for v in self.vendor_urls]) + return "\n".join("\t{desc:<{width}} : {url}".format( + desc=u[0], width=width, url=u[1]) + for u in self.vendor_urls + ) + def register_presets(self, presets, replace=False): """Add new presets to this policy object. @@ -1252,511 +610,4 @@ any third party. self.presets.pop(name) -class GenericPolicy(Policy): - """This Policy will be returned if no other policy can be loaded. This - should allow for IndependentPlugins to be executed on any system""" - - def get_msg(self): - return self.msg % {'distro': self.system} - - -class LinuxPolicy(Policy): - """This policy is meant to be an abc class that provides common - implementations used in Linux distros""" - - distro = "Linux" - vendor = "None" - PATH = "/bin:/sbin:/usr/bin:/usr/sbin" - init = None - # _ prefixed class attrs are used for storing any vendor-defined defaults - # the non-prefixed attrs are used by the upload methods, and will be set - # to the cmdline/config file values, if provided. If not provided, then - # those attrs will be set to the _ prefixed values as a fallback. - # TL;DR Use _upload_* for policy default values, use upload_* when wanting - # to actual use the value in a method/override - _upload_url = None - _upload_directory = '/' - _upload_user = None - _upload_password = None - _use_https_streaming = False - default_container_runtime = 'docker' - _preferred_hash_name = None - upload_url = None - upload_user = None - upload_password = None - # collector-focused class attrs - containerized = False - container_image = None - sos_path_strip = None - sos_pkg_name = None - sos_bin_path = None - sos_container_name = 'sos-collector-tmp' - container_version_command = None - - def __init__(self, sysroot=None, init=None, probe_runtime=True): - super(LinuxPolicy, self).__init__(sysroot=sysroot, - probe_runtime=probe_runtime) - self.init_kernel_modules() - - if init is not None: - self.init_system = init - elif os.path.isdir("/run/systemd/system/"): - self.init_system = SystemdInit() - else: - self.init_system = InitSystem() - - self.runtimes = {} - if self.probe_runtime: - _crun = [ - PodmanContainerRuntime(policy=self), - DockerContainerRuntime(policy=self) - ] - for runtime in _crun: - if runtime.check_is_active(): - self.runtimes[runtime.name] = runtime - if runtime.name == self.default_container_runtime: - self.runtimes['default'] = self.runtimes[runtime.name] - self.runtimes[runtime.name].load_container_info() - - if self.runtimes and 'default' not in self.runtimes.keys(): - # still allow plugins to query a runtime present on the system - # even if that is not the policy default one - idx = list(self.runtimes.keys()) - self.runtimes['default'] = self.runtimes[idx[0]] - - def get_preferred_hash_name(self): - - if self._preferred_hash_name: - return self._preferred_hash_name - - checksum = "md5" - try: - fp = open("/proc/sys/crypto/fips_enabled", "r") - except IOError: - self._preferred_hash_name = checksum - return checksum - - fips_enabled = fp.read() - if fips_enabled.find("1") >= 0: - checksum = "sha256" - fp.close() - self._preferred_hash_name = checksum - return checksum - - def default_runlevel(self): - try: - with open("/etc/inittab") as fp: - pattern = r"id:(\d{1}):initdefault:" - text = fp.read() - return int(re.findall(pattern, text)[0]) - except (IndexError, IOError): - return 3 - - def kernel_version(self): - return self.release - - def host_name(self): - return self.hostname - - def is_kernel_smp(self): - return self.smp - - def get_arch(self): - return self.machine - - def get_local_name(self): - """Returns the name usd in the pre_work step""" - return self.host_name() - - def sanitize_filename(self, name): - return re.sub(r"[^-a-z,A-Z.0-9]", "", name) - - def init_kernel_modules(self): - """Obtain a list of loaded kernel modules to reference later for plugin - enablement and SoSPredicate checks - """ - lines = shell_out("lsmod", timeout=0).splitlines() - self.kernel_mods = [line.split()[0].strip() for line in lines] - - def pre_work(self): - # this method will be called before the gathering begins - - cmdline_opts = self.commons['cmdlineopts'] - caseid = cmdline_opts.case_id if cmdline_opts.case_id else "" - - # Set the cmdline settings to the class attrs that are referenced later - # The policy default '_' prefixed versions of these are untouched to - # allow fallback - self.upload_url = cmdline_opts.upload_url - self.upload_user = cmdline_opts.upload_user - self.upload_directory = cmdline_opts.upload_directory - self.upload_password = cmdline_opts.upload_pass - - if not cmdline_opts.batch and not \ - cmdline_opts.quiet: - try: - if caseid: - self.case_id = caseid - else: - self.case_id = input(_("Please enter the case id " - "that you are generating this " - "report for [%s]: ") % caseid) - # Policies will need to handle the prompts for user information - if cmdline_opts.upload or self.upload_url: - self.prompt_for_upload_user() - self.prompt_for_upload_password() - self._print() - except KeyboardInterrupt: - self._print() - raise - - if cmdline_opts.case_id: - self.case_id = cmdline_opts.case_id - - return - - def prompt_for_upload_user(self): - """Should be overridden by policies to determine if a user needs to - be provided or not - """ - if not self.upload_user and not self._upload_user: - msg = "Please provide upload user for %s: " % self.get_upload_url() - self.upload_user = input(_(msg)) - - def prompt_for_upload_password(self): - """Should be overridden by policies to determine if a password needs to - be provided for upload or not - """ - if ((not self.upload_password and not self._upload_password) and - self.upload_user): - msg = ( - "Please provide the upload password for %s: " - % self.upload_user - ) - self.upload_password = getpass(msg) - - def upload_archive(self, archive): - """ - Entry point for sos attempts to upload the generated archive to a - policy or user specified location. - - Curerntly there is support for HTTPS, SFTP, and FTP. HTTPS uploads are - preferred for policy-defined defaults. - - Policies that need to override uploading methods should override the - respective upload_https(), upload_sftp(), and/or upload_ftp() methods - and should NOT override this method. - - :param archive: The archive filepath to use for upload - :type archive: ``str`` - - In order to enable this for a policy, that policy needs to implement - the following: - - Required Class Attrs - - :_upload_url: The default location to use. Note these MUST include - protocol header - :_upload_user: Default username, if any else None - :_upload_password: Default password, if any else None - :_use_https_streaming: Set to True if the HTTPS endpoint supports - streaming data - - The following Class Attrs may optionally be overidden by the Policy - - :_upload_directory: Default FTP server directory, if any - - - The following methods may be overridden by ``Policy`` as needed - - `prompt_for_upload_user()` - Determines if sos should prompt for a username or not. - - `get_upload_user()` - Determines if the default or a different username should be used - - `get_upload_https_auth()` - Format authentication data for HTTPS uploads - - `get_upload_url_string()` - Print a more human-friendly string than vendor URLs - """ - self.upload_archive = archive - self.upload_url = self.get_upload_url() - if not self.upload_url: - raise Exception("No upload destination provided by policy or by " - "--upload-url") - upload_func = self._determine_upload_type() - print(_("Attempting upload to %s" % self.get_upload_url_string())) - return upload_func() - - def _determine_upload_type(self): - """Based on the url provided, determine what type of upload to attempt. - - Note that this requires users to provide a FQDN address, such as - https://myvendor.com/api or ftp://myvendor.com instead of - myvendor.com/api or myvendor.com - """ - prots = { - 'ftp': self.upload_ftp, - 'sftp': self.upload_sftp, - 'https': self.upload_https - } - if '://' not in self.upload_url: - raise Exception("Must provide protocol in upload URL") - prot, url = self.upload_url.split('://') - if prot not in prots.keys(): - raise Exception("Unsupported or unrecognized protocol: %s" % prot) - return prots[prot] - - def get_upload_https_auth(self, user=None, password=None): - """Formats the user/password credentials using basic auth - - :param user: The username for upload - :type user: ``str`` - - :param password: Password for `user` to use for upload - :type password: ``str`` - - :returns: The user/password auth suitable for use in reqests calls - :rtype: ``requests.auth.HTTPBasicAuth()`` - """ - if not user: - user = self.get_upload_user() - if not password: - password = self.get_upload_password() - - return requests.auth.HTTPBasicAuth(user, password) - - def get_upload_url(self): - """Helper function to determine if we should use the policy default - upload url or one provided by the user - - :returns: The URL to use for upload - :rtype: ``str`` - """ - return self.upload_url or self._upload_url - - def get_upload_url_string(self): - """Used by distro policies to potentially change the string used to - report upload location from the URL to a more human-friendly string - """ - return self.get_upload_url() - - def get_upload_user(self): - """Helper function to determine if we should use the policy default - upload user or one provided by the user - - :returns: The username to use for upload - :rtype: ``str`` - """ - return self.upload_user or self._upload_user - - def get_upload_password(self): - """Helper function to determine if we should use the policy default - upload password or one provided by the user - - :returns: The password to use for upload - :rtype: ``str`` - """ - return self.upload_password or self._upload_password - - def upload_sftp(self): - """Attempts to upload the archive to an SFTP location. - - Due to the lack of well maintained, secure, and generally widespread - python libraries for SFTP, sos will shell-out to the system's local ssh - installation in order to handle these uploads. - - Do not override this method with one that uses python-paramiko, as the - upstream sos team will reject any PR that includes that dependency. - """ - raise NotImplementedError("SFTP support is not yet implemented") - - def _upload_https_streaming(self, archive): - """If upload_https() needs to use requests.put(), this method is used - to provide streaming functionality - - Policies should override this method instead of the base upload_https() - - :param archive: The open archive file object - """ - return requests.put(self.get_upload_url(), data=archive, - auth=self.get_upload_https_auth()) - - def _get_upload_headers(self): - """Define any needed headers to be passed with the POST request here - """ - return {} - - def _upload_https_no_stream(self, archive): - """If upload_https() needs to use requests.post(), this method is used - to provide non-streaming functionality - - Policies should override this method instead of the base upload_https() - - :param archive: The open archive file object - """ - files = { - 'file': (archive.name.split('/')[-1], archive, - self._get_upload_headers()) - } - return requests.post(self.get_upload_url(), files=files, - auth=self.get_upload_https_auth()) - - def upload_https(self): - """Attempts to upload the archive to an HTTPS location. - - Policies may define whether this upload attempt should use streaming - or non-streaming data by setting the `use_https_streaming` class - attr to True - - :returns: ``True`` if upload is successful - :rtype: ``bool`` - - :raises: ``Exception`` if upload was unsuccessful - """ - if not REQUESTS_LOADED: - raise Exception("Unable to upload due to missing python requests " - "library") - - with open(self.upload_archive, 'rb') as arc: - if not self._use_https_streaming: - r = self._upload_https_no_stream(arc) - else: - r = self._upload_https_streaming(arc) - if r.status_code != 201: - if r.status_code == 401: - raise Exception( - "Authentication failed: invalid user credentials" - ) - raise Exception("POST request returned %s: %s" - % (r.status_code, r.reason)) - return True - - def upload_ftp(self, url=None, directory=None, user=None, password=None): - """Attempts to upload the archive to either the policy defined or user - provided FTP location. - - :param url: The URL to upload to - :type url: ``str`` - - :param directory: The directory on the FTP server to write to - :type directory: ``str`` or ``None`` - - :param user: The user to authenticate with - :type user: ``str`` - - :param password: The password to use for `user` - :type password: ``str`` - - :returns: ``True`` if upload is successful - :rtype: ``bool`` - - :raises: ``Exception`` if upload in unsuccessful - """ - try: - import ftplib - import socket - except ImportError: - # socket is part of the standard library, should only fail here on - # ftplib - raise Exception("missing python ftplib library") - - if not url: - url = self.get_upload_url() - if url is None: - raise Exception("no FTP server specified by policy, use --upload-" - "url to specify a location") - - url = url.replace('ftp://', '') - - if not user: - user = self.get_upload_user() - - if not password: - password = self.get_upload_password() - - if not directory: - directory = self._upload_directory - - try: - session = ftplib.FTP(url, user, password) - session.cwd(directory) - except socket.gaierror: - raise Exception("unable to connect to %s" % url) - except ftplib.error_perm as err: - errno = str(err).split()[0] - if errno == 503: - raise Exception("could not login as '%s'" % user) - if errno == 550: - raise Exception("could not set upload directory to %s" - % directory) - - try: - with open(self.upload_archive, 'rb') as _arcfile: - session.storbinary( - "STOR %s" % self.upload_archive.split('/')[-1], - _arcfile - ) - session.quit() - return True - except IOError: - raise Exception("could not open archive file") - - def set_sos_prefix(self): - """If sosreport commands need to always be prefixed with something, - for example running in a specific container image, then it should be - defined here. - - If no prefix should be set, return an empty string instead of None. - """ - return '' - - def set_cleanup_cmd(self): - """If a host requires additional cleanup, the command should be set and - returned here - """ - return '' - - def create_sos_container(self): - """Returns the command that will create the container that will be - used for running commands inside a container on hosts that require it. - - This will use the container runtime defined for the host type to - launch a container. From there, we use the defined runtime to exec into - the container's namespace. - """ - return '' - - def restart_sos_container(self): - """Restarts the container created for sos collect if it has stopped. - - This is called immediately after create_sos_container() as the command - to create the container will exit and the container will stop. For - current container runtimes, subsequently starting the container will - default to opening a bash shell in the container to keep it running, - thus allowing us to exec into it again. - """ - return "%s start %s" % (self.container_runtime, - self.sos_container_name) - - def format_container_command(self, cmd): - """Returns the command that allows us to exec into the created - container for sos collect. - - :param cmd: The command to run in the sos container - :type cmd: ``str`` - - :returns: The command to execute to run `cmd` in the container - :rtype: ``str`` - """ - if self.container_runtime: - return '%s exec %s %s' % (self.container_runtime, - self.sos_container_name, - cmd) - else: - return cmd - - # vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/amazon.py 4.5.3ubuntu2/sos/policies/amazon.py --- 4.0-2/sos/policies/amazon.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/amazon.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,43 +0,0 @@ -# Copyright (C) Red Hat, Inc. 2019 - -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information. - -from sos.policies.redhat import RedHatPolicy, OS_RELEASE -import os - - -class AmazonPolicy(RedHatPolicy): - - distro = "Amazon Linux" - vendor = "Amazon" - vendor_url = "https://aws.amazon.com" - - def __init__(self, sysroot=None, init=None, probe_runtime=True, - remote_exec=None): - super(AmazonPolicy, self).__init__(sysroot=sysroot, init=init, - probe_runtime=probe_runtime, - remote_exec=remote_exec) - - @classmethod - def check(cls, remote=''): - - if remote: - return cls.distro in remote - - if not os.path.exists(OS_RELEASE): - return False - - with open(OS_RELEASE, 'r') as f: - for line in f: - if line.startswith('NAME'): - if 'Amazon Linux' in line: - return True - return False - -# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/cos.py 4.5.3ubuntu2/sos/policies/cos.py --- 4.0-2/sos/policies/cos.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/cos.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,49 +0,0 @@ -# Copyright (C) Red Hat, Inc. 2020 - -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information. - -from sos.report.plugins import CosPlugin -from sos.policies import LinuxPolicy - - -def _blank_or_comment(line): - """Test whether line is empty of contains a comment. - - Test whether the ``line`` argument is either blank, or a - whole-line comment. - - :param line: the line of text to be checked. - :returns: ``True`` if the line is blank or a comment, - and ``False`` otherwise. - :rtype: bool - """ - return not line.strip() or line.lstrip().startswith('#') - - -class CosPolicy(LinuxPolicy): - distro = "Container-Optimized OS" - vendor = "Google Cloud Platform" - vendor_url = "https://cloud.google.com/container-optimized-os/" - valid_subclasses = [CosPlugin] - PATH = "/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin" - - @classmethod - def check(cls, remote=''): - if remote: - return cls.distro in remote - - try: - with open('/etc/os-release', 'r') as fp: - os_release = dict(line.strip().split('=') for line in fp - if not _blank_or_comment(line)) - return os_release['ID'] == 'cos' - except (IOError, KeyError): - return False - -# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/debian.py 4.5.3ubuntu2/sos/policies/debian.py --- 4.0-2/sos/policies/debian.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/debian.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,63 +0,0 @@ -from sos.report.plugins import DebianPlugin -from sos.policies import PackageManager, LinuxPolicy - -import os - - -class DebianPolicy(LinuxPolicy): - distro = "Debian" - vendor = "the Debian project" - vendor_url = "https://www.debian.org/" - ticket_number = "" - _debq_cmd = "dpkg-query -W -f='${Package}|${Version}\\n'" - _debv_cmd = "dpkg --verify" - _debv_filter = "" - name_pattern = 'friendly' - valid_subclasses = [DebianPlugin] - PATH = "/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" \ - + ":/usr/local/sbin:/usr/local/bin" - sos_pkg_name = 'sosreport' - sos_bin_path = '/usr/bin/sosreport' - - def __init__(self, sysroot=None, init=None, probe_runtime=True, - remote_exec=None): - super(DebianPolicy, self).__init__(sysroot=sysroot, init=init, - probe_runtime=probe_runtime) - self.ticket_number = "" - self.package_manager = PackageManager(query_command=self._debq_cmd, - verify_command=self._debv_cmd, - verify_filter=self._debv_filter, - chroot=sysroot, - remote_exec=remote_exec) - - self.valid_subclasses = [DebianPlugin] - - def _get_pkg_name_for_binary(self, binary): - # for binary not specified inside {..}, return binary itself - return { - "xz": "xz-utils" - }.get(binary, binary) - - @classmethod - def check(cls, remote=''): - """This method checks to see if we are running on Debian. - It returns True or False.""" - - if remote: - return cls.distro in remote - - return os.path.isfile('/etc/debian_version') - - def dist_version(self): - try: - with open('/etc/lsb-release', 'r') as fp: - rel_string = fp.read() - if "wheezy/sid" in rel_string: - return 6 - elif "jessie/sid" in rel_string: - return 7 - return False - except IOError: - return False - -# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/distros/__init__.py 4.5.3ubuntu2/sos/policies/distros/__init__.py --- 4.0-2/sos/policies/distros/__init__.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/distros/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,844 @@ +# Copyright (C) 2020 Red Hat, Inc., Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import os +import re + +from getpass import getpass + +from sos import _sos as _ +from sos.policies import Policy +from sos.policies.init_systems import InitSystem +from sos.policies.init_systems.systemd import SystemdInit +from sos.policies.runtimes.crio import CrioContainerRuntime +from sos.policies.runtimes.podman import PodmanContainerRuntime +from sos.policies.runtimes.docker import DockerContainerRuntime + +from sos.utilities import (shell_out, is_executable, bold, + sos_get_command_output) + + +try: + import requests + REQUESTS_LOADED = True +except ImportError: + REQUESTS_LOADED = False + +# Container environment variables for detecting if we're in a container +ENV_CONTAINER = 'container' +ENV_HOST_SYSROOT = 'HOST' + + +class LinuxPolicy(Policy): + """This policy is meant to be an abc class that provides common + implementations used in Linux distros""" + + distro = "Linux" + vendor = "None" + PATH = "/bin:/sbin:/usr/bin:/usr/sbin" + init = None + # _ prefixed class attrs are used for storing any vendor-defined defaults + # the non-prefixed attrs are used by the upload methods, and will be set + # to the cmdline/config file values, if provided. If not provided, then + # those attrs will be set to the _ prefixed values as a fallback. + # TL;DR Use _upload_* for policy default values, use upload_* when wanting + # to actual use the value in a method/override + _upload_url = None + _upload_directory = '/' + _upload_user = None + _upload_password = None + _upload_method = None + default_container_runtime = 'docker' + _preferred_hash_name = None + upload_url = None + upload_user = None + upload_password = None + # collector-focused class attrs + containerized = False + container_image = None + sos_path_strip = None + sos_pkg_name = None + sos_bin_path = '/usr/bin' + sos_container_name = 'sos-collector-tmp' + container_version_command = None + container_authfile = None + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(LinuxPolicy, self).__init__(sysroot=sysroot, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + + if sysroot: + self.sysroot = sysroot + else: + self.sysroot = self._container_init() or '/' + + self.init_kernel_modules() + + if init is not None: + self.init_system = init + elif os.path.isdir("/run/systemd/system/"): + self.init_system = SystemdInit(chroot=self.sysroot) + else: + self.init_system = InitSystem() + + self.runtimes = {} + if self.probe_runtime: + _crun = [ + PodmanContainerRuntime(policy=self), + DockerContainerRuntime(policy=self), + CrioContainerRuntime(policy=self) + ] + for runtime in _crun: + if runtime.check_is_active(): + self.runtimes[runtime.name] = runtime + if runtime.name == self.default_container_runtime: + self.runtimes['default'] = self.runtimes[runtime.name] + self.runtimes[runtime.name].load_container_info() + + if self.runtimes and 'default' not in self.runtimes.keys(): + # still allow plugins to query a runtime present on the system + # even if that is not the policy default one + idx = list(self.runtimes.keys()) + self.runtimes['default'] = self.runtimes[idx[0]] + + @classmethod + def set_forbidden_paths(cls): + return [ + '/etc/passwd', + '/etc/shadow' + ] + + def kernel_version(self): + return self.release + + def host_name(self): + return self.hostname + + def is_kernel_smp(self): + return self.smp + + def get_arch(self): + return self.machine + + def get_local_name(self): + """Returns the name usd in the pre_work step""" + return self.host_name() + + def sanitize_filename(self, name): + return re.sub(r"[^-a-z,A-Z.0-9]", "", name) + + @classmethod + def display_help(cls, section): + if cls == LinuxPolicy: + cls.display_self_help(section) + else: + section.set_title("%s Distribution Policy" % cls.distro) + cls.display_distro_help(section) + + @classmethod + def display_self_help(cls, section): + section.set_title("SoS Distribution Policies") + section.add_text( + 'Distributions supported by SoS will each have a specific policy ' + 'defined for them, to ensure proper operation of SoS on those ' + 'systems.' + ) + + @classmethod + def display_distro_help(cls, section): + if cls.__doc__ and cls.__doc__ is not LinuxPolicy.__doc__: + section.add_text(cls.__doc__) + else: + section.add_text( + '\nDetailed help information for this policy is not available' + ) + + # instantiate the requested policy so we can report more interesting + # information like $PATH and loaded presets + _pol = cls(None, None, False) + section.add_text( + "Default --upload location: %s" % _pol._upload_url + ) + section.add_text( + "Default container runtime: %s" % _pol.default_container_runtime, + newline=False + ) + section.add_text( + "$PATH used when running report: %s" % _pol.PATH, + newline=False + ) + + refsec = section.add_section('Reference URLs') + for url in cls.vendor_urls: + refsec.add_text( + "{:>8}{:<30}{:<40}".format(' ', url[0], url[1]), + newline=False + ) + + presec = section.add_section('Presets Available With This Policy\n') + presec.add_text( + bold( + "{:>8}{:<20}{:<45}{:<30}".format(' ', 'Preset Name', + 'Description', + 'Enabled Options') + ), + newline=False + ) + for preset in _pol.presets: + _preset = _pol.presets[preset] + _opts = ' '.join(_preset.opts.to_args()) + presec.add_text( + "{:>8}{:<20}{:<45}{:<30}".format( + ' ', preset, _preset.desc, _opts + ), + newline=False + ) + + def _container_init(self): + """Check if sos is running in a container and perform container + specific initialisation based on ENV_HOST_SYSROOT. + """ + if ENV_CONTAINER in os.environ: + if os.environ[ENV_CONTAINER] in ['docker', 'oci', 'podman']: + self._in_container = True + if ENV_HOST_SYSROOT in os.environ: + if not os.environ[ENV_HOST_SYSROOT]: + # guard against blank/improperly unset values + return None + self._tmp_dir = os.path.abspath( + os.environ[ENV_HOST_SYSROOT] + self._tmp_dir + ) + return os.environ[ENV_HOST_SYSROOT] + return None + + def init_kernel_modules(self): + """Obtain a list of loaded kernel modules to reference later for plugin + enablement and SoSPredicate checks + """ + self.kernel_mods = [] + release = os.uname().release + + # first load modules from lsmod + lines = shell_out("lsmod", timeout=0, chroot=self.sysroot).splitlines() + self.kernel_mods.extend([ + line.split()[0].strip() for line in lines[1:] + ]) + + # next, include kernel builtins + builtins = self.join_sysroot( + "/usr/lib/modules/%s/modules.builtin" % release + ) + try: + with open(builtins, "r") as mfile: + for line in mfile: + kmod = line.split('/')[-1].split('.ko')[0] + self.kernel_mods.append(kmod) + except IOError: + pass + + # finally, parse kconfig looking for specific kconfig strings that + # have been verified to not appear in either lsmod or modules.builtin + # regardless of how they are built + config_strings = { + 'devlink': 'CONFIG_NET_DEVLINK', + 'dm_mod': 'CONFIG_BLK_DEV_DM' + } + + booted_config = self.join_sysroot("/boot/config-%s" % release) + kconfigs = [] + try: + with open(booted_config, "r") as kfile: + for line in kfile: + if '=y' in line: + kconfigs.append(line.split('=y')[0]) + except IOError: + pass + + for builtin in config_strings: + if config_strings[builtin] in kconfigs: + self.kernel_mods.append(builtin) + + def join_sysroot(self, path): + if self.sysroot and self.sysroot != '/': + path = os.path.join(self.sysroot, path.lstrip('/')) + return path + + def pre_work(self): + # this method will be called before the gathering begins + + cmdline_opts = self.commons['cmdlineopts'] + caseid = cmdline_opts.case_id if cmdline_opts.case_id else "" + + if cmdline_opts.low_priority: + self._configure_low_priority() + + # Set the cmdline settings to the class attrs that are referenced later + # The policy default '_' prefixed versions of these are untouched to + # allow fallback + self.upload_url = cmdline_opts.upload_url + self.upload_user = cmdline_opts.upload_user + self.upload_directory = cmdline_opts.upload_directory + self.upload_password = cmdline_opts.upload_pass + self.upload_archive_name = '' + + # set or query for case id + if not cmdline_opts.batch and not \ + cmdline_opts.quiet: + try: + if caseid: + self.commons['cmdlineopts'].case_id = caseid + else: + self.commons['cmdlineopts'].case_id = input( + _("Optionally, please enter the case id that you are " + "generating this report for [%s]: ") % caseid + ) + except KeyboardInterrupt: + raise + if cmdline_opts.case_id: + self.case_id = cmdline_opts.case_id + + # set or query for upload credentials; this needs to be done after + # setting case id, as below methods might rely on detection of it + if not cmdline_opts.batch and not \ + cmdline_opts.quiet: + try: + # Policies will need to handle the prompts for user information + if cmdline_opts.upload and self.get_upload_url(): + self.prompt_for_upload_user() + self.prompt_for_upload_password() + self.ui_log.info('') + except KeyboardInterrupt: + raise + + return + + def _configure_low_priority(self): + """Used to constrain sos to a 'low priority' execution, potentially + letting individual policies set their own definition of what that is. + + By default, this will attempt to assign sos to an idle io class via + ionice if available. We will also renice our own pid to 19 in order to + not cause competition with other host processes for CPU time. + """ + _pid = os.getpid() + if is_executable('ionice'): + ret = sos_get_command_output( + f"ionice -c3 -p {_pid}", timeout=5 + ) + if ret['status'] == 0: + self.soslog.info('Set IO class to idle') + else: + msg = (f"Error setting IO class to idle: {ret['output']} " + f"(exit code {ret['status']})") + self.soslog.error(msg) + else: + self.ui_log.warning( + "Warning: unable to constrain report to idle IO class: " + "ionice is not available." + ) + + try: + os.nice(20) + self.soslog.info('Set niceness of report to 19') + except Exception as err: + self.soslog.error(f"Error setting report niceness to 19: {err}") + + def prompt_for_upload_user(self): + """Should be overridden by policies to determine if a user needs to + be provided or not + """ + if not self.get_upload_user(): + msg = "Please provide upload user for %s: " % self.get_upload_url() + self.upload_user = input(_(msg)) + + def prompt_for_upload_password(self): + """Should be overridden by policies to determine if a password needs to + be provided for upload or not + """ + if not self.get_upload_password() and (self.get_upload_user() != + self._upload_user): + msg = ("Please provide the upload password for %s: " + % self.get_upload_user()) + self.upload_password = getpass(msg) + + def upload_archive(self, archive): + """ + Entry point for sos attempts to upload the generated archive to a + policy or user specified location. + + Curerntly there is support for HTTPS, SFTP, and FTP. HTTPS uploads are + preferred for policy-defined defaults. + + Policies that need to override uploading methods should override the + respective upload_https(), upload_sftp(), and/or upload_ftp() methods + and should NOT override this method. + + :param archive: The archive filepath to use for upload + :type archive: ``str`` + + In order to enable this for a policy, that policy needs to implement + the following: + + Required Class Attrs + + :_upload_url: The default location to use. Note these MUST include + protocol header + :_upload_user: Default username, if any else None + :_upload_password: Default password, if any else None + + The following Class Attrs may optionally be overidden by the Policy + + :_upload_directory: Default FTP server directory, if any + + + The following methods may be overridden by ``Policy`` as needed + + `prompt_for_upload_user()` + Determines if sos should prompt for a username or not. + + `get_upload_user()` + Determines if the default or a different username should be used + + `get_upload_https_auth()` + Format authentication data for HTTPS uploads + + `get_upload_url_string()` + Print a more human-friendly string than vendor URLs + """ + self.upload_archive_name = archive + if not self.upload_url: + self.upload_url = self.get_upload_url() + if not self.upload_url: + raise Exception("No upload destination provided by policy or by " + "--upload-url") + upload_func = self._determine_upload_type() + self.ui_log.info( + _(f"Attempting upload to {self.get_upload_url_string()}") + ) + return upload_func() + + def _determine_upload_type(self): + """Based on the url provided, determine what type of upload to attempt. + + Note that this requires users to provide a FQDN address, such as + https://myvendor.com/api or ftp://myvendor.com instead of + myvendor.com/api or myvendor.com + """ + prots = { + 'ftp': self.upload_ftp, + 'sftp': self.upload_sftp, + 'https': self.upload_https + } + if self.commons['cmdlineopts'].upload_protocol in prots.keys(): + return prots[self.commons['cmdlineopts'].upload_protocol] + elif '://' not in self.upload_url: + raise Exception("Must provide protocol in upload URL") + prot, url = self.upload_url.split('://') + if prot not in prots.keys(): + raise Exception("Unsupported or unrecognized protocol: %s" % prot) + return prots[prot] + + def get_upload_https_auth(self, user=None, password=None): + """Formats the user/password credentials using basic auth + + :param user: The username for upload + :type user: ``str`` + + :param password: Password for `user` to use for upload + :type password: ``str`` + + :returns: The user/password auth suitable for use in reqests calls + :rtype: ``requests.auth.HTTPBasicAuth()`` + """ + if not user: + user = self.get_upload_user() + if not password: + password = self.get_upload_password() + + return requests.auth.HTTPBasicAuth(user, password) + + def get_upload_url(self): + """Helper function to determine if we should use the policy default + upload url or one provided by the user + + :returns: The URL to use for upload + :rtype: ``str`` + """ + return self.upload_url or self._upload_url + + def get_upload_url_string(self): + """Used by distro policies to potentially change the string used to + report upload location from the URL to a more human-friendly string + """ + return self.get_upload_url() + + def get_upload_user(self): + """Helper function to determine if we should use the policy default + upload user or one provided by the user + + :returns: The username to use for upload + :rtype: ``str`` + """ + return (os.getenv('SOSUPLOADUSER', None) or + self.upload_user or + self._upload_user) + + def get_upload_password(self): + """Helper function to determine if we should use the policy default + upload password or one provided by the user + + A user provided password, either via option or the 'SOSUPLOADPASSWORD' + environment variable will have precendent over any policy value + + :returns: The password to use for upload + :rtype: ``str`` + """ + return (os.getenv('SOSUPLOADPASSWORD', None) or + self.upload_password or + self._upload_password) + + def upload_sftp(self, user=None, password=None): + """Attempts to upload the archive to an SFTP location. + + Due to the lack of well maintained, secure, and generally widespread + python libraries for SFTP, sos will shell-out to the system's local ssh + installation in order to handle these uploads. + + Do not override this method with one that uses python-paramiko, as the + upstream sos team will reject any PR that includes that dependency. + """ + # if we somehow don't have sftp available locally, fail early + if not is_executable('sftp'): + raise Exception('SFTP is not locally supported') + + # soft dependency on python3-pexpect, which we need to use to control + # sftp login since as of this writing we don't have a viable solution + # via ssh python bindings commonly available among downstreams + try: + import pexpect + except ImportError: + raise Exception('SFTP upload requires python3-pexpect, which is ' + 'not currently installed') + + sftp_connected = False + + if not user: + user = self.get_upload_user() + if not password: + password = self.get_upload_password() + + # need to strip the protocol prefix here + sftp_url = self.get_upload_url().replace('sftp://', '') + sftp_cmd = "sftp -oStrictHostKeyChecking=no %s@%s" % (user, sftp_url) + ret = pexpect.spawn(sftp_cmd, encoding='utf-8') + + sftp_expects = [ + u'sftp>', + u'password:', + u'Connection refused', + pexpect.TIMEOUT, + pexpect.EOF + ] + + idx = ret.expect(sftp_expects, timeout=15) + + if idx == 0: + sftp_connected = True + elif idx == 1: + ret.sendline(password) + pass_expects = [ + u'sftp>', + u'Permission denied', + pexpect.TIMEOUT, + pexpect.EOF + ] + sftp_connected = ret.expect(pass_expects, timeout=10) == 0 + if not sftp_connected: + ret.close() + raise Exception("Incorrect username or password for %s" + % self.get_upload_url_string()) + elif idx == 2: + raise Exception("Connection refused by %s. Incorrect port?" + % self.get_upload_url_string()) + elif idx == 3: + raise Exception("Timeout hit trying to connect to %s" + % self.get_upload_url_string()) + elif idx == 4: + raise Exception("Unexpected error trying to connect to sftp: %s" + % ret.before) + + if not sftp_connected: + ret.close() + raise Exception("Unable to connect via SFTP to %s" + % self.get_upload_url_string()) + + put_cmd = 'put %s %s' % (self.upload_archive_name, + self._get_sftp_upload_name()) + ret.sendline(put_cmd) + + put_expects = [ + u'100%', + pexpect.TIMEOUT, + pexpect.EOF, + u'No such file or directory' + ] + + put_success = ret.expect(put_expects, timeout=180) + + if put_success == 0: + ret.sendline('bye') + return True + elif put_success == 1: + raise Exception("Timeout expired while uploading") + elif put_success == 2: + raise Exception("Unknown error during upload: %s" % ret.before) + elif put_success == 3: + raise Exception("Unable to write archive to destination") + else: + raise Exception("Unexpected response from server: %s" % ret.before) + + def _get_sftp_upload_name(self): + """If a specific file name pattern is required by the SFTP server, + override this method in the relevant Policy. Otherwise the archive's + name on disk will be used + + :returns: Filename as it will exist on the SFTP server + :rtype: ``str`` + """ + fname = self.upload_archive_name.split('/')[-1] + if self.upload_directory: + fname = os.path.join(self.upload_directory, fname) + return fname + + def _upload_https_put(self, archive, verify=True): + """If upload_https() needs to use requests.put(), use this method. + + Policies should override this method instead of the base upload_https() + + :param archive: The open archive file object + """ + return requests.put(self.get_upload_url(), data=archive, + auth=self.get_upload_https_auth(), + verify=verify) + + def _get_upload_headers(self): + """Define any needed headers to be passed with the POST request here + """ + return {} + + def _upload_https_post(self, archive, verify=True): + """If upload_https() needs to use requests.post(), use this method. + + Policies should override this method instead of the base upload_https() + + :param archive: The open archive file object + """ + files = { + 'file': (archive.name.split('/')[-1], archive, + self._get_upload_headers()) + } + return requests.post(self.get_upload_url(), files=files, + auth=self.get_upload_https_auth(), + verify=verify) + + def upload_https(self): + """Attempts to upload the archive to an HTTPS location. + + :returns: ``True`` if upload is successful + :rtype: ``bool`` + + :raises: ``Exception`` if upload was unsuccessful + """ + if not REQUESTS_LOADED: + raise Exception("Unable to upload due to missing python requests " + "library") + + with open(self.upload_archive_name, 'rb') as arc: + if self.commons['cmdlineopts'].upload_method == 'auto': + method = self._upload_method + else: + method = self.commons['cmdlineopts'].upload_method + verify = self.commons['cmdlineopts'].upload_no_ssl_verify is False + if method == 'put': + r = self._upload_https_put(arc, verify) + else: + r = self._upload_https_post(arc, verify) + if r.status_code != 200 and r.status_code != 201: + if r.status_code == 401: + raise Exception( + "Authentication failed: invalid user credentials" + ) + raise Exception("POST request returned %s: %s" + % (r.status_code, r.reason)) + return True + + def upload_ftp(self, url=None, directory=None, user=None, password=None): + """Attempts to upload the archive to either the policy defined or user + provided FTP location. + + :param url: The URL to upload to + :type url: ``str`` + + :param directory: The directory on the FTP server to write to + :type directory: ``str`` or ``None`` + + :param user: The user to authenticate with + :type user: ``str`` + + :param password: The password to use for `user` + :type password: ``str`` + + :returns: ``True`` if upload is successful + :rtype: ``bool`` + + :raises: ``Exception`` if upload in unsuccessful + """ + try: + import ftplib + import socket + except ImportError: + # socket is part of the standard library, should only fail here on + # ftplib + raise Exception("missing python ftplib library") + + if not url: + url = self.get_upload_url() + if url is None: + raise Exception("no FTP server specified by policy, use --upload-" + "url to specify a location") + + url = url.replace('ftp://', '') + + if not user: + user = self.get_upload_user() + + if not password: + password = self.get_upload_password() + + if not directory: + directory = self.upload_directory or self._upload_directory + + try: + session = ftplib.FTP(url, user, password, timeout=15) + if not session: + raise Exception("connection failed, did you set a user and " + "password?") + session.cwd(directory) + except socket.timeout: + raise Exception("timeout hit while connecting to %s" % url) + except socket.gaierror: + raise Exception("unable to connect to %s" % url) + except ftplib.error_perm as err: + errno = str(err).split()[0] + if errno == '503': + raise Exception("could not login as '%s'" % user) + if errno == '530': + raise Exception("invalid password for user '%s'" % user) + if errno == '550': + raise Exception("could not set upload directory to %s" + % directory) + raise Exception("error trying to establish session: %s" + % str(err)) + + try: + with open(self.upload_archive_name, 'rb') as _arcfile: + session.storbinary( + "STOR %s" % self.upload_archive_name.split('/')[-1], + _arcfile + ) + session.quit() + return True + except IOError: + raise Exception("could not open archive file") + + def set_sos_prefix(self): + """If sosreport commands need to always be prefixed with something, + for example running in a specific container image, then it should be + defined here. + + If no prefix should be set, return an empty string instead of None. + """ + return '' + + def set_cleanup_cmd(self): + """If a host requires additional cleanup, the command should be set and + returned here + """ + return '' + + def create_sos_container(self, image=None, auth=None, force_pull=False): + """Returns the command that will create the container that will be + used for running commands inside a container on hosts that require it. + + This will use the container runtime defined for the host type to + launch a container. From there, we use the defined runtime to exec into + the container's namespace. + + :param image: The name of the image if not using the policy default + :type image: ``str`` or ``None`` + + :param auth: The auth string required by the runtime to pull an + image from the registry + :type auth: ``str`` or ``None`` + + :param force_pull: Should the runtime forcibly pull the image + :type force_pull: ``bool`` + + :returns: The command to execute to launch the temp container + :rtype: ``str`` + """ + return '' + + def restart_sos_container(self): + """Restarts the container created for sos collect if it has stopped. + + This is called immediately after create_sos_container() as the command + to create the container will exit and the container will stop. For + current container runtimes, subsequently starting the container will + default to opening a bash shell in the container to keep it running, + thus allowing us to exec into it again. + """ + return "%s start %s" % (self.container_runtime, + self.sos_container_name) + + def format_container_command(self, cmd): + """Returns the command that allows us to exec into the created + container for sos collect. + + :param cmd: The command to run in the sos container + :type cmd: ``str`` + + :returns: The command to execute to run `cmd` in the container + :rtype: ``str`` + """ + if self.container_runtime: + return '%s exec %s %s' % (self.container_runtime, + self.sos_container_name, + cmd) + else: + return cmd + + +class GenericLinuxPolicy(LinuxPolicy): + """This Policy will be returned if no other policy can be loaded. This + should allow for IndependentPlugins to be executed on any system""" + + vendor_urls = [('Upstream Project', 'https://github.com/sosreport/sos')] + vendor = 'SoS' + vendor_text = ('SoS was unable to determine that the distribution of this ' + 'system is supported, and has loaded a generic ' + 'configuration. This may not provide desired behavior, and ' + 'users are encouraged to request a new distribution-specifc' + ' policy at the GitHub project above.\n') + + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/distros/amazon.py 4.5.3ubuntu2/sos/policies/distros/amazon.py --- 4.0-2/sos/policies/distros/amazon.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/distros/amazon.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,43 @@ +# Copyright (C) Red Hat, Inc. 2019 + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.policies.distros.redhat import RedHatPolicy, OS_RELEASE +import os + + +class AmazonPolicy(RedHatPolicy): + + distro = "Amazon Linux" + vendor = "Amazon" + vendor_urls = [('Distribution Website', 'https://aws.amazon.com')] + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(AmazonPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + + @classmethod + def check(cls, remote=''): + + if remote: + return cls.distro in remote + + if not os.path.exists(OS_RELEASE): + return False + + with open(OS_RELEASE, 'r') as f: + for line in f: + if line.startswith('NAME'): + if 'Amazon Linux' in line: + return True + return False + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/distros/anolis.py 4.5.3ubuntu2/sos/policies/distros/anolis.py --- 4.0-2/sos/policies/distros/anolis.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/distros/anolis.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,46 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.policies.distros.redhat import RedHatPolicy, OS_RELEASE +import os + + +class AnolisPolicy(RedHatPolicy): + + distro = "Anolis OS" + vendor = "The OpenAnolis Project" + vendor_urls = [('Distribution Website', 'https://openanolis.org/')] + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(AnolisPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + + @classmethod + def check(cls, remote=''): + + if remote: + return cls.distro in remote + + # Return False if /etc/os-release is missing + if not os.path.exists(OS_RELEASE): + return False + + # Return False if /etc/anolis-release is missing + if not os.path.isfile('/etc/anolis-release'): + return False + + with open(OS_RELEASE, 'r') as f: + for line in f: + if line.startswith('NAME'): + if 'Anolis OS' in line: + return True + return False + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/distros/circle.py 4.5.3ubuntu2/sos/policies/distros/circle.py --- 4.0-2/sos/policies/distros/circle.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/distros/circle.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,49 @@ +# Copyright (C) Bella Zhang + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.policies.distros.redhat import RedHatPolicy, OS_RELEASE +import os + + +class CirclePolicy(RedHatPolicy): + + distro = "Circle Linux" + vendor = "The Circle Linux Project" + vendor_urls = [('Distribution Website', 'https://cclinux.org')] + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(CirclePolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + + @classmethod + def check(cls, remote=''): + + if remote: + return cls.distro in remote + + # Return False if /etc/os-release is missing + if not os.path.exists(OS_RELEASE): + return False + + # Return False if /etc/circle-release is missing + if not os.path.isfile('/etc/circle-release'): + return False + + with open(OS_RELEASE, 'r') as f: + for line in f: + if line.startswith('NAME'): + if 'Circle Linux' in line: + return True + + return False + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/distros/cos.py 4.5.3ubuntu2/sos/policies/distros/cos.py --- 4.0-2/sos/policies/distros/cos.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/distros/cos.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,59 @@ +# Copyright (C) Red Hat, Inc. 2020 + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import CosPlugin, IndependentPlugin +from sos.policies.distros import LinuxPolicy + + +def _blank_or_comment(line): + """Test whether line is empty of contains a comment. + + Test whether the ``line`` argument is either blank, or a + whole-line comment. + + :param line: the line of text to be checked. + :returns: ``True`` if the line is blank or a comment, + and ``False`` otherwise. + :rtype: bool + """ + return not line.strip() or line.lstrip().startswith('#') + + +class CosPolicy(LinuxPolicy): + distro = "Container-Optimized OS" + vendor = "Google Cloud Platform" + vendor_urls = [ + ('Distribution Website', + 'https://cloud.google.com/container-optimized-os/') + ] + valid_subclasses = [CosPlugin, IndependentPlugin] + PATH = "/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin" + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(CosPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + self.valid_subclasses += [CosPolicy] + + @classmethod + def check(cls, remote=''): + if remote: + return cls.distro in remote + + try: + with open('/etc/os-release', 'r') as fp: + os_release = dict(line.strip().split('=') for line in fp + if not _blank_or_comment(line)) + return os_release['ID'] == 'cos' + except (IOError, KeyError): + return False + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/distros/debian.py 4.5.3ubuntu2/sos/policies/distros/debian.py --- 4.0-2/sos/policies/distros/debian.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/distros/debian.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,78 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import DebianPlugin +from sos.policies.distros import LinuxPolicy +from sos.policies.package_managers.dpkg import DpkgPackageManager + +import os + + +class DebianPolicy(LinuxPolicy): + distro = "Debian" + vendor = "the Debian project" + vendor_urls = [('Community Website', 'https://www.debian.org/')] + name_pattern = 'friendly' + valid_subclasses = [DebianPlugin] + PATH = "/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" \ + + ":/usr/local/sbin:/usr/local/bin" + sos_pkg_name = 'sosreport' + + deb_versions = { + 'squeeze': 6, + 'wheezy': 7, + 'jessie': 8, + 'stretch': 9, + 'buster': 10, + 'bullseye': 11, + 'bookworm': 12, + 'trixie': 13, + 'forky': 14, + } + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(DebianPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + self.package_manager = DpkgPackageManager(chroot=self.sysroot, + remote_exec=remote_exec) + self.valid_subclasses += [DebianPlugin] + + def _get_pkg_name_for_binary(self, binary): + # for binary not specified inside {..}, return binary itself + return { + "xz": "xz-utils" + }.get(binary, binary) + + @classmethod + def check(cls, remote=''): + """This method checks to see if we are running on Debian. + It returns True or False.""" + + if remote: + return cls.distro in remote + + return os.path.isfile('/etc/debian_version') + + def dist_version(self): + try: + with open('/etc/os-release', 'r') as fp: + rel_string = "" + lines = fp.readlines() + for line in lines: + if "VERSION_CODENAME" in line: + rel_string = line.split("=")[1].strip() + break + if rel_string in self.deb_versions: + return self.deb_versions[rel_string] + return False + except IOError: + return False + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/distros/mariner.py 4.5.3ubuntu2/sos/policies/distros/mariner.py --- 4.0-2/sos/policies/distros/mariner.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/distros/mariner.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,45 @@ +# Copyright (C) Eric Desrochers + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.policies.distros.redhat import RedHatPolicy, OS_RELEASE +import os + + +class MarinerPolicy(RedHatPolicy): + + distro = "CBL-Mariner" + vendor = "Microsoft" + vendor_urls = [ + ('Distribution Website', 'https://github.com/microsoft/CBL-Mariner') + ] + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(MarinerPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + + @classmethod + def check(cls, remote=''): + + if remote: + return cls.distro in remote + + if not os.path.exists(OS_RELEASE): + return False + + with open(OS_RELEASE, 'r') as f: + for line in f: + if line.startswith('NAME'): + if 'Common Base Linux Mariner' in line: + return True + return False + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/distros/opencloudos.py 4.5.3ubuntu2/sos/policies/distros/opencloudos.py --- 4.0-2/sos/policies/distros/opencloudos.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/distros/opencloudos.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,42 @@ +# Copyright (c) 2022 Tencent., ZoeDong +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.policies.distros.redhat import RedHatPolicy, OS_RELEASE +import os + + +class OpenCloudOSPolicy(RedHatPolicy): + distro = "OpenCloudOS Stream" + vendor = "OpenCloudOS" + vendor_urls = [('Distribution Website', 'https://www.opencloudos.org/')] + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(OpenCloudOSPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + + @classmethod + def check(cls, remote=''): + + if remote: + return cls.distro in remote + + if not os.path.exists(OS_RELEASE): + return False + + with open(OS_RELEASE, 'r') as f: + for line in f: + if line.startswith('NAME'): + if 'OpenCloudOS Stream' in line: + return True + + return False + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/distros/openeuler.py 4.5.3ubuntu2/sos/policies/distros/openeuler.py --- 4.0-2/sos/policies/distros/openeuler.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/distros/openeuler.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,43 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import OpenEulerPlugin +from sos.policies.distros.redhat import RedHatPolicy, OS_RELEASE +import os + + +class OpenEulerPolicy(RedHatPolicy): + distro = "openEuler" + vendor = "The openEuler Project" + vendor_urls = [('Distribution Website', 'https://openeuler.org/')] + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(OpenEulerPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + + self.valid_subclasses += [OpenEulerPlugin] + + @classmethod + def check(cls, remote=''): + + if remote: + return cls.distro in remote + + if not os.path.exists(OS_RELEASE): + return False + + with open(OS_RELEASE, 'r') as f: + for line in f: + if line.startswith('NAME'): + if 'openEuler' in line: + return True + return False + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/distros/redhat.py 4.5.3ubuntu2/sos/policies/distros/redhat.py --- 4.0-2/sos/policies/distros/redhat.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/distros/redhat.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,621 @@ +# Copyright (C) Steve Conklin + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import json +import os +import sys +import re + +from sos.report.plugins import RedHatPlugin +from sos.presets.redhat import (RHEL_PRESETS, ATOMIC_PRESETS, RHV, RHEL, + CB, RHOSP, RHOCP, RH_CFME, RH_SATELLITE, + ATOMIC) +from sos.policies.distros import LinuxPolicy, ENV_HOST_SYSROOT +from sos.policies.package_managers.rpm import RpmPackageManager +from sos.utilities import bold +from sos import _sos as _ + +try: + import requests + REQUESTS_LOADED = True +except ImportError: + REQUESTS_LOADED = False + +OS_RELEASE = "/etc/os-release" +RHEL_RELEASE_STR = "Red Hat Enterprise Linux" +ATOMIC_RELEASE_STR = "Atomic" + + +class RedHatPolicy(LinuxPolicy): + distro = "Red Hat" + vendor = "Red Hat" + vendor_urls = [ + ('Distribution Website', 'https://www.redhat.com/'), + ('Commercial Support', 'https://access.redhat.com/') + ] + _tmp_dir = "/var/tmp" + _in_container = False + default_scl_prefix = '/opt/rh' + name_pattern = 'friendly' + upload_url = None + upload_user = None + default_container_runtime = 'podman' + sos_pkg_name = 'sos' + sos_bin_path = '/usr/sbin' + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(RedHatPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + self.usrmove = False + + self.package_manager = RpmPackageManager(chroot=self.sysroot, + remote_exec=remote_exec) + + self.valid_subclasses += [RedHatPlugin] + + self.pkgs = self.package_manager.packages + + # If rpm query failed, exit + if not self.pkgs: + sys.stderr.write("Could not obtain installed package list") + sys.exit(1) + + self.usrmove = self.check_usrmove(self.pkgs) + + if self.usrmove: + self.PATH = "/usr/sbin:/usr/bin:/root/bin" + else: + self.PATH = "/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" + self.PATH += os.pathsep + "/usr/local/bin" + self.PATH += os.pathsep + "/usr/local/sbin" + if not self.remote_exec: + self.set_exec_path() + self.load_presets() + + @classmethod + def check(cls, remote=''): + """This method checks to see if we are running on Red Hat. It must be + overriden by concrete subclasses to return True when running on a + Fedora, RHEL or other Red Hat distribution or False otherwise. + + If `remote` is provided, it should be the contents of a remote host's + os-release, or comparable, file to be used in place of the locally + available one. + """ + return False + + @classmethod + def display_distro_help(cls, section): + if cls is not RedHatPolicy: + super(RedHatPolicy, cls).display_distro_help(section) + return + section.add_text( + 'This policy is a building block for all other Red Hat family ' + 'distributions. You are likely looking for one of the ' + 'distributions listed below.\n' + ) + + subs = { + 'centos': CentOsPolicy, + 'rhel': RHELPolicy, + 'redhatcoreos': RedHatCoreOSPolicy, + 'fedora': FedoraPolicy + } + + for subc in subs: + subln = bold("policies.%s" % subc) + section.add_text( + "{:>8}{:<35}{:<30}".format(' ', subln, subs[subc].distro), + newline=False + ) + + def check_usrmove(self, pkgs): + """Test whether the running system implements UsrMove. + + If the 'filesystem' package is present, it will check that the + version is greater than 3. If the package is not present the + '/bin' and '/sbin' paths are checked and UsrMove is assumed + if both are symbolic links. + + :param pkgs: a packages dictionary + """ + if 'filesystem' not in pkgs: + return os.path.islink('/bin') and os.path.islink('/sbin') + else: + filesys_version = pkgs['filesystem']['version'] + return True if filesys_version[0] == '3' else False + + def mangle_package_path(self, files): + """Mangle paths for post-UsrMove systems. + + If the system implements UsrMove, all files will be in + '/usr/[s]bin'. This method substitutes all the /[s]bin + references in the 'files' list with '/usr/[s]bin'. + + :param files: the list of package managed files + """ + paths = [] + + def transform_path(path): + # Some packages actually own paths in /bin: in this case, + # duplicate the path as both the / and /usr version. + skip_paths = ["/bin/rpm", "/bin/mailx"] + if path in skip_paths: + return (path, os.path.join("/usr", path[1:])) + return (re.sub(r'(^)(/s?bin)', r'\1/usr\2', path),) + + if self.usrmove: + for f in files: + paths.extend(transform_path(f)) + return paths + else: + return files + + def get_tmp_dir(self, opt_tmp_dir): + if not opt_tmp_dir: + return self._tmp_dir + return opt_tmp_dir + + +# Legal disclaimer text for Red Hat products +disclaimer_text = """ +Any information provided to %(vendor)s will be treated in \ +accordance with the published support policies at:\n + %(vendor_urls)s + +The generated archive may contain data considered sensitive \ +and its content should be reviewed by the originating \ +organization before being passed to any third party. + +No changes will be made to system configuration. +""" + +RH_API_HOST = "https://api.access.redhat.com" +RH_SFTP_HOST = "sftp://sftp.access.redhat.com" + + +class RHELPolicy(RedHatPolicy): + """ + The RHEL policy is used specifically for Red Hat Enterprise Linux, of + any release, and not forks or derivative distributions. For example, this + policy will be loaded for any RHEL 8 installation, but will not be loaded + for CentOS Stream 8 or Red Hat CoreOS, for which there are separate + policies. + + Plugins activated by installed packages will only be activated if those + packages are installed via RPM (dnf/yum inclusive). Packages installed by + other means are not considered by this policy. + + By default, --upload will be directed to using the SFTP location provided + by Red Hat for technical support cases. Users who provide login credentials + for their Red Hat Customer Portal account will have their archives uploaded + to a user-specific directory. + + If users provide those credentials as well as a case number, --upload will + instead attempt to directly upload archives to the referenced case, thus + streamlining the process of providing data to technical support engineers. + + If either or both of the credentials or case number are omitted or are + incorrect, then a temporary anonymous user will be used for upload to the + SFTP server, and users will need to provide that information to their + technical support engineer. This information will be printed at the end of + the upload process for any sos report execution. + """ + distro = RHEL_RELEASE_STR + vendor = "Red Hat" + msg = _("""\ +This command will collect diagnostic and configuration \ +information from this %(distro)s system and installed \ +applications. + +An archive containing the collected information will be \ +generated in %(tmpdir)s and may be provided to a %(vendor)s \ +support representative. +""" + disclaimer_text + "%(vendor_text)s\n") + _upload_url = RH_SFTP_HOST + _upload_method = 'post' + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(RHELPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + self.register_presets(RHEL_PRESETS) + + @classmethod + def check(cls, remote=''): + """Test to see if the running host is a RHEL installation. + + Checks for the presence of the "Red Hat Enterprise Linux" + release string at the beginning of the NAME field in the + `/etc/os-release` file and returns ``True`` if it is + found, and ``False`` otherwise. + + :returns: ``True`` if the host is running RHEL or ``False`` + otherwise. + """ + + if remote: + return cls.distro in remote + + if not os.path.exists(OS_RELEASE): + return False + + with open(OS_RELEASE, "r") as f: + for line in f: + if line.startswith("NAME"): + (name, value) = line.split("=") + value = value.strip("\"'") + if value.startswith(cls.distro): + return True + return False + + def prompt_for_upload_user(self): + if self.commons['cmdlineopts'].upload_user: + return + # Not using the default, so don't call this prompt for RHCP + if self.commons['cmdlineopts'].upload_url: + super(RHELPolicy, self).prompt_for_upload_user() + return + if not self.get_upload_user(): + if self.case_id: + self.upload_user = input(_( + "Enter your Red Hat Customer Portal username for " + "uploading [empty for anonymous SFTP]: ") + ) + else: # no case id provided => failover to SFTP + self.upload_url = RH_SFTP_HOST + self.ui_log.info("No case id provided, uploading to SFTP") + self.upload_user = input(_( + "Enter your Red Hat Customer Portal username for " + "uploading to SFTP [empty for anonymous]: ") + ) + + def get_upload_url(self): + if self.upload_url: + return self.upload_url + elif self.commons['cmdlineopts'].upload_url: + return self.commons['cmdlineopts'].upload_url + elif self.commons['cmdlineopts'].upload_protocol == 'sftp': + return RH_SFTP_HOST + else: + rh_case_api = "/support/v1/cases/%s/attachments" + return RH_API_HOST + rh_case_api % self.case_id + + def _get_upload_headers(self): + if self.get_upload_url().startswith(RH_API_HOST): + return {'isPrivate': 'false', 'cache-control': 'no-cache'} + return {} + + def get_upload_url_string(self): + if self.get_upload_url().startswith(RH_API_HOST): + return "Red Hat Customer Portal" + elif self.get_upload_url().startswith(RH_SFTP_HOST): + return "Red Hat Secure FTP" + return self.upload_url + + def _get_sftp_upload_name(self): + """The RH SFTP server will only automatically connect file uploads to + cases if the filename _starts_ with the case number + """ + fname = self.upload_archive_name.split('/')[-1] + if self.case_id: + fname = "%s_%s" % (self.case_id, fname) + if self.upload_directory: + fname = os.path.join(self.upload_directory, fname) + return fname + + def upload_sftp(self): + """Override the base upload_sftp to allow for setting an on-demand + generated anonymous login for the RH SFTP server if a username and + password are not given + """ + if RH_SFTP_HOST.split('//')[1] not in self.get_upload_url(): + return super(RHELPolicy, self).upload_sftp() + + if not REQUESTS_LOADED: + raise Exception("python3-requests is not installed and is required" + " for obtaining SFTP auth token.") + _token = None + _user = None + url = RH_API_HOST + '/support/v2/sftp/token' + # we have a username and password, but we need to reset the password + # to be the token returned from the auth endpoint + if self.get_upload_user() and self.get_upload_password(): + auth = self.get_upload_https_auth() + ret = requests.post(url, auth=auth, timeout=10) + if ret.status_code == 200: + # credentials are valid + _user = self.get_upload_user() + _token = json.loads(ret.text)['token'] + else: + self.ui_log.error( + "Unable to retrieve Red Hat auth token using provided " + "credentials. Will try anonymous." + ) + # we either do not have a username or password/token, or both + if not _token: + adata = {"isAnonymous": True} + anon = requests.post(url, data=json.dumps(adata), timeout=10) + if anon.status_code == 200: + resp = json.loads(anon.text) + _user = resp['username'] + _token = resp['token'] + self.ui_log.info( + _(f"User {_user} used for anonymous upload. Please inform " + f"your support engineer so they may retrieve the data.") + ) + if _user and _token: + return super(RHELPolicy, self).upload_sftp(user=_user, + password=_token) + raise Exception("Could not retrieve valid or anonymous credentials") + + def upload_archive(self, archive): + """Override the base upload_archive to provide for automatic failover + from RHCP failures to the public RH dropbox + """ + try: + if self.upload_url and self.upload_url.startswith(RH_API_HOST) and\ + (not self.get_upload_user() or not self.get_upload_password()): + self.upload_url = RH_SFTP_HOST + uploaded = super(RHELPolicy, self).upload_archive(archive) + except Exception: + uploaded = False + if not self.upload_url.startswith(RH_API_HOST): + raise + else: + self.ui_log.error( + _(f"Upload to Red Hat Customer Portal failed. Trying " + f"{RH_SFTP_HOST}") + ) + self.upload_url = RH_SFTP_HOST + uploaded = super(RHELPolicy, self).upload_archive(archive) + return uploaded + + def dist_version(self): + try: + rr = self.package_manager.all_pkgs_by_name_regex("redhat-release*") + pkgname = self.pkgs[rr[0]]["version"] + # this should always map to the major version number. This will not + # be so on RHEL 5, but RHEL 5 does not support python3 and thus + # should never run a version of sos with this check + return int(pkgname[0]) + except Exception: + pass + return False + + def probe_preset(self): + # Emergency or rescue mode? + for target in ["rescue", "emergency"]: + if self.init_system.is_running("%s.target" % target, False): + return self.find_preset(CB) + # Package based checks + if self.pkg_by_name("satellite-common") is not None: + return self.find_preset(RH_SATELLITE) + if self.pkg_by_name("rhosp-release") is not None: + return self.find_preset(RHOSP) + if self.pkg_by_name("cfme") is not None: + return self.find_preset(RH_CFME) + if self.pkg_by_name("ovirt-engine") is not None or \ + self.pkg_by_name("vdsm") is not None: + return self.find_preset(RHV) + + # Vanilla RHEL is default + return self.find_preset(RHEL) + + +class CentOsPolicy(RHELPolicy): + distro = "CentOS" + vendor = "CentOS" + vendor_urls = [('Community Website', 'https://www.centos.org/')] + + +class RedHatAtomicPolicy(RHELPolicy): + distro = "Red Hat Atomic Host" + msg = _("""\ +This command will collect diagnostic and configuration \ +information from this %(distro)s system. + +An archive containing the collected information will be \ +generated in %(tmpdir)s and may be provided to a %(vendor)s \ +support representative. +""" + disclaimer_text + "%(vendor_text)s\n") + + containerzed = True + container_runtime = 'docker' + container_image = 'registry.access.redhat.com/rhel7/support-tools' + sos_path_strip = '/host' + container_version_command = 'rpm -q sos' + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(RedHatAtomicPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + self.register_presets(ATOMIC_PRESETS) + + @classmethod + def check(cls, remote=''): + + if remote: + return cls.distro in remote + + atomic = False + if ENV_HOST_SYSROOT not in os.environ: + return atomic + host_release = os.environ[ENV_HOST_SYSROOT] + OS_RELEASE + if not os.path.exists(host_release): + return False + try: + for line in open(host_release, "r").read().splitlines(): + atomic |= ATOMIC_RELEASE_STR in line + except IOError: + pass + return atomic + + def probe_preset(self): + if self.pkg_by_name('atomic-openshift'): + return self.find_preset(RHOCP) + + return self.find_preset(ATOMIC) + + def create_sos_container(self, image=None, auth=None, force_pull=False): + _cmd = ("{runtime} run -di --name {name} --privileged --ipc=host" + " --net=host --pid=host -e HOST=/host -e NAME={name} -e " + "IMAGE={image} {pull} -v /run:/run -v /var/log:/var/log -v " + "/etc/machine-id:/etc/machine-id -v " + "/etc/localtime:/etc/localtime -v /:/host {auth} {image}") + _image = image or self.container_image + _pull = '--pull=always' if force_pull else '' + return _cmd.format(runtime=self.container_runtime, + name=self.sos_container_name, + image=_image, + pull=_pull, + auth=auth or '') + + def set_cleanup_cmd(self): + return 'docker rm --force sos-collector-tmp' + + +class RedHatCoreOSPolicy(RHELPolicy): + """ + Red Hat CoreOS is a containerized host built upon Red Hat Enterprise Linux + and as such this policy is built on top of the RHEL policy. For users, this + should be entirely transparent as any behavior exhibited or influenced on + RHEL systems by that policy will be seen on RHCOS systems as well. + + The one change is that this policy ensures that sos collect will deploy a + container on RHCOS systems in order to facilitate sos report collection, + as RHCOS discourages non-default package installation via rpm-ostree which + is used to maintain atomicity for RHCOS nodes. The default container image + used by this policy is the support-tools image maintained by Red Hat on + registry.redhat.io. + + Note that this policy is only loaded when sos is directly run on an RHCOS + node - if sos collect uses the `oc` transport (the default transport that + will be attempted by the ocp cluster profile), then the policy loaded + inside the launched pod will be RHEL. Again, this is expected and will not + impact how sos report collections are performed. + """ + + distro = "Red Hat CoreOS" + msg = _("""\ +This command will collect diagnostic and configuration \ +information from this %(distro)s system. + +An archive containing the collected information will be \ +generated in %(tmpdir)s and may be provided to a %(vendor)s \ +support representative. +""" + disclaimer_text + "%(vendor_text)s\n") + + containerized = True + container_runtime = 'podman' + container_image = 'registry.redhat.io/rhel8/support-tools' + sos_path_strip = '/host' + container_version_command = 'rpm -q sos' + container_authfile = '/var/lib/kubelet/config.json' + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(RedHatCoreOSPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + + @classmethod + def check(cls, remote=''): + + if remote: + return 'CoreOS' in remote + + coreos = False + if ENV_HOST_SYSROOT not in os.environ: + return coreos + host_release = os.environ[ENV_HOST_SYSROOT] + OS_RELEASE + try: + for line in open(host_release, 'r').read().splitlines(): + coreos |= 'Red Hat Enterprise Linux CoreOS' in line + except IOError: + pass + return coreos + + def probe_preset(self): + # As of the creation of this policy, RHCOS is only available for + # RH OCP environments. + return self.find_preset(RHOCP) + + def create_sos_container(self, image=None, auth=None, force_pull=False): + _cmd = ("{runtime} run -di --name {name} --privileged --ipc=host" + " --net=host --pid=host -e HOST=/host -e NAME={name} -e " + "IMAGE={image} {pull} -v /run:/run -v /var/log:/var/log -v " + "/etc/machine-id:/etc/machine-id -v " + "/etc/localtime:/etc/localtime -v /:/host {auth} {image}") + _image = image or self.container_image + _pull = '--pull=always' if force_pull else '' + return _cmd.format(runtime=self.container_runtime, + name=self.sos_container_name, + image=_image, + pull=_pull, + auth=auth or '') + + def set_cleanup_cmd(self): + return 'podman rm --force %s' % self.sos_container_name + + +class CentOsAtomicPolicy(RedHatAtomicPolicy): + distro = "CentOS Atomic Host" + vendor = "CentOS" + vendor_urls = [('Community Website', 'https://www.centos.org/')] + + +class FedoraPolicy(RedHatPolicy): + """ + The policy for Fedora based systems, regardless of spin/edition. This + policy is based on the parent Red Hat policy, and thus will only check for + RPM packages when considering packaged-based plugin enablement. Packages + installed by other sources are not considered. + + There is no default --upload location for this policy. If users need to + upload an sos report archive from a Fedora system, they will need to + provide the location via --upload-url, and optionally login credentials + for that location via --upload-user and --upload-pass (or the appropriate + environment variables). + """ + + distro = "Fedora" + vendor = "the Fedora Project" + vendor_urls = [ + ('Community Website', 'https://fedoraproject.org/'), + ('Community Forums', 'https://discussion.fedoraproject.org/') + ] + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(FedoraPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + + @classmethod + def check(cls, remote=''): + """This method checks to see if we are running on Fedora. It returns + True or False.""" + + if remote: + return cls.distro in remote + + return os.path.isfile('/etc/fedora-release') + + def fedora_version(self): + pkg = self.pkg_by_name("fedora-release") or \ + self.all_pkgs_by_name_regex("fedora-release-.*")[-1] + return int(pkg["version"]) + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/distros/rocky.py 4.5.3ubuntu2/sos/policies/distros/rocky.py --- 4.0-2/sos/policies/distros/rocky.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/distros/rocky.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,52 @@ +# Copyright (C) Louis Abel + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.policies.distros.redhat import RedHatPolicy, OS_RELEASE +import os + + +class RockyPolicy(RedHatPolicy): + distro = "Rocky Linux" + vendor = "Rocky Enterprise Software Foundation" + vendor_urls = [ + ('Distribution Website', 'https://rockylinux.org'), + ('Vendor Website', 'https://resf.org') + ] + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(RockyPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + + @classmethod + def check(cls, remote=''): + if remote: + return cls.distro in remote + + # Return False if /etc/os-release is missing + if not os.path.exists(OS_RELEASE): + return False + + # Return False if /etc/rocky-release is missing + if not os.path.isfile('/etc/rocky-release'): + return False + + # If we've gotten this far, check for Rocky in + # /etc/os-release + with open(OS_RELEASE, 'r') as f: + for line in f: + if line.startswith('NAME'): + if 'Rocky Linux' in line: + return True + + return False + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/distros/suse.py 4.5.3ubuntu2/sos/policies/distros/suse.py --- 4.0-2/sos/policies/distros/suse.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/distros/suse.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,92 @@ +# Copyright (C) 2015 Red Hat, Inc. Bryn M. Reeves +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import os +import sys + +from sos.report.plugins import RedHatPlugin, SuSEPlugin +from sos.policies.distros import LinuxPolicy +from sos.policies.package_managers.rpm import RpmPackageManager +from sos import _sos as _ + + +class SuSEPolicy(LinuxPolicy): + distro = "SuSE" + vendor = "SuSE" + vendor_urls = [('Distribution Website', 'https://www.suse.com/')] + _tmp_dir = "/var/tmp" + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(SuSEPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + self.valid_subclasses += [SuSEPlugin, RedHatPlugin] + + self.usrmove = False + self.package_manager = RpmPackageManager() + + # If rpm query timed out after timeout duration exit + if not self.package_manager.packages: + self.ui_log.error("Could not obtain installed package list.") + sys.exit(1) + + self.PATH = "/usr/sbin:/usr/bin:/root/bin:/sbin" + self.PATH += os.pathsep + "/usr/local/bin" + self.PATH += os.pathsep + "/usr/local/sbin" + self.set_exec_path() + + @classmethod + def check(cls, remote=''): + """This method checks to see if we are running on SuSE. It must be + overriden by concrete subclasses to return True when running on an + OpenSuSE, SLES or other Suse distribution and False otherwise.""" + return False + + def get_tmp_dir(self, opt_tmp_dir): + if not opt_tmp_dir: + return self._tmp_dir + return opt_tmp_dir + + def get_local_name(self): + return self.host_name() + + +class OpenSuSEPolicy(SuSEPolicy): + distro = "OpenSuSE" + vendor = "SuSE" + vendor_urls = [('Community Website', 'https://www.opensuse.org/')] + msg = _("""\ +This command will collect diagnostic and configuration \ +information from this %(distro)s system and installed \ +applications. + +An archive containing the collected information will be \ +generated in %(tmpdir)s and may be provided to a %(vendor)s \ +support representative. + +No changes will be made to system configuration. +%(vendor_text)s +""") + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(OpenSuSEPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + + @classmethod + def check(cls, remote): + """This method checks to see if we are running on SuSE. + """ + + if remote: + return cls.distro in remote + + return os.path.isfile('/etc/SUSE-brand') diff -pruN 4.0-2/sos/policies/distros/ubuntu.py 4.5.3ubuntu2/sos/policies/distros/ubuntu.py --- 4.0-2/sos/policies/distros/ubuntu.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/distros/ubuntu.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,83 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import UbuntuPlugin +from sos.policies.distros.debian import DebianPolicy + +import os + + +class UbuntuPolicy(DebianPolicy): + distro = "Ubuntu" + vendor = "Canonical" + vendor_urls = [ + ('Community Website', 'https://www.ubuntu.com/'), + ('Commercial Support', 'https://www.canonical.com') + ] + PATH = "/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" \ + + ":/usr/local/sbin:/usr/local/bin:/snap/bin" + _upload_url = "https://files.support.canonical.com/uploads/" + _upload_user = "ubuntu" + _upload_password = "ubuntu" + _upload_method = 'put' + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(UbuntuPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + self.valid_subclasses += [UbuntuPlugin] + + @classmethod + def check(cls, remote=''): + """This method checks to see if we are running on Ubuntu. + It returns True or False.""" + + if remote: + return cls.distro in remote + + try: + with open('/etc/lsb-release', 'r') as fp: + return "Ubuntu" in fp.read() + except IOError: + return False + + def dist_version(self): + """ Returns the version stated in DISTRIB_RELEASE + """ + try: + with open('/etc/lsb-release', 'r') as fp: + lines = fp.readlines() + for line in lines: + if "DISTRIB_RELEASE" in line: + return int(line.split("=")[1].strip()) + return False + except (IOError, ValueError): + return False + + def get_upload_https_auth(self): + if self.upload_url.startswith(self._upload_url): + return (self._upload_user, self._upload_password) + else: + return super(UbuntuPolicy, self).get_upload_https_auth() + + def get_upload_url_string(self): + if self.upload_url.startswith(self._upload_url): + return "Canonical Support File Server" + else: + return self.get_upload_url() + + def get_upload_url(self): + if not self.upload_url or self.upload_url.startswith(self._upload_url): + if not self.upload_archive_name: + return self._upload_url + fname = os.path.basename(self.upload_archive_name) + return self._upload_url + fname + return super(UbuntuPolicy, self).get_upload_url() + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/distros/uniontechserver.py 4.5.3ubuntu2/sos/policies/distros/uniontechserver.py --- 4.0-2/sos/policies/distros/uniontechserver.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/distros/uniontechserver.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,40 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.policies.distros.redhat import RedHatPolicy, OS_RELEASE +import os + + +class UnionTechPolicy(RedHatPolicy): + distro = "UnionTech OS Server" + vendor = "The UnionTech Project" + vendor_urls = [('Distribution Website', 'https://www.chinauos.com/')] + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(UnionTechPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + + @classmethod + def check(cls, remote=''): + + if remote: + return cls.distro in remote + + if not os.path.exists(OS_RELEASE): + return False + + with open(OS_RELEASE, 'r') as f: + for line in f: + if line.startswith('NAME'): + if 'UnionTech OS Server' in line: + return True + return False + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/ibmkvm.py 4.5.3ubuntu2/sos/policies/ibmkvm.py --- 4.0-2/sos/policies/ibmkvm.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/ibmkvm.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,78 +0,0 @@ -# Copyright (C) IBM Corporation, 2015 -# -# Authors: Kamalesh Babulal -# -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information. - -from sos.report.plugins import PowerKVMPlugin, ZKVMPlugin, RedHatPlugin -from sos.policies.redhat import RedHatPolicy - -import os - - -class PowerKVMPolicy(RedHatPolicy): - distro = "PowerKVM" - vendor = "IBM" - vendor_url = "http://www-03.ibm.com/systems/power/software/linux/powerkvm" - - def __init__(self, sysroot=None, init=None, probe_runtime=True, - remote_exec=None): - super(PowerKVMPolicy, self).__init__(sysroot=sysroot, init=init, - probe_runtime=probe_runtime, - remote_exec=remote_exec) - self.valid_subclasses = [PowerKVMPlugin, RedHatPlugin] - - @classmethod - def check(cls, remote=''): - """This method checks to see if we are running on PowerKVM. - It returns True or False.""" - - if remote: - return cls.distro in remote - - return os.path.isfile('/etc/ibm_powerkvm-release') - - def dist_version(self): - try: - with open('/etc/ibm_powerkvm-release', 'r') as fp: - version_string = fp.read() - return version_string[2][0] - except IOError: - return False - - -class ZKVMPolicy(RedHatPolicy): - distro = "IBM Hypervisor" - vendor = "IBM Hypervisor" - vendor_url = "http://www.ibm.com/systems/z/linux/IBMHypervisor/support/" - - def __init__(self, sysroot=None): - super(ZKVMPolicy, self).__init__(sysroot=sysroot) - self.valid_subclasses = [ZKVMPlugin, RedHatPlugin] - - @classmethod - def check(cls, remote=''): - """This method checks to see if we are running on IBM Z KVM. It - returns True or False.""" - - if remote: - return cls.distro in remote - - return os.path.isfile('/etc/base-release') - - def dist_version(self): - try: - with open('/etc/base-release', 'r') as fp: - version_string = fp.read() - return version_string.split(' ', 4)[3][0] - except IOError: - return False - - -# vim: set ts=4 sw=4 diff -pruN 4.0-2/sos/policies/init_systems/__init__.py 4.5.3ubuntu2/sos/policies/init_systems/__init__.py --- 4.0-2/sos/policies/init_systems/__init__.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/init_systems/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,181 @@ +# Copyright (C) 2020 Red Hat, Inc., Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import re +from sos.utilities import sos_get_command_output + + +class InitSystem(): + """Encapsulates an init system to provide service-oriented functions to + sos. + + This should be used to query the status of services, such as if they are + enabled or disabled on boot, or if the service is currently running. + + :param init_cmd: The binary used to interact with the init system + :type init_cmd: ``str`` + + :param list_cmd: The list subcmd given to `init_cmd` to list services + :type list_cmd: ``str`` + + :param query_cmd: The query subcmd given to `query_cmd` to query the + status of services + :type query_cmd: ``str`` + + :param chroot: Location to chroot to for any command execution, i.e. the + sysroot if we're running in a container + :type chroot: ``str`` or ``None`` + + """ + + def __init__(self, init_cmd=None, list_cmd=None, query_cmd=None, + chroot=None): + """Initialize a new InitSystem()""" + + self.services = {} + + self.init_cmd = init_cmd + self.list_cmd = "%s %s" % (self.init_cmd, list_cmd) or None + self.query_cmd = "%s %s" % (self.init_cmd, query_cmd) or None + self.chroot = chroot + + def is_enabled(self, name): + """Check if given service name is enabled + + :param name: The name of the service + :type name: ``str`` + + :returns: ``True`` if the service is enabled, else ``False`` + :rtype: ``bool`` + """ + if self.services and name in self.services: + return self.services[name]['config'] == 'enabled' + return False + + def is_disabled(self, name): + """Check if a given service name is disabled + :param name: The name of the service + :type name: ``str`` + + :returns: ``True`` if the service is disabled, else ``False`` + :rtype: ``bool`` + """ + if self.services and name in self.services: + return self.services[name]['config'] == 'disabled' + return False + + def is_service(self, name): + """Checks if the given service name exists on the system at all, this + does not check for the service status + + :param name: The name of the service + :type name: ``str`` + + :returns: ``True`` if the service exists, else ``False`` + :rtype: ``bool`` + """ + return name in self.services + + def is_running(self, name, default=True): + """Checks if the given service name is in a running state. + + This should be overridden by initsystems that subclass InitSystem + + :param name: The name of the service + :type name: ``str`` + + :param default: The default response in case the check fails + :type default: ``bool` + + :returns: ``True`` if the service is running, else ``default`` + :rtype: ``bool`` + """ + # This is going to be primarily used in gating if service related + # commands are going to be run or not. Default to always returning + # True when an actual init system is not specified by policy so that + # we don't inadvertantly restrict sosreports on those systems + return default + + def load_all_services(self): + """This loads all services known to the init system into a dict. + The dict should be keyed by the service name, and contain a dict of the + name and service status + + This must be overridden by anything that subclasses `InitSystem` in + order for service methods to function properly + """ + pass + + def _query_service(self, name): + """Query an individual service""" + if self.query_cmd: + try: + return sos_get_command_output( + "%s %s" % (self.query_cmd, name), + chroot=self.chroot + ) + except Exception: + return None + return None + + def parse_query(self, output): + """Parses the output returned by the query command to make a + determination of what the state of the service is + + This should be overriden by anything that subclasses InitSystem + + :param output: The raw output from querying the service with the + configured `query_cmd` + :type output: ``str`` + + :returns: A state for the service, e.g. 'active', 'disabled', etc... + :rtype: ``str`` + """ + return output + + def get_service_names(self, regex): + """Get a list of all services discovered on the system that match the + given regex. + + :param regex: The service name regex to match against + :type regex: ``str`` + """ + reg = re.compile(regex, re.I) + return [s for s in self.services.keys() if reg.match(s)] + + def get_service_status(self, name): + """Get the status for the given service name along with the output + of the query command + + :param name: The name of the service + :type name: ``str`` + + :returns: Service status and query_cmd output from the init system + :rtype: ``dict`` with keys `name`, `status`, and `output` + """ + _default = { + 'name': name, + 'status': 'missing', + 'output': '' + } + if name not in self.services: + return _default + if 'status' in self.services[name]: + # service status has been queried before, return existing info + return self.services[name] + svc = self._query_service(name) + if svc is not None: + self.services[name]['status'] = self.parse_query(svc['output']) + self.services[name]['output'] = svc['output'] + return self.services[name] + return _default + + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/init_systems/systemd.py 4.5.3ubuntu2/sos/policies/init_systems/systemd.py --- 4.0-2/sos/policies/init_systems/systemd.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/init_systems/systemd.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,53 @@ +# Copyright (C) 2020 Red Hat, Inc., Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.policies.init_systems import InitSystem +from sos.utilities import shell_out + + +class SystemdInit(InitSystem): + """InitSystem abstraction for SystemD systems""" + + def __init__(self, chroot=None): + super(SystemdInit, self).__init__( + init_cmd='systemctl', + list_cmd='list-unit-files --type=service', + query_cmd='status', + chroot=chroot + ) + self.load_all_services() + + def parse_query(self, output): + for line in output.splitlines(): + if line.strip().startswith('Active:'): + return line.split()[1] + return 'unknown' + + def load_all_services(self): + svcs = shell_out(self.list_cmd, chroot=self.chroot).splitlines()[1:] + for line in svcs: + try: + name = line.split('.service')[0] + config = line.split()[1] + self.services[name] = { + 'name': name, + 'config': config + } + except IndexError: + pass + + def is_running(self, name, default=False): + try: + svc = self.get_service_status(name) + return svc['status'] == 'active' + except Exception: + return default + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/package_managers/__init__.py 4.5.3ubuntu2/sos/policies/package_managers/__init__.py --- 4.0-2/sos/policies/package_managers/__init__.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/package_managers/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,288 @@ +# Copyright 2020 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import re +import fnmatch + +from sos.utilities import sos_get_command_output + + +class PackageManager(): + """Encapsulates a package manager. If you provide a query_command to the + constructor it should print each package on the system in the following + format:: + + package name|package.version + + You may also subclass this class and provide a _generate_pkg_list method to + build the list of packages and versions. + + :cvar query_command: The command to use for querying packages + :vartype query_command: ``str`` or ``None`` + + :cvar verify_command: The command to use for verifying packages + :vartype verify_command: ``str`` or ``None`` + + :cvar verify_filter: Optional filter to use for controlling package + verification + :vartype verify_filter: ``str or ``None`` + + :cvar files_command: The command to use for getting file lists for packages + :vartype files_command: ``str`` or ``None`` + + :cvar chroot: Perform a chroot when executing `files_command` + :vartype chroot: ``bool`` + + :cvar remote_exec: If package manager is on a remote system (e.g. for + sos collect), use this to execute commands + :vartype remote_exec: ``SoSTransport.run_command()`` + """ + + query_command = None + verify_command = None + verify_filter = None + files_command = None + query_path_command = None + chroot = None + files = None + + def __init__(self, chroot=None, query_command=None, verify_command=None, + verify_filter=None, files_command=None, + query_path_command=None, remote_exec=None): + self._packages = {} + self.files = [] + self.remote_exec = remote_exec + + self.query_command = query_command or self.query_command + self.verify_command = verify_command or self.verify_command + self.verify_filter = verify_filter or self.verify_filter + self.files_command = files_command or self.files_command + self.query_path_command = query_path_command or self.query_path_command + + if chroot: + self.chroot = chroot + + @property + def packages(self): + if not self._packages: + self._generate_pkg_list() + return self._packages + + def exec_cmd(self, command, timeout=30, need_root=False, env=None, + get_pty=False, chroot=None): + """ + Runs a package manager command, either via sos_get_command_output() if + local, or via a SoSTransport's run_command() if this needs to be run + remotely, as in the case of remote nodes for use during `sos collect`. + + :param command: The command to execute + :type command: ``str`` + + :param timeout: Timeout for command to run, in seconds + :type timeout: ``int`` + + :param need_root: Does the command require root privileges? + :type need_root: ``bool`` + + :param env: Environment variables to set + :type env: ``dict`` with keys being env vars to define + + :param get_pty: If running remotely, does the command require + obtaining a pty? + :type get_pty: ``bool`` + + :param chroot: If necessary, chroot command execution to here + :type chroot: ``None`` or ``str`` + + :returns: The output of the command + :rtype: ``str`` + """ + if self.remote_exec: + ret = self.remote_exec(command, timeout, need_root, env, get_pty) + else: + ret = sos_get_command_output(command, timeout, chroot=chroot, + env=env) + if ret['status'] == 0: + return ret['output'] + # In the case of package managers, we don't want to potentially iterate + # over stderr, so prevent the package methods from doing anything at + # all by returning nothing. + return '' + + def all_pkgs_by_name(self, name): + """ + Get a list of packages that match name. + + :param name: The name of the package + :type name: ``str`` + + :returns: List of all packages matching `name` + :rtype: ``list`` + """ + return fnmatch.filter(self.packages.keys(), name) + + def all_pkgs_by_name_regex(self, regex_name, flags=0): + """ + Get a list of packages that match regex_name. + + :param regex_name: The regex to use for matching package names against + :type regex_name: ``str`` + + :param flags: Flags for the `re` module when matching `regex_name` + + :returns: All packages matching `regex_name` + :rtype: ``list`` + """ + reg = re.compile(regex_name, flags) + return [pkg for pkg in self.packages.keys() if reg.match(pkg)] + + def pkg_by_name(self, name): + """ + Get a single package that matches name. + + :param name: The name of the package + :type name: ``str`` + + :returns: The first package that matches `name` + :rtype: ``str`` + """ + try: + return self.packages[name] + except Exception: + return None + + def _generate_pkg_list(self): + """Generates a dictionary of packages for internal use by the package + manager in the format:: + + {'package_name': {'name': 'package_name', + 'version': 'major.minor.version'}} + + """ + if self.query_command: + cmd = self.query_command + pkg_list = self.exec_cmd(cmd, timeout=30, chroot=self.chroot) + + for pkg in pkg_list.splitlines(): + if '|' not in pkg: + continue + elif pkg.count("|") == 1: + name, version = pkg.split("|") + release = None + elif pkg.count("|") == 2: + name, version, release = pkg.split("|") + self._packages[name] = { + 'name': name, + 'version': version.split(".") + } + release = release if release else None + self._packages[name]['release'] = release + + def pkg_version(self, pkg): + """Returns the entry in self.packages for pkg if it exists + + :param pkg: The name of the package + :type pkg: ``str`` + + :returns: Package name and version, if package exists + :rtype: ``dict`` if found, else ``None`` + """ + if pkg in self.packages: + return self.packages[pkg] + return None + + def pkg_nvra(self, pkg): + """Get the name, version, release, and architecture for a package + + :param pkg: The name of the package + :type pkg: ``str`` + + :returns: name, version, release, and arch of the package + :rtype: ``tuple`` + """ + fields = pkg.split("-") + version, release, arch = fields[-3:] + name = "-".join(fields[:-3]) + return (name, version, release, arch) + + def all_files(self): + """ + Get a list of files known by the package manager + + :returns: All files known by the package manager + :rtype: ``list`` + """ + if self.files_command and not self.files: + cmd = self.files_command + files = self.exec_cmd(cmd, timeout=180, chroot=self.chroot) + self.files = files.splitlines() + return self.files + + def pkg_by_path(self, path): + """Given a path, return the package that owns that path. + + :param path: The filepath to check for package ownership + :type path: ``str`` + + :returns: The package name or 'unknown' + :rtype: ``str`` + """ + if not self.query_path_command: + return 'unknown' + try: + cmd = f"{self.query_path_command} {path}" + pkg = self.exec_cmd(cmd, timeout=5, chroot=self.chroot) + return pkg.splitlines() or 'unknown' + except Exception: + return 'unknown' + + def build_verify_command(self, packages): + """build_verify_command(self, packages) -> str + Generate a command to verify the list of packages given + in ``packages`` using the native package manager's + verification tool. + + The command to be executed is returned as a string that + may be passed to a command execution routine (for e.g. + ``sos_get_command_output()``. + + :param packages: a string, or a list of strings giving + package names to be verified. + :returns: a string containing an executable command + that will perform verification of the given + packages. + :rtype: str or ``NoneType`` + """ + if not self.verify_command: + return None + + # The re.match(pkg) used by all_pkgs_by_name_regex() may return + # an empty list (`[[]]`) when no package matches: avoid building + # an rpm -V command line with the empty string as the package + # list in this case. + by_regex = self.all_pkgs_by_name_regex + verify_list = filter(None, map(by_regex, packages)) + + # No packages after regex match? + if not verify_list: + return None + + verify_packages = "" + for package_list in verify_list: + for package in package_list: + if any([f in package for f in self.verify_filter]): + continue + if len(verify_packages): + verify_packages += " " + verify_packages += package + return self.verify_command + " " + verify_packages + + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/package_managers/dpkg.py 4.5.3ubuntu2/sos/policies/package_managers/dpkg.py --- 4.0-2/sos/policies/package_managers/dpkg.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/package_managers/dpkg.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,24 @@ +# Copyright 2020 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.policies.package_managers import PackageManager + + +class DpkgPackageManager(PackageManager): + """Subclass for dpkg-based distrubitons + """ + + query_command = "dpkg-query -W -f='${Package}|${Version}\\n'" + query_path_command = "dpkg -S" + verify_command = "dpkg --verify" + verify_filter = "" + + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/package_managers/rpm.py 4.5.3ubuntu2/sos/policies/package_managers/rpm.py --- 4.0-2/sos/policies/package_managers/rpm.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/package_managers/rpm.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,25 @@ +# Copyright 2020 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.policies.package_managers import PackageManager + + +class RpmPackageManager(PackageManager): + """Package Manager for RPM-based distributions + """ + + query_command = 'rpm -qa --queryformat "%{NAME}|%{VERSION}|%{RELEASE}\\n"' + query_path_command = 'rpm -qf' + files_command = 'rpm -qal' + verify_command = 'rpm -V' + verify_filter = ["debuginfo", "-devel"] + + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/redhat.py 4.5.3ubuntu2/sos/policies/redhat.py --- 4.0-2/sos/policies/redhat.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/redhat.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,563 +0,0 @@ -# Copyright (C) Steve Conklin - -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information. - -import os -import sys -import re - -from sos.report.plugins import RedHatPlugin -from sos.policies import LinuxPolicy, PackageManager, PresetDefaults -from sos import _sos as _ -from sos.options import SoSOptions - -OS_RELEASE = "/etc/os-release" - - -class RedHatPolicy(LinuxPolicy): - distro = "Red Hat" - vendor = "Red Hat" - vendor_url = "https://www.redhat.com/" - _redhat_release = '/etc/redhat-release' - _tmp_dir = "/var/tmp" - _rpmq_cmd = 'rpm -qa --queryformat "%{NAME}|%{VERSION}|%{RELEASE}\\n"' - _rpmql_cmd = 'rpm -qal' - _rpmv_cmd = 'rpm -V' - _rpmv_filter = ["debuginfo", "-devel"] - _in_container = False - _host_sysroot = '/' - default_scl_prefix = '/opt/rh' - name_pattern = 'friendly' - upload_url = 'dropbox.redhat.com' - upload_user = 'anonymous' - upload_directory = '/incoming' - default_container_runtime = 'podman' - sos_pkg_name = 'sos' - sos_bin_path = '/usr/sbin/sosreport' - - def __init__(self, sysroot=None, init=None, probe_runtime=True, - remote_exec=None): - super(RedHatPolicy, self).__init__(sysroot=sysroot, init=init, - probe_runtime=probe_runtime) - self.ticket_number = "" - self.usrmove = False - # need to set _host_sysroot before PackageManager() - if sysroot: - self._container_init() - self._host_sysroot = sysroot - else: - sysroot = self._container_init() - - self.package_manager = PackageManager(query_command=self._rpmq_cmd, - verify_command=self._rpmv_cmd, - verify_filter=self._rpmv_filter, - files_command=self._rpmql_cmd, - chroot=sysroot, - remote_exec=remote_exec) - - self.valid_subclasses = [RedHatPlugin] - - self.pkgs = self.package_manager.all_pkgs() - - # If rpm query failed, exit - if not self.pkgs: - sys.stderr.write("Could not obtain installed package list") - sys.exit(1) - - self.usrmove = self.check_usrmove(self.pkgs) - - if self.usrmove: - self.PATH = "/usr/sbin:/usr/bin:/root/bin" - else: - self.PATH = "/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" - self.PATH += os.pathsep + "/usr/local/bin" - self.PATH += os.pathsep + "/usr/local/sbin" - self.set_exec_path() - self.load_presets() - - @classmethod - def check(cls, remote=''): - """This method checks to see if we are running on Red Hat. It must be - overriden by concrete subclasses to return True when running on a - Fedora, RHEL or other Red Hat distribution or False otherwise. - - If `remote` is provided, it should be the contents of a remote host's - os-release, or comparable, file to be used in place of the locally - available one. - """ - return False - - def check_usrmove(self, pkgs): - """Test whether the running system implements UsrMove. - - If the 'filesystem' package is present, it will check that the - version is greater than 3. If the package is not present the - '/bin' and '/sbin' paths are checked and UsrMove is assumed - if both are symbolic links. - - :param pkgs: a packages dictionary - """ - if 'filesystem' not in pkgs: - return os.path.islink('/bin') and os.path.islink('/sbin') - else: - filesys_version = pkgs['filesystem']['version'] - return True if filesys_version[0] == '3' else False - - def mangle_package_path(self, files): - """Mangle paths for post-UsrMove systems. - - If the system implements UsrMove, all files will be in - '/usr/[s]bin'. This method substitutes all the /[s]bin - references in the 'files' list with '/usr/[s]bin'. - - :param files: the list of package managed files - """ - paths = [] - - def transform_path(path): - # Some packages actually own paths in /bin: in this case, - # duplicate the path as both the / and /usr version. - skip_paths = ["/bin/rpm", "/bin/mailx"] - if path in skip_paths: - return (path, os.path.join("/usr", path[1:])) - return (re.sub(r'(^)(/s?bin)', r'\1/usr\2', path),) - - if self.usrmove: - for f in files: - paths.extend(transform_path(f)) - return paths - else: - return files - - def _container_init(self): - """Check if sos is running in a container and perform container - specific initialisation based on ENV_HOST_SYSROOT. - """ - if ENV_CONTAINER in os.environ: - if os.environ[ENV_CONTAINER] in ['docker', 'oci']: - self._in_container = True - if ENV_HOST_SYSROOT in os.environ: - self._host_sysroot = os.environ[ENV_HOST_SYSROOT] - use_sysroot = self._in_container and self._host_sysroot is not None - if use_sysroot: - host_tmp_dir = os.path.abspath(self._host_sysroot + self._tmp_dir) - self._tmp_dir = host_tmp_dir - return self._host_sysroot if use_sysroot else None - - def runlevel_by_service(self, name): - from subprocess import Popen, PIPE - ret = [] - p = Popen("LC_ALL=C /sbin/chkconfig --list %s" % name, - shell=True, - stdout=PIPE, - stderr=PIPE, - bufsize=-1, - close_fds=True) - out, err = p.communicate() - if err: - return ret - for tabs in out.split()[1:]: - try: - (runlevel, onoff) = tabs.split(":", 1) - except IndexError: - pass - else: - if onoff == "on": - ret.append(int(runlevel)) - return ret - - def get_tmp_dir(self, opt_tmp_dir): - if not opt_tmp_dir: - return self._tmp_dir - return opt_tmp_dir - - -# Container environment variables on Red Hat systems. -ENV_CONTAINER = 'container' -ENV_HOST_SYSROOT = 'HOST' - -_opts_verify = SoSOptions(verify=True) -_opts_all_logs = SoSOptions(all_logs=True) -_opts_all_logs_verify = SoSOptions(all_logs=True, verify=True) -_cb_profiles = ['boot', 'storage', 'system'] -_cb_plugopts = ['boot.all-images=on', 'rpm.rpmva=on', 'rpm.rpmdb=on'] - -RHEL_RELEASE_STR = "Red Hat Enterprise Linux" - -RHV = "rhv" -RHV_DESC = "Red Hat Virtualization" - -RHEL = "rhel" -RHEL_DESC = RHEL_RELEASE_STR - -RHOSP = "rhosp" -RHOSP_DESC = "Red Hat OpenStack Platform" - -RHOCP = "ocp" -RHOCP_DESC = "OpenShift Container Platform by Red Hat" -RHOSP_OPTS = SoSOptions(plugopts=[ - 'process.lsof=off', - 'networking.ethtool_namespaces=False', - 'networking.namespaces=200']) - -RH_CFME = "cfme" -RH_CFME_DESC = "Red Hat CloudForms" - -RH_SATELLITE = "satellite" -RH_SATELLITE_DESC = "Red Hat Satellite" -SAT_OPTS = SoSOptions(log_size=100, plugopts=['apache.log=on']) - -CB = "cantboot" -CB_DESC = "For use when normal system startup fails" -CB_OPTS = SoSOptions( - verify=True, all_logs=True, profiles=_cb_profiles, - plugopts=_cb_plugopts - ) -CB_NOTE = ("Data collection will be limited to a boot-affecting scope") - -NOTE_SIZE = "This preset may increase report size" -NOTE_TIME = "This preset may increase report run time" -NOTE_SIZE_TIME = "This preset may increase report size and run time" - -rhel_presets = { - RHV: PresetDefaults(name=RHV, desc=RHV_DESC, note=NOTE_TIME, - opts=_opts_verify), - RHEL: PresetDefaults(name=RHEL, desc=RHEL_DESC), - RHOSP: PresetDefaults(name=RHOSP, desc=RHOSP_DESC, opts=RHOSP_OPTS), - RHOCP: PresetDefaults(name=RHOCP, desc=RHOCP_DESC, note=NOTE_SIZE_TIME, - opts=_opts_all_logs_verify), - RH_CFME: PresetDefaults(name=RH_CFME, desc=RH_CFME_DESC, note=NOTE_TIME, - opts=_opts_verify), - RH_SATELLITE: PresetDefaults(name=RH_SATELLITE, desc=RH_SATELLITE_DESC, - note=NOTE_TIME, opts=SAT_OPTS), - CB: PresetDefaults(name=CB, desc=CB_DESC, note=CB_NOTE, opts=CB_OPTS) -} - -# Legal disclaimer text for Red Hat products -disclaimer_text = """ -Any information provided to %(vendor)s will be treated in \ -accordance with the published support policies at:\n - %(vendor_url)s - -The generated archive may contain data considered sensitive \ -and its content should be reviewed by the originating \ -organization before being passed to any third party. - -No changes will be made to system configuration. -""" - -RH_API_HOST = "https://access.redhat.com" -RH_FTP_HOST = "ftp://dropbox.redhat.com" - - -class RHELPolicy(RedHatPolicy): - distro = RHEL_RELEASE_STR - vendor = "Red Hat" - vendor_url = "https://access.redhat.com/support/" - msg = _("""\ -This command will collect diagnostic and configuration \ -information from this %(distro)s system and installed \ -applications. - -An archive containing the collected information will be \ -generated in %(tmpdir)s and may be provided to a %(vendor)s \ -support representative. -""" + disclaimer_text + "%(vendor_text)s\n") - _upload_url = RH_FTP_HOST - _upload_user = 'anonymous' - _upload_directory = '/incoming' - - def __init__(self, sysroot=None, init=None, probe_runtime=True, - remote_exec=None): - super(RHELPolicy, self).__init__(sysroot=sysroot, init=init, - probe_runtime=probe_runtime, - remote_exec=remote_exec) - self.register_presets(rhel_presets) - - @classmethod - def check(cls, remote=''): - """Test to see if the running host is a RHEL installation. - - Checks for the presence of the "Red Hat Enterprise Linux" - release string at the beginning of the NAME field in the - `/etc/os-release` file and returns ``True`` if it is - found, and ``False`` otherwise. - - :returns: ``True`` if the host is running RHEL or ``False`` - otherwise. - """ - - if remote: - return cls.distro in remote - - if not os.path.exists(OS_RELEASE): - return False - - with open(OS_RELEASE, "r") as f: - for line in f: - if line.startswith("NAME"): - (name, value) = line.split("=") - value = value.strip("\"'") - if value.startswith(cls.distro): - return True - return False - - def prompt_for_upload_user(self): - if self.commons['cmdlineopts'].upload_user: - return - # Not using the default, so don't call this prompt for RHCP - if self.commons['cmdlineopts'].upload_url: - super(RHELPolicy, self).prompt_for_upload_user() - return - if self.case_id: - self.upload_user = input(_( - "Enter your Red Hat Customer Portal username (empty to use " - "public dropbox): ") - ) - - def get_upload_url(self): - if self.commons['cmdlineopts'].upload_url: - return self.commons['cmdlineopts'].upload_url - if (not self.case_id or not self.upload_user or not - self.upload_password): - # Cannot use the RHCP. Use anonymous dropbox - self.upload_user = self._upload_user - self.upload_directory = self._upload_directory - self.upload_password = None - return RH_FTP_HOST - else: - rh_case_api = "/hydra/rest/cases/%s/attachments" - return RH_API_HOST + rh_case_api % self.case_id - - def _get_upload_headers(self): - if self.get_upload_url().startswith(RH_API_HOST): - return {'isPrivate': 'false', 'cache-control': 'no-cache'} - return {} - - def get_upload_url_string(self): - if self.get_upload_url().startswith(RH_API_HOST): - return "Red Hat Customer Portal" - return self.upload_url or RH_FTP_HOST - - def get_upload_user(self): - # if this is anything other than dropbox, annonymous won't work - if self.upload_url != RH_FTP_HOST: - return self.upload_user - return self._upload_user - - def dist_version(self): - try: - rr = self.package_manager.all_pkgs_by_name_regex("redhat-release*") - pkgname = self.pkgs[rr[0]]["version"] - if pkgname[0] == "4": - return 4 - elif pkgname[0] in ["5Server", "5Client"]: - return 5 - elif pkgname[0] == "6": - return 6 - elif pkgname[0] == "7": - return 7 - elif pkgname[0] == "8": - return 8 - except Exception: - pass - return False - - def probe_preset(self): - # Emergency or rescue mode? - for target in ["rescue", "emergency"]: - if self.init_system.is_running("%s.target" % target): - return self.find_preset(CB) - # Package based checks - if self.pkg_by_name("satellite-common") is not None: - return self.find_preset(RH_SATELLITE) - if self.pkg_by_name("rhosp-release") is not None: - return self.find_preset(RHOSP) - if self.pkg_by_name("cfme") is not None: - return self.find_preset(RH_CFME) - if self.pkg_by_name("ovirt-engine") is not None or \ - self.pkg_by_name("vdsm") is not None: - return self.find_preset(RHV) - - # Vanilla RHEL is default - return self.find_preset(RHEL) - - -class CentOsPolicy(RHELPolicy): - distro = "CentOS" - vendor = "CentOS" - vendor_url = "https://www.centos.org/" - - -ATOMIC = "atomic" -ATOMIC_RELEASE_STR = "Atomic" -ATOMIC_DESC = "Red Hat Enterprise Linux Atomic Host" - -atomic_presets = { - ATOMIC: PresetDefaults(name=ATOMIC, desc=ATOMIC_DESC, note=NOTE_TIME, - opts=_opts_verify) -} - - -class RedHatAtomicPolicy(RHELPolicy): - distro = "Red Hat Atomic Host" - msg = _("""\ -This command will collect diagnostic and configuration \ -information from this %(distro)s system. - -An archive containing the collected information will be \ -generated in %(tmpdir)s and may be provided to a %(vendor)s \ -support representative. -""" + disclaimer_text + "%(vendor_text)s\n") - - containerzed = True - container_runtime = 'docker' - container_image = 'registry.access.redhat.com/rhel7/support-tools' - sos_path_strip = '/host' - container_version_command = 'rpm -q sos' - - def __init__(self, sysroot=None, init=None, probe_runtime=True, - remote_exec=None): - super(RedHatAtomicPolicy, self).__init__(sysroot=sysroot, init=init, - probe_runtime=probe_runtime, - remote_exec=remote_exec) - self.register_presets(atomic_presets) - - @classmethod - def check(cls, remote=''): - - if remote: - return cls.distro in remote - - atomic = False - if ENV_HOST_SYSROOT not in os.environ: - return atomic - host_release = os.environ[ENV_HOST_SYSROOT] + cls._redhat_release - if not os.path.exists(host_release): - return False - try: - for line in open(host_release, "r").read().splitlines(): - atomic |= ATOMIC_RELEASE_STR in line - except IOError: - pass - return atomic - - def probe_preset(self): - if self.pkg_by_name('atomic-openshift'): - return self.find_preset(RHOCP) - - return self.find_preset(ATOMIC) - - def create_sos_container(self): - _cmd = ("{runtime} run -di --name {name} --privileged --ipc=host" - " --net=host --pid=host -e HOST=/host -e NAME={name} -e " - "IMAGE={image} -v /run:/run -v /var/log:/var/log -v " - "/etc/machine-id:/etc/machine-id -v " - "/etc/localtime:/etc/localtime -v /:/host {image}") - return _cmd.format(runtime=self.container_runtime, - name=self.sos_container_name, - image=self.container_image) - - def set_cleanup_cmd(self): - return 'docker rm --force sos-collector-tmp' - - -class RedHatCoreOSPolicy(RHELPolicy): - distro = "Red Hat CoreOS" - msg = _("""\ -This command will collect diagnostic and configuration \ -information from this %(distro)s system. - -An archive containing the collected information will be \ -generated in %(tmpdir)s and may be provided to a %(vendor)s \ -support representative. -""" + disclaimer_text + "%(vendor_text)s\n") - - containerized = True - container_runtime = 'podman' - container_image = 'registry.redhat.io/rhel8/support-tools' - sos_path_strip = '/host' - container_version_command = 'rpm -q sos' - - def __init__(self, sysroot=None, init=None, probe_runtime=True, - remote_exec=None): - super(RedHatCoreOSPolicy, self).__init__(sysroot=sysroot, init=init, - probe_runtime=probe_runtime, - remote_exec=remote_exec) - - @classmethod - def check(cls, remote=''): - - if remote: - return 'CoreOS' in remote - - coreos = False - if ENV_HOST_SYSROOT not in os.environ: - return coreos - host_release = os.environ[ENV_HOST_SYSROOT] + cls._redhat_release - try: - for line in open(host_release, 'r').read().splitlines(): - coreos |= 'Red Hat Enterprise Linux CoreOS' in line - except IOError: - pass - return coreos - - def probe_preset(self): - # As of the creation of this policy, RHCOS is only available for - # RH OCP environments. - return self.find_preset(RHOCP) - - def create_sos_container(self): - _cmd = ("{runtime} run -di --name {name} --privileged --ipc=host" - " --net=host --pid=host -e HOST=/host -e NAME={name} -e " - "IMAGE={image} -v /run:/run -v /var/log:/var/log -v " - "/etc/machine-id:/etc/machine-id -v " - "/etc/localtime:/etc/localtime -v /:/host {image}") - return _cmd.format(runtime=self.container_runtime, - name=self.sos_container_name, - image=self.container_image) - - def set_cleanup_cmd(self): - return 'podman rm --force %s' % self.sos_container_name - - -class CentOsAtomicPolicy(RedHatAtomicPolicy): - distro = "CentOS Atomic Host" - vendor = "CentOS" - vendor_url = "https://www.centos.org/" - - -class FedoraPolicy(RedHatPolicy): - - distro = "Fedora" - vendor = "the Fedora Project" - vendor_url = "https://fedoraproject.org/" - - def __init__(self, sysroot=None, init=None, probe_runtime=True, - remote_exec=None): - super(FedoraPolicy, self).__init__(sysroot=sysroot, init=init, - probe_runtime=probe_runtime, - remote_exec=remote_exec) - - @classmethod - def check(cls, remote=''): - """This method checks to see if we are running on Fedora. It returns - True or False.""" - - if remote: - return cls.distro in remote - - return os.path.isfile('/etc/fedora-release') - - def fedora_version(self): - pkg = self.pkg_by_name("fedora-release") or \ - self.all_pkgs_by_name_regex("fedora-release-.*")[-1] - return int(pkg["version"]) - -# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/runtimes/__init__.py 4.5.3ubuntu2/sos/policies/runtimes/__init__.py --- 4.0-2/sos/policies/runtimes/__init__.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/runtimes/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,252 @@ +# Copyright (C) 2020 Red Hat, Inc., Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import re + +from pipes import quote +from sos.utilities import sos_get_command_output, is_executable + + +class ContainerRuntime(): + """Encapsulates a container runtime that provides the ability to plugins to + check runtime status, check for the presence of specific containers, and + to format commands to run in those containers + + :param policy: The loaded policy for the system + :type policy: ``Policy()`` + + :cvar name: The name of the container runtime, e.g. 'podman' + :vartype name: ``str`` + + :cvar containers: A list of containers known to the runtime + :vartype containers: ``list`` + + :cvar images: A list of images known to the runtime + :vartype images: ``list`` + + :cvar binary: The binary command to run for the runtime, must exit within + $PATH + :vartype binary: ``str`` + """ + + name = 'Undefined' + containers = [] + images = [] + volumes = [] + binary = '' + active = False + + def __init__(self, policy=None): + self.policy = policy + self.run_cmd = "%s exec " % self.binary + + def load_container_info(self): + """If this runtime is found to be active, attempt to load information + on the objects existing in the runtime. + """ + self.containers = self.get_containers() + self.images = self.get_images() + self.volumes = self.get_volumes() + + def check_is_active(self): + """Check to see if the container runtime is both present AND active. + + Active in this sense means that the runtime can be used to glean + information about the runtime itself and containers that are running. + + :returns: ``True`` if the runtime is active, else ``False`` + :rtype: ``bool`` + """ + if is_executable(self.binary, self.policy.sysroot): + self.active = True + return True + return False + + def check_can_copy(self): + """Check if the runtime supports copying files out of containers and + onto the host filesystem + """ + return True + + def get_containers(self, get_all=False): + """Get a list of containers present on the system. + + :param get_all: If set, include stopped containers as well + :type get_all: ``bool`` + """ + containers = [] + _cmd = "%s ps %s" % (self.binary, '-a' if get_all else '') + if self.active: + out = sos_get_command_output(_cmd, chroot=self.policy.sysroot) + if out['status'] == 0: + for ent in out['output'].splitlines()[1:]: + ent = ent.split() + # takes the form (container_id, container_name) + containers.append((ent[0], ent[-1])) + return containers + + def get_container_by_name(self, name): + """Get the container ID for the container matching the provided + name + + :param name: The name of the container, note this can be a regex + :type name: ``str`` + + :returns: The id of the first container to match `name`, else ``None`` + :rtype: ``str`` + """ + if not self.active or name is None: + return None + for c in self.containers: + if re.match(name, c[1]): + return c[0] + return None + + def get_images(self): + """Get a list of images present on the system + + :returns: A list of 2-tuples containing (image_name, image_id) + :rtype: ``list`` + """ + images = [] + fmt = '{{lower .Repository}}:{{lower .Tag}} {{lower .ID}}' + if self.active: + out = sos_get_command_output( + "%s images --format '%s'" % (self.binary, fmt), + chroot=self.policy.sysroot + ) + if out['status'] == 0: + for ent in out['output'].splitlines(): + ent = ent.split() + # takes the form (image_name, image_id) + images.append((ent[0], ent[1])) + return images + + def get_volumes(self): + """Get a list of container volumes present on the system + + :returns: A list of volume IDs on the system + :rtype: ``list`` + """ + vols = [] + if self.active: + out = sos_get_command_output( + "%s volume ls" % self.binary, + chroot=self.policy.sysroot + ) + if out['status'] == 0: + for ent in out['output'].splitlines()[1:]: + ent = ent.split() + vols.append(ent[-1]) + return vols + + def container_exists(self, container): + """Check if a given container ID or name exists on the system from the + perspective of the container runtime. + + Note that this will only check _running_ containers + + :param container: The name or ID of the container + :type container: ``str`` + + :returns: True if the container exists, else False + :rtype: ``bool`` + """ + for _contup in self.containers: + if container in _contup: + return True + return False + + def fmt_container_cmd(self, container, cmd, quotecmd): + """Format a command to run inside a container using the runtime + + :param container: The name or ID of the container in which to run + :type container: ``str`` + + :param cmd: The command to run inside `container` + :type cmd: ``str`` + + :param quotecmd: Whether the cmd should be quoted. + :type quotecmd: ``bool`` + + :returns: Formatted string to run `cmd` inside `container` + :rtype: ``str`` + """ + if quotecmd: + quoted_cmd = quote(cmd) + else: + quoted_cmd = cmd + return "%s %s %s" % (self.run_cmd, container, quoted_cmd) + + def fmt_registry_credentials(self, username, password): + """Format a string to pass to the 'run' command of the runtime to + enable authorization for pulling the image during `sos collect`, if + needed using username and optional password creds + + :param username: The name of the registry user + :type username: ``str`` + + :param password: The password of the registry user + :type password: ``str`` or ``None`` + + :returns: The string to use to enable a run command to pull the image + :rtype: ``str`` + """ + return "--creds=%s%s" % (username, ':' + password if password else '') + + def fmt_registry_authfile(self, authfile): + """Format a string to pass to the 'run' command of the runtime to + enable authorization for pulling the image during `sos collect`, if + needed using an authfile. + """ + if authfile: + return "--authfile %s" % authfile + return '' + + def get_logs_command(self, container): + """Get the command string used to dump container logs from the + runtime + + :param container: The name or ID of the container to get logs for + :type container: ``str`` + + :returns: Formatted runtime command to get logs from `container` + :type: ``str`` + """ + return "%s logs -t %s" % (self.binary, container) + + def get_copy_command(self, container, path, dest, sizelimit=None): + """Generate the command string used to copy a file out of a container + by way of the runtime. + + :param container: The name or ID of the container + :type container: ``str`` + + :param path: The path to copy from the container. Note that at + this time, no supported runtime supports globbing + :type path: ``str`` + + :param dest: The destination on the *host* filesystem to write + the file to + :type dest: ``str`` + + :param sizelimit: Limit the collection to the last X bytes of the + file at PATH + :type sizelimit: ``int`` + + :returns: Formatted runtime command to copy a file from a container + :rtype: ``str`` + """ + if sizelimit: + return "%s %s tail -c %s %s" % (self.run_cmd, container, sizelimit, + path) + return "%s cp %s:%s %s" % (self.binary, container, path, dest) + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/runtimes/crio.py 4.5.3ubuntu2/sos/policies/runtimes/crio.py --- 4.0-2/sos/policies/runtimes/crio.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/runtimes/crio.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,92 @@ +# Copyright (C) 2021 Red Hat, Inc., Nadia Pinaeva + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. +import json + +from sos.policies.runtimes import ContainerRuntime +from sos.utilities import sos_get_command_output +from pipes import quote + + +class CrioContainerRuntime(ContainerRuntime): + """Runtime class to use for systems running crio""" + + name = 'crio' + binary = 'crictl' + + def check_can_copy(self): + return False + + def get_containers(self, get_all=False): + """Get a list of containers present on the system. + + :param get_all: If set, include stopped containers as well + :type get_all: ``bool`` + """ + containers = [] + _cmd = "%s ps %s -o json" % (self.binary, '-a' if get_all else '') + if self.active: + out = sos_get_command_output(_cmd, chroot=self.policy.sysroot) + if out["status"] == 0: + out_json = json.loads(out["output"]) + for container in out_json["containers"]: + # takes the form (container_id, container_name) + containers.append( + (container["id"], container["metadata"]["name"])) + return containers + + def get_images(self): + """Get a list of images present on the system + + :returns: A list of 2-tuples containing (image_name, image_id) + :rtype: ``list`` + """ + images = [] + if self.active: + out = sos_get_command_output("%s images -o json" % self.binary, + chroot=self.policy.sysroot) + if out['status'] == 0: + out_json = json.loads(out["output"]) + for image in out_json["images"]: + # takes the form (repository:tag, image_id) + if len(image["repoTags"]) > 0: + for repo_tag in image["repoTags"]: + images.append((repo_tag, image["id"])) + else: + if len(image["repoDigests"]) == 0: + image_name = "" + else: + image_name = image["repoDigests"][0].split("@")[0] + images.append((image_name + ":", image["id"])) + return images + + def fmt_container_cmd(self, container, cmd, quotecmd): + """Format a command to run inside a container using the runtime + + :param container: The name or ID of the container in which to run + :type container: ``str`` + + :param cmd: The command to run inside `container` + :type cmd: ``str`` + + :param quotecmd: Whether the cmd should be quoted. + :type quotecmd: ``bool`` + + :returns: Formatted string to run `cmd` inside `container` + :rtype: ``str`` + """ + if quotecmd: + quoted_cmd = quote(cmd) + else: + quoted_cmd = cmd + container_id = self.get_container_by_name(container) + return "%s %s %s" % (self.run_cmd, container_id, + quoted_cmd) if container_id is not None else '' + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/runtimes/docker.py 4.5.3ubuntu2/sos/policies/runtimes/docker.py --- 4.0-2/sos/policies/runtimes/docker.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/runtimes/docker.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,33 @@ +# Copyright (C) 2020 Red Hat, Inc., Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.policies.runtimes import ContainerRuntime +from sos.utilities import is_executable + + +class DockerContainerRuntime(ContainerRuntime): + """Runtime class to use for systems running Docker""" + + name = 'docker' + binary = 'docker' + + def check_is_active(self, sysroot=None): + # the daemon must be running + if (is_executable('docker', sysroot) and + (self.policy.init_system.is_running('docker') or + self.policy.init_system.is_running('snap.docker.dockerd'))): + self.active = True + return True + return False + + def check_can_copy(self): + return self.check_is_active(sysroot=self.policy.sysroot) + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/runtimes/podman.py 4.5.3ubuntu2/sos/policies/runtimes/podman.py --- 4.0-2/sos/policies/runtimes/podman.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/runtimes/podman.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,21 @@ +# Copyright (C) 2020 Red Hat, Inc., Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.policies.runtimes import ContainerRuntime + + +class PodmanContainerRuntime(ContainerRuntime): + """Runtime class to use for systems running Podman""" + + name = 'podman' + binary = 'podman' + + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/policies/suse.py 4.5.3ubuntu2/sos/policies/suse.py --- 4.0-2/sos/policies/suse.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/suse.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,113 +0,0 @@ -# Copyright (C) 2015 Red Hat, Inc. Bryn M. Reeves -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information. - -import os -import sys - -from sos.report.plugins import RedHatPlugin, SuSEPlugin -from sos.policies import LinuxPolicy, PackageManager -from sos import _sos as _ - - -class SuSEPolicy(LinuxPolicy): - distro = "SuSE" - vendor = "SuSE" - vendor_url = "https://www.suse.com/" - _tmp_dir = "/var/tmp" - - def __init__(self, sysroot=None, init=None, probe_runtime=True, - remote_exec=None): - super(SuSEPolicy, self).__init__(sysroot=sysroot, init=init, - probe_runtime=probe_runtime) - self.ticket_number = "" - self.package_manager = PackageManager( - 'rpm -qa --queryformat "%{NAME}|%{VERSION}\\n"', - remote_exec=remote_exec) - self.valid_subclasses = [SuSEPlugin, RedHatPlugin] - - pkgs = self.package_manager.all_pkgs() - - # If rpm query timed out after timeout duration exit - if not pkgs: - print("Could not obtain installed package list", file=sys.stderr) - sys.exit(1) - - self.PATH = "/usr/sbin:/usr/bin:/root/bin" - self.PATH += os.pathsep + "/usr/local/bin" - self.PATH += os.pathsep + "/usr/local/sbin" - self.set_exec_path() - - @classmethod - def check(cls, remote=''): - """This method checks to see if we are running on SuSE. It must be - overriden by concrete subclasses to return True when running on an - OpenSuSE, SLES or other Suse distribution and False otherwise.""" - return False - - def runlevel_by_service(self, name): - from subprocess import Popen, PIPE - ret = [] - p = Popen("LC_ALL=C /sbin/chkconfig --list %s" % name, - shell=True, - stdout=PIPE, - stderr=PIPE, - bufsize=-1, - close_fds=True) - out, err = p.communicate() - if err: - return ret - for tabs in out.split()[1:]: - try: - (runlevel, onoff) = tabs.split(":", 1) - except IndexError: - pass - else: - if onoff == "on": - ret.append(int(runlevel)) - return ret - - def get_tmp_dir(self, opt_tmp_dir): - if not opt_tmp_dir: - return self._tmp_dir - return opt_tmp_dir - - def get_local_name(self): - return self.host_name() - - -class OpenSuSEPolicy(SuSEPolicy): - distro = "OpenSuSE" - vendor = "SuSE" - vendor_url = "https://www.opensuse.org/" - msg = _("""\ -This command will collect diagnostic and configuration \ -information from this %(distro)s system and installed \ -applications. - -An archive containing the collected information will be \ -generated in %(tmpdir)s and may be provided to a %(vendor)s \ -support representative. - -No changes will be made to system configuration. -%(vendor_text)s -""") - - def __init__(self, sysroot=None, init=None, probe_runtime=True): - super(OpenSuSEPolicy, self).__init__(sysroot=sysroot, init=init, - probe_runtime=probe_runtime) - - @classmethod - def check(cls, remote): - """This method checks to see if we are running on SuSE. - """ - - if remote: - return cls.distro in remote - - return (os.path.isfile('/etc/SuSE-release')) diff -pruN 4.0-2/sos/policies/ubuntu.py 4.5.3ubuntu2/sos/policies/ubuntu.py --- 4.0-2/sos/policies/ubuntu.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/policies/ubuntu.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,70 +0,0 @@ -from sos.report.plugins import UbuntuPlugin, DebianPlugin -from sos.policies.debian import DebianPolicy - -import os - - -class UbuntuPolicy(DebianPolicy): - distro = "Ubuntu" - vendor = "Canonical" - vendor_url = "https://www.ubuntu.com/" - PATH = "/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" \ - + ":/usr/local/sbin:/usr/local/bin:/snap/bin" - _upload_url = "https://files.support.canonical.com/uploads/" - _upload_user = "ubuntu" - _upload_password = "ubuntu" - _use_https_streaming = True - - def __init__(self, sysroot=None, init=None, probe_runtime=True, - remote_exec=None): - super(UbuntuPolicy, self).__init__(sysroot=sysroot, init=init, - probe_runtime=probe_runtime, - remote_exec=remote_exec) - self.valid_subclasses = [UbuntuPlugin, DebianPlugin] - - @classmethod - def check(cls, remote=''): - """This method checks to see if we are running on Ubuntu. - It returns True or False.""" - - if remote: - return cls.distro in remote - - try: - with open('/etc/lsb-release', 'r') as fp: - return "Ubuntu" in fp.read() - except IOError: - return False - - def dist_version(self): - """ Returns the version stated in DISTRIB_RELEASE - """ - try: - with open('/etc/lsb-release', 'r') as fp: - lines = fp.readlines() - for line in lines: - if "DISTRIB_RELEASE" in line: - return line.split("=")[1].strip() - return False - except IOError: - return False - - def get_upload_https_auth(self): - if self.upload_url.startswith(self._upload_url): - return (self._upload_user, self._upload_password) - else: - return super(UbuntuPolicy, self).get_upload_https_auth() - - def get_upload_url_string(self): - if self.upload_url.startswith(self._upload_url): - return "Canonical Support File Server" - else: - return self.get_upload_url() - - def get_upload_url(self): - if not self.upload_url or self.upload_url.startswith(self._upload_url): - fname = os.path.basename(self.upload_archive) - return self._upload_url + fname - super(UbuntuPolicy, self).get_upload_url() - -# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/presets/__init__.py 4.5.3ubuntu2/sos/presets/__init__.py --- 4.0-2/sos/presets/__init__.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/presets/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,129 @@ +# Copyright (C) 2020 Red Hat, Inc., Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import json +import os + +from sos.options import SoSOptions + +PRESETS_PATH = "/etc/sos/presets.d" + +#: Constants for on-disk preset fields +DESC = "desc" +NOTE = "note" +OPTS = "args" + + +class PresetDefaults(): + """Preset command line defaults to allow for quick reference to sets of + commonly used options + + :param name: The name of the new preset + :type name: ``str`` + + :param desc: A description for the new preset + :type desc: ``str`` + + :param note: Note for the new preset + :type note: ``str`` + + :param opts: Options set for the new preset + :type opts: ``SoSOptions`` + """ + #: Preset name, used for selection + name = None + #: Human readable preset description + desc = None + #: Notes on preset behaviour + note = None + #: Options set for this preset + opts = SoSOptions() + + #: ``True`` if this preset if built-in or ``False`` otherwise. + builtin = True + + def __str__(self): + """Return a human readable string representation of this + ``PresetDefaults`` object. + """ + return ("name=%s desc=%s note=%s opts=(%s)" % + (self.name, self.desc, self.note, str(self.opts))) + + def __repr__(self): + """Return a machine readable string representation of this + ``PresetDefaults`` object. + """ + return ("PresetDefaults(name='%s' desc='%s' note='%s' opts=(%s)" % + (self.name, self.desc, self.note, repr(self.opts))) + + def __init__(self, name="", desc="", note=None, opts=SoSOptions()): + """Initialise a new ``PresetDefaults`` object with the specified + arguments. + + :returns: The newly initialised ``PresetDefaults`` + """ + self.name = name + self.desc = desc + self.note = note + self.opts = opts + + def write(self, presets_path): + """Write this preset to disk in JSON notation. + + :param presets_path: the directory where the preset will be written + :type presets_path: ``str`` + """ + if self.builtin: + raise TypeError("Cannot write built-in preset") + + # Make dictionaries of PresetDefaults values + odict = self.opts.dict() + pdict = {self.name: {DESC: self.desc, NOTE: self.note, OPTS: odict}} + + if not os.path.exists(presets_path): + os.makedirs(presets_path, mode=0o755) + + with open(os.path.join(presets_path, self.name), "w") as pfile: + json.dump(pdict, pfile) + + def delete(self, presets_path): + """Delete a preset from disk + + :param presets_path: the directory where the preset is saved + :type presets_path: ``str`` + """ + os.unlink(os.path.join(presets_path, self.name)) + + +NO_PRESET = 'none' +NO_PRESET_DESC = 'Do not load a preset' +NO_PRESET_NOTE = 'Use to disable automatically loaded presets' + +SMALL_PRESET = 'minimal' +SMALL_PRESET_DESC = ('Small and quick report that reduces sos report resource ' + 'consumption') +SMALL_PRESET_NOTE = ('May be useful for low-resource systems, but may not ' + 'provide sufficient data for analysis') + +SMALL_PRESET_OPTS = SoSOptions(log_size=10, journal_size=10, plugin_timeout=30, + command_timeout=30, low_priority=True) + +GENERIC_PRESETS = { + NO_PRESET: PresetDefaults( + name=NO_PRESET, desc=NO_PRESET_DESC, note=NO_PRESET_NOTE, + opts=SoSOptions() + ), + SMALL_PRESET: PresetDefaults( + name=SMALL_PRESET, desc=SMALL_PRESET_DESC, note=SMALL_PRESET_NOTE, + opts=SMALL_PRESET_OPTS + ) +} + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/presets/redhat/__init__.py 4.5.3ubuntu2/sos/presets/redhat/__init__.py --- 4.0-2/sos/presets/redhat/__init__.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/presets/redhat/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,93 @@ +# Copyright (C) 2020 Red Hat, Inc., Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.options import SoSOptions +from sos.presets import PresetDefaults + + +RHEL_RELEASE_STR = "Red Hat Enterprise Linux" + +_opts_verify = SoSOptions(verify=True) +_opts_all_logs = SoSOptions(all_logs=True) +_opts_all_logs_verify = SoSOptions(all_logs=True, verify=True) +_cb_profiles = ['boot', 'storage', 'system'] +_cb_plugopts = ['boot.all-images=on', 'rpm.rpmva=on', 'rpm.rpmdb=on'] + + +RHV = "rhv" +RHV_DESC = "Red Hat Virtualization" + +RHEL = "rhel" +RHEL_DESC = RHEL_RELEASE_STR + +RHOSP = "rhosp" +RHOSP_DESC = "Red Hat OpenStack Platform" +RHOSP_OPTS = SoSOptions(plugopts=[ + 'process.lsof=off', + 'networking.ethtool_namespaces=False', + 'networking.namespaces=200']) + +RHOCP = "ocp" +RHOCP_DESC = "OpenShift Container Platform by Red Hat" +RHOCP_OPTS = SoSOptions( + skip_plugins=['cgroups'], container_runtime='crio', no_report=True, + log_size=100, + plugopts=[ + 'crio.timeout=600', + 'networking.timeout=600', + 'networking.ethtool_namespaces=False', + 'networking.namespaces=200' + ]) + +RH_CFME = "cfme" +RH_CFME_DESC = "Red Hat CloudForms" + +RH_SATELLITE = "satellite" +RH_SATELLITE_DESC = "Red Hat Satellite" +SAT_OPTS = SoSOptions(log_size=100, plugopts=['apache.log=on']) + +CB = "cantboot" +CB_DESC = "For use when normal system startup fails" +CB_OPTS = SoSOptions( + verify=True, all_logs=True, profiles=_cb_profiles, + plugopts=_cb_plugopts + ) +CB_NOTE = ("Data collection will be limited to a boot-affecting scope") + +NOTE_SIZE = "This preset may increase report size" +NOTE_TIME = "This preset may increase report run time" +NOTE_SIZE_TIME = "This preset may increase report size and run time" + +RHEL_PRESETS = { + RHV: PresetDefaults(name=RHV, desc=RHV_DESC, note=NOTE_TIME, + opts=_opts_verify), + RHEL: PresetDefaults(name=RHEL, desc=RHEL_DESC), + RHOSP: PresetDefaults(name=RHOSP, desc=RHOSP_DESC, opts=RHOSP_OPTS), + RHOCP: PresetDefaults(name=RHOCP, desc=RHOCP_DESC, note=NOTE_SIZE_TIME, + opts=RHOCP_OPTS), + RH_CFME: PresetDefaults(name=RH_CFME, desc=RH_CFME_DESC, note=NOTE_TIME, + opts=_opts_verify), + RH_SATELLITE: PresetDefaults(name=RH_SATELLITE, desc=RH_SATELLITE_DESC, + note=NOTE_TIME, opts=SAT_OPTS), + CB: PresetDefaults(name=CB, desc=CB_DESC, note=CB_NOTE, opts=CB_OPTS) +} + + +ATOMIC = "atomic" +ATOMIC_RELEASE_STR = "Atomic" +ATOMIC_DESC = "Red Hat Enterprise Linux Atomic Host" + +ATOMIC_PRESETS = { + ATOMIC: PresetDefaults(name=ATOMIC, desc=ATOMIC_DESC, note=NOTE_TIME, + opts=_opts_verify) +} + + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/__init__.py 4.5.3ubuntu2/sos/report/__init__.py --- 4.0-2/sos/report/__init__.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -17,7 +17,9 @@ import logging from datetime import datetime import glob import sos.report.plugins -from sos.utilities import ImporterHelper, SoSTimeoutError +from sos.utilities import (ImporterHelper, SoSTimeoutError, bold, + sos_get_command_output, TIMEOUT_DEFAULT, listdir, + is_executable) from shutil import rmtree import hashlib from concurrent.futures import ThreadPoolExecutor, TimeoutError @@ -81,37 +83,52 @@ class SoSReport(SoSComponent): 'case_id': '', 'chroot': 'auto', 'clean': False, + 'container_runtime': 'auto', + 'keep_binary_files': False, 'desc': '', 'domains': [], + 'disable_parsers': [], 'dry_run': False, + 'estimate_only': False, 'experimental': False, 'enable_plugins': [], + 'journal_size': 100, 'keywords': [], + 'keyword_file': None, 'plugopts': [], 'label': '', 'list_plugins': False, 'list_presets': False, 'list_profiles': False, 'log_size': 25, + 'low_priority': False, 'map_file': '/etc/sos/cleaner/default_mapping', + 'skip_commands': [], + 'skip_files': [], 'skip_plugins': [], - 'noreport': False, + 'namespaces': None, + 'no_report': False, 'no_env_vars': False, 'no_postproc': False, 'no_update': False, 'note': '', 'only_plugins': [], 'preset': 'auto', - 'plugin_timeout': 300, + 'plugin_timeout': TIMEOUT_DEFAULT, + 'cmd_timeout': TIMEOUT_DEFAULT, 'profiles': [], 'since': None, 'verify': False, 'allow_system_changes': False, + 'usernames': [], 'upload': False, 'upload_url': None, 'upload_directory': None, 'upload_user': None, 'upload_pass': None, + 'upload_method': 'auto', + 'upload_no_ssl_verify': False, + 'upload_protocol': 'auto', 'add_preset': '', 'del_preset': '' } @@ -126,6 +143,7 @@ class SoSReport(SoSComponent): self._args = args self.sysroot = "/" self.preset = None + self.estimated_plugsizes = {} self.print_header() self._set_debug() @@ -150,18 +168,19 @@ class SoSReport(SoSComponent): self.opts.merge(self.preset.opts) # re-apply any cmdline overrides to the preset self.opts = self.apply_options_from_cmdline(self.opts) + if hasattr(self.preset.opts, 'verbosity') and \ + self.preset.opts.verbosity > 0: + self.set_loggers_verbosity(self.preset.opts.verbosity) self._set_directories() msg = "default" - host_sysroot = self.policy.host_sysroot() + self.sysroot = self.policy.sysroot # set alternate system root directory if self.opts.sysroot: msg = "cmdline" - self.sysroot = self.opts.sysroot - elif self.policy.in_container() and host_sysroot != os.sep: + elif self.policy.in_container() and self.sysroot != os.sep: msg = "policy" - self.sysroot = host_sysroot self.soslog.debug("set sysroot to '%s' (%s)" % (self.sysroot, msg)) if self.opts.chroot not in chroot_modes: @@ -170,6 +189,8 @@ class SoSReport(SoSComponent): self.tempfile_util.clean() self._exit(1) + self._check_container_runtime() + self._get_namespaces() self._get_hardware_devices() @classmethod @@ -200,18 +221,31 @@ class SoSReport(SoSComponent): dest="chroot", default='auto', help="chroot executed commands to SYSROOT " "[auto, always, never] (default=auto)") + report_grp.add_argument("--container-runtime", default="auto", + help="Default container runtime to use for " + "collections. 'auto' for policy control.") report_grp.add_argument("--desc", "--description", type=str, action="store", default="", help="Description for a new preset",) report_grp.add_argument("--dry-run", action="store_true", help="Run plugins but do not collect data") + report_grp.add_argument("--estimate-only", action="store_true", + help="Approximate disk space requirements for " + "a real sos run; disables --clean and " + "--collect, sets --threads=1 and " + "--no-postproc") report_grp.add_argument("--experimental", action="store_true", dest="experimental", default=False, help="enable experimental plugins") report_grp.add_argument("-e", "--enable-plugins", action="extend", dest="enable_plugins", type=str, help="enable these plugins", default=[]) - report_grp.add_argument("-k", "--plugin-option", action="extend", + report_grp.add_argument("--journal-size", type=int, default=100, + dest="journal_size", + help="limit the size of collected journals " + "in MiB") + report_grp.add_argument("-k", "--plugin-option", "--plugopts", + action="extend", dest="plugopts", type=str, help="plugin options in plugname.option=value " "format (see -l)", default=[]) @@ -231,12 +265,19 @@ class SoSReport(SoSComponent): report_grp.add_argument("--log-size", action="store", dest="log_size", type=int, default=25, help="limit the size of collected logs " - "(in MiB)") + "(not journals) in MiB") + report_grp.add_argument("--low-priority", action="store_true", + default=False, + help="generate report with low system priority" + ) + report_grp.add_argument("--namespaces", default=None, + help="limit number of namespaces to collect " + "output for - 0 means unlimited") report_grp.add_argument("-n", "--skip-plugins", action="extend", dest="skip_plugins", type=str, help="disable these plugins", default=[]) report_grp.add_argument("--no-report", action="store_true", - dest="noreport", default=False, + dest="no_report", default=False, help="disable plaintext/HTML reporting") report_grp.add_argument("--no-env-vars", action="store_true", dest="no_env_vars", default=False, @@ -253,10 +294,19 @@ class SoSReport(SoSComponent): help="A preset identifier", default="auto") report_grp.add_argument("--plugin-timeout", default=None, help="set a timeout for all plugins") - report_grp.add_argument("-p", "--profile", action="extend", - dest="profiles", type=str, default=[], + report_grp.add_argument("--cmd-timeout", default=None, + help="set a command timeout for all plugins") + report_grp.add_argument("-p", "--profile", "--profiles", + action="extend", dest="profiles", type=str, + default=[], help="enable plugins used by the given " "profiles") + report_grp.add_argument('--skip-commands', default=[], action='extend', + dest='skip_commands', + help="do not execute these commands") + report_grp.add_argument('--skip-files', default=[], action='extend', + dest='skip_files', + help="do not collect these files") report_grp.add_argument("--verify", action="store_true", dest="verify", default=False, help="perform data verification during " @@ -276,6 +326,15 @@ class SoSReport(SoSComponent): help="Username to authenticate to server with") report_grp.add_argument("--upload-pass", default=None, help="Password to authenticate to server with") + report_grp.add_argument("--upload-method", default='auto', + choices=['auto', 'put', 'post'], + help="HTTP method to use for uploading") + report_grp.add_argument("--upload-no-ssl-verify", default=False, + action='store_true', + help="Disable SSL verification for upload url") + report_grp.add_argument("--upload-protocol", default='auto', + choices=['auto', 'https', 'ftp', 'sftp'], + help="Manually specify the upload protocol") # Group to make add/del preset exclusive preset_grp = report_grp.add_mutually_exclusive_group() @@ -289,38 +348,144 @@ class SoSReport(SoSComponent): 'Cleaner/Masking Options', 'These options control how data obfuscation is performed' ) - cleaner_grp.add_argument('--clean', '--mask', dest='clean', + cleaner_grp.add_argument('--clean', '--cleaner', '--mask', + dest='clean', default=False, action='store_true', - help='Obfuscate sensistive information') + help='Obfuscate sensitive information') cleaner_grp.add_argument('--domains', dest='domains', default=[], action='extend', help='Additional domain names to obfuscate') + cleaner_grp.add_argument('--disable-parsers', action='extend', + default=[], dest='disable_parsers', + help=('Disable specific parsers, so that ' + 'those elements are not obfuscated')) cleaner_grp.add_argument('--keywords', action='extend', default=[], dest='keywords', help='List of keywords to obfuscate') + cleaner_grp.add_argument('--keyword-file', default=None, + dest='keyword_file', + help='Provide a file a keywords to obfuscate') cleaner_grp.add_argument('--no-update', action='store_true', default=False, dest='no_update', help='Do not update the default cleaner map') - cleaner_grp.add_argument('--map', dest='map_file', + cleaner_grp.add_argument('--map-file', dest='map_file', default='/etc/sos/cleaner/default_mapping', help=('Provide a previously generated mapping' ' file for obfuscation')) + cleaner_grp.add_argument('--keep-binary-files', default=False, + action='store_true', + dest='keep_binary_files', + help='Keep unprocessable binary files in the ' + 'archive instead of removing them') + cleaner_grp.add_argument('--usernames', dest='usernames', default=[], + action='extend', + help='List of usernames to obfuscate') + + @classmethod + def display_help(cls, section): + section.set_title('SoS Report Detailed Help') + section.add_text( + 'The report command is the most common use case for SoS, and aims ' + 'to collect relevant diagnostic and troubleshooting data to assist' + ' with issue analysis without actively performing that analysis on' + ' the system while it is in use.' + ) + section.add_text( + 'Additionally, sos report archives can be used for ongoing ' + 'inspection for pre-emptive issue monitoring, such as that done ' + 'by the Insights project.' + ) + + section.add_text( + 'The typical result of an execution of \'sos report\' is a tarball' + ' that contains troubleshooting command output, copies of config ' + 'files, and copies of relevant sections of the host filesystem. ' + 'Root privileges are required for collections.' + ) + + psec = section.add_section(title='How Collections Are Determined') + psec.add_text( + 'SoS report performs it\'s collections by way of \'plugins\' that ' + 'individually specify what files to copy and what commands to run.' + ' Plugins typically map to specific components or software ' + 'packages.' + ) + psec.add_text( + 'Plugins may specify different collections on different distribu' + 'tions, and some plugins may only be for specific distributions. ' + 'Distributions are represented within SoS by \'policies\' and may ' + 'influence how other SoS commands or options function. For example' + 'policies can alter where the --upload option defaults to or ' + 'functions.' + ) + + ssec = section.add_section(title='See Also') + ssec.add_text( + "For information on available options for report, see %s and %s" + % (bold('sos report --help'), bold('man sos-report')) + ) + ssec.add_text("The following %s sections may be of interest:\n" + % bold('sos help')) + help_lines = { + 'report.plugins': 'Information on the plugin design of sos', + 'report.plugins.$plugin': 'Information on a specific $plugin', + 'policies': 'How sos operates on different distributions' + } + helpln = '' + for ln in help_lines: + ssec.add_text("\t{:<36}{}".format(ln, help_lines[ln]), + newline=False) + ssec.add_text(helpln) def print_header(self): print("\n%s\n" % _("sosreport (version %s)" % (__version__,))) def _get_hardware_devices(self): self.devices = { - 'block': self.get_block_devs(), - 'fibre': self.get_fibre_devs() + 'storage': { + 'block': self._get_block_devs(), + 'fibre': self._get_fibre_devs() + }, + 'network': self._get_network_devs(), + 'namespaced_network': self._get_network_namespace_devices() } - # TODO: enumerate network devices, preferably with devtype info - def get_fibre_devs(self): + def _check_container_runtime(self): + """Check the loaded container runtimes, and the policy default runtime + (if set), against any requested --container-runtime value. This can be + useful for systems that have multiple runtimes, such as RHCOS, but do + not have a clearly defined 'default' (or one that is determined based + entirely on configuration). + """ + if self.opts.container_runtime != 'auto': + crun = self.opts.container_runtime.lower() + if crun in ['none', 'off', 'diabled']: + self.policy.runtimes = {} + self.soslog.info( + "Disabled all container runtimes per user option." + ) + elif not self.policy.runtimes: + msg = ("WARNING: No container runtimes are active, ignoring " + "option to set default runtime to '%s'\n" % crun) + self.soslog.warning(msg) + elif crun not in self.policy.runtimes.keys(): + valid = ', '.join(p for p in self.policy.runtimes.keys() + if p != 'default') + raise Exception("Cannot use container runtime '%s': no such " + "runtime detected. Available runtimes: %s" + % (crun, valid)) + else: + self.policy.runtimes['default'] = self.policy.runtimes[crun] + self.soslog.info( + "Set default container runtime to '%s'" + % self.policy.runtimes['default'].name + ) + + def _get_fibre_devs(self): """Enumerate a list of fibrechannel devices on this system so that plugins can iterate over them - These devices are used by add_fibredev_cmd() in the Plugin class. + These devices are used by add_device_cmd() in the Plugin class. """ try: devs = [] @@ -338,18 +503,177 @@ class SoSReport(SoSComponent): self.soslog.error("Could not get fibre device list: %s" % err) return [] - def get_block_devs(self): + def _get_block_devs(self): """Enumerate a list of block devices on this system so that plugins can iterate over them - These devices are used by add_blockdev_cmd() in the Plugin class. + These devices are used by add_device_cmd() in the Plugin class. """ try: - return os.listdir('/sys/block/') + device_list = ["/dev/%s" % d for d in os.listdir('/sys/block')] + loop_devices = sos_get_command_output('losetup --all --noheadings') + real_loop_devices = [] + if loop_devices['status'] == 0: + for loop_dev in loop_devices['output'].splitlines(): + loop_device = loop_dev.split()[0].replace(':', '') + real_loop_devices.append(loop_device) + ghost_loop_devs = [dev for dev in device_list + if dev.startswith("loop") + if dev not in real_loop_devices] + dev_list = list(set(device_list) - set(ghost_loop_devs)) + return dev_list except Exception as err: self.soslog.error("Could not get block device list: %s" % err) return [] + def _get_namespaces(self): + self.namespaces = { + 'network': self._get_network_namespaces() + } + + def _get_network_devs(self): + """Helper to encapsulate network devices probed by sos. Rather than + probing lists of distinct device types like we do for storage, we can + do some introspection of device enumeration where a single interface + may have multiple device types. E.G an 'ethernet' device could also be + a bond, and that is pertinent information for device iteration. + + :returns: A collection of enumerated devices sorted by device type + :rtype: ``dict`` with keys being device types + """ + _devs = { + 'ethernet': [], + 'bridge': [], + 'team': [], + 'bond': [] + } + try: + if is_executable('nmcli', sysroot=self.opts.sysroot): + _devs.update(self._get_nmcli_devs()) + # if nmcli failed for some reason, fallback + if not _devs['ethernet']: + self.soslog.debug( + 'Network devices not enumerated by nmcli. Will attempt to ' + 'manually compile list of devices.' + ) + _devs.update(self._get_eth_devs()) + _devs['bridge'] = self._get_bridge_devs() + except Exception as err: + self.soslog.warning(f"Could not enumerate network devices: {err}") + return _devs + + def _get_network_namespace_devices(self): + """Enumerate the network devices that exist within each network + namespace that exists on the system + """ + _nmdevs = {} + for nmsp in self.namespaces['network']: + _nmdevs[nmsp] = { + 'ethernet': self._get_eth_devs(nmsp) + } + return _nmdevs + + def _get_nmcli_devs(self): + """Use nmcli, if available, to enumerate network devices. From this + output, manually grok together lists of devices. + """ + _devs = {} + try: + _ndevs = sos_get_command_output('nmcli --fields DEVICE,TYPE dev') + if _ndevs['status'] == 0: + for dev in _ndevs['output'].splitlines()[1:]: + dname, dtype = dev.split() + if dtype not in _devs: + _devs[dtype] = [dname] + else: + _devs[dtype].append(dname) + _devs['ethernet'].append(dname) + _devs['ethernet'] = list(set(_devs['ethernet'])) + except Exception as err: + self.soslog.debug("Could not parse nmcli devices: %s" % err) + return _devs + + def _get_eth_devs(self, namespace=None): + """Enumerate a list of ethernet network devices so that plugins can + reliably iterate over the same set of devices without doing piecemeal + discovery. + + These devices are used by `add_device_cmd()` when `devices` includes + "ethernet" or "network". + + :param namespace: Inspect this existing network namespace, if + provided + :type namespace: ``str`` + + :returns: All valid ethernet devices found, potentially within a + given namespace + :rtype: ``list`` + """ + filt_devs = ['bonding_masters'] + _eth_devs = [] + if not namespace: + _eth_devs = [ + dev for dev in listdir('/sys/class/net', self.opts.sysroot) + if dev not in filt_devs + ] + else: + try: + _nscmd = "ip netns exec %s ls /sys/class/net" % namespace + _nsout = sos_get_command_output(_nscmd) + if _nsout['status'] == 0: + for _nseth in _nsout['output'].split(): + if _nseth not in filt_devs: + _eth_devs.append(_nseth) + except Exception as err: + self.soslog.warning( + "Could not determine network namespace '%s' devices: %s" + % (namespace, err) + ) + return { + 'ethernet': _eth_devs, + 'bond': [bd for bd in _eth_devs if bd.startswith('bond')], + 'tun': [td for td in _eth_devs if td.startswith('tun')] + } + + def _get_bridge_devs(self): + """Enumerate a list of bridge devices so that plugins can reliably + iterate over the same set of bridges. + + These devices are used by `add_device_cmd()` when `devices` includes + "bridge" or "network". + """ + _bridges = [] + try: + _bout = sos_get_command_output('brctl show', timeout=15) + except Exception as err: + self.soslog.warning("Unable to enumerate bridge devices: %s" % err) + if _bout['status'] == 0: + for _bline in _bout['output'].splitlines()[1:]: + try: + _bridges.append(_bline.split()[0]) + except Exception as err: + self.soslog.info( + "Could not parse device from line '%s': %s" + % (_bline, err) + ) + return _bridges + + def _get_network_namespaces(self): + """Enumerate a list of network namespaces on this system so that + plugins can iterate over them + + Note that stderr is not collected, so no handling of error lines. + """ + out_ns = [] + + ip_netns = sos_get_command_output("ip netns") + if ip_netns['status'] == 0: + for line in ip_netns['output'].splitlines(): + if line.isspace() or line[:1].isspace(): + continue + out_ns.append(line.partition(' ')[0]) + return out_ns + def get_commons(self): return { 'cmddir': self.cmddir, @@ -361,7 +685,8 @@ class SoSReport(SoSComponent): 'sysroot': self.sysroot, 'verbosity': self.opts.verbosity, 'cmdlineopts': self.opts, - 'devices': self.devices + 'devices': self.devices, + 'namespaces': self.namespaces } def get_temp_file(self): @@ -545,21 +870,24 @@ class SoSReport(SoSComponent): def _set_all_options(self): if self.opts.alloptions: for plugname, plug in self.loaded_plugins: - for name, parms in zip(plug.opt_names, plug.opt_parms): - if type(parms["enabled"]) == bool: - parms["enabled"] = True + for opt in plug.options.values(): + if bool in opt.val_type: + opt.value = True def _set_tunables(self): if self.opts.plugopts: opts = {} for opt in self.opts.plugopts: - # split up "general.syslogsize=5" try: opt, val = opt.split("=") except ValueError: val = True - else: - if val.lower() in ["off", "disable", "disabled", "false"]: + + if isinstance(val, str): + val = val.lower() + if val in ["on", "enable", "enabled", "true", "yes"]: + val = True + elif val in ["off", "disable", "disabled", "false", "no"]: val = False else: # try to convert string "val" to int() @@ -568,7 +896,6 @@ class SoSReport(SoSComponent): except ValueError: pass - # split up "general.syslogsize" try: plug, opt = opt.split(".") except ValueError: @@ -578,20 +905,29 @@ class SoSReport(SoSComponent): try: opts[plug] except KeyError: - opts[plug] = [] - opts[plug].append((opt, val)) + opts[plug] = {} + opts[plug][opt] = val for plugname, plug in self.loaded_plugins: if plugname in opts: - for opt, val in opts[plugname]: - if not plug.set_option(opt, val): + for opt in opts[plugname]: + if opt not in plug.options: self.soslog.error('no such option "%s" for plugin ' '(%s)' % (opt, plugname)) self._exit(1) + try: + plug.options[opt].set_value(opts[plugname][opt]) + self.soslog.debug( + "Set %s plugin option to %s" + % (plugname, plug.options[opt]) + ) + except Exception as err: + self.soslog.error(err) + self._exit(1) del opts[plugname] for plugname in opts.keys(): self.soslog.error('WARNING: unable to set option for disabled ' - 'or non-existing plugin (%s)' % (plugname)) + 'or non-existing plugin (%s).' % (plugname)) # in case we printed warnings above, visually intend them from # subsequent header text if opts.keys(): @@ -600,20 +936,49 @@ class SoSReport(SoSComponent): def _check_for_unknown_plugins(self): import itertools for plugin in itertools.chain(self.opts.only_plugins, - self.opts.skip_plugins, self.opts.enable_plugins): plugin_name = plugin.split(".")[0] if plugin_name not in self.plugin_names: self.soslog.fatal('a non-existing plugin (%s) was specified ' - 'in the command line' % (plugin_name)) + 'in the command line.' % (plugin_name)) self._exit(1) + for plugin in self.opts.skip_plugins: + if plugin not in self.plugin_names: + self.soslog.warning( + "Requested to skip non-existing plugin '%s'." % plugin + ) def _set_plugin_options(self): for plugin_name, plugin in self.loaded_plugins: - names, parms = plugin.get_all_options() - for optname, optparm in zip(names, parms): - self.all_options.append((plugin, plugin_name, optname, - optparm)) + for opt in plugin.options: + self.all_options.append(plugin.options[opt]) + + def _set_estimate_only(self): + # set estimate-only mode by enforcing some options settings + # and return a corresponding log messages string + msg = "\nEstimate-only mode enabled" + ext_msg = [] + if self.opts.threads > 1: + ext_msg += ["--threads=%s overriden to 1" % self.opts.threads, ] + self.opts.threads = 1 + if not self.opts.build: + ext_msg += ["--build enabled", ] + self.opts.build = True + if not self.opts.no_postproc: + ext_msg += ["--no-postproc enabled", ] + self.opts.no_postproc = True + if self.opts.clean: + ext_msg += ["--clean disabled", ] + self.opts.clean = False + if self.opts.upload: + ext_msg += ["--upload* options disabled", ] + self.opts.upload = False + if ext_msg: + msg += ", which overrides some options:\n " + "\n ".join(ext_msg) + else: + msg += "." + msg += "\n\n" + return msg def _report_profiles_and_plugins(self): self.ui_log.info("") @@ -654,25 +1019,41 @@ class SoSReport(SoSComponent): if self.all_options: self.ui_log.info(_("The following options are available for ALL " "plugins:")) - for opt in self.all_options[0][0]._default_plug_opts: - self.ui_log.info(" %-25s %-15s %s" % (opt[0], opt[3], opt[1])) + _defaults = self.loaded_plugins[0][1].get_default_plugin_opts() + for _opt in _defaults: + opt = _defaults[_opt] + val = opt.value + if opt.value == -1: + if _opt == 'timeout': + val = self.opts.plugin_timeout or TIMEOUT_DEFAULT + elif _opt == 'cmd-timeout': + val = self.opts.cmd_timeout or TIMEOUT_DEFAULT + else: + val = TIMEOUT_DEFAULT + if opt.name == 'postproc': + val = not self.opts.no_postproc + self.ui_log.info(" %-25s %-15s %s" % (opt.name, val, opt.desc)) self.ui_log.info("") self.ui_log.info(_("The following plugin options are available:")) - for (plug, plugname, optname, optparm) in self.all_options: - if optname in ('timeout', 'postproc'): - continue + for opt in self.all_options: + if opt.name in ('timeout', 'postproc', 'cmd-timeout'): + if opt.value == opt.default: + continue # format option value based on its type (int or bool) - if type(optparm["enabled"]) == bool: - if optparm["enabled"] is True: + if isinstance(opt.value, bool): + if opt.value is True: tmpopt = "on" else: tmpopt = "off" else: - tmpopt = optparm["enabled"] + tmpopt = opt.value + + if tmpopt is None: + tmpopt = 0 self.ui_log.info(" %-25s %-15s %s" % ( - plugname + "." + optname, tmpopt, optparm["desc"])) + opt.plugin + "." + opt.name, tmpopt, opt.desc)) else: self.ui_log.info(_("No plugin options available.")) @@ -780,10 +1161,12 @@ class SoSReport(SoSComponent): return True def batch(self): + msg = self.policy.get_msg() + if self.opts.estimate_only: + msg += self._set_estimate_only() if self.opts.batch: - self.ui_log.info(self.policy.get_msg()) + self.ui_log.info(msg) else: - msg = self.policy.get_msg() msg += _("Press ENTER to continue, or CTRL-C to quit.\n") try: input(msg) @@ -791,9 +1174,7 @@ class SoSReport(SoSComponent): self.ui_log.error("Exiting on user cancel") self._exit(130) except Exception as e: - self.ui_log.info("") - self.ui_log.error(e) - self._exit(e) + self._exit(1, e) def _log_plugin_exception(self, plugin, method): trace = traceback.format_exc() @@ -808,15 +1189,6 @@ class SoSReport(SoSComponent): self.policy.pre_work() try: self.ui_log.info(_(" Setting up archive ...")) - compression_methods = ('auto', 'bzip2', 'gzip', 'xz') - method = self.opts.compression_type - if method not in compression_methods: - compression_list = ', '.join(compression_methods) - self.ui_log.error("") - self.ui_log.error("Invalid compression specified: " + method) - self.ui_log.error("Valid types are: " + compression_list) - self.ui_log.error("") - self._exit(1) self.setup_archive() self._make_archive_paths() return @@ -839,20 +1211,6 @@ class SoSReport(SoSComponent): self._exit(1) def setup(self): - # Log command line options - msg = "[%s:%s] executing 'sos %s'" - self.soslog.info(msg % (__name__, "setup", " ".join(self.cmdline))) - - # Log active preset defaults - preset_args = self.preset.opts.to_args() - msg = ("[%s:%s] using '%s' preset defaults (%s)" % - (__name__, "setup", self.preset.name, " ".join(preset_args))) - self.soslog.info(msg) - - # Log effective options after applying preset defaults - self.soslog.info("[%s:%s] effective options now: %s" % - (__name__, "setup", " ".join(self.opts.to_args()))) - self.ui_log.info(_(" Setting up plugins ...")) for plugname, plug in self.loaded_plugins: try: @@ -890,9 +1248,6 @@ class SoSReport(SoSComponent): versions = [] versions.append("sosreport: %s" % __version__) - for plugname, plug in self.loaded_plugins: - versions.append("%s: %s" % (plugname, plug.version)) - self.archive.add_string(content="\n".join(versions), dest='version.txt') @@ -907,11 +1262,10 @@ class SoSReport(SoSComponent): plugruncount += 1 self.pluglist.append((plugruncount, i[0])) try: - self.plugpool = ThreadPoolExecutor(self.opts.threads) - # Pass the plugpool its own private copy of self.pluglist - results = self.plugpool.map(self._collect_plugin, - list(self.pluglist)) - self.plugpool.shutdown(wait=True) + results = [] + with ThreadPoolExecutor(self.opts.threads) as executor: + results = executor.map(self._collect_plugin, + list(self.pluglist)) for res in results: if not res: self.soslog.debug("Unexpected plugin task result: %s" % @@ -939,10 +1293,36 @@ class SoSReport(SoSComponent): _plug.manifest.add_field('end_time', end) _plug.manifest.add_field('run_time', end - start) except TimeoutError: - self.ui_log.error("\n Plugin %s timed out\n" % plugin[1]) + msg = "Plugin %s timed out" % plugin[1] + # log to ui_log.error to show the user, log to soslog.info + # so that someone investigating the sos execution has it all + # in one place, but without double notifying the user. + self.ui_log.error("\n %s\n" % msg) + self.soslog.info(msg) self.running_plugs.remove(plugin[1]) self.loaded_plugins[plugin[0]-1][1].set_timeout_hit() + pool.shutdown(wait=True) pool._threads.clear() + if self.opts.estimate_only: + # call "du -s -B1" for the tmp dir to get the disk usage of the + # data collected by the plugin - if the command fails, count with 0 + tmpdir = self.archive.get_tmp_dir() + try: + du = sos_get_command_output('du -sB1 %s' % tmpdir) + self.estimated_plugsizes[plugin[1]] = \ + int(du['output'].split()[0]) + except Exception: + self.estimated_plugsizes[plugin[1]] = 0 + # remove whole tmp_dir content - including "sos_commands" and + # similar dirs that will be re-created on demand by next plugin + # if needed; it is less error-prone approach than skipping + # deletion of some dirs but deleting their content + for f in os.listdir(tmpdir): + f = os.path.join(tmpdir, f) + if os.path.isdir(f) and not os.path.islink(f): + rmtree(f) + else: + os.unlink(f) return True def collect_plugin(self, plugin): @@ -960,7 +1340,7 @@ class SoSReport(SoSComponent): ) self.ui_progress(status_line) try: - plug.collect() + plug.collect_plugin() # certain exceptions can cause either of these lists to no # longer contain the plugin, which will result in sos hanging # so we can't blindly call remove() on these two. @@ -1042,13 +1422,9 @@ class SoSReport(SoSComponent): cmd['file'] ))) - for content, f in plug.copy_strings: + for content, f, tags in plug.copy_strings: section.add(CreatedFile(name=f, - href=os.path.join( - "..", - "sos_strings", - plugname, - f))) + href=os.path.join("..", f))) report.add(section) @@ -1062,6 +1438,8 @@ class SoSReport(SoSComponent): try: fd = self.get_temp_file() output = class_(report).unicode() + # safeguard against non-UTF characters + output = output.encode('utf-8', 'replace').decode() fd.write(output) fd.flush() self.archive.add_file(fd, dest=os.path.join('sos_reports', @@ -1123,6 +1501,8 @@ class SoSReport(SoSComponent): directory = None # report directory path (--build) map_file = None # path of the map file generated for the report + self.generate_manifest_tag_summary() + # use this instead of self.opts.clean beyond the initial check if # cleaning was requested in case SoSCleaner fails for some reason do_clean = False @@ -1158,6 +1538,40 @@ class SoSReport(SoSComponent): short_name='manifest.json' ) + # Now, just (optionally) pack the report and print work outcome; let + # print ui_log to stdout also in quiet mode. For non-quiet mode we + # already added the handler + if self.opts.quiet: + self.add_ui_log_to_stdout() + + # print results in estimate mode (to include also just added manifest) + if self.opts.estimate_only: + from sos.utilities import get_human_readable + from pathlib import Path + # add sos_logs, sos_reports dirs, etc., basically everything + # that remained in self.tmpdir after plugins' contents removal + # that still will be moved to the sos report final directory path + tmpdir_path = Path(self.tmpdir) + self.estimated_plugsizes['sos_logs_reports'] = sum( + [f.lstat().st_size for f in tmpdir_path.glob('**/*')]) + + _sum = get_human_readable(sum(self.estimated_plugsizes.values())) + self.ui_log.info("Estimated disk space requirement for whole " + "uncompressed sos report directory: %s" % _sum) + bigplugins = sorted(self.estimated_plugsizes.items(), + key=lambda x: x[1], reverse=True)[:5] + bp_out = ", ".join("%s: %s" % + (p, get_human_readable(v, precision=0)) + for p, v in bigplugins) + self.ui_log.info("Five biggest plugins: %s" % bp_out) + self.ui_log.info("") + self.ui_log.info("Please note the estimation is relevant to the " + "current options.") + self.ui_log.info("Be aware that the real disk space requirements " + "might be different. A rule of thumb is to " + "reserve at least double the estimation.") + self.ui_log.info("") + # package up and compress the results if not self.opts.build: old_umask = os.umask(0o077) @@ -1184,13 +1598,17 @@ class SoSReport(SoSComponent): finally: os.umask(old_umask) else: + if self.opts.encrypt_pass or self.opts.encrypt_key: + self.ui_log.warning("\nUnable to encrypt when using --build. " + "Encryption is only available for " + "archives.") # move the archive root out of the private tmp directory. directory = self.archive.get_archive_path() dir_name = os.path.basename(directory) + if do_clean: + dir_name = cleaner.obfuscate_string(dir_name) try: final_dir = os.path.join(self.sys_tmp, dir_name) - if do_clean: - final_dir = cleaner.obfuscate_string(final_dir) os.rename(directory, final_dir) directory = final_dir except (OSError, IOError): @@ -1205,9 +1623,14 @@ class SoSReport(SoSComponent): if not archive: print("Creating archive tarball failed.") else: - # compute and store the archive checksum - hash_name = self.policy.get_preferred_hash_name() - checksum = self._create_checksum(archive, hash_name) + try: + # compute and store the archive checksum + hash_name = self.policy.get_preferred_hash_name() + checksum = self._create_checksum(archive, hash_name) + except Exception: + print(_("Error generating archive checksum after " + "archive creation.\n")) + return False try: self._write_checksum(archive, hash_name, checksum) except (OSError, IOError): @@ -1215,12 +1638,12 @@ class SoSReport(SoSComponent): # output filename is in the private tmpdir - move it to the # containing directory. - final_name = os.path.join(self.sys_tmp, - os.path.basename(archive)) + base_archive = os.path.basename(archive) if do_clean: - final_name = cleaner.obfuscate_string( - final_name.replace('.tar', '-obfuscated.tar') + base_archive = cleaner.obfuscate_string( + base_archive.replace('.tar', '-obfuscated.tar') ) + final_name = os.path.join(self.sys_tmp, base_archive) # Get stat on the archive archivestat = os.stat(archive) @@ -1252,9 +1675,8 @@ class SoSReport(SoSComponent): except (OSError, IOError): print(_("Error moving checksum file: %s" % archive_hash)) - if not self.opts.build: - self.policy.display_results(archive, directory, checksum, - archivestat, map_file=map_file) + self.policy.display_results(archive, directory, checksum, + archivestat, map_file=map_file) else: self.policy.display_results(archive, directory, checksum, map_file=map_file) @@ -1294,18 +1716,76 @@ class SoSReport(SoSComponent): self.report_md.add_field('preset', self.preset.name if self.preset else 'unset') self.report_md.add_list('profiles', self.opts.profiles) + + _io_class = 'unknown' + if is_executable('ionice'): + _io = sos_get_command_output(f"ionice -p {os.getpid()}") + if _io['status'] == 0: + _io_class = _io['output'].split()[0].strip(':') + self.report_md.add_section('priority') + self.report_md.priority.add_field('io_class', _io_class) + self.report_md.priority.add_field('niceness', os.nice(0)) + self.report_md.add_section('devices') for key, value in self.devices.items(): - self.report_md.devices.add_list(key, value) + self.report_md.devices.add_field(key, value) self.report_md.add_list('enabled_plugins', self.opts.enable_plugins) self.report_md.add_list('disabled_plugins', self.opts.skip_plugins) self.report_md.add_section('plugins') + def generate_manifest_tag_summary(self): + """Add a section to the manifest that contains a dict summarizing the + tags that were created and assigned during this report's creation. + + This summary dict can be used for easier inspection of tagged items by + inspection/analyzer projects such as Red Hat Insights. The format of + this dict is `{tag_name: [file_list]}`. + """ + def compile_tags(ent, key='filepath'): + for tag in ent['tags']: + if not ent[key] or not tag: + continue + try: + path = tag_summary[tag] + except KeyError: + path = [] + path.extend( + ent[key] if isinstance(ent[key], list) else [ent[key]] + ) + tag_summary[tag] = sorted(list(set(path))) + + tag_summary = {} + for plug in self.report_md.plugins: + for cmd in plug.commands: + compile_tags(cmd) + for _file in plug.files: + compile_tags(_file, 'files_copied') + for collection in plug.collections: + compile_tags(collection) + self.report_md.add_field('tag_summary', + dict(sorted(tag_summary.items()))) + + def _merge_preset_options(self): + # Log command line options + msg = "[%s:%s] executing 'sos %s'" + self.soslog.info(msg % (__name__, "setup", " ".join(self.cmdline))) + + # Log active preset defaults + preset_args = self.preset.opts.to_args() + msg = ("[%s:%s] using '%s' preset defaults (%s)" % + (__name__, "setup", self.preset.name, " ".join(preset_args))) + self.soslog.info(msg) + + # Log effective options after applying preset defaults + self.soslog.info("[%s:%s] effective options now: %s" % + (__name__, "setup", " ".join(self.opts.to_args()))) + def execute(self): try: self.policy.set_commons(self.get_commons()) self.load_plugins() self._set_all_options() + self._merge_preset_options() self._set_tunables() self._check_for_unknown_plugins() self._set_plugin_options() @@ -1327,14 +1807,14 @@ class SoSReport(SoSComponent): if not self.verify_plugins(): return False - self.add_manifest_data() self.batch() self.prework() + self.add_manifest_data() self.setup() self.collect() if not self.opts.no_env_vars: self.collect_env_vars() - if not self.opts.noreport: + if not self.opts.no_report: self.generate_reports() if not self.opts.no_postproc: self.postproc() @@ -1346,13 +1826,15 @@ class SoSReport(SoSComponent): except (OSError): if self.opts.debug: raise - self.cleanup() + if not os.getenv('SOS_TEST_LOGS', None) == 'keep': + self.cleanup() except (KeyboardInterrupt): self.ui_log.error("\nExiting on user cancel") self.cleanup() self._exit(130) except (SystemExit) as e: - self.cleanup() + if not os.getenv('SOS_TEST_LOGS', None) == 'keep': + self.cleanup() sys.exit(e.code) self._exit(1) diff -pruN 4.0-2/sos/report/plugins/__init__.py 4.5.3ubuntu2/sos/report/plugins/__init__.py --- 4.0-2/sos/report/plugins/__init__.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/__init__.py 2023-04-28 17:16:21.000000000 +0000 @@ -11,7 +11,13 @@ """ This exports methods available for use by plugins for sos """ from sos.utilities import (sos_get_command_output, import_module, grep, - fileobj, tail, is_executable) + fileobj, tail, is_executable, TIMEOUT_DEFAULT, + path_exists, path_isdir, path_isfile, path_islink, + listdir, path_join, bold, file_is_binary, + recursive_dict_values_by_key) + +from sos.archive import P_FILE, P_LINK +import contextlib import os import glob import re @@ -20,6 +26,7 @@ from time import time import logging import fnmatch import errno +import textwrap from datetime import datetime @@ -41,11 +48,6 @@ def _mangle_command(command, name_max): return mangledname -def _path_in_path_list(path, path_list): - return any((p == path or path.startswith(os.path.abspath(p)+os.sep) - for p in path_list)) - - def _node_type(st): """ return a string indicating the type of special node represented by the stat buffer st (block, character, fifo, socket). @@ -61,16 +63,6 @@ def _node_type(st): return t[1] -def _file_is_compressed(path): - """Check if a file appears to be compressed - - Return True if the file specified by path appears to be compressed, - or False otherwise by testing the file name extension against a - list of known file compression extentions. - """ - return path.endswith(('.gz', '.xz', '.bz', '.bz2')) - - _certmatch = re.compile("-*BEGIN.*?-*END", re.DOTALL) _cert_replace = "-----SCRUBBED" @@ -325,10 +317,14 @@ class SoSPredicate(object): """Used by `Plugin()` to obtain the error string based on if the reason was a failed check or a forbidden check """ - msg = [self._report_failed(), self._report_forbidden()] + msg = [ + self._report_failed(), + self._report_forbidden(), + '(dry run)' if self.dry_run else '' + ] return " ".join(msg).lstrip() - def __nonzero__(self): + def __bool__(self): """Predicate evaluation hook. """ @@ -342,11 +338,6 @@ class SoSPredicate(object): self._eval_arch()) and not self.dry_run) - def __bool__(self): - # Py3 evaluation ends in a __bool__() call where py2 ends in a call - # to __nonzero__(). Wrap the latter here, to support both versions - return self.__nonzero__() - def __init__(self, owner, dry_run=False, kmods=[], services=[], packages=[], cmd_outputs=[], arch=[], required={}): """Initialise a new SoSPredicate object @@ -399,7 +390,90 @@ class SoSCommand(object): sorted(self.__dict__.items())) -class Plugin(object): +class PluginOpt(): + """This is used to define options available to plugins. Plugins will need + to define options alongside their distro-specific classes in order to add + support for user-controlled changes in Plugin behavior. + + :param name: The name of the plugin option + :type name: ``str`` + + :param default: The default value of the option + :type default: Any + + :param desc: A short description of the effect of the option + :type desc: ``str`` + + :param long_desc: A detailed description of the option. Will be used by + `sos info` + :type long_desc: ``str`` + + :param val_type: The type of object the option accepts for values. If + not provided, auto-detect from the type of ``default`` + :type val_type: A single type or a ``list`` of types + """ + + name = '' + default = None + enabled = False + desc = '' + long_desc = '' + value = None + val_type = [None] + plugin = '' + + def __init__(self, name='undefined', default=None, desc='', long_desc='', + val_type=None): + self.name = name + self.default = default + self.desc = desc + self.long_desc = long_desc + self.value = self.default + if val_type is not None: + if not isinstance(val_type, list): + val_type = [val_type] + else: + val_type = [default.__class__] + self.val_type = val_type + + def __str__(self): + items = [ + 'name=%s' % self.name, + 'desc=\'%s\'' % self.desc, + 'value=%s' % self.value, + 'default=%s' % self.default + ] + return '(' + ', '.join(items) + ')' + + def __repr__(self): + return self.__str__() + + def set_value(self, val): + # 'str' type accepts any value, incl. numbers + if type('') in self.val_type: + self.value = str(val) + return + if not any([type(val) == _t for _t in self.val_type]): + valid = [] + for t in self.val_type: + if t is None: + continue + if t.__name__ == 'bool': + valid.append("boolean true/false (on/off, etc)") + elif t.__name__ == 'str': + valid.append("string (no spaces)") + elif t.__name__ == 'int': + valid.append("integer values") + raise Exception( + "Plugin option '%s.%s' takes %s, not %s" % ( + self.plugin, self.name, ', '.join(valid), + type(val).__name__ + ) + ) + self.value = val + + +class Plugin(): """This is the base class for sosreport plugins. Plugins should subclass this and set the class variables where applicable. @@ -417,9 +491,6 @@ class Plugin(object): :cvar plugin_name: The name of the plugin, will be returned by `name()` :vartype plugin_name: ``str`` - :cvar version: The version of the plugin, defaults to 'unversioned' - :vartype version: ``str`` - :cvar packages: Package name(s) that, if installed, enable this plugin :vartype packages: ``tuple`` @@ -450,67 +521,80 @@ class Plugin(object): """ plugin_name = None - version = 'unversioned' packages = () files = () commands = () kernel_mods = () services = () + containers = () architectures = None archive = None profiles = () sysroot = '/' - plugin_timeout = 300 - cmd_timeout = 300 + plugin_timeout = TIMEOUT_DEFAULT + cmd_timeout = TIMEOUT_DEFAULT _timeout_hit = False cmdtags = {} filetags = {} + option_list = [] # Default predicates predicate = None cmd_predicate = None - _default_plug_opts = [ - ('timeout', 'Timeout in seconds for plugin', 'fast', -1), - ('postproc', 'Enable post-processing collected plugin data', 'fast', - True) - ] def __init__(self, commons): - if not getattr(self, "option_list", False): - self.option_list = [] self.copied_files = [] self.executed_commands = [] self._env_vars = set() self.alerts = [] self.custom_text = "" - self.opt_names = [] - self.opt_parms = [] self.commons = commons self.forbidden_paths = [] self.copy_paths = set() + self.container_copy_paths = [] self.copy_strings = [] self.collect_cmds = [] + self.options = {} self.sysroot = commons['sysroot'] self.policy = commons['policy'] self.devices = commons['devices'] self.manifest = None + self.skip_files = commons['cmdlineopts'].skip_files + self.skip_commands = commons['cmdlineopts'].skip_commands + self.default_environment = {} + self._tail_files_list = [] self.soslog = self.commons['soslog'] if 'soslog' in self.commons \ else logging.getLogger('sos') # add the default plugin opts - self.option_list.extend(self._default_plug_opts) - - # get the option list into a dictionary + self.options.update(self.get_default_plugin_opts()) + for popt in self.options: + self.options[popt].plugin = self.name() for opt in self.option_list: - self.opt_names.append(opt[0]) - self.opt_parms.append({'desc': opt[1], 'speed': opt[2], - 'enabled': opt[3]}) + opt.plugin = self.name() + self.options[opt.name] = opt # Initialise the default --dry-run predicate self.set_predicate(SoSPredicate(self)) + def get_default_plugin_opts(self): + return { + 'timeout': PluginOpt( + 'timeout', default=-1, val_type=int, + desc='Timeout in seconds for plugin to finish all collections' + ), + 'cmd-timeout': PluginOpt( + 'cmd-timeout', default=-1, val_type=int, + desc='Timeout in seconds for individual commands to finish' + ), + 'postproc': PluginOpt( + 'postproc', default=True, val_type=bool, + desc='Enable post-processing of collected data' + ) + } + def set_plugin_manifest(self, manifest): """Pass in a manifest object to the plugin to write to @@ -525,33 +609,124 @@ class Plugin(object): self.manifest.add_field('setup_start', '') self.manifest.add_field('setup_end', '') self.manifest.add_field('setup_time', '') + self.manifest.add_field('timeout', self.timeout) self.manifest.add_field('timeout_hit', False) + self.manifest.add_field('command_timeout', self.cmdtimeout) self.manifest.add_list('commands', []) self.manifest.add_list('files', []) + self.manifest.add_field('strings', {}) + self.manifest.add_field('containers', {}) + self.manifest.add_list('collections', []) - @property - def timeout(self): - """Returns either the default plugin timeout value, the value as - provided on the commandline via -k plugin.timeout=value, or the value - of the global --plugin-timeout option. + def set_default_cmd_environment(self, env_vars): + """ + Specify a collection of environment variables that should always be + passed to commands being executed by this plugin. + + :param env_vars: The environment variables and their values to set + :type env_vars: ``dict{ENV_VAR_NAME: ENV_VAR_VALUE}`` + """ + if not isinstance(env_vars, dict): + raise TypeError( + "Environment variables for Plugin must be specified by dict" + ) + self.default_environment = env_vars + self._log_debug("Default environment for all commands now set to %s" + % self.default_environment) + + def add_default_cmd_environment(self, env_vars): + """ + Add or modify a specific environment variable in the set of default + environment variables used by this Plugin. + + :param env_vars: The environment variables to add to the current + set of env vars in use + :type env_vars: ``dict`` + """ + if not isinstance(env_vars, dict): + raise TypeError("Environment variables must be added via dict") + self._log_debug("Adding %s to default environment" % env_vars) + self.default_environment.update(env_vars) + + def _get_cmd_environment(self, env=None): + """ + Get the merged set of environment variables for a command about to be + executed by this plugin. + + :returns: The set of env vars to use for a command + :rtype: ``dict`` + """ + if env is None: + return self.default_environment + if not isinstance(env, dict): + raise TypeError("Command env vars must be passed as dict") + _env = self.default_environment.copy() + _env.update(env) + return _env + + def timeout_from_options(self, optname, plugoptname, default_timeout): + """ + Get the timeout value for either the plugin or a command, as + determined by either the value provided via the + plugin.timeout or plugin.cmd-timeout option, the global timeout or + cmd-timeout options, or the default value set by the plugin or the + collection, in that order of precendence. + + :param optname: The name of the cmdline option being checked, either + 'plugin_timeout' or 'timeout' + :type optname: ``str`` + + :param plugoptname: The name of the plugin option name being checked, + either 'timeout' or 'cmd-timeout' + :type plugoptname: ``str`` + + :param default_timeout: The timeout to default to if determination is + inconclusive or hits an error + :type default_timeout: ``int`` + + :returns: The timeout value in seconds + :rtype: ``int`` """ _timeout = None try: - opt_timeout = self.get_option('plugin_timeout') - own_timeout = int(self.get_option('timeout')) + opt_timeout = self.get_option(optname) + own_timeout = int(self.get_option(plugoptname)) if opt_timeout is None: _timeout = own_timeout elif opt_timeout is not None and own_timeout == -1: - _timeout = int(opt_timeout) + if opt_timeout == TIMEOUT_DEFAULT: + _timeout = default_timeout + else: + _timeout = int(opt_timeout) elif opt_timeout is not None and own_timeout > -1: _timeout = own_timeout else: return None except ValueError: - return self.plugin_timeout # Default to known safe value + return default_timeout # Default to known safe value if _timeout is not None and _timeout > -1: return _timeout - return self.plugin_timeout + return default_timeout + + @property + def timeout(self): + """Returns either the default plugin timeout value, the value as + provided on the commandline via -k plugin.timeout=value, or the value + of the global --plugin-timeout option. + """ + _timeout = self.timeout_from_options('plugin_timeout', 'timeout', + self.plugin_timeout) + return _timeout + + @property + def cmdtimeout(self): + """Returns either the default command timeout value, the value as + provided on the commandline via -k plugin.cmd-timeout=value, or the + value of the global --cmd-timeout option. + """ + _cmdtimeout = self.timeout_from_options('cmd_timeout', 'cmd-timeout', + self.cmd_timeout) + return _cmdtimeout def set_timeout_hit(self): self._timeout_hit = True @@ -588,8 +763,180 @@ class Plugin(object): return cls.plugin_name return cls.__name__.lower() + @classmethod + def display_help(cls, section): + if cls.plugin_name is None: + cls.display_self_help(section) + else: + cls.display_plugin_help(section) + + @classmethod + def display_plugin_help(cls, section): + from sos.help import TERMSIZE + section.set_title("%s Plugin Information - %s" + % (cls.plugin_name.title(), cls.short_desc)) + missing = '\nDetailed information is not available for this plugin.\n' + + # Concatenate the docstrings of distro-specific plugins with their + # base classes, if available. + try: + _doc = '' + _sc = cls.__mro__[1] + if _sc != Plugin and _sc.__doc__: + _doc = _sc.__doc__ + if cls.__doc__: + _doc += cls.__doc__ + except Exception: + _doc = None + + section.add_text('\n %s' % _doc if _doc else missing) + + if not any([cls.packages, cls.commands, cls.files, cls.kernel_mods, + cls.services, cls.containers]): + section.add_text("This plugin is always enabled by default.") + else: + for trig in ['packages', 'commands', 'files', 'kernel_mods', + 'services']: + if getattr(cls, trig, None): + section.add_text( + "Enabled by %s: %s" + % (trig, ', '.join(getattr(cls, trig))), + newline=False + ) + if getattr(cls, 'containers'): + section.add_text( + "Enabled by containers with names matching: %s" + % ', '.join(c for c in cls.containers), + newline=False + ) + + if cls.profiles: + section.add_text( + "Enabled with the following profiles: %s" + % ', '.join(p for p in cls.profiles), + newline=False + ) + + if hasattr(cls, 'verify_packages'): + section.add_text( + "\nVerfies packages (when using --verify): %s" + % ', '.join(pkg for pkg in cls.verify_packages), + newline=False, + ) + + if cls.postproc is not Plugin.postproc: + section.add_text( + 'This plugin performs post-processing on potentially ' + 'sensitive collections. Disabling post-processing may' + ' leave sensitive data in plaintext.' + ) + + if not cls.option_list: + return + + optsec = section.add_section('Plugin Options') + optsec.add_text( + "These options may be toggled or changed using '%s'" + % bold("-k %s.option_name=$value" % cls.plugin_name) + ) + optsec.add_text(bold( + "\n{:<4}{:<20}{:<30}{:<20}\n".format( + ' ', "Option Name", "Default", "Description") + ), newline=False + ) + + opt_indent = ' ' * 54 + for opt in cls.option_list: + _def = opt.default + # convert certain values to text meanings + if _def is None or _def == '': + _def = "None/Unset" + if isinstance(opt.default, bool): + if opt.default: + _def = "True/On" + else: + _def = "False/Off" + _ln = "{:<4}{:<20}{:<30}{:<20}".format(' ', opt.name, _def, + opt.desc) + optsec.add_text( + textwrap.fill(_ln, width=TERMSIZE, + subsequent_indent=opt_indent), + newline=False + ) + if opt.long_desc: + _size = TERMSIZE - 10 + space = ' ' * 8 + optsec.add_text( + textwrap.fill(opt.long_desc, width=_size, + initial_indent=space, + subsequent_indent=space), + newline=False + ) + + @classmethod + def display_self_help(cls, section): + section.set_title("SoS Plugin Detailed Help") + section.add_text( + "Plugins are what define what collections occur for a given %s " + "execution. Plugins are generally representative of a single " + "system component (e.g. kernel), package (e.g. podman), or similar" + " construct. Plugins will typically specify multiple files or " + "directories to copy, as well as commands to execute and collect " + "the output of for further analysis." + % bold('sos report') + ) + + subsec = section.add_section('Plugin Enablement') + subsec.add_text( + 'Plugins will be automatically enabled based on one of several ' + 'triggers - a certain package being installed, a command or file ' + 'existing, a kernel module being loaded, etc...' + ) + subsec.add_text( + "Plugins may also be enabled or disabled by name using the %s or " + "%s options respectively." + % (bold('-e $name'), bold('-n $name')) + ) + + subsec.add_text( + "Certain plugins may only be available for specific distributions " + "or may behave differently on different distributions based on how" + " the component for that plugin is installed or how it operates." + " When using %s, help will be displayed for the version of the " + "plugin appropriate for your distribution." + % bold('sos help report.plugins.$plugin') + ) + + optsec = section.add_section('Using Plugin Options') + optsec.add_text( + "Many plugins support additional options to enable/disable or in " + "some other way modify the collections it makes. Plugin options " + "are set using the %s syntax. Options that are on/off toggles " + "may exclude setting a value, which will be interpreted as " + "enabling that option.\n\nSee specific plugin help sections " + "or %s for more information on these options" + % (bold('-k $plugin_name.$option_name=$value'), + bold('sos report -l')) + ) + + seealso = section.add_section('See Also') + _also = { + 'report.plugins.$plugin': 'Help for a specific $plugin', + 'policies': 'Information on distribution policies' + } + seealso.add_text( + "Additional relevant information may be available in these " + "help sections:\n\n%s" % "\n".join( + "{:>8}{:<30}{:<30}".format(' ', sec, desc) + for sec, desc in _also.items() + ), newline=False + ) + def _format_msg(self, msg): - return "[plugin:%s] %s" % (self.name(), msg) + return "[plugin:%s] %s" % (self.name(), + # safeguard against non-UTF logging, see + # #2790 for reference + msg.encode('utf-8', 'replace').decode()) def _log_error(self, msg): self.soslog.error(self._format_msg(msg)) @@ -603,19 +950,6 @@ class Plugin(object): def _log_debug(self, msg): self.soslog.debug(self._format_msg(msg)) - def join_sysroot(self, path): - """Join a given path with the configured sysroot - - :param path: The filesystem path that needs to be joined - :type path: ``str`` - - :returns: The joined filesystem path - :rtype: ``str`` - """ - if path[0] == os.sep: - path = path[1:] - return os.path.join(self.sysroot, path) - def strip_sysroot(self, path): """Remove the configured sysroot from a filesystem path @@ -627,7 +961,7 @@ class Plugin(object): """ if not self.use_sysroot(): return path - if path.startswith(self.sysroot): + if self.sysroot and path.startswith(self.sysroot): return path[len(self.sysroot):] return path @@ -646,8 +980,10 @@ class Plugin(object): ``False`` :rtype: ``bool`` """ - paths = [self.sysroot, self.archive.get_tmp_dir()] - return os.path.commonprefix(paths) == self.sysroot + # if sysroot is still None, that implies '/' + _sysroot = self.sysroot or '/' + paths = [_sysroot, self.archive.get_tmp_dir()] + return os.path.commonprefix(paths) == _sysroot def is_installed(self, package_name): """Is the package $package_name installed? @@ -658,7 +994,9 @@ class Plugin(object): :returns: ``True`` id the package is installed, else ``False`` :rtype: ``bool`` """ - return self.policy.pkg_by_name(package_name) is not None + return ( + len(self.policy.package_manager.all_pkgs_by_name(package_name)) > 0 + ) def is_service(self, name): """Does the service $name exist on the system? @@ -784,8 +1122,7 @@ class Plugin(object): return bool(pred) return False - def log_skipped_cmd(self, pred, cmd, kmods=False, services=False, - changes=False): + def log_skipped_cmd(self, cmd, pred, changes=False): """Log that a command was skipped due to predicate evaluation. Emit a warning message indicating that a command was skipped due @@ -795,21 +1132,17 @@ class Plugin(object): message indicating that the missing data can be collected by using the "--allow-system-changes" command line option will be included. - :param pred: The predicate that caused the command to be skipped - :type pred: ``SoSPredicate`` - :param cmd: The command that was skipped :type cmd: ``str`` - :param kmods: Did kernel modules cause the command to be skipped - :type kmods: ``bool`` - - :param services: Did services cause the command to be skipped - :type services: ``bool`` + :param pred: The predicate that caused the command to be skipped + :type pred: ``SoSPredicate`` :param changes: Is the `--allow-system-changes` enabled :type changes: ``bool`` """ + if pred is None: + pred = SoSPredicate(self) msg = "skipped command '%s': %s" % (cmd, pred.report_failure()) if changes: @@ -951,7 +1284,8 @@ class Plugin(object): content = readable.read() if not isinstance(content, str): content = content.decode('utf8', 'ignore') - result, replacements = re.subn(regexp, subst, content) + result, replacements = re.subn(regexp, subst, content, + flags=re.IGNORECASE) if replacements: self.archive.add_string(result, srcpath) else: @@ -1014,13 +1348,13 @@ class Plugin(object): reldest = linkdest self._log_debug("copying link '%s' pointing to '%s' with isdir=%s" - % (srcpath, linkdest, os.path.isdir(absdest))) + % (srcpath, linkdest, self.path_isdir(absdest))) dstpath = self.strip_sysroot(srcpath) # use the relative target path in the tarball self.archive.add_link(reldest, dstpath) - if os.path.isdir(absdest): + if self.path_isdir(absdest): self._log_debug("link '%s' is a directory, skipping..." % linkdest) return @@ -1053,7 +1387,7 @@ class Plugin(object): def _copy_dir(self, srcpath): try: - for name in os.listdir(srcpath): + for name in self.listdir(srcpath): self._log_debug("recursively adding '%s' from '%s'" % (name, srcpath)) path = os.path.join(srcpath, name) @@ -1071,14 +1405,35 @@ class Plugin(object): def _get_dest_for_srcpath(self, srcpath): if self.use_sysroot(): - srcpath = self.join_sysroot(srcpath) + srcpath = self.path_join(srcpath) for copied in self.copied_files: if srcpath == copied["srcpath"]: return copied["dstpath"] return None def _is_forbidden_path(self, path): - return _path_in_path_list(path, self.forbidden_paths) + return any( + re.match(forbid, path) for forbid in self.forbidden_paths + ) + + def _is_policy_forbidden_path(self, path): + return any([ + fnmatch.fnmatch(path, fp) for fp in self.policy.forbidden_paths + ]) + + def _is_skipped_path(self, path): + """Check if the given path matches a user-provided specification to + ignore collection of via the ``--skip-files`` option + + :param path: The filepath being collected + :type path: ``str`` + + :returns: ``True`` if file should be skipped, else ``False`` + """ + for _skip_path in self.skip_files: + if fnmatch.fnmatch(path, _skip_path): + return True + return False def _copy_node(self, path, st): dev_maj = os.major(st.st_rdev) @@ -1117,7 +1472,7 @@ class Plugin(object): else: if stat.S_ISDIR(st.st_mode) and os.access(srcpath, os.R_OK): # copy empty directory - if not os.listdir(srcpath): + if not self.listdir(srcpath): self.archive.add_dir(dest) return self._copy_dir(srcpath) @@ -1158,16 +1513,15 @@ class Plugin(object): forbidden = [forbidden] if self.use_sysroot(): - forbidden = [self.join_sysroot(f) for f in forbidden] + forbidden = [self.path_join(f) for f in forbidden] for forbid in forbidden: self._log_info("adding forbidden path '%s'" % forbid) - for path in glob.glob(forbid): - self.forbidden_paths.append(path) - - def get_all_options(self): - """return a list of all options selected""" - return (self.opt_names, self.opt_parms) + if "*" in forbid: + # calling translate() here on a dir-level path will break the + # re.match() call during path comparison + forbid = fnmatch.translate(forbid) + self.forbidden_paths.append(forbid) def set_option(self, optionname, value): """Set the named option to value. Ensure the original type of the @@ -1181,18 +1535,13 @@ class Plugin(object): :returns: ``True`` if the option is successfully set, else ``False`` :rtype: ``bool`` """ - for name, parms in zip(self.opt_names, self.opt_parms): - if name == optionname: - # FIXME: ensure that the resulting type of the set option - # matches that of the default value. This prevents a string - # option from being coerced to int simply because it holds - # a numeric value (e.g. a password). - # See PR #1526 and Issue #1597 - defaulttype = type(parms['enabled']) - if defaulttype != type(value) and defaulttype != type(None): - value = (defaulttype)(value) - parms['enabled'] = value + if optionname in self.options: + try: + self.options[optionname].set_value(value) return True + except Exception as err: + self._log_error(err) + raise return False def get_option(self, optionname, default=0): @@ -1213,31 +1562,19 @@ class Plugin(object): """ global_options = ( - 'all_logs', 'allow_system_changes', 'log_size', 'plugin_timeout', - 'since', 'verify' + 'all_logs', 'allow_system_changes', 'cmd_timeout', 'journal_size', + 'log_size', 'plugin_timeout', 'since', 'verify' ) if optionname in global_options: return getattr(self.commons['cmdlineopts'], optionname) - for name, parms in zip(self.opt_names, self.opt_parms): - if name == optionname: - val = parms['enabled'] - if val is not None: - return val - - return default - - def get_option_as_list(self, optionname, delimiter=",", default=None): - """Will try to return the option as a list separated by the - delimiter. - """ - option = self.get_option(optionname) - try: - opt_list = [opt.strip() for opt in option.split(delimiter)] - return list(filter(None, opt_list)) - except Exception: + if optionname in self.options: + opt = self.options[optionname] + if not default or opt.value is not None: + return opt.value return default + return default def _add_copy_paths(self, copy_paths): self.copy_paths.update(copy_paths) @@ -1269,10 +1606,11 @@ class Plugin(object): :returns: The tag(s) associated with `fname` :rtype: ``list`` of strings """ + tags = [] for key, val in self.filetags.items(): if re.match(key, fname): - return val - return [] + tags.extend(val) + return tags def generate_copyspec_tags(self): """After file collections have completed, retroactively generate @@ -1287,16 +1625,16 @@ class Plugin(object): matched_files = [] for cfile in self.copied_files: if re.match(file_regex, cfile['srcpath']): - matched_files.append(cfile['dstpath']) + matched_files.append(cfile['dstpath'].lstrip('/')) if matched_files: manifest_data['files_copied'] = matched_files self.manifest.files.append(manifest_data) def add_copy_spec(self, copyspecs, sizelimit=None, maxage=None, - tailit=True, pred=None, tags=[]): - """Add a file, directory, or regex matching filepaths to the archive + tailit=True, pred=None, tags=[], container=None): + """Add a file, directory, or globs matching filepaths to the archive - :param copyspecs: A file, directory, or regex matching filepaths + :param copyspecs: Files, directories, or globs matching filepaths :type copyspecs: ``str`` or a ``list`` of strings :param sizelimit: Limit the total size of collections from `copyspecs` @@ -1318,10 +1656,17 @@ class Plugin(object): for this collection :type tags: ``str`` or a ``list`` of strings + :param container: Container(s) from which this file should be copied + :type container: ``str`` or a ``list`` of strings + `copyspecs` will be expanded and/or globbed as appropriate. Specifying a directory here will cause the plugin to attempt to collect the entire directory, recursively. + If `container` is specified, `copyspecs` may only be explicit paths, + not globs as currently container runtimes do not support glob expansion + as part of the copy operation. + Note that `sizelimit` is applied to each `copyspec`, not each file individually. For example, a copyspec of ``['/etc/foo', '/etc/bar.conf']`` and a `sizelimit` of 25 means that @@ -1333,7 +1678,7 @@ class Plugin(object): since = self.get_option('since') logarchive_pattern = re.compile(r'.*((\.(zip|gz|bz2|xz))|[-.][\d]+)$') - configfile_pattern = re.compile(r"^%s/*" % self.join_sysroot("etc")) + configfile_pattern = re.compile(r"^%s/*" % self.path_join("etc")) if not self.test_predicate(pred=pred): self._log_info("skipped copy spec '%s' due to predicate (%s)" % @@ -1358,30 +1703,87 @@ class Plugin(object): if isinstance(tags, str): tags = [tags] + def get_filename_tag(fname): + """Generate a tag to add for a single file copyspec + + This tag will be set to the filename, minus any extensions + except for special extensions like .conf or .log, which will be + mangled to _conf or similar. + """ + if fname.startswith(('/proc', '/sys')): + return + _fname = fname.split('/')[-1] + _fname = _fname.replace('-', '_') + if _fname.endswith(('.conf', '.log', '.txt')): + return _fname.replace('.', '_') + for copyspec in copyspecs: if not (copyspec and len(copyspec)): return False - if self.use_sysroot(): - copyspec = self.join_sysroot(copyspec) - - files = self._expand_copy_spec(copyspec) + if not container: + if self.use_sysroot(): + copyspec = self.path_join(copyspec) + files = self._expand_copy_spec(copyspec) + if len(files) == 0: + continue + else: + files = [copyspec] - if len(files) == 0: - continue + _spec_tags = [] + if len(files) == 1: + _spec = get_filename_tag(files[0]) + if _spec: + _spec_tags.append(_spec) + _spec_tags.extend(self.get_tags_for_file(files[0])) - def get_filename_tag(fname): - """Generate a tag to add for a single file copyspec + _spec_tags.extend(tags) + _spec_tags = list(set(_spec_tags)) - This tag will be set to the filename, minus any extensions - except '.conf' which will be converted to '_conf' - """ - fname = fname.replace('-', '_') - if fname.endswith('.conf'): - return fname.replace('.', '_') - return fname.split('.')[0] + if container: + if isinstance(container, str): + container = [container] + for con in container: + if not self.container_exists(con): + continue + _tail = False + if sizelimit: + # to get just the size, stat requires a literal '%s' + # which conflicts with python string formatting + cmd = "stat -c %s " + copyspec + ret = self.exec_cmd(cmd, container=con) + if ret['status'] == 0: + try: + consize = int(ret['output']) + if consize > sizelimit: + _tail = True + except ValueError: + self._log_info( + "unable to determine size of '%s' in " + "container '%s'. Skipping collection." + % (copyspec, con) + ) + continue + else: + self._log_debug( + "stat of '%s' in container '%s' failed, " + "skipping collection: %s" + % (copyspec, con, ret['output']) + ) + continue + self.container_copy_paths.append( + (con, copyspec, sizelimit, _tail, _spec_tags) + ) + self._log_info( + "added collection of '%s' from container '%s'" + % (copyspec, con) + ) + # break out of the normal flow here as container file + # copies are done via command execution, not raw cp/mv + # operations + continue - # Files hould be sorted in most-recently-modified order, so that + # Files should be sorted in most-recently-modified order, so that # we collect the newest data first before reaching the limit. def getmtime(path): try: @@ -1403,12 +1805,6 @@ class Plugin(object): return False return True - _spec_tags = [] - if len(files) == 1: - _spec_tags = [get_filename_tag(files[0].split('/')[-1])] - - _spec_tags.extend(tags) - if since or maxage: files = list(filter(lambda f: time_filter(f), files)) @@ -1425,70 +1821,82 @@ class Plugin(object): if self._is_forbidden_path(_file): self._log_debug("skipping forbidden path '%s'" % _file) continue + if self._is_policy_forbidden_path(_file): + self._log_debug("skipping policy forbidden path '%s'" + % _file) + continue + if self._is_skipped_path(_file): + self._log_debug("skipping excluded path '%s'" % _file) + continue if limit_reached: self._log_info("skipping '%s' over size limit" % _file) continue try: - filestat = os.stat(_file) + file_size = os.stat(_file)[stat.ST_SIZE] except OSError: - self._log_info("failed to stat '%s'" % _file) - continue - current_size += filestat[stat.ST_SIZE] + # if _file is a broken symlink, we should collect it, + # otherwise skip it + if self.path_islink(_file): + file_size = 0 + else: + self._log_info("failed to stat '%s', skipping" % _file) + continue + current_size += file_size if sizelimit and current_size > sizelimit: limit_reached = True - if tailit and not _file_is_compressed(_file): - self._log_info("collecting tail of '%s' due to size " - "limit" % _file) - file_name = _file - if file_name[0] == os.sep: - file_name = file_name.lstrip(os.sep) - strfile = ( - file_name.replace(os.path.sep, ".") + ".tailed" + if tailit: + if file_is_binary(_file): + self._log_info( + "File '%s' is over size limit and is binary. " + "Skipping collection." % _file + ) + continue + + self._log_info( + "File '%s' is over size limit, will instead tail " + "the file during collection phase." % _file ) - add_size = (sizelimit + filestat[stat.ST_SIZE] - - current_size) - self.add_string_as_file(tail(_file, add_size), strfile) - rel_path = os.path.relpath('/', os.path.dirname(_file)) - link_path = os.path.join(rel_path, 'sos_strings', - self.name(), strfile) - self.archive.add_link(link_path, _file) + add_size = sizelimit + file_size - current_size + self._tail_files_list.append((_file, add_size)) _manifest_files.append(_file.lstrip('/')) - else: - self._log_info("skipping '%s' over size limit" % _file) else: # size limit not exceeded, copy the file _manifest_files.append(_file.lstrip('/')) self._add_copy_paths([_file]) # in the corner case we just reached the sizelimit, we # should collect the whole file and stop - limit_reached = (current_size == sizelimit) - if self.manifest: - self.manifest.files.append({ - 'specification': copyspec, - 'files_copied': _manifest_files, - 'tags': _spec_tags - }) - - def add_blockdev_cmd(self, cmds, devices='block', timeout=300, - sizelimit=None, chroot=True, runat=None, env=None, - binary=False, prepend_path=None, whitelist=[], - blacklist=[], tags=[]): - """Run a command or list of commands against storage-related devices. + limit_reached = (sizelimit and current_size == sizelimit) + + if not container: + # container collection manifest additions are handled later + if self.manifest: + self.manifest.files.append({ + 'specification': copyspec, + 'files_copied': _manifest_files, + 'tags': _spec_tags + }) + + def add_device_cmd(self, cmds, devices, timeout=None, sizelimit=None, + chroot=True, runat=None, env=None, binary=False, + prepend_path=None, whitelist=[], blacklist=[], tags=[], + priority=10, subdir=None): + """Run a command or list of commands against devices discovered during + sos initialization. Any commands specified by cmd will be iterated over the list of the specified devices. Commands passed to this should include a '%(dev)s' variable for substitution. - :param cmds: The command(s) to run against the list of devices - :type cmds: ``str`` or a ``list`` of strings + :param cmds: The command(s) to run against the list of devices + :type cmds: ``str`` or a ``list`` of strings - :param devices: The device paths to run `cmd` against. If set to - `block` or `fibre`, the commands will be run against - the matching list of discovered devices - :type devices: ``str`` or a ``list`` of device paths + :param devices: The device paths to run `cmd` against. This should be + either a list of devices/device paths or a key in the + devices dict discovered by sos during initialization. + :type devices: ``str`` or a ``list`` of devices or device paths. :param timeout: Timeout in seconds to allow each `cmd` to run :type timeout: ``int`` @@ -1518,78 +1926,95 @@ class Plugin(object): :param blacklist: Do not run `cmds` against devices matching these item(s) :type blacklist: ``list`` of ``str`` + + :param subdir: Write the command output to this subdir within the + Plugin directory + :type subdir: ``str`` """ + _dev_tags = [] if isinstance(tags, str): tags = [tags] - if devices == 'block': - prepend_path = prepend_path or '/dev/' - devices = self.devices['block'] - _dev_tags.append('block') - if devices == 'fibre': - devices = self.devices['fibre'] - _dev_tags.append('fibre') + if isinstance(devices, str): + devices = [devices] + + _devs = recursive_dict_values_by_key(self.devices, devices) + + if whitelist: + if isinstance(whitelist, str): + whitelist = [whitelist] + + _devs = [d for d in _devs if + any(re.match("(.*)?%s" % wl, d) for wl in whitelist)] + + if blacklist: + if isinstance(blacklist, str): + blacklist = [blacklist] + + _devs = [d for d in _devs if not + any(re.match("(.*)?%s" % bl, d) for bl in blacklist)] + _dev_tags.extend(tags) - self._add_device_cmd(cmds, devices, timeout=timeout, + self._add_device_cmd(cmds, _devs, timeout=timeout, sizelimit=sizelimit, chroot=chroot, runat=runat, env=env, binary=binary, prepend_path=prepend_path, - whitelist=whitelist, blacklist=blacklist, - tags=_dev_tags) + tags=_dev_tags, priority=priority, subdir=subdir) - def _add_device_cmd(self, cmds, devices, timeout=300, sizelimit=None, + def _add_device_cmd(self, cmds, devices, timeout=None, sizelimit=None, chroot=True, runat=None, env=None, binary=False, - prepend_path=None, whitelist=[], blacklist=[], - tags=[]): + prepend_path=None, tags=[], priority=10, subdir=None): """Run a command against all specified devices on the system. """ if isinstance(cmds, str): cmds = [cmds] if isinstance(devices, str): devices = [devices] - if isinstance(whitelist, str): - whitelist = [whitelist] - if isinstance(blacklist, str): - blacklist = [blacklist] sizelimit = sizelimit or self.get_option('log_size') for cmd in cmds: for device in devices: - _dev_ok = True _dev_tags = [device] _dev_tags.extend(tags) - if whitelist: - if not any(re.match(wl, device) for wl in whitelist): - _dev_ok = False - if blacklist: - if any(re.match(blist, device) for blist in blacklist): - _dev_ok = False - if not _dev_ok: - continue if prepend_path: - device = os.path.join(prepend_path, device) + device = self.path_join(prepend_path, device) _cmd = cmd % {'dev': device} self._add_cmd_output(cmd=_cmd, timeout=timeout, sizelimit=sizelimit, chroot=chroot, runat=runat, env=env, binary=binary, - tags=_dev_tags) + tags=_dev_tags, priority=priority, + subdir=subdir) def _add_cmd_output(self, **kwargs): """Internal helper to add a single command to the collection list.""" - pred = kwargs.pop('pred') if 'pred' in kwargs else None + pred = kwargs.pop('pred') if 'pred' in kwargs else SoSPredicate(self) + if 'priority' not in kwargs: + kwargs['priority'] = 10 + if 'changes' not in kwargs: + kwargs['changes'] = False + if self.get_option('all_logs') or kwargs['sizelimit'] == 0: + kwargs['to_file'] = True soscmd = SoSCommand(**kwargs) self._log_debug("packed command: " + soscmd.__str__()) + for _skip_cmd in self.skip_commands: + # This probably seems weird to be doing filename matching on the + # commands, however we want to remain consistent with our regex + # matching with file paths, which sysadmins are almost guaranteed + # to assume will use shell-style unix matching + if fnmatch.fnmatch(soscmd.cmd, _skip_cmd): + self._log_debug("skipping excluded command '%s'" % soscmd.cmd) + return if self.test_predicate(cmd=True, pred=pred): self.collect_cmds.append(soscmd) self._log_info("added cmd output '%s'" % soscmd.cmd) else: - self.log_skipped_cmd(pred, soscmd.cmd, kmods=bool(pred.kmods), - services=bool(pred.services), - changes=soscmd.changes) + self.log_skipped_cmd(soscmd.cmd, pred, changes=soscmd.changes) def add_cmd_output(self, cmds, suggest_filename=None, - root_symlink=None, timeout=cmd_timeout, stderr=True, + root_symlink=None, timeout=None, stderr=True, chroot=True, runat=None, env=None, binary=False, sizelimit=None, pred=None, subdir=None, - changes=False, foreground=False, tags=[]): + changes=False, foreground=False, tags=[], + priority=10, cmd_as_tag=False, container=None, + to_file=False): """Run a program or a list of programs and collect the output Output will be limited to `sizelimit`, collecting the last X amount @@ -1646,6 +2071,22 @@ class Plugin(object): :param tags: A tag or set of tags to add to the metadata entries for the `cmds` being run :type tags: ``str`` or a ``list`` of strings + + :param priority: The priority with which this command should be run, + lower values will run before higher values + :type priority: ``int`` + + :param cmd_as_tag: Should the command string be automatically formatted + to a tag? + :type cmd_as_tag: ``bool`` + + :param container: Run the specified `cmds` inside a container with this + ID or name + :type container: ``str`` + + :param to_file: Should command output be written directly to a new + file rather than stored in memory? + :type to_file: ``bool`` """ if isinstance(cmds, str): cmds = [cmds] @@ -1656,12 +2097,24 @@ class Plugin(object): if pred is None: pred = self.get_predicate(cmd=True) for cmd in cmds: + container_cmd = None + if container: + ocmd = cmd + container_cmd = (ocmd, container) + cmd = self.fmt_container_cmd(container, cmd) + if not cmd: + self._log_debug("Skipping command '%s' as the requested " + "container '%s' does not exist." + % (ocmd, container)) + continue self._add_cmd_output(cmd=cmd, suggest_filename=suggest_filename, root_symlink=root_symlink, timeout=timeout, stderr=stderr, chroot=chroot, runat=runat, env=env, binary=binary, sizelimit=sizelimit, pred=pred, subdir=subdir, tags=tags, - changes=changes, foreground=foreground) + changes=changes, foreground=foreground, + priority=priority, cmd_as_tag=cmd_as_tag, + to_file=to_file, container_cmd=container_cmd) def add_cmd_tags(self, tagdict): """Retroactively add tags to any commands that have been run by this @@ -1780,7 +2233,8 @@ class Plugin(object): # adds a mixed case variable name, still get that as well self._env_vars.update([env, env.upper(), env.lower()]) - def add_string_as_file(self, content, filename, pred=None): + def add_string_as_file(self, content, filename, pred=None, plug_dir=False, + tags=[]): """Add a string to the archive as a file :param content: The string to write to the archive @@ -1792,26 +2246,37 @@ class Plugin(object): :param pred: A predicate to gate if the string should be added to the archive or not :type pred: ``SoSPredicate`` - """ - # Generate summary string for logging - summary = content.splitlines()[0] if content else '' - if not isinstance(summary, str): - summary = content.decode('utf8', 'ignore') + :param plug_dir: Should the string be saved under the plugin's dir in + sos_commands/? If false, save to sos_strings/ + :type plug_dir: ``bool`` + + :param tags: A tag or set of tags to add to the manifest entry for this + collection + :type tags: ``str`` or a ``list`` of strings + """ if not self.test_predicate(cmd=False, pred=pred): - self._log_info("skipped string ...'%s' due to predicate (%s)" % - (summary, self.get_predicate(pred=pred))) + self._log_info("skipped string due to predicate (%s)" % + (self.get_predicate(pred=pred))) return - self.copy_strings.append((content, filename)) - self._log_debug("added string ...'%s' as '%s'" % (summary, filename)) + sos_dir = 'sos_commands' if plug_dir else 'sos_strings' + filename = os.path.join(sos_dir, self.name(), filename) + + if isinstance(tags, str): + tags = [tags] + + self.copy_strings.append((content, filename, tags)) + self._log_debug("added string as '%s'" % filename) def _collect_cmd_output(self, cmd, suggest_filename=None, - root_symlink=False, timeout=cmd_timeout, + root_symlink=False, timeout=None, stderr=True, chroot=True, runat=None, env=None, binary=False, sizelimit=None, subdir=None, - changes=False, foreground=False, tags=[]): + changes=False, foreground=False, tags=[], + priority=10, cmd_as_tag=False, to_file=False, + container_cmd=False): """Execute a command and save the output to a file for inclusion in the report. @@ -1833,7 +2298,12 @@ class Plugin(object): :param subdir: Subdir in plugin directory to save to :param changes: Does this cmd potentially make a change on the system? + :param foreground: Run the `cmd` in the foreground with a + TTY :param tags: Add tags in the archive manifest + :param cmd_as_tag: Format command string to tag + :param to_file: Write output directly to file instead + of saving in memory :returns: dict containing status, output, and filename in the archive for the executed cmd @@ -1842,40 +2312,74 @@ class Plugin(object): if self._timeout_hit: return + if timeout is None: + timeout = self.cmdtimeout _tags = [] if isinstance(tags, str): tags = [tags] _tags.extend(tags) - _tags.append(cmd.split(' ')[0]) _tags.extend(self.get_tags_for_cmd(cmd)) + if cmd_as_tag: + _tags.append(re.sub(r"[^\w\.]+", "_", cmd)) + + _tags = list(set(_tags)) + + _env = self._get_cmd_environment(env) + if chroot or self.commons['cmdlineopts'].chroot == 'always': root = self.sysroot else: root = None + if suggest_filename: + outfn = self._make_command_filename(suggest_filename, subdir) + else: + outfn = self._make_command_filename(cmd, subdir) + + outfn_strip = outfn[len(self.commons['cmddir'])+1:] + + if to_file: + self._log_debug("collecting '%s' output directly to disk" + % cmd) + self.archive.check_path(outfn, P_FILE) + out_file = os.path.join(self.archive.get_archive_path(), outfn) + else: + out_file = False + start = time() result = sos_get_command_output( cmd, timeout=timeout, stderr=stderr, chroot=root, - chdir=runat, env=env, binary=binary, sizelimit=sizelimit, - poller=self.check_timeout, foreground=foreground + chdir=runat, env=_env, binary=binary, sizelimit=sizelimit, + poller=self.check_timeout, foreground=foreground, + to_file=out_file ) + end = time() + run_time = end - start + if result['status'] == 124: - self._log_warn( - "command '%s' timed out after %ds" % (cmd, timeout) - ) + warn = "command '%s' timed out after %ds" % (cmd, timeout) + self._log_warn(warn) + if to_file: + msg = (" - output up until the timeout may be available at " + "%s" % outfn) + self._log_debug("%s%s" % (warn, msg)) manifest_cmd = { 'command': cmd.split(' ')[0], 'parameters': cmd.split(' ')[1:], 'exec': cmd, - 'filepath': None, + 'filepath': outfn if to_file else None, + 'truncated': result['truncated'], 'return_code': result['status'], - 'run_time': time() - start, + 'priority': priority, + 'start_time': start, + 'end_time': end, + 'run_time': run_time, 'tags': _tags } @@ -1890,8 +2394,9 @@ class Plugin(object): result = sos_get_command_output( cmd, timeout=timeout, chroot=False, chdir=runat, env=env, binary=binary, sizelimit=sizelimit, - poller=self.check_timeout + poller=self.check_timeout, to_file=out_file ) + run_time = time() - start self._log_debug("could not run '%s': command not found" % cmd) # Exit here if the command was not found in the chroot check above # as otherwise we will create a blank file in the archive @@ -1900,22 +2405,29 @@ class Plugin(object): self.manifest.commands.append(manifest_cmd) return result - run_time = time() - start - self._log_debug("collected output of '%s' in %s (changes=%s)" % (cmd.split()[0], run_time, changes)) - if suggest_filename: - outfn = self._make_command_filename(suggest_filename, subdir) - else: - outfn = self._make_command_filename(cmd, subdir) - - outfn_strip = outfn[len(self.commons['cmddir'])+1:] + if result['truncated']: + self._log_info("collected output of '%s' was truncated" + % cmd.split()[0]) + linkfn = outfn + outfn = outfn.replace('sos_commands', 'sos_strings') + '.tailed' + + if not to_file: + if binary: + self.archive.add_binary(result['output'], outfn) + else: + self.archive.add_string(result['output'], outfn) - if binary: - self.archive.add_binary(result['output'], outfn) - else: - self.archive.add_string(result['output'], outfn) + if result['truncated']: + # we need to manually build the relative path from the paths that + # exist within the build dir to properly drop these symlinks + _outfn_path = os.path.join(self.archive.get_archive_path(), outfn) + _link_path = os.path.join(self.archive.get_archive_path(), linkfn) + rpath = os.path.relpath(_outfn_path, _link_path) + rpath = rpath.replace('../', '', 1) + self.archive.add_link(rpath, linkfn) if root_symlink: self.archive.add_link(outfn, root_symlink) @@ -1927,17 +2439,22 @@ class Plugin(object): os.path.join(self.archive.get_archive_path(), outfn) if outfn else '' ) + if self.manifest: manifest_cmd['filepath'] = outfn manifest_cmd['run_time'] = run_time self.manifest.commands.append(manifest_cmd) + if container_cmd: + self._add_container_cmd_to_manifest(manifest_cmd.copy(), + container_cmd) return result def collect_cmd_output(self, cmd, suggest_filename=None, - root_symlink=False, timeout=cmd_timeout, + root_symlink=False, timeout=None, stderr=True, chroot=True, runat=None, env=None, binary=False, sizelimit=None, pred=None, - subdir=None, tags=[]): + changes=False, foreground=False, subdir=None, + tags=[]): """Execute a command and save the output to a file for inclusion in the report, then return the results for further use by the plugin @@ -1980,6 +2497,9 @@ class Plugin(object): on the system? :type changes: ``bool`` + :param foreground: Run the `cmd` in the foreground with a TTY + :type foreground: ``bool`` + :param tags: Add tags in the archive manifest :type tags: ``str`` or a ``list`` of strings @@ -1988,8 +2508,7 @@ class Plugin(object): :rtype: ``dict`` """ if not self.test_predicate(cmd=True, pred=pred): - self._log_info("skipped cmd output '%s' due to predicate (%s)" % - (cmd, self.get_predicate(cmd=True, pred=pred))) + self.log_skipped_cmd(cmd, pred, changes=changes) return { 'status': None, # don't match on if result['status'] checks 'output': '', @@ -1999,13 +2518,13 @@ class Plugin(object): return self._collect_cmd_output( cmd, suggest_filename=suggest_filename, root_symlink=root_symlink, timeout=timeout, stderr=stderr, chroot=chroot, runat=runat, - env=env, binary=binary, sizelimit=sizelimit, subdir=subdir, - tags=tags + env=env, binary=binary, sizelimit=sizelimit, foreground=foreground, + subdir=subdir, tags=tags ) - def exec_cmd(self, cmd, timeout=cmd_timeout, stderr=True, chroot=True, + def exec_cmd(self, cmd, timeout=None, stderr=True, chroot=True, runat=None, env=None, binary=False, pred=None, - foreground=False, container=False): + foreground=False, container=False, quotecmd=False): """Execute a command right now and return the output and status, but do not save the output within the archive. @@ -2044,6 +2563,9 @@ class Plugin(object): this name :type container: ``str`` + :param quotecmd: Whether the cmd should be quoted. + :type quotecmd: ``bool`` + :returns: Command exit status and output :rtype: ``dict`` """ @@ -2051,25 +2573,87 @@ class Plugin(object): if not self.test_predicate(cmd=True, pred=pred): return _default + if timeout is None: + timeout = self.cmdtimeout + if chroot or self.commons['cmdlineopts'].chroot == 'always': root = self.sysroot else: root = None + _env = self._get_cmd_environment(env) + if container: if self._get_container_runtime() is None: self._log_info("Cannot run cmd '%s' in container %s: no " "runtime detected on host." % (cmd, container)) return _default if self.container_exists(container): - cmd = self.fmt_container_cmd(container, cmd) + cmd = self.fmt_container_cmd(container, cmd, quotecmd) else: self._log_info("Cannot run cmd '%s' in container %s: no such " "container is running." % (cmd, container)) return sos_get_command_output(cmd, timeout=timeout, chroot=root, - chdir=runat, binary=binary, env=env, - foreground=foreground) + chdir=runat, binary=binary, env=_env, + foreground=foreground, stderr=stderr) + + def _add_container_file_to_manifest(self, container, path, arcpath, tags): + """Adds a file collection to the manifest for a particular container + and file path. + + :param container: The name of the container + :type container: ``str`` + + :param path: The filename from the container filesystem + :type path: ``str`` + + :param arcpath: Where in the archive the file is written to + :type arcpath: ``str`` + + :param tags: Metadata tags for this collection + :type tags: ``str`` or ``list`` of strings + """ + if container not in self.manifest.containers: + self.manifest.containers[container] = {'files': [], 'commands': []} + self.manifest.containers[container]['files'].append({ + 'specification': path, + 'files_copied': arcpath, + 'tags': tags + }) + + def _add_container_cmd_to_manifest(self, manifest, contup): + """Adds a command collection to the manifest for a particular container + and creates a symlink to that collection from the relevant + sos_containers/ location + + :param manifest: The manifest entry for the command + :type manifest: ``dict`` + + :param contup: A tuple of (original_cmd, container_name) + :type contup: ``tuple`` + """ + + cmd, container = contup + if container not in self.manifest.containers: + self.manifest.containers[container] = {'files': [], 'commands': []} + manifest['exec'] = cmd + manifest['command'] = cmd.split(' ')[0] + manifest['parameters'] = cmd.split(' ')[1:] + + _cdir = "sos_containers/%s/sos_commands/%s" % (container, self.name()) + _outloc = "../../../../%s" % manifest['filepath'] + cmdfn = self._mangle_command(cmd) + conlnk = "%s/%s" % (_cdir, cmdfn) + + # If check_path return None, it means that the sym link already exits, + # so to avoid Error 17, trying to recreate, we will skip creation and + # trust on the existing sym link (e.g. duplicate command) + if self.archive.check_path(conlnk, P_LINK): + os.symlink(_outloc, self.archive.dest_path(conlnk)) + + manifest['filepath'] = conlnk + self.manifest.containers[container]['commands'].append(manifest) def _get_container_runtime(self, runtime=None): """Based on policy and request by the plugin, return a usable @@ -2088,7 +2672,7 @@ class Plugin(object): """If a container runtime is present, check to see if a container with a given name is currently running - :param name: The name of the container to check presence of + :param name: The name or ID of the container to check presence of :type name: ``str`` :returns: ``True`` if `name` exists, else ``False`` @@ -2096,10 +2680,28 @@ class Plugin(object): """ _runtime = self._get_container_runtime() if _runtime is not None: - con = _runtime.get_container_by_name(name) - return con is not None + return (_runtime.container_exists(name) or + _runtime.get_container_by_name(name) is not None) return False + def get_all_containers_by_regex(self, regex, get_all=False): + """Get a list of all container names and ID matching a regex + + :param regex: The regular expression to match + :type regex: ``str`` + + :param get_all: Return all containers found, even terminated ones + :type get_all: ``bool`` + + :returns: All container IDs and names matching ``regex`` + :rtype: ``list`` of ``tuples`` as (id, name) + """ + _runtime = self._get_container_runtime() + if _runtime is not None: + _containers = _runtime.get_containers(get_all=get_all) + return [c for c in _containers if re.match(regex, c[1])] + return [] + def get_container_by_name(self, name): """Get the container ID for a specific container @@ -2177,22 +2779,34 @@ class Plugin(object): return _runtime.volumes return [] - def get_container_logs(self, container, **kwargs): - """Helper to get the ``logs`` output for a given container + def add_container_logs(self, containers, get_all=False, **kwargs): + """Helper to get the ``logs`` output for a given container or list + of container names and/or regexes. Supports passthru of add_cmd_output() options - :param container: The name of the container to retrieve logs from - :type container: ``str`` + :param containers: The name of the container to retrieve logs from, + may be a single name or a regex + :type containers: ``str`` or ``list`` of strs + + :param get_all: Should non-running containers also be queried? + Default: False + :type get_all: ``bool`` :param kwargs: Any kwargs supported by ``add_cmd_output()`` are supported here """ _runtime = self._get_container_runtime() if _runtime is not None: - self.add_cmd_output(_runtime.get_logs_command(container), **kwargs) + if isinstance(containers, str): + containers = [containers] + for container in containers: + _cons = self.get_all_containers_by_regex(container, get_all) + for _con in _cons: + cmd = _runtime.get_logs_command(_con[1]) + self.add_cmd_output(cmd, **kwargs) - def fmt_container_cmd(self, container, cmd): + def fmt_container_cmd(self, container, cmd, quotecmd=False): """Format a command to be executed by the loaded ``ContainerRuntime`` in a specified container @@ -2202,14 +2816,17 @@ class Plugin(object): :param cmd: The command to run within the container :type cmd: ``str`` + :param quotecmd: Whether the cmd should be quoted. + :type quotecmd: ``bool`` + :returns: The command to execute so that the specified `cmd` will run within the `container` and not on the host :rtype: ``str`` """ if self.container_exists(container): _runtime = self._get_container_runtime() - return _runtime.fmt_container_cmd(container, cmd) - return cmd + return _runtime.fmt_container_cmd(container, cmd, quotecmd) + return '' def is_module_loaded(self, module_name): """Determine whether specified module is loaded or not @@ -2247,7 +2864,7 @@ class Plugin(object): :param services: Service name(s) to collect statuses for :type services: ``str`` or a ``list`` of strings - :param kwargs: Optional arguments to pass to _add_cmd_output + :param kwargs: Optional arguments to pass to add_cmd_output (timeout, predicate, suggest_filename,..) """ @@ -2262,12 +2879,12 @@ class Plugin(object): return for service in services: - self._add_cmd_output(cmd="%s %s" % (query, service), **kwargs) + self.add_cmd_output("%s %s" % (query, service), **kwargs) def add_journal(self, units=None, boot=None, since=None, until=None, lines=None, allfields=False, output=None, - timeout=cmd_timeout, identifier=None, catalog=None, - sizelimit=None, pred=None, tags=[]): + timeout=None, identifier=None, catalog=None, + sizelimit=None, pred=None, tags=None, priority=10): """Collect journald logs from one of more units. :param units: Which journald units to collect @@ -2316,17 +2933,24 @@ class Plugin(object): identifier_opt = " --identifier %s" catalog_opt = " --catalog" - journal_size = 100 - all_logs = self.get_option("all_logs") - log_size = sizelimit or self.get_option("log_size") - log_size = max(log_size, journal_size) if not all_logs else 0 + if sizelimit == 0 or self.get_option("all_logs"): + # allow for specific sizelimit overrides in plugins + log_size = 0 + else: + log_size = sizelimit or self.get_option('journal_size') if isinstance(units, str): units = [units] + if isinstance(tags, str): + tags = [tags] + elif not tags: + tags = [] + if units: for unit in units: journal_cmd += unit_opt % unit + tags.append("journal_%s" % unit) if identifier: journal_cmd += identifier_opt % identifier @@ -2358,7 +2982,8 @@ class Plugin(object): self._log_debug("collecting journal: %s" % journal_cmd) self._add_cmd_output(cmd=journal_cmd, timeout=timeout, - sizelimit=log_size, pred=pred, tags=tags) + sizelimit=log_size, pred=pred, tags=tags, + priority=priority) def _expand_copy_spec(self, copyspec): def __expand(paths): @@ -2367,10 +2992,10 @@ class Plugin(object): for path in paths: try: # avoid recursive symlink dirs - if os.path.isfile(path) or os.path.islink(path): + if self.path_isfile(path) or self.path_islink(path): found_paths.append(path) - elif os.path.isdir(path) and os.listdir(path): - found_paths.extend(__expand(os.path.join(path, '*'))) + elif self.path_isdir(path) and self.listdir(path): + found_paths.extend(__expand(self.path_join(path, '*'))) else: found_paths.append(path) except PermissionError: @@ -2383,15 +3008,15 @@ class Plugin(object): pass return list(set(found_paths)) - if (os.access(copyspec, os.R_OK) and os.path.isdir(copyspec) and - os.listdir(copyspec)): + if (os.access(copyspec, os.R_OK) and self.path_isdir(copyspec) and + self.listdir(copyspec)): # the directory exists and is non-empty, recurse through it - copyspec = os.path.join(copyspec, '*') + copyspec = self.path_join(copyspec, '*') expanded = glob.glob(copyspec, recursive=True) recursed_files = [] for _path in expanded: try: - if os.path.isdir(_path) and os.listdir(_path): + if self.path_isdir(_path) and self.listdir(_path): # remove the top level dir to avoid duplicate attempts to # copy the dir and its contents expanded.remove(_path) @@ -2405,43 +3030,165 @@ class Plugin(object): return list(set(expanded)) def _collect_copy_specs(self): - for path in self.copy_paths: + for path in sorted(self.copy_paths, reverse=True): self._log_info("collecting path '%s'" % path) self._do_copy_path(path) self.generate_copyspec_tags() + def _collect_container_copy_specs(self): + """Copy any requested files from containers here. This is done + separately from normal file collection as this requires the use of + a container runtime. + + This method will iterate over self.container_copy_paths which is a set + of 5-tuples as (container, path, sizelimit, stdout, tags). + """ + if not self.container_copy_paths: + return + rt = self._get_container_runtime() + if not rt: + self._log_info("Cannot collect container based files - no runtime " + "is present/active.") + return + if not rt.check_can_copy(): + self._log_info("Loaded runtime '%s' does not support copying " + "files from containers. Skipping collections.") + return + for contup in self.container_copy_paths: + con, path, sizelimit, tailit, tags = contup + self._log_info("collecting '%s' from container '%s'" % (path, con)) + + arcdest = "sos_containers/%s/%s" % (con, path.lstrip('/')) + self.archive.check_path(arcdest, P_FILE) + dest = self.archive.dest_path(arcdest) + + cpcmd = rt.get_copy_command( + con, path, dest, sizelimit=sizelimit if tailit else None + ) + cpret = self.exec_cmd(cpcmd, timeout=10) + + if cpret['status'] == 0: + if tailit: + # current runtimes convert files sent to stdout to tar + # archives, with no way to control that + self.archive.add_string(cpret['output'], arcdest) + self._add_container_file_to_manifest(con, path, arcdest, tags) + else: + self._log_info("error copying '%s' from container '%s': %s" + % (path, con, cpret['output'])) + def _collect_cmds(self): + self.collect_cmds.sort(key=lambda x: x.priority) for soscmd in self.collect_cmds: self._log_debug("unpacked command: " + soscmd.__str__()) self._log_info("collecting output of '%s'" % soscmd.cmd) self._collect_cmd_output(**soscmd.__dict__) + def _collect_tailed_files(self): + for _file, _size in self._tail_files_list: + self._log_info(f"collecting tail of '{_file}' due to size limit") + file_name = _file + if file_name[0] == os.sep: + file_name = file_name.lstrip(os.sep) + strfile = ( + file_name.replace(os.path.sep, ".") + ".tailed" + ) + self.add_string_as_file(tail(_file, _size), strfile) + rel_path = os.path.relpath('/', os.path.dirname(_file)) + link_path = os.path.join(rel_path, 'sos_strings', + self.name(), strfile) + self.archive.add_link(link_path, _file) + def _collect_strings(self): - for string, file_name in self.copy_strings: + for string, file_name, tags in self.copy_strings: if self._timeout_hit: return - content = '' - if string: - content = string.splitlines()[0] - if not isinstance(content, str): - content = content.decode('utf8', 'ignore') - self._log_info("collecting string ...'%s' as '%s'" - % (content, file_name)) + self._log_info("collecting string as '%s'" % file_name) try: - self.archive.add_string(string, - os.path.join('sos_strings', - self.name(), - file_name)) + self.archive.add_string(string, file_name) + _name = file_name.split('/')[-1].replace('.', '_') + self.manifest.strings[_name] = { + 'path': file_name, + 'tags': tags + } except Exception as e: self._log_debug("could not add string '%s': %s" % (file_name, e)) + def _collect_manual(self): + """Kick off manual collections performed by the plugin. These manual + collections are anything the plugin collects outside of existing + files and/or command output. Anything the plugin manually compiles or + constructs for data that is included in the final archive. + + Plugins will need to define these collections by overriding the + ``collect()`` method, similar to how plugins define their own + ``setup()`` methods. + """ + try: + self.collect() + except Exception as err: + self._log_error(f"Error during plugin collections: {err}") + def collect(self): + """If a plugin needs to manually compile some data for a collection, + that should be specified here by overriding this method. + + These collections are run last during a plugin's execution, and as such + are more likely to be interrupted by timeouts than file or command + output collections. + """ + pass + + @contextlib.contextmanager + def collection_file(self, fname, subdir=None, tags=[]): + """Handles creating and managing files within a plugin's subdirectory + within the archive, and is intended to be used to save manually + compiled data generated during a plugin's ``_collect_manual()`` step + of the collection phase. + + Plugins should call this method using a ``with`` context manager. + + :param fname: The name of the file within the plugin directory + :type fname: ``str`` + + :param subdir: If needed, specify a subdir to write the file to + :type subdir: ``str`` + + :param tags: Tags to be added to this file in the manifest + :type tags: ``str`` or ``list`` of ``str``s + """ + try: + start = time() + _pfname = self._make_command_filename(fname, subdir=subdir) + self.archive.check_path(_pfname, P_FILE) + _name = self.archive.dest_path(_pfname) + _file = open(_name, 'w') + self._log_debug(f"manual collection file opened: {_name}") + yield _file + _file.close() + end = time() + run = end - start + self._log_info(f"manual collection '{fname}' finished in {run}") + if isinstance(tags, str): + tags = [tags] + self.manifest.collections.append({ + 'name': fname, + 'filepath': _pfname, + 'tags': tags + }) + except Exception as err: + self._log_info(f"Error with collection file '{fname}': {err}") + + def collect_plugin(self): """Collect the data for a plugin.""" start = time() self._collect_copy_specs() + self._collect_container_copy_specs() + self._collect_tailed_files() self._collect_cmds() self._collect_strings() + self._collect_manual() fields = (self.name(), time() - start) self._log_debug("collected plugin '%s' in %s" % fields) @@ -2475,7 +3222,7 @@ class Plugin(object): """ # some files or packages have been specified for this package if any([self.files, self.packages, self.commands, self.kernel_mods, - self.services, self.architectures]): + self.services, self.containers, self.architectures]): if isinstance(self.files, str): self.files = [self.files] @@ -2502,14 +3249,18 @@ class Plugin(object): if self._check_plugin_triggers(files, packages, commands, - services): + services, + # SCL containers don't exist + ()): type(self)._scls_matched.append(scl) - return len(type(self)._scls_matched) > 0 + if type(self)._scls_matched: + return True return self._check_plugin_triggers(self.files, self.packages, self.commands, - self.services) + self.services, + self.containers) if isinstance(self, SCLPlugin): # if files and packages weren't specified, we take all SCLs @@ -2517,17 +3268,19 @@ class Plugin(object): return True - def _check_plugin_triggers(self, files, packages, commands, services): + def _check_plugin_triggers(self, files, packages, commands, services, + containers): - if not any([files, packages, commands, services]): + if not any([files, packages, commands, services, containers]): # no checks beyond architecture restrictions return self.check_is_architecture() - return ((any(os.path.exists(fname) for fname in files) or + return ((any(self.path_exists(fname) for fname in files) or any(self.is_installed(pkg) for pkg in packages) or - any(is_executable(cmd) for cmd in commands) or + any(is_executable(cmd, self.sysroot) for cmd in commands) or any(self.is_module_loaded(mod) for mod in self.kernel_mods) or - any(self.is_service(svc) for svc in services)) and + any(self.is_service(svc) for svc in services) or + any(self.container_exists(cntr) for cntr in containers)) and self.check_is_architecture()) def check_is_architecture(self): @@ -2559,6 +3312,10 @@ class Plugin(object): for service in self.services: if self.is_service(service): self.add_service_status(service) + self.add_journal(service) + for kmod in self.kernel_mods: + if self.is_module_loaded(kmod): + self.add_cmd_output(f"modinfo {kmod}") def setup(self): """Collect the list of files declared by the plugin. This method @@ -2580,6 +3337,85 @@ class Plugin(object): if verify_cmd: self.add_cmd_output(verify_cmd) + def path_exists(self, path): + """Helper to call the sos.utilities wrapper that allows the + corresponding `os` call to account for sysroot + + :param path: The canonical path for a specific file/directory + :type path: ``str`` + + + :returns: True if the path exists in sysroot, else False + :rtype: ``bool`` + """ + return path_exists(path, self.sysroot) + + def path_isdir(self, path): + """Helper to call the sos.utilities wrapper that allows the + corresponding `os` call to account for sysroot + + :param path: The canonical path for a specific file/directory + :type path: ``str`` + + + :returns: True if the path is a dir, else False + :rtype: ``bool`` + """ + return path_isdir(path, self.sysroot) + + def path_isfile(self, path): + """Helper to call the sos.utilities wrapper that allows the + corresponding `os` call to account for sysroot + + :param path: The canonical path for a specific file/directory + :type path: ``str`` + + + :returns: True if the path is a file, else False + :rtype: ``bool`` + """ + return path_isfile(path, self.sysroot) + + def path_islink(self, path): + """Helper to call the sos.utilities wrapper that allows the + corresponding `os` call to account for sysroot + + :param path: The canonical path for a specific file/directory + :type path: ``str`` + + + :returns: True if the path is a link, else False + :rtype: ``bool`` + """ + return path_islink(path, self.sysroot) + + def listdir(self, path): + """Helper to call the sos.utilities wrapper that allows the + corresponding `os` call to account for sysroot + + :param path: The canonical path for a specific file/directory + :type path: ``str`` + + + :returns: Contents of path, if it is a directory + :rtype: ``list`` + """ + return listdir(path, self.sysroot) + + def path_join(self, path, *p): + """Helper to call the sos.utilities wrapper that allows the + corresponding `os` call to account for sysroot + + :param path: The leading path passed to os.path.join() + :type path: ``str`` + + :param p: Following path section(s) to be joined with ``path``, + an empty parameter will result in a path that ends with + a separator + :type p: ``str`` + """ + return path_join(path, *p, sysroot=self.sysroot) + def postproc(self): """Perform any postprocessing. To be replaced by a plugin if required. """ @@ -2599,7 +3435,7 @@ class Plugin(object): try: cmd_line_paths = glob.glob(cmd_line_glob) for path in cmd_line_paths: - f = open(path, 'r') + f = open(self.path_join(path), 'r') cmd_line = f.read().strip() if process in cmd_line: status = True @@ -2629,12 +3465,91 @@ class Plugin(object): continue return pids + def get_network_namespaces(self, ns_pattern=None, ns_max=None): + if ns_max is None and self.commons['cmdlineopts'].namespaces: + ns_max = self.commons['cmdlineopts'].namespaces + return self.filter_namespaces(self.commons['namespaces']['network'], + ns_pattern, ns_max) + + def filter_namespaces(self, ns_list, ns_pattern=None, ns_max=None): + """Filter a list of namespaces by regex pattern or max number of + namespaces (options originally present in the networking plugin.) + """ + out_ns = [] + + # Regex initialization outside of for loop + if ns_pattern: + pattern = ( + '(?:%s$)' % '$|'.join(ns_pattern.split()).replace('*', '.*') + ) + for ns in ns_list: + # if ns_pattern defined, skip namespaces not matching the pattern + if ns_pattern and not bool(re.match(pattern, ns)): + continue + out_ns.append(ns) + + # if ns_max is defined at all, break the loop when the limit is + # reached + # this allows the use of both '0' and `None` to mean unlimited + if ns_max: + if len(out_ns) == ns_max: + self._log_warn("Limiting namespace iteration " + "to first %s namespaces found" + % ns_max) + break + + return out_ns + + +class PluginDistroTag(): + """The base tagging class for distro-specific classes used to signify that + a Plugin is written for that distro. -class RedHatPlugin(object): + Use IndependentPlugin for plugins that are distribution agnostic + """ + pass + + +class RedHatPlugin(PluginDistroTag): """Tagging class for Red Hat's Linux distributions""" pass +class UbuntuPlugin(PluginDistroTag): + """Tagging class for Ubuntu Linux""" + pass + + +class DebianPlugin(PluginDistroTag): + """Tagging class for Debian Linux""" + pass + + +class SuSEPlugin(PluginDistroTag): + """Tagging class for SuSE Linux distributions""" + pass + + +class OpenEulerPlugin(PluginDistroTag): + """Tagging class for openEuler linux distributions""" + pass + + +class CosPlugin(PluginDistroTag): + """Tagging class for Container-Optimized OS""" + pass + + +class IndependentPlugin(PluginDistroTag): + """Tagging class for plugins that can run on any platform""" + pass + + +class ExperimentalPlugin(PluginDistroTag): + """Tagging class that indicates that this plugin is experimental""" + pass + + class SCLPlugin(RedHatPlugin): """Superclass for plugins operating on Software Collections (SCLs). @@ -2670,30 +3585,17 @@ class SCLPlugin(RedHatPlugin): return [scl.strip() for scl in output.splitlines()] def convert_cmd_scl(self, scl, cmd): - """wrapping command in "scl enable" call and adds proper PATH + """wrapping command in "scl enable" call """ - # load default SCL prefix to PATH - prefix = self.policy.get_default_scl_prefix() - # read prefix from /etc/scl/prefixes/${scl} and strip trailing '\n' - try: - prefix = open('/etc/scl/prefixes/%s' % scl, 'r').read()\ - .rstrip('\n') - except Exception as e: - self._log_error("Failed to find prefix for SCL %s using %s: %s" - % (scl, prefix, e)) - - # expand PATH by equivalent prefixes under the SCL tree - path = os.environ["PATH"] - for p in path.split(':'): - path = '%s/%s%s:%s' % (prefix, scl, p, path) - - scl_cmd = "scl enable %s \"PATH=%s %s\"" % (scl, path, cmd) + scl_cmd = "scl enable %s \"%s\"" % (scl, cmd) return scl_cmd def add_cmd_output_scl(self, scl, cmds, **kwargs): """Same as add_cmd_output, except that it wraps command in "scl enable" call and sets proper PATH. """ + if scl not in self.scls_matched: + return if isinstance(cmds, str): cmds = [cmds] scl_cmds = [] @@ -2718,6 +3620,8 @@ class SCLPlugin(RedHatPlugin): """Same as add_copy_spec, except that it prepends path to SCL root to "copyspecs". """ + if scl not in self.scls_matched: + return if isinstance(copyspecs, str): copyspecs = [copyspecs] scl_copyspecs = [] @@ -2725,55 +3629,6 @@ class SCLPlugin(RedHatPlugin): scl_copyspecs.append(self.convert_copyspec_scl(scl, copyspec)) self.add_copy_spec(scl_copyspecs) - def add_copy_spec_limit_scl(self, scl, copyspec, **kwargs): - """Same as add_copy_spec_limit, except that it prepends path to SCL - root to "copyspec". - """ - self.add_copy_spec_limit( - self.convert_copyspec_scl(scl, copyspec), - **kwargs - ) - - -class PowerKVMPlugin(RedHatPlugin): - """Tagging class for IBM PowerKVM Linux""" - pass - - -class ZKVMPlugin(RedHatPlugin): - """Tagging class for IBM ZKVM Linux""" - pass - - -class UbuntuPlugin(object): - """Tagging class for Ubuntu Linux""" - pass - - -class DebianPlugin(object): - """Tagging class for Debian Linux""" - pass - - -class SuSEPlugin(object): - """Tagging class for SuSE Linux distributions""" - pass - - -class CosPlugin(object): - """Tagging class for Container-Optimized OS""" - pass - - -class IndependentPlugin(object): - """Tagging class for plugins that can run on any platform""" - pass - - -class ExperimentalPlugin(object): - """Tagging class that indicates that this plugin is experimental""" - pass - def import_plugin(name, superclasses=None): """Import name as a module and return a list of all classes defined in that diff -pruN 4.0-2/sos/report/plugins/abrt.py 4.5.3ubuntu2/sos/report/plugins/abrt.py --- 4.0-2/sos/report/plugins/abrt.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/abrt.py 2023-04-28 17:16:21.000000000 +0000 @@ -8,7 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt class Abrt(Plugin, RedHatPlugin): @@ -21,11 +21,13 @@ class Abrt(Plugin, RedHatPlugin): files = ('/var/spool/abrt',) option_list = [ - ("detailed", 'collect detailed info for every report', 'slow', False) + PluginOpt("detailed", default=False, + desc="collect detailed information for every report") ] def setup(self): - self.add_cmd_output("abrt-cli status") + self.add_cmd_output("abrt-cli status", + tags=["abrt_status", "abrt_status_bare"]) abrt_list = self.collect_cmd_output("abrt-cli list") if self.get_option("detailed") and abrt_list['status'] == 0: for line in abrt_list["output"].splitlines(): diff -pruN 4.0-2/sos/report/plugins/alternatives.py 4.5.3ubuntu2/sos/report/plugins/alternatives.py --- 4.0-2/sos/report/plugins/alternatives.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/alternatives.py 2023-04-28 17:16:21.000000000 +0000 @@ -14,11 +14,18 @@ from sos.report.plugins import Plugin, R class Alternatives(Plugin, RedHatPlugin): short_desc = 'System alternatives' - + plugin_name = 'alternatives' packages = ('chkconfig',) commands = ('alternatives',) def setup(self): + + self.add_cmd_tags({ + "alternatives --display java.*": 'display_java', + "alternatives --display python.*": + 'alternatives_display_python' + }) + self.add_cmd_output('alternatives --version') alts = [] diff -pruN 4.0-2/sos/report/plugins/anaconda.py 4.5.3ubuntu2/sos/report/plugins/anaconda.py --- 4.0-2/sos/report/plugins/anaconda.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/anaconda.py 2023-04-28 17:16:21.000000000 +0000 @@ -7,7 +7,6 @@ # See the LICENSE file in the source distribution for further information. from sos.report.plugins import Plugin, RedHatPlugin -import os class Anaconda(Plugin, RedHatPlugin): @@ -29,7 +28,7 @@ class Anaconda(Plugin, RedHatPlugin): "/root/anaconda-ks.cfg" ] - if os.path.isdir('/var/log/anaconda'): + if self.path_isdir('/var/log/anaconda'): # new anaconda paths.append('/var/log/anaconda') else: diff -pruN 4.0-2/sos/report/plugins/anacron.py 4.5.3ubuntu2/sos/report/plugins/anacron.py --- 4.0-2/sos/report/plugins/anacron.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/anacron.py 2023-04-28 17:16:21.000000000 +0000 @@ -6,10 +6,10 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import Plugin, IndependentPlugin -class Anacron(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin): +class Anacron(Plugin, IndependentPlugin): short_desc = 'Anacron job scheduling service' diff -pruN 4.0-2/sos/report/plugins/ansible.py 4.5.3ubuntu2/sos/report/plugins/ansible.py --- 4.0-2/sos/report/plugins/ansible.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/ansible.py 2023-04-28 17:16:21.000000000 +0000 @@ -29,4 +29,7 @@ class Ansible(Plugin, RedHatPlugin, Ubun "ansible --version" ]) + # let rhui plugin collects the RHUI specific files + self.add_forbidden_path("/etc/ansible/facts.d/rhui_*.fact") + # vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/apache.py 4.5.3ubuntu2/sos/report/plugins/apache.py --- 4.0-2/sos/report/plugins/apache.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/apache.py 2023-04-28 17:16:21.000000000 +0000 @@ -6,10 +6,24 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, + UbuntuPlugin, PluginOpt) class Apache(Plugin): + """The Apache plugin covers the upstream Apache webserver project, + regardless of the packaged name; apache2 for Debian and Ubuntu, or httpd + for Red Hat family distributions. + + The aim of this plugin is for Apache-specific information, not necessarily + other projects that happen to place logs or similar files within the + standardized apache directories. For example, OpenStack components that log + to apache logging directories are excluded from this plugin and collected + via their respective OpenStack plugins. + + Users can expect the collection of apachectl command output, apache server + logs, and apache configuration files from this plugin. + """ short_desc = 'Apache http daemon' plugin_name = "apache" @@ -18,23 +32,45 @@ class Apache(Plugin): files = ('/var/www/',) option_list = [ - ("log", "gathers all apache logs", "slow", False) + PluginOpt(name="log", default=False, desc="gathers all apache logs") ] def setup(self): # collect list of installed modules and verify config syntax. self.add_cmd_output([ - "apachectl -M", "apachectl -S", "apachectl -t" - ]) + ], cmd_as_tag=True) + self.add_cmd_output("apachectl -M", tags="httpd_M") - # The foreman plugin collects these files with a greater size limit: + # Other plugins collect these files; # do not collect them here to avoid collisions in the archive paths. - self.add_forbidden_path("/var/log/{}*/foreman*".format(self.apachepkg)) + subdirs = [ + 'aodh', + 'ceilometer', + 'cinder', + 'foreman', + 'horizon', + 'keystone', + 'nova', + 'placement', + 'pulp' + ] + self.add_forbidden_path([ + "/var/log/%s*/%s*" % (self.apachepkg, sub) for sub in subdirs + ]) class RedHatApache(Apache, RedHatPlugin): + """ + On Red Hat distributions, the Apache plugin will also attempt to collect + JBoss Web Server logs and configuration files. + + Note that for Red Hat distributions, this plugin explicitly collects for + 'httpd' installations. If you have installed apache from source or via any + method that uses the name 'apache' instead of 'httpd', these collections + will fail. + """ files = ( '/etc/httpd/conf/httpd.conf', '/etc/httpd22/conf/httpd.conf', @@ -43,50 +79,54 @@ class RedHatApache(Apache, RedHatPlugin) apachepkg = 'httpd' def setup(self): - super(RedHatApache, self).setup() - self.add_copy_spec([ - "/etc/httpd/conf/httpd.conf", - "/etc/httpd/conf.d/*.conf", - "/etc/httpd/conf.modules.d/*.conf", - # JBoss Enterprise Web Server 2.x - "/etc/httpd22/conf/httpd.conf", - "/etc/httpd22/conf.d/*.conf", - # Red Hat JBoss Web Server 3.x - "/etc/httpd24/conf/httpd.conf", - "/etc/httpd24/conf.d/*.conf", - "/etc/httpd24/conf.modules.d/*.conf", - ]) + self.add_file_tags({ + "/var/log/httpd/access_log": 'httpd_access_log', + "/var/log/httpd/error_log": 'httpd_error_log', + "/var/log/httpd/ssl_access_log": 'httpd_ssl_access_log', + "/var/log/httpd/ssl_error_log": 'httpd_ssl_error_log' + }) - self.add_forbidden_path("/etc/httpd/conf/password.conf") + super(RedHatApache, self).setup() - self.add_service_status('httpd') + # httpd versions, including those used for JBoss Web Server + vers = ['', '22', '24'] - # collect only the current log set by default - self.add_copy_spec([ - "/var/log/httpd/access_log", - "/var/log/httpd/error_log", - "/var/log/httpd/ssl_access_log", - "/var/log/httpd/ssl_error_log", - # JBoss Enterprise Web Server 2.x - "/var/log/httpd22/access_log", - "/var/log/httpd22/error_log", - "/var/log/httpd22/ssl_access_log", - "/var/log/httpd22/ssl_error_log", - # Red Hat JBoss Web Server 3.x - "/var/log/httpd24/access_log", - "/var/log/httpd24/error_log", - "/var/log/httpd24/ssl_access_log", - "/var/log/httpd24/ssl_error_log", + # Extrapolate all top-level config directories for each version, and + # relevant config files within each + etcdirs = ["/etc/httpd%s" % ver for ver in vers] + confs = [ + "conf/httpd.conf", + "conf.d/*.conf", + "conf.modules.d/*.conf" + ] + + # Extrapolate top-level logging directories for each version, and the + # relevant log files within each + logdirs = ["/var/log/httpd%s" % ver for ver in vers] + logs = [ + "access_log", + "error_log", + "ssl_access_log", + "ssl_error_log" + ] + + self.add_forbidden_path([ + "%s/conf/password.conf" % etc for etc in etcdirs ]) + + for edir in etcdirs: + for conf in confs: + self.add_copy_spec("%s/%s" % (edir, conf), tags="httpd_conf") + if self.get_option("log") or self.get_option("all_logs"): - self.add_copy_spec([ - "/var/log/httpd/*", - # JBoss Enterprise Web Server 2.x - "/var/log/httpd22/*", - # Red Hat JBoss Web Server 3.x - "/var/log/httpd24/*" - ]) + self.add_copy_spec(logdirs) + else: + for ldir in logdirs: + for log in logs: + self.add_copy_spec("%s/%s" % (ldir, log)) + + self.add_service_status('httpd', tags='systemctl_httpd') class DebianApache(Apache, DebianPlugin, UbuntuPlugin): diff -pruN 4.0-2/sos/report/plugins/apt.py 4.5.3ubuntu2/sos/report/plugins/apt.py --- 4.0-2/sos/report/plugins/apt.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/apt.py 2023-04-28 17:16:21.000000000 +0000 @@ -20,7 +20,9 @@ class Apt(Plugin, DebianPlugin, UbuntuPl def setup(self): self.add_copy_spec([ - "/etc/apt", "/var/log/apt" + "/etc/apt", + "/var/log/apt", + "/var/log/unattended-upgrades" ]) self.add_forbidden_path("/etc/apt/auth.conf") diff -pruN 4.0-2/sos/report/plugins/arcconf.py 4.5.3ubuntu2/sos/report/plugins/arcconf.py --- 4.0-2/sos/report/plugins/arcconf.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/arcconf.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,31 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + + +# This sosreport plugin is meant for sas adapters. +# This plugin logs inforamtion on each adapter it finds. + +from sos.report.plugins import Plugin, IndependentPlugin + + +class arcconf(Plugin, IndependentPlugin): + + short_desc = 'arcconf Integrated RAID adapter information' + + plugin_name = "arcconf" + commands = ("arcconf",) + + def setup(self): + + # get list of adapters + self.add_cmd_output([ + "arcconf getconfig 1", + "arcconf list", + "arcconf GETLOGS 1 UART" + ]) +# vim: et ts=4 sw=4 diff -pruN 4.0-2/sos/report/plugins/ata.py 4.5.3ubuntu2/sos/report/plugins/ata.py --- 4.0-2/sos/report/plugins/ata.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/ata.py 2023-04-28 17:16:21.000000000 +0000 @@ -6,11 +6,10 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin, DebianPlugin -import os +from sos.report.plugins import Plugin, IndependentPlugin -class Ata(Plugin, RedHatPlugin, UbuntuPlugin, DebianPlugin): +class Ata(Plugin, IndependentPlugin): short_desc = 'ATA and IDE information' @@ -20,17 +19,16 @@ class Ata(Plugin, RedHatPlugin, UbuntuPl packages = ('hdparm', 'smartmontools') def setup(self): - dev_path = '/dev' - sys_block = '/sys/block' self.add_copy_spec('/proc/ide') - if os.path.isdir(sys_block): - for disk in os.listdir(sys_block): - if disk.startswith("sd") or disk.startswith("hd"): - disk_path = os.path.join(dev_path, disk) - self.add_cmd_output([ - "hdparm %s" % disk_path, - "smartctl -a %s" % disk_path - ]) + cmd_list = [ + "hdparm %(dev)s", + "smartctl -a %(dev)s", + "smartctl -a %(dev)s -j", + "smartctl -l scterc %(dev)s", + "smartctl -l scterc %(dev)s -j" + ] + self.add_device_cmd(cmd_list, devices='block', + whitelist=['sd.*', 'hd.*']) # vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/atomichost.py 4.5.3ubuntu2/sos/report/plugins/atomichost.py --- 4.0-2/sos/report/plugins/atomichost.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/atomichost.py 2023-04-28 17:16:21.000000000 +0000 @@ -8,7 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt class AtomicHost(Plugin, RedHatPlugin): @@ -18,7 +18,8 @@ class AtomicHost(Plugin, RedHatPlugin): plugin_name = "atomichost" profiles = ('container',) option_list = [ - ("info", "gather atomic info for each image", "fast", False) + PluginOpt("info", default=False, + desc="gather atomic info for each image") ] def check_enabled(self): diff -pruN 4.0-2/sos/report/plugins/auditd.py 4.5.3ubuntu2/sos/report/plugins/auditd.py --- 4.0-2/sos/report/plugins/auditd.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/auditd.py 2023-04-28 17:16:21.000000000 +0000 @@ -6,10 +6,10 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import Plugin, IndependentPlugin -class Auditd(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin): +class Auditd(Plugin, IndependentPlugin): short_desc = 'Audit daemon information' @@ -22,18 +22,35 @@ class Auditd(Plugin, RedHatPlugin, Debia self.add_copy_spec([ "/etc/audit/auditd.conf", "/etc/audit/audit.rules", + "/etc/audit/audit-stop.rules", + "/etc/audit/rules.d/", "/etc/audit/plugins.d/", "/etc/audisp/", ]) - self.add_cmd_output([ - "ausearch --input-logs -m avc,user_avc -ts today", - "auditctl -s", - "auditctl -l" - ]) + + self.add_cmd_output( + "ausearch --input-logs -m avc,user_avc,fanotify -ts today" + ) + self.add_cmd_output("auditctl -l", tags="auditctl_rules") + self.add_cmd_output("auditctl -s", tags="auditctl_status") + + config_file = "/etc/audit/auditd.conf" + log_file = "/var/log/audit/audit.log" + try: + with open(config_file, 'r') as cf: + for line in cf.read().splitlines(): + if not line: + continue + words = line.split('=') + if words[0].strip() == 'log_file': + log_file = words[1].strip() + except IOError as error: + self._log_error('Could not open conf file %s: %s' % + (config_file, error)) if not self.get_option("all_logs"): - self.add_copy_spec("/var/log/audit/audit.log") + self.add_copy_spec(log_file) else: - self.add_copy_spec("/var/log/audit") + self.add_copy_spec(log_file+'*') # vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/autofs.py 4.5.3ubuntu2/sos/report/plugins/autofs.py --- 4.0-2/sos/report/plugins/autofs.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/autofs.py 2023-04-28 17:16:21.000000000 +0000 @@ -43,6 +43,7 @@ class Autofs(Plugin): def setup(self): self.add_copy_spec("/etc/auto*") + self.add_file_tags({"/etc/autofs.conf": "autofs_conf"}) self.add_service_status("autofs") self.add_cmd_output("automount -m") if self.checkdebug(): @@ -54,6 +55,25 @@ class Autofs(Plugin): r"(password=)[^,\s]*", r"\1********" ) + # Hide secrets in the LDAP authentication config + # + # Example of scrubbing of the secret: + # + # secret="abc" + # or + # encoded_secret = 'abc' + # + # to: + # + # secret="********" + # or + # encoded_secret = '********' + # + self.do_file_sub( + "/etc/autofs_ldap_auth.conf", + r"(secret[\s]*[=]+[\s]*)(\'|\").*(\'|\")", + r"\1\2********\3" + ) self.do_cmd_output_sub( "automount -m", r"(password=)[^,\s]*", diff -pruN 4.0-2/sos/report/plugins/azure.py 4.5.3ubuntu2/sos/report/plugins/azure.py --- 4.0-2/sos/report/plugins/azure.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/azure.py 2023-04-28 17:16:21.000000000 +0000 @@ -8,8 +8,8 @@ # # See the LICENSE file in the source distribution for further information. -import os from sos.report.plugins import Plugin, UbuntuPlugin, RedHatPlugin +import os class Azure(Plugin, UbuntuPlugin): @@ -27,7 +27,8 @@ class Azure(Plugin, UbuntuPlugin): "/etc/default/kv-kvp-daemon-init", "/etc/waagent.conf", "/sys/module/hv_netvsc/parameters/ring_size", - "/sys/module/hv_storvsc/parameters/storvsc_ringbuffer_size" + "/sys/module/hv_storvsc/parameters/storvsc_ringbuffer_size", + "/var/lib/AzureEnhancedMonitor" ]) # Adds all files under /var/log/azure to the sosreport @@ -37,12 +38,12 @@ class Azure(Plugin, UbuntuPlugin): for path, subdirs, files in os.walk("/var/log/azure"): for name in files: - self.add_copy_spec(os.path.join(path, name), sizelimit=limit) + self.add_copy_spec(self.path_join(path, name), sizelimit=limit) self.add_cmd_output(( - 'curl -s -H Metadata:true ' + 'curl -s -H Metadata:true --noproxy "*" ' '"http://169.254.169.254/metadata/instance/compute?' - 'api-version=2019-11-01"' + 'api-version=2021-01-01&format=json"' ), suggest_filename='instance_metadata.json') @@ -51,7 +52,7 @@ class RedHatAzure(Azure, RedHatPlugin): def setup(self): super(RedHatAzure, self).setup() - if os.path.isfile('/etc/yum.repos.d/rh-cloud.repo'): + if self.path_isfile('/etc/yum.repos.d/rh-cloud.repo'): curl_cmd = ('curl -s -m 5 -vvv ' 'https://rhui-%s.microsoft.com/pulp/repos/%s') self.add_cmd_output([ @@ -61,7 +62,7 @@ class RedHatAzure(Azure, RedHatPlugin): ]) crt_path = '/etc/pki/rhui/product/content.crt' - if os.path.isfile(crt_path): + if self.path_isfile(crt_path): self.add_cmd_output([ 'openssl x509 -noout -text -in ' + crt_path ]) diff -pruN 4.0-2/sos/report/plugins/bcache.py 4.5.3ubuntu2/sos/report/plugins/bcache.py --- 4.0-2/sos/report/plugins/bcache.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/bcache.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,56 @@ +# Copyright (C) 2021, Canonical ltd +# Ponnuvel Palaniyappan + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, IndependentPlugin, SoSPredicate + + +class Bcache(Plugin, IndependentPlugin): + + short_desc = 'Bcache statistics' + + plugin_name = 'bcache' + profiles = ('storage', 'hardware') + files = ('/sys/fs/bcache',) + + def setup(self): + + # Caution: reading /sys/fs/bcache/*/cache0/priority_stats is known + # to degrade performance on old kernels. Needs care if that's ever + # considered for inclusion here. + # see: https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1840043 + self.add_forbidden_path([ + '/sys/fs/bcache/*/*/priority_stats', + ]) + + self.add_copy_spec([ + '/sys/block/bcache*/bcache/cache/internal/copy_gc_enabled', + '/sys/block/bcache*/bcache/cache_mode', + '/sys/block/bcache*/bcache/dirty_data', + '/sys/block/bcache*/bcache/io_errors', + '/sys/block/bcache*/bcache/sequential_cutoff', + '/sys/block/bcache*/bcache/stats_hour/bypassed', + '/sys/block/bcache*/bcache/stats_hour/cache_hit_ratio', + '/sys/block/bcache*/bcache/stats_hour/cache_hits', + '/sys/block/bcache*/bcache/stats_hour/cache_misses', + '/sys/block/bcache*/bcache/writeback_percent', + '/sys/fs/bcache/*/average_key_size', + '/sys/fs/bcache/*/bdev*/*', + '/sys/fs/bcache/*/bdev*/stat_*/*', + '/sys/fs/bcache/*/block_size', + '/sys/fs/bcache/*/bucket_size', + '/sys/fs/bcache/*/cache_available_percent', + '/sys/fs/bcache/*/congested_*_threshold_us', + '/sys/fs/bcache/*/internal/*', + '/sys/fs/bcache/*/stats_*/*', + '/sys/fs/bcache/*/tree_depth', + ], pred=SoSPredicate(self, kmods=['bcache'])) + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/block.py 4.5.3ubuntu2/sos/report/plugins/block.py --- 4.0-2/sos/report/plugins/block.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/block.py 2023-04-28 17:16:21.000000000 +0000 @@ -6,10 +6,10 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import Plugin, IndependentPlugin -class Block(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin): +class Block(Plugin, IndependentPlugin): short_desc = 'Block device information' @@ -21,14 +21,20 @@ class Block(Plugin, RedHatPlugin, Debian def setup(self): self.add_forbidden_path("/sys/block/*/queue/iosched") + self.add_file_tags({ + '/sys/block/.*/queue/scheduler': 'scheduler' + }) + + self.add_cmd_output("blkid -c /dev/null", tags="blkid") + self.add_cmd_output("ls -lanR /dev", tags="ls_dev") + self.add_cmd_output("lsblk", tags="lsblk") + self.add_cmd_output("lsblk -O -P", tags="lsblk_pairs") self.add_cmd_output([ - "lsblk", "lsblk -t", "lsblk -D", - "blkid -c /dev/null", "blockdev --report", - "ls -lanR /dev", - "ls -lanR /sys/block" + "ls -lanR /sys/block", + "losetup -a", ]) # legacy location for non-/run distributions @@ -37,16 +43,21 @@ class Block(Plugin, RedHatPlugin, Debian "/run/blkid/blkid.tab", "/proc/partitions", "/proc/diskstats", - "/sys/block/*/queue/" + "/sys/block/*/queue/", + "/sys/block/sd*/device/timeout", + "/sys/block/hd*/device/timeout", + "/sys/block/sd*/device/state", + "/sys/block/loop*/loop/", ]) cmds = [ "parted -s %(dev)s unit s print", - "fdisk -l %(dev)s", "udevadm info %(dev)s", "udevadm info -a %(dev)s" ] - self.add_blockdev_cmd(cmds, blacklist='ram.*') + self.add_device_cmd(cmds, devices='block', blacklist='ram.*') + self.add_device_cmd("fdisk -l %(dev)s", blacklist="ram.*", + devices="block", tags="fdisk_l_sos") lsblk = self.collect_cmd_output("lsblk -f -a -l") # for LUKS devices, collect cryptsetup luksDump diff -pruN 4.0-2/sos/report/plugins/boot.py 4.5.3ubuntu2/sos/report/plugins/boot.py --- 4.0-2/sos/report/plugins/boot.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/boot.py 2023-04-28 17:16:21.000000000 +0000 @@ -6,11 +6,11 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt from glob import glob -class Boot(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin): +class Boot(Plugin, IndependentPlugin): short_desc = 'Bootloader information' @@ -19,7 +19,8 @@ class Boot(Plugin, RedHatPlugin, DebianP packages = ('grub', 'grub2', 'grub-common', 'grub2-common', 'zipl') option_list = [ - ("all-images", "collect lsinitrd for all images", "slow", False) + PluginOpt("all-images", default=False, + desc="collect lsinitrd for all images") ] def setup(self): @@ -31,21 +32,27 @@ class Boot(Plugin, RedHatPlugin, DebianP "/etc/yaboot.conf", "/boot/yaboot.conf" ]) - self.add_cmd_output([ - "ls -lanR /boot", - "lsinitrd" - ]) + + self.add_cmd_output("ls -lanR /boot", tags="ls_boot") + self.add_cmd_output("ls -lanR /sys/firmware", + tags="ls_sys_firmware") + self.add_cmd_output("lsinitrd", tags="lsinitrd") + self.add_cmd_output("mokutil --sb-state", + tags="mokutil_sbstate") self.add_cmd_output([ "efibootmgr -v", - "mokutil --sb-state" + "ls -l /initrd.img /boot/initrd.img", + "lsinitramfs -l /initrd.img", + "lsinitramfs -l /boot/initrd.img" ]) if self.get_option("all-images"): - for image in glob('/boot/initr*.img'): + for image in glob('/boot/initr*.img*'): if image[-9:] == "kdump.img": continue - self.add_cmd_output("lsinitrd %s" % image) + self.add_cmd_output("lsinitrd %s" % image, priority=100) + self.add_cmd_output("lsinitramfs -l %s" % image, priority=100) # vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/btrfs.py 4.5.3ubuntu2/sos/report/plugins/btrfs.py --- 4.0-2/sos/report/plugins/btrfs.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/btrfs.py 2023-04-28 17:16:21.000000000 +0000 @@ -6,10 +6,10 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin, DebianPlugin +from sos.report.plugins import Plugin, IndependentPlugin -class Btrfs(Plugin, RedHatPlugin, UbuntuPlugin, DebianPlugin): +class Btrfs(Plugin, IndependentPlugin): short_desc = 'Btrfs filesystem' diff -pruN 4.0-2/sos/report/plugins/candlepin.py 4.5.3ubuntu2/sos/report/plugins/candlepin.py --- 4.0-2/sos/report/plugins/candlepin.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/candlepin.py 2023-04-28 17:16:21.000000000 +0000 @@ -44,6 +44,13 @@ class Candlepin(Plugin, RedHatPlugin): except (IOError, IndexError): # fallback when the cfg file is not accessible or parseable pass + + self.add_file_tags({ + '/var/log/candlepin/candlepin.log.*': 'candlepin_log', + '/var/log/candlepin/err.log.*': 'candlepin_error_log', + '/etc/candlepin/candlepin.conf': 'candlepin_conf' + }) + # set the password to os.environ when calling psql commands to prevent # printing it in sos logs # we can't set os.environ directly now: other plugins can overwrite it @@ -58,32 +65,48 @@ class Candlepin(Plugin, RedHatPlugin): # Allow limiting on logrotated logs self.add_copy_spec([ "/etc/candlepin/candlepin.conf", + "/etc/candlepin/broker.xml", "/var/log/candlepin/audit*.log*", "/var/log/candlepin/candlepin.log[.-]*", "/var/log/candlepin/cpdb*.log*", "/var/log/candlepin/cpinit*.log*", - "/var/log/candlepin/error.log[.-]*" + "/var/log/candlepin/error.log[.-]*", + # Specific to candlepin, ALL catalina logs are relevant. Adding it + # here rather than the tomcat plugin to ease maintenance and not + # pollute non-candlepin sosreports that enable the tomcat plugin + "/var/log/tomcat*/catalina*log*", + "/var/log/tomcat*/host-manager*log*", + "/var/log/tomcat*/localhost*log*", + "/var/log/tomcat*/manager*log*", ]) self.add_cmd_output("du -sh /var/lib/candlepin/*/*") # collect tables sizes, ordered - _cmd = self.build_query_cmd("\ - SELECT schema_name, relname, \ - pg_size_pretty(table_size) AS size, table_size \ - FROM ( \ - SELECT \ - pg_catalog.pg_namespace.nspname AS schema_name, \ - relname, \ - pg_relation_size(pg_catalog.pg_class.oid) AS table_size \ - FROM pg_catalog.pg_class \ - JOIN pg_catalog.pg_namespace \ - ON relnamespace = pg_catalog.pg_namespace.oid \ - ) t \ - WHERE schema_name NOT LIKE 'pg_%' \ - ORDER BY table_size DESC;") + _cmd = self.build_query_cmd( + "SELECT table_name, pg_size_pretty(total_bytes) AS total, " + "pg_size_pretty(index_bytes) AS INDEX , " + "pg_size_pretty(toast_bytes) AS toast, pg_size_pretty(table_bytes)" + " AS TABLE FROM ( SELECT *, " + "total_bytes-index_bytes-COALESCE(toast_bytes,0) AS table_bytes " + "FROM (SELECT c.oid,nspname AS table_schema, relname AS " + "TABLE_NAME, c.reltuples AS row_estimate, " + "pg_total_relation_size(c.oid) AS total_bytes, " + "pg_indexes_size(c.oid) AS index_bytes, " + "pg_total_relation_size(reltoastrelid) AS toast_bytes " + "FROM pg_class c LEFT JOIN pg_namespace n ON " + "n.oid = c.relnamespace WHERE relkind = 'r') a) a order by " + "total_bytes DESC" + ) self.add_cmd_output(_cmd, suggest_filename='candlepin_db_tables_sizes', env=self.env) + _cmd = self.build_query_cmd("\ + SELECT displayname, content_access_mode \ + FROM cp_owner;") + self.add_cmd_output(_cmd, + suggest_filename='simple_content_access', + env=self.env) + def build_query_cmd(self, query, csv=False): """ Builds the command needed to invoke the pgsql query as the postgres @@ -93,7 +116,8 @@ class Candlepin(Plugin, RedHatPlugin): a large amount of quoting in sos logs referencing the command being run """ csvformat = "-A -F , -X" if csv else "" - _dbcmd = "psql -h %s -p 5432 -U candlepin -d candlepin %s -c %s" + _dbcmd = "psql --no-password -h %s -p 5432 -U candlepin \ + -d candlepin %s -c %s" return _dbcmd % (self.dbhost, csvformat, quote(query)) def postproc(self): @@ -102,5 +126,9 @@ class Candlepin(Plugin, RedHatPlugin): self.do_file_sub("/etc/candlepin/candlepin.conf", reg, repl) cpdbreg = r"(--password=)([a-zA-Z0-9]*)" self.do_file_sub("/var/log/candlepin/cpdb.log", cpdbreg, repl) + for key in ["trustStorePassword", "keyStorePassword"]: + self.do_file_sub("/etc/candlepin/broker.xml", + r"(%s)=(\w*)([;<])" % key, + r"\1=********\3") # vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/ceph.py 4.5.3ubuntu2/sos/report/plugins/ceph.py --- 4.0-2/sos/report/plugins/ceph.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/ceph.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,135 +0,0 @@ -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information. - -from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin -from socket import gethostname - - -class Ceph(Plugin, RedHatPlugin, UbuntuPlugin): - - short_desc = 'CEPH distributed storage' - - plugin_name = 'ceph' - profiles = ('storage', 'virt') - ceph_hostname = gethostname() - - packages = ( - 'ceph', - 'ceph-mds', - 'ceph-common', - 'libcephfs1', - 'ceph-fs-common', - 'calamari-server', - 'librados2' - ) - - services = ( - 'ceph-nfs@pacemaker', - 'ceph-mds@%s' % ceph_hostname, - 'ceph-mon@%s' % ceph_hostname, - 'ceph-mgr@%s' % ceph_hostname, - 'ceph-radosgw@*', - 'ceph-osd@*' - ) - - def setup(self): - all_logs = self.get_option("all_logs") - - if not all_logs: - self.add_copy_spec([ - "/var/log/ceph/*.log", - "/var/log/radosgw/*.log", - "/var/log/calamari/*.log" - ]) - else: - self.add_copy_spec([ - "/var/log/ceph/", - "/var/log/calamari", - "/var/log/radosgw" - ]) - - self.add_copy_spec([ - "/etc/ceph/", - "/etc/calamari/", - "/var/lib/ceph/", - "/run/ceph/" - ]) - - self.add_cmd_output([ - "ceph mon stat", - "ceph mon_status", - "ceph quorum_status", - "ceph mgr module ls", - "ceph mgr metadata", - "ceph osd metadata", - "ceph osd erasure-code-profile ls", - "ceph report", - "ceph osd crush show-tunables", - "ceph-disk list", - "ceph versions", - "ceph features", - "ceph insights", - "ceph osd crush dump", - "ceph -v", - "ceph-volume lvm list", - "ceph crash stat", - "ceph crash ls", - "ceph config log", - "ceph config generate-minimal-conf", - "ceph config-key dump", - ]) - - ceph_cmds = [ - "status", - "health detail", - "osd tree", - "osd stat", - "osd df tree", - "osd dump", - "osd df", - "osd perf", - "osd blocked-by", - "osd pool ls detail", - "osd numa-status", - "device ls", - "mon dump", - "mgr dump", - "mds stat", - "df", - "df detail", - "fs ls", - "fs dump", - "pg dump", - "pg stat", - ] - - self.add_cmd_output([ - "ceph %s" % s for s in ceph_cmds - ]) - - self.add_cmd_output([ - "ceph %s --format json-pretty" % s for s in ceph_cmds - ], subdir="json_output") - - for service in self.services: - self.add_journal(units=service) - - self.add_forbidden_path([ - "/etc/ceph/*keyring*", - "/var/lib/ceph/*keyring*", - "/var/lib/ceph/*/*keyring*", - "/var/lib/ceph/*/*/*keyring*", - "/var/lib/ceph/osd", - "/var/lib/ceph/mon", - # Excludes temporary ceph-osd mount location like - # /var/lib/ceph/tmp/mnt.XXXX from sos collection. - "/var/lib/ceph/tmp/*mnt*", - "/etc/ceph/*bindpass*" - ]) - -# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/ceph_common.py 4.5.3ubuntu2/sos/report/plugins/ceph_common.py --- 4.0-2/sos/report/plugins/ceph_common.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/ceph_common.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,85 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin +from socket import gethostname + + +class Ceph_Common(Plugin, RedHatPlugin, UbuntuPlugin): + + short_desc = 'CEPH common' + + plugin_name = 'ceph_common' + profiles = ('storage', 'virt', 'container') + + containers = ('ceph-(.*-)?(mon|rgw|osd).*',) + ceph_hostname = gethostname() + + packages = ( + 'ceph', + 'ceph-mds', + 'ceph-common', + 'libcephfs1', + 'ceph-fs-common', + 'calamari-server', + 'librados2' + ) + + services = ( + 'ceph-nfs@pacemaker', + 'ceph-mds@%s' % ceph_hostname, + 'ceph-mon@%s' % ceph_hostname, + 'ceph-mgr@%s' % ceph_hostname, + 'ceph-radosgw@*', + 'ceph-osd@*' + ) + + # This check will enable the plugin regardless of being + # containerized or not + files = ('/etc/ceph/ceph.conf',) + + def setup(self): + all_logs = self.get_option("all_logs") + + self.add_file_tags({ + '.*/ceph.conf': 'ceph_conf', + '/var/log/ceph(.*)?/ceph.log.*': 'ceph_log', + }) + + if not all_logs: + self.add_copy_spec("/var/log/calamari/*.log",) + else: + self.add_copy_spec("/var/log/calamari",) + + self.add_copy_spec([ + "/var/log/ceph/**/ceph.log", + "/var/log/ceph/**/ceph.audit.log*", + "/var/log/calamari/*.log", + "/etc/ceph/", + "/etc/calamari/", + "/var/lib/ceph/tmp/", + ]) + + self.add_cmd_output([ + "ceph -v", + ]) + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/*keyring*", + "/var/lib/ceph/*/*keyring*", + "/var/lib/ceph/*/*/*keyring*", + "/var/lib/ceph/osd", + "/var/lib/ceph/mon", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/tmp/*mnt*", + "/etc/ceph/*bindpass*" + ]) + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/ceph_iscsi.py 4.5.3ubuntu2/sos/report/plugins/ceph_iscsi.py --- 4.0-2/sos/report/plugins/ceph_iscsi.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/ceph_iscsi.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,37 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin + + +class CephISCSI(Plugin, RedHatPlugin, UbuntuPlugin): + + short_desc = "CEPH iSCSI" + + plugin_name = "ceph_iscsi" + profiles = ("storage", "virt", "container") + packages = ("ceph-iscsi",) + services = ("rbd-target-api", "rbd-target-gw") + containers = ("rbd-target-api.*", "rbd-target-gw.*") + + def setup(self): + self.add_copy_spec([ + "/etc/tcmu/tcmu.conf", + "/var/log/**/ceph-client.*.log", + "/var/log/**/rbd-target-api.log", + "/var/log/**/rbd-target-gw.log", + "/var/log/**/tcmu-runner.log", + "/var/log/tcmu-runner.log" + ]) + + self.add_cmd_output([ + "gwcli info", + "gwcli ls" + ]) + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/ceph_mds.py 4.5.3ubuntu2/sos/report/plugins/ceph_mds.py --- 4.0-2/sos/report/plugins/ceph_mds.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/ceph_mds.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,94 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin + + +class CephMDS(Plugin, RedHatPlugin, UbuntuPlugin): + short_desc = 'CEPH mds' + plugin_name = 'ceph_mds' + profiles = ('storage', 'virt', 'container') + containers = ('ceph-(.*-)?fs.*',) + files = ('/var/lib/ceph/mds/',) + + def setup(self): + self.add_file_tags({ + '/var/log/ceph/ceph-mds.*.log': 'ceph_mds_log', + }) + + self.add_copy_spec([ + "/var/log/ceph/ceph-mds*.log", + "/var/lib/ceph/bootstrap-mds/", + "/var/lib/ceph/mds/", + "/run/ceph/ceph-mds*", + ]) + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/*keyring*", + "/var/lib/ceph/*/*keyring*", + "/var/lib/ceph/*/*/*keyring*", + "/var/lib/ceph/osd", + "/var/lib/ceph/mon", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/tmp/*mnt*", + "/etc/ceph/*bindpass*" + ]) + + ceph_cmds = [ + "cache status", + "client ls", + "config diff", + "config show", + "damage ls", + "dump loads", + "dump tree", + "dump_blocked_ops", + "dump_historic_ops", + "dump_historic_ops_by_duration", + "dump_mempools", + "dump_ops_in_flight", + "get subtrees", + "objecter_requests", + "ops", + "perf histogram dump", + "perf histogram schema", + "perf schema", + "perf dump", + "status", + "version", + "session ls" + ] + + mds_ids = [] + # Get the ceph user processes + out = self.exec_cmd('ps -u ceph -o args') + + if out['status'] == 0: + # Extract the OSD ids from valid output lines + for procs in out['output'].splitlines(): + proc = procs.split() + if len(proc) < 6: + continue + if proc[4] == '--id' and "ceph-mds" in proc[0]: + mds_ids.append("mds.%s" % proc[5]) + + # If containerized, run commands in containers + try: + cname = self.get_all_containers_by_regex("ceph-mds*")[0][1] + except Exception: + cname = None + + self.add_cmd_output([ + "ceph daemon %s %s" + % (mdsid, cmd) for mdsid in mds_ids for cmd in ceph_cmds + ], container=cname) + + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/ceph_mgr.py 4.5.3ubuntu2/sos/report/plugins/ceph_mgr.py --- 4.0-2/sos/report/plugins/ceph_mgr.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/ceph_mgr.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,122 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import os + +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin + + +class CephMGR(Plugin, RedHatPlugin, UbuntuPlugin): + """ + This plugin is for capturing information from Ceph mgr nodes. While the + majority of this plugin should be version-agnostic, several collections are + dependent upon the version of Ceph installed. Versions that correlate to + RHCS 4 or RHCS 5 are explicitly handled for differences such as those + pertaining to log locations on the host filesystem. + + Note that while this plugin will activate based on the presence of Ceph + containers, commands are run directly on the host as those containers are + often not configured to successfully run the `ceph` commands collected by + this plugin. These commands are majorily `ceph daemon` commands that will + reference discovered admin sockets under /var/run/ceph. + + Users may expect to see several collections twice - once in standard output + from the `ceph` command, and again in JSON format. The latter of which will + be placed in the `json_output/` subdirectory within this plugin's directory + in the report archive. These JSON formatted collections are intended to + aid in automated analysis. + """ + + short_desc = 'CEPH mgr' + + plugin_name = 'ceph_mgr' + profiles = ('storage', 'virt', 'container') + files = ('/var/lib/ceph/mgr/', '/var/lib/ceph/*/mgr*') + containers = ('ceph-(.*-)?mgr.*',) + + def setup(self): + + self.add_file_tags({ + '/var/log/ceph/(.*/)?ceph-mgr.*.log': 'ceph_mgr_log', + }) + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/**/*keyring*", + "/var/lib/ceph/**/osd*", + "/var/lib/ceph/**/mon*", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/**/tmp/*mnt*", + "/etc/ceph/*bindpass*", + ]) + + self.add_copy_spec([ + "/var/log/ceph/**/ceph-mgr*.log", + "/var/lib/ceph/**/mgr*", + "/var/lib/ceph/**/bootstrap-mgr/", + "/run/ceph/**/ceph-mgr*", + ]) + + # more commands to be added later + ceph_mgr_cmds = ([ + "balancer status", + "orch host ls", + "orch device ls", + "orch ls", + "orch ls --export", + "orch ps", + "orch status --detail", + "orch upgrade status", + "log last cephadm" + ]) + + self.add_cmd_output( + [f"ceph {cmd}" for cmd in ceph_mgr_cmds]) + # get ceph_cmds again as json for easier automation parsing + self.add_cmd_output( + [f"ceph {cmd} --format json-pretty" for cmd in ceph_mgr_cmds], + subdir="json_output", + ) + + cmds = [ + "config diff", + "config show", + "dump_cache", + "dump_mempools", + "dump_osd_network", + "mds_requests", + "mds_sessions", + "objecter_requests", + "mds_requests", + "mds_sessions", + "perf dump", + "perf histogram dump", + "perf histogram schema", + "perf schema", + "status", + "version" + ] + + self.add_cmd_output([ + f"ceph daemon {m} {cmd}" for m in self.get_socks() for cmd in cmds] + ) + + def get_socks(self): + """ + Find any available admin sockets under /var/run/ceph (or subdirs for + later versions of Ceph) which can be used for ceph daemon commands + """ + ceph_sockets = [] + for rdir, dirs, files in os.walk('/var/run/ceph/'): + for file in files: + if file.startswith('ceph-mgr') and file.endswith('.asok'): + ceph_sockets.append(self.path_join(rdir, file)) + return ceph_sockets + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/ceph_mon.py 4.5.3ubuntu2/sos/report/plugins/ceph_mon.py --- 4.0-2/sos/report/plugins/ceph_mon.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/ceph_mon.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,225 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import re + +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin + + +class CephMON(Plugin, RedHatPlugin, UbuntuPlugin): + """ + This plugin serves to collect information on monitor nodes within a Ceph + cluster. It is designed to collect from several versions of Ceph, including + those versions that serve as the basis for RHCS 4 and RHCS 5. + + Older versions of Ceph will have collections from locations such as + /var/log/ceph, whereas newer versions (as of this plugin's latest update) + will have collections from /var/log/ceph//. This plugin attempts to + account for this where possible across the host's filesystem. + + Users may expect to see several collections twice - once in standard output + from the `ceph` command, and again in JSON format. The latter of which will + be placed in the `json_output/` subdirectory within this plugin's directory + in the report archive. These JSON formatted collections are intended to + aid in automated analysis. + """ + + short_desc = 'CEPH mon' + + plugin_name = 'ceph_mon' + profiles = ('storage', 'virt', 'container') + # note: for RHCS 5 / Ceph v16 the containers serve as an enablement trigger + # but by default they are not capable of running various ceph commands in + # this plugin - the `ceph` binary is functional directly on the host + containers = ('ceph-(.*-)?mon.*',) + files = ('/var/lib/ceph/mon/', '/var/lib/ceph/*/mon*') + ceph_version = 0 + + def setup(self): + + self.ceph_version = self.get_ceph_version() + + self.add_file_tags({ + '.*/ceph.conf': 'ceph_conf', + "/var/log/ceph/(.*/)?ceph-.*mon.*.log": 'ceph_mon_log' + }) + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/**/*keyring*", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/**/tmp/*mnt*", + "/etc/ceph/*bindpass*" + ]) + + self.add_copy_spec([ + "/run/ceph/**/ceph-mon*", + "/var/lib/ceph/**/kv_backend", + "/var/log/ceph/**/*ceph-mon*.log" + ]) + + self.add_cmd_output("ceph report", tags="ceph_report") + self.add_cmd_output([ + # The ceph_mon plugin will collect all the "ceph ..." commands + # which typically require the keyring. + + "ceph mon stat", + "ceph quorum_status", + "ceph-disk list", + "ceph versions", + "ceph features", + "ceph insights", + "ceph crash stat", + "ceph config dump", + "ceph config log", + "ceph config generate-minimal-conf", + "ceph config-key dump", + "ceph osd metadata", + "ceph osd erasure-code-profile ls", + "ceph osd crush dump", + "ceph osd crush show-tunables", + "ceph osd crush tree --show-shadow", + "ceph mgr dump", + "ceph mgr metadata", + "ceph mgr module ls", + "ceph mgr services", + "ceph mgr versions", + "ceph log last 10000 debug cluster", + "ceph log last 10000 debug audit" + ]) + + crashes = self.collect_cmd_output('ceph crash ls') + if crashes['status'] == 0: + for crashln in crashes['output'].splitlines(): + if crashln.endswith('*'): + cid = crashln.split()[0] + self.add_cmd_output(f"ceph crash info {cid}") + + ceph_cmds = [ + "mon dump", + "status", + "device ls", + "df", + "df detail", + "fs ls", + "fs dump", + "pg dump", + "pg stat", + "time-sync-status", + "osd stat", + "osd df tree", + "osd dump", + "osd df", + "osd perf", + "osd blocked-by", + "osd pool ls detail", + "osd pool autoscale-status", + "mds stat", + "osd numa-status" + ] + + self.add_cmd_output("ceph health detail --format json-pretty", + subdir="json_output", + tags="ceph_health_detail") + self.add_cmd_output("ceph osd tree --format json-pretty", + subdir="json_output", + tags="ceph_osd_tree") + self.add_cmd_output( + [f"ceph tell mon.{mid} mon_status" for mid in self.get_ceph_ids()], + subdir="json_output", + ) + + self.add_cmd_output([f"ceph {cmd}" for cmd in ceph_cmds]) + + # get ceph_cmds again as json for easier automation parsing + self.add_cmd_output( + [f"ceph {cmd} --format json-pretty" for cmd in ceph_cmds], + subdir="json_output", + ) + + def get_ceph_version(self): + ver = self.exec_cmd('ceph --version') + if ver['status'] == 0: + try: + _ver = ver['output'].split()[2] + return int(_ver.split('.')[0]) + except Exception as err: + self._log_debug(f"Could not determine ceph version: {err}") + self._log_error( + 'Failed to find ceph version, command collection will be limited' + ) + return 0 + + def get_ceph_ids(self): + ceph_ids = [] + # ceph version 14 correlates to RHCS 4 + if self.ceph_version == 14 or self.ceph_version == 15: + # Get the ceph user processes + out = self.exec_cmd('ps -u ceph -o args') + + if out['status'] == 0: + # Extract the mon ids + for procs in out['output'].splitlines(): + proc = procs.split() + # Locate the '--id' value of the process + if proc and proc[0].endswith("ceph-mon"): + try: + id_index = proc.index("--id") + ceph_ids.append(proc[id_index + 1]) + except (IndexError, ValueError): + self._log_warn('Unable to find ceph IDs') + + # ceph version 16 is RHCS 5 + elif self.ceph_version >= 16: + stats = self.exec_cmd('ceph status') + if stats['status'] == 0: + try: + ret = re.search(r'(\s*mon: .* quorum) (.*) (\(.*\))', + stats['output']) + ceph_ids.extend(ret.groups()[1].split(',')) + except Exception as err: + self._log_debug(f"id determination failed: {err}") + return ceph_ids + + def postproc(self): + + if self.ceph_version >= 16: + keys = [ + 'key', + 'username', + 'password', + '_secret', + 'rbd/mirror/peer/.*' + ] + # we need to do this iteratively, as config-key dump here contains + # nested json data written as strings, which may have multiple hits + # within the same line + for key in keys: + creg = fr'(((.*)({key}\\\": ))((\\\"(.*?)\\\")(.*)))' + self.do_cmd_output_sub( + 'ceph config-key dump', creg, r'\2\"******\"\8' + ) + else: + keys = [ + 'API_PASSWORD', + 'API_USER.*', + 'API_.*_KEY', + 'key', + '_secret', + 'rbd/mirror/peer/.*' + ] + + creg = fr"((\".*({'|'.join(keys)})\":) \")(.*)(\".*)" + self.do_cmd_output_sub( + 'ceph config-key dump', creg, r'\1*******\5' + ) + + self.do_cmd_private_sub('ceph config-key dump') + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/ceph_osd.py 4.5.3ubuntu2/sos/report/plugins/ceph_osd.py --- 4.0-2/sos/report/plugins/ceph_osd.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/ceph_osd.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,103 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import os + +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin + + +class CephOSD(Plugin, RedHatPlugin, UbuntuPlugin): + """ + This plugin is for capturing information from Ceph OSD nodes. While the + majority of this plugin should be version agnotics, several collections are + dependent upon the version of Ceph installed. Versions that correlate to + RHCS 4 or RHCS 5 are explicitly handled for differences such as those + pertaining to log locations on the host filesystem. + + Note that while this plugin will activate based on the presence of Ceph + containers, commands are run directly on the host as those containers are + often not configured to successfully run the `ceph` commands collected by + this plugin. These commands are majorily `ceph daemon` commands that will + reference discovered admin sockets under /var/run/ceph. + """ + + short_desc = 'CEPH osd' + + plugin_name = 'ceph_osd' + profiles = ('storage', 'virt', 'container') + containers = ('ceph-(.*-)?osd.*',) + files = ('/var/lib/ceph/osd/', '/var/lib/ceph/*/osd*') + + def setup(self): + + self.add_file_tags({ + "/var/log/ceph/(.*/)?ceph-(.*-)?osd.*.log": 'ceph_osd_log', + }) + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/**/*keyring*", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/**/tmp/*mnt*", + "/etc/ceph/*bindpass*" + ]) + + # Only collect OSD specific files + self.add_copy_spec([ + "/run/ceph/**/ceph-osd*", + "/var/lib/ceph/**/kv_backend", + "/var/log/ceph/**/ceph-osd*.log", + "/var/log/ceph/**/ceph-volume*.log", + ]) + + self.add_cmd_output([ + "ceph-disk list", + "ceph-volume lvm list" + ]) + + cmds = [ + "bluestore bluefs available", + "config diff", + "config show", + "dump_blacklist", + "dump_blocked_ops", + "dump_historic_ops_by_duration", + "dump_historic_slow_ops", + "dump_mempools", + "dump_ops_in_flight", + "dump_op_pq_state", + "dump_osd_network", + "dump_reservations", + "dump_watchers", + "log dump", + "perf dump", + "perf histogram dump", + "objecter_requests", + "ops", + "status", + "version", + ] + + self.add_cmd_output( + [f"ceph daemon {i} {c}" for i in self.get_socks() for c in cmds] + ) + + def get_socks(self): + """ + Find any available admin sockets under /var/run/ceph (or subdirs for + later versions of Ceph) which can be used for ceph daemon commands + """ + ceph_sockets = [] + for rdir, dirs, files in os.walk('/var/run/ceph/'): + for file in files: + if file.endswith('.asok'): + ceph_sockets.append(self.path_join(rdir, file)) + return ceph_sockets + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/ceph_rgw.py 4.5.3ubuntu2/sos/report/plugins/ceph_rgw.py --- 4.0-2/sos/report/plugins/ceph_rgw.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/ceph_rgw.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,38 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin + + +class CephRGW(Plugin, RedHatPlugin, UbuntuPlugin): + + short_desc = 'CEPH rgw' + + plugin_name = 'ceph_rgw' + profiles = ('storage', 'virt', 'container', 'webserver') + containers = ('ceph-(.*)?rgw.*',) + files = ('/var/lib/ceph/radosgw',) + + def setup(self): + self.add_copy_spec('/var/log/ceph/ceph-client.rgw*.log', + tags='ceph_rgw_log') + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/*keyring*", + "/var/lib/ceph/*/*keyring*", + "/var/lib/ceph/*/*/*keyring*", + "/var/lib/ceph/osd", + "/var/lib/ceph/mon", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/tmp/*mnt*", + "/etc/ceph/*bindpass*" + ]) + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/cgroups.py 4.5.3ubuntu2/sos/report/plugins/cgroups.py --- 4.0-2/sos/report/plugins/cgroups.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/cgroups.py 2023-04-28 17:16:21.000000000 +0000 @@ -19,14 +19,20 @@ class Cgroups(Plugin, DebianPlugin, Ubun files = ('/proc/cgroups',) def setup(self): + + self.add_file_tags({ + '/proc/1/cgroups': 'init_process_cgroup' + }) + self.add_copy_spec([ "/proc/cgroups", "/sys/fs/cgroup" ]) self.add_cmd_output("systemd-cgls") - - return + self.add_forbidden_path( + "/sys/fs/cgroup/memory/**/memory.kmem.slabinfo" + ) class RedHatCgroups(Cgroups, RedHatPlugin): diff -pruN 4.0-2/sos/report/plugins/chrony.py 4.5.3ubuntu2/sos/report/plugins/chrony.py --- 4.0-2/sos/report/plugins/chrony.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/chrony.py 2023-04-28 17:16:21.000000000 +0000 @@ -22,12 +22,12 @@ class Chrony(Plugin): self.add_cmd_output([ "chronyc activity", "chronyc tracking", - "chronyc -n sources", "chronyc sourcestats", "chronyc serverstats", "chronyc ntpdata", "chronyc -n clients" ]) + self.add_cmd_output("chronyc -n sources", tags="chronyc_sources") class RedHatChrony(Chrony, RedHatPlugin): @@ -45,6 +45,8 @@ class DebianChrony(Chrony, DebianPlugin, super(DebianChrony, self).setup() self.add_copy_spec([ "/etc/chrony/chrony.conf", + "/etc/chrony/conf.d", + "/etc/chrony/sources.d", "/var/lib/chrony/chrony.drift", "/etc/default/chrony" ]) diff -pruN 4.0-2/sos/report/plugins/cifs.py 4.5.3ubuntu2/sos/report/plugins/cifs.py --- 4.0-2/sos/report/plugins/cifs.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/cifs.py 2023-04-28 17:16:21.000000000 +0000 @@ -6,15 +6,15 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import Plugin, IndependentPlugin -class Cifs(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin): +class Cifs(Plugin, IndependentPlugin): short_desc = 'SMB file system information' plugin_name = 'cifs' profiles = ('storage', 'network', 'cifs') - packages = ['cifs-utils'] + packages = ('cifs-utils',) def setup(self): self.add_copy_spec([ diff -pruN 4.0-2/sos/report/plugins/clear_containers.py 4.5.3ubuntu2/sos/report/plugins/clear_containers.py --- 4.0-2/sos/report/plugins/clear_containers.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/clear_containers.py 2023-04-28 17:16:21.000000000 +0000 @@ -8,12 +8,10 @@ import re -from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, - UbuntuPlugin, SuSEPlugin) +from sos.report.plugins import Plugin, IndependentPlugin -class ClearContainers(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin, - SuSEPlugin): +class ClearContainers(Plugin, IndependentPlugin): short_desc = 'Intel(R) Clear Containers configuration' @@ -22,6 +20,7 @@ class ClearContainers(Plugin, RedHatPlug runtime = 'cc-runtime' packages = (runtime,) + services = ('cc-proxy',) def attach_cc_config_files(self): @@ -77,7 +76,6 @@ class ClearContainers(Plugin, RedHatPlug self.attach_cc_config_files() self.attach_cc_log_files() - self.add_journal(units="cc-proxy") self.add_journal(identifier="cc-shim") # vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/cloud_init.py 4.5.3ubuntu2/sos/report/plugins/cloud_init.py --- 4.0-2/sos/report/plugins/cloud_init.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/cloud_init.py 2023-04-28 17:16:21.000000000 +0000 @@ -8,10 +8,10 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin, DebianPlugin +from sos.report.plugins import Plugin, IndependentPlugin -class CloudInit(Plugin, RedHatPlugin, UbuntuPlugin, DebianPlugin): +class CloudInit(Plugin, IndependentPlugin): short_desc = 'cloud-init instance configurations' @@ -20,11 +20,21 @@ class CloudInit(Plugin, RedHatPlugin, Ub services = ('cloud-init',) def setup(self): + + self.add_cmd_output([ + 'cloud-init --version', + 'cloud-init features', + 'cloud-init status' + ]) + self.add_copy_spec([ - '/var/lib/cloud/', '/etc/cloud/', - '/run/cloud-init/cloud-init-generator.log', + '/run/cloud-init/', '/var/log/cloud-init*' ]) + self.add_file_tags({ + "/etc/cloud/cloud.cfg": "cloud_cfg_filtered" + }) + # vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/cman.py 4.5.3ubuntu2/sos/report/plugins/cman.py --- 4.0-2/sos/report/plugins/cman.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/cman.py 2023-04-28 17:16:21.000000000 +0000 @@ -17,16 +17,9 @@ class Cman(Plugin, RedHatPlugin): plugin_name = "cman" profiles = ("cluster",) - packages = [ - "luci", - "cman", - "clusterlib", - ] + packages = ("luci", "cman", "clusterlib") - files = ["/etc/cluster/cluster.conf"] - - debugfs_path = "/sys/kernel/debug" - _debugfs_cleanup = False + files = ("/etc/cluster/cluster.conf",) def setup(self): @@ -60,7 +53,7 @@ class Cman(Plugin, RedHatPlugin): self.do_file_sub( cluster_conf, r"(\s*\ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt + + +class Conntrack(Plugin, IndependentPlugin): + + short_desc = 'conntrack - netfilter connection tracking' + + plugin_name = 'conntrack' + profiles = ('network', 'cluster') + packages = ('conntrack-tools', 'conntrack', 'conntrackd') + + option_list = [ + PluginOpt("namespaces", default=None, val_type=int, + desc="Number of namespaces to collect, 0 for unlimited"), + ] + + def setup(self): + # Collect info from conntrackd + self.add_copy_spec("/etc/conntrackd/conntrackd.conf") + self.add_cmd_output([ + "conntrackd -s network", + "conntrackd -s cache", + "conntrackd -s runtime", + "conntrackd -s link", + "conntrackd -s rsqueue", + "conntrackd -s queue", + "conntrackd -s ct", + "conntrackd -s expect", + ]) + + # Collect info from conntrack + self.add_cmd_output([ + "conntrack -L -o extended", + "conntrack -S", + ]) + + # Capture additional data from namespaces; each command is run + # per-namespace + cmd_prefix = "ip netns exec " + nsps = self.get_option('namespaces') + for namespace in self.get_network_namespaces(ns_max=nsps): + ns_cmd_prefix = cmd_prefix + namespace + " " + self.add_cmd_output(ns_cmd_prefix + "conntrack -L -o extended") + self.add_cmd_output(ns_cmd_prefix + "conntrack -S") + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/conntrackd.py 4.5.3ubuntu2/sos/report/plugins/conntrackd.py --- 4.0-2/sos/report/plugins/conntrackd.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/conntrackd.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,36 +0,0 @@ -# Copyright (C) 2017 Red Hat, Inc., Marcus Linden -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information. - -from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, - UbuntuPlugin, SuSEPlugin) - - -class Conntrackd(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin, SuSEPlugin): - - short_desc = 'conntrackd - netfilter connection tracking user-space daemon' - - plugin_name = 'conntrackd' - profiles = ('network', 'cluster') - - packages = ('conntrack-tools', 'conntrackd') - - def setup(self): - self.add_copy_spec("/etc/conntrackd/conntrackd.conf") - self.add_cmd_output([ - "conntrackd -s network", - "conntrackd -s cache", - "conntrackd -s runtime", - "conntrackd -s link", - "conntrackd -s rsqueue", - "conntrackd -s queue", - "conntrackd -s ct", - "conntrackd -s expect", - ]) - -# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/container_log.py 4.5.3ubuntu2/sos/report/plugins/container_log.py --- 4.0-2/sos/report/plugins/container_log.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/container_log.py 2023-04-28 17:16:21.000000000 +0000 @@ -8,11 +8,11 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import Plugin, IndependentPlugin import os -class ContainerLog(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin): +class ContainerLog(Plugin, IndependentPlugin): short_desc = 'All logs under /var/log/containers' plugin_name = 'container_log' @@ -29,6 +29,6 @@ class ContainerLog(Plugin, RedHatPlugin, """Collect *.log files from subdirs of passed root path """ for dirName, _, _ in os.walk(root): - self.add_copy_spec(os.path.join(dirName, '*.log')) + self.add_copy_spec(self.path_join(dirName, '*.log')) # vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/containerd.py 4.5.3ubuntu2/sos/report/plugins/containerd.py --- 4.0-2/sos/report/plugins/containerd.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/containerd.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,29 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import (Plugin, RedHatPlugin, UbuntuPlugin, CosPlugin) + + +class Containerd(Plugin, RedHatPlugin, UbuntuPlugin, CosPlugin): + + short_desc = 'Containerd containers' + plugin_name = 'containerd' + profiles = ('container',) + packages = ('containerd',) + + def setup(self): + self.add_copy_spec([ + "/etc/containerd/", + ]) + + self.add_cmd_output('containerd config dump') + + # collect the containerd logs. + self.add_journal(units='containerd') + +# vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/containers_common.py 4.5.3ubuntu2/sos/report/plugins/containers_common.py --- 4.0-2/sos/report/plugins/containers_common.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/containers_common.py 2023-04-28 17:16:21.000000000 +0000 @@ -8,7 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin, PluginOpt import os @@ -19,8 +19,8 @@ class ContainersCommon(Plugin, RedHatPlu profiles = ('container', ) packages = ('containers-common', ) option_list = [ - ('rootlessusers', 'colon-separated list of users\' containers info', - '', ''), + PluginOpt('rootlessusers', default='', val_type=str, + desc='colon-delimited list of users to collect for') ] def setup(self): @@ -31,6 +31,10 @@ class ContainersCommon(Plugin, RedHatPlu '/etc/subgid', ]) + self.add_file_tags({ + "/etc/containers/policy.json": "containers_policy" + }) + users_opt = self.get_option('rootlessusers') users_list = [] if users_opt: @@ -41,6 +45,7 @@ class ContainersCommon(Plugin, RedHatPlu 'podman unshare cat /proc/self/uid_map', 'podman unshare cat /proc/self/gid_map', 'podman images', + 'podman images --digests', 'podman pod ps', 'podman port --all', 'podman ps', diff -pruN 4.0-2/sos/report/plugins/convert2rhel.py 4.5.3ubuntu2/sos/report/plugins/convert2rhel.py --- 4.0-2/sos/report/plugins/convert2rhel.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/convert2rhel.py 2023-04-28 17:16:21.000000000 +0000 @@ -13,15 +13,16 @@ class convert2rhel(Plugin, RedHatPlugin) short_desc = 'Convert2RHEL' plugin_name = 'convert2rhel' - profiles = ('system') - packages = ('convert2rhel') + profiles = ('system',) + packages = ('convert2rhel',) verify_packages = ('convert2rhel$',) def setup(self): self.add_copy_spec([ "/var/log/convert2rhel/convert2rhel.log", - "/var/log/convert2rhel/rpm_va.log" + "/var/log/convert2rhel/archive/convert2rhel-*.log", + "/var/log/convert2rhel/rpm_va.log", ]) diff -pruN 4.0-2/sos/report/plugins/corosync.py 4.5.3ubuntu2/sos/report/plugins/corosync.py --- 4.0-2/sos/report/plugins/corosync.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/corosync.py 2023-04-28 17:16:21.000000000 +0000 @@ -7,7 +7,6 @@ # See the LICENSE file in the source distribution for further information. from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin -import os.path import re @@ -23,7 +22,7 @@ class Corosync(Plugin): self.add_copy_spec([ "/etc/corosync", "/var/lib/corosync/fdata", - "/var/log/cluster/corosync.log" + "/var/log/cluster/corosync.log*" ]) self.add_cmd_output([ "corosync-quorumtool -l", @@ -32,13 +31,14 @@ class Corosync(Plugin): "corosync-cfgtool -s", "corosync-blackbox", "corosync-objctl -a", - "corosync-cmapctl", "corosync-cmapctl -m stats" ]) + self.add_cmd_output("corosync-cmapctl", + tags="corosync_cmapctl") self.exec_cmd("killall -USR2 corosync") corosync_conf = "/etc/corosync/corosync.conf" - if not os.path.exists(corosync_conf): + if not self.path_exists(corosync_conf): return # collect user-defined logfiles, matching either of pattern: @@ -48,7 +48,7 @@ class Corosync(Plugin): # (it isnt precise but sufficient) pattern = r'^\s*(logging.)?logfile:\s*(\S+)$' try: - with open("/etc/corosync/corosync.conf") as f: + with open(self.path_join("/etc/corosync/corosync.conf"), 'r') as f: for line in f: if re.match(pattern, line): self.add_copy_spec(re.search(pattern, line).group(2)) diff -pruN 4.0-2/sos/report/plugins/crio.py 4.5.3ubuntu2/sos/report/plugins/crio.py --- 4.0-2/sos/report/plugins/crio.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/crio.py 2023-04-28 17:16:21.000000000 +0000 @@ -8,21 +8,23 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin, SoSPredicate +from sos.report.plugins import (Plugin, RedHatPlugin, UbuntuPlugin, + SoSPredicate, PluginOpt, CosPlugin) -class CRIO(Plugin, RedHatPlugin, UbuntuPlugin): +class CRIO(Plugin, RedHatPlugin, UbuntuPlugin, CosPlugin): short_desc = 'CRI-O containers' plugin_name = 'crio' profiles = ('container',) packages = ('cri-o', 'cri-tools') + services = ('crio',) option_list = [ - ("all", "enable capture for all containers, even containers " - "that have terminated", 'fast', False), - ("logs", "capture logs for running containers", - 'fast', False), + PluginOpt('all', default=False, + desc='collect for all containers, even terminated ones'), + PluginOpt('logs', default=False, + desc='collect stdout/stderr logs for containers') ] def setup(self): @@ -31,6 +33,7 @@ class CRIO(Plugin, RedHatPlugin, UbuntuP "/etc/crictl.yaml", "/etc/crio/crio.conf", "/etc/crio/seccomp.json", + "/etc/crio/crio.conf.d/", "/etc/systemd/system/cri-o.service", "/etc/sysconfig/crio-*" ]) @@ -42,8 +45,10 @@ class CRIO(Plugin, RedHatPlugin, UbuntuP 'ALL_PROXY' ]) - self.add_journal(units="crio") - self.add_cmd_output("ls -alhR /etc/cni") + self.add_cmd_output([ + "ls -alhR /etc/cni", + "crio config" + ]) # base cri-o installation does not require cri-tools, which is what # supplies the crictl utility @@ -73,11 +78,15 @@ class CRIO(Plugin, RedHatPlugin, UbuntuP images = self._get_crio_list(img_cmd) pods = self._get_crio_list(pod_cmd) + self._get_crio_goroutine_stacks() + for container in containers: - self.add_cmd_output("crictl inspect %s" % container) + self.add_cmd_output("crictl inspect %s" % container, + subdir="containers") if self.get_option('logs'): self.add_cmd_output("crictl logs -t %s" % container, - subdir="containers") + subdir="containers/logs", priority=100, + tags="crictl_logs") for image in images: self.add_cmd_output("crictl inspecti %s" % image, subdir="images") @@ -96,4 +105,13 @@ class CRIO(Plugin, RedHatPlugin, UbuntuP ret.pop(0) return ret + def _get_crio_goroutine_stacks(self): + result = self.exec_cmd("pidof crio") + if result['status'] != 0: + return + pid = result['output'].strip() + result = self.exec_cmd("kill -USR1 " + pid) + if result['status'] == 0: + self.add_copy_spec("/tmp/crio-goroutine-stacks*.log") + # vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/cron.py 4.5.3ubuntu2/sos/report/plugins/cron.py --- 4.0-2/sos/report/plugins/cron.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/cron.py 2023-04-28 17:16:21.000000000 +0000 @@ -6,10 +6,10 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import Plugin, IndependentPlugin -class Cron(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin): +class Cron(Plugin, IndependentPlugin): short_desc = 'Cron job scheduler' @@ -17,7 +17,7 @@ class Cron(Plugin, RedHatPlugin, DebianP profiles = ('system',) packages = ('cron', 'anacron', 'chronie') - files = ('/etc/crontab') + files = ('/etc/crontab',) def setup(self): self.add_copy_spec([ @@ -30,6 +30,7 @@ class Cron(Plugin, RedHatPlugin, DebianP self.add_copy_spec("/var/log/cron*") self.add_cmd_output("crontab -l -u root", - suggest_filename="root_crontab") + suggest_filename="root_crontab", + tags="root_crontab") # vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/crypto.py 4.5.3ubuntu2/sos/report/plugins/crypto.py --- 4.0-2/sos/report/plugins/crypto.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/crypto.py 2023-04-28 17:16:21.000000000 +0000 @@ -8,10 +8,10 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import Plugin, IndependentPlugin -class Crypto(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin): +class Crypto(Plugin, IndependentPlugin): short_desc = 'System crypto services information' @@ -19,6 +19,16 @@ class Crypto(Plugin, RedHatPlugin, Debia profiles = ('system', 'hardware') def setup(self): + + cpth = '/etc/crypto-policies/back-ends' + + self.add_file_tags({ + "%s/bind.config" % cpth: 'crypto_policies_bind', + "%s/opensshserver.config" % cpth: 'crypto_policies_opensshserver', + '/etc/crypto-policies/.*/current': 'crypto_policies_state_current', + '/etc/crypto-policies/config': 'crypto_policies_config' + }) + self.add_copy_spec([ "/proc/crypto", "/proc/sys/crypto/fips_enabled", diff -pruN 4.0-2/sos/report/plugins/cs.py 4.5.3ubuntu2/sos/report/plugins/cs.py --- 4.0-2/sos/report/plugins/cs.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/cs.py 2023-04-28 17:16:21.000000000 +0000 @@ -11,7 +11,6 @@ # See the LICENSE file in the source distribution for further information. from sos.report.plugins import Plugin, RedHatPlugin -from os.path import exists from glob import glob @@ -38,7 +37,8 @@ class CertificateSystem(Plugin, RedHatPl ) def checkversion(self): - if self.is_installed("redhat-cs") or exists("/opt/redhat-cs"): + if (self.is_installed("redhat-cs") or + self.path_exists("/opt/redhat-cs")): return 71 elif self.is_installed("rhpki-common") or \ len(glob("/var/lib/rhpki-*")): @@ -88,6 +88,9 @@ class CertificateSystem(Plugin, RedHatPl "/var/log/dirsrv/slapd-*/access", "/var/log/dirsrv/slapd-*/errors" ]) + self.add_file_tags({ + "/var/log/dirsrv/*/access": "dirsrv_access" + }) if csversion == 8: self.add_copy_spec([ "/etc/pki-*/CS.cfg", diff -pruN 4.0-2/sos/report/plugins/cups.py 4.5.3ubuntu2/sos/report/plugins/cups.py --- 4.0-2/sos/report/plugins/cups.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/cups.py 2023-04-28 17:16:21.000000000 +0000 @@ -6,16 +6,16 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import Plugin, IndependentPlugin -class Cups(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin): +class Cups(Plugin, IndependentPlugin): short_desc = 'CUPS IPP print service' plugin_name = 'cups' profiles = ('hardware',) - + services = ('cups', 'cups-browsed') packages = ('cups',) def setup(self): @@ -39,6 +39,4 @@ class Cups(Plugin, RedHatPlugin, DebianP "lpstat -d" ]) - self.add_journal(units="cups") - # vim: set et ts=4 sw=4 : diff -pruN 4.0-2/sos/report/plugins/date.py 4.5.3ubuntu2/sos/report/plugins/date.py --- 4.0-2/sos/report/plugins/date.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/date.py 2023-04-28 17:16:21.000000000 +0000 @@ -8,22 +8,22 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin +from sos.report.plugins import Plugin, IndependentPlugin -class Date(Plugin, RedHatPlugin, DebianPlugin): +class Date(Plugin, IndependentPlugin): short_desc = 'Basic system time information' plugin_name = 'date' def setup(self): - self.add_cmd_output('date', root_symlink='date') self.add_cmd_output([ + 'date', 'date --utc', 'hwclock' - ]) + ], cmd_as_tag=True) self.add_copy_spec([ '/etc/localtime', diff -pruN 4.0-2/sos/report/plugins/dbus.py 4.5.3ubuntu2/sos/report/plugins/dbus.py --- 4.0-2/sos/report/plugins/dbus.py 2020-08-17 21:41:02.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/dbus.py 2023-04-28 17:16:21.000000000 +0000 @@ -6,10 +6,10 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import Plugin, IndependentPlugin -class Dbus(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin): +class Dbus(Plugin, IndependentPlugin): short_desc = 'D-Bus message bus' diff -pruN 4.0-2/sos/report/plugins/dellrac.py 4.5.3ubuntu2/sos/report/plugins/dellrac.py --- 4.0-2/sos/report/plugins/dellrac.py 1970-01-01 00:00:00.000000000 +0000 +++ 4.5.3ubuntu2/sos/report/plugins/dellrac.py 2023-04-28 17:16:21.000000000 +0000 @@ -0,0 +1,49 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt + + +class DellRAC(Plugin, IndependentPlugin): + + short_desc = 'Dell Remote Access Controller Administration' + + plugin_name = 'dellrac' + profiles = ('system', 'storage', 'hardware',) + packages = ('srvadmin-idracadm7',) + + option_list = [ + PluginOpt('debug', default=False, desc='capture support assist data') + ] + + racadm = '/opt/dell/srvadmin/bin/idracadm7' + prefix = 'idracadm7' + + def setup(self): + for subcmd in ['getniccfg', 'getsysinfo']: + self.add_cmd_output( + '%s %s' % (self.racadm, subcmd), + suggest_filename='%s_%s' % (self.prefix, subcmd)) + + if self.get_option("debug"): + self.do_debug() + + def do_debug(self): + # ensure the sos_commands/dellrac directory does exist in either case + # as we will need to run the command at that dir, and also