From 25471134f1c79a03b5ef42a27afdf69f1632488a Mon Sep 17 00:00:00 2001 From: Christer Edwards Date: Fri, 7 Feb 2020 10:29:59 -0700 Subject: [PATCH 01/11] Initial support for FreeBSD --- CHANGELOG.md | 4 ++++ datadog/install.sls | 8 +++++++- datadog/map.jinja | 5 +++++ pillar.example | 2 +- 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1df3bab..a2b4b8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changes +## 3.1 / 2020-02-07 + +Initial support for FreeBSD + ## 3.0 / 2019-12-18 **This formula will install Agent v7 by default.** Datadog Agent v7 runs checks with Python 3, so if you were running any custom checks written in Python, they must be compatible with Python 3. If you were not running any custom checks or if your custom checks are already compatible with Python 3, then it is safe to upgrade to Agent v7. diff --git a/datadog/install.sls b/datadog/install.sls index 5ac42f0..c982075 100644 --- a/datadog/install.sls +++ b/datadog/install.sls @@ -6,6 +6,7 @@ datadog-apt-https: - name: apt-transport-https {%- endif %} +{%- if grains['os_family'].lower() != 'freebsd' %} datadog-repo: pkgrepo.managed: - humanname: "Datadog, Inc." @@ -60,6 +61,7 @@ datadog-repo: {%- endif %} - sslverify: '1' {% endif %} +{%- endif %} datadog-pkg: pkg.installed: @@ -70,8 +72,12 @@ datadog-pkg: - version: 1:{{ datadog_install_settings.agent_version }}-1 {%- elif grains['os_family'].lower() == 'redhat' %} - version: {{ datadog_install_settings.agent_version }}-1 + {%- elif grains['os_family'].lower() == 'freebsd' %} + - version: {{ datadog_install_settings.agent_version }}-1 {%- endif %} - ignore_epoch: True - refresh: True + {%- if grains['os_family'].lower() != 'freebsd' %} - require: - - pkgrepo: datadog-repo \ No newline at end of file + - pkgrepo: datadog-repo + {%- endif %} diff --git a/datadog/map.jinja b/datadog/map.jinja index 091ad7d..51c933e 100644 --- a/datadog/map.jinja +++ b/datadog/map.jinja @@ -1,6 +1,7 @@ {% set os_family_map = salt['grains.filter_by']({ 'Debian': {}, 'RedHat': {}, + 'FreeBSD': {}, }, grain="os_family") %} @@ -35,6 +36,8 @@ {# Determine defaults depending on specified version #} {%- if latest_agent_version or parsed_version[1] != '5' %} {% do datadog_install_settings.update({'config_folder': '/etc/datadog-agent'}) %} +{%- elif grains['os_family'].lower() == 'freebsd' %} + {% do datadog_install_settings.update({'config_folder': '/usr/local/etc/datadog'}) %} {%- else %} {% do datadog_install_settings.update({'config_folder': '/etc/dd-agent'}) %} {%- endif %} @@ -50,6 +53,8 @@ {%- else %} {%- if latest_agent_version or parsed_version[1] != '5' %} {% do datadog_install_settings.update({'confd_path': '/etc/datadog-agent/conf.d'}) %} + {%- elif grains['os_family'].lower() == 'freebsd' %} + {% do datadog_install_settings.update({'confd_path': '/usr/local/etc/datadog/conf.d'}) %} {%- else %} {% do datadog_install_settings.update({'confd_path': '/etc/dd-agent/conf.d'}) %} {%- endif %} diff --git a/pillar.example b/pillar.example index 538cfc2..3a58543 100644 --- a/pillar.example +++ b/pillar.example @@ -2,7 +2,7 @@ datadog: config: api_key: aaaaaaaabbbbbbbbccccccccdddddddd site: datadoghq.com - python_version: 2 + python_version: 3 checks: process: From 62980dc2dcd896d5a06b49f268b17c96dde1b8d3 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 18 Mar 2020 14:45:59 -0300 Subject: [PATCH 02/11] Add kitchen and support for freebsd Signed-off-by: Felipe Zipitria --- .github/workflows/kitchen.yml | 31 + Gemfile | 7 + kitchen.yml | 120 + test/base/top.sls | 4 - test/base/top_install.sls | 3 - test/base/top_uninstall.sls | 3 - test/dist/debian_Dockerfile | 17 - test/dist/redhat_Dockerfile | 16 - test/docker-compose.yaml | 13 - test/integration/datadog/README.md | 50 + test/integration/datadog/controls/datadog.rb | 64 + test/integration/datadog/inspec.yml | 17 + test/minion.d/masterless.conf | 9 - test/pillar/top.sls | 3 - test/{ => salt}/pillar/datadog.sls | 0 test/{ => salt}/pillar/datadog5.sls | 0 test/{ => salt}/pillar/datadog6.sls | 0 test/{ => salt}/pillar/datadog7.sls | 0 test/start.sh | 7 - test/utils/check_apt_install.py | 40 - test/utils/check_yum_install.py | 34 - test/utils/helpers.py | 30 - test/utils/systemctl.py | 4544 ------------------ 23 files changed, 289 insertions(+), 4723 deletions(-) create mode 100644 .github/workflows/kitchen.yml create mode 100644 Gemfile create mode 100644 kitchen.yml delete mode 100644 test/base/top.sls delete mode 100644 test/base/top_install.sls delete mode 100644 test/base/top_uninstall.sls delete mode 100644 test/dist/debian_Dockerfile delete mode 100644 test/dist/redhat_Dockerfile delete mode 100644 test/docker-compose.yaml create mode 100644 test/integration/datadog/README.md create mode 100644 test/integration/datadog/controls/datadog.rb create mode 100644 test/integration/datadog/inspec.yml delete mode 100644 test/minion.d/masterless.conf delete mode 100644 test/pillar/top.sls rename test/{ => salt}/pillar/datadog.sls (100%) rename test/{ => salt}/pillar/datadog5.sls (100%) rename test/{ => salt}/pillar/datadog6.sls (100%) rename test/{ => salt}/pillar/datadog7.sls (100%) delete mode 100755 test/start.sh delete mode 100644 test/utils/check_apt_install.py delete mode 100644 test/utils/check_yum_install.py delete mode 100644 test/utils/helpers.py delete mode 100755 test/utils/systemctl.py diff --git a/.github/workflows/kitchen.yml b/.github/workflows/kitchen.yml new file mode 100644 index 0000000..2298584 --- /dev/null +++ b/.github/workflows/kitchen.yml @@ -0,0 +1,31 @@ +name: Integration + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + datadog_version: [ 5, 6, 7 ] + os: [ debian-9-2019-2-py3, ubuntu-1804-2019-2-py3, centos-7-2019-2-py3 ] + steps: + - uses: actions/checkout@v2 + + - name: Set up Ruby 2.6 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.6.x + + - name: Build and test with Kitchen + run: | + gem install bundler + bundle install --jobs 4 --retry 3 + + - name: "Running tests for ${{ matrix.datadog_version }}-${{ matrix.os }}" + run: | + bundle exec kitchen test datadog${{ matrix.datadog_version }}-${{ matrix.os }} diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..5a232b6 --- /dev/null +++ b/Gemfile @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +source 'https://rubygems.org' + +gem 'kitchen-docker', '>= 2.9' +gem 'kitchen-inspec', '>= 1.1' +gem 'kitchen-salt', '>= 0.6.0' diff --git a/kitchen.yml b/kitchen.yml new file mode 100644 index 0000000..2d729a4 --- /dev/null +++ b/kitchen.yml @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +--- +# For help on this file's format, see https://kitchen.ci/ +driver: + name: docker + use_sudo: false + privileged: true + run_command: /lib/systemd/systemd + +# Make sure the platforms listed below match up with +# the `env.matrix` instances defined in `.travis.yml` +platforms: + ## SALT `master` + - name: debian-10-master-py3 + driver: + image: netmanagers/salt-master-py3:debian-10 + provision_command: + - curl -o bootstrap-salt.sh -L https://bootstrap.saltstack.com + - sh bootstrap-salt.sh -XdPbfrq -x python3 git master + - name: ubuntu-1804-master-py3 + driver: + image: netmanagers/salt-master-py3:ubuntu-18.04 + provision_command: + - curl -o bootstrap-salt.sh -L https://bootstrap.saltstack.com + - sh bootstrap-salt.sh -XdPbfrq -x python3 git master + + ## SALT `2019.2` + - name: debian-9-2019-2-py3 + driver: + image: netmanagers/salt-2019.2-py3:debian-9 + - name: ubuntu-1804-2019-2-py3 + driver: + image: netmanagers/salt-2019.2-py3:ubuntu-18.04 + + - name: centos-8-2019-2-py3 + driver: + image: netmanagers/salt-2019.2-py3:centos-8 + + - name: centos-7-2019-2-py3 + driver: + image: netmanagers/salt-2019.2-py3:centos-7 + + +provisioner: + name: salt_solo + log_level: debug + salt_install: none + require_chef: false + formula: datadog + salt_copy_filter: + - .kitchen + - .git + +verifier: + # https://www.inspec.io/ + name: inspec + sudo: true + # cli, documentation, html, progress, json, json-min, json-rspec, junit + reporter: + - cli + +suites: + - name: datadog5 + provisioner: + state_top: + base: + '*': + - datadog + pillars: + top.sls: + base: + '*': + - datadog + pillars_from_files: + datadog.sls: test/salt/pillar/datadog5.sls + verifier: + inspec_tests: + - path: test/integration/datadog + inputs: + version: 5 + + - name: datadog6 + provisioner: + state_top: + base: + '*': + - datadog + pillars: + top.sls: + base: + '*': + - datadog + pillars_from_files: + datadog.sls: test/salt/pillar/datadog6.sls + verifier: + inspec_tests: + - path: test/integration/datadog + inputs: + version: 6 + + - name: datadog7 + provisioner: + state_top: + base: + '*': + - datadog + pillars: + top.sls: + base: + '*': + - datadog + pillars_from_files: + datadog.sls: test/salt/pillar/datadog7.sls + verifier: + inspec_tests: + - path: test/integration/datadog + inputs: + version: 6 + diff --git a/test/base/top.sls b/test/base/top.sls deleted file mode 100644 index b63f0d9..0000000 --- a/test/base/top.sls +++ /dev/null @@ -1,4 +0,0 @@ -base: - '*': - - datadog - - datadog.uninstall diff --git a/test/base/top_install.sls b/test/base/top_install.sls deleted file mode 100644 index a5c7a07..0000000 --- a/test/base/top_install.sls +++ /dev/null @@ -1,3 +0,0 @@ -base: - '*': - - datadog \ No newline at end of file diff --git a/test/base/top_uninstall.sls b/test/base/top_uninstall.sls deleted file mode 100644 index 0066eec..0000000 --- a/test/base/top_uninstall.sls +++ /dev/null @@ -1,3 +0,0 @@ -base: - '*': - - datadog.uninstall \ No newline at end of file diff --git a/test/dist/debian_Dockerfile b/test/dist/debian_Dockerfile deleted file mode 100644 index e4171e2..0000000 --- a/test/dist/debian_Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM ubuntu:16.04 -LABEL maintainer="package@datadoghq.com" - -# preparation for saltstack -RUN apt-get update &&\ - apt-get install -y curl - -# enable systemd, thanks to @gdraheim (https://github.com/gdraheim/) -ADD utils/systemctl.py /bin/systemctl -ADD utils/systemctl.py /bin/systemd - -# install salt -RUN curl -L https://bootstrap.saltstack.com | sh -s -- -d -X stable; exit 0 - -# add the start test script -ADD start.sh /start.sh -CMD ["bash", "start.sh"] diff --git a/test/dist/redhat_Dockerfile b/test/dist/redhat_Dockerfile deleted file mode 100644 index f73453c..0000000 --- a/test/dist/redhat_Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM centos:7 -LABEL maintainer="package@datadoghq.com" - -# preparation for saltstack -RUN yum -y update && yum -y install curl - -# enable systemd, thanks to @gdraheim (https://github.com/gdraheim/) -ADD utils/systemctl.py /usr/bin/systemctl -ADD utils/systemctl.py /usr/bin/systemd - -# install salt -RUN curl -L https://bootstrap.saltstack.com | sh -s -- -d -X stable; exit 0 - -# add the start test script -ADD start.sh /start.sh -CMD ["bash", "start.sh"] diff --git a/test/docker-compose.yaml b/test/docker-compose.yaml deleted file mode 100644 index 7588fcf..0000000 --- a/test/docker-compose.yaml +++ /dev/null @@ -1,13 +0,0 @@ -version: '3' - -services: - masterless: - image: dd-salt-${TEST_DIST}-masterless - build: - context: . - dockerfile: ./dist/${TEST_DIST}_Dockerfile - volumes: - - ./minion.d:/etc/salt/minion.d - - ./pillar:/srv/pillar - - ./base:/srv/salt/base - - ../datadog:/srv/salt/base/datadog diff --git a/test/integration/datadog/README.md b/test/integration/datadog/README.md new file mode 100644 index 0000000..857aca5 --- /dev/null +++ b/test/integration/datadog/README.md @@ -0,0 +1,50 @@ +# InSpec Profile: `datadog` + +This shows the implementation of the `datadog` InSpec [profile](https://github.com/inspec/inspec/blob/master/docs/profiles.md). + +## Verify a profile + +InSpec ships with built-in features to verify a profile structure. + +```bash +$ inspec check datadog +Summary +------- +Location: datadog +Profile: profile +Controls: 4 +Timestamp: 2019-06-24T23:09:01+00:00 +Valid: true + +Errors +------ + +Warnings +-------- +``` + +## Execute a profile + +To run all **supported** controls on a local machine use `inspec exec /path/to/profile`. + +```bash +$ inspec exec datadog +.. + +Finished in 0.0025 seconds (files took 0.12449 seconds to load) +8 examples, 0 failures +``` + +## Execute a specific control from a profile + +To run one control from the profile use `inspec exec /path/to/profile --controls name`. + +```bash +$ inspec exec datadog --controls package +. + +Finished in 0.0025 seconds (files took 0.12449 seconds to load) +1 examples, 0 failures +``` + +See an [example control here](https://github.com/inspec/inspec/blob/master/examples/profile/controls/example.rb). diff --git a/test/integration/datadog/controls/datadog.rb b/test/integration/datadog/controls/datadog.rb new file mode 100644 index 0000000..99d09d2 --- /dev/null +++ b/test/integration/datadog/controls/datadog.rb @@ -0,0 +1,64 @@ +# Get datadog major as input variable +datadog_version = input('version') + +if os.debian? + describe apt('https://apt.datadoghq.com') do + it { should exist } + it { should be_enabled } + end +elsif os.redhat? + describe yum.repo('datadog') do + it { should exist } + it { should be_enabled } + its('baseurl') { should include 'yum.datadoghq.com' } + end +end + +describe user('dd-agent') do + it { should exist } + its('group') { should eq 'dd-agent' } +end + +describe package('datadog-agent') do + it { should be_installed } +end + +describe service('datadog-agent') do + it { should be_enabled } + it { should be_running } +end + +if datadog_version >= 6 + describe port(5001) do + it { should be_listening } + its('protocols') { should include 'tcp' } + its('processes') { should include 'agent' } + end +end + +if datadog_version == 5 + describe file('/etc/dd-agent/datadog.conf') do + it { should be_file } + its('owner') { should eq 'dd-agent' } + its('group') { should eq 'dd-agent' } + end + + describe file('/etc/dd-agent/conf.d') do + it { should be_directory } + its('owner') { should eq 'dd-agent' } + its('group') { should eq 'dd-agent' } + end +else + describe file('/etc/datadog-agent/datadog.yaml') do + it { should be_file } + its('owner') { should eq 'dd-agent' } + its('group') { should eq 'dd-agent' } + end + + describe file('/etc/datadog-agent/conf.d') do + it { should be_directory } + its('owner') { should eq 'dd-agent' } + its('group') { should eq 'dd-agent' } + end +end + diff --git a/test/integration/datadog/inspec.yml b/test/integration/datadog/inspec.yml new file mode 100644 index 0000000..824664c --- /dev/null +++ b/test/integration/datadog/inspec.yml @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +--- +name: datadog +title: datadog Formula +maintainer: SaltStack Formulas +license: Apache-2.0 +summary: Verify that the datadog is setup and configured correctly +supports: + - platform-name: debian + - platform-name: ubuntu + - platform-name: centos + - platform-name: fedora + - platform-name: opensuse + - platform-name: suse + - platform-name: freebsd + - platform-name: amazon diff --git a/test/minion.d/masterless.conf b/test/minion.d/masterless.conf deleted file mode 100644 index d87f7d3..0000000 --- a/test/minion.d/masterless.conf +++ /dev/null @@ -1,9 +0,0 @@ -file_client: local -file_roots: - base: - - /srv/salt/base -pillar_roots: - base: - - /srv/pillar -providers: - service: systemd_service diff --git a/test/pillar/top.sls b/test/pillar/top.sls deleted file mode 100644 index 85e4974..0000000 --- a/test/pillar/top.sls +++ /dev/null @@ -1,3 +0,0 @@ -base: - '*': - - datadog diff --git a/test/pillar/datadog.sls b/test/salt/pillar/datadog.sls similarity index 100% rename from test/pillar/datadog.sls rename to test/salt/pillar/datadog.sls diff --git a/test/pillar/datadog5.sls b/test/salt/pillar/datadog5.sls similarity index 100% rename from test/pillar/datadog5.sls rename to test/salt/pillar/datadog5.sls diff --git a/test/pillar/datadog6.sls b/test/salt/pillar/datadog6.sls similarity index 100% rename from test/pillar/datadog6.sls rename to test/salt/pillar/datadog6.sls diff --git a/test/pillar/datadog7.sls b/test/salt/pillar/datadog7.sls similarity index 100% rename from test/pillar/datadog7.sls rename to test/salt/pillar/datadog7.sls diff --git a/test/start.sh b/test/start.sh deleted file mode 100755 index 2841673..0000000 --- a/test/start.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -# Install & uninstall datadog-agent -salt-call --local state.highstate -l debug - -echo "==== Done ====" -sleep infinity diff --git a/test/utils/check_apt_install.py b/test/utils/check_apt_install.py deleted file mode 100644 index 4576aa5..0000000 --- a/test/utils/check_apt_install.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python - -import apt, re, sys -from helpers import get_options, check_major_version - -def get_apt_package_version(package_name): - cache = apt.cache.Cache() - cache.update() - cache.open() - - installed_version = None - try: - pkg = cache["datadog-agent"] - if pkg.is_installed: - # pkg.installed has the form datadog-agent=1:x.y.z-1, we only want x.y.z - installed_version = re.match("datadog-agent=[0-9]+:(.*)-[0-9]+", str(pkg.installed)).groups()[0] - except KeyError: - # datadog-agent is not installed - pass - - return installed_version - - -def main(argv): - expected_major_version = get_options(argv[1:]) - print("Expected major version: {}".format(expected_major_version)) - - installed_version = get_apt_package_version("datadog-agent") - print("Installed Agent version: {}".format(installed_version)) - - result = check_major_version(installed_version, expected_major_version) - if result: - print("Agent version check successful!") - sys.exit() - else: - print("Agent version check failed.") - sys.exit(1) - -if __name__ == "__main__": - main(sys.argv) diff --git a/test/utils/check_yum_install.py b/test/utils/check_yum_install.py deleted file mode 100644 index 7c92faa..0000000 --- a/test/utils/check_yum_install.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python - -import yum, sys -from helpers import get_options, check_major_version - -def get_yum_package_version(package_name): - yb = yum.YumBase() - try: - # Use next to stop at the first match - pkg = next(p for p in yb.rpmdb.returnPackages() if p.name == "datadog-agent") - installed_version = pkg.version - except StopIteration: - # datadog-agent is not in the list of installed packages - installed_version = None - - return installed_version - -def main(argv): - expected_major_version = get_options(argv[1:]) - print("Expected major version: {}".format(expected_major_version)) - - installed_version = get_yum_package_version("datadog-agent") - print("Installed Agent version: {}".format(installed_version)) - - result = check_major_version(installed_version, expected_major_version) - if result: - print("Agent version check successful!") - sys.exit() - else: - print("Agent version check failed.") - sys.exit(1) - -if __name__ == "__main__": - main(sys.argv) diff --git a/test/utils/helpers.py b/test/utils/helpers.py deleted file mode 100644 index d287179..0000000 --- a/test/utils/helpers.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python - -import getopt, re - -def get_options(args): - expected_major_version = None - try: - opts, _ = getopt.getopt(args, "hnm:", ["not-installed", "major-version="]) - except getopt.GetoptError: - print('check_install.py [-n] [-m ]') - sys.exit(2) - for opt, arg in opts: - if opt == '-h': - print('check_apt_install.py [-n] [-m ]') - sys.exit() - elif opt in ("-n", "--not-installed"): - expected_major_version = None - elif opt in ("-m", "--major-version"): - expected_major_version = arg - - return expected_major_version - -def check_major_version(installed_version, expected_major_version): - installed_major_version = None - - if installed_version is not None: - installed_major_version = re.match("([0-9]+).*", installed_version).groups()[0] - - print("Installed Agent major version: {}".format(installed_major_version)) - return installed_major_version == expected_major_version diff --git a/test/utils/systemctl.py b/test/utils/systemctl.py deleted file mode 100755 index e909024..0000000 --- a/test/utils/systemctl.py +++ /dev/null @@ -1,4544 +0,0 @@ -#! /usr/bin/python -from __future__ import print_function - -__copyright__ = "(C) 2016-2019 Guido U. Draheim, licensed under the EUPL" -__version__ = "1.4.3424" - -import logging -logg = logging.getLogger("systemctl") - -import re -import fnmatch -import shlex -import collections -import errno -import os -import sys -import signal -import time -import socket -import datetime -import fcntl - -if sys.version[0] == '2': - string_types = basestring - BlockingIOError = IOError -else: - string_types = str - xrange = range - -COVERAGE = os.environ.get("SYSTEMCTL_COVERAGE", "") -DEBUG_AFTER = os.environ.get("SYSTEMCTL_DEBUG_AFTER", "") or False -EXIT_WHEN_NO_MORE_PROCS = os.environ.get("SYSTEMCTL_EXIT_WHEN_NO_MORE_PROCS", "") or False -EXIT_WHEN_NO_MORE_SERVICES = os.environ.get("SYSTEMCTL_EXIT_WHEN_NO_MORE_SERVICES", "") or False - -FOUND_OK = 0 -FOUND_INACTIVE = 2 -FOUND_UNKNOWN = 4 - -# defaults for options -_extra_vars = [] -_force = False -_full = False -_now = False -_no_legend = False -_no_ask_password = False -_preset_mode = "all" -_quiet = False -_root = "" -_unit_type = None -_unit_state = None -_unit_property = None -_show_all = False -_user_mode = False - -# common default paths -_default_target = "multi-user.target" -_system_folder1 = "/etc/systemd/system" -_system_folder2 = "/var/run/systemd/system" -_system_folder3 = "/usr/lib/systemd/system" -_system_folder4 = "/lib/systemd/system" -_system_folder9 = None -_user_folder1 = "~/.config/systemd/user" -_user_folder2 = "/etc/systemd/user" -_user_folder3 = "~.local/share/systemd/user" -_user_folder4 = "/usr/lib/systemd/user" -_user_folder9 = None -_init_folder1 = "/etc/init.d" -_init_folder2 = "/var/run/init.d" -_init_folder9 = None -_preset_folder1 = "/etc/systemd/system-preset" -_preset_folder2 = "/var/run/systemd/system-preset" -_preset_folder3 = "/usr/lib/systemd/system-preset" -_preset_folder4 = "/lib/systemd/system-preset" -_preset_folder9 = None - -SystemCompatibilityVersion = 219 -SysInitTarget = "sysinit.target" -SysInitWait = 5 # max for target -EpsilonTime = 0.1 -MinimumYield = 0.5 -MinimumTimeoutStartSec = 4 -MinimumTimeoutStopSec = 4 -DefaultTimeoutStartSec = int(os.environ.get("SYSTEMCTL_TIMEOUT_START_SEC", 90)) # official value -DefaultTimeoutStopSec = int(os.environ.get("SYSTEMCTL_TIMEOUT_STOP_SEC", 90)) # official value -DefaultMaximumTimeout = int(os.environ.get("SYSTEMCTL_MAXIMUM_TIMEOUT", 200)) # overrides all other -InitLoopSleep = int(os.environ.get("SYSTEMCTL_INITLOOP", 5)) -ProcMaxDepth = 100 -MaxLockWait = None # equals DefaultMaximumTimeout -DefaultPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" -ResetLocale = ["LANG", "LANGUAGE", "LC_CTYPE", "LC_NUMERIC", "LC_TIME", "LC_COLLATE", "LC_MONETARY", - "LC_MESSAGES", "LC_PAPER", "LC_NAME", "LC_ADDRESS", "LC_TELEPHONE", "LC_MEASUREMENT", - "LC_IDENTIFICATION", "LC_ALL"] - -# The systemd default is NOTIFY_SOCKET="/var/run/systemd/notify" -_notify_socket_folder = "/var/run/systemd" # alias /run/systemd -_pid_file_folder = "/var/run" -_journal_log_folder = "/var/log/journal" - -_systemctl_debug_log = "/var/log/systemctl.debug.log" -_systemctl_extra_log = "/var/log/systemctl.log" - -_default_targets = [ "poweroff.target", "rescue.target", "sysinit.target", "basic.target", "multi-user.target", "graphical.target", "reboot.target" ] -_feature_targets = [ "network.target", "remote-fs.target", "local-fs.target", "timers.target", "nfs-client.target" ] -_all_common_targets = [ "default.target" ] + _default_targets + _feature_targets - -# inside a docker we pretend the following -_all_common_enabled = [ "default.target", "multi-user.target", "remote-fs.target" ] -_all_common_disabled = [ "graphical.target", "resue.target", "nfs-client.target" ] - -_runlevel_mappings = {} # the official list -_runlevel_mappings["0"] = "poweroff.target" -_runlevel_mappings["1"] = "rescue.target" -_runlevel_mappings["2"] = "multi-user.target" -_runlevel_mappings["3"] = "multi-user.target" -_runlevel_mappings["4"] = "multi-user.target" -_runlevel_mappings["5"] = "graphical.target" -_runlevel_mappings["6"] = "reboot.target" - -_sysv_mappings = {} # by rule of thumb -_sysv_mappings["$local_fs"] = "local-fs.target" -_sysv_mappings["$network"] = "network.target" -_sysv_mappings["$remote_fs"] = "remote-fs.target" -_sysv_mappings["$timer"] = "timers.target" - -def shell_cmd(cmd): - return " ".join(["'%s'" % part for part in cmd]) -def to_int(value, default = 0): - try: - return int(value) - except: - return default -def to_list(value): - if isinstance(value, string_types): - return [ value ] - return value -def unit_of(module): - if "." not in module: - return module + ".service" - return module - -def os_path(root, path): - if not root: - return path - if not path: - return path - while path.startswith(os.path.sep): - path = path[1:] - return os.path.join(root, path) - -def os_getlogin(): - """ NOT using os.getlogin() """ - import pwd - return pwd.getpwuid(os.geteuid()).pw_name - -def get_runtime_dir(): - explicit = os.environ.get("XDG_RUNTIME_DIR", "") - if explicit: return explicit - user = os_getlogin() - return "/tmp/run-"+user - -def get_home(): - explicit = os.environ.get("HOME", "") - if explicit: return explicit - return os.path.expanduser("~") - -def _var_path(path): - """ assumes that the path starts with /var - when in - user mode it shall be moved to /run/user/1001/run/ - or as a fallback path to /tmp/run-{user}/ so that - you may find /var/log in /tmp/run-{user}/log ..""" - if path.startswith("/var"): - runtime = get_runtime_dir() # $XDG_RUNTIME_DIR - if not os.path.isdir(runtime): - os.makedirs(runtime) - os.chmod(runtime, 0o700) - return re.sub("^(/var)?", get_runtime_dir(), path) - return path - - -def shutil_setuid(user = None, group = None): - """ set fork-child uid/gid (returns pw-info env-settings)""" - if group: - import grp - gid = grp.getgrnam(group).gr_gid - os.setgid(gid) - logg.debug("setgid %s '%s'", gid, group) - if user: - import pwd - pw = pwd.getpwnam(user) - if not group: - gid = pw.pw_gid - os.setgid(gid) - logg.debug("setgid %s", gid) - uid = pw.pw_uid - os.setuid(uid) - logg.debug("setuid %s '%s'", uid, user) - home = pw.pw_dir - shell = pw.pw_shell - logname = pw.pw_name - return { "USER": user, "LOGNAME": logname, "HOME": home, "SHELL": shell } - return {} - -def shutil_truncate(filename): - """ truncates the file (or creates a new empty file)""" - filedir = os.path.dirname(filename) - if not os.path.isdir(filedir): - os.makedirs(filedir) - f = open(filename, "w") - f.write("") - f.close() - -# http://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a-process-with-a-given-pid -def pid_exists(pid): - """Check whether pid exists in the current process table.""" - if pid is None: - return False - return _pid_exists(int(pid)) -def _pid_exists(pid): - """Check whether pid exists in the current process table. - UNIX only. - """ - if pid < 0: - return False - if pid == 0: - # According to "man 2 kill" PID 0 refers to every process - # in the process group of the calling process. - # On certain systems 0 is a valid PID but we have no way - # to know that in a portable fashion. - raise ValueError('invalid PID 0') - try: - os.kill(pid, 0) - except OSError as err: - if err.errno == errno.ESRCH: - # ESRCH == No such process - return False - elif err.errno == errno.EPERM: - # EPERM clearly means there's a process to deny access to - return True - else: - # According to "man 2 kill" possible error values are - # (EINVAL, EPERM, ESRCH) - raise - else: - return True -def pid_zombie(pid): - """ may be a pid exists but it is only a zombie """ - if pid is None: - return False - return _pid_zombie(int(pid)) -def _pid_zombie(pid): - """ may be a pid exists but it is only a zombie """ - if pid < 0: - return False - if pid == 0: - # According to "man 2 kill" PID 0 refers to every process - # in the process group of the calling process. - # On certain systems 0 is a valid PID but we have no way - # to know that in a portable fashion. - raise ValueError('invalid PID 0') - check = "/proc/%s/status" % pid - try: - for line in open(check): - if line.startswith("State:"): - return "Z" in line - except IOError as e: - if e.errno != errno.ENOENT: - logg.error("%s (%s): %s", check, e.errno, e) - return False - return False - -def checkstatus(cmd): - if cmd.startswith("-"): - return False, cmd[1:] - else: - return True, cmd - -# https://github.com/phusion/baseimage-docker/blob/rel-0.9.16/image/bin/my_init -def ignore_signals_and_raise_keyboard_interrupt(signame): - signal.signal(signal.SIGTERM, signal.SIG_IGN) - signal.signal(signal.SIGINT, signal.SIG_IGN) - raise KeyboardInterrupt(signame) - -class SystemctlConfigParser: - """ A *.service files has a structure similar to an *.ini file but it is - actually not like it. Settings may occur multiple times in each section - and they create an implicit list. In reality all the settings are - globally uniqute, so that an 'environment' can be printed without - adding prefixes. Settings are continued with a backslash at the end - of the line. """ - def __init__(self, defaults=None, dict_type=None, allow_no_value=False): - self._defaults = defaults or {} - self._dict_type = dict_type or collections.OrderedDict - self._allow_no_value = allow_no_value - self._conf = self._dict_type() - self._files = [] - def defaults(self): - return self._defaults - def sections(self): - return list(self._conf.keys()) - def add_section(self, section): - if section not in self._conf: - self._conf[section] = self._dict_type() - def has_section(self, section): - return section in self._conf - def has_option(self, section, option): - if section not in self._conf: - return False - return option in self._conf[section] - def set(self, section, option, value): - if section not in self._conf: - self._conf[section] = self._dict_type() - if option not in self._conf[section]: - self._conf[section][option] = [ value ] - else: - self._conf[section][option].append(value) - if value is None: - self._conf[section][option] = [] - def get(self, section, option, default = None, allow_no_value = False): - allow_no_value = allow_no_value or self._allow_no_value - if section not in self._conf: - if default is not None: - return default - if allow_no_value: - return None - logg.warning("section {} does not exist".format(section)) - logg.warning(" have {}".format(self.sections())) - raise AttributeError("section {} does not exist".format(section)) - if option not in self._conf[section]: - if default is not None: - return default - if allow_no_value: - return None - raise AttributeError("option {} in {} does not exist".format(option, section)) - if not self._conf[section][option]: # i.e. an empty list - if default is not None: - return default - if allow_no_value: - return None - raise AttributeError("option {} in {} is None".format(option, section)) - return self._conf[section][option][0] # the first line in the list of configs - def getlist(self, section, option, default = None, allow_no_value = False): - allow_no_value = allow_no_value or self._allow_no_value - if section not in self._conf: - if default is not None: - return default - if allow_no_value: - return [] - logg.warning("section {} does not exist".format(section)) - logg.warning(" have {}".format(self.sections())) - raise AttributeError("section {} does not exist".format(section)) - if option not in self._conf[section]: - if default is not None: - return default - if allow_no_value: - return [] - raise AttributeError("option {} in {} does not exist".format(option, section)) - return self._conf[section][option] # returns a list, possibly empty - def read(self, filename): - return self.read_sysd(filename) - def read_sysd(self, filename): - initscript = False - initinfo = False - section = None - nextline = False - name, text = "", "" - if os.path.isfile(filename): - self._files.append(filename) - for orig_line in open(filename): - if nextline: - text += orig_line - if text.rstrip().endswith("\\") or text.rstrip().endswith("\\\n"): - text = text.rstrip() + "\n" - else: - self.set(section, name, text) - nextline = False - continue - line = orig_line.strip() - if not line: - continue - if line.startswith("#"): - continue - if line.startswith(";"): - continue - if line.startswith(".include"): - logg.error("the '.include' syntax is deprecated. Use x.service.d/ drop-in files!") - includefile = re.sub(r'^\.include[ ]*', '', line).rstrip() - if not os.path.isfile(includefile): - raise Exception("tried to include file that doesn't exist: %s" % includefile) - self.read_sysd(includefile) - continue - if line.startswith("["): - x = line.find("]") - if x > 0: - section = line[1:x] - self.add_section(section) - continue - m = re.match(r"(\w+) *=(.*)", line) - if not m: - logg.warning("bad ini line: %s", line) - raise Exception("bad ini line") - name, text = m.group(1), m.group(2).strip() - if text.endswith("\\") or text.endswith("\\\n"): - nextline = True - text = text + "\n" - else: - # hint: an empty line shall reset the value-list - self.set(section, name, text and text or None) - def read_sysv(self, filename): - """ an LSB header is scanned and converted to (almost) - equivalent settings of a SystemD ini-style input """ - initscript = False - initinfo = False - section = None - if os.path.isfile(filename): - self._files.append(filename) - for orig_line in open(filename): - line = orig_line.strip() - if line.startswith("#"): - if " BEGIN INIT INFO" in line: - initinfo = True - section = "init.d" - if " END INIT INFO" in line: - initinfo = False - if initinfo: - m = re.match(r"\S+\s*(\w[\w_-]*):(.*)", line) - if m: - key, val = m.group(1), m.group(2).strip() - self.set(section, key, val) - continue - description = self.get("init.d", "Description", "") - if description: - self.set("Unit", "Description", description) - check = self.get("init.d", "Required-Start","") - if check: - for item in check.split(" "): - if item.strip() in _sysv_mappings: - self.set("Unit", "Requires", _sysv_mappings[item.strip()]) - provides = self.get("init.d", "Provides", "") - if provides: - self.set("Install", "Alias", provides) - # if already in multi-user.target then start it there. - runlevels = self.get("init.d", "Default-Start","") - if runlevels: - for item in runlevels.split(" "): - if item.strip() in _runlevel_mappings: - self.set("Install", "WantedBy", _runlevel_mappings[item.strip()]) - self.set("Service", "Type", "sysv") - def filenames(self): - return self._files - -# UnitConfParser = ConfigParser.RawConfigParser -UnitConfParser = SystemctlConfigParser - -class SystemctlConf: - def __init__(self, data, module = None): - self.data = data # UnitConfParser - self.env = {} - self.status = None - self.masked = None - self.module = module - self.drop_in_files = {} - self._root = _root - self._user_mode = _user_mode - def os_path(self, path): - return os_path(self._root, path) - def os_path_var(self, path): - if self._user_mode: - return os_path(self._root, _var_path(path)) - return os_path(self._root, path) - def loaded(self): - files = self.data.filenames() - if self.masked: - return "masked" - if len(files): - return "loaded" - return "" - def filename(self): - """ returns the last filename that was parsed """ - files = self.data.filenames() - if files: - return files[0] - return None - def overrides(self): - """ drop-in files are loaded alphabetically by name, not by full path """ - return [ self.drop_in_files[name] for name in sorted(self.drop_in_files) ] - def name(self): - """ the unit id or defaults to the file name """ - name = self.module or "" - filename = self.filename() - if filename: - name = os.path.basename(filename) - return self.get("Unit", "Id", name) - def set(self, section, name, value): - return self.data.set(section, name, value) - def get(self, section, name, default, allow_no_value = False): - return self.data.get(section, name, default, allow_no_value) - def getlist(self, section, name, default = None, allow_no_value = False): - return self.data.getlist(section, name, default or [], allow_no_value) - def getbool(self, section, name, default = None): - value = self.data.get(section, name, default or "no") - if value: - if value[0] in "TtYy123456789": - return True - return False - -class PresetFile: - def __init__(self): - self._files = [] - self._lines = [] - def filename(self): - """ returns the last filename that was parsed """ - if self._files: - return self._files[-1] - return None - def read(self, filename): - self._files.append(filename) - for line in open(filename): - self._lines.append(line.strip()) - return self - def get_preset(self, unit): - for line in self._lines: - m = re.match(r"(enable|disable)\s+(\S+)", line) - if m: - status, pattern = m.group(1), m.group(2) - if fnmatch.fnmatchcase(unit, pattern): - logg.debug("%s %s => %s [%s]", status, pattern, unit, self.filename()) - return status - return None - -## with waitlock(conf): self.start() -class waitlock: - def __init__(self, conf): - self.conf = conf # currently unused - self.opened = None - self.lockfolder = conf.os_path_var(_notify_socket_folder) - try: - folder = self.lockfolder - if not os.path.isdir(folder): - os.makedirs(folder) - except Exception as e: - logg.warning("oops, %s", e) - def lockfile(self): - unit = "" - if self.conf: - unit = self.conf.name() - return os.path.join(self.lockfolder, str(unit or "global") + ".lock") - def __enter__(self): - try: - lockfile = self.lockfile() - lockname = os.path.basename(lockfile) - self.opened = os.open(lockfile, os.O_RDWR | os.O_CREAT, 0o600) - for attempt in xrange(int(MaxLockWait or DefaultMaximumTimeout)): - try: - logg.debug("[%s] %s. trying %s _______ ", os.getpid(), attempt, lockname) - fcntl.flock(self.opened, fcntl.LOCK_EX | fcntl.LOCK_NB) - st = os.fstat(self.opened) - if not st.st_nlink: - logg.debug("[%s] %s. %s got deleted, trying again", os.getpid(), attempt, lockname) - os.close(self.opened) - self.opened = os.open(lockfile, os.O_RDWR | os.O_CREAT, 0o600) - continue - content = "{ 'systemctl': %s, 'lock': '%s' }\n" % (os.getpid(), lockname) - os.write(self.opened, content.encode("utf-8")) - logg.debug("[%s] %s. holding lock on %s", os.getpid(), attempt, lockname) - return True - except BlockingIOError as e: - whom = os.read(self.opened, 4096) - os.lseek(self.opened, 0, os.SEEK_SET) - logg.info("[%s] %s. systemctl locked by %s", os.getpid(), attempt, whom.rstrip()) - time.sleep(1) # until MaxLockWait - continue - logg.error("[%s] not able to get the lock to %s", os.getpid(), lockname) - except Exception as e: - logg.warning("[%s] oops %s, %s", os.getpid(), str(type(e)), e) - #TODO# raise Exception("no lock for %s", self.unit or "global") - return False - def __exit__(self, type, value, traceback): - try: - os.lseek(self.opened, 0, os.SEEK_SET) - os.ftruncate(self.opened, 0) - if "removelockfile" in COVERAGE: # actually an optional implementation - lockfile = self.lockfile() - lockname = os.path.basename(lockfile) - os.unlink(lockfile) # ino is kept allocated because opened by this process - logg.debug("[%s] lockfile removed for %s", os.getpid(), lockname) - fcntl.flock(self.opened, fcntl.LOCK_UN) - os.close(self.opened) # implies an unlock but that has happend like 6 seconds later - self.opened = None - except Exception as e: - logg.warning("oops, %s", e) - -def must_have_failed(waitpid, cmd): - # found to be needed on ubuntu:16.04 to match test result from ubuntu:18.04 and other distros - # .... I have tracked it down that python's os.waitpid() returns an exitcode==0 even when the - # .... underlying process has actually failed with an exitcode<>0. It is unknown where that - # .... bug comes from but it seems a bit serious to trash some very basic unix functionality. - # .... Essentially a parent process does not get the correct exitcode from its own children. - if cmd and cmd[0] == "/bin/kill": - pid = None - for arg in cmd[1:]: - if not arg.startswith("-"): - pid = arg - if pid is None: # unknown $MAINPID - if not waitpid.returncode: - logg.error("waitpid %s did return %s => correcting as 11", cmd, waitpid.returncode) - waitpidNEW = collections.namedtuple("waitpidNEW", ["pid", "returncode", "signal" ]) - waitpid = waitpidNEW(waitpid.pid, 11, waitpid.signal) - return waitpid - -def subprocess_waitpid(pid): - waitpid = collections.namedtuple("waitpid", ["pid", "returncode", "signal" ]) - run_pid, run_stat = os.waitpid(pid, 0) - return waitpid(run_pid, os.WEXITSTATUS(run_stat), os.WTERMSIG(run_stat)) -def subprocess_testpid(pid): - testpid = collections.namedtuple("testpid", ["pid", "returncode", "signal" ]) - run_pid, run_stat = os.waitpid(pid, os.WNOHANG) - if run_pid: - return testpid(run_pid, os.WEXITSTATUS(run_stat), os.WTERMSIG(run_stat)) - else: - return testpid(pid, None, 0) - -def parse_unit(name): # -> object(prefix, instance, suffix, ...., name, component) - unit_name, suffix = name, "" - has_suffix = name.rfind(".") - if has_suffix > 0: - unit_name = name[:has_suffix] - suffix = name[has_suffix+1:] - prefix, instance = unit_name, "" - has_instance = unit_name.find("@") - if has_instance > 0: - prefix = unit_name[:has_instance] - instance = unit_name[has_instance+1:] - component = "" - has_component = prefix.rfind("-") - if has_component > 0: - component = prefix[has_component+1:] - UnitName = collections.namedtuple("UnitName", ["name", "prefix", "instance", "suffix", "component" ]) - return UnitName(name, prefix, instance, suffix, component) - -def time_to_seconds(text, maximum = None): - if maximum is None: - maximum = DefaultMaximumTimeout - value = 0 - for part in str(text).split(" "): - item = part.strip() - if item == "infinity": - return maximum - if item.endswith("m"): - try: value += 60 * int(item[:-1]) - except: pass # pragma: no cover - if item.endswith("min"): - try: value += 60 * int(item[:-3]) - except: pass # pragma: no cover - elif item.endswith("ms"): - try: value += int(item[:-2]) / 1000. - except: pass # pragma: no cover - elif item.endswith("s"): - try: value += int(item[:-1]) - except: pass # pragma: no cover - elif item: - try: value += int(item) - except: pass # pragma: no cover - if value > maximum: - return maximum - if not value: - return 1 - return value -def seconds_to_time(seconds): - seconds = float(seconds) - mins = int(int(seconds) / 60) - secs = int(int(seconds) - (mins * 60)) - msecs = int(int(seconds * 1000) - (secs * 1000 + mins * 60000)) - if mins and secs and msecs: - return "%smin %ss %sms" % (mins, secs, msecs) - elif mins and secs: - return "%smin %ss" % (mins, secs) - elif secs and msecs: - return "%ss %sms" % (secs, msecs) - elif mins and msecs: - return "%smin %sms" % (mins, msecs) - elif mins: - return "%smin" % (mins) - else: - return "%ss" % (secs) - -def getBefore(conf): - result = [] - beforelist = conf.getlist("Unit", "Before", []) - for befores in beforelist: - for before in befores.split(" "): - name = before.strip() - if name and name not in result: - result.append(name) - return result - -def getAfter(conf): - result = [] - afterlist = conf.getlist("Unit", "After", []) - for afters in afterlist: - for after in afters.split(" "): - name = after.strip() - if name and name not in result: - result.append(name) - return result - -def compareAfter(confA, confB): - idA = confA.name() - idB = confB.name() - for after in getAfter(confA): - if after == idB: - logg.debug("%s After %s", idA, idB) - return -1 - for after in getAfter(confB): - if after == idA: - logg.debug("%s After %s", idB, idA) - return 1 - for before in getBefore(confA): - if before == idB: - logg.debug("%s Before %s", idA, idB) - return 1 - for before in getBefore(confB): - if before == idA: - logg.debug("%s Before %s", idB, idA) - return -1 - return 0 - -def sortedAfter(conflist, cmp = compareAfter): - # the normal sorted() does only look at two items - # so if "A after C" and a list [A, B, C] then - # it will see "A = B" and "B = C" assuming that - # "A = C" and the list is already sorted. - # - # To make a totalsorted we have to create a marker - # that informs sorted() that also B has a relation. - # It only works when 'after' has a direction, so - # anything without 'before' is a 'after'. In that - # case we find that "B after C". - class SortTuple: - def __init__(self, rank, conf): - self.rank = rank - self.conf = conf - sortlist = [ SortTuple(0, conf) for conf in conflist] - for check in xrange(len(sortlist)): # maxrank = len(sortlist) - changed = 0 - for A in xrange(len(sortlist)): - for B in xrange(len(sortlist)): - if A != B: - itemA = sortlist[A] - itemB = sortlist[B] - before = compareAfter(itemA.conf, itemB.conf) - if before > 0 and itemA.rank <= itemB.rank: - if DEBUG_AFTER: # pragma: no cover - logg.info(" %-30s before %s", itemA.conf.name(), itemB.conf.name()) - itemA.rank = itemB.rank + 1 - changed += 1 - if before < 0 and itemB.rank <= itemA.rank: - if DEBUG_AFTER: # pragma: no cover - logg.info(" %-30s before %s", itemB.conf.name(), itemA.conf.name()) - itemB.rank = itemA.rank + 1 - changed += 1 - if not changed: - if DEBUG_AFTER: # pragma: no cover - logg.info("done in check %s of %s", check, len(sortlist)) - break - # because Requires is almost always the same as the After clauses - # we are mostly done in round 1 as the list is in required order - for conf in conflist: - if DEBUG_AFTER: # pragma: no cover - logg.debug(".. %s", conf.name()) - for item in sortlist: - if DEBUG_AFTER: # pragma: no cover - logg.info("(%s) %s", item.rank, item.conf.name()) - sortedlist = sorted(sortlist, key = lambda item: -item.rank) - for item in sortedlist: - if DEBUG_AFTER: # pragma: no cover - logg.info("[%s] %s", item.rank, item.conf.name()) - return [ item.conf for item in sortedlist ] - -class Systemctl: - def __init__(self): - # from command line options or the defaults - self._extra_vars = _extra_vars - self._force = _force - self._full = _full - self._init = _init - self._no_ask_password = _no_ask_password - self._no_legend = _no_legend - self._now = _now - self._preset_mode = _preset_mode - self._quiet = _quiet - self._root = _root - self._show_all = _show_all - self._unit_property = _unit_property - self._unit_state = _unit_state - self._unit_type = _unit_type - # some common constants that may be changed - self._systemd_version = SystemCompatibilityVersion - self._pid_file_folder = _pid_file_folder - self._journal_log_folder = _journal_log_folder - # and the actual internal runtime state - self._loaded_file_sysv = {} # /etc/init.d/name => config data - self._loaded_file_sysd = {} # /etc/systemd/system/name.service => config data - self._file_for_unit_sysv = None # name.service => /etc/init.d/name - self._file_for_unit_sysd = None # name.service => /etc/systemd/system/name.service - self._preset_file_list = None # /etc/systemd/system-preset/* => file content - self._default_target = _default_target - self._sysinit_target = None - self.exit_when_no_more_procs = EXIT_WHEN_NO_MORE_PROCS or False - self.exit_when_no_more_services = EXIT_WHEN_NO_MORE_SERVICES or False - self._user_mode = _user_mode - self._user_getlogin = os_getlogin() - self._log_file = {} # init-loop - self._log_hold = {} # init-loop - def user(self): - return self._user_getlogin - def user_mode(self): - return self._user_mode - def user_folder(self): - for folder in self.user_folders(): - if folder: return folder - raise Exception("did not find any systemd/user folder") - def system_folder(self): - for folder in self.system_folders(): - if folder: return folder - raise Exception("did not find any systemd/system folder") - def init_folders(self): - if _init_folder1: yield _init_folder1 - if _init_folder2: yield _init_folder2 - if _init_folder9: yield _init_folder9 - def preset_folders(self): - if _preset_folder1: yield _preset_folder1 - if _preset_folder2: yield _preset_folder2 - if _preset_folder3: yield _preset_folder3 - if _preset_folder4: yield _preset_folder4 - if _preset_folder9: yield _preset_folder9 - def user_folders(self): - if _user_folder1: yield os.path.expanduser(_user_folder1) - if _user_folder2: yield os.path.expanduser(_user_folder2) - if _user_folder3: yield os.path.expanduser(_user_folder3) - if _user_folder4: yield os.path.expanduser(_user_folder4) - if _user_folder9: yield os.path.expanduser(_user_folder9) - def system_folders(self): - if _system_folder1: yield _system_folder1 - if _system_folder2: yield _system_folder2 - if _system_folder3: yield _system_folder3 - if _system_folder4: yield _system_folder4 - if _system_folder9: yield _system_folder9 - def sysd_folders(self): - """ if --user then these folders are preferred """ - if self.user_mode(): - for folder in self.user_folders(): - yield folder - if True: - for folder in self.system_folders(): - yield folder - def scan_unit_sysd_files(self, module = None): # -> [ unit-names,... ] - """ reads all unit files, returns the first filename for the unit given """ - if self._file_for_unit_sysd is None: - self._file_for_unit_sysd = {} - for folder in self.sysd_folders(): - if not folder: - continue - folder = os_path(self._root, folder) - if not os.path.isdir(folder): - continue - for name in os.listdir(folder): - path = os.path.join(folder, name) - if os.path.isdir(path): - continue - service_name = name - if service_name not in self._file_for_unit_sysd: - self._file_for_unit_sysd[service_name] = path - logg.debug("found %s sysd files", len(self._file_for_unit_sysd)) - return list(self._file_for_unit_sysd.keys()) - def scan_unit_sysv_files(self, module = None): # -> [ unit-names,... ] - """ reads all init.d files, returns the first filename when unit is a '.service' """ - if self._file_for_unit_sysv is None: - self._file_for_unit_sysv = {} - for folder in self.init_folders(): - if not folder: - continue - folder = os_path(self._root, folder) - if not os.path.isdir(folder): - continue - for name in os.listdir(folder): - path = os.path.join(folder, name) - if os.path.isdir(path): - continue - service_name = name + ".service" # simulate systemd - if service_name not in self._file_for_unit_sysv: - self._file_for_unit_sysv[service_name] = path - logg.debug("found %s sysv files", len(self._file_for_unit_sysv)) - return list(self._file_for_unit_sysv.keys()) - def unit_sysd_file(self, module = None): # -> filename? - """ file path for the given module (systemd) """ - self.scan_unit_sysd_files() - if module and module in self._file_for_unit_sysd: - return self._file_for_unit_sysd[module] - if module and unit_of(module) in self._file_for_unit_sysd: - return self._file_for_unit_sysd[unit_of(module)] - return None - def unit_sysv_file(self, module = None): # -> filename? - """ file path for the given module (sysv) """ - self.scan_unit_sysv_files() - if module and module in self._file_for_unit_sysv: - return self._file_for_unit_sysv[module] - if module and unit_of(module) in self._file_for_unit_sysv: - return self._file_for_unit_sysv[unit_of(module)] - return None - def unit_file(self, module = None): # -> filename? - """ file path for the given module (sysv or systemd) """ - path = self.unit_sysd_file(module) - if path is not None: return path - path = self.unit_sysv_file(module) - if path is not None: return path - return None - def is_sysv_file(self, filename): - """ for routines that have a special treatment for init.d services """ - self.unit_file() # scan all - if not filename: return None - if filename in self._file_for_unit_sysd.values(): return False - if filename in self._file_for_unit_sysv.values(): return True - return None # not True - def is_user_conf(self, conf): - if not conf: - return False # no such conf >> ignored - filename = conf.filename() - if filename and "/user/" in filename: - return True - return False - def not_user_conf(self, conf): - """ conf can not be started as user service (when --user)""" - if not conf: - return True # no such conf >> ignored - if not self.user_mode(): - logg.debug("%s no --user mode >> accept", conf.filename()) - return False - if self.is_user_conf(conf): - logg.debug("%s is /user/ conf >> accept", conf.filename()) - return False - # to allow for 'docker run -u user' with system services - user = self.expand_special(conf.get("Service", "User", ""), conf) - if user and user == self.user(): - logg.debug("%s with User=%s >> accept", conf.filename(), user) - return False - return True - def find_drop_in_files(self, unit): - """ search for some.service.d/extra.conf files """ - result = {} - basename_d = unit + ".d" - for folder in self.sysd_folders(): - if not folder: - continue - folder = os_path(self._root, folder) - override_d = os_path(folder, basename_d) - if not os.path.isdir(override_d): - continue - for name in os.listdir(override_d): - path = os.path.join(override_d, name) - if os.path.isdir(path): - continue - if not path.endswith(".conf"): - continue - if name not in result: - result[name] = path - return result - def load_sysd_template_conf(self, module): # -> conf? - """ read the unit template with a UnitConfParser (systemd) """ - if "@" in module: - unit = parse_unit(module) - service = "%s@.service" % unit.prefix - return self.load_sysd_unit_conf(service) - return None - def load_sysd_unit_conf(self, module): # -> conf? - """ read the unit file with a UnitConfParser (systemd) """ - path = self.unit_sysd_file(module) - if not path: return None - if path in self._loaded_file_sysd: - return self._loaded_file_sysd[path] - masked = None - if os.path.islink(path) and os.readlink(path).startswith("/dev"): - masked = os.readlink(path) - drop_in_files = {} - data = UnitConfParser() - if not masked: - data.read_sysd(path) - drop_in_files = self.find_drop_in_files(os.path.basename(path)) - # load in alphabetic order, irrespective of location - for name in sorted(drop_in_files): - path = drop_in_files[name] - data.read_sysd(path) - conf = SystemctlConf(data, module) - conf.masked = masked - conf.drop_in_files = drop_in_files - conf._root = self._root - self._loaded_file_sysd[path] = conf - return conf - def load_sysv_unit_conf(self, module): # -> conf? - """ read the unit file with a UnitConfParser (sysv) """ - path = self.unit_sysv_file(module) - if not path: return None - if path in self._loaded_file_sysv: - return self._loaded_file_sysv[path] - data = UnitConfParser() - data.read_sysv(path) - conf = SystemctlConf(data, module) - conf._root = self._root - self._loaded_file_sysv[path] = conf - return conf - def load_unit_conf(self, module): # -> conf | None(not-found) - """ read the unit file with a UnitConfParser (sysv or systemd) """ - try: - conf = self.load_sysd_unit_conf(module) - if conf is not None: - return conf - conf = self.load_sysd_template_conf(module) - if conf is not None: - return conf - conf = self.load_sysv_unit_conf(module) - if conf is not None: - return conf - except Exception as e: - logg.warning("%s not loaded: %s", module, e) - return None - def default_unit_conf(self, module, description = None): # -> conf - """ a unit conf that can be printed to the user where - attributes are empty and loaded() is False """ - data = UnitConfParser() - data.set("Unit","Id", module) - data.set("Unit", "Names", module) - data.set("Unit", "Description", description or ("NOT-FOUND "+module)) - # assert(not data.loaded()) - conf = SystemctlConf(data, module) - conf._root = self._root - return conf - def get_unit_conf(self, module): # -> conf (conf | default-conf) - """ accept that a unit does not exist - and return a unit conf that says 'not-loaded' """ - conf = self.load_unit_conf(module) - if conf is not None: - return conf - return self.default_unit_conf(module) - def match_sysd_templates(self, modules = None, suffix=".service"): # -> generate[ unit ] - """ make a file glob on all known template units (systemd areas). - It returns no modules (!!) if no modules pattern were given. - The module string should contain an instance name already. """ - modules = to_list(modules) - if not modules: - return - self.scan_unit_sysd_files() - for item in sorted(self._file_for_unit_sysd.keys()): - if "@" not in item: - continue - service_unit = parse_unit(item) - for module in modules: - if "@" not in module: - continue - module_unit = parse_unit(module) - if service_unit.prefix == module_unit.prefix: - yield "%s@%s.%s" % (service_unit.prefix, module_unit.instance, service_unit.suffix) - def match_sysd_units(self, modules = None, suffix=".service"): # -> generate[ unit ] - """ make a file glob on all known units (systemd areas). - It returns all modules if no modules pattern were given. - Also a single string as one module pattern may be given. """ - modules = to_list(modules) - self.scan_unit_sysd_files() - for item in sorted(self._file_for_unit_sysd.keys()): - if not modules: - yield item - elif [ module for module in modules if fnmatch.fnmatchcase(item, module) ]: - yield item - elif [ module for module in modules if module+suffix == item ]: - yield item - def match_sysv_units(self, modules = None, suffix=".service"): # -> generate[ unit ] - """ make a file glob on all known units (sysv areas). - It returns all modules if no modules pattern were given. - Also a single string as one module pattern may be given. """ - modules = to_list(modules) - self.scan_unit_sysv_files() - for item in sorted(self._file_for_unit_sysv.keys()): - if not modules: - yield item - elif [ module for module in modules if fnmatch.fnmatchcase(item, module) ]: - yield item - elif [ module for module in modules if module+suffix == item ]: - yield item - def match_units(self, modules = None, suffix=".service"): # -> [ units,.. ] - """ Helper for about any command with multiple units which can - actually be glob patterns on their respective unit name. - It returns all modules if no modules pattern were given. - Also a single string as one module pattern may be given. """ - found = [] - for unit in self.match_sysd_units(modules, suffix): - if unit not in found: - found.append(unit) - for unit in self.match_sysd_templates(modules, suffix): - if unit not in found: - found.append(unit) - for unit in self.match_sysv_units(modules, suffix): - if unit not in found: - found.append(unit) - return found - def list_service_unit_basics(self): - """ show all the basic loading state of services """ - filename = self.unit_file() # scan all - result = [] - for name, value in self._file_for_unit_sysd.items(): - result += [ (name, "SysD", value) ] - for name, value in self._file_for_unit_sysv.items(): - result += [ (name, "SysV", value) ] - return result - def list_service_units(self, *modules): # -> [ (unit,loaded+active+substate,description) ] - """ show all the service units """ - result = {} - active = {} - substate = {} - description = {} - for unit in self.match_units(modules): - result[unit] = "not-found" - active[unit] = "inactive" - substate[unit] = "dead" - description[unit] = "" - try: - conf = self.get_unit_conf(unit) - result[unit] = "loaded" - description[unit] = self.get_description_from(conf) - active[unit] = self.get_active_from(conf) - substate[unit] = self.get_substate_from(conf) - except Exception as e: - logg.warning("list-units: %s", e) - if self._unit_state: - if self._unit_state not in [ result[unit], active[unit], substate[unit] ]: - del result[unit] - return [ (unit, result[unit] + " " + active[unit] + " " + substate[unit], description[unit]) for unit in sorted(result) ] - def show_list_units(self, *modules): # -> [ (unit,loaded,description) ] - """ [PATTERN]... -- List loaded units. - If one or more PATTERNs are specified, only units matching one of - them are shown. NOTE: This is the default command.""" - hint = "To show all installed unit files use 'systemctl list-unit-files'." - result = self.list_service_units(*modules) - if self._no_legend: - return result - found = "%s loaded units listed." % len(result) - return result + [ "", found, hint ] - def list_service_unit_files(self, *modules): # -> [ (unit,enabled) ] - """ show all the service units and the enabled status""" - logg.debug("list service unit files for %s", modules) - result = {} - enabled = {} - for unit in self.match_units(modules): - result[unit] = None - enabled[unit] = "" - try: - conf = self.get_unit_conf(unit) - if self.not_user_conf(conf): - result[unit] = None - continue - result[unit] = conf - enabled[unit] = self.enabled_from(conf) - except Exception as e: - logg.warning("list-units: %s", e) - return [ (unit, enabled[unit]) for unit in sorted(result) if result[unit] ] - def each_target_file(self): - folders = self.system_folders() - if self.user_mode(): - folders = self.user_folders() - for folder in folders: - if not os.path.isdir(folder): - continue - for filename in os.listdir(folder): - if filename.endswith(".target"): - yield (filename, os.path.join(folder, filename)) - def list_target_unit_files(self, *modules): # -> [ (unit,enabled) ] - """ show all the target units and the enabled status""" - enabled = {} - targets = {} - for target, filepath in self.each_target_file(): - logg.info("target %s", filepath) - targets[target] = filepath - enabled[target] = "static" - for unit in _all_common_targets: - targets[unit] = None - enabled[unit] = "static" - if unit in _all_common_enabled: - enabled[unit] = "enabled" - if unit in _all_common_disabled: - enabled[unit] = "disabled" - return [ (unit, enabled[unit]) for unit in sorted(targets) ] - def show_list_unit_files(self, *modules): # -> [ (unit,enabled) ] - """[PATTERN]... -- List installed unit files - List installed unit files and their enablement state (as reported - by is-enabled). If one or more PATTERNs are specified, only units - whose filename (just the last component of the path) matches one of - them are shown. This command reacts to limitations of --type being - --type=service or --type=target (and --now for some basics).""" - if self._now: - result = self.list_service_unit_basics() - elif self._unit_type == "target": - result = self.list_target_unit_files() - elif self._unit_type == "service": - result = self.list_service_unit_files() - elif self._unit_type: - logg.warning("unsupported unit --type=%s", self._unit_type) - result = [] - else: - result = self.list_target_unit_files() - result += self.list_service_unit_files(*modules) - if self._no_legend: - return result - found = "%s unit files listed." % len(result) - return [ ("UNIT FILE", "STATE") ] + result + [ "", found ] - ## - ## - def get_description(self, unit, default = None): - return self.get_description_from(self.load_unit_conf(unit)) - def get_description_from(self, conf, default = None): # -> text - """ Unit.Description could be empty sometimes """ - if not conf: return default or "" - description = conf.get("Unit", "Description", default or "") - return self.expand_special(description, conf) - def read_pid_file(self, pid_file, default = None): - pid = default - if not pid_file: - return default - if not os.path.isfile(pid_file): - return default - if self.truncate_old(pid_file): - return default - try: - # some pid-files from applications contain multiple lines - for line in open(pid_file): - if line.strip(): - pid = to_int(line.strip()) - break - except Exception as e: - logg.warning("bad read of pid file '%s': %s", pid_file, e) - return pid - def wait_pid_file(self, pid_file, timeout = None): # -> pid? - """ wait some seconds for the pid file to appear and return the pid """ - timeout = int(timeout or (DefaultTimeoutStartSec/2)) - timeout = max(timeout, (MinimumTimeoutStartSec)) - dirpath = os.path.dirname(os.path.abspath(pid_file)) - for x in xrange(timeout): - if not os.path.isdir(dirpath): - time.sleep(1) # until TimeoutStartSec/2 - continue - pid = self.read_pid_file(pid_file) - if not pid: - time.sleep(1) # until TimeoutStartSec/2 - continue - if not pid_exists(pid): - time.sleep(1) # until TimeoutStartSec/2 - continue - return pid - return None - def test_pid_file(self, unit): # -> text - """ support for the testsuite.py """ - conf = self.get_unit_conf(unit) - return self.pid_file_from(conf) or self.status_file_from(conf) - def pid_file_from(self, conf, default = ""): - """ get the specified pid file path (not a computed default) """ - pid_file = conf.get("Service", "PIDFile", default) - return self.expand_special(pid_file, conf) - def read_mainpid_from(self, conf, default): - """ MAINPID is either the PIDFile content written from the application - or it is the value in the status file written by this systemctl.py code """ - pid_file = self.pid_file_from(conf) - if pid_file: - return self.read_pid_file(pid_file, default) - status = self.read_status_from(conf) - return status.get("MainPID", default) - def clean_pid_file_from(self, conf): - pid_file = self.pid_file_from(conf) - if pid_file and os.path.isfile(pid_file): - try: - os.remove(pid_file) - except OSError as e: - logg.warning("while rm %s: %s", pid_file, e) - self.write_status_from(conf, MainPID=None) - def get_status_file(self, unit): # for testing - conf = self.get_unit_conf(unit) - return self.status_file_from(conf) - def status_file_from(self, conf, default = None): - if default is None: - default = self.default_status_file(conf) - if conf is None: return default - status_file = conf.get("Service", "StatusFile", default) - # this not a real setting, but do the expand_special anyway - return self.expand_special(status_file, conf) - def default_status_file(self, conf): # -> text - """ default file pattern where to store a status mark """ - folder = conf.os_path_var(self._pid_file_folder) - name = "%s.status" % conf.name() - return os.path.join(folder, name) - def clean_status_from(self, conf): - status_file = self.status_file_from(conf) - if os.path.exists(status_file): - os.remove(status_file) - conf.status = {} - def write_status_from(self, conf, **status): # -> bool(written) - """ if a status_file is known then path is created and the - give status is written as the only content. """ - status_file = self.status_file_from(conf) - if not status_file: - logg.debug("status %s but no status_file", conf.name()) - return False - dirpath = os.path.dirname(os.path.abspath(status_file)) - if not os.path.isdir(dirpath): - os.makedirs(dirpath) - if conf.status is None: - conf.status = self.read_status_from(conf) - if True: - for key in sorted(status.keys()): - value = status[key] - if key.upper() == "AS": key = "ActiveState" - if key.upper() == "EXIT": key = "ExecMainCode" - if value is None: - try: del conf.status[key] - except KeyError: pass - else: - conf.status[key] = value - try: - with open(status_file, "w") as f: - for key in sorted(conf.status): - value = conf.status[key] - if key == "MainPID" and str(value) == "0": - logg.warning("ignore writing MainPID=0") - continue - content = "{}={}\n".format(key, str(value)) - logg.debug("writing to %s\n\t%s", status_file, content.strip()) - f.write(content) - except IOError as e: - logg.error("writing STATUS %s: %s\n\t to status file %s", status, e, status_file) - return True - def read_status_from(self, conf, defaults = None): - status_file = self.status_file_from(conf) - status = {} - if hasattr(defaults, "keys"): - for key in defaults.keys(): - status[key] = defaults[key] - elif isinstance(defaults, string_types): - status["ActiveState"] = defaults - if not status_file: - logg.debug("no status file. returning %s", status) - return status - if not os.path.isfile(status_file): - logg.debug("no status file: %s\n returning %s", status_file, status) - return status - if self.truncate_old(status_file): - logg.debug("old status file: %s\n returning %s", status_file, status) - return status - try: - logg.debug("reading %s", status_file) - for line in open(status_file): - if line.strip(): - m = re.match(r"(\w+)[:=](.*)", line) - if m: - key, value = m.group(1), m.group(2) - if key.strip(): - status[key.strip()] = value.strip() - elif line in [ "active", "inactive", "failed"]: - status["ActiveState"] = line - else: - logg.warning("ignored %s", line.strip()) - except: - logg.warning("bad read of status file '%s'", status_file) - return status - def get_status_from(self, conf, name, default = None): - if conf.status is None: - conf.status = self.read_status_from(conf) - return conf.status.get(name, default) - def set_status_from(self, conf, name, value): - if conf.status is None: - conf.status = self.read_status_from(conf) - if value is None: - try: del conf.status[name] - except KeyError: pass - else: - conf.status[name] = value - # - def wait_boot(self, hint = None): - booted = self.get_boottime() - while True: - now = time.time() - if booted + EpsilonTime <= now: - break - time.sleep(EpsilonTime) - logg.info(" %s ................. boot sleep %ss", hint or "", EpsilonTime) - def get_boottime(self): - if "oldest" in COVERAGE: - return self.get_boottime_oldest() - for pid in xrange(10): - proc = "/proc/%s/status" % pid - try: - if os.path.exists(proc): - return os.path.getmtime(proc) - except Exception as e: # pragma: nocover - logg.warning("could not access %s: %s", proc, e) - return self.get_boottime_oldest() - def get_boottime_oldest(self): - # otherwise get the oldest entry in /proc - booted = time.time() - for name in os.listdir("/proc"): - proc = "/proc/%s/status" % name - try: - if os.path.exists(proc): - ctime = os.path.getmtime(proc) - if ctime < booted: - booted = ctime - except Exception as e: # pragma: nocover - logg.warning("could not access %s: %s", proc, e) - return booted - def get_filetime(self, filename): - return os.path.getmtime(filename) - def truncate_old(self, filename): - filetime = self.get_filetime(filename) - boottime = self.get_boottime() - if isinstance(filetime, float): - filetime -= EpsilonTime - if filetime >= boottime : - logg.debug(" file time: %s", datetime.datetime.fromtimestamp(filetime)) - logg.debug(" boot time: %s", datetime.datetime.fromtimestamp(boottime)) - return False # OK - logg.info("truncate old %s", filename) - logg.info(" file time: %s", datetime.datetime.fromtimestamp(filetime)) - logg.info(" boot time: %s", datetime.datetime.fromtimestamp(boottime)) - try: - shutil_truncate(filename) - except Exception as e: - logg.warning("while truncating: %s", e) - return True # truncated - def getsize(self, filename): - if not filename: - return 0 - if not os.path.isfile(filename): - return 0 - if self.truncate_old(filename): - return 0 - try: - return os.path.getsize(filename) - except Exception as e: - logg.warning("while reading file size: %s\n of %s", e, filename) - return 0 - # - def read_env_file(self, env_file): # -> generate[ (name,value) ] - """ EnvironmentFile= is being scanned """ - if env_file.startswith("-"): - env_file = env_file[1:] - if not os.path.isfile(os_path(self._root, env_file)): - return - try: - for real_line in open(os_path(self._root, env_file)): - line = real_line.strip() - if not line or line.startswith("#"): - continue - m = re.match(r"(?:export +)?([\w_]+)[=]'([^']*)'", line) - if m: - yield m.group(1), m.group(2) - continue - m = re.match(r'(?:export +)?([\w_]+)[=]"([^"]*)"', line) - if m: - yield m.group(1), m.group(2) - continue - m = re.match(r'(?:export +)?([\w_]+)[=](.*)', line) - if m: - yield m.group(1), m.group(2) - continue - except Exception as e: - logg.info("while reading %s: %s", env_file, e) - def read_env_part(self, env_part): # -> generate[ (name, value) ] - """ Environment== is being scanned """ - ## systemd Environment= spec says it is a space-seperated list of - ## assignments. In order to use a space or an equals sign in a value - ## one should enclose the whole assignment with double quotes: - ## Environment="VAR1=word word" VAR2=word3 "VAR3=$word 5 6" - ## and the $word is not expanded by other environment variables. - try: - for real_line in env_part.split("\n"): - line = real_line.strip() - for found in re.finditer(r'\s*("[\w_]+=[^"]*"|[\w_]+=\S*)', line): - part = found.group(1) - if part.startswith('"'): - part = part[1:-1] - name, value = part.split("=", 1) - yield name, value - except Exception as e: - logg.info("while reading %s: %s", env_part, e) - def show_environment(self, unit): - """ [UNIT]. -- show environment parts """ - conf = self.load_unit_conf(unit) - if conf is None: - logg.error("Unit %s could not be found.", unit) - return False - if _unit_property: - return conf.getlist("Service", _unit_property) - return self.get_env(conf) - def extra_vars(self): - return self._extra_vars # from command line - def get_env(self, conf): - env = os.environ.copy() - for env_part in conf.getlist("Service", "Environment", []): - for name, value in self.read_env_part(self.expand_special(env_part, conf)): - env[name] = value # a '$word' is not special here - for env_file in conf.getlist("Service", "EnvironmentFile", []): - for name, value in self.read_env_file(self.expand_special(env_file, conf)): - env[name] = self.expand_env(value, env) - logg.debug("extra-vars %s", self.extra_vars()) - for extra in self.extra_vars(): - if extra.startswith("@"): - for name, value in self.read_env_file(extra[1:]): - logg.info("override %s=%s", name, value) - env[name] = self.expand_env(value, env) - else: - for name, value in self.read_env_part(extra): - logg.info("override %s=%s", name, value) - env[name] = value # a '$word' is not special here - return env - def expand_env(self, cmd, env): - def get_env1(m): - if m.group(1) in env: - return env[m.group(1)] - logg.debug("can not expand $%s", m.group(1)) - return "" # empty string - def get_env2(m): - if m.group(1) in env: - return env[m.group(1)] - logg.debug("can not expand ${%s}", m.group(1)) - return "" # empty string - # - maxdepth = 20 - expanded = re.sub("[$](\w+)", lambda m: get_env1(m), cmd.replace("\\\n","")) - for depth in xrange(maxdepth): - new_text = re.sub("[$][{](\w+)[}]", lambda m: get_env2(m), expanded) - if new_text == expanded: - return expanded - expanded = new_text - logg.error("shell variable expansion exceeded maxdepth %s", maxdepth) - return expanded - def expand_special(self, cmd, conf = None): - """ expand %i %t and similar special vars. They are being expanded - before any other expand_env takes place which handles shell-style - $HOME references. """ - def sh_escape(value): - return "'" + value.replace("'","\\'") + "'" - def get_confs(conf): - confs={ "%": "%" } - if not conf: - return confs - unit = parse_unit(conf.name()) - confs["N"] = unit.name - confs["n"] = sh_escape(unit.name) - confs["P"] = unit.prefix - confs["p"] = sh_escape(unit.prefix) - confs["I"] = unit.instance - confs["i"] = sh_escape(unit.instance) - confs["J"] = unit.component - confs["j"] = sh_escape(unit.component) - confs["f"] = sh_escape(conf.filename()) - VARTMP = "/var/tmp" - TMP = "/tmp" - RUN = "/run" - DAT = "/var/lib" - LOG = "/var/log" - CACHE = "/var/cache" - CONFIG = "/etc" - HOME = "/root" - USER = "root" - UID = 0 - SHELL = "/bin/sh" - if self.is_user_conf(conf): - USER = os_getlogin() - HOME = get_home() - RUN = os.environ.get("XDG_RUNTIME_DIR", get_runtime_dir()) - CONFIG = os.environ.get("XDG_CONFIG_HOME", HOME + "/.config") - CACHE = os.environ.get("XDG_CACHE_HOME", HOME + "/.cache") - SHARE = os.environ.get("XDG_DATA_HOME", HOME + "/.local/share") - DAT = CONFIG - LOG = os.path.join(CONFIG, "log") - SHELL = os.environ.get("SHELL", SHELL) - VARTMP = os.environ.get("TMPDIR", os.environ.get("TEMP", os.environ.get("TMP", VARTMP))) - TMP = os.environ.get("TMPDIR", os.environ.get("TEMP", os.environ.get("TMP", TMP))) - confs["V"] = os_path(self._root, VARTMP) - confs["T"] = os_path(self._root, TMP) - confs["t"] = os_path(self._root, RUN) - confs["S"] = os_path(self._root, DAT) - confs["s"] = SHELL - confs["h"] = HOME - confs["u"] = USER - confs["C"] = os_path(self._root, CACHE) - confs["E"] = os_path(self._root, CONFIG) - return confs - def get_conf1(m): - confs = get_confs(conf) - if m.group(1) in confs: - return confs[m.group(1)] - logg.warning("can not expand %%%s", m.group(1)) - return "''" # empty escaped string - return re.sub("[%](.)", lambda m: get_conf1(m), cmd) - def exec_cmd(self, cmd, env, conf = None): - """ expand ExecCmd statements including %i and $MAINPID """ - cmd1 = cmd.replace("\\\n","") - # according to documentation the %n / %% need to be expanded where in - # most cases they are shell-escaped values. So we do it before shlex. - cmd2 = self.expand_special(cmd1, conf) - # according to documentation, when bar="one two" then the expansion - # of '$bar' is ["one","two"] and '${bar}' becomes ["one two"]. We - # tackle that by expand $bar before shlex, and the rest thereafter. - def get_env1(m): - if m.group(1) in env: - return env[m.group(1)] - logg.debug("can not expand $%s", m.group(1)) - return "" # empty string - def get_env2(m): - if m.group(1) in env: - return env[m.group(1)] - logg.debug("can not expand ${%s}", m.group(1)) - return "" # empty string - cmd3 = re.sub("[$](\w+)", lambda m: get_env1(m), cmd2) - newcmd = [] - for part in shlex.split(cmd3): - newcmd += [ re.sub("[$][{](\w+)[}]", lambda m: get_env2(m), part) ] - return newcmd - def path_journal_log(self, conf): # never None - """ /var/log/zzz.service.log or /var/log/default.unit.log """ - filename = os.path.basename(conf.filename() or "") - unitname = (conf.name() or "default")+".unit" - name = filename or unitname - log_folder = conf.os_path_var(self._journal_log_folder) - log_file = name.replace(os.path.sep,".") + ".log" - if log_file.startswith("."): - log_file = "dot."+log_file - return os.path.join(log_folder, log_file) - def open_journal_log(self, conf): - log_file = self.path_journal_log(conf) - log_folder = os.path.dirname(log_file) - if not os.path.isdir(log_folder): - os.makedirs(log_folder) - return open(os.path.join(log_file), "a") - def chdir_workingdir(self, conf): - """ if specified then change the working directory """ - # the original systemd will start in '/' even if User= is given - if self._root: - os.chdir(self._root) - workingdir = conf.get("Service", "WorkingDirectory", "") - if workingdir: - ignore = False - if workingdir.startswith("-"): - workingdir = workingdir[1:] - ignore = True - into = os_path(self._root, self.expand_special(workingdir, conf)) - try: - logg.debug("chdir workingdir '%s'", into) - os.chdir(into) - return False - except Exception as e: - if not ignore: - logg.error("chdir workingdir '%s': %s", into, e) - return into - else: - logg.debug("chdir workingdir '%s': %s", into, e) - return None - return None - def notify_socket_from(self, conf, socketfile = None): - """ creates a notify-socket for the (non-privileged) user """ - NotifySocket = collections.namedtuple("NotifySocket", ["socket", "socketfile" ]) - notify_socket_folder = conf.os_path_var(_notify_socket_folder) - notify_name = "notify." + str(conf.name() or "systemctl") - notify_socket = os.path.join(notify_socket_folder, notify_name) - socketfile = socketfile or notify_socket - if len(socketfile) > 100: - logg.debug("https://unix.stackexchange.com/questions/367008/%s", - "why-is-socket-path-length-limited-to-a-hundred-chars") - logg.debug("old notify socketfile (%s) = %s", len(socketfile), socketfile) - notify_socket_folder = re.sub("^(/var)?", get_runtime_dir(), _notify_socket_folder) - notify_name = notify_name[0:min(100-len(notify_socket_folder),len(notify_name))] - socketfile = os.path.join(notify_socket_folder, notify_name) - # occurs during testsuite.py for ~user/test.tmp/root path - logg.info("new notify socketfile (%s) = %s", len(socketfile), socketfile) - try: - if not os.path.isdir(os.path.dirname(socketfile)): - os.makedirs(os.path.dirname(socketfile)) - if os.path.exists(socketfile): - os.unlink(socketfile) - except Exception as e: - logg.warning("error %s: %s", socketfile, e) - sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - sock.bind(socketfile) - os.chmod(socketfile, 0o777) # the service my run under some User=setting - return NotifySocket(sock, socketfile) - def read_notify_socket(self, notify, timeout): - notify.socket.settimeout(timeout or DefaultMaximumTimeout) - result = "" - try: - result, client_address = notify.socket.recvfrom(4096) - if result: - result = result.decode("utf-8") - result_txt = result.replace("\n","|") - result_len = len(result) - logg.debug("read_notify_socket(%s):%s", result_len, result_txt) - except socket.timeout as e: - if timeout > 2: - logg.debug("socket.timeout %s", e) - return result - def wait_notify_socket(self, notify, timeout, pid = None): - if not os.path.exists(notify.socketfile): - logg.info("no $NOTIFY_SOCKET exists") - return {} - # - logg.info("wait $NOTIFY_SOCKET, timeout %s", timeout) - results = {} - seenREADY = None - for attempt in xrange(timeout+1): - if pid and not self.is_active_pid(pid): - logg.info("dead PID %s", pid) - return results - if not attempt: # first one - time.sleep(1) # until TimeoutStartSec - continue - result = self.read_notify_socket(notify, 1) # sleep max 1 second - if not result: # timeout - time.sleep(1) # until TimeoutStartSec - continue - for name, value in self.read_env_part(result): - results[name] = value - if name == "READY": - seenREADY = value - if name in ["STATUS", "ACTIVESTATE"]: - logg.debug("%s: %s", name, value) # TODO: update STATUS -> SubState - if seenREADY: - break - if not seenREADY: - logg.info(".... timeout while waiting for 'READY=1' status on $NOTIFY_SOCKET") - logg.debug("notify = %s", results) - try: - notify.socket.close() - except Exception as e: - logg.debug("socket.close %s", e) - return results - def start_modules(self, *modules): - """ [UNIT]... -- start these units - /// SPECIAL: with --now or --init it will - run the init-loop and stop the units afterwards """ - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - init = self._now or self._init - return self.start_units(units, init) and found_all - def start_units(self, units, init = None): - """ fails if any unit does not start - /// SPECIAL: may run the init-loop and - stop the named units afterwards """ - self.wait_system() - done = True - started_units = [] - for unit in self.sortedAfter(units): - started_units.append(unit) - if not self.start_unit(unit): - done = False - if init: - logg.info("init-loop start") - sig = self.init_loop_until_stop(started_units) - logg.info("init-loop %s", sig) - for unit in reversed(started_units): - self.stop_unit(unit) - return done - def start_unit(self, unit): - conf = self.load_unit_conf(unit) - if conf is None: - logg.debug("unit could not be loaded (%s)", unit) - logg.error("Unit %s could not be found.", unit) - return False - if self.not_user_conf(conf): - logg.error("Unit %s not for --user mode", unit) - return False - return self.start_unit_from(conf) - def get_TimeoutStartSec(self, conf): - timeout = conf.get("Service", "TimeoutSec", DefaultTimeoutStartSec) - timeout = conf.get("Service", "TimeoutStartSec", timeout) - return time_to_seconds(timeout, DefaultMaximumTimeout) - def start_unit_from(self, conf): - if not conf: return False - if self.syntax_check(conf) > 100: return False - with waitlock(conf): - logg.debug(" start unit %s => %s", conf.name(), conf.filename()) - return self.do_start_unit_from(conf) - def do_start_unit_from(self, conf): - timeout = self.get_TimeoutStartSec(conf) - doRemainAfterExit = conf.getbool("Service", "RemainAfterExit", "no") - runs = conf.get("Service", "Type", "simple").lower() - env = self.get_env(conf) - self.exec_check_service(conf, env, "Exec") # all... - # for StopPost on failure: - returncode = 0 - service_result = "success" - if True: - if runs in [ "simple", "forking", "notify" ]: - env["MAINPID"] = str(self.read_mainpid_from(conf, "")) - for cmd in conf.getlist("Service", "ExecStartPre", []): - check, cmd = checkstatus(cmd) - newcmd = self.exec_cmd(cmd, env, conf) - logg.info(" pre-start %s", shell_cmd(newcmd)) - forkpid = os.fork() - if not forkpid: - self.execve_from(conf, newcmd, env) # pragma: nocover - run = subprocess_waitpid(forkpid) - logg.debug(" pre-start done (%s) <-%s>", - run.returncode or "OK", run.signal or "") - if run.returncode and check: - logg.error("the ExecStartPre control process exited with error code") - active = "failed" - self.write_status_from(conf, AS=active ) - return False - if runs in [ "sysv" ]: - status_file = self.status_file_from(conf) - if True: - exe = conf.filename() - cmd = "'%s' start" % exe - env["SYSTEMCTL_SKIP_REDIRECT"] = "yes" - newcmd = self.exec_cmd(cmd, env, conf) - logg.info("%s start %s", runs, shell_cmd(newcmd)) - forkpid = os.fork() - if not forkpid: # pragma: no cover - os.setsid() # detach child process from parent - self.execve_from(conf, newcmd, env) - run = subprocess_waitpid(forkpid) - self.set_status_from(conf, "ExecMainCode", run.returncode) - logg.info("%s start done (%s) <-%s>", runs, - run.returncode or "OK", run.signal or "") - active = run.returncode and "failed" or "active" - self.write_status_from(conf, AS=active ) - return True - elif runs in [ "oneshot" ]: - status_file = self.status_file_from(conf) - if self.get_status_from(conf, "ActiveState", "unknown") == "active": - logg.warning("the service was already up once") - return True - for cmd in conf.getlist("Service", "ExecStart", []): - check, cmd = checkstatus(cmd) - newcmd = self.exec_cmd(cmd, env, conf) - logg.info("%s start %s", runs, shell_cmd(newcmd)) - forkpid = os.fork() - if not forkpid: # pragma: no cover - os.setsid() # detach child process from parent - self.execve_from(conf, newcmd, env) - run = subprocess_waitpid(forkpid) - if run.returncode and check: - returncode = run.returncode - service_result = "failed" - logg.error("%s start %s (%s) <-%s>", runs, service_result, - run.returncode or "OK", run.signal or "") - break - logg.info("%s start done (%s) <-%s>", runs, - run.returncode or "OK", run.signal or "") - if True: - self.set_status_from(conf, "ExecMainCode", returncode) - active = returncode and "failed" or "active" - self.write_status_from(conf, AS=active) - elif runs in [ "simple" ]: - status_file = self.status_file_from(conf) - pid = self.read_mainpid_from(conf, "") - if self.is_active_pid(pid): - logg.warning("the service is already running on PID %s", pid) - return True - if doRemainAfterExit: - logg.debug("%s RemainAfterExit -> AS=active", runs) - self.write_status_from(conf, AS="active") - cmdlist = conf.getlist("Service", "ExecStart", []) - for idx, cmd in enumerate(cmdlist): - logg.debug("ExecStart[%s]: %s", idx, cmd) - for cmd in cmdlist: - pid = self.read_mainpid_from(conf, "") - env["MAINPID"] = str(pid) - newcmd = self.exec_cmd(cmd, env, conf) - logg.info("%s start %s", runs, shell_cmd(newcmd)) - forkpid = os.fork() - if not forkpid: # pragma: no cover - os.setsid() # detach child process from parent - self.execve_from(conf, newcmd, env) - self.write_status_from(conf, MainPID=forkpid) - logg.info("%s started PID %s", runs, forkpid) - env["MAINPID"] = str(forkpid) - time.sleep(MinimumYield) - run = subprocess_testpid(forkpid) - if run.returncode is not None: - logg.info("%s stopped PID %s (%s) <-%s>", runs, run.pid, - run.returncode or "OK", run.signal or "") - if doRemainAfterExit: - self.set_status_from(conf, "ExecMainCode", run.returncode) - active = run.returncode and "failed" or "active" - self.write_status_from(conf, AS=active) - if run.returncode: - service_result = "failed" - break - elif runs in [ "notify" ]: - # "notify" is the same as "simple" but we create a $NOTIFY_SOCKET - # and wait for startup completion by checking the socket messages - pid = self.read_mainpid_from(conf, "") - if self.is_active_pid(pid): - logg.error("the service is already running on PID %s", pid) - return False - notify = self.notify_socket_from(conf) - if notify: - env["NOTIFY_SOCKET"] = notify.socketfile - logg.debug("use NOTIFY_SOCKET=%s", notify.socketfile) - if doRemainAfterExit: - logg.debug("%s RemainAfterExit -> AS=active", runs) - self.write_status_from(conf, AS="active") - cmdlist = conf.getlist("Service", "ExecStart", []) - for idx, cmd in enumerate(cmdlist): - logg.debug("ExecStart[%s]: %s", idx, cmd) - mainpid = None - for cmd in cmdlist: - mainpid = self.read_mainpid_from(conf, "") - env["MAINPID"] = str(mainpid) - newcmd = self.exec_cmd(cmd, env, conf) - logg.info("%s start %s", runs, shell_cmd(newcmd)) - forkpid = os.fork() - if not forkpid: # pragma: no cover - os.setsid() # detach child process from parent - self.execve_from(conf, newcmd, env) - # via NOTIFY # self.write_status_from(conf, MainPID=forkpid) - logg.info("%s started PID %s", runs, forkpid) - mainpid = forkpid - self.write_status_from(conf, MainPID=mainpid) - env["MAINPID"] = str(mainpid) - time.sleep(MinimumYield) - run = subprocess_testpid(forkpid) - if run.returncode is not None: - logg.info("%s stopped PID %s (%s) <-%s>", runs, run.pid, - run.returncode or "OK", run.signal or "") - if doRemainAfterExit: - self.set_status_from(conf, "ExecMainCode", run.returncode or 0) - active = run.returncode and "failed" or "active" - self.write_status_from(conf, AS=active) - if run.returncode: - service_result = "failed" - break - if service_result in [ "success" ] and mainpid: - logg.debug("okay, wating on socket for %ss", timeout) - results = self.wait_notify_socket(notify, timeout, mainpid) - if "MAINPID" in results: - new_pid = results["MAINPID"] - if new_pid and to_int(new_pid) != mainpid: - logg.info("NEW PID %s from sd_notify (was PID %s)", new_pid, mainpid) - self.write_status_from(conf, MainPID=new_pid) - mainpid = new_pid - logg.info("%s start done %s", runs, mainpid) - pid = self.read_mainpid_from(conf, "") - if pid: - env["MAINPID"] = str(pid) - else: - service_result = "timeout" # "could not start service" - elif runs in [ "forking" ]: - pid_file = self.pid_file_from(conf) - for cmd in conf.getlist("Service", "ExecStart", []): - check, cmd = checkstatus(cmd) - newcmd = self.exec_cmd(cmd, env, conf) - if not newcmd: continue - logg.info("%s start %s", runs, shell_cmd(newcmd)) - forkpid = os.fork() - if not forkpid: # pragma: no cover - os.setsid() # detach child process from parent - self.execve_from(conf, newcmd, env) - logg.info("%s started PID %s", runs, forkpid) - run = subprocess_waitpid(forkpid) - if run.returncode and check: - returncode = run.returncode - service_result = "failed" - logg.info("%s stopped PID %s (%s) <-%s>", runs, run.pid, - run.returncode or "OK", run.signal or "") - if pid_file and service_result in [ "success" ]: - pid = self.wait_pid_file(pid_file) # application PIDFile - logg.info("%s start done PID %s [%s]", runs, pid, pid_file) - if pid: - env["MAINPID"] = str(pid) - if not pid_file: - time.sleep(MinimumTimeoutStartSec) - logg.warning("No PIDFile for forking %s", conf.filename()) - status_file = self.status_file_from(conf) - self.set_status_from(conf, "ExecMainCode", returncode) - active = returncode and "failed" or "active" - self.write_status_from(conf, AS=active) - else: - logg.error("unsupported run type '%s'", runs) - return False - # POST sequence - active = self.is_active_from(conf) - if not active: - logg.warning("%s start not active", runs) - # according to the systemd documentation, a failed start-sequence - # should execute the ExecStopPost sequence allowing some cleanup. - env["SERVICE_RESULT"] = service_result - for cmd in conf.getlist("Service", "ExecStopPost", []): - check, cmd = checkstatus(cmd) - newcmd = self.exec_cmd(cmd, env, conf) - logg.info("post-fail %s", shell_cmd(newcmd)) - forkpid = os.fork() - if not forkpid: - self.execve_from(conf, newcmd, env) # pragma: nocover - run = subprocess_waitpid(forkpid) - logg.debug("post-fail done (%s) <-%s>", - run.returncode or "OK", run.signal or "") - return False - else: - for cmd in conf.getlist("Service", "ExecStartPost", []): - check, cmd = checkstatus(cmd) - newcmd = self.exec_cmd(cmd, env, conf) - logg.info("post-start %s", shell_cmd(newcmd)) - forkpid = os.fork() - if not forkpid: - self.execve_from(conf, newcmd, env) # pragma: nocover - run = subprocess_waitpid(forkpid) - logg.debug("post-start done (%s) <-%s>", - run.returncode or "OK", run.signal or "") - return True - def extend_exec_env(self, env): - env = env.copy() - # implant DefaultPath into $PATH - path = env.get("PATH", DefaultPath) - parts = path.split(os.pathsep) - for part in DefaultPath.split(os.pathsep): - if part and part not in parts: - parts.append(part) - env["PATH"] = str(os.pathsep).join(parts) - # reset locale to system default - for name in ResetLocale: - if name in env: - del env[name] - locale = {} - for var, val in self.read_env_file("/etc/locale.conf"): - locale[var] = val - env[var] = val - if "LANG" not in locale: - env["LANG"] = locale.get("LANGUAGE", locale.get("LC_CTYPE", "C")) - return env - def execve_from(self, conf, cmd, env): - """ this code is commonly run in a child process // returns exit-code""" - runs = conf.get("Service", "Type", "simple").lower() - logg.debug("%s process for %s", runs, conf.filename()) - inp = open("/dev/zero") - out = self.open_journal_log(conf) - os.dup2(inp.fileno(), sys.stdin.fileno()) - os.dup2(out.fileno(), sys.stdout.fileno()) - os.dup2(out.fileno(), sys.stderr.fileno()) - runuser = self.expand_special(conf.get("Service", "User", ""), conf) - rungroup = self.expand_special(conf.get("Service", "Group", ""), conf) - envs = shutil_setuid(runuser, rungroup) - badpath = self.chdir_workingdir(conf) # some dirs need setuid before - if badpath: - logg.error("(%s): bad workingdir: '%s'", shell_cmd(cmd), badpath) - sys.exit(1) - env = self.extend_exec_env(env) - env.update(envs) # set $HOME to ~$USER - try: - if "spawn" in COVERAGE: - os.spawnvpe(os.P_WAIT, cmd[0], cmd, env) - sys.exit(0) - else: # pragma: nocover - os.execve(cmd[0], cmd, env) - except Exception as e: - logg.error("(%s): %s", shell_cmd(cmd), e) - sys.exit(1) - def test_start_unit(self, unit): - """ helper function to test the code that is normally forked off """ - conf = self.load_unit_conf(unit) - env = self.get_env(conf) - for cmd in conf.getlist("Service", "ExecStart", []): - newcmd = self.exec_cmd(cmd, env, conf) - return self.execve_from(conf, newcmd, env) - return None - def stop_modules(self, *modules): - """ [UNIT]... -- stop these units """ - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - return self.stop_units(units) and found_all - def stop_units(self, units): - """ fails if any unit fails to stop """ - self.wait_system() - done = True - for unit in self.sortedBefore(units): - if not self.stop_unit(unit): - done = False - return done - def stop_unit(self, unit): - conf = self.load_unit_conf(unit) - if conf is None: - logg.error("Unit %s could not be found.", unit) - return False - if self.not_user_conf(conf): - logg.error("Unit %s not for --user mode", unit) - return False - return self.stop_unit_from(conf) - - def get_TimeoutStopSec(self, conf): - timeout = conf.get("Service", "TimeoutSec", DefaultTimeoutStartSec) - timeout = conf.get("Service", "TimeoutStopSec", timeout) - return time_to_seconds(timeout, DefaultMaximumTimeout) - def stop_unit_from(self, conf): - if not conf: return False - if self.syntax_check(conf) > 100: return False - with waitlock(conf): - logg.info(" stop unit %s => %s", conf.name(), conf.filename()) - return self.do_stop_unit_from(conf) - def do_stop_unit_from(self, conf): - timeout = self.get_TimeoutStopSec(conf) - runs = conf.get("Service", "Type", "simple").lower() - env = self.get_env(conf) - self.exec_check_service(conf, env, "ExecStop") - returncode = 0 - service_result = "success" - if runs in [ "sysv" ]: - status_file = self.status_file_from(conf) - if True: - exe = conf.filename() - cmd = "'%s' stop" % exe - env["SYSTEMCTL_SKIP_REDIRECT"] = "yes" - newcmd = self.exec_cmd(cmd, env, conf) - logg.info("%s stop %s", runs, shell_cmd(newcmd)) - forkpid = os.fork() - if not forkpid: - self.execve_from(conf, newcmd, env) # pragma: nocover - run = subprocess_waitpid(forkpid) - if run.returncode: - self.set_status_from(conf, "ExecStopCode", run.returncode) - self.write_status_from(conf, AS="failed") - else: - self.clean_status_from(conf) # "inactive" - return True - elif runs in [ "oneshot" ]: - status_file = self.status_file_from(conf) - if self.get_status_from(conf, "ActiveState", "unknown") == "inactive": - logg.warning("the service is already down once") - return True - for cmd in conf.getlist("Service", "ExecStop", []): - check, cmd = checkstatus(cmd) - logg.debug("{env} %s", env) - newcmd = self.exec_cmd(cmd, env, conf) - logg.info("%s stop %s", runs, shell_cmd(newcmd)) - forkpid = os.fork() - if not forkpid: - self.execve_from(conf, newcmd, env) # pragma: nocover - run = subprocess_waitpid(forkpid) - if run.returncode and check: - returncode = run.returncode - service_result = "failed" - break - if True: - if returncode: - self.set_status_from(conf, "ExecStopCode", returncode) - self.write_status_from(conf, AS="failed") - else: - self.clean_status_from(conf) # "inactive" - ### fallback Stop => Kill for ["simple","notify","forking"] - elif not conf.getlist("Service", "ExecStop", []): - logg.info("no ExecStop => systemctl kill") - if True: - self.do_kill_unit_from(conf) - self.clean_pid_file_from(conf) - self.clean_status_from(conf) # "inactive" - elif runs in [ "simple", "notify" ]: - status_file = self.status_file_from(conf) - size = os.path.exists(status_file) and os.path.getsize(status_file) - logg.info("STATUS %s %s", status_file, size) - pid = 0 - for cmd in conf.getlist("Service", "ExecStop", []): - check, cmd = checkstatus(cmd) - env["MAINPID"] = str(self.read_mainpid_from(conf, "")) - newcmd = self.exec_cmd(cmd, env, conf) - logg.info("%s stop %s", runs, shell_cmd(newcmd)) - forkpid = os.fork() - if not forkpid: - self.execve_from(conf, newcmd, env) # pragma: nocover - run = subprocess_waitpid(forkpid) - run = must_have_failed(run, newcmd) # TODO: a workaround - # self.write_status_from(conf, MainPID=run.pid) # no ExecStop - if run.returncode and check: - returncode = run.returncode - service_result = "failed" - break - pid = env.get("MAINPID",0) - if pid: - if self.wait_vanished_pid(pid, timeout): - self.clean_pid_file_from(conf) - self.clean_status_from(conf) # "inactive" - else: - logg.info("%s sleep as no PID was found on Stop", runs) - time.sleep(MinimumTimeoutStopSec) - pid = self.read_mainpid_from(conf, "") - if not pid or not pid_exists(pid) or pid_zombie(pid): - self.clean_pid_file_from(conf) - self.clean_status_from(conf) # "inactive" - elif runs in [ "forking" ]: - status_file = self.status_file_from(conf) - pid_file = self.pid_file_from(conf) - for cmd in conf.getlist("Service", "ExecStop", []): - active = self.is_active_from(conf) - if pid_file: - new_pid = self.read_mainpid_from(conf, "") - if new_pid: - env["MAINPID"] = str(new_pid) - check, cmd = checkstatus(cmd) - logg.debug("{env} %s", env) - newcmd = self.exec_cmd(cmd, env, conf) - logg.info("fork stop %s", shell_cmd(newcmd)) - forkpid = os.fork() - if not forkpid: - self.execve_from(conf, newcmd, env) # pragma: nocover - run = subprocess_waitpid(forkpid) - if run.returncode and check: - returncode = run.returncode - service_result = "failed" - break - pid = env.get("MAINPID",0) - if pid: - if self.wait_vanished_pid(pid, timeout): - self.clean_pid_file_from(conf) - else: - logg.info("%s sleep as no PID was found on Stop", runs) - time.sleep(MinimumTimeoutStopSec) - pid = self.read_mainpid_from(conf, "") - if not pid or not pid_exists(pid) or pid_zombie(pid): - self.clean_pid_file_from(conf) - if returncode: - if os.path.isfile(status_file): - self.set_status_from(conf, "ExecStopCode", returncode) - self.write_status_from(conf, AS="failed") - else: - self.clean_status_from(conf) # "inactive" - else: - logg.error("unsupported run type '%s'", runs) - return False - # POST sequence - active = self.is_active_from(conf) - if not active: - env["SERVICE_RESULT"] = service_result - for cmd in conf.getlist("Service", "ExecStopPost", []): - check, cmd = checkstatus(cmd) - newcmd = self.exec_cmd(cmd, env, conf) - logg.info("post-stop %s", shell_cmd(newcmd)) - forkpid = os.fork() - if not forkpid: - self.execve_from(conf, newcmd, env) # pragma: nocover - run = subprocess_waitpid(forkpid) - logg.debug("post-stop done (%s) <-%s>", - run.returncode or "OK", run.signal or "") - return service_result == "success" - def wait_vanished_pid(self, pid, timeout): - if not pid: - return True - logg.info("wait for PID %s to vanish (%ss)", pid, timeout) - for x in xrange(int(timeout)): - if not self.is_active_pid(pid): - logg.info("wait for PID %s is done (%s.)", pid, x) - return True - time.sleep(1) # until TimeoutStopSec - logg.info("wait for PID %s failed (%s.)", pid, x) - return False - def reload_modules(self, *modules): - """ [UNIT]... -- reload these units """ - self.wait_system() - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - return self.reload_units(units) and found_all - def reload_units(self, units): - """ fails if any unit fails to reload """ - self.wait_system() - done = True - for unit in self.sortedAfter(units): - if not self.reload_unit(unit): - done = False - return done - def reload_unit(self, unit): - conf = self.load_unit_conf(unit) - if conf is None: - logg.error("Unit %s could not be found.", unit) - return False - if self.not_user_conf(conf): - logg.error("Unit %s not for --user mode", unit) - return False - return self.reload_unit_from(conf) - def reload_unit_from(self, conf): - if not conf: return False - if self.syntax_check(conf) > 100: return False - with waitlock(conf): - logg.info(" reload unit %s => %s", conf.name(), conf.filename()) - return self.do_reload_unit_from(conf) - def do_reload_unit_from(self, conf): - runs = conf.get("Service", "Type", "simple").lower() - env = self.get_env(conf) - self.exec_check_service(conf, env, "ExecReload") - if runs in [ "sysv" ]: - status_file = self.status_file_from(conf) - if True: - exe = conf.filename() - cmd = "'%s' reload" % exe - env["SYSTEMCTL_SKIP_REDIRECT"] = "yes" - newcmd = self.exec_cmd(cmd, env, conf) - logg.info("%s reload %s", runs, shell_cmd(newcmd)) - forkpid = os.fork() - if not forkpid: - self.execve_from(conf, newcmd, env) # pragma: nocover - run = subprocess_waitpid(forkpid) - self.set_status_from(conf, "ExecReloadCode", run.returncode) - if run.returncode: - self.write_status_from(conf, AS="failed") - return False - else: - self.write_status_from(conf, AS="active") - return True - elif runs in [ "simple", "notify", "forking" ]: - if not self.is_active_from(conf): - logg.info("no reload on inactive service %s", conf.name()) - return True - for cmd in conf.getlist("Service", "ExecReload", []): - env["MAINPID"] = str(self.read_mainpid_from(conf, "")) - check, cmd = checkstatus(cmd) - newcmd = self.exec_cmd(cmd, env, conf) - logg.info("%s reload %s", runs, shell_cmd(newcmd)) - forkpid = os.fork() - if not forkpid: - self.execve_from(conf, newcmd, env) # pragma: nocover - run = subprocess_waitpid(forkpid) - if check and run.returncode: - logg.error("Job for %s failed because the control process exited with error code. (%s)", - conf.name(), run.returncode) - return False - time.sleep(MinimumYield) - return True - elif runs in [ "oneshot" ]: - logg.debug("ignored run type '%s' for reload", runs) - return True - else: - logg.error("unsupported run type '%s'", runs) - return False - def restart_modules(self, *modules): - """ [UNIT]... -- restart these units """ - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - return self.restart_units(units) and found_all - def restart_units(self, units): - """ fails if any unit fails to restart """ - self.wait_system() - done = True - for unit in self.sortedAfter(units): - if not self.restart_unit(unit): - done = False - return done - def restart_unit(self, unit): - conf = self.load_unit_conf(unit) - if conf is None: - logg.error("Unit %s could not be found.", unit) - return False - if self.not_user_conf(conf): - logg.error("Unit %s not for --user mode", unit) - return False - return self.restart_unit_from(conf) - def restart_unit_from(self, conf): - if not conf: return False - if self.syntax_check(conf) > 100: return False - with waitlock(conf): - logg.info(" restart unit %s => %s", conf.name(), conf.filename()) - if not self.is_active_from(conf): - return self.do_start_unit_from(conf) - else: - return self.do_restart_unit_from(conf) - def do_restart_unit_from(self, conf): - logg.info("(restart) => stop/start") - self.do_stop_unit_from(conf) - return self.do_start_unit_from(conf) - def try_restart_modules(self, *modules): - """ [UNIT]... -- try-restart these units """ - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - return self.try_restart_units(units) and found_all - def try_restart_units(self, units): - """ fails if any module fails to try-restart """ - self.wait_system() - done = True - for unit in self.sortedAfter(units): - if not self.try_restart_unit(unit): - done = False - return done - def try_restart_unit(self, unit): - """ only do 'restart' if 'active' """ - conf = self.load_unit_conf(unit) - if conf is None: - logg.error("Unit %s could not be found.", unit) - return False - if self.not_user_conf(conf): - logg.error("Unit %s not for --user mode", unit) - return False - with waitlock(conf): - logg.info(" try-restart unit %s => %s", conf.name(), conf.filename()) - if self.is_active_from(conf): - return self.do_restart_unit_from(conf) - return True - def reload_or_restart_modules(self, *modules): - """ [UNIT]... -- reload-or-restart these units """ - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - return self.reload_or_restart_units(units) and found_all - def reload_or_restart_units(self, units): - """ fails if any unit does not reload-or-restart """ - self.wait_system() - done = True - for unit in self.sortedAfter(units): - if not self.reload_or_restart_unit(unit): - done = False - return done - def reload_or_restart_unit(self, unit): - """ do 'reload' if specified, otherwise do 'restart' """ - conf = self.load_unit_conf(unit) - if conf is None: - logg.error("Unit %s could not be found.", unit) - return False - if self.not_user_conf(conf): - logg.error("Unit %s not for --user mode", unit) - return False - return self.reload_or_restart_unit_from(conf) - def reload_or_restart_unit_from(self, conf): - """ do 'reload' if specified, otherwise do 'restart' """ - if not conf: return False - with waitlock(conf): - logg.info(" reload-or-restart unit %s => %s", conf.name(), conf.filename()) - return self.do_reload_or_restart_unit_from(conf) - def do_reload_or_restart_unit_from(self, conf): - if not self.is_active_from(conf): - # try: self.stop_unit_from(conf) - # except Exception as e: pass - return self.do_start_unit_from(conf) - elif conf.getlist("Service", "ExecReload", []): - logg.info("found service to have ExecReload -> 'reload'") - return self.do_reload_unit_from(conf) - else: - logg.info("found service without ExecReload -> 'restart'") - return self.do_restart_unit_from(conf) - def reload_or_try_restart_modules(self, *modules): - """ [UNIT]... -- reload-or-try-restart these units """ - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - return self.reload_or_try_restart_units(units) and found_all - def reload_or_try_restart_units(self, units): - """ fails if any unit fails to reload-or-try-restart """ - self.wait_system() - done = True - for unit in self.sortedAfter(units): - if not self.reload_or_try_restart_unit(unit): - done = False - return done - def reload_or_try_restart_unit(self, unit): - conf = self.load_unit_conf(unit) - if conf is None: - logg.error("Unit %s could not be found.", unit) - return False - if self.not_user_conf(conf): - logg.error("Unit %s not for --user mode", unit) - return False - return self.reload_or_try_restart_unit_from(conf) - def reload_or_try_restart_unit_from(self, conf): - with waitlock(conf): - logg.info(" reload-or-try-restart unit %s => %s", conf.name(), conf.filename()) - return self.do_reload_or_try_restart_unit_from(conf) - def do_reload_or_try_restart_unit_from(self, conf): - if conf.getlist("Service", "ExecReload", []): - return self.do_reload_unit_from(conf) - elif not self.is_active_from(conf): - return True - else: - return self.do_restart_unit_from(conf) - def kill_modules(self, *modules): - """ [UNIT]... -- kill these units """ - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - return self.kill_units(units) and found_all - def kill_units(self, units): - """ fails if any unit could not be killed """ - self.wait_system() - done = True - for unit in self.sortedBefore(units): - if not self.kill_unit(unit): - done = False - return done - def kill_unit(self, unit): - conf = self.load_unit_conf(unit) - if conf is None: - logg.error("Unit %s could not be found.", unit) - return False - if self.not_user_conf(conf): - logg.error("Unit %s not for --user mode", unit) - return False - return self.kill_unit_from(conf) - def kill_unit_from(self, conf): - if not conf: return False - with waitlock(conf): - logg.info(" kill unit %s => %s", conf.name(), conf.filename()) - return self.do_kill_unit_from(conf) - def do_kill_unit_from(self, conf): - started = time.time() - doSendSIGKILL = conf.getbool("Service", "SendSIGKILL", "yes") - doSendSIGHUP = conf.getbool("Service", "SendSIGHUP", "no") - useKillMode = conf.get("Service", "KillMode", "control-group") - useKillSignal = conf.get("Service", "KillSignal", "SIGTERM") - kill_signal = getattr(signal, useKillSignal) - timeout = self.get_TimeoutStopSec(conf) - status_file = self.status_file_from(conf) - size = os.path.exists(status_file) and os.path.getsize(status_file) - logg.info("STATUS %s %s", status_file, size) - mainpid = to_int(self.read_mainpid_from(conf, "")) - self.clean_status_from(conf) # clear RemainAfterExit and TimeoutStartSec - if not mainpid: - if useKillMode in ["control-group"]: - logg.warning("no main PID [%s]", conf.filename()) - logg.warning("and there is no control-group here") - else: - logg.info("no main PID [%s]", conf.filename()) - return False - if not pid_exists(mainpid) or pid_zombie(mainpid): - logg.debug("ignoring children when mainpid is already dead") - # because we list child processes, not processes in control-group - return True - pidlist = self.pidlist_of(mainpid) # here - if pid_exists(mainpid): - logg.info("stop kill PID %s", mainpid) - self._kill_pid(mainpid, kill_signal) - if useKillMode in ["control-group"]: - if len(pidlist) > 1: - logg.info("stop control-group PIDs %s", pidlist) - for pid in pidlist: - if pid != mainpid: - self._kill_pid(pid, kill_signal) - if doSendSIGHUP: - logg.info("stop SendSIGHUP to PIDs %s", pidlist) - for pid in pidlist: - self._kill_pid(pid, signal.SIGHUP) - # wait for the processes to have exited - while True: - dead = True - for pid in pidlist: - if pid_exists(pid) and not pid_zombie(pid): - dead = False - break - if dead: - break - if time.time() > started + timeout: - logg.info("service PIDs not stopped after %s", timeout) - break - time.sleep(1) # until TimeoutStopSec - if dead or not doSendSIGKILL: - logg.info("done kill PID %s %s", mainpid, dead and "OK") - return dead - if useKillMode in [ "control-group", "mixed" ]: - logg.info("hard kill PIDs %s", pidlist) - for pid in pidlist: - if pid != mainpid: - self._kill_pid(pid, signal.SIGKILL) - time.sleep(MinimumYield) - # useKillMode in [ "control-group", "mixed", "process" ] - if pid_exists(mainpid): - logg.info("hard kill PID %s", mainpid) - self._kill_pid(mainpid, signal.SIGKILL) - time.sleep(MinimumYield) - dead = not pid_exists(mainpid) or pid_zombie(mainpid) - logg.info("done hard kill PID %s %s", mainpid, dead and "OK") - return dead - def _kill_pid(self, pid, kill_signal = None): - try: - sig = kill_signal or signal.SIGTERM - os.kill(pid, sig) - except OSError as e: - if e.errno == errno.ESRCH or e.errno == errno.ENOENT: - logg.debug("kill PID %s => No such process", pid) - return True - else: - logg.error("kill PID %s => %s", pid, str(e)) - return False - return not pid_exists(pid) or pid_zombie(pid) - def is_active_modules(self, *modules): - """ [UNIT].. -- check if these units are in active state - implements True if all is-active = True """ - # systemctl returns multiple lines, one for each argument - # "active" when is_active - # "inactive" when not is_active - # "unknown" when not enabled - # The return code is set to - # 0 when "active" - # 1 when unit is not found - # 3 when any "inactive" or "unknown" - # However: # TODO!!!!! BUG in original systemctl!! - # documentation says " exit code 0 if at least one is active" - # and "Unless --quiet is specified, print the unit state" - units = [] - results = [] - for module in modules: - units = self.match_units([ module ]) - if not units: - logg.error("Unit %s could not be found.", unit_of(module)) - results += [ "unknown" ] - continue - for unit in units: - active = self.get_active_unit(unit) - enabled = self.enabled_unit(unit) - if enabled != "enabled": active = "unknown" - results += [ active ] - break - ## how it should work: - status = "active" in results - ## how 'systemctl' works: - non_active = [ result for result in results if result != "active" ] - status = not non_active - if not status: - status = 3 - if not _quiet: - return status, results - else: - return status - def is_active_from(self, conf): - """ used in try-restart/other commands to check if needed. """ - if not conf: return False - return self.get_active_from(conf) == "active" - def active_pid_from(self, conf): - if not conf: return False - pid = self.read_mainpid_from(conf, "") - return self.is_active_pid(pid) - def is_active_pid(self, pid): - """ returns pid if the pid is still an active process """ - if pid and pid_exists(pid) and not pid_zombie(pid): - return pid # usually a string (not null) - return None - def get_active_unit(self, unit): - """ returns 'active' 'inactive' 'failed' 'unknown' """ - conf = self.get_unit_conf(unit) - if not conf.loaded(): - logg.warning("Unit %s could not be found.", unit) - return "unknown" - return self.get_active_from(conf) - def get_active_from(self, conf): - """ returns 'active' 'inactive' 'failed' 'unknown' """ - # used in try-restart/other commands to check if needed. - if not conf: return "unknown" - pid_file = self.pid_file_from(conf) - if pid_file: # application PIDFile - if not os.path.exists(pid_file): - return "inactive" - status_file = self.status_file_from(conf) - if self.getsize(status_file): - state = self.get_status_from(conf, "ActiveState", "") - if state: - logg.info("get_status_from %s => %s", conf.name(), state) - return state - pid = self.read_mainpid_from(conf, "") - logg.debug("pid_file '%s' => PID %s", pid_file or status_file, pid) - if pid: - if not pid_exists(pid) or pid_zombie(pid): - return "failed" - return "active" - else: - return "inactive" - def get_substate_from(self, conf): - """ returns 'running' 'exited' 'dead' 'failed' 'plugged' 'mounted' """ - if not conf: return False - pid_file = self.pid_file_from(conf) - if pid_file: - if not os.path.exists(pid_file): - return "dead" - status_file = self.status_file_from(conf) - if self.getsize(status_file): - state = self.get_status_from(conf, "ActiveState", "") - if state: - if state in [ "active" ]: - return self.get_status_from(conf, "SubState", "running") - else: - return self.get_status_from(conf, "SubState", "dead") - pid = self.read_mainpid_from(conf, "") - logg.debug("pid_file '%s' => PID %s", pid_file or status_file, pid) - if pid: - if not pid_exists(pid) or pid_zombie(pid): - return "failed" - return "running" - else: - return "dead" - def is_failed_modules(self, *modules): - """ [UNIT]... -- check if these units are in failes state - implements True if any is-active = True """ - units = [] - results = [] - for module in modules: - units = self.match_units([ module ]) - if not units: - logg.error("Unit %s could not be found.", unit_of(module)) - results += [ "unknown" ] - continue - for unit in units: - active = self.get_active_unit(unit) - enabled = self.enabled_unit(unit) - if enabled != "enabled": active = "unknown" - results += [ active ] - break - status = "failed" in results - if not _quiet: - return status, results - else: - return status - def is_failed_from(self, conf): - if conf is None: return True - return self.get_active_from(conf) == "failed" - def reset_failed_modules(self, *modules): - """ [UNIT]... -- Reset failed state for all, one, or more units """ - units = [] - status = True - for module in modules: - units = self.match_units([ module ]) - if not units: - logg.error("Unit %s could not be found.", unit_of(module)) - return 1 - for unit in units: - if not self.reset_failed_unit(unit): - logg.error("Unit %s could not be reset.", unit_of(module)) - status = False - break - return status - def reset_failed_unit(self, unit): - conf = self.get_unit_conf(unit) - if not conf.loaded(): - logg.warning("Unit %s could not be found.", unit) - return False - if self.not_user_conf(conf): - logg.error("Unit %s not for --user mode", unit) - return False - return self.reset_failed_from(conf) - def reset_failed_from(self, conf): - if conf is None: return True - if not self.is_failed_from(conf): return False - done = False - status_file = self.status_file_from(conf) - if status_file and os.path.exists(status_file): - try: - os.remove(status_file) - done = True - logg.debug("done rm %s", status_file) - except Exception as e: - logg.error("while rm %s: %s", status_file, e) - pid_file = self.pid_file_from(conf) - if pid_file and os.path.exists(pid_file): - try: - os.remove(pid_file) - done = True - logg.debug("done rm %s", pid_file) - except Exception as e: - logg.error("while rm %s: %s", pid_file, e) - return done - def status_modules(self, *modules): - """ [UNIT]... check the status of these units. - """ - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - status, result = self.status_units(units) - if not found_all: - status = 3 # same as (dead) # original behaviour - return (status, result) - def status_units(self, units): - """ concatenates the status output of all units - and the last non-successful statuscode """ - status, result = 0, "" - for unit in units: - status1, result1 = self.status_unit(unit) - if status1: status = status1 - if result: result += "\n\n" - result += result1 - return status, result - def status_unit(self, unit): - conf = self.get_unit_conf(unit) - result = "%s - %s" % (unit, self.get_description_from(conf)) - loaded = conf.loaded() - if loaded: - filename = conf.filename() - enabled = self.enabled_from(conf) - result += "\n Loaded: {loaded} ({filename}, {enabled})".format(**locals()) - for path in conf.overrides(): - result += "\n Drop-In: {path}".format(**locals()) - else: - result += "\n Loaded: failed" - return 3, result - active = self.get_active_from(conf) - substate = self.get_substate_from(conf) - result += "\n Active: {} ({})".format(active, substate) - if active == "active": - return 0, result - else: - return 3, result - def cat_modules(self, *modules): - """ [UNIT]... show the *.system file for these" - """ - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - done, result = self.cat_units(units) - return (done and found_all, result) - def cat_units(self, units): - done = True - result = "" - for unit in units: - text = self.cat_unit(unit) - if not text: - done = False - else: - if result: - result += "\n\n" - result += text - return done, result - def cat_unit(self, unit): - try: - unit_file = self.unit_file(unit) - if unit_file: - return open(unit_file).read() - logg.error("no file for unit '%s'", unit) - except Exception as e: - print("Unit {} is not-loaded: {}".format(unit, e)) - return False - ## - ## - def load_preset_files(self, module = None): # -> [ preset-file-names,... ] - """ reads all preset files, returns the scanned files """ - if self._preset_file_list is None: - self._preset_file_list = {} - for folder in self.preset_folders(): - if not folder: - continue - if self._root: - folder = os_path(self._root, folder) - if not os.path.isdir(folder): - continue - for name in os.listdir(folder): - if not name.endswith(".preset"): - continue - if name not in self._preset_file_list: - path = os.path.join(folder, name) - if os.path.isdir(path): - continue - preset = PresetFile().read(path) - self._preset_file_list[name] = preset - logg.debug("found %s preset files", len(self._preset_file_list)) - return sorted(self._preset_file_list.keys()) - def get_preset_of_unit(self, unit): - """ [UNIT] check the *.preset of this unit - """ - self.load_preset_files() - for filename in sorted(self._preset_file_list.keys()): - preset = self._preset_file_list[filename] - status = preset.get_preset(unit) - if status: - return status - return None - def preset_modules(self, *modules): - """ [UNIT]... -- set 'enabled' when in *.preset - """ - if self.user_mode(): - logg.warning("preset makes no sense in --user mode") - return True - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - return self.preset_units(units) and found_all - def preset_units(self, units): - """ fails if any unit could not be changed """ - self.wait_system() - fails = 0 - found = 0 - for unit in units: - status = self.get_preset_of_unit(unit) - if not status: continue - found += 1 - if status.startswith("enable"): - if self._preset_mode == "disable": continue - logg.info("preset enable %s", unit) - if not self.enable_unit(unit): - logg.warning("failed to enable %s", unit) - fails += 1 - if status.startswith("disable"): - if self._preset_mode == "enable": continue - logg.info("preset disable %s", unit) - if not self.disable_unit(unit): - logg.warning("failed to disable %s", unit) - fails += 1 - return not fails and not not found - def system_preset_all(self, *modules): - """ 'preset' all services - enable or disable services according to *.preset files - """ - if self.user_mode(): - logg.warning("preset-all makes no sense in --user mode") - return True - found_all = True - units = self.match_units() # TODO: how to handle module arguments - return self.preset_units(units) and found_all - def wanted_from(self, conf, default = None): - if not conf: return default - return conf.get("Install", "WantedBy", default, True) - def enablefolders(self, wanted): - if self.user_mode(): - for folder in self.user_folders(): - yield self.default_enablefolder(wanted, folder) - if True: - for folder in self.system_folders(): - yield self.default_enablefolder(wanted, folder) - def enablefolder(self, wanted = None): - if self.user_mode(): - user_folder = self.user_folder() - return self.default_enablefolder(wanted, user_folder) - else: - return self.default_enablefolder(wanted) - def default_enablefolder(self, wanted = None, basefolder = None): - basefolder = basefolder or self.system_folder() - if not wanted: - return wanted - if not wanted.endswith(".wants"): - wanted = wanted + ".wants" - return os.path.join(basefolder, wanted) - def enable_modules(self, *modules): - """ [UNIT]... -- enable these units """ - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - logg.info("matched %s", unit) #++ - if unit not in units: - units += [ unit ] - return self.enable_units(units) and found_all - def enable_units(self, units): - self.wait_system() - done = True - for unit in units: - if not self.enable_unit(unit): - done = False - elif self._now: - self.start_unit(unit) - return done - def enable_unit(self, unit): - unit_file = self.unit_file(unit) - if not unit_file: - logg.error("Unit %s could not be found.", unit) - return False - if self.is_sysv_file(unit_file): - if self.user_mode(): - logg.error("Initscript %s not for --user mode", unit) - return False - return self.enable_unit_sysv(unit_file) - conf = self.get_unit_conf(unit) - if self.not_user_conf(conf): - logg.error("Unit %s not for --user mode", unit) - return False - wanted = self.wanted_from(self.get_unit_conf(unit)) - if not wanted: - return False # "static" is-enabled - folder = self.enablefolder(wanted) - if self._root: - folder = os_path(self._root, folder) - if not os.path.isdir(folder): - os.makedirs(folder) - target = os.path.join(folder, os.path.basename(unit_file)) - if True: - _f = self._force and "-f" or "" - logg.info("ln -s {_f} '{unit_file}' '{target}'".format(**locals())) - if self._force and os.path.islink(target): - os.remove(target) - if not os.path.islink(target): - os.symlink(unit_file, target) - return True - def rc3_root_folder(self): - old_folder = "/etc/rc3.d" - new_folder = "/etc/init.d/rc3.d" - if self._root: - old_folder = os_path(self._root, old_folder) - new_folder = os_path(self._root, new_folder) - if os.path.isdir(old_folder): - return old_folder - return new_folder - def rc5_root_folder(self): - old_folder = "/etc/rc5.d" - new_folder = "/etc/init.d/rc5.d" - if self._root: - old_folder = os_path(self._root, old_folder) - new_folder = os_path(self._root, new_folder) - if os.path.isdir(old_folder): - return old_folder - return new_folder - def enable_unit_sysv(self, unit_file): - # a "multi-user.target"/rc3 is also started in /rc5 - rc3 = self._enable_unit_sysv(unit_file, self.rc3_root_folder()) - rc5 = self._enable_unit_sysv(unit_file, self.rc5_root_folder()) - return rc3 and rc5 - def _enable_unit_sysv(self, unit_file, rc_folder): - name = os.path.basename(unit_file) - nameS = "S50"+name - nameK = "K50"+name - if not os.path.isdir(rc_folder): - os.makedirs(rc_folder) - # do not double existing entries - for found in os.listdir(rc_folder): - m = re.match(r"S\d\d(.*)", found) - if m and m.group(1) == name: - nameS = found - m = re.match(r"K\d\d(.*)", found) - if m and m.group(1) == name: - nameK = found - target = os.path.join(rc_folder, nameS) - if not os.path.exists(target): - os.symlink(unit_file, target) - target = os.path.join(rc_folder, nameK) - if not os.path.exists(target): - os.symlink(unit_file, target) - return True - def disable_modules(self, *modules): - """ [UNIT]... -- disable these units """ - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - return self.disable_units(units) and found_all - def disable_units(self, units): - self.wait_system() - done = True - for unit in units: - if not self.disable_unit(unit): - done = False - return done - def disable_unit(self, unit): - unit_file = self.unit_file(unit) - if not unit_file: - logg.error("Unit %s could not be found.", unit) - return False - if self.is_sysv_file(unit_file): - if self.user_mode(): - logg.error("Initscript %s not for --user mode", unit) - return False - return self.disable_unit_sysv(unit_file) - conf = self.get_unit_conf(unit) - if self.not_user_conf(conf): - logg.error("Unit %s not for --user mode", unit) - return False - wanted = self.wanted_from(self.get_unit_conf(unit)) - if not wanted: - return False # "static" is-enabled - for folder in self.enablefolders(wanted): - if self._root: - folder = os_path(self._root, folder) - target = os.path.join(folder, os.path.basename(unit_file)) - if os.path.isfile(target): - try: - _f = self._force and "-f" or "" - logg.info("rm {_f} '{target}'".format(**locals())) - os.remove(target) - except IOError as e: - logg.error("disable %s: %s", target, e) - except OSError as e: - logg.error("disable %s: %s", target, e) - return True - def disable_unit_sysv(self, unit_file): - rc3 = self._disable_unit_sysv(unit_file, self.rc3_root_folder()) - rc5 = self._disable_unit_sysv(unit_file, self.rc5_root_folder()) - return rc3 and rc5 - def _disable_unit_sysv(self, unit_file, rc_folder): - # a "multi-user.target"/rc3 is also started in /rc5 - name = os.path.basename(unit_file) - nameS = "S50"+name - nameK = "K50"+name - # do not forget the existing entries - for found in os.listdir(rc_folder): - m = re.match(r"S\d\d(.*)", found) - if m and m.group(1) == name: - nameS = found - m = re.match(r"K\d\d(.*)", found) - if m and m.group(1) == name: - nameK = found - target = os.path.join(rc_folder, nameS) - if os.path.exists(target): - os.unlink(target) - target = os.path.join(rc_folder, nameK) - if os.path.exists(target): - os.unlink(target) - return True - def is_enabled_sysv(self, unit_file): - name = os.path.basename(unit_file) - target = os.path.join(self.rc3_root_folder(), "S50%s" % name) - if os.path.exists(target): - return True - return False - def is_enabled_modules(self, *modules): - """ [UNIT]... -- check if these units are enabled - returns True if any of them is enabled.""" - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - return self.is_enabled_units(units) # and found_all - def is_enabled_units(self, units): - """ true if any is enabled, and a list of infos """ - result = False - infos = [] - for unit in units: - infos += [ self.enabled_unit(unit) ] - if self.is_enabled(unit): - result = True - return result, infos - def is_enabled(self, unit): - unit_file = self.unit_file(unit) - if not unit_file: - logg.error("Unit %s could not be found.", unit) - return False - if self.is_sysv_file(unit_file): - return self.is_enabled_sysv(unit_file) - wanted = self.wanted_from(self.get_unit_conf(unit)) - if not wanted: - return True # "static" - for folder in self.enablefolders(wanted): - if self._root: - folder = os_path(self._root, folder) - target = os.path.join(folder, os.path.basename(unit_file)) - if os.path.isfile(target): - return True - return False - def enabled_unit(self, unit): - conf = self.get_unit_conf(unit) - return self.enabled_from(conf) - def enabled_from(self, conf): - unit_file = conf.filename() - if self.is_sysv_file(unit_file): - state = self.is_enabled_sysv(unit_file) - if state: - return "enabled" - return "disabled" - if conf.masked: - return "masked" - wanted = self.wanted_from(conf) - if not wanted: - return "static" - for folder in self.enablefolders(wanted): - if self._root: - folder = os_path(self._root, folder) - target = os.path.join(folder, os.path.basename(unit_file)) - if os.path.isfile(target): - return "enabled" - return "disabled" - def mask_modules(self, *modules): - """ [UNIT]... -- mask non-startable units """ - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - return self.mask_units(units) and found_all - def mask_units(self, units): - self.wait_system() - done = True - for unit in units: - if not self.mask_unit(unit): - done = False - return done - def mask_unit(self, unit): - unit_file = self.unit_file(unit) - if not unit_file: - logg.error("Unit %s could not be found.", unit) - return False - if self.is_sysv_file(unit_file): - logg.error("Initscript %s can not be masked", unit) - return False - conf = self.get_unit_conf(unit) - if self.not_user_conf(conf): - logg.error("Unit %s not for --user mode", unit) - return False - folder = self.mask_folder() - if self._root: - folder = os_path(self._root, folder) - if not os.path.isdir(folder): - os.makedirs(folder) - target = os.path.join(folder, os.path.basename(unit_file)) - if True: - _f = self._force and "-f" or "" - logg.debug("ln -s {_f} /dev/null '{target}'".format(**locals())) - if self._force and os.path.islink(target): - os.remove(target) - if not os.path.exists(target): - os.symlink("/dev/null", target) - logg.info("Created symlink {target} -> /dev/null".format(**locals())) - return True - elif os.path.islink(target): - logg.debug("mask symlink does already exist: %s", target) - return True - else: - logg.error("mask target does already exist: %s", target) - return False - def mask_folder(self): - for folder in self.mask_folders(): - if folder: return folder - raise Exception("did not find any systemd/system folder") - def mask_folders(self): - if self.user_mode(): - for folder in self.user_folders(): - yield folder - if True: - for folder in self.system_folders(): - yield folder - def unmask_modules(self, *modules): - """ [UNIT]... -- unmask non-startable units """ - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - return self.unmask_units(units) and found_all - def unmask_units(self, units): - self.wait_system() - done = True - for unit in units: - if not self.unmask_unit(unit): - done = False - return done - def unmask_unit(self, unit): - unit_file = self.unit_file(unit) - if not unit_file: - logg.error("Unit %s could not be found.", unit) - return False - if self.is_sysv_file(unit_file): - logg.error("Initscript %s can not be un/masked", unit) - return False - conf = self.get_unit_conf(unit) - if self.not_user_conf(conf): - logg.error("Unit %s not for --user mode", unit) - return False - folder = self.mask_folder() - if self._root: - folder = os_path(self._root, folder) - target = os.path.join(folder, os.path.basename(unit_file)) - if True: - _f = self._force and "-f" or "" - logg.info("rm {_f} '{target}'".format(**locals())) - if os.path.islink(target): - os.remove(target) - return True - elif not os.path.exists(target): - logg.debug("Symlink did exist anymore: %s", target) - return True - else: - logg.warning("target is not a symlink: %s", target) - return True - def list_dependencies_modules(self, *modules): - """ [UNIT]... show the dependency tree" - """ - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - return self.list_dependencies_units(units) # and found_all - def list_dependencies_units(self, units): - if self._now: - return self.list_start_dependencies_units(units) - result = [] - for unit in units: - if result: - result += [ "", "" ] - result += self.list_dependencies_unit(unit) - return result - def list_dependencies_unit(self, unit): - result = [] - for line in self.list_dependencies(unit, ""): - result += [ line ] - return result - def list_dependencies(self, unit, indent = None, mark = None, loop = []): - mapping = {} - mapping["Requires"] = "required to start" - mapping["Wants"] = "wanted to start" - mapping["Requisite"] = "required started" - mapping["Bindsto"] = "binds to start" - mapping["PartOf"] = "part of started" - mapping[".requires"] = ".required to start" - mapping[".wants"] = ".wanted to start" - mapping["PropagateReloadTo"] = "(to be reloaded as well)" - mapping["Conflicts"] = "(to be stopped on conflict)" - restrict = ["Requires", "Requisite", "ConsistsOf", "Wants", - "BindsTo", ".requires", ".wants"] - indent = indent or "" - mark = mark or "" - deps = self.get_dependencies_unit(unit) - conf = self.get_unit_conf(unit) - if not conf.loaded(): - if not self._show_all: - return - yield "%s(%s): %s" % (indent, unit, mark) - else: - yield "%s%s: %s" % (indent, unit, mark) - for stop_recursion in [ "Conflict", "conflict", "reloaded", "Propagate" ]: - if stop_recursion in mark: - return - for dep in deps: - if dep in loop: - logg.debug("detected loop at %s", dep) - continue - new_loop = loop + list(deps.keys()) - new_indent = indent + "| " - new_mark = deps[dep] - if not self._show_all: - if new_mark not in restrict: - continue - if new_mark in mapping: - new_mark = mapping[new_mark] - restrict = ["Requires", "Requisite", "ConsistsOf", "Wants", - "BindsTo", ".requires", ".wants"] - for line in self.list_dependencies(dep, new_indent, new_mark, new_loop): - yield line - def get_dependencies_unit(self, unit): - conf = self.get_unit_conf(unit) - deps = {} - for style in [ "Requires", "Wants", "Requisite", "BindsTo", "PartOf", - ".requires", ".wants", "PropagateReloadTo", "Conflicts", ]: - if style.startswith("."): - for folder in self.sysd_folders(): - if not folder: - continue - require_path = os.path.join(folder, unit + style) - if self._root: - require_path = os_path(self._root, require_path) - if os.path.isdir(require_path): - for required in os.listdir(require_path): - if required not in deps: - deps[required] = style - else: - for requirelist in conf.getlist("Unit", style, []): - for required in requirelist.strip().split(" "): - deps[required.strip()] = style - return deps - def get_start_dependencies(self, unit): # pragma: no cover - """ the list of services to be started as well / TODO: unused """ - deps = {} - unit_deps = self.get_dependencies_unit(unit) - for dep_unit, dep_style in unit_deps.items(): - restrict = ["Requires", "Requisite", "ConsistsOf", "Wants", - "BindsTo", ".requires", ".wants"] - if dep_style in restrict: - if dep_unit in deps: - if dep_style not in deps[dep_unit]: - deps[dep_unit].append( dep_style) - else: - deps[dep_unit] = [ dep_style ] - next_deps = self.get_start_dependencies(dep_unit) - for dep, styles in next_deps.items(): - for style in styles: - if dep in deps: - if style not in deps[dep]: - deps[dep].append(style) - else: - deps[dep] = [ style ] - return deps - def list_start_dependencies_units(self, units): - unit_order = [] - deps = {} - for unit in units: - unit_order.append(unit) - # unit_deps = self.get_start_dependencies(unit) # TODO - unit_deps = self.get_dependencies_unit(unit) - for dep_unit, styles in unit_deps.items(): - styles = to_list(styles) - for dep_style in styles: - if dep_unit in deps: - if dep_style not in deps[dep_unit]: - deps[dep_unit].append( dep_style) - else: - deps[dep_unit] = [ dep_style ] - deps_conf = [] - for dep in deps: - if dep in unit_order: - continue - conf = self.get_unit_conf(dep) - if conf.loaded(): - deps_conf.append(conf) - for unit in unit_order: - deps[unit] = [ "Requested" ] - conf = self.get_unit_conf(unit) - if conf.loaded(): - deps_conf.append(conf) - result = [] - for dep in sortedAfter(deps_conf, cmp=compareAfter): - line = (dep.name(), "(%s)" % (" ".join(deps[dep.name()]))) - result.append(line) - return result - def sortedAfter(self, unitlist): - """ get correct start order for the unit list (ignoring masked units) """ - conflist = [ self.get_unit_conf(unit) for unit in unitlist ] - if True: - conflist = [] - for unit in unitlist: - conf = self.get_unit_conf(unit) - if conf.masked: - logg.debug("ignoring masked unit %s", unit) - continue - conflist.append(conf) - sortlist = sortedAfter(conflist) - return [ item.name() for item in sortlist ] - def sortedBefore(self, unitlist): - """ get correct start order for the unit list (ignoring masked units) """ - conflist = [ self.get_unit_conf(unit) for unit in unitlist ] - if True: - conflist = [] - for unit in unitlist: - conf = self.get_unit_conf(unit) - if conf.masked: - logg.debug("ignoring masked unit %s", unit) - continue - conflist.append(conf) - sortlist = sortedAfter(reversed(conflist)) - return [ item.name() for item in reversed(sortlist) ] - def system_daemon_reload(self): - """ reload does will only check the service files here. - The returncode will tell the number of warnings, - and it is over 100 if it can not continue even - for the relaxed systemctl.py style of execution. """ - errors = 0 - for unit in self.match_units(): - try: - conf = self.get_unit_conf(unit) - except Exception as e: - logg.error("%s: can not read unit file %s\n\t%s", - unit, conf.filename(), e) - continue - errors += self.syntax_check(conf) - if errors: - logg.warning(" (%s) found %s problems", errors, errors % 100) - return True # errors - def syntax_check(self, conf): - if conf.filename() and conf.filename().endswith(".service"): - return self.syntax_check_service(conf) - return 0 - def syntax_check_service(self, conf): - unit = conf.name() - if not conf.data.has_section("Service"): - logg.error(" %s: a .service file without [Service] section", unit) - return 101 - errors = 0 - haveType = conf.get("Service", "Type", "simple") - haveExecStart = conf.getlist("Service", "ExecStart", []) - haveExecStop = conf.getlist("Service", "ExecStop", []) - haveExecReload = conf.getlist("Service", "ExecReload", []) - usedExecStart = [] - usedExecStop = [] - usedExecReload = [] - if haveType not in [ "simple", "forking", "notify", "oneshot", "dbus", "idle", "sysv"]: - logg.error(" %s: Failed to parse service type, ignoring: %s", unit, haveType) - errors += 100 - for line in haveExecStart: - if not line.startswith("/") and not line.startswith("-/"): - logg.error(" %s: Executable path is not absolute, ignoring: %s", unit, line.strip()) - errors += 1 - usedExecStart.append(line) - for line in haveExecStop: - if not line.startswith("/") and not line.startswith("-/"): - logg.error(" %s: Executable path is not absolute, ignoring: %s", unit, line.strip()) - errors += 1 - usedExecStop.append(line) - for line in haveExecReload: - if not line.startswith("/") and not line.startswith("-/"): - logg.error(" %s: Executable path is not absolute, ignoring: %s", unit, line.strip()) - errors += 1 - usedExecReload.append(line) - if haveType in ["simple", "notify", "forking"]: - if not usedExecStart and not usedExecStop: - logg.error(" %s: Service lacks both ExecStart and ExecStop= setting. Refusing.", unit) - errors += 101 - elif not usedExecStart and haveType != "oneshot": - logg.error(" %s: Service has no ExecStart= setting, which is only allowed for Type=oneshot services. Refusing.", unit) - errors += 101 - if len(usedExecStart) > 1 and haveType != "oneshot": - logg.error(" %s: there may be only one ExecStart statement (unless for 'oneshot' services)." - + "\n\t\t\tYou can use ExecStartPre / ExecStartPost to add additional commands.", unit) - errors += 1 - if len(usedExecStop) > 1 and haveType != "oneshot": - logg.info(" %s: there should be only one ExecStop statement (unless for 'oneshot' services)." - + "\n\t\t\tYou can use ExecStopPost to add additional commands (also executed on failed Start)", unit) - if len(usedExecReload) > 1: - logg.info(" %s: there should be only one ExecReload statement." - + "\n\t\t\tUse ' ; ' for multiple commands (ExecReloadPost or ExedReloadPre do not exist)", unit) - if len(usedExecReload) > 0 and "/bin/kill " in usedExecReload[0]: - logg.warning(" %s: the use of /bin/kill is not recommended for ExecReload as it is asychronous." - + "\n\t\t\tThat means all the dependencies will perform the reload simultanously / out of order.", unit) - if conf.getlist("Service", "ExecRestart", []): #pragma: no cover - logg.error(" %s: there no such thing as an ExecRestart (ignored)", unit) - if conf.getlist("Service", "ExecRestartPre", []): #pragma: no cover - logg.error(" %s: there no such thing as an ExecRestartPre (ignored)", unit) - if conf.getlist("Service", "ExecRestartPost", []): #pragma: no cover - logg.error(" %s: there no such thing as an ExecRestartPost (ignored)", unit) - if conf.getlist("Service", "ExecReloadPre", []): #pragma: no cover - logg.error(" %s: there no such thing as an ExecReloadPre (ignored)", unit) - if conf.getlist("Service", "ExecReloadPost", []): #pragma: no cover - logg.error(" %s: there no such thing as an ExecReloadPost (ignored)", unit) - if conf.getlist("Service", "ExecStopPre", []): #pragma: no cover - logg.error(" %s: there no such thing as an ExecStopPre (ignored)", unit) - for env_file in conf.getlist("Service", "EnvironmentFile", []): - if env_file.startswith("-"): continue - if not os.path.isfile(os_path(self._root, env_file)): - logg.error(" %s: Failed to load environment files: %s", unit, env_file) - errors += 101 - return errors - def exec_check_service(self, conf, env, exectype = ""): - if not conf: - return True - if not conf.data.has_section("Service"): - return True #pragma: no cover - haveType = conf.get("Service", "Type", "simple") - if haveType in [ "sysv" ]: - return True # we don't care about that - abspath = 0 - notexists = 0 - for execs in [ "ExecStartPre", "ExecStart", "ExecStartPost", "ExecStop", "ExecStopPost", "ExecReload" ]: - if not execs.startswith(exectype): - continue - for cmd in conf.getlist("Service", execs, []): - check, cmd = checkstatus(cmd) - newcmd = self.exec_cmd(cmd, env, conf) - if not newcmd: - continue - exe = newcmd[0] - if not exe: - continue - if exe[0] != "/": - logg.error(" Exec is not an absolute path: %s=%s", execs, cmd) - abspath += 1 - if not os.path.isfile(exe): - logg.error(" Exec command does not exist: (%s) %s", execs, exe) - notexists += 1 - newexe1 = os.path.join("/usr/bin", exe) - newexe2 = os.path.join("/bin", exe) - if os.path.exists(newexe1): - logg.error(" but this does exist: %s %s", " " * len(execs), newexe1) - elif os.path.exists(newexe2): - logg.error(" but this does exist: %s %s", " " * len(execs), newexe2) - if not abspath and not notexists: - return True - if True: - filename = conf.filename() - if len(filename) > 45: filename = "..." + filename[-42:] - logg.error(" !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - logg.error(" Found %s problems in %s", abspath + notexists, filename) - time.sleep(1) - if abspath: - logg.error(" The SystemD commands must always be absolute paths by definition.") - time.sleep(1) - logg.error(" Earlier versions of systemctl.py did use a subshell thus using $PATH") - time.sleep(1) - logg.error(" however newer versions use execve just like the real SystemD daemon") - time.sleep(1) - logg.error(" so that your docker-only service scripts may start to fail suddenly.") - time.sleep(1) - if notexists: - logg.error(" Now %s executable paths were not found in the current environment.", notexists) - time.sleep(1) - logg.error(" !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - return False - def show_modules(self, *modules): - """ [PATTERN]... -- Show properties of one or more units - Show properties of one or more units (or the manager itself). - If no argument is specified, properties of the manager will be - shown. If a unit name is specified, properties of the unit is - shown. By default, empty properties are suppressed. Use --all to - show those too. To select specific properties to show, use - --property=. This command is intended to be used whenever - computer-parsable output is required. Use status if you are looking - for formatted human-readable output. - - NOTE: only a subset of properties is implemented """ - notfound = [] - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - units += [ module ] - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - return self.show_units(units) + notfound # and found_all - def show_units(self, units): - logg.debug("show --property=%s", self._unit_property) - result = [] - for unit in units: - if result: result += [ "" ] - for var, value in self.show_unit_items(unit): - if self._unit_property: - if self._unit_property != var: - continue - else: - if not value and not self._show_all: - continue - result += [ "%s=%s" % (var, value) ] - return result - def show_unit_items(self, unit): - """ [UNIT]... -- show properties of a unit. - """ - logg.info("try read unit %s", unit) - conf = self.get_unit_conf(unit) - for entry in self.each_unit_items(unit, conf): - yield entry - def each_unit_items(self, unit, conf): - loaded = conf.loaded() - if not loaded: - loaded = "not-loaded" - if "NOT-FOUND" in self.get_description_from(conf): - loaded = "not-found" - yield "Id", unit - yield "Names", unit - yield "Description", self.get_description_from(conf) # conf.get("Unit", "Description") - yield "PIDFile", self.pid_file_from(conf) # not self.pid_file_from w/o default location - yield "MainPID", self.active_pid_from(conf) or "0" # status["MainPID"] or PIDFile-read - yield "SubState", self.get_substate_from(conf) # status["SubState"] or notify-result - yield "ActiveState", self.get_active_from(conf) # status["ActiveState"] - yield "LoadState", loaded - yield "UnitFileState", self.enabled_from(conf) - yield "TimeoutStartUSec", seconds_to_time(self.get_TimeoutStartSec(conf)) - yield "TimeoutStopUSec", seconds_to_time(self.get_TimeoutStopSec(conf)) - env_parts = [] - for env_part in conf.getlist("Service", "Environment", []): - env_parts.append(self.expand_special(env_part, conf)) - if env_parts: - yield "Environment", " ".join(env_parts) - env_files = [] - for env_file in conf.getlist("Service", "EnvironmentFile", []): - env_files.append(self.expand_special(env_file, conf)) - if env_files: - yield "EnvironmentFile", " ".join(env_files) - # - igno_centos = [ "netconsole", "network" ] - igno_opensuse = [ "raw", "pppoe", "*.local", "boot.*", "rpmconf*", "purge-kernels.service", "after-local.service", "postfix*" ] - igno_ubuntu = [ "mount*", "umount*", "ondemand", "*.local" ] - igno_always = [ "network*", "dbus", "systemd-*" ] - def _ignored_unit(self, unit, ignore_list): - for ignore in ignore_list: - if fnmatch.fnmatchcase(unit, ignore): - return True # ignore - if fnmatch.fnmatchcase(unit, ignore+".service"): - return True # ignore - return False - def system_default_services(self, sysv = "S", default_target = None): - """ show the default services - This is used internally to know the list of service to be started in 'default' - runlevel when the container is started through default initialisation. It will - ignore a number of services - use '--all' to show a longer list of services and - use '--all --force' if not even a minimal filter shall be used. - """ - igno = self.igno_centos + self.igno_opensuse + self.igno_ubuntu + self.igno_always - if self._show_all: - igno = self.igno_always - if self._force: - igno = [] - logg.debug("ignored services filter for default.target:\n\t%s", igno) - return self.enabled_default_services(sysv, default_target, igno) - def enabled_default_services(self, sysv = "S", default_target = None, igno = []): - if self.user_mode(): - return self.enabled_default_user_services(sysv, default_target, igno) - else: - return self.enabled_default_system_services(sysv, default_target, igno) - def enabled_default_user_services(self, sysv = "S", default_target = None, igno = []): - logg.debug("check for default user services") - default_target = default_target or self._default_target - default_services = [] - for basefolder in self.user_folders(): - if not basefolder: - continue - folder = self.default_enablefolder(default_target, basefolder) - if self._root: - folder = os_path(self._root, folder) - if os.path.isdir(folder): - for unit in sorted(os.listdir(folder)): - path = os.path.join(folder, unit) - if os.path.isdir(path): continue - if self._ignored_unit(unit, igno): - continue # ignore - if unit.endswith(".service"): - default_services.append(unit) - for basefolder in self.system_folders(): - if not basefolder: - continue - folder = self.default_enablefolder(default_target, basefolder) - if self._root: - folder = os_path(self._root, folder) - if os.path.isdir(folder): - for unit in sorted(os.listdir(folder)): - path = os.path.join(folder, unit) - if os.path.isdir(path): continue - if self._ignored_unit(unit, igno): - continue # ignore - if unit.endswith(".service"): - conf = self.load_unit_conf(unit) - if self.not_user_conf(conf): - pass - else: - default_services.append(unit) - return default_services - def enabled_default_system_services(self, sysv = "S", default_target = None, igno = []): - logg.debug("check for default system services") - default_target = default_target or self._default_target - default_services = [] - for basefolder in self.system_folders(): - if not basefolder: - continue - folder = self.default_enablefolder(default_target, basefolder) - if self._root: - folder = os_path(self._root, folder) - if os.path.isdir(folder): - for unit in sorted(os.listdir(folder)): - path = os.path.join(folder, unit) - if os.path.isdir(path): continue - if self._ignored_unit(unit, igno): - continue # ignore - if unit.endswith(".service"): - default_services.append(unit) - for folder in [ self.rc3_root_folder() ]: - if not os.path.isdir(folder): - logg.warning("non-existant %s", folder) - continue - for unit in sorted(os.listdir(folder)): - path = os.path.join(folder, unit) - if os.path.isdir(path): continue - m = re.match(sysv+r"\d\d(.*)", unit) - if m: - service = m.group(1) - unit = service + ".service" - if self._ignored_unit(unit, igno): - continue # ignore - default_services.append(unit) - return default_services - def system_default(self, arg = True): - """ start units for default system level - This will go through the enabled services in the default 'multi-user.target'. - However some services are ignored as being known to be installation garbage - from unintended services. Use '--all' so start all of the installed services - and with '--all --force' even those services that are otherwise wrong. - /// SPECIAL: with --now or --init the init-loop is run and afterwards - a system_halt is performed with the enabled services to be stopped.""" - self.sysinit_status(SubState = "initializing") - logg.info("system default requested - %s", arg) - init = self._now or self._init - self.start_system_default(init = init) - def start_system_default(self, init = False): - """ detect the default.target services and start them. - When --init is given then the init-loop is run and - the services are stopped again by 'systemctl halt'.""" - default_target = self._default_target - default_services = self.system_default_services("S", default_target) - self.sysinit_status(SubState = "starting") - self.start_units(default_services) - logg.info(" -- system is up") - if init: - logg.info("init-loop start") - sig = self.init_loop_until_stop(default_services) - logg.info("init-loop %s", sig) - self.stop_system_default() - def stop_system_default(self): - """ detect the default.target services and stop them. - This is commonly run through 'systemctl halt' or - at the end of a 'systemctl --init default' loop.""" - default_target = self._default_target - default_services = self.system_default_services("K", default_target) - self.sysinit_status(SubState = "stopping") - self.stop_units(default_services) - logg.info(" -- system is down") - def system_halt(self, arg = True): - """ stop units from default system level """ - logg.info("system halt requested - %s", arg) - self.stop_system_default() - try: - os.kill(1, signal.SIGQUIT) # exit init-loop on no_more_procs - except Exception as e: - logg.warning("SIGQUIT to init-loop on PID-1: %s", e) - def system_get_default(self): - """ get current default run-level""" - current = self._default_target - folder = os_path(self._root, self.mask_folder()) - target = os.path.join(folder, "default.target") - if os.path.islink(target): - current = os.path.basename(os.readlink(target)) - return current - def set_default_modules(self, *modules): - """ set current default run-level""" - if not modules: - logg.debug(".. no runlevel given") - return (1, "Too few arguments") - current = self._default_target - folder = os_path(self._root, self.mask_folder()) - target = os.path.join(folder, "default.target") - if os.path.islink(target): - current = os.path.basename(os.readlink(target)) - err, msg = 0, "" - for module in modules: - if module == current: - continue - targetfile = None - for targetname, targetpath in self.each_target_file(): - if targetname == module: - targetfile = targetpath - if not targetfile: - err, msg = 3, "No such runlevel %s" % (module) - continue - # - if os.path.islink(target): - os.unlink(target) - if not os.path.isdir(os.path.dirname(target)): - os.makedirs(os.path.dirname(target)) - os.symlink(targetfile, target) - msg = "Created symlink from %s -> %s" % (target, targetfile) - logg.debug("%s", msg) - return (err, msg) - def init_modules(self, *modules): - """ [UNIT*] -- init loop: '--init default' or '--init start UNIT*' - The systemctl init service will start the enabled 'default' services, - and then wait for any zombies to be reaped. When a SIGINT is received - then a clean shutdown of the enabled services is ensured. A Control-C in - in interactive mode will also run 'stop' on all the enabled services. // - When a UNIT name is given then only that one is started instead of the - services in the 'default.target'. Using 'init UNIT' is better than - '--init start UNIT' because the UNIT is also stopped cleanly even when - it was never enabled in the system. - /// SPECIAL: when using --now then only the init-loop is started, - with the reap-zombies function and waiting for an interrupt. - (and no unit is started/stoppped wether given or not). - """ - if self._now: - return self.init_loop_until_stop([]) - if not modules: - # like 'systemctl --init default' - if self._now or self._show_all: - logg.debug("init default --now --all => no_more_procs") - self.exit_when_no_more_procs = True - return self.start_system_default(init = True) - # - # otherwise quit when all the init-services have died - self.exit_when_no_more_services = True - if self._now or self._show_all: - logg.debug("init services --now --all => no_more_procs") - self.exit_when_no_more_procs = True - found_all = True - units = [] - for module in modules: - matched = self.match_units([ module ]) - if not matched: - logg.error("Unit %s could not be found.", unit_of(module)) - found_all = False - continue - for unit in matched: - if unit not in units: - units += [ unit ] - logg.info("init %s -> start %s", ",".join(modules), ",".join(units)) - done = self.start_units(units, init = True) - logg.info("-- init is done") - return done # and found_all - def start_log_files(self, units): - self._log_file = {} - self._log_hold = {} - for unit in units: - conf = self.load_unit_conf(unit) - if not conf: continue - log_path = self.path_journal_log(conf) - try: - opened = os.open(log_path, os.O_RDONLY | os.O_NONBLOCK) - self._log_file[unit] = opened - self._log_hold[unit] = b"" - except Exception as e: - logg.error("can not open %s log: %s\n\t%s", unit, log_path, e) - def read_log_files(self, units): - BUFSIZE=8192 - for unit in units: - if unit in self._log_file: - new_text = b"" - while True: - buf = os.read(self._log_file[unit], BUFSIZE) - if not buf: break - new_text += buf - continue - text = self._log_hold[unit] + new_text - if not text: continue - lines = text.split(b"\n") - if not text.endswith(b"\n"): - self._log_hold[unit] = lines[-1] - lines = lines[:-1] - for line in lines: - prefix = unit.encode("utf-8") - content = prefix+b": "+line+b"\n" - os.write(1, content) - try: os.fsync(1) - except: pass - def stop_log_files(self, units): - for unit in units: - try: - if unit in self._log_file: - if self._log_file[unit]: - os.close(self._log_file[unit]) - except Exception as e: - logg.error("can not close log: %s\n\t%s", unit, e) - self._log_file = {} - self._log_hold = {} - def init_loop_until_stop(self, units): - """ this is the init-loop - it checks for any zombies to be reaped and - waits for an interrupt. When a SIGTERM /SIGINT /Control-C signal - is received then the signal name is returned. Any other signal will - just raise an Exception like one would normally expect. As a special - the 'systemctl halt' emits SIGQUIT which puts it into no_more_procs mode.""" - signal.signal(signal.SIGQUIT, lambda signum, frame: ignore_signals_and_raise_keyboard_interrupt("SIGQUIT")) - signal.signal(signal.SIGINT, lambda signum, frame: ignore_signals_and_raise_keyboard_interrupt("SIGINT")) - signal.signal(signal.SIGTERM, lambda signum, frame: ignore_signals_and_raise_keyboard_interrupt("SIGTERM")) - self.start_log_files(units) - self.sysinit_status(ActiveState = "active", SubState = "running") - result = None - while True: - try: - time.sleep(InitLoopSleep) - self.read_log_files(units) - ##### the reaper goes round - running = self.system_reap_zombies() - # logg.debug("reap zombies - init-loop found %s running procs", running) - if self.exit_when_no_more_services: - active = False - for unit in units: - conf = self.load_unit_conf(unit) - if not conf: continue - if self.is_active_from(conf): - active = True - if not active: - logg.info("no more services - exit init-loop") - break - if self.exit_when_no_more_procs: - if not running: - logg.info("no more procs - exit init-loop") - break - except KeyboardInterrupt as e: - if e.args and e.args[0] == "SIGQUIT": - # the original systemd puts a coredump on that signal. - logg.info("SIGQUIT - switch to no more procs check") - self.exit_when_no_more_procs = True - continue - signal.signal(signal.SIGTERM, signal.SIG_DFL) - signal.signal(signal.SIGINT, signal.SIG_DFL) - logg.info("interrupted - exit init-loop") - result = e.message or "STOPPED" - break - except Exception as e: - logg.info("interrupted - exception %s", e) - raise - self.sysinit_status(ActiveState = None, SubState = "degraded") - self.read_log_files(units) - self.read_log_files(units) - self.stop_log_files(units) - logg.debug("done - init loop") - return result - def system_reap_zombies(self): - """ check to reap children """ - selfpid = os.getpid() - running = 0 - for pid in os.listdir("/proc"): - try: pid = int(pid) - except: continue - if pid == selfpid: - continue - proc_status = "/proc/%s/status" % pid - if os.path.isfile(proc_status): - zombie = False - ppid = -1 - try: - for line in open(proc_status): - m = re.match(r"State:\s*Z.*", line) - if m: zombie = True - m = re.match(r"PPid:\s*(\d+)", line) - if m: ppid = int(m.group(1)) - except IOError as e: - logg.warning("%s : %s", proc_status, e) - continue - if zombie and ppid == os.getpid(): - logg.info("reap zombie %s", pid) - try: os.waitpid(pid, os.WNOHANG) - except OSError as e: - logg.warning("reap zombie %s: %s", e.strerror) - if os.path.isfile(proc_status): - if pid > 1: - running += 1 - return running # except PID 0 and PID 1 - def sysinit_status(self, **status): - conf = self.sysinit_target() - self.write_status_from(conf, **status) - def sysinit_target(self): - if not self._sysinit_target: - self._sysinit_target = self.default_unit_conf("sysinit.target", "System Initialization") - return self._sysinit_target - def is_system_running(self): - conf = self.sysinit_target() - status_file = self.status_file_from(conf) - if not os.path.isfile(status_file): - time.sleep(EpsilonTime) - if not os.path.isfile(status_file): - return "offline" - status = self.read_status_from(conf) - return status.get("SubState", "unknown") - def system_is_system_running(self): - state = self.is_system_running() - if self._quiet: - return state in [ "running" ] - else: - if state in [ "running" ]: - return True, state - else: - return False, state - def wait_system(self, target = None): - target = target or SysInitTarget - for attempt in xrange(int(SysInitWait)): - state = self.is_system_running() - if "init" in state: - if target in [ "sysinit.target", "basic.target" ]: - logg.info("system not initialized - wait %s", target) - time.sleep(1) - continue - if "start" in state or "stop" in state: - if target in [ "basic.target" ]: - logg.info("system not running - wait %s", target) - time.sleep(1) - continue - if "running" not in state: - logg.info("system is %s", state) - break - def pidlist_of(self, pid): - try: pid = int(pid) - except: return [] - pidlist = [ pid ] - pids = [ pid ] - for depth in xrange(ProcMaxDepth): - for pid in os.listdir("/proc"): - try: pid = int(pid) - except: continue - proc_status = "/proc/%s/status" % pid - if os.path.isfile(proc_status): - try: - for line in open(proc_status): - if line.startswith("PPid:"): - ppid = line[len("PPid:"):].strip() - try: ppid = int(ppid) - except: continue - if ppid in pidlist and pid not in pids: - pids += [ pid ] - except IOError as e: - logg.warning("%s : %s", proc_status, e) - continue - if len(pids) != len(pidlist): - pidlist = pids[:] - continue - return pids - def etc_hosts(self): - path = "/etc/hosts" - if self._root: - return os_path(self._root, path) - return path - def force_ipv4(self, *args): - """ only ipv4 localhost in /etc/hosts """ - logg.debug("checking /etc/hosts for '::1 localhost'") - lines = [] - for line in open(self.etc_hosts()): - if "::1" in line: - newline = re.sub("\\slocalhost\\s", " ", line) - if line != newline: - logg.info("/etc/hosts: '%s' => '%s'", line.rstrip(), newline.rstrip()) - line = newline - lines.append(line) - f = open(self.etc_hosts(), "w") - for line in lines: - f.write(line) - f.close() - def force_ipv6(self, *args): - """ only ipv4 localhost in /etc/hosts """ - logg.debug("checking /etc/hosts for '127.0.0.1 localhost'") - lines = [] - for line in open(self.etc_hosts()): - if "127.0.0.1" in line: - newline = re.sub("\\slocalhost\\s", " ", line) - if line != newline: - logg.info("/etc/hosts: '%s' => '%s'", line.rstrip(), newline.rstrip()) - line = newline - lines.append(line) - f = open(self.etc_hosts(), "w") - for line in lines: - f.write(line) - f.close() - def show_help(self, *args): - """[command] -- show this help - """ - lines = [] - okay = True - prog = os.path.basename(sys.argv[0]) - if not args: - argz = {} - for name in dir(self): - arg = None - if name.startswith("system_"): - arg = name[len("system_"):].replace("_","-") - if name.startswith("show_"): - arg = name[len("show_"):].replace("_","-") - if name.endswith("_of_unit"): - arg = name[:-len("_of_unit")].replace("_","-") - if name.endswith("_modules"): - arg = name[:-len("_modules")].replace("_","-") - if arg: - argz[arg] = name - lines.append("%s command [options]..." % prog) - lines.append("") - lines.append("Commands:") - for arg in sorted(argz): - name = argz[arg] - method = getattr(self, name) - doc = "..." - doctext = getattr(method, "__doc__") - if doctext: - doc = doctext - elif not self._show_all: - continue # pragma: nocover - firstline = doc.split("\n")[0] - doc_text = firstline.strip() - if "--" not in firstline: - doc_text = "-- " + doc_text - lines.append(" %s %s" % (arg, firstline.strip())) - return lines - for arg in args: - arg = arg.replace("-","_") - func1 = getattr(self.__class__, arg+"_modules", None) - func2 = getattr(self.__class__, arg+"_of_unit", None) - func3 = getattr(self.__class__, "show_"+arg, None) - func4 = getattr(self.__class__, "system_"+arg, None) - func = func1 or func2 or func3 or func4 - if func is None: - print("error: no such command '%s'" % arg) - okay = False - else: - doc_text = "..." - doc = getattr(func, "__doc__", None) - if doc: - doc_text = doc.replace("\n","\n\n", 1).strip() - if "--" not in doc_text: - doc_text = "-- " + doc_text - else: - logg.debug("__doc__ of %s is none", func_name) - if not self._show_all: continue - lines.append("%s %s %s" % (prog, arg, doc_text)) - if not okay: - self.show_help() - return False - return lines - def systemd_version(self): - """ the version line for systemd compatibility """ - return "systemd %s\n - via systemctl.py %s" % (self._systemd_version, __version__) - def systemd_features(self): - """ the info line for systemd features """ - features1 = "-PAM -AUDIT -SELINUX -IMA -APPARMOR -SMACK" - features2 = " +SYSVINIT -UTMP -LIBCRYPTSETUP -GCRYPT -GNUTLS" - features3 = " -ACL -XZ -LZ4 -SECCOMP -BLKID -ELFUTILS -KMOD -IDN" - return features1+features2+features3 - def systems_version(self): - return [ self.systemd_version(), self.systemd_features() ] - -def print_result(result): - # logg_info = logg.info - # logg_debug = logg.debug - def logg_info(*msg): pass - def logg_debug(*msg): pass - exitcode = 0 - if result is None: - logg_info("EXEC END None") - elif result is True: - logg_info("EXEC END True") - result = None - exitcode = 0 - elif result is False: - logg_info("EXEC END False") - result = None - exitcode = 1 - elif isinstance(result, tuple) and len(result) == 2: - exitcode, status = result - logg_info("EXEC END %s '%s'", exitcode, status) - if exitcode is True: exitcode = 0 - if exitcode is False: exitcode = 1 - result = status - elif isinstance(result, int): - logg_info("EXEC END %s", result) - exitcode = result - result = None - # - if result is None: - pass - elif isinstance(result, string_types): - print(result) - result1 = result.split("\n")[0][:-20] - if result == result1: - logg_info("EXEC END '%s'", result) - else: - logg_info("EXEC END '%s...'", result1) - logg_debug(" END '%s'", result) - elif isinstance(result, list) or hasattr(result, "next") or hasattr(result, "__next__"): - shown = 0 - for element in result: - if isinstance(element, tuple): - print("\t".join([ str(elem) for elem in element] )) - else: - print(element) - shown += 1 - logg_info("EXEC END %s items", shown) - logg_debug(" END %s", result) - elif hasattr(result, "keys"): - shown = 0 - for key in sorted(result.keys()): - element = result[key] - if isinstance(element, tuple): - print(key,"=","\t".join([ str(elem) for elem in element])) - else: - print("%s=%s" % (key,element)) - shown += 1 - logg_info("EXEC END %s items", shown) - logg_debug(" END %s", result) - else: - logg.warning("EXEC END Unknown result type %s", str(type(result))) - return exitcode - -if __name__ == "__main__": - import optparse - _o = optparse.OptionParser("%prog [options] command [name...]", - epilog="use 'help' command for more information") - _o.add_option("--version", action="store_true", - help="Show package version") - _o.add_option("--system", action="store_true", default=False, - help="Connect to system manager (default)") # overrides --user - _o.add_option("--user", action="store_true", default=_user_mode, - help="Connect to user service manager") - # _o.add_option("-H", "--host", metavar="[USER@]HOST", - # help="Operate on remote host*") - # _o.add_option("-M", "--machine", metavar="CONTAINER", - # help="Operate on local container*") - _o.add_option("-t","--type", metavar="TYPE", dest="unit_type", default=_unit_type, - help="List units of a particual type") - _o.add_option("--state", metavar="STATE", default=_unit_state, - help="List units with particular LOAD or SUB or ACTIVE state") - _o.add_option("-p", "--property", metavar="NAME", dest="unit_property", default=_unit_property, - help="Show only properties by this name") - _o.add_option("-a", "--all", action="store_true", dest="show_all", default=_show_all, - help="Show all loaded units/properties, including dead empty ones. To list all units installed on the system, use the 'list-unit-files' command instead") - _o.add_option("-l","--full", action="store_true", default=_full, - help="Don't ellipsize unit names on output (never ellipsized)") - _o.add_option("--reverse", action="store_true", - help="Show reverse dependencies with 'list-dependencies' (ignored)") - _o.add_option("--job-mode", metavar="MODE", - help="Specifiy how to deal with already queued jobs, when queuing a new job (ignored)") - _o.add_option("--show-types", action="store_true", - help="When showing sockets, explicitly show their type (ignored)") - _o.add_option("-i","--ignore-inhibitors", action="store_true", - help="When shutting down or sleeping, ignore inhibitors (ignored)") - _o.add_option("--kill-who", metavar="WHO", - help="Who to send signal to (ignored)") - _o.add_option("-s", "--signal", metavar="SIG", - help="Which signal to send (ignored)") - _o.add_option("--now", action="store_true", default=_now, - help="Start or stop unit in addition to enabling or disabling it") - _o.add_option("-q","--quiet", action="store_true", default=_quiet, - help="Suppress output") - _o.add_option("--no-block", action="store_true", default=False, - help="Do not wait until operation finished (ignored)") - _o.add_option("--no-legend", action="store_true", default=_no_legend, - help="Do not print a legend (column headers and hints)") - _o.add_option("--no-wall", action="store_true", default=False, - help="Don't send wall message before halt/power-off/reboot (ignored)") - _o.add_option("--no-reload", action="store_true", - help="Don't reload daemon after en-/dis-abling unit files (ignored)") - _o.add_option("--no-ask-password", action="store_true", default=_no_ask_password, - help="Do not ask for system passwords") - # _o.add_option("--global", action="store_true", dest="globally", default=_globally, - # help="Enable/disable unit files globally") # for all user logins - # _o.add_option("--runtime", action="store_true", - # help="Enable unit files only temporarily until next reboot") - _o.add_option("--force", action="store_true", default=_force, - help="When enabling unit files, override existing symblinks / When shutting down, execute action immediately") - _o.add_option("--preset-mode", metavar="TYPE", default=_preset_mode, - help="Apply only enable, only disable, or all presets [%default]") - _o.add_option("--root", metavar="PATH", default=_root, - help="Enable unit files in the specified root directory (used for alternative root prefix)") - _o.add_option("-n","--lines", metavar="NUM", - help="Number of journal entries to show (ignored)") - _o.add_option("-o","--output", metavar="CAT", - help="change journal output mode [short, ..., cat] (ignored)") - _o.add_option("--plain", action="store_true", - help="Print unit dependencies as a list instead of a tree (ignored)") - _o.add_option("--no-pager", action="store_true", - help="Do not pipe output into pager (ignored)") - # - _o.add_option("--coverage", metavar="OPTIONLIST", default=COVERAGE, - help="..support for coverage (e.g. spawn,oldest,sleep) [%default]") - _o.add_option("-e","--extra-vars", "--environment", metavar="NAME=VAL", action="append", default=[], - help="..override settings in the syntax of 'Environment='") - _o.add_option("-v","--verbose", action="count", default=0, - help="..increase debugging information level") - _o.add_option("-4","--ipv4", action="store_true", default=False, - help="..only keep ipv4 localhost in /etc/hosts") - _o.add_option("-6","--ipv6", action="store_true", default=False, - help="..only keep ipv6 localhost in /etc/hosts") - _o.add_option("-1","--init", action="store_true", default=False, - help="..keep running as init-process (default if PID 1)") - opt, args = _o.parse_args() - logging.basicConfig(level = max(0, logging.FATAL - 10 * opt.verbose)) - logg.setLevel(max(0, logging.ERROR - 10 * opt.verbose)) - # - COVERAGE = opt.coverage - if "sleep" in COVERAGE: - MinimumTimeoutStartSec = 7 - MinimumTimeoutStopSec = 7 - if "quick" in COVERAGE: - MinimumTimeoutStartSec = 4 - MinimumTimeoutStopSec = 4 - DefaultTimeoutStartSec = 9 - DefaultTimeoutStopSec = 9 - _extra_vars = opt.extra_vars - _force = opt.force - _full = opt.full - _no_legend = opt.no_legend - _no_ask_password = opt.no_ask_password - _now = opt.now - _preset_mode = opt.preset_mode - _quiet = opt.quiet - _root = opt.root - _show_all = opt.show_all - _unit_state = opt.state - _unit_type = opt.unit_type - _unit_property = opt.unit_property - # being PID 1 (or 0) in a container will imply --init - _pid = os.getpid() - _init = opt.init or _pid in [ 1, 0 ] - _user_mode = opt.user - if os.geteuid() and _pid in [ 1, 0 ]: - _user_mode = True - if opt.system: - _user_mode = False # override --user - # - if _user_mode: - systemctl_debug_log = os_path(_root, _var_path(_systemctl_debug_log)) - systemctl_extra_log = os_path(_root, _var_path(_systemctl_extra_log)) - else: - systemctl_debug_log = os_path(_root, _systemctl_debug_log) - systemctl_extra_log = os_path(_root, _systemctl_extra_log) - if os.access(systemctl_extra_log, os.W_OK): - loggfile = logging.FileHandler(systemctl_extra_log) - loggfile.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s")) - logg.addHandler(loggfile) - logg.setLevel(max(0, logging.INFO - 10 * opt.verbose)) - if os.access(systemctl_debug_log, os.W_OK): - loggfile = logging.FileHandler(systemctl_debug_log) - loggfile.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s")) - logg.addHandler(loggfile) - logg.setLevel(logging.DEBUG) - logg.info("EXEC BEGIN %s %s%s%s", os.path.realpath(sys.argv[0]), " ".join(args), - _user_mode and " --user" or " --system", _init and " --init" or "", ) - # - # - systemctl = Systemctl() - if opt.version: - args = [ "version" ] - if not args: - if _init: - args = [ "default" ] - else: - args = [ "list-units" ] - logg.debug("======= systemctl.py " + " ".join(args)) - command = args[0] - modules = args[1:] - if opt.ipv4: - systemctl.force_ipv4() - elif opt.ipv6: - systemctl.force_ipv6() - found = False - # command NAME - if command.startswith("__"): - command_name = command[2:] - command_func = getattr(systemctl, command_name, None) - if callable(command_func) and not found: - found = True - result = command_func(*modules) - command_name = command.replace("-","_").replace(".","_")+"_modules" - command_func = getattr(systemctl, command_name, None) - if callable(command_func) and not found: - systemctl.wait_boot(command_name) - found = True - result = command_func(*modules) - command_name = "show_"+command.replace("-","_").replace(".","_") - command_func = getattr(systemctl, command_name, None) - if callable(command_func) and not found: - systemctl.wait_boot(command_name) - found = True - result = command_func(*modules) - command_name = "system_"+command.replace("-","_").replace(".","_") - command_func = getattr(systemctl, command_name, None) - if callable(command_func) and not found: - systemctl.wait_boot(command_name) - found = True - result = command_func() - command_name = "systems_"+command.replace("-","_").replace(".","_") - command_func = getattr(systemctl, command_name, None) - if callable(command_func) and not found: - systemctl.wait_boot(command_name) - found = True - result = command_func() - if not found: - logg.error("Unknown operation %s.", command) - sys.exit(1) - # - sys.exit(print_result(result)) \ No newline at end of file From 5ab2b17f91fa1a06f570ba9e023608e660a927b1 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Thu, 19 Mar 2020 08:40:02 -0300 Subject: [PATCH 03/11] feat: add stanza from upstream example formula Signed-off-by: Felipe Zipitria --- .rubocop.yml | 11 ++ .salt-lint | 14 +++ .yamllint | 36 ++++++ FORMULA | 9 ++ commitlint.config.js | 3 + datadog/map.jinja | 2 +- kitchen.yml | 8 +- pre-commit_semantic-release.sh | 30 +++++ release-rules.js | 18 +++ release.config.js | 106 ++++++++++++++++++ .../{datadog => default}/README.md | 0 .../{datadog => default}/controls/datadog.rb | 3 +- .../{datadog => default}/inspec.yml | 0 13 files changed, 234 insertions(+), 6 deletions(-) create mode 100644 .rubocop.yml create mode 100644 .salt-lint create mode 100644 .yamllint create mode 100644 FORMULA create mode 100644 commitlint.config.js create mode 100755 pre-commit_semantic-release.sh create mode 100644 release-rules.js create mode 100644 release.config.js rename test/integration/{datadog => default}/README.md (100%) rename test/integration/{datadog => default}/controls/datadog.rb (98%) rename test/integration/{datadog => default}/inspec.yml (100%) diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 0000000..7c701f5 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +--- +# General overrides used across formulas in the org +Layout/LineLength: + # Increase from default of `80` + # Based on https://github.com/PyCQA/flake8-bugbear#opinionated-warnings (`B950`) + Max: 88 + +# Any offenses that should be fixed, e.g. collected via. `rubocop --auto-gen-config` + diff --git a/.salt-lint b/.salt-lint new file mode 100644 index 0000000..3715677 --- /dev/null +++ b/.salt-lint @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +--- +exclude_paths: [] +rules: {} +skip_list: + # Using `salt-lint` for linting other files as well, such as Jinja macros/templates + - 205 # Use ".sls" as a Salt State file extension + # Skipping `207` and `208` because `210` is sufficient, at least for the time-being + # I.e. Allows 3-digit unquoted codes to still be used, such as `644` and `755` + - 207 # File modes should always be encapsulated in quotation marks + - 208 # File modes should always contain a leading zero +tags: [] +verbosity: 1 diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..740beca --- /dev/null +++ b/.yamllint @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +--- +# Extend the `default` configuration provided by `yamllint` +extends: default + +# Files to ignore completely +# 1. All YAML files under directory `node_modules/`, introduced during the Travis run +# 2. Any SLS files under directory `test/`, which are actually state files +# 3. Any YAML files under directory `.kitchen/`, introduced during local testing +ignore: | + node_modules/ + test/**/states/**/*.sls + .kitchen/ + +yaml-files: + # Default settings + - '*.yaml' + - '*.yml' + - .salt-lint + - .yamllint + # SaltStack Formulas additional settings + - '*.example' + - test/**/*.sls + +rules: + empty-values: + forbid-in-block-mappings: true + forbid-in-flow-mappings: true + line-length: + # Increase from default of `80` + # Based on https://github.com/PyCQA/flake8-bugbear#opinionated-warnings (`B950`) + max: 88 + octal-values: + forbid-implicit-octal: true + forbid-explicit-octal: true diff --git a/FORMULA b/FORMULA new file mode 100644 index 0000000..f11ef1d --- /dev/null +++ b/FORMULA @@ -0,0 +1,9 @@ +name: datadog +os: Debian, Ubuntu, Raspbian, RedHat, Fedora, CentOS, Amazon, Suse, openSUSE, Gentoo, Funtoo, Arch, Manjaro, Alpine, FreeBSD, OpenBSD, Solaris, SmartOS, Windows, MacOS +os_family: Debian, RedHat, Suse, Gentoo, Arch, Alpine, FreeBSD, OpenBSD, Solaris, Windows, MacOS +version: 1.0.0 +release: 1 +minimum_version: 2017.7 +summary: datadog formula +description: Formula to use as a template for other formulas +top_level_dir: datadog diff --git a/commitlint.config.js b/commitlint.config.js new file mode 100644 index 0000000..2f9d1aa --- /dev/null +++ b/commitlint.config.js @@ -0,0 +1,3 @@ +module.exports = { + extends: ['@commitlint/config-conventional'], +}; diff --git a/datadog/map.jinja b/datadog/map.jinja index 51c933e..7f7fab8 100644 --- a/datadog/map.jinja +++ b/datadog/map.jinja @@ -14,7 +14,7 @@ 'agent_version': 'latest', }, } -}%} +} %} {# Merge os_family_map into the default settings #} {% do default_settings.datadog.update(os_family_map) %} diff --git a/kitchen.yml b/kitchen.yml index 2d729a4..01932e7 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -76,7 +76,7 @@ suites: datadog.sls: test/salt/pillar/datadog5.sls verifier: inspec_tests: - - path: test/integration/datadog + - path: test/integration/default inputs: version: 5 @@ -95,7 +95,7 @@ suites: datadog.sls: test/salt/pillar/datadog6.sls verifier: inspec_tests: - - path: test/integration/datadog + - path: test/integration/default inputs: version: 6 @@ -114,7 +114,7 @@ suites: datadog.sls: test/salt/pillar/datadog7.sls verifier: inspec_tests: - - path: test/integration/datadog + - path: test/integration/default inputs: - version: 6 + version: 7 diff --git a/pre-commit_semantic-release.sh b/pre-commit_semantic-release.sh new file mode 100755 index 0000000..ba80535 --- /dev/null +++ b/pre-commit_semantic-release.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +############################################################################### +# (A) Update `FORMULA` with `${nextRelease.version}` +############################################################################### +sed -i -e "s_^\(version:\).*_\1 ${1}_" FORMULA + + +############################################################################### +# (B) Use `m2r` to convert automatically produced `.md` docs to `.rst` +############################################################################### + +# Install `m2r` +sudo -H pip install m2r + +# Copy and then convert the `.md` docs +cp ./*.md docs/ +cd docs/ || exit +m2r --overwrite ./*.md + +# Change excess `H1` headings to `H2` in converted `CHANGELOG.rst` +sed -i -e '/^=.*$/s/=/-/g' CHANGELOG.rst +sed -i -e '1,4s/-/=/g' CHANGELOG.rst + +# Use for debugging output, when required +# cat AUTHORS.rst +# cat CHANGELOG.rst + +# Return back to the main directory +cd .. diff --git a/release-rules.js b/release-rules.js new file mode 100644 index 0000000..c63c850 --- /dev/null +++ b/release-rules.js @@ -0,0 +1,18 @@ +// No release is triggered for the types commented out below. +// Commits using these types will be incorporated into the next release. +// +// NOTE: Any changes here must be reflected in `CONTRIBUTING.md`. +module.exports = [ + {breaking: true, release: 'major'}, + // {type: 'build', release: 'patch'}, + // {type: 'chore', release: 'patch'}, + // {type: 'ci', release: 'patch'}, + {type: 'docs', release: 'patch'}, + {type: 'feat', release: 'minor'}, + {type: 'fix', release: 'patch'}, + {type: 'perf', release: 'patch'}, + {type: 'refactor', release: 'patch'}, + {type: 'revert', release: 'patch'}, + {type: 'style', release: 'patch'}, + {type: 'test', release: 'patch'}, +]; diff --git a/release.config.js b/release.config.js new file mode 100644 index 0000000..6af7aa8 --- /dev/null +++ b/release.config.js @@ -0,0 +1,106 @@ +module.exports = { + branch: 'master', + plugins: [ + ['@semantic-release/commit-analyzer', { + preset: 'angular', + releaseRules: './release-rules.js', + }], + '@semantic-release/release-notes-generator', + ['@semantic-release/changelog', { + changelogFile: 'CHANGELOG.md', + changelogTitle: '# Changelog', + }], + ['@semantic-release/exec', { + prepareCmd: 'sh ./pre-commit_semantic-release.sh ${nextRelease.version}', + }], + ['@semantic-release/git', { + assets: ['*.md', 'docs/*.rst', 'FORMULA'], + }], + '@semantic-release/github', + ], + generateNotes: { + preset: 'angular', + writerOpts: { + // Required due to upstream bug preventing all types being displayed. + // Bug: https://github.com/conventional-changelog/conventional-changelog/issues/317 + // Fix: https://github.com/conventional-changelog/conventional-changelog/pull/410 + transform: (commit, context) => { + const issues = [] + + commit.notes.forEach(note => { + note.title = `BREAKING CHANGES` + }) + + // NOTE: Any changes here must be reflected in `CONTRIBUTING.md`. + if (commit.type === `feat`) { + commit.type = `Features` + } else if (commit.type === `fix`) { + commit.type = `Bug Fixes` + } else if (commit.type === `perf`) { + commit.type = `Performance Improvements` + } else if (commit.type === `revert`) { + commit.type = `Reverts` + } else if (commit.type === `docs`) { + commit.type = `Documentation` + } else if (commit.type === `style`) { + commit.type = `Styles` + } else if (commit.type === `refactor`) { + commit.type = `Code Refactoring` + } else if (commit.type === `test`) { + commit.type = `Tests` + } else if (commit.type === `build`) { + commit.type = `Build System` + // } else if (commit.type === `chore`) { + // commit.type = `Maintenance` + } else if (commit.type === `ci`) { + commit.type = `Continuous Integration` + } else { + return + } + + if (commit.scope === `*`) { + commit.scope = `` + } + + if (typeof commit.hash === `string`) { + commit.shortHash = commit.hash.substring(0, 7) + } + + if (typeof commit.subject === `string`) { + let url = context.repository + ? `${context.host}/${context.owner}/${context.repository}` + : context.repoUrl + if (url) { + url = `${url}/issues/` + // Issue URLs. + commit.subject = commit.subject.replace(/#([0-9]+)/g, (_, issue) => { + issues.push(issue) + return `[#${issue}](${url}${issue})` + }) + } + if (context.host) { + // User URLs. + commit.subject = commit.subject.replace(/\B@([a-z0-9](?:-?[a-z0-9/]){0,38})/g, (_, username) => { + if (username.includes('/')) { + return `@${username}` + } + + return `[@${username}](${context.host}/${username})` + }) + } + } + + // remove references that already appear in the subject + commit.references = commit.references.filter(reference => { + if (issues.indexOf(reference.issue) === -1) { + return true + } + + return false + }) + + return commit + }, + }, + }, +}; diff --git a/test/integration/datadog/README.md b/test/integration/default/README.md similarity index 100% rename from test/integration/datadog/README.md rename to test/integration/default/README.md diff --git a/test/integration/datadog/controls/datadog.rb b/test/integration/default/controls/datadog.rb similarity index 98% rename from test/integration/datadog/controls/datadog.rb rename to test/integration/default/controls/datadog.rb index 99d09d2..095d353 100644 --- a/test/integration/datadog/controls/datadog.rb +++ b/test/integration/default/controls/datadog.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Get datadog major as input variable datadog_version = input('version') @@ -61,4 +63,3 @@ its('group') { should eq 'dd-agent' } end end - diff --git a/test/integration/datadog/inspec.yml b/test/integration/default/inspec.yml similarity index 100% rename from test/integration/datadog/inspec.yml rename to test/integration/default/inspec.yml From 8c184f2ac7b072fb525646c1e3891a520648f08d Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Thu, 19 Mar 2020 11:18:26 -0300 Subject: [PATCH 04/11] docs: merge README contents with the one from template --- .rubocop.yml | 1 - README.rst | 152 ------------- docs/README.rst | 186 +++++++++++++++ docs/TOFS_pattern.rst | 518 ++++++++++++++++++++++++++++++++++++++++++ kitchen.yml | 1 - 5 files changed, 704 insertions(+), 154 deletions(-) delete mode 100644 README.rst create mode 100644 docs/README.rst create mode 100644 docs/TOFS_pattern.rst diff --git a/.rubocop.yml b/.rubocop.yml index 7c701f5..96fd6e5 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -8,4 +8,3 @@ Layout/LineLength: Max: 88 # Any offenses that should be fixed, e.g. collected via. `rubocop --auto-gen-config` - diff --git a/README.rst b/README.rst deleted file mode 100644 index 8ecb1a8..0000000 --- a/README.rst +++ /dev/null @@ -1,152 +0,0 @@ -Datadog Formula -=============== - -SaltStack Formula to install the Datadog Agent and the Agent based integrations, -also called Checks. See the full `Salt Formulas installation and usage instructions `_. - -Available States -================ - -.. contents:: - :local: - -``datadog`` ------------ - -Installs, configures and starts the Datadog Agent service. - -``datadog.install`` ------------------- - -Configure the right repo and installs the Agent. - -``datadog.config`` ------------------- - -Configures Agent and Integrations using pillar data. See `pillar.example`. - -**NOTE:** in order to split a single check type's configuration among multiple -pillar files (eg. to configure different check instances on different machines), -the `pillar_merge_lists` option must be set to `True` in the Salt master config -(or the salt minion config if running masterless) (see -https://docs.saltstack.com/en/latest/ref/configuration/master.html#pillar-merge-lists). - -``datadog.service`` ------------------- - -Runs the Datadog Agent service, watching for changes to the config files for the -Agent itself and the checks. - -``datadog.uninstall`` ------------------- - -Stops the service and uninstalls Datadog Agent. - -Pillar configuration -==================== - -The formula configuration must be written in the ``datadog`` key of the pillar file. - -The formula configuration contains three parts: ``config``, ``install_settings``, and ``checks``. - -``config`` ----------- -The ``config`` option contains the configuration options which will be written in the minions' Agent configuration file (``datadog.yaml`` for Agent v6 & v7, ``datadog.conf`` for Agent v5). - -Depending on the Agent version installed, different options can be set: - -- Agent v6 & v7: all options supported by the Agent's configuration file are supported. -- Agent v5: only the ``api_key`` option is supported. - -Example: set the API key, and the site option to ``datadoghq.eu`` (Agent v6 only) - -.. code:: - - datadog: - config: - api_key: - site: datadoghq.eu - -``install_settings`` --------------------- -The ``install_settings`` option contains the Agent installation configuration options. -It has the following option: - -- ``agent_version``: the version of the Agent which will be installed. Default: latest (will install the latest Agent v7 package). - -Example: install the Agent version ``6.14.1`` - -.. code:: - - datadog: - install_settings: - agent_version: 6.14.1 - - -``checks`` ----------- -The ``checks`` option contains configuration for the Agent Checks. - -To add an Agent Check, add an entry in the ``checks`` option with the check's name as the key. - -Each check has two options: - -- ``config``: contains the check's configuration, which will be written to the check's configuration file (``/.d/conf.yaml`` for Agent v6/v7, ``/.yaml`` for Agent v5). -- ``version``: the version of the check which will be installed (Agent v6 and v7 only). Default: the version bundled with the agent. - -Example: ``directory`` check version ``1.4.0``, monitoring the ``/srv/pillar`` directory - -.. code:: - - datadog: - checks: - directory: - config: - instances: - - directory: "/srv/pillar" - name: "pillars" - version: 1.4.0 - -Development -=========== - -To ease the development of the formula, you can use Docker and Docker Compose with -the compose file in `test/docker-compose.yaml`. - -First, build and run a Docker container to create a masterless SaltStack minion. You have the option of choosing either -a Debian- or Redhat-based minion. Then, get a shell running in the container. - -.. code-block:: shell - - $ cd test/ - $ TEST_DIST=debian docker-compose run masterless /bin/bash - -Once you've built the container and have a shell up and running, you need to apply the SaltStack state on your minion: - -.. code-block:: shell - - $ # On your SaltStack minion - $ salt-call --local state.highstate -l debug - -Testing -========= - -A proper integration test suite is still a Work in Progress, in the meantime you -can use the Docker Compose file provided in the `test` directory to easily check -out the formula in action. - -Requirements ------------- - -* Docker -* Docker Compose - -Run the formula ---------------- - -.. code-block:: shell - - $ cd test/ - $ TEST_DIST=debian docker-compose up --build - -You should be able to see from the logs if all the states completed successfully. diff --git a/docs/README.rst b/docs/README.rst new file mode 100644 index 0000000..04576ab --- /dev/null +++ b/docs/README.rst @@ -0,0 +1,186 @@ +.. _readme: + +Datadog Formula +================ + +|img_travis| |img_sr| + +.. |img_travis| image:: ![Integration](https://github.com/Perceptyx/datadog-formula/workflows/Integration/badge.svg) + :alt: GitHub Actions Build Status + :scale: 100% + :target: https://github.com/Perceptyx/datadog-formula/actions + +SaltStack Formula to install the Datadog Agent and the Agent based integrations, also called Checks. + +.. contents:: **Table of Contents** + +General notes +------------- + +See the full `SaltStack Formulas installation and usage instructions +`_. + +If you are interested in writing or contributing to formulas, please pay attention to the `Writing Formula Section +`_. + +If you want to use this formula, please pay attention to the ``FORMULA`` file and/or ``git tag``, +which contains the currently released version. This formula is versioned according to `Semantic Versioning `_. + +See `Formula Versioning Section `_ for more details. + +If you need (non-default) configuration, please pay attention to the ``pillar.example`` file and/or `Special notes`_ section. + +Contributing to this repo +------------------------- + +**Commit message formatting is significant!!** + +Please see `How to contribute `_ for more details. + +Special notes +------------- + +None + +Available states +---------------- + +.. contents:: + :local: + +``datadog`` +^^^^^^^^^^^^ + +*Meta-state (This is a state that includes other states)*. + +This installs the datadog package, +manages the datadog configuration file and then +starts the associated Datadog Agent service. + +``datadog.package`` +^^^^^^^^^^^^^^^^^^^^ + +This state will configure repos and install the datadog package only. + +``datadog.config`` +^^^^^^^^^^^^^^^^^^^ + +This state will configure the datadog service and has a dependency on ``datadog.install`` +via include list. + +**NOTE:** in order to split a single check type's configuration among multiple +pillar files (eg. to configure different check instances on different machines), +the `pillar_merge_lists` option must be set to `True` in the Salt master config +(or the salt minion config if running masterless) (see +https://docs.saltstack.com/en/latest/ref/configuration/master.html#pillar-merge-lists). + + +``datadog.service`` +^^^^^^^^^^^^^^^^^^^^ + +This state will start the datadog service and has a dependency on ``datadog.config`` +via include list. + +``datadog.uninstall`` +^^^^^^^^^^^^^^^^^^ + +*Meta-state (This is a state that includes other states)*. + +this state will undo everything performed in the ``datadog`` meta-state in reverse order, i.e. +stops the service, removes the configuration file and then uninstalls the package. + +Pillar configuration +==================== + +The formula configuration must be written in the ``datadog`` key of the pillar file. + +The formula configuration contains three parts: ``config``, ``install_settings``, and ``checks``. + +``config`` +---------- +The ``config`` option contains the configuration options which will be written in the minions' Agent configuration file (``datadog.yaml`` for Agent v6 & v7, ``datadog.conf`` for Agent v5). + +Depending on the Agent version installed, different options can be set: + +- Agent v6 & v7: all options supported by the Agent's configuration file are supported. +- Agent v5: only the ``api_key`` option is supported. + +Example: set the API key, and the site option to ``datadoghq.eu`` (Agent v6 only) +.. code:: + + datadog: + install_settings: + agent_version: 6.14.1 + + +``checks`` +---------- +The ``checks`` option contains configuration for the Agent Checks. + +To add an Agent Check, add an entry in the ``checks`` option with the check's name as the key. + +Each check has two options: + +- ``config``: contains the check's configuration, which will be written to the check's configuration file (``/.d/conf.yaml`` for Agent v6/v7, ``/.yaml`` for Agent v5). +- ``version``: the version of the check which will be installed (Agent v6 and v7 only). Default: the version bundled with the agent. + +Example: ``directory`` check version ``1.4.0``, monitoring the ``/srv/pillar`` directory + +.. code:: + + datadog: + checks: + directory: + config: + instances: + - directory: "/srv/pillar" + name: "pillars" + version: 1.4.0 + + + +Testing +------- + +Linux testing is done with ``kitchen-salt``. + +Requirements +^^^^^^^^^^^^ + +* Ruby +* Docker + +.. code-block:: bash + + $ gem install bundler + $ bundle install + $ bin/kitchen test [platform] + +Where ``[platform]`` is the platform name defined in ``kitchen.yml``, +e.g. ``debian-9-2019-2-py3``. + +``bin/kitchen converge`` +^^^^^^^^^^^^^^^^^^^^^^^^ + +Creates the docker instance and runs the ``datadog`` main state, ready for testing. + +``bin/kitchen verify`` +^^^^^^^^^^^^^^^^^^^^^^ + +Runs the ``inspec`` tests on the actual instance. + +``bin/kitchen destroy`` +^^^^^^^^^^^^^^^^^^^^^^^ + +Removes the docker instance. + +``bin/kitchen test`` +^^^^^^^^^^^^^^^^^^^^ + +Runs all of the stages above in one go: i.e. ``destroy`` + ``converge`` + ``verify`` + ``destroy``. + +``bin/kitchen login`` +^^^^^^^^^^^^^^^^^^^^^ + +Gives you SSH access to the instance for manual testing. + diff --git a/docs/TOFS_pattern.rst b/docs/TOFS_pattern.rst new file mode 100644 index 0000000..4fea5dd --- /dev/null +++ b/docs/TOFS_pattern.rst @@ -0,0 +1,518 @@ +.. _tofs_pattern: + +TOFS: A pattern for using SaltStack +=================================== + +.. list-table:: + :name: tofs-authors + :header-rows: 1 + :stub-columns: 1 + :widths: 2,2,3,2 + + * - + - Person + - Contact + - Date + * - Authored by + - Roberto Moreda + - moreda@allenta.com + - 29/12/2014 + * - Modified by + - Daniel Dehennin + - daniel.dehennin@baby-gnu.org + - 07/02/2019 + * - Modified by + - Imran Iqbal + - https://github.com/myii + - 23/02/2019 + +All that follows is a proposal based on my experience with `SaltStack `_. The good thing of a piece of software like this is that you can "bend it" to suit your needs in many possible ways, and this is one of them. All the recommendations and thoughts are given "as it is" with no warranty of any type. + +.. contents:: **Table of Contents** + +Usage of values in pillar vs templates in ``file_roots`` +-------------------------------------------------------- + +Among other functions, the *master* (or *salt-master*) serves files to the *minions* (or *salt-minions*). The `file_roots `_ is the list of directories used in sequence to find a file when a minion requires it: the first match is served to the minion. Those files could be `state files `_ or configuration templates, among others. + +Using SaltStack is a simple and effective way to implement configuration management, but even in a `non-multitenant `_ scenario, it is not a good idea to generally access some data (e.g. the database password in our `Zabbix `_ server configuration file or the private key of our `Nginx `_ TLS certificate). + +To avoid this situation we can use the `pillar mechanism `_, which is designed to provide controlled access to data from the minions based on some selection rules. As pillar data could be easily integrated in the `Jinja `_ templates, it is a good mechanism to store values to be used in the final rendering of state files and templates. + +There are a variety of approaches on the usage of pillar and templates as seen in the `saltstack-formulas `_' repositories. `Some `_ `developments `_ stress the initial purpose of pillar data into a storage for most of the possible variables for a determined system configuration. This, in my opinion, is shifting too much load from the original template files approach. Adding up some `non-trivial Jinja `_ code as essential part of composing the state file definitely makes SaltStack state files (hence formulas) more difficult to read. The extreme of this approach is that we could end up with a new render mechanism, implemented in Jinja, storing everything needed in pillar data to compose configurations. Additionally, we are establishing a strong dependency with the Jinja renderer. + +In opposition to the *put the code in file_roots and the data in pillars* approach, there is the *pillar as a store for a set of key-values* approach. A full-blown configuration file abstracted in pillar and jinja is complicated to develop, understand and maintain. I think a better and simpler approach is to keep a configuration file templated using just a basic (non-extensive but extensible) set of pillar values. + +On the reusability of SaltStack state files +------------------------------------------- + +There is a brilliant initiative of the SaltStack community called `salt-formulas `_. Their goal is to provide state files, pillar examples and configuration templates ready to be used for provisioning. I am a contributor for two small ones: `zabbix-formula `_ and `varnish-formula `_. + +The `design guidelines `_ for formulas are clear in many aspects and it is a recommended reading for anyone willing to write state files, even non-formulaic ones. + +In the next section, I am going to describe my proposal to extend further the reusability of formulas, suggesting some patterns of usage. + +The Template Override and Files Switch (TOFS) pattern +----------------------------------------------------- + +I understand a formula as a **complete, independent set of SaltStack state and configuration template files sufficient to configure a system**. A system could be something as simple as an NTP server or some other much more complex service that requires many state and configuration template files. + +The customization of a formula should be done mainly by providing pillar data used later to render either the state or the configuration template files. + +Example: NTP before applying TOFS +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Let's work with the NTP example. A basic formula that follows the `design guidelines `_ has the following files and directories tree: + +.. code-block:: + + /srv/saltstack/salt-formulas/ntp-saltstack-formula/ + ntp/ + map.jinja + init.sls + conf.sls + files/ + default/ + etc/ + ntp.conf.jinja + +In order to use it, let's assume a `masterless configuration `_ and this relevant section of ``/etc/salt/minion``: + +.. code-block:: yaml + + pillar_roots: + base: + - /srv/saltstack/pillar + file_client: local + file_roots: + base: + - /srv/saltstack/salt + - /srv/saltstack/salt-formulas/ntp-saltstack-formula + +.. code-block:: jinja + + {#- /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/map.jinja #} + {%- set ntp = salt['grains.filter_by']({ + 'default': { + 'pkg': 'ntp', + 'service': 'ntp', + 'config': '/etc/ntp.conf', + }, + }, merge=salt['pillar.get']('ntp:lookup')) %} + +In ``init.sls`` we have the minimal states required to have NTP configured. In many cases ``init.sls`` is almost equivalent to an ``apt-get install`` or a ``yum install`` of the package. + +.. code-block:: sls + + ## /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/init.sls + {%- from 'ntp/map.jinja' import ntp with context %} + + Install NTP: + pkg.installed: + - name: {{ ntp.pkg }} + + Enable and start NTP: + service.running: + - name: {{ ntp.service }} + - enabled: True + - require: + - pkg: Install NTP package + +In ``conf.sls`` we have the configuration states. In most cases, that is just managing configuration file templates and making them to be watched by the service. + +.. code-block:: sls + + ## /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/conf.sls + include: + - ntp + + {%- from 'ntp/map.jinja' import ntp with context %} + + Configure NTP: + file.managed: + - name: {{ ntp.config }} + - template: jinja + - source: salt://ntp/files/default/etc/ntp.conf.jinja + - watch_in: + - service: Enable and start NTP service + - require: + - pkg: Install NTP package + +Under ``files/default``, there is a structure that mimics the one in the minion in order to avoid clashes and confusion on where to put the needed templates. There you can find a mostly standard template for the configuration file. + +.. code-block:: jinja + + {#- /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/files/default/etc/ntp.conf.jinja #} + {#- Managed by saltstack #} + {#- Edit pillars or override this template in saltstack if you need customization #} + {%- set settings = salt['pillar.get']('ntp', {}) %} + {%- set default_servers = ['0.ubuntu.pool.ntp.org', + '1.ubuntu.pool.ntp.org', + '2.ubuntu.pool.ntp.org', + '3.ubuntu.pool.ntp.org'] %} + + driftfile /var/lib/ntp/ntp.drift + statistics loopstats peerstats clockstats + filegen loopstats file loopstats type day enable + filegen peerstats file peerstats type day enable + filegen clockstats file clockstats type day enable + + {%- for server in settings.get('servers', default_servers) %} + server {{ server }} + {%- endfor %} + + restrict -4 default kod notrap nomodify nopeer noquery + restrict -6 default kod notrap nomodify nopeer noquery + + restrict 127.0.0.1 + restrict ::1 + +With all this, it is easy to install and configure a simple NTP server by just running ``salt-call state.sls ntp.conf``: the package will be installed, the service will be running and the configuration should be correct for most of cases, even without pillar data. + +Alternatively, you can define a highstate in ``/srv/saltstack/salt/top.sls`` and run ``salt-call state.highstate``. + +.. code-block:: sls + + ## /srv/saltstack/salt/top.sls + base: + '*': + - ntp.conf + +**Customizing the formula just with pillar data**, we have the option to define the NTP servers. + +.. code-block:: sls + + ## /srv/saltstack/pillar/top.sls + base: + '*': + - ntp + +.. code-block:: sls + + ## /srv/saltstack/pillar/ntp.sls + ntp: + servers: + - 0.ch.pool.ntp.org + - 1.ch.pool.ntp.org + - 2.ch.pool.ntp.org + - 3.ch.pool.ntp.org + +Template Override +^^^^^^^^^^^^^^^^^ + +If the customization based on pillar data is not enough, we can override the template by creating a new one in ``/srv/saltstack/salt/ntp/files/default/etc/ntp.conf.jinja`` + +.. code-block:: jinja + + {#- /srv/saltstack/salt/ntp/files/default/etc/ntp.conf.jinja #} + {#- Managed by saltstack #} + {#- Edit pillars or override this template in saltstack if you need customization #} + + {#- Some bizarre configurations here #} + {#- ... #} + + {%- for server in settings.get('servers', default_servers) %} + server {{ server }} + {%- endfor %} + +This way we are locally **overriding the template files** offered by the formula in order to make a more complex adaptation. Of course, this could be applied as well to any of the files, including the state files. + +Files Switch +^^^^^^^^^^^^ + +To bring some order into the set of template files included in a formula, as we commented, we suggest having a similar structure to a normal final file system under ``files/default``. + +We can make different templates coexist for different minions, classified by any `grain `_ value, by simply creating new directories under ``files``. This mechanism is based on **using values of some grains as a switch for the directories under** ``files/``. + +If we decide that we want ``os_family`` as switch, then we could provide the formula template variants for both the ``RedHat`` and ``Debian`` families. + +.. code-block:: + + /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/files/ + default/ + etc/ + ntp.conf.jinja + RedHat/ + etc/ + ntp.conf.jinja + Debian/ + etc/ + ntp.conf.jinja + +To make this work we need a ``conf.sls`` state file that takes a list of possible files as the configuration template. + +.. code-block:: sls + + ## /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/conf.sls + include: + - ntp + + {%- from 'ntp/map.jinja' import ntp with context %} + + Configure NTP: + file.managed: + - name: {{ ntp.config }} + - template: jinja + - source: + - salt://ntp/files/{{ grains.get('os_family', 'default') }}/etc/ntp.conf.jinja + - salt://ntp/files/default/etc/ntp.conf.jinja + - watch_in: + - service: Enable and start NTP service + - require: + - pkg: Install NTP package + +If we want to cover the possibility of a special template for a minion identified by ``node01`` then we could have a specific template in ``/srv/saltstack/salt/ntp/files/node01/etc/ntp.conf.jinja``. + +.. code-block:: jinja + + {#- /srv/saltstack/salt/ntp/files/node01/etc/ntp.conf.jinja #} + {#- Managed by saltstack #} + {#- Edit pillars or override this template in saltstack if you need customization #} + + {#- Some crazy configurations here for node01 #} + {#- ... #} + +To make this work we could write a specially crafted ``conf.sls``. + +.. code-block:: sls + + ## /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/conf.sls + include: + - ntp + + {%- from 'ntp/map.jinja' import ntp with context %} + + Configure NTP: + file.managed: + - name: {{ ntp.config }} + - template: jinja + - source: + - salt://ntp/files/{{ grains.get('id') }}/etc/ntp.conf.jinja + - salt://ntp/files/{{ grains.get('os_family') }}/etc/ntp.conf.jinja + - salt://ntp/files/default/etc/ntp.conf.jinja + - watch_in: + - service: Enable and start NTP service + - require: + - pkg: Install NTP package + +Using the ``files_switch`` macro +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We can simplify the ``conf.sls`` with the new ``files_switch`` macro to use in the ``source`` parameter for the ``file.managed`` state. + +.. code-block:: sls + + ## /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/conf.sls + include: + - ntp + + {%- set tplroot = tpldir.split('/')[0] %} + {%- from 'ntp/map.jinja' import ntp with context %} + {%- from 'ntp/libtofs.jinja' import files_switch %} + + Configure NTP: + file.managed: + - name: {{ ntp.config }} + - template: jinja + - source: {{ files_switch(['/etc/ntp.conf.jinja'], + lookup='Configure NTP' + ) + }} + - watch_in: + - service: Enable and start NTP service + - require: + - pkg: Install NTP package + + +* This uses ``config.get``, searching for ``ntp:tofs:source_files:Configure NTP`` to determine the list of template files to use. +* If this returns a result, the default of ``['/etc/ntp.conf.jinja']`` will be appended to it. +* If this does not yield any results, the default of ``['/etc/ntp.conf.jinja']`` will be used. + +In ``libtofs.jinja``, we define this new macro ``files_switch``. + +.. literalinclude:: ../template/libtofs.jinja + :caption: /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/libtofs.jinja + :language: jinja + +How to customise the ``source`` further +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The examples below are based on an ``Ubuntu`` minion called ``theminion`` being configured via. pillar. + +Using the default settings of the ``files_switch`` macro above, +the ``source`` will be: + +.. code-block:: sls + + - source: + - salt://ntp/files/theminion/etc/ntp.conf.jinja + - salt://ntp/files/Debian/etc/ntp.conf.jinja + - salt://ntp/files/default/etc/ntp.conf.jinja + +Customise ``files`` +~~~~~~~~~~~~~~~~~~~ + +The ``files`` portion can be customised: + +.. code-block:: sls + + ntp: + tofs: + dirs: + files: files_alt + +Resulting in: + +.. code-block:: sls + + - source: + - salt://ntp/files_alt/theminion/etc/ntp.conf.jinja + - salt://ntp/files_alt/Debian/etc/ntp.conf.jinja + - salt://ntp/files_alt/default/etc/ntp.conf.jinja + +Customise the use of grains +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Grains can be customised and even arbitrary paths can be supplied: + +.. code-block:: sls + + ntp: + tofs: + files_switch: + - any/path/can/be/used/here + - id + - os + - os_family + +Resulting in: + +.. code-block:: sls + + - source: + - salt://ntp/files/any/path/can/be/used/here/etc/ntp.conf.jinja + - salt://ntp/files/theminion/etc/ntp.conf.jinja + - salt://ntp/files/Ubuntu/etc/ntp.conf.jinja + - salt://ntp/files/Debian/etc/ntp.conf.jinja + - salt://ntp/files/default/etc/ntp.conf.jinja + +Customise the ``default`` path +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``default`` portion of the path can be customised: + +.. code-block:: sls + + ntp: + tofs: + dirs: + default: default_alt + +Resulting in: + +.. code-block:: sls + + - source: + ... + - salt://ntp/files/default_alt/etc/ntp.conf.jinja + +Customise the list of ``source_files`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The list of ``source_files`` can be given: + +.. code-block:: sls + + ntp: + tofs: + source_files: + Configure NTP: + - '/etc/ntp.conf_alt.jinja' + +Resulting in: + +.. code-block:: sls + + - source: + - salt://ntp/files/theminion/etc/ntp.conf_alt.jinja + - salt://ntp/files/theminion/etc/ntp.conf.jinja + - salt://ntp/files/Debian/etc/ntp.conf_alt.jinja + - salt://ntp/files/Debian/etc/ntp.conf.jinja + - salt://ntp/files/default/etc/ntp.conf_alt.jinja + - salt://ntp/files/default/etc/ntp.conf.jinja + +Note: This does *not* override the default value. +Rather, the value from the pillar/config is prepended to the default. + +Using sub-directories for ``components`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If your formula is composed of several components, you may prefer to provides files under sub-directories, like in the `systemd-formula `_. + +.. code-block:: + + /srv/saltstack/systemd-formula/ + systemd/ + init.sls + libtofs.jinja + map.jinja + networkd/ + init.sls + files/ + default/ + network/ + 99-default.link + resolved/ + init.sls + files/ + default/ + resolved.conf + timesyncd/ + init.sls + files/ + Arch/ + resolved.conf + Debian/ + resolved.conf + default/ + resolved.conf + Ubuntu/ + resolved.conf + +For example, the following ``formula.component.config`` SLS: + +.. code-block:: sls + + {%- from "formula/libtofs.jinja" import files_switch with context %} + + formula configuration file: + file.managed: + - name: /etc/formula.conf + - user: root + - group: root + - mode: 644 + - template: jinja + - source: {{ files_switch(['formula.conf'], + lookup='formula', + use_subpath=True + ) + }} + +will be rendered on a ``Debian`` minion named ``salt-formula.ci.local`` as: + +.. code-block:: sls + + formula configuration file: + file.managed: + - name: /etc/formula.conf + - user: root + - group: root + - mode: 644 + - template: jinja + - source: + - salt://formula/component/files/salt-formula.ci.local/formula.conf + - salt://formula/component/files/Debian/formula.conf + - salt://formula/component/files/default/formula.conf + - salt://formula/files/salt-formula.ci.local/formula.conf + - salt://formula/files/Debian/formula.conf + - salt://formula/files/default/formula.conf diff --git a/kitchen.yml b/kitchen.yml index 01932e7..38b7c4a 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -117,4 +117,3 @@ suites: - path: test/integration/default inputs: version: 7 - From 1f910aff0a9971cc0391a7ef54b8c0c621b06957 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Fri, 20 Mar 2020 08:20:44 -0300 Subject: [PATCH 05/11] fix(freebsd): change mapping to use prefix Signed-off-by: Felipe Zipitria --- datadog/map.jinja | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/datadog/map.jinja b/datadog/map.jinja index 7f7fab8..7adff4d 100644 --- a/datadog/map.jinja +++ b/datadog/map.jinja @@ -1,7 +1,22 @@ {% set os_family_map = salt['grains.filter_by']({ - 'Debian': {}, - 'RedHat': {}, - 'FreeBSD': {}, + 'Debian': { + 'config': { + 'prefix': '', + 'root_group': 'root', + }, + }, + 'RedHat': { + 'config': { + 'prefix': '', + 'root_group': 'root', + }, + }, + 'FreeBSD': { + 'config': { + 'prefix': '/usr/local', + 'root_group': 'wheel', + }, + }, }, grain="os_family") %} @@ -35,11 +50,9 @@ {# Determine defaults depending on specified version #} {%- if latest_agent_version or parsed_version[1] != '5' %} - {% do datadog_install_settings.update({'config_folder': '/etc/datadog-agent'}) %} -{%- elif grains['os_family'].lower() == 'freebsd' %} - {% do datadog_install_settings.update({'config_folder': '/usr/local/etc/datadog'}) %} + {% do datadog_install_settings.update({'config_folder': datadog_config.prefix ~ '/etc/datadog-agent'}) %} {%- else %} - {% do datadog_install_settings.update({'config_folder': '/etc/dd-agent'}) %} + {% do datadog_install_settings.update({'config_folder': datadog_config.prefix ~ '/etc/dd-agent'}) %} {%- endif %} {%- if latest_agent_version or parsed_version[1] != '5' %} @@ -52,10 +65,8 @@ {% do datadog_install_settings.update({'confd_path': datadog_config.confd_path }) %} {%- else %} {%- if latest_agent_version or parsed_version[1] != '5' %} - {% do datadog_install_settings.update({'confd_path': '/etc/datadog-agent/conf.d'}) %} - {%- elif grains['os_family'].lower() == 'freebsd' %} - {% do datadog_install_settings.update({'confd_path': '/usr/local/etc/datadog/conf.d'}) %} + {% do datadog_install_settings.update({'confd_path': datadog_config.prefix ~ '/etc/datadog-agent/conf.d'}) %} {%- else %} - {% do datadog_install_settings.update({'confd_path': '/etc/dd-agent/conf.d'}) %} + {% do datadog_install_settings.update({'confd_path': datadog_config.prefix ~ '/etc/dd-agent/conf.d'}) %} {%- endif %} {%- endif %} From 75b6f03c630b52c64bc654c2d2078970233d722f Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Fri, 20 Mar 2020 08:23:34 -0300 Subject: [PATCH 06/11] fix(config): use map for file group Signed-off-by: Felipe Zipitria --- datadog/config.sls | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/datadog/config.sls b/datadog/config.sls index 3bc7f30..6210dea 100644 --- a/datadog/config.sls +++ b/datadog/config.sls @@ -8,7 +8,7 @@ datadog_conf_installed: - source: salt://datadog/files/datadog.conf.jinja - user: dd-agent - group: dd-agent - - mode: 600 + - mode: '0600' - template: jinja - require: - pkg: datadog-pkg @@ -19,7 +19,7 @@ datadog_yaml_installed: - source: salt://datadog/files/datadog.yaml.jinja - user: dd-agent - group: dd-agent - - mode: 600 + - mode: '0600' - template: jinja - require: - pkg: datadog-pkg @@ -34,8 +34,8 @@ datadog_{{ check_name }}_folder_installed: file.directory: - name: {{ datadog_install_settings.confd_path }}/{{ check_name }}.d - user: dd-agent - - group: root - - mode: 700 + - group: {{ datadog_config.root_group }} + - mode: '0700' # Remove the old config file (if it exists) datadog_{{ check_name }}_old_yaml_removed: @@ -52,8 +52,8 @@ datadog_{{ check_name }}_yaml_installed: {%- endif %} - source: salt://datadog/files/conf.yaml.jinja - user: dd-agent - - group: root - - mode: 600 + - group: {{ datadog_config.root_group }} + - mode: '0600' - template: jinja - context: check_name: {{ check_name }} From f219b9cc2f0436de683936dfeb3675d342f4612e Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Fri, 20 Mar 2020 08:24:12 -0300 Subject: [PATCH 07/11] docs(authors): add authors file Signed-off-by: Felipe Zipitria --- AUTHORS.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 AUTHORS.md diff --git a/AUTHORS.md b/AUTHORS.md new file mode 100644 index 0000000..b3c405f --- /dev/null +++ b/AUTHORS.md @@ -0,0 +1,28 @@ +# Authors + +This list is sorted by the number of commits per contributor in _descending_ order. + +Avatar|Contributor|Contributions +:-:|---|:-: +@KSerrania|[@KSerrania](https://github.com/KSerrania)|25 +@masci|[@masci](https://github.com/masci)|18 +@alq666|[@alq666](https://github.com/alq666)|11 +@onlyanegg|[@onlyanegg](https://github.com/onlyanegg)|4 +@fzipi|[@fzipi](https://github.com/fzipi)|4 +@albertvaka|[@albertvaka](https://github.com/albertvaka)|3 +@remh|[@remh](https://github.com/remh)|3 +@MSeven|[@MSeven](https://github.com/MSeven)|2 +@talwai|[@talwai](https://github.com/talwai)|1 +@norrland|[@norrland](https://github.com/norrland)|1 +@cedwards|[@cedwards](https://github.com/cedwards)|1 +@davedash|[@davedash](https://github.com/davedash)|1 +@irabinovitch|[@irabinovitch](https://github.com/irabinovitch)|1 +@nicomfer|[@nicomfer](https://github.com/nicomfer)|1 +@olivielpeau|[@olivielpeau](https://github.com/olivielpeau)|1 +@remicalixte|[@remicalixte](https://github.com/remicalixte)|1 +@blacksmith77|[@blacksmith77](https://github.com/blacksmith77)|1 +@sc250024|[@sc250024](https://github.com/sc250024)|1 + +--- + +Auto-generated by a [forked version](https://github.com/myii/maintainer) of [gaocegege/maintainer](https://github.com/gaocegege/maintainer) on 2020-03-19. From bb1553dec1701813d052162ec0fe861b2a41267f Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Fri, 20 Mar 2020 09:44:59 -0300 Subject: [PATCH 08/11] fix(freebsd): port revision uses underscore Signed-off-by: Felipe Zipitria --- datadog/install.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datadog/install.sls b/datadog/install.sls index c982075..d48df8b 100644 --- a/datadog/install.sls +++ b/datadog/install.sls @@ -73,7 +73,7 @@ datadog-pkg: {%- elif grains['os_family'].lower() == 'redhat' %} - version: {{ datadog_install_settings.agent_version }}-1 {%- elif grains['os_family'].lower() == 'freebsd' %} - - version: {{ datadog_install_settings.agent_version }}-1 + - version: {{ datadog_install_settings.agent_version }}_1 {%- endif %} - ignore_epoch: True - refresh: True From 0db5c97517ffcf8730953f1bbd615565ffc75557 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Sat, 21 Mar 2020 09:11:08 -0300 Subject: [PATCH 09/11] fix(freebsd): use pkg.latest when no version specified Signed-off-by: Felipe Zipitria --- datadog/install.sls | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/datadog/install.sls b/datadog/install.sls index d48df8b..5a68681 100644 --- a/datadog/install.sls +++ b/datadog/install.sls @@ -64,17 +64,20 @@ datadog-repo: {%- endif %} datadog-pkg: + {%- if latest_agent_version %} + pkg.latest: + - name: datadog-agent + {%- else %} pkg.installed: - name: datadog-agent - {%- if latest_agent_version %} - - version: 'latest' - {%- elif grains['os_family'].lower() == 'debian' %} + {%- if grains['os_family'].lower() == 'debian' %} - version: 1:{{ datadog_install_settings.agent_version }}-1 {%- elif grains['os_family'].lower() == 'redhat' %} - version: {{ datadog_install_settings.agent_version }}-1 {%- elif grains['os_family'].lower() == 'freebsd' %} - - version: {{ datadog_install_settings.agent_version }}_1 + - version: {{ datadog_install_settings.agent_version }} {%- endif %} + {%- endif %} - ignore_epoch: True - refresh: True {%- if grains['os_family'].lower() != 'freebsd' %} From bc647fd32ab4e61cb09167b362a395dc9d00847e Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Fri, 27 Mar 2020 10:15:32 -0300 Subject: [PATCH 10/11] fix(freebsd): there is no service dependency, so we need to start all daemons Signed-off-by: Felipe Zipitria --- datadog/map.jinja | 5 +++++ datadog/service.sls | 7 +++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/datadog/map.jinja b/datadog/map.jinja index 7adff4d..5c4d6f9 100644 --- a/datadog/map.jinja +++ b/datadog/map.jinja @@ -16,6 +16,9 @@ 'prefix': '/usr/local', 'root_group': 'wheel', }, + 'install_settings': { + 'services': ['datadog-agent-trace', 'datadog-agent-process'] + }, }, }, grain="os_family") @@ -25,8 +28,10 @@ 'datadog': { 'config': {}, 'checks': {}, + 'services': [], 'install_settings': { 'agent_version': 'latest', + 'services': ['datadog-agent'] }, } } %} diff --git a/datadog/service.sls b/datadog/service.sls index 5a23b47..ac2c0a3 100644 --- a/datadog/service.sls +++ b/datadog/service.sls @@ -1,9 +1,11 @@ {% from "datadog/map.jinja" import datadog_install_settings, datadog_checks with context %} {% set config_file_path = '%s/%s'|format(datadog_install_settings.config_folder, datadog_install_settings.config_file) -%} -datadog-agent-service: +{# freebsd has no notion of dependent services as upstart of systemd #} +{% for service in datadog_install_settings.services %} +{{ service }}-service: service.running: - - name: datadog-agent + - name: {{ service }} - enable: True - watch: - pkg: datadog-agent @@ -11,3 +13,4 @@ datadog-agent-service: {%- if datadog_checks | length %} - file: {{ datadog_install_settings.confd_path }}/* {% endif %} +{% endfor %} From 2201dea53bfa918c37425422ae82edf98c5b76c9 Mon Sep 17 00:00:00 2001 From: Marek Knappe Date: Wed, 22 Apr 2020 20:47:22 +1000 Subject: [PATCH 11/11] Added makedirs to config dirs in case someone put dirname that doesn't have parent dir yet --- datadog/config.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/datadog/config.sls b/datadog/config.sls index 6210dea..eb7f8b2 100644 --- a/datadog/config.sls +++ b/datadog/config.sls @@ -36,6 +36,7 @@ datadog_{{ check_name }}_folder_installed: - user: dd-agent - group: {{ datadog_config.root_group }} - mode: '0700' + - makedirs: True # Remove the old config file (if it exists) datadog_{{ check_name }}_old_yaml_removed: