diff --git a/.travis.yml b/.travis.yml index ec14974..6258f28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ python: # - "3.5" - "3.6" - "3.7" + - "3.8" install: - make init diff --git a/.vscode/settings.json b/.vscode/settings.json index 7ef6e03..9cfbbbd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,27 +2,44 @@ "python.linting.flake8Enabled": true, "python.linting.enabled": true, "cSpell.words": [ + "CACERTDIR", + "CACERTFILE", "PYTHONPATH", "Postgres", + "TABLENAMES", "apscheduler", "apscheduler's", + "auditlogs", "bcrypt", + "bdist", "checkpw", "corescheduler", + "dateutil", "dockerized", + "funcsigs", "hashpw", "htpasswd", "ioloop", + "jobauditlog", "keyout", + "ldaps", "mrkdwn", "ndscheduler", "newkey", "palto", + "pypi", "rtype", "sched", + "sdist", "selfsigned", "sendgrid", + "sqlite", + "sslmode", + "tablename", "urlencode", - "venv" - ] + "venv", + "webcron", + "wheel" + ], + "python.pythonPath": ".venv/bin/python" } \ No newline at end of file diff --git a/Makefile b/Makefile index 7067408..ecfebab 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ test: make install make flake8 # Hacky way to ensure mock is installed before running setup.py - $(SOURCE_VENV) && pip install mock==1.1.2 && $(PYTHON) setup.py test + $(SOURCE_VENV) && pip install -r test_requirements.txt && $(PYTHON) setup.py test install: make init diff --git a/README.md b/README.md index 40a5771..1a9010d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Nextdoor Scheduler [![License](https://img.shields.io/badge/License-BSD%202--Clause-orange.svg)](LICENSE.txt) -[![Build Status](https://api.travis-ci.org/palto42/ndscheduler.svg)](https://travis-ci.org/palto42/ndscheduler) +[![Build Status](https://api.travis-ci.com/palto42/ndscheduler.svg)](https://travis-ci.com/palto42/ndscheduler) ``ndscheduler`` is a flexible python library for building your own cron-like system to schedule jobs, which is to run a tornado process to serve REST APIs and a web ui. @@ -42,10 +42,11 @@ Note: ``corescheduler`` can also be used independently within your own service i * pip install . * Install scheduler implementation like [simple_scheduler](https://github.com/palto42/simple_scheduler) 3. Configure ~/.config/ndscheduler/config.yaml - * See [example configuration](config_example.yaml) - * Passwords must be hashed with bcrypt - * See [Python bcrypt tutorial](http://zetcode.com/python/bcrypt/) - * More ideas for basic_auth [Tornado basic auth example](https://gist.github.com/notsobad/5771635) + * See [example configuration](config_example.yaml) and [default configuration](ndscheduler/config_default.yaml) for available options. + * Optionally, enable authentication + * For local authentication, configure users and passwords as in the [example configuration](config_example.yaml). Passwords must be hashed with bcrypt, which can be done with the command `python -m ndscheduler --encrypt` + * For LDAP authentication, configure the LDAP server settings and the list of allowed users. + * If LDAP authentication should be used, the Python package `python-ldap`must be installed. 4. Start scheduler implementation 5. Launch web browser at configured URL and authenticate with configured account diff --git a/config_example.yaml b/config_example.yaml index ac1ffd0..21ece17 100644 --- a/config_example.yaml +++ b/config_example.yaml @@ -18,7 +18,7 @@ SSL_KEY: /path/to/key # required for HTTPS # "jobs_tablename": confuse.String(default="scheduler_jobs"), # "executions_tablename": confuse.String(default="scheduler_execution"), # "auditlogs_tablename": confuse.String(default="scheduler_jobauditlog"), -# DATABASE_CLASS: sqlite # supporte: sqlite, postgres, mysql +# DATABASE_CLASS: sqlite # supported: sqlite, postgres, mysql DATABASE_CONFIG_DICT: "file_path": datastore.db # SQlite # additional attributes for MySQL and Postgres diff --git a/ndscheduler/__init__.py b/ndscheduler/__init__.py index f00a7f9..38d8e96 100644 --- a/ndscheduler/__init__.py +++ b/ndscheduler/__init__.py @@ -21,6 +21,7 @@ import bcrypt from getpass import getpass from time import sleep +import pkg_resources from ndscheduler import default_settings @@ -85,6 +86,13 @@ def get_cli_args(): parser.add_argument( "--encrypt", "-e", help="Create hash value from password for use in AUTH_CREDENTIALS.", action="store_true", ) + parser.add_argument( + "--version", + "-V", + action="version", + help="Show version", + version=f"%(prog)s fla{pkg_resources.get_distribution('construct').version}", + ) args, _ = parser.parse_known_args() @@ -186,6 +194,16 @@ def load_yaml_config( "MAIL_SERVER": confuse.StrSeq(), "ADMIN_MAIL": confuse.StrSeq(), "SERVER_MAIL": confuse.String(default=""), + # LDAP server addess in the format "ldap://my.ldap.server" "ldaps://my.ldap.server" + # Non-standard ports can be specified like "ldap://my.ldap.server:1234" + "LDAP_SERVER": confuse.String(default=""), + "LDAP_REQUIRE_CERT": confuse.Choice(["demand", "allow", "never"], default="demand",), + "LDAP_CERT_DIR": confuse.String(default=None), + "LDAP_CERT_FILE": confuse.String(default=None), + # Define LDAP dn format for login, {username} will be replaced with the entered user name + "LDAP_LOGIN_DN": confuse.String(default="uid={username},ou=people,o=MyCompany,dc=net"), + # List of permitted LDAP users. If none are specified, any authenticated used is allowed + "LDAP_USERS": confuse.StrSeq(default=[]), } yaml_template.update(yaml_extras) diff --git a/ndscheduler/config_default.yaml b/ndscheduler/config_default.yaml index 0ffdd77..aa9a3b3 100644 --- a/ndscheduler/config_default.yaml +++ b/ndscheduler/config_default.yaml @@ -85,3 +85,14 @@ MAIL_SERVER: [] # Server sender mail address SERVER_MAIL: "" + +# LDAP server address in the format "ldap://my.ldap.server" "ldaps://my.ldap.server" +# Non-standard ports can be specified like "ldap://my.ldap.server:1234" +# LDAP_SERVER: ldaps://my.ldap.server +LDAP_REQUIRE_CERT: demand +# LDAP_CERT_DIR: None +# LDAP_CERT_FILE: None +# Define LDAP dn format for login, {username} will be replaced with the entered user name +LDAP_LOGIN_DN: uid={username},ou=users,dc=example,dc=com +# List of permitted LDAP users. If none are specified, any authenticated used is allowed +LDAP_USERS: [] diff --git a/ndscheduler/corescheduler/core/base.py b/ndscheduler/corescheduler/core/base.py index 5c56f3f..0fd86d7 100644 --- a/ndscheduler/corescheduler/core/base.py +++ b/ndscheduler/corescheduler/core/base.py @@ -52,10 +52,7 @@ def run_job( execution_id = utils.generate_uuid() datastore = utils.get_datastore_instance(db_class_path, db_config, db_tablenames) datastore.add_execution( - execution_id, - job_id, - constants.EXECUTION_STATUS_SCHEDULED, - description=JobBase.get_scheduled_description(), + execution_id, job_id, constants.EXECUTION_STATUS_SCHEDULED, description=JobBase.get_scheduled_description(), ) try: job_class = utils.import_from_path(job_class_path) diff --git a/ndscheduler/corescheduler/datastore/base_test.py b/ndscheduler/corescheduler/datastore/base_test.py index 20b1712..dc432bc 100644 --- a/ndscheduler/corescheduler/datastore/base_test.py +++ b/ndscheduler/corescheduler/datastore/base_test.py @@ -41,16 +41,10 @@ def test_get_executions_by_time_interval(self): "12", "34", state=constants.EXECUTION_STATUS_SCHEDULED, scheduled_time=now + datetime.timedelta(minutes=5), ) self.store.add_execution( - "13", - "34", - state=constants.EXECUTION_STATUS_SCHEDULED, - scheduled_time=now + datetime.timedelta(minutes=50), + "13", "34", state=constants.EXECUTION_STATUS_SCHEDULED, scheduled_time=now + datetime.timedelta(minutes=50), ) self.store.add_execution( - "14", - "34", - state=constants.EXECUTION_STATUS_SCHEDULED, - scheduled_time=now + datetime.timedelta(minutes=70), + "14", "34", state=constants.EXECUTION_STATUS_SCHEDULED, scheduled_time=now + datetime.timedelta(minutes=70), ) self.store.add_execution( "15", diff --git a/ndscheduler/default_settings.py b/ndscheduler/default_settings.py index c1163c1..c288f06 100644 --- a/ndscheduler/default_settings.py +++ b/ndscheduler/default_settings.py @@ -118,7 +118,6 @@ # "user": "$2y$11$MCw3cm9Tp.8zF/hmPILW3.1hGMtP0UV8kUevfaxrzM7JzXdoyFi6.", # Very$ecret } - # List of admin users ADMIN_USER = [] @@ -130,3 +129,16 @@ # Server sender mail address SERVER_MAIL = "" + +# LDAP server address in the format "ldap://my.ldap.server" "ldaps://my.ldap.server" +# Non-standard ports can be specified like "ldap://my.ldap.server:1234" +LDAP_SERVER = "" +# If "ldaps://" is used, specify of the SSL certificate should be verified +# Possible options are "demand", "allow" or "never" +LDAP_REQUIRE_CERT = "demand" +LDAP_CERT_DIR = None +LDAP_CERT_File = None +# Define LDAP dn format for login, {username} will be replaced with the entered user name +LDAP_LOGIN_DN = "uid={username},ou=users,dc=example,dc=com" +# List of permitted LDAP users. If none are specified, any authenticated used is allowed +LDAP_USERS = [] diff --git a/ndscheduler/server/handlers/audit_logs.py b/ndscheduler/server/handlers/audit_logs.py index 237bbde..b2fcd1e 100644 --- a/ndscheduler/server/handlers/audit_logs.py +++ b/ndscheduler/server/handlers/audit_logs.py @@ -57,8 +57,9 @@ def get(self): "event": "modified", "user": "", "created_time": "", - "description": (""), + "description": ( + "" + ), } ] } diff --git a/ndscheduler/server/handlers/base.py b/ndscheduler/server/handlers/base.py index 15c00bd..892e7f9 100644 --- a/ndscheduler/server/handlers/base.py +++ b/ndscheduler/server/handlers/base.py @@ -7,6 +7,7 @@ import logging import json import bcrypt +import ldap from concurrent import futures @@ -70,18 +71,75 @@ def get(self): def post(self): username = self.get_argument("username") + password = self.get_argument("password") hashed = self.auth_credentials.get(username) logger.debug(f"Received login for user '{username}'") - if hashed is not None and bcrypt.checkpw(self.get_argument("password").encode(), hashed.encode()): - # 6h = 0.25 days - # 1h = 0.041666667 days - # 1min = 0.000694444 days - self.set_secure_cookie(settings.COOKIE_NAME, username, expires_days=settings.COOKIE_MAX_AGE) - logger.debug(f"Set cookie for user {username}, expires: {settings.COOKIE_MAX_AGE * 1440} minutes") - self.redirect("/") + if settings.LDAP_SERVER and self.ldap_login(username, password): + self.set_user_cookie(username) + elif hashed is not None and bcrypt.checkpw(password.encode(), hashed.encode()): + logger.debug("Try local authentication") + self.set_user_cookie(username) else: logger.debug("Wrong username or password") - self.redirect("/") + self.redirect("/") + + def set_user_cookie(self, username): + # 6h = 0.25 days + # 1h = 0.041666667 days + # 1min = 0.000694444 days + self.set_secure_cookie(settings.COOKIE_NAME, username, expires_days=settings.COOKIE_MAX_AGE) + logger.debug(f"Set cookie for user {username}, expires: {settings.COOKIE_MAX_AGE * 1440} minutes") + + def ldap_login(self, username, password): + """Verifies credentials for username and password. + + Parameters + ---------- + username : str + User ID (uid) to be used for login + password : str + User password + + Returns + ------- + bool + True if login was successful + """ + if settings.LDAP_USERS and username not in settings.LDAP_USERS: + logging.warning(f"User {username} not allowed for LDAP login") + return False + LDAP_SERVER = settings.LDAP_SERVER + # Create fully qualified DN for user + LDAP_DN = settings.LDAP_LOGIN_DN.replace("{username}", username) + logger.debug(f"LDAP dn: {LDAP_DN}") + # disable certificate check + ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_ALLOW) + + # specify certificate dir or file + if settings.LDAP_CERT_DIR: + ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, settings.LDAP_CERT_DIR) + if settings.LDAP_CERT_FILE: + ldap.set_option(ldap.OPT_X_TLS_CACERTFILE, settings.LDAP_CERT_FILE) + try: + # build a client + ldap_client = ldap.initialize(LDAP_SERVER) + ldap_client.set_option(ldap.OPT_REFERRALS, 0) + # perform a synchronous bind to test authentication + ldap_client.simple_bind_s(LDAP_DN, password) + logger.info(f"User '{username}' successfully authenticated via LDAP") + ldap_client.unbind_s() + return True + except (ldap.INVALID_CREDENTIALS, ldap.NO_SUCH_OBJECT): + ldap_client.unbind() + logger.warning("LDAP: wrong username or password") + except ldap.SERVER_DOWN: + logger.warning("LDAP server not available") + except ldap.LDAPError as e: + if isinstance(e, dict) and "desc" in e: + logger.warning(f"LDAP error: {e['desc']}") + else: + logger.warning(f"LDAP error: {e}") + return False class LogoutHandler(BaseHandler): diff --git a/ndscheduler/server/handlers/jobs.py b/ndscheduler/server/handlers/jobs.py index b5bfdf2..2db55bd 100644 --- a/ndscheduler/server/handlers/jobs.py +++ b/ndscheduler/server/handlers/jobs.py @@ -329,6 +329,5 @@ def _validate_post_data(self): if not valid_cron_string: raise tornado.web.HTTPError( - 400, - reason=("Require at least one of following parameters:" " %s" % str(at_least_one_required_fields)), + 400, reason=("Require at least one of following parameters:" " %s" % str(at_least_one_required_fields)), ) diff --git a/ndscheduler/version.py b/ndscheduler/version.py index 953cf71..6426181 100644 --- a/ndscheduler/version.py +++ b/ndscheduler/version.py @@ -1 +1 @@ -__version__ = "0.6.0" # http://semver.org/ +__version__ = "0.6.1" # http://semver.org/ diff --git a/setup.py b/setup.py index eef1abc..e11d981 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ multiprocessing -PACKAGE = "ndscheduler" +PACKAGE = "ndscheduler-fork" __version__ = None exec(open(os.path.join("ndscheduler", "version.py")).read()) # set __version__ @@ -61,10 +61,10 @@ def maybe_rm(path): version=__version__, description="ndscheduler: A cron-replacement library from Nextdoor", long_description=open("README.md").read(), - author="Nextdoor Engineering", - author_email="eng@nextdoor.com", - url="https://github.com/Nextdoor/ndscheduler", - license="Apache License, Version 2", + author="Matthias Homann (original: Nextdoor Engineering)", + author_email="palto@mailbox.org", + url="https://github.com/palto42/ndscheduler", + license="BSD 2-Clause 'Simplified' License", keywords="scheduler nextdoor cron python", packages=find_packages(), include_package_data=True, @@ -79,6 +79,8 @@ def maybe_rm(path): "python-dateutil >= 2.2", "bcrypt >= 3.1.7", # for user authentication "confuse >= 1.1.0", # for yaml config support + # python-ldap is only required if LDAP authentication is used + # "python-ldap >= 3.3.1", ], classifiers=classifiers, cmdclass={"clean": CleanHook}, diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 0000000..225e31e --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,3 @@ +mock==1.1.2 +construct>=2.10 +python-ldap >= "3.3.1"