Webhook

README.md

@@ -6,9 +6,14 @@ This is a Python package that integrates Superset and CommCare HQ.
 Local Development
 -----------------
 
-Follow below instructions.
+### Preparing CommCare HQ
 
-### Setup env
+The 'User configurable reports UI' feature flag must be enabled for the
+domain in CommCare HQ, even if the data sources to be imported were
+created by Report Builder, not a UCR.
+
+
+### Setting up a dev environment
 
 While doing development on top of this integration, it's useful to
 install this via `pip -e` option so that any changes made get reflected
@@ -51,11 +56,12 @@ directly without another `pip install`.
 Read through the initialization instructions at
 https://superset.apache.org/docs/installation/installing-superset-from-scratch/#installing-and-initializing-superset.
 
-Create the database. These instructions assume that PostgreSQL is
-running on localhost, and that its user is "commcarehq". Adapt
-accordingly:
+Create a database for Superset, and a database for storing data from
+CommCare HQ. Adapt the username and database names to suit your
+environment.
 ```bash
-$ createdb -h localhost -p 5432 -U commcarehq superset_meta
+$ createdb -h localhost -p 5432 -U postgres superset
+$ createdb -h localhost -p 5432 -U postgres superset_hq_data
 ```
 
 Set the following environment variables:
@@ -64,10 +70,11 @@ $ export FLASK_APP=superset
 $ export SUPERSET_CONFIG_PATH=/path/to/superset_config.py
 ```
 
-Initialize the database. Create an administrator. Create default roles
+Initialize the databases. Create an administrator. Create default roles
 and permissions:
 ```bash
 $ superset db upgrade
+$ superset db upgrade --directory hq_superset/migrations/
 $ superset fab create-admin
 $ superset load_examples  # (Optional)
 $ superset init
@@ -78,27 +85,8 @@ You should now be able to run superset using the `superset run` command:
 ```bash
 $ superset run -p 8088 --with-threads --reload --debugger
 ```
-However, OAuth login does not work yet as hq-superset needs a Postgres
-database created to store CommCare HQ data.
-
-
-### Create a Postgres Database Connection for storing HQ data
-
-- Create a Postgres database. e.g.
-  ```bash
-  $ createdb -h localhost -p 5432 -U commcarehq hq_data
-  ```
-- Log into Superset as the admin user created in the Superset
-  installation and initialization. Note that you will need to update
-  `AUTH_TYPE = AUTH_DB` to log in as admin user. `AUTH_TYPE` should be
-  otherwise set to `AUTH_OAUTH`.
-- Go to 'Data' -> 'Databases' or http://127.0.0.1:8088/databaseview/list/
-- Create a database connection by clicking '+ DATABASE' button at the top.
-- The name of the DISPLAY NAME should be 'HQ Data' exactly, as this is
-  the name by which this codebase refers to the Postgres DB.
 
-OAuth integration should now be working. You can log in as a CommCare
-HQ web user.
+You can now log in as a CommCare HQ web user.
 
 
 ### Importing UCRs using Redis and Celery
@@ -129,6 +117,41 @@ code you want to test will need to be in a module whose dependencies
 don't include Superset.
 
 
+### Creating a migration
+
+You will need to create an Alembic migration for any new SQLAlchemy
+models that you add. The Superset CLI should allow you to do this:
+
+```shell
+$ superset db revision --autogenerate -m "Add table for Foo model"
+```
+
+However, problems with this approach have occurred in the past. You
+might have more success by using Alembic directly. You will need to
+modify the configuration a little to do this:
+
+1. Copy the "HQ_DATA" database URI from `superset_config.py`.
+
+2. Paste it as the value of `sqlalchemy.url` in
+   `hq_superset/migrations/alembic.ini`.
+
+3. Edit `env.py` and comment out the following lines:
+   ```
+   hq_data_uri = current_app.config['SQLALCHEMY_BINDS'][HQ_DATA]
+   decoded_uri = urllib.parse.unquote(hq_data_uri)
+   config.set_main_option('sqlalchemy.url', decoded_uri)
+   ```
+
+Those changes will allow Alembic to connect to the "HD Data" database
+without the need to instantiate Superset's Flask app. You can now
+autogenerate your new table with:
+
+```shell
+$ cd hq_superset/migrations/
+$ alembic revision --autogenerate -m "Add table for Foo model"
+```
+
+
 Upgrading Superset
 ------------------
 

hq_superset/__init__.py

@@ -8,10 +8,12 @@ def flask_app_mutator(app):
     # Import the views (which assumes the app is initialized) here
     # return
     from superset.extensions import appbuilder
+    from . import api, hq_domain, views
 
-    from . import hq_domain, views
     appbuilder.add_view(views.HQDatasourceView, 'Update HQ Datasource', menu_cond=lambda *_: False)
     appbuilder.add_view(views.SelectDomainView, 'Select a Domain', menu_cond=lambda *_: False)
+    appbuilder.add_api(api.OAuth)
+    appbuilder.add_api(api.DataSetChangeAPI)
     app.before_request_funcs.setdefault(None, []).append(
         hq_domain.before_request_hook
     )
@@ -40,4 +42,4 @@ def override_jinja2_template_loader(app):
         'images'
     ))
     blueprint = Blueprint('Static', __name__, static_url_path='/static/images', static_folder=images_path)
-    app.register_blueprint(blueprint)
+    app.register_blueprint(blueprint)

hq_superset/api.py

@@ -0,0 +1,126 @@
+import json
+from datetime import datetime, timedelta
+from http import HTTPStatus
+
+from authlib.integrations.flask_oauth2 import (
+    AuthorizationServer,
+    ResourceProtector,
+)
+from authlib.oauth2.rfc6749 import grants
+from authlib.oauth2.rfc6750 import BearerTokenValidator
+from flask import jsonify, request
+from flask_appbuilder.api import BaseApi, expose
+from flask_appbuilder.baseviews import expose_api
+from sqlalchemy.orm.exc import NoResultFound
+from superset import db
+from superset.extensions import appbuilder, csrf
+from superset.superset_typing import FlaskResponse
+from superset.views.base import handle_api_exception, json_error_response
+
+from .models import DataSetChange, HQClient, Token
+from .utils import update_dataset
+
+require_oauth = ResourceProtector()
+app = appbuilder.app
+
+
+def query_client(client_id):
+    return HQClient.get_by_client_id(client_id)
+
+
+def save_token(token, request):
+    client = request.client
+    client.revoke_tokens()
+
+    expires_at = datetime.utcnow() + timedelta(days=1)
+    tok = Token(
+        client_id=client.client_id,
+        expires_at=expires_at,
+        access_token=token['access_token'],
+        token_type=token['token_type'],
+        scope=client.domain,
+    )
+    db.session.add(tok)
+    db.session.commit()
+
+
+class HQBearerTokenValidator(BearerTokenValidator):
+    def authenticate_token(self, token_string):
+        return db.session.query(Token).filter_by(access_token=token_string).first()
+
+
+require_oauth.register_token_validator(HQBearerTokenValidator())
+
+authorization = AuthorizationServer(
+    app=app,
+    query_client=query_client,
+    save_token=save_token,
+)
+authorization.register_grant(grants.ClientCredentialsGrant)
+
+
+class OAuth(BaseApi):
+
+    def __init__(self):
+        super().__init__()
+        self.route_base = "/oauth"
+
+    @expose("/token", methods=('POST',))
+    def issue_access_token(self):
+        try:
+            response = authorization.create_token_response()
+        except NoResultFound:
+            return jsonify({"error": "Invalid client"}), 401
+
+        if response.status_code >= 400:
+            return response
+
+        data = json.loads(response.data.decode("utf-8"))
+        return jsonify(data)
+
+
+class DataSetChangeAPI(BaseApi):
+    """
+    Accepts changes to datasets from CommCare HQ data forwarding
+    """
+
+    MAX_REQUEST_LENGTH = 10_485_760  # reject >10MB JSON requests
+
+    def __init__(self):
+        self.route_base = '/hq_webhook'
+        self.default_view = 'post_dataset_change'
+        super().__init__()
+
+    # http://localhost:8088/hq_webhook/change/
+    @expose_api(url='/change/', methods=('POST',))
+    @handle_api_exception
+    @csrf.exempt
+    @require_oauth
+    def post_dataset_change(self) -> FlaskResponse:
+        if request.content_length > self.MAX_REQUEST_LENGTH:
+            return json_error_response(
+                HTTPStatus.REQUEST_ENTITY_TOO_LARGE.description,
+                status=HTTPStatus.REQUEST_ENTITY_TOO_LARGE.value,
+            )
+
+        try:
+            request_json = json.loads(request.get_data(as_text=True))
+            change = DataSetChange(**request_json)
+            update_dataset(change)
+            return self.json_response(
+                'Request accepted; updating dataset',
+                status=HTTPStatus.ACCEPTED.value,
+            )
+        except json.JSONDecodeError:
+            return json_error_response(
+                'Invalid JSON syntax',
+                status=HTTPStatus.BAD_REQUEST.value,
+            )
+        except (TypeError, ValueError) as err:
+            return json_error_response(
+                str(err),
+                status=HTTPStatus.BAD_REQUEST.value,
+            )
+        # `@handle_api_exception` will return other exceptions as JSON
+        # with status code 500, e.g.
+        #     {"error": "CommCare HQ database missing"}

hq_superset/const.py

@@ -0,0 +1,2 @@
+# The name of the database for storing data related to CommCare HQ
+HQ_DATA = "HQ Data"

hq_superset/hq_domain.py

@@ -36,6 +36,7 @@ def is_user_admin():
     return security_manager.is_admin()
 
 
+
 def ensure_domain_selected():
     # Check if a hq_domain cookie is set
     #   Ensure necessary roles, permissions and DB schemas are created for the domain

hq_superset/hq_requests.py

@@ -0,0 +1,48 @@
+import superset
+from hq_superset.oauth import get_valid_cchq_oauth_token
+
+
+class HqUrl:
+    @classmethod
+    def datasource_export_url(cls, domain, datasource_id):
+        return f"a/{domain}/configurable_reports/data_sources/export/{datasource_id}/?format=csv"
+
+    @classmethod
+    def datasource_list_url(cls, domain):
+        return f"a/{domain}/api/v0.5/ucr_data_source/"
+
+    @classmethod
+    def datasource_details_url(cls, domain, datasource_id):
+        return f"a/{domain}/api/v0.5/ucr_data_source/{datasource_id}/"
+
+    @classmethod
+    def subscribe_to_datasource_url(cls, domain, datasource_id):
+        return f"a/{domain}/configurable_reports/data_sources/subscribe/{datasource_id}/"
+
+
+class HQRequest:
+
+    def __init__(self, url):
+        self.url = url
+
+    @property
+    def oauth_token(self):
+        return get_valid_cchq_oauth_token()
+
+    @property
+    def commcare_provider(self):
+        return superset.appbuilder.sm.oauth_remotes["commcare"]
+
+    @property
+    def api_base_url(self):
+        return self.commcare_provider.api_base_url
+
+    @property
+    def absolute_url(self):
+        return f"{self.api_base_url}{self.url}"
+
+    def get(self):
+        return self.commcare_provider.get(self.url, token=self.oauth_token)
+
+    def post(self, data):
+        return self.commcare_provider.post(self.url, data=data, token=self.oauth_token)

hq_superset/migrations/README

@@ -0,0 +1 @@
+Generic single-database configuration.