MM-61359: sql query to channel

tests/utils/db/conftest.py

@@ -1,3 +1,4 @@
+import pandas as pd
 import pytest
 from sqlalchemy import VARCHAR, Column, DateTime, Integer, MetaData, Table
 
@@ -44,3 +45,12 @@ def delta_table_2():
         Column('extra', VARCHAR(255)),
         Column('column_b', Integer()),
     )
+
+
+@pytest.fixture
+def test_data(sqlalchemy_memory_engine):
+    with sqlalchemy_memory_engine.connect() as conn, conn.begin():
+        df = pd.DataFrame({'id': [1, 2], 'title': ['The Great Gatsby', 'The Lord of the Rings']})
+        df.to_sql('books', conn)
+
+        return df

tests/utils/db/test_helpers.py

@@ -2,13 +2,15 @@
 from datetime import datetime, timezone
 from textwrap import dedent
 
+import pandas as pd
 import pytest
 from mock.mock import call
 
 from utils.db.helpers import (
     TableStats,
     clone_table,
     get_table_stats_for_schema,
+    load_query,
     load_table_definition,
     merge_event_delta_table_into,
     move_table,
@@ -363,3 +365,9 @@ def test_should_add_new_columns_and_merge_table(mocker, base_table, delta_table_
 
     # THEN: expect the merge statement to have the correct parameters
     assert mock_exec.call_args[0][0].compile().params == {"first_duplicate_date": "2022-12-21T23:55:59"}
+
+
+def test_load_query(sqlalchemy_memory_engine, test_data):
+    with sqlalchemy_memory_engine.connect() as conn, conn.begin():
+        df = load_query(conn, "SELECT id, title FROM books")
+        pd.testing.assert_frame_equal(df, test_data)

tests/utils/db/test_service.py

@@ -0,0 +1,42 @@
+import json
+from textwrap import dedent
+
+from responses import Response
+
+from utils.db.service import post_query_results
+
+
+def test_post_query_results(sqlalchemy_memory_engine, test_data, responses):
+    # GIVEN: a mattermost server waiting for a request
+
+    response = Response(
+        method="POST",
+        url='https://mattermost.a-test-server.com',
+        status=200,
+        content_type='application/json',
+    )
+    responses.add(response)
+
+    # GIVEN: a database with some data
+    with sqlalchemy_memory_engine.connect() as conn, conn.begin():
+        # WHEN: request to post query results to Mattermost
+        post_query_results(
+            conn,
+            'select id, title from books order by id',
+            ['ISBN', 'Title'],
+            'https://mattermost.a-test-server.com',
+            'book-club',
+        )
+
+    # THEN: a request is made to the Mattermost server
+    assert json.loads(responses.calls[0].request.body) == {
+        'text': dedent(
+            '''
+            |   ISBN | Title                 |
+            |--------|-----------------------|
+            |      1 | The Great Gatsby      |
+            |      2 | The Lord of the Rings |
+        '''
+        ).strip(),
+        'channel': 'book-club',
+    }

tests/utils/test_helpers.py

@@ -1,6 +1,11 @@
+import json
 from datetime import date
+from textwrap import dedent
 
-from utils.helpers import daterange
+import pandas as pd
+from responses import Response
+
+from utils.helpers import daterange, post_to_mattermost
 
 
 def test_include_start_exclude_end():
@@ -21,3 +26,37 @@ def test_start_date_end_date_match():
 
 def test_end_date_next_day_of_start_date():
     assert list(daterange(date(2024, 2, 18), date(2024, 2, 19))) == [date(2024, 2, 18)]
+
+
+def test_post_to_mattermost(responses):
+    # GIVEN: A Mattermost URL, and channel that returns HTTP OK
+    url = 'https://mattermost.a-test-server.com/path/to/hookid'
+    channel = 'test-channel'
+
+    response = Response(
+        method="POST",
+        url='https://mattermost.a-test-server.com/path/to/hookid',
+        status=200,
+        content_type='application/json',
+    )
+    responses.add(response)
+
+    # GIVEN: Some data to post
+    df = pd.DataFrame({'id': [1, 2], 'title': ['The Great Gatsby', 'The Lord of the Rings']})
+
+    # WHEN: request is made to Mattermost
+    post_to_mattermost(url, channel, df, headers=['ISBN', 'Book Title'])
+
+    # THEN: A request is made to the Mattermost server
+    assert len(responses.calls) == 1
+    assert json.loads(responses.calls[0].request.body) == {
+        'text': dedent(
+            '''
+            |   ISBN | Book Title            |
+            |--------|-----------------------|
+            |      1 | The Great Gatsby      |
+            |      2 | The Lord of the Rings |
+        '''
+        ).strip(),
+        'channel': 'test-channel',
+    }

utils/db/__main__.py

@@ -5,6 +5,7 @@
 from utils.cli.context import snowflake_engine_context
 from utils.cli.logging import initialize_cli_logging
 from utils.db.helpers import clone_table, merge_event_delta_table_into
+from utils.db.service import post_query_results
 
 initialize_cli_logging(logging.INFO, 'stderr')
 
@@ -111,5 +112,38 @@ def merge(
         )
 
 
+@snowflake.group()
+def post_query():
+    pass
+
+
+@post_query.command()
+@click.pass_context
+@click.argument('url', type=str)
+@click.argument('channel', type=str)
+def feedback(ctx: click.Context, url: str, channel: str):
+    with ctx.obj['engine'].begin() as conn, conn.begin():
+        post_query_results(
+            conn,
+            '''
+                SELECT
+                    to_date(timestamp) as date
+                    , COALESCE(feedback, 'No feedback') as feedback
+                    , rating
+                    , page_path
+                    , ua_browser_family
+                    , ua_os_family
+                    , geolocated_country_name
+                FROM "REPORTS_DOCS".rpt_docs_feedback
+                WHERE
+                    timestamp::date >= current_date() - 7 AND timestamp::date <= current_date() - 1
+                ORDER BY timestamp DESC
+            ''',
+            ['Date', 'Feedback', 'Rating', 'Path', 'Browser', 'OS', 'Country'],
+            url,
+            channel,
+        )
+
+
 if __name__ == '__main__':
     snowflake()

utils/db/helpers.py

@@ -319,3 +319,10 @@ def merge_event_delta_table_into(
     # Workaround for https://github.com/snowflakedb/snowflake-sqlalchemy/issues/536
     stmt = text(merge.__repr__()).bindparams(first_duplicate_date=first_duplicate_date)
     conn.execute(stmt)
+
+
+def load_query(conn: Connection, query: str) -> pd.DataFrame:
+    """
+    Loads a query into a DataFrame.
+    """
+    return pd.read_sql(query, conn)

utils/db/service.py

@@ -0,0 +1,25 @@
+import logging
+from typing import List
+
+from sqlalchemy.engine import Connection
+
+from utils.db.helpers import load_query
+from utils.helpers import post_to_mattermost
+
+logger = logging.getLogger(__name__)
+
+
+def post_query_results(
+    conn: Connection,
+    query: str,
+    headers: List[str],
+    url: str,
+    channel: str,
+):
+    """
+    Post query results to Mattermost.
+    """
+    logger.info('Querying data...')
+    df = load_query(conn, query)
+    logger.info('Sending dataframe to mattermost...')
+    post_to_mattermost(url, channel, df, headers)

utils/helpers.py

@@ -1,6 +1,22 @@
 from datetime import timedelta
+from typing import List
+
+import pandas as pd
+import requests
+from tabulate import tabulate
 
 
 def daterange(start_date, end_date):
     for n in range(int((end_date - start_date).days)):
         yield start_date + timedelta(n)
+
+
+def post_to_mattermost(url: str, channel: str, df: pd.DataFrame, headers: List[str]):
+    msg = tabulate(df, headers=headers, tablefmt='github', showindex='never')
+    response = requests.post(
+        url,
+        json={"text": msg, "channel": channel},
+    )
+
+    if response.status_code != 200:
+        raise ValueError(f'Request to Mattermost returned {response.status_code}, the response is:\n{response.text}')