Merge pull request #1 from datakind/feat/integration-scaffolding

Feat/integration scaffolding

.env.example

@@ -23,8 +23,31 @@ DOMAIN_SERVER=http://localhost:3080
 
 NO_INDEX=true
 
-# COonection string for where the data DB resides. Can be remote or local DB, in form postgresql://username:password@host:port/database
-DATA_DB_CONN_STRING=
+# =========== Local Docker containers START ==========
+POSTGRES_DATA_HOST=postgres-prototypes.postgres.database.azure.com
+POSTGRES_DATA_PORT=5432
+POSTGRES_DATA_DB=hdexpert-alpha
+POSTGRES_DATA_USER=
+POSTGRES_DATA_PASSWORD=
+DATA_DB_CONN_STRING=postgresql://${POSTGRES_DATA_USER}:${POSTGRES_DATA_PASSWORD}@${POSTGRES_DATA_HOST}:${POSTGRES_DATA_PORT}/${POSTGRES_DATA_DB}
+
+POSTGRES_RECIPE_HOST=recipedb
+POSTGRES_RECIPE_PORT=5432
+POSTGRES_RECIPE_DB=recipes
+POSTGRES_RECIPE_USER=
+POSTGRES_RECIPE_PASSWORD=
+RECIPE_DB_CONN_STRING=postgresql://${POSTGRES_RECIPE_USER}:${POSTGRES_RECIPE_PASSWORD}@${POSTGRES_RECIPE_HOST}:${POSTGRES_RECIPE_PORT}/${POSTGRES_RECIPE_DB}
+# =========== Local Docker containers END ==========
+
+#==================================================#
+#               Get Memory Action Configuration    #
+#==================================================#
+OPENAI_API_TYPE=azure
+OPENAI_API_ENDPOINT=
+OPENAI_API_VERSION_MEMORY=2024-02-15-preview
+BASE_URL_MEMORY=
+MODEL_MEMORY=gpt-4-turbo
+OPENAI_TEXT_COMPLETION_DEPLOYMENT_NAME=text-embedding-ada-002
 
 #===============#
 # JSON Logging  #
@@ -88,22 +111,22 @@ ANTHROPIC_API_KEY=sk-ant-api03-6b0Vg33VrXpxmVALpTlGJu90IvfrIbW5Q8wEamdnk1Es4mg6e
 # Azure      #
 #============#
 
-AZURE_API_KEY=c3e37d558bea46ca917dd2be40ee69d4
+AZURE_API_KEY=
 
 # Note: these variables are DEPRECATED
 # Use the `librechat.yaml` configuration for `azureOpenAI` instead
 # You may also continue to use them if you opt out of using the `librechat.yaml` configuration
 
-AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo # Deprecated
-AZURE_OPENAI_MODELS=gpt-35-turbo,gpt-4 
+#AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo # Deprecated
+#AZURE_OPENAI_MODELS=gpt-35-turbo,gpt-4 
 # AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE
 # AZURE_API_KEY= # Deprecated
-AZURE_OPENAI_API_INSTANCE_NAME=DK-DS-Team
-AZURE_OPENAI_API_DEPLOYMENT_NAME=gpt-4
-AZURE_OPENAI_API_VERSION=2023-07-01-preview
+#AZURE_OPENAI_API_INSTANCE_NAME=DK-DS-Team
+#AZURE_OPENAI_API_DEPLOYMENT_NAME=gpt-4
+#AZURE_OPENAI_API_VERSION=2023-07-01-preview
 # AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME= # Deprecated
 # AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME= # Deprecated
-PLUGINS_USE_AZURE="true" # Deprecated
+#PLUGINS_USE_AZURE="true" # Deprecated
 
 #============#
 # BingAI     #
@@ -363,6 +386,32 @@ HELP_AND_FAQ_URL=https://librechat.ai
 
 # SHOW_BIRTHDAY_ICON=true
 
+#==================================================#
+#               VectorDB Configuration               #
+#==================================================#
+POSTGRES_DB=mydatabase
+POSTGRES_USER=myuser
+POSTGRES_PASSWORD=mypassword
+
+
+#==================================================#
+#               RecipeDB Configuration               #
+#==================================================#
+POSTGRES_RECIPE_DB=myrecipedatabase
+POSTGRES_RECIPE_USER=myrecipeuser
+POSTGRES_RECIPE_PASSWORD=myrecipepassword
+POSTGRES_RECIPE_HOST=data_recipe_db
+POSTGRES_RECIPE_PORT=5432
+
+#==================================================#
+#               Get Memory Action Configuration    #
+#==================================================#
+OPENAI_API_TYPE=
+OPENAI_API_ENDPOINT=
+OPENAI_API_VERSION_MEMORY=
+BASE_URL_MEMORY=
+MODEL_MEMORY=
+OPENAI_TEXT_COMPLETION_DEPLOYMENT_NAME=
 #==================================================#
 #                      Others                      #
 #==================================================#

.gitignore

@@ -6,4 +6,6 @@ logs
 meili_data_v1.7
 images
 pgdata2
-tmp
+tmp
+ingestion/api/hapi/*
+ingestion/api/hdx/*

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.terminal.activateEnvironment": false
+}

README.md

@@ -32,23 +32,30 @@ Robocorp AI Actions API - [http://localhost:3001/](http://localhost:3001/)
 
 TODO: This will be automated, but for now ...
 
-1. Initialize the DB connection by going to [http://localhost:4001/](http://localhost:4001/) and running action `init_postgres_connection` to set Recipes DB in Azure (TO DO will be changed once we finish ingestion folders)
 1. Got to  [chat app](http://localhost:3080/) and register a user on the login page
 2. Log in
-3. Select Assistants, choose Humanitarian AI Assistant (alpha) (Note: As this is still a work in progress, the names might change over time)
-4. Under actions, create a new action and use the function definition from [here](http://localhost:4001/openapi.json). You'll need to remove the comments at the top and change the host to be 'url' in 'servers' to be "http://actions:8080"
-5. Save the action
-6. Update the agent
+3. `docker exec -it haa-ingestion /bin/bash`
+4. `python3 ingest.py`
+5. Select Assistants, choose HDeXpert SQL
+6. Under actions, create a new action and use the function definition from [here](http://localhost:4001/openapi.json). You'll need to remove the comments at the top and change the host to be 'url' in 'servers' to be "http://actions:8080"
+7. Save the action
+8. Update the agent
 
 Note: You can reset Libre chat by removing contents of `ui/recipes_assistant_chat/data-node/`. This is sometimes neccesary due to a bug in specifying actions.
 
-## Testing connection to actions server
+## Reseting your environment
+
+If running locally, you can reset your environment - removing any data for your databases, which means re-registration - by running `./cleanuop.sh`.
 
-1. `exec -it LibreChat /bin/sh`
-2. `curl -X POST -H "Content-Type: application/json" \
-    -d '{"dsn": "postgresql://username:password@host:port/database"}' \
-    "http://actions:8080/api/actions/postgresql-universal-actions/init-postgres-connection/run"` .... replacing with correct postgres credentials`
+## Testing connection to actions server
 
+1. `docker exec -it haa-libre-chat  /bin/sh`
+2. To test the SQL query action, run `curl -X POST -H "Content-Type: application/json" \
+    -d '{"query": "select 1"}' \
+    "http://actions:8080/api/actions/postgresql-universal-actions/execute-query/run"`
+3. To get get-memory action, run ... `curl -X POST -H "Content-Type: application/json" \
+    -d '{"chat_history": "[]", "user_input":"population of Mali", "generate_intent":"true"}' \
+    "http://actions:8080/api/actions/get-data-recipe-memory/get-memory/run"``
 
 ## Deploying to Azure
 

actions/Dockerfile

@@ -38,8 +38,11 @@ ENV DATA_DB_CONN_STRING=$DATA_DB_CONN_STRING
 # RUN action-server import --datadir=/action-server/datadir
 # Load individually
 RUN action-server import --dir=./actions_plugins/postgres-universal --datadir=/action-server/datadir
+RUN action-server import --dir=./actions_plugins/recipe-server --datadir=/action-server/datadir
 
-RUN echo "{\"dsn\": \"$DATA_DB_CONN_STRING\"}" > ./actions_plugins/postgres-universal/postgres_connection.json
+RUN echo "{\"dsn\": \"$DATA_DB_CONN_STRING\"}" > ./actions_plugins/postgres-universal/postgres_data_connection.json
+
+RUN echo "{\"dsn\": \"$RECIPE_DB_CONN_STRING\"}" > ./actions_plugins/postgres-universal/postgres_recipe_connection.json
 
 EXPOSE 8080
 

actions/actions_plugins/postgres-universal/actions.py

@@ -3,10 +3,11 @@
 import psycopg2
 from robocorp.actions import action
 
-CONNECTION_FILE_PATH = 'postgres_connection.json'
+CONNECTION_FILE_PATH = "postgres_data_connection.json"
 
 MAX_CHARS_TOT = 10000
 
+
 class ReadOnlyConnection:
     def __init__(self, conn_params):
         self.conn_params = conn_params
@@ -24,26 +25,33 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         self.conn.close()
 
 
-def truncate_output_with_beginning_clue(output: str, max_chars: int = MAX_CHARS_TOT) -> str:
-    beginning_clue = "[Cut] "  # A very short clue at the beginning to indicate possible truncation
+def truncate_output_with_beginning_clue(
+    output: str, max_chars: int = MAX_CHARS_TOT
+) -> str:
+    beginning_clue = (
+        "[Cut] "  # A very short clue at the beginning to indicate possible truncation
+    )
 
     if len(output) > max_chars:
-        truncated_output = output[:max_chars - len(beginning_clue)]
+        truncated_output = output[: max_chars - len(beginning_clue)]
         chars_missed = len(output) - len(truncated_output)
         truncated_message = f"[+{chars_missed}]"
         return beginning_clue + truncated_output + truncated_message
     else:
         return output
 
+
 def get_database_schema(conn):
     schema_info = "Database Schema:\n"
 
     with conn.cursor() as cursor:
         # Retrieve triggers
-        cursor.execute("""
+        cursor.execute(
+            """
             SELECT event_object_table AS table_name, trigger_name
             FROM information_schema.triggers
-        """)
+        """
+        )
         trigger_records = cursor.fetchall()
 
         table_trigger_dict = {}
@@ -55,26 +63,33 @@ def get_database_schema(conn):
                     table_trigger_dict[table_name].append(trigger_name)
 
         # Retrieve tables
-        cursor.execute("""
+        cursor.execute(
+            """
             SELECT table_name
             FROM information_schema.tables
             WHERE table_schema = 'public'
-        """)
+        """
+        )
         tables = cursor.fetchall()
 
         for table in tables:
             table_name = table[0]
             schema_info += f"\nTable: {table_name}\n"
 
             if table_name in table_trigger_dict.keys():
-                schema_info += f"Triggers: {', '.join(table_trigger_dict[table_name])}\n"
-
+                schema_info += (
+                    f"Triggers: {', '.join(table_trigger_dict[table_name])}\n"
+                )
+
             # Get columns and primary key info
-            cursor.execute("""
+            cursor.execute(
+                """
                 SELECT column_name, data_type, is_nullable, column_default
                 FROM information_schema.columns
                 WHERE table_name = %s
-            """, (table_name,))
+            """,
+                (table_name,),
+            )
             columns = cursor.fetchall()
 
             for col in columns:
@@ -85,25 +100,31 @@ def get_database_schema(conn):
                 schema_info += "\n"
 
             # Get primary key info
-            cursor.execute("""
+            cursor.execute(
+                """
                 SELECT kcu.column_name
                 FROM information_schema.table_constraints tc
                 JOIN information_schema.key_column_usage kcu
                 ON tc.constraint_name = kcu.constraint_name
                 WHERE tc.table_name = %s AND tc.constraint_type = 'PRIMARY KEY'
-            """, (table_name,))
+            """,
+                (table_name,),
+            )
             primary_keys = cursor.fetchall()
 
             # Get foreign key info
-            cursor.execute("""
+            cursor.execute(
+                """
                 SELECT kcu.column_name, ccu.table_name AS foreign_table_name, ccu.column_name AS foreign_column_name
                 FROM information_schema.table_constraints AS tc 
                 JOIN information_schema.key_column_usage AS kcu
                 ON tc.constraint_name = kcu.constraint_name
                 JOIN information_schema.constraint_column_usage AS ccu
                 ON ccu.constraint_name = tc.constraint_name
                 WHERE tc.table_name = %s AND tc.constraint_type = 'FOREIGN KEY'
-            """, (table_name,))
+            """,
+                (table_name,),
+            )
             foreign_keys = cursor.fetchall()
 
             for pk in primary_keys:
@@ -114,7 +135,7 @@ def get_database_schema(conn):
     return schema_info
 
 
-def truncate_query_results(results, max_chars = MAX_CHARS_TOT):
+def truncate_query_results(results, max_chars=MAX_CHARS_TOT):
     if not results:
         return ""
 
@@ -142,7 +163,7 @@ def truncate_query_results(results, max_chars = MAX_CHARS_TOT):
             cell_output = str(cell)
             # Truncate cell if necessary
             if len(cell_output) > cell_max:
-                cell_output = cell_output[:cell_max - 3] + "..."
+                cell_output = cell_output[: cell_max - 3] + "..."
             row_output += cell_output + ", "
 
         # Remove last comma and space, add newline
@@ -151,36 +172,36 @@ def truncate_query_results(results, max_chars = MAX_CHARS_TOT):
         # Check if we've reached the max characters
         if len(truncated_output) >= max_chars:
             # Further truncate and end the loop
-            truncated_output = truncated_output[:max_chars - 3] + "..."
+            truncated_output = truncated_output[: max_chars - 3] + "..."
             break
 
     return truncated_output
 
 
+# @action
+# def init_postgres_connection(dsn: str) -> str:
+#     """
+#     Initializes a connection to a PostgreSQL database using the provided Data Source Name (DSN).
+#
+#     Args:
+#         dsn (str): The connection string for the PostgreSQL database.
+#                    This should be in the format 'postgresql://username:password@host:port/database'.
+#
+#     Returns:
+#         str: A textual representation of the database schema.
+#     """
+#     conn_params = {"dsn": dsn}
+#
+#     # Save connection parameters to a file
+#     try:
+#         with open(CONNECTION_FILE_PATH, "w") as file:
+#             json.dump(conn_params, file)
+#         with ReadOnlyConnection(conn_params) as conn:
+#             schema = get_database_schema(conn)
+#         return truncate_output_with_beginning_clue(schema)
+#     except Exception as e:
+#         return f"Failed: {e}"
 
-@action
-def init_postgres_connection(dsn: str) -> str:
-    """
-    Initializes a connection to a PostgreSQL database using the provided Data Source Name (DSN).
-
-    Args:
-        dsn (str): The connection string for the PostgreSQL database.
-                   This should be in the format 'postgresql://username:password@host:port/database'.
-
-    Returns:
-        str: A textual representation of the database schema.
-    """
-    conn_params = {'dsn': dsn}
-
-    # Save connection parameters to a file
-    try:
-        with open(CONNECTION_FILE_PATH, 'w') as file:
-            json.dump(conn_params, file)
-        with ReadOnlyConnection(conn_params) as conn:
-            schema = get_database_schema(conn)
-        return truncate_output_with_beginning_clue(schema)
-    except Exception as e:
-        return f"Failed: {e}"
 
 @action
 def execute_query(query: str) -> str:
@@ -196,7 +217,7 @@ def execute_query(query: str) -> str:
     try:
         # Read connection parameters from the file
         if os.path.exists(CONNECTION_FILE_PATH):
-            with open(CONNECTION_FILE_PATH, 'r') as file:
+            with open(CONNECTION_FILE_PATH, "r") as file:
                 conn_params = json.load(file)
         else:
             return "Connection parameters file not found."
@@ -208,4 +229,3 @@ def execute_query(query: str) -> str:
             return truncate_query_results(results)
     except Exception as e:
         return f"An error occurred while executing the query: {e}"
-