Merge pull request #133 from microsoft/0.2-Beta

0.2 Beta Release Candidate

.gitignore

@@ -372,5 +372,6 @@ random.txt
 app/backend/cd
 app/backend/lib
 app/backend/lib64
+app/backend/shared_code
 
 packages-microsoft*

.vscode/launch.json

@@ -4,6 +4,13 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal"
+        },
         {
             "name": "Python: Flask",
             "type": "python",

.vscode/settings.json

@@ -4,5 +4,14 @@
     "azureFunctions.pythonVenv": ".venv",
     "azureFunctions.projectLanguage": "Python",
     "azureFunctions.projectRuntime": "~4",
-    "debug.internalConsoleOptions": "neverOpen"
+    "debug.internalConsoleOptions": "neverOpen",
+    "spellright.language": [
+        "en"
+    ],
+    "spellright.documentTypes": [
+        "markdown",
+        "latex",
+        "plaintext",
+        "shellscript"
+    ]
 }

Makefile

@@ -30,10 +30,10 @@ deploy-search-indexes: extract-env ## Deploy search indexes
 	@./scripts/deploy-search-indexes.sh
 
 extract-env-debug-webapp: ## Extract infrastructure.debug.env file from BICEP output
-	@./scripts/json-to-env.debug.sh < infra_output.json > ./scripts/environments/infrastructure.debug.env
+	@./scripts/json-to-env.webapp.debug.sh < infra_output.json > ./scripts/environments/infrastructure.debug.env
 
 extract-env-debug-functions: ## Extract local.settings.json to debug functions from BICEP output
-	@./scripts/json-to-local.debug.sh < infra_output.json > ./functions/local.settings.json
+	@./scripts/json-to-env.function.debug.sh < infra_output.json > ./functions/local.settings.json
 
 # Utils (used by other Makefile rules)
 check-subscription:

NOTICE.md

@@ -73,6 +73,12 @@ From the following locations:
 
 <https://github.com/Azure/azure-sdk-for-python>
 
+
+### tenacity
+
+<https://github.com/jd/tenacity>
+
+
 ## NPM Modules
 
 From the following locations:

README.md

@@ -1,13 +1,9 @@
 # Information Assistant Accelerator
 
-This sample demonstrates a few approaches for creating ChatGPT-like experiences over your own data using the Retrieval Augmented Generation pattern. It uses Azure OpenAI Service to access the ChatGPT model (gpt-35-turbo), and Azure Cognitive Search for data indexing and retrieval.
-
-> **NOTICE:** This codebase relies on the Azure OpenAI Service which must be procured first separately, subject to any applicable license agreement. Access to this code does not grant you a license or right to use Azure OpenAI Service. You can request access [here](https://aka.ms/oaiapply).
+This accelerator demonstrates a few approaches for creating ChatGPT-like experiences over your own data using the Retrieval Augmented Generation pattern. It uses Azure OpenAI Service to access the ChatGPT model (gpt-35-turbo), and Azure Cognitive Search for data indexing and retrieval.
 
 ---
-
 ## Responsible AI
-
 The Information Assistant (IA) Accelerator and Microsoft are committed to the advancement of AI driven by ethical principles that put people first. 
 
 **Read our [Transparency Note](./docs/transparency.md)**
@@ -22,41 +18,48 @@ Find out more with Microsoft's [Responsible AI resources](https://www.microsoft.
 
 * Chat and Q&A interfaces
 * File Upload and automated chunking and indexing for PDF, HTML, and DOCX
+* Monitoring the status of files uploaded and processed by the accelerator
+* Interacting with your data in supported native languages*
 * Explores various options to help users evaluate the trustworthiness of responses with citations, tracking of source content, etc.
 * Shows possible approaches for data preparation, prompt construction, and orchestration of interaction between model (ChatGPT) and retriever (Cognitive Search)
 * Settings directly in the UX to tweak the behavior and experiment with options
 
+*\*See [Configuring your own language ENV file](/docs/features/configuring_language_env_files.md) for supported languages*
+
 ![Chat screen](docs/images/chatscreen.png)
 
 For a detailed review see our [Features](/docs/features/features.md) page.
 
+---
 # Getting Started
 
-
-## Prerequisites
+The IA Accelerator relies on multiple Azure services and has certain prerequisites that need to be met before deployment. It's essential to procure these prerequisites prior to proceeding with the deployment instructions in this guide.
 
 ---
-
-The IA Accelerator requires a variety of services in Azure. There are a few prerequisites required to deploy and configure the IA Accelerator. Please obtain these items first before continuing on to the deployment section of this guide.
-
->To get started with the IA Accelerator you will need the following:
+## Prerequisites
+To get started with the IA Accelerator you will need the following:
 >
->* An active Azure Subscription *
+>* An azure subscription with access enabled for the Azure OpenAI service.
+You can request access [here](https://aka.ms/oaiapply) *
 >* Administrative rights on the Azure Subscription
+>* [Visual studio code](https://code.visualstudio.com/)
+>
 >
->\* *The Azure Subscription needs to be a paid subscription. While you can create a free tier of any of the Cognitive Services, you can only create one free Cognitive Service in an account so the trial would therefore not be able to handle all of the services in this example.*
+You can sign up for an Azure subscription [here](https://azure.microsoft.com/en-us/free/). 
+
+Once you have your prerequisite items, please move on to the Deployment Configuration step.
 
-You can sign up for an Azure subscription [here](https://azure.microsoft.com/en-us/free/). Once you have your prerequisite items, please move on to the Deployment Configuration step.
+**NOTICE:** This codebase relies on the Azure OpenAI Service which must be procured first separately, subject to any applicable license agreement. Access to this code does not grant you a license or right to use Azure OpenAI Service. 
 
 ## Deployment Configuration
 
-The deployment process for the IA Accelerator, uses a concept of **Developing inside a Container** to containerize all the necessary pre-requisite component without requiring them to be installed on the local machine. The environment you will work in will be created using a development container, or dev container, hosted on a virtual machine using GitHub Codespaces.
+The deployment process for the IA Accelerator, uses a concept of **Developing inside a Container** to containerize all the necessary pre-requisite component without requiring them to be installed on the local machine. The environment you will work in will be created using a development container, or dev container hosted on a virtual machine using GitHub Codespaces.
 
 [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/microsoft/PubSec-Info-Assistant)
 
 Begin by setting up your own Codespace using our  [Developing in a Codespaces](docs/developing_in_a_codespaces.md) documentation.
 
-*More information can be found at [Developing inside a Container](https://code.visualstudio.com/docs/remote/containers).*
+*If you want to configure your local deskop for development container, follow our [Configuring your System for Development Containers](/docs/configure_local_dev_environment.md) guide. More information can be found at [Developing inside a Container](https://code.visualstudio.com/docs/remote/containers).*
 
 Once you have the completed the setting up Codespaces, please move on to the Sizing Estimation step.
 

app/backend/app.py

@@ -16,6 +16,7 @@
 from approaches.readdecomposeask import ReadDecomposeAsk
 from approaches.chatreadretrieveread import ChatReadRetrieveReadApproach
 from azure.storage.blob import BlobServiceClient, generate_account_sas, ResourceTypes, AccountSasPermissions
+from shared_code.status_log import StatusLog, State
 
 # Replace these with your own values, either in environment variables or directly here
 AZURE_BLOB_STORAGE_ACCOUNT = os.environ.get("AZURE_BLOB_STORAGE_ACCOUNT") or "mystorageaccount"
@@ -33,6 +34,13 @@
 KB_FIELDS_CATEGORY = os.environ.get("KB_FIELDS_CATEGORY") or "category"
 KB_FIELDS_SOURCEPAGE = os.environ.get("KB_FIELDS_SOURCEPAGE") or "file_storage_path"
 
+COSMOSDB_URL = os.environ.get("COSMOSDB_URL")
+COSMODB_KEY = os.environ.get("COSMOSDB_KEY")
+COSMOSDB_DATABASE_NAME = os.environ.get("COSMOSDB_DATABASE_NAME") or "statusdb"
+COSMOSDB_CONTAINER_NAME = os.environ.get("COSMOSDB_CONTAINER_NAME") or "statuscontainer"
+
+QUERY_TERM_LANGUAGE = os.environ.get("QUERY_TERM_LANGUAGE") or "English"
+
 # Use the current user identity to authenticate with Azure OpenAI, Cognitive Search and Blob Storage (no secrets needed, 
 # just use 'az login' locally, and managed identity when deployed on Azure). If you need to use keys, use separate AzureKeyCredential instances with the 
 # keys for each service
@@ -43,7 +51,10 @@
 # Used by the OpenAI SDK
 openai.api_type = "azure"
 openai.api_base = f"https://{AZURE_OPENAI_SERVICE}.openai.azure.com"
-openai.api_version = "2023-03-15-preview"
+openai.api_version = "2023-06-01-preview"
+
+# Setup StatusLog to allow access to CosmosDB for logging
+statusLog = StatusLog(COSMOSDB_URL, COSMODB_KEY, COSMOSDB_DATABASE_NAME, COSMOSDB_CONTAINER_NAME)
 
 # Comment these two lines out if using keys, set your API key in the OPENAI_API_KEY environment variable instead
 #openai.api_type = "azure_ad"
@@ -69,7 +80,7 @@
 }
 
 chat_approaches = {
-    "rrr": ChatReadRetrieveReadApproach(search_client, AZURE_OPENAI_SERVICE, AZURE_OPENAI_SERVICE_KEY, AZURE_OPENAI_CHATGPT_DEPLOYMENT, AZURE_OPENAI_GPT_DEPLOYMENT, KB_FIELDS_SOURCEPAGE, KB_FIELDS_CONTENT, blob_client)
+    "rrr": ChatReadRetrieveReadApproach(search_client, AZURE_OPENAI_SERVICE, AZURE_OPENAI_SERVICE_KEY, AZURE_OPENAI_CHATGPT_DEPLOYMENT, AZURE_OPENAI_GPT_DEPLOYMENT, KB_FIELDS_SOURCEPAGE, KB_FIELDS_CONTENT, blob_client, QUERY_TERM_LANGUAGE)
 }
 
 app = Flask(__name__)
@@ -113,7 +124,17 @@ def chat():
         if not impl:
             return jsonify({"error": "unknown approach"}), 400
         r = impl.run(request.json["history"], request.json.get("overrides") or {})
-        return jsonify(r)
+
+        # return jsonify(r)
+        #To fix citation bug,below code is added.aparmar            
+        return jsonify({
+                "data_points": r["data_points"],
+                "answer": r["answer"],
+                "thoughts": r["thoughts"],
+                "citation_lookup": r["citation_lookup"]
+            })
+
+
     except Exception as e:
         logging.exception("Exception in /chat")
         return jsonify({"error": str(e)}), 500
@@ -129,3 +150,14 @@ def get_blob_client_url():
 
 if __name__ == "__main__":
     app.run()
+
+@app.route("/getalluploadstatus", methods=["POST"])
+def get_all_upload_status():
+    timeframe = request.json["timeframe"]
+    state = request.json["state"]
+    try:
+        results = statusLog.read_files_status_by_timeframe(timeframe,State[state])
+    except Exception as e:
+        logging.exception("Exception in /getalluploadstatus")
+        return jsonify({"error": str(e)}), 500
+    return jsonify(results)