feat: CEC in M365 Copilot: implement citation in rag ts&js template (#…

…12378) * implement citation in rag ts&js template * fix eslint issue * remove unuse code * use module.exports * add error handling * update error log
OfficeDev · Sep 12, 2024 · ec3398e · ec3398e
1 parent ea03ef7
commit ec3398e
Show file tree

Hide file tree

Showing 24 changed files with 715 additions and 35 deletions.
diff --git a/templates/js/custom-copilot-rag-azure-ai-search/src/app/app.js.tpl b/templates/js/custom-copilot-rag-azure-ai-search/src/app/app.js.tpl
@@ -1,9 +1,10 @@
 const { MemoryStorage, MessageFactory } = require("botbuilder");
 const path = require("path");
 const config = require("../config");
+const customSayCommand = require("./customSayCommand");
 
 // See https://aka.ms/teams-ai-library to learn more about the Teams AI library.
-const { Application, ActionPlanner, OpenAIModel, PromptManager } = require("@microsoft/teams-ai");
+const { AI, Application, ActionPlanner, OpenAIModel, PromptManager } = require("@microsoft/teams-ai");
 const { AzureAISearchDataSource } = require("./azureAISearchDataSource");
 
 // Create AI components
@@ -58,6 +59,7 @@ const app = new Application({
     enable_feedback_loop: true,
   },
 });
+app.ai.action(AI.SayCommandActionName, customSayCommand.sayCommand(true));
 
 app.conversationUpdate("membersAdded", async (turnContext) => {
   const welcomeText = "How can I help you today?";

diff --git a/templates/js/custom-copilot-rag-azure-ai-search/src/app/azureAISearchDataSource.js.tpl b/templates/js/custom-copilot-rag-azure-ai-search/src/app/azureAISearchDataSource.js.tpl
@@ -61,7 +61,7 @@ class AzureAISearchDataSource {
         let usedTokens = 0;
         let doc = "";
         for await (const result of searchResults.results) {
-            const formattedResult = this.formatDocument(result.document.description);
+            const formattedResult = this.formatDocument(`${result.document.description}\n Citation title:${result.document.docTitle}.`);
             const tokens = tokenizer.encode(formattedResult).length;
 
             if (usedTokens + tokens > maxTokens) {

diff --git a/templates/js/custom-copilot-rag-azure-ai-search/src/app/customSayCommand.js b/templates/js/custom-copilot-rag-azure-ai-search/src/app/customSayCommand.js
@@ -0,0 +1,90 @@
+const botbuilder = require("botbuilder");
+const Utilities = require("@microsoft/teams-ai");
+
+function sayCommand(feedbackLoopEnabled = false) {
+  return async (context, _state, data) => {
+    if (!data.response?.content) {
+      return "";
+    }
+    const isTeamsChannel = context.activity.channelId === botbuilder.Channels.Msteams;
+    let content = "";
+    let result = undefined;
+    try {
+      result = JSON.parse(data.response.content);
+    } catch (error) {
+      console.error(`Response is not valid json, send the raw text. error: ${error}`);
+      await context.sendActivity({
+        type: botbuilder.ActivityTypes.Message,
+        text: data.response.content,
+        ...(isTeamsChannel ? { channelData: { feedbackLoopEnabled } } : {}),
+        entities: [
+          {
+            type: "https://schema.org/Message",
+            "@type": "Message",
+            "@context": "https://schema.org",
+            "@id": "",
+            additionalType: ["AIGeneratedContent"],
+          },
+        ],
+      });
+      return "";
+    }
+    // If the response from AI includes citations, those citations will be parsed and added to the SAY command.
+    let citations = [];
+    let position = 1;
+    if (result.results && result.results.length > 0) {
+      result.results.forEach((contentItem) => {
+        if (contentItem.citationTitle && contentItem.citationTitle.length > 0) {
+          const clientCitation = {
+            "@type": "Claim",
+            position: `${position}`,
+            appearance: {
+              "@type": "DigitalDocument",
+              name: contentItem.citationTitle || `Document #${position}`,
+              url: contentItem.citationUrl,
+              abstract: Utilities.Utilities.snippet(contentItem.citationContent, 500),
+            },
+          };
+          content += `${contentItem.answer}[${position}]<br>`;
+          position++;
+          citations.push(clientCitation);
+        } else {
+          content += `${contentItem.answer}<br>`;
+        }
+      });
+    } else {
+      content = data.response.content;
+    }
+
+    if (isTeamsChannel) {
+      content = content.split("\n").join("<br>");
+    }
+    // If there are citations, modify the content so that the sources are numbers instead of [doc1], [doc2], etc.
+    const contentText =
+      citations.length < 1 ? content : Utilities.Utilities.formatCitationsResponse(content);
+    // If there are citations, filter out the citations unused in content.
+    const referencedCitations =
+      citations.length > 0
+        ? Utilities.Utilities.getUsedCitations(contentText, citations)
+        : undefined;
+    await context.sendActivity({
+      type: botbuilder.ActivityTypes.Message,
+      text: contentText,
+      ...(isTeamsChannel ? { channelData: { feedbackLoopEnabled } } : {}),
+      entities: [
+        {
+          type: "https://schema.org/Message",
+          "@type": "Message",
+          "@context": "https://schema.org",
+          "@id": "",
+          additionalType: ["AIGeneratedContent"],
+          ...(referencedCitations ? { citation: referencedCitations } : {}),
+        },
+      ],
+    });
+    return "";
+  };
+}
+module.exports = {
+  sayCommand,
+};
diff --git a/templates/js/custom-copilot-rag-azure-ai-search/src/prompts/chat/skprompt.txt b/templates/js/custom-copilot-rag-azure-ai-search/src/prompts/chat/skprompt.txt
@@ -1,3 +1,21 @@
 The following is a conversation with an AI assistant, who is an expert on answering questions over the given context.
-Responses should be in a short journalistic style with no more than 80 words.
-Use the context provided in the `<context></context>` tags as the source for your answers.
+Responses should be in a short journalistic style with no more than 80 words, and provide citations.
+Use the context provided in the `<context></context>` tags as the source for your answers.
+Response should be a json array, list all the answers and citations. 
+If the answer no citation, set the citationTitle and citationContent as empty.
+Data format:
+{
+    "results":[
+        {
+            "answer":"{$answer1}",
+            "citationTitle":"{$citationTitle1}",
+            "citationContent":"{$citationContent1}"
+        },
+        {
+            "answer":"{$answer2}",
+            "citationTitle":"{$citationTitle2}",
+            "citationContent":"{$citationContent2}"
+        },
+        ...
+    ]
+} 
diff --git a/templates/js/custom-copilot-rag-customize/src/app/app.js.tpl b/templates/js/custom-copilot-rag-customize/src/app/app.js.tpl
@@ -1,9 +1,10 @@
 const { MemoryStorage, MessageFactory } = require("botbuilder");
 const path = require("path");
 const config = require("../config");
+const customSayCommand = require("./customSayCommand");
 
 // See https://aka.ms/teams-ai-library to learn more about the Teams AI library.
-const { Application, ActionPlanner, OpenAIModel, PromptManager } = require("@microsoft/teams-ai");
+const { AI, Application, ActionPlanner, OpenAIModel, PromptManager } = require("@microsoft/teams-ai");
 const { MyDataSource } = require("./myDataSource");
 
 // Create AI components
@@ -44,6 +45,7 @@ const app = new Application({
     enable_feedback_loop: true,
   },
 });
+app.ai.action(AI.SayCommandActionName, customSayCommand.sayCommand(true));
 
 app.conversationUpdate("membersAdded", async (turnContext) => {
   const welcomeText = "How can I help you today?";

diff --git a/templates/js/custom-copilot-rag-customize/src/app/customSayCommand.js b/templates/js/custom-copilot-rag-customize/src/app/customSayCommand.js
@@ -0,0 +1,90 @@
+const botbuilder = require("botbuilder");
+const Utilities = require("@microsoft/teams-ai");
+
+function sayCommand(feedbackLoopEnabled = false) {
+  return async (context, _state, data) => {
+    if (!data.response?.content) {
+      return "";
+    }
+    const isTeamsChannel = context.activity.channelId === botbuilder.Channels.Msteams;
+    let content = "";
+    let result = undefined;
+    try {
+      result = JSON.parse(data.response.content);
+    } catch (error) {
+      console.error(`Response is not valid json, send the raw text. error: ${error}`);
+      await context.sendActivity({
+        type: botbuilder.ActivityTypes.Message,
+        text: data.response.content,
+        ...(isTeamsChannel ? { channelData: { feedbackLoopEnabled } } : {}),
+        entities: [
+          {
+            type: "https://schema.org/Message",
+            "@type": "Message",
+            "@context": "https://schema.org",
+            "@id": "",
+            additionalType: ["AIGeneratedContent"],
+          },
+        ],
+      });
+      return "";
+    }
+    // If the response from AI includes citations, those citations will be parsed and added to the SAY command.
+    let citations = [];
+    let position = 1;
+    if (result.results && result.results.length > 0) {
+      result.results.forEach((contentItem) => {
+        if (contentItem.citationTitle && contentItem.citationTitle.length > 0) {
+          const clientCitation = {
+            "@type": "Claim",
+            position: `${position}`,
+            appearance: {
+              "@type": "DigitalDocument",
+              name: contentItem.citationTitle || `Document #${position}`,
+              url: contentItem.citationUrl,
+              abstract: Utilities.Utilities.snippet(contentItem.citationContent, 500),
+            },
+          };
+          content += `${contentItem.answer}[${position}]<br>`;
+          position++;
+          citations.push(clientCitation);
+        } else {
+          content += `${contentItem.answer}<br>`;
+        }
+      });
+    } else {
+      content = data.response.content;
+    }
+
+    if (isTeamsChannel) {
+      content = content.split("\n").join("<br>");
+    }
+    // If there are citations, modify the content so that the sources are numbers instead of [doc1], [doc2], etc.
+    const contentText =
+      citations.length < 1 ? content : Utilities.Utilities.formatCitationsResponse(content);
+    // If there are citations, filter out the citations unused in content.
+    const referencedCitations =
+      citations.length > 0
+        ? Utilities.Utilities.getUsedCitations(contentText, citations)
+        : undefined;
+    await context.sendActivity({
+      type: botbuilder.ActivityTypes.Message,
+      text: contentText,
+      ...(isTeamsChannel ? { channelData: { feedbackLoopEnabled } } : {}),
+      entities: [
+        {
+          type: "https://schema.org/Message",
+          "@type": "Message",
+          "@context": "https://schema.org",
+          "@id": "",
+          additionalType: ["AIGeneratedContent"],
+          ...(referencedCitations ? { citation: referencedCitations } : {}),
+        },
+      ],
+    });
+    return "";
+  };
+}
+module.exports = {
+  sayCommand,
+};
diff --git a/templates/js/custom-copilot-rag-customize/src/app/myDataSource.js.tpl b/templates/js/custom-copilot-rag-customize/src/app/myDataSource.js.tpl
@@ -19,7 +19,12 @@ class MyDataSource {
         const filePath = path.join(__dirname, "../data");
         const files = fs.readdirSync(filePath);
         this._data = files.map(file => {
-            return fs.readFileSync(path.join(filePath, file), "utf-8");
+            const data = 
+            {
+                content:fs.readFileSync(path.join(filePath, file), "utf-8"),
+                citation:file
+            };
+            return data;
         });
     }
 
@@ -32,16 +37,16 @@ class MyDataSource {
             return { output: "", length: 0, tooLong: false };
         }
         for (let data of this._data) {
-            if (data.includes(query)) {
-                return { output: this.formatDocument(data), length: data.length, tooLong: false };
+            if (data.content.includes(query)) {
+                return { output: this.formatDocument(`${data.content}\n Citation title:${data.citation}`), length: data.content.length, tooLong: false };
             }
         }
         if (query.toLocaleLowerCase().includes("perksplus")) {
-            return { output: this.formatDocument(this._data[0]), length: this._data[0].length, tooLong: false };
+            return { output: this.formatDocument(`${this._data[0].content}\n Citation title:${this._data[0].citation}`), length: this._data[0].content.length, tooLong: false };
         } else if (query.toLocaleLowerCase().includes("company") || query.toLocaleLowerCase().includes("history")) {
-            return { output: this.formatDocument(this._data[1]), length: this._data[1].length, tooLong: false };
+            return { output: this.formatDocument(`${this._data[1].content}\n Citation title:${this._data[1].citation}`), length: this._data[1].content.length, tooLong: false };
         } else if (query.toLocaleLowerCase().includes("northwind") || query.toLocaleLowerCase().includes("health")) {
-            return { output: this.formatDocument(this._data[2]), length: this._data[2].length, tooLong: false };
+            return { output: this.formatDocument(`${this._data[2].content}\n Citation title:${this._data[2].citation}`), length: this._data[2].content.length, tooLong: false };
         }
         return { output: "", length: 0, tooLong: false };
     }

diff --git a/templates/js/custom-copilot-rag-customize/src/prompts/chat/skprompt.txt b/templates/js/custom-copilot-rag-customize/src/prompts/chat/skprompt.txt
@@ -1,3 +1,21 @@
 The following is a conversation with an AI assistant, who is an expert on answering questions over the given context.
-Responses should be in a short journalistic style with no more than 80 words.
-Use the context provided in the `<context></context>` tags as the source for your answers.
+Responses should be in a short journalistic style with no more than 80 words, and provide citations.
+Use the context provided in the `<context></context>` tags as the source for your answers.
+Response should be a json array, list all the answers and citations. 
+If the answer no citation, set the citationTitle and citationContent as empty.
+Data format:
+{
+    "results":[
+        {
+            "answer":"{$answer1}",
+            "citationTitle":"{$citationTitle1}",
+            "citationContent":"{$citationContent1}"
+        },
+        {
+            "answer":"{$answer2}",
+            "citationTitle":"{$citationTitle2}",
+            "citationContent":"{$citationContent2}"
+        },
+        ...
+    ]
+} 
diff --git a/templates/js/custom-copilot-rag-microsoft365/src/app/app.js.tpl b/templates/js/custom-copilot-rag-microsoft365/src/app/app.js.tpl
@@ -1,9 +1,10 @@
 const { MemoryStorage, MessageFactory } = require("botbuilder");
 const path = require("path");
 const config = require("../config");
+const customSayCommand = require("./customSayCommand");
 
 // See https://aka.ms/teams-ai-library to learn more about the Teams AI library.
-const { Application, ActionPlanner, OpenAIModel, PromptManager } = require("@microsoft/teams-ai");
+const { AI, Application, ActionPlanner, OpenAIModel, PromptManager } = require("@microsoft/teams-ai");
 const { GraphDataSource } = require("./graphDataSource");
 
 // Create AI components
@@ -59,6 +60,7 @@ const app = new Application({
     autoSignIn: true,
   }
 });
+app.ai.action(AI.SayCommandActionName, customSayCommand.sayCommand(true));
 
 app.conversationUpdate("membersAdded", async (turnContext) => {
   const welcomeText = "How can I help you today?";