From f6fb9f94d713a798907ceee3be4f9f04f6c00e71 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Wed, 6 Sep 2023 09:08:05 -0700 Subject: [PATCH] .Net: StepwisePlanner Chat support (#2504) ### Motivation and Context This pull request introduces a series of improvements and updates to the StepwisePlanner, StepwisePlannerTests, and WebSearchEngineSkill. The changes include better handling of action invocation, function retrieval, logging, chat history management, execution helpers, and the addition of an offset parameter to the WebSearchEngineSkill. Additionally, the StepwisePlannerTests have been enhanced with more test cases and refined expected minimum steps, while the ChatRequestSettings class has been improved with a method to create a new settings object from another settings object. Resolves #2367 Fixes #2466 Fixes #2553 ### Description 1. Improved the StepwisePlanner class with better action invocation, function retrieval, logging, chat history management, and execution helpers. 2. Enhanced the StepwisePlannerTests with more test cases and refined expected minimum steps. 3. Added an offset parameter to the WebSearchEngineSkill class for improved search functionality. 4. Improved the ChatRequestSettings class with a method to create a new settings object from another settings object. 5. Updated the Example51_StepwisePlanner.cs file with new questions, improved result handling, and additional output for better analysis. 6. Enhanced the ParseResultTests with new test cases for the ParseResult class. 7. Updated the WebSearchEngineSkill to include parameter descriptions for the SearchAsync method. 8. Added project references and minor changes to the SemanticKernel.MetaPackage project. ### Contribution Checklist - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone :smile: Co-authored-by: Andrew Hesky @andhesky --- dotnet/SK-dotnet.sln | 9 + .../Example15_TextMemorySkill.cs | 16 +- .../Example51_StepwisePlanner.cs | 223 ++++-- .../.editorconfig | 5 + ...xtensions.StepwisePlanner.UnitTests.csproj | 30 + .../StepwisePlanner/ParseResultTests.cs | 43 +- .../Planning.StepwisePlanner.csproj | 12 +- .../Skills/RenderFunctionManual/config.json | 14 + .../Skills/RenderFunctionManual/skprompt.txt | 8 + .../Skills/RenderQuestion/config.json | 14 + .../Skills/RenderQuestion/skprompt.txt | 2 + .../Skills/StepwiseStep/config.json | 15 +- .../Skills/StepwiseStep/skprompt.txt | 58 +- .../StepwisePlanner.cs | 693 ++++++++++++------ .../StepwisePlannerConfig.cs | 66 +- .../Planning.StepwisePlanner/SystemStep.cs | 12 +- .../StepwisePlanner/StepwisePlannerTests.cs | 8 +- .../AI/ChatCompletion/ChatRequestSettings.cs | 19 + .../SemanticKernel.MetaPackage.csproj | 8 +- .../Skills/Skills.Web/WebSearchEngineSkill.cs | 4 +- 20 files changed, 892 insertions(+), 367 deletions(-) create mode 100644 dotnet/src/Extensions/Extensions.StepwisePlanner.UnitTests/.editorconfig create mode 100644 dotnet/src/Extensions/Extensions.StepwisePlanner.UnitTests/Extensions.StepwisePlanner.UnitTests.csproj rename dotnet/src/Extensions/{Extensions.UnitTests => Extensions.StepwisePlanner.UnitTests}/Planning/StepwisePlanner/ParseResultTests.cs (53%) create mode 100644 dotnet/src/Extensions/Planning.StepwisePlanner/Skills/RenderFunctionManual/config.json create mode 100644 dotnet/src/Extensions/Planning.StepwisePlanner/Skills/RenderFunctionManual/skprompt.txt create mode 100644 dotnet/src/Extensions/Planning.StepwisePlanner/Skills/RenderQuestion/config.json create mode 100644 dotnet/src/Extensions/Planning.StepwisePlanner/Skills/RenderQuestion/skprompt.txt diff --git a/dotnet/SK-dotnet.sln b/dotnet/SK-dotnet.sln index 84cccdc1fba3..da836c907297 100644 --- a/dotnet/SK-dotnet.sln +++ b/dotnet/SK-dotnet.sln @@ -158,6 +158,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Reliability.Polly", "src\Ex EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Reliability.Basic", "src\Extensions\Reliability.Basic\Reliability.Basic.csproj", "{3DC4DBD8-20A5-4937-B4F5-BB5E24E7A567}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Extensions.StepwisePlanner.UnitTests", "src\Extensions\Extensions.StepwisePlanner.UnitTests\Extensions.StepwisePlanner.UnitTests.csproj", "{6F651E87-F16E-407B-AF7F-B3475F850E9A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -393,6 +395,12 @@ Global {3DC4DBD8-20A5-4937-B4F5-BB5E24E7A567}.Publish|Any CPU.Build.0 = Publish|Any CPU {3DC4DBD8-20A5-4937-B4F5-BB5E24E7A567}.Release|Any CPU.ActiveCfg = Release|Any CPU {3DC4DBD8-20A5-4937-B4F5-BB5E24E7A567}.Release|Any CPU.Build.0 = Release|Any CPU + {6F651E87-F16E-407B-AF7F-B3475F850E9A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6F651E87-F16E-407B-AF7F-B3475F850E9A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6F651E87-F16E-407B-AF7F-B3475F850E9A}.Publish|Any CPU.ActiveCfg = Debug|Any CPU + {6F651E87-F16E-407B-AF7F-B3475F850E9A}.Publish|Any CPU.Build.0 = Debug|Any CPU + {6F651E87-F16E-407B-AF7F-B3475F850E9A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6F651E87-F16E-407B-AF7F-B3475F850E9A}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -449,6 +457,7 @@ Global {10E4B697-D4E8-468D-872D-49670FD150FB} = {078F96B4-09E1-4E0E-B214-F71A4F4BF633} {D4540A0F-98E3-4E70-9093-1948AE5B2AAD} = {078F96B4-09E1-4E0E-B214-F71A4F4BF633} {3DC4DBD8-20A5-4937-B4F5-BB5E24E7A567} = {078F96B4-09E1-4E0E-B214-F71A4F4BF633} + {6F651E87-F16E-407B-AF7F-B3475F850E9A} = {078F96B4-09E1-4E0E-B214-F71A4F4BF633} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {FBDC56A3-86AD-4323-AA0F-201E59123B83} diff --git a/dotnet/samples/KernelSyntaxExamples/Example15_TextMemorySkill.cs b/dotnet/samples/KernelSyntaxExamples/Example15_TextMemorySkill.cs index 623ea340f170..c5f3f64ec51c 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example15_TextMemorySkill.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example15_TextMemorySkill.cs @@ -146,7 +146,7 @@ private static async Task RunWithStoreAsync(IMemoryStore memoryStore, Cancellati .WithOpenAITextEmbeddingGenerationService(TestConfiguration.OpenAI.EmbeddingModelId, TestConfiguration.OpenAI.ApiKey) .Build(); - // Create an embedding generator to use for semantic memory. + // Create an embedding generator to use for semantic memory. var embeddingGenerator = new OpenAITextEmbeddingGeneration(TestConfiguration.OpenAI.EmbeddingModelId, TestConfiguration.OpenAI.ApiKey); // The combination of the text embedding generator and the memory store makes up the 'SemanticTextMemory' object used to @@ -155,7 +155,7 @@ private static async Task RunWithStoreAsync(IMemoryStore memoryStore, Cancellati ///////////////////////////////////////////////////////////////////////////////////////////////////// // PART 1: Store and retrieve memories using the ISemanticTextMemory (textMemory) object. - // + // // This is a simple way to store memories from a code perspective, without using the Kernel. ///////////////////////////////////////////////////////////////////////////////////////////////////// Console.WriteLine("== PART 1a: Saving Memories through the ISemanticTextMemory object =="); @@ -175,12 +175,12 @@ private static async Task RunWithStoreAsync(IMemoryStore memoryStore, Cancellati // Retrieve a memory Console.WriteLine("== PART 1b: Retrieving Memories through the ISemanticTextMemory object =="); MemoryQueryResult? lookup = await textMemory.GetAsync(MemoryCollectionName, "info1", cancellationToken: cancellationToken); - Console.WriteLine("Memory with key 'info3':" + lookup?.Metadata.Text ?? "ERROR: memory not found"); + Console.WriteLine("Memory with key 'info1':" + lookup?.Metadata.Text ?? "ERROR: memory not found"); Console.WriteLine(); ///////////////////////////////////////////////////////////////////////////////////////////////////// // PART 2: Create TextMemorySkill, store and retrieve memories through the Kernel. - // + // // This enables semantic functions and the AI (via Planners) to access memories ///////////////////////////////////////////////////////////////////////////////////////////////////// @@ -212,8 +212,8 @@ private static async Task RunWithStoreAsync(IMemoryStore memoryStore, Cancellati ///////////////////////////////////////////////////////////////////////////////////////////////////// // PART 3: Recall similar ideas with semantic search - // - // Uses AI Embeddings for fuzzy lookup of memories based on intent, rather than a specific key. + // + // Uses AI Embeddings for fuzzy lookup of memories based on intent, rather than a specific key. ///////////////////////////////////////////////////////////////////////////////////////////////////// Console.WriteLine("== PART 3: Recall (similarity search) with AI Embeddings =="); @@ -260,7 +260,7 @@ private static async Task RunWithStoreAsync(IMemoryStore memoryStore, Cancellati ///////////////////////////////////////////////////////////////////////////////////////////////////// // PART 3: TextMemorySkill Recall in a Semantic Function - // + // // Looks up related memories when rendering a prompt template, then sends the rendered prompt to // the text completion model to answer a natural language query. ///////////////////////////////////////////////////////////////////////////////////////////////////// @@ -300,7 +300,7 @@ END FACTS ///////////////////////////////////////////////////////////////////////////////////////////////////// // PART 5: Cleanup, deleting database collection - // + // ///////////////////////////////////////////////////////////////////////////////////////////////////// Console.WriteLine("== PART 5: Cleanup, deleting database collection =="); diff --git a/dotnet/samples/KernelSyntaxExamples/Example51_StepwisePlanner.cs b/dotnet/samples/KernelSyntaxExamples/Example51_StepwisePlanner.cs index d29a872cd151..c59dd16ed927 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example51_StepwisePlanner.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example51_StepwisePlanner.cs @@ -1,9 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using System.Threading.Tasks; using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.Planning; using Microsoft.SemanticKernel.Skills.Core; using Microsoft.SemanticKernel.Skills.Web; @@ -12,77 +15,216 @@ using RepoUtils; /** - * This example shows how to use Stepwise Planner to create a plan for a given goal. + * This example shows how to use Stepwise Planner to create and run a stepwise plan for a given goal. */ // ReSharper disable once InconsistentNaming public static class Example51_StepwisePlanner { + // Used to override the max allowed tokens when running the plan + internal static int? ChatMaxTokens = null; + internal static int? TextMaxTokens = null; + + // Used to quickly modify the chat model used by the planner + internal static string? ChatModelOverride = null; //"gpt-35-turbo"; + internal static string? TextModelOverride = null; //"text-davinci-003"; + + internal static string? Suffix = null; + public static async Task RunAsync() { string[] questions = new string[] { - "Who is the current president of the United States? What is his current age divided by 2", - // "Who is Leo DiCaprio's girlfriend? What is her current age raised to the (his current age)/100 power?", - // "What is the capital of France? Who is that city's current mayor? What percentage of their life has been in the 21st century as of today?", - // "What is the current day of the calendar year? Using that as an angle in degrees, what is the area of a unit circle with that angle?" + "What color is the sky?", + "What is the weather in Seattle?", + "What is the tallest mountain on Earth? How tall is it divided by 2?", + "What is the capital of France? Who is that city's current mayor? What percentage of their life has been in the 21st century as of today?", + "What is the current day of the calendar year? Using that as an angle in degrees, what is the area of a unit circle with that angle?", + "If a spacecraft travels at 0.99 the speed of light and embarks on a journey to the nearest star system, Alpha Centauri, which is approximately 4.37 light-years away, how much time would pass on Earth during the spacecraft's voyage?" }; foreach (var question in questions) { - var kernel = GetKernel(); - await RunWithQuestion(kernel, question); + for (int i = 0; i < 1; i++) + { + await RunTextCompletion(question); + await RunChatCompletion(question); + } + } + + PrintResults(); + } + + // print out summary table of ExecutionResults + private static void PrintResults() + { + Console.WriteLine("**************************"); + Console.WriteLine("Execution Results Summary:"); + Console.WriteLine("**************************"); + + foreach (var question in ExecutionResults.Select(s => s.question).Distinct()) + { + Console.WriteLine("Question: " + question); + Console.WriteLine("Mode\tModel\tAnswer\tStepsTaken\tIterations\tTimeTaken"); + foreach (var er in ExecutionResults.OrderByDescending(s => s.model).Where(s => s.question == question)) + { + Console.WriteLine($"{er.mode}\t{er.model}\t{er.stepsTaken}\t{er.iterations}\t{er.timeTaken}\t{er.answer}"); + } } } - private static async Task RunWithQuestion(IKernel kernel, string question) + private struct ExecutionResult { + public string mode; + public string? model; + public string? question; + public string? answer; + public string? stepsTaken; + public string? iterations; + public string? timeTaken; + } + + private static List ExecutionResults = new(); + + private static async Task RunTextCompletion(string question) + { + Console.WriteLine("RunTextCompletion"); + ExecutionResult currentExecutionResult = default; + currentExecutionResult.mode = "RunTextCompletion"; + var kernel = GetKernel(ref currentExecutionResult); + await RunWithQuestion(kernel, currentExecutionResult, question, TextMaxTokens); + } + + private static async Task RunChatCompletion(string question, string? model = null) + { + Console.WriteLine("RunChatCompletion"); + ExecutionResult currentExecutionResult = default; + currentExecutionResult.mode = "RunChatCompletion"; + var kernel = GetKernel(ref currentExecutionResult, true, model); + await RunWithQuestion(kernel, currentExecutionResult, question, ChatMaxTokens); + } + + private static async Task RunWithQuestion(IKernel kernel, ExecutionResult currentExecutionResult, string question, int? MaxTokens = null) + { + currentExecutionResult.question = question; var bingConnector = new BingConnector(TestConfiguration.Bing.ApiKey); var webSearchEngineSkill = new WebSearchEngineSkill(bingConnector); kernel.ImportSkill(webSearchEngineSkill, "WebSearch"); - kernel.ImportSkill(new LanguageCalculatorSkill(kernel), "advancedCalculator"); + kernel.ImportSkill(new LanguageCalculatorSkill(kernel), "semanticCalculator"); kernel.ImportSkill(new TimeSkill(), "time"); + // StepwisePlanner is instructed to depend on available functions. + // We expose this function to increase the flexibility in it's ability to answer + // given the relatively small number of functions we have in this example. + // This seems to be particularly helpful in these examples with gpt-35-turbo -- even though it + // does not *use* this function. It seems to help the planner find a better path to the answer. + kernel.CreateSemanticFunction( + "Generate an answer for the following question: {{$input}}", + functionName: "GetAnswerForQuestion", + skillName: "AnswerBot", + description: "Given a question, get an answer and return it as the result of the function"); + Console.WriteLine("*****************************************************"); Stopwatch sw = new(); Console.WriteLine("Question: " + question); var plannerConfig = new Microsoft.SemanticKernel.Planning.Stepwise.StepwisePlannerConfig(); plannerConfig.ExcludedFunctions.Add("TranslateMathProblem"); + plannerConfig.ExcludedFunctions.Add("DaysAgo"); + plannerConfig.ExcludedFunctions.Add("DateMatchingLastDayName"); plannerConfig.MinIterationTimeMs = 1500; - plannerConfig.MaxTokens = 4000; + plannerConfig.MaxIterations = 25; - StepwisePlanner planner = new(kernel, plannerConfig); - sw.Start(); - var plan = planner.CreatePlan(question); + if (!string.IsNullOrEmpty(Suffix)) + { + plannerConfig.Suffix = $"{Suffix}\n{plannerConfig.Suffix}"; + currentExecutionResult.question = $"[Assisted] - {question}"; + } - var result = await plan.InvokeAsync(kernel.CreateNewContext()); - Console.WriteLine("Result: " + result); - if (result.Variables.TryGetValue("stepCount", out string? stepCount)) + if (MaxTokens.HasValue) { - Console.WriteLine("Steps Taken: " + stepCount); + plannerConfig.MaxTokens = MaxTokens.Value; } - if (result.Variables.TryGetValue("skillCount", out string? skillCount)) + SKContext result; + sw.Start(); + + try { - Console.WriteLine("Skills Used: " + skillCount); + StepwisePlanner planner = new(kernel: kernel, config: plannerConfig); + var plan = planner.CreatePlan(question); + + result = await plan.InvokeAsync(kernel.CreateNewContext()); + + if (result.Result.Contains("Result not found, review _stepsTaken to see what", StringComparison.OrdinalIgnoreCase)) + { + Console.WriteLine("Could not answer question in " + plannerConfig.MaxIterations + " iterations"); + currentExecutionResult.answer = "Could not answer question in " + plannerConfig.MaxIterations + " iterations"; + } + else + { + Console.WriteLine("Result: " + result.Result); + currentExecutionResult.answer = result.Result; + } + + if (result.Variables.TryGetValue("stepCount", out string? stepCount)) + { + Console.WriteLine("Steps Taken: " + stepCount); + currentExecutionResult.stepsTaken = stepCount; + } + + if (result.Variables.TryGetValue("skillCount", out string? skillCount)) + { + Console.WriteLine("Skills Used: " + skillCount); + } + + if (result.Variables.TryGetValue("iterations", out string? iterations)) + { + Console.WriteLine("Iterations: " + iterations); + currentExecutionResult.iterations = iterations; + } + } +#pragma warning disable CA1031 + catch (Exception ex) + { + Console.WriteLine("Exception: " + ex); } Console.WriteLine("Time Taken: " + sw.Elapsed); + currentExecutionResult.timeTaken = sw.Elapsed.ToString(); + ExecutionResults.Add(currentExecutionResult); Console.WriteLine("*****************************************************"); } - private static IKernel GetKernel() + private static IKernel GetKernel(ref ExecutionResult result, bool useChat = false, string? model = null) { var builder = new KernelBuilder(); + var maxTokens = 0; + if (useChat) + { + builder.WithAzureChatCompletionService( + model ?? ChatModelOverride ?? TestConfiguration.AzureOpenAI.ChatDeploymentName, + TestConfiguration.AzureOpenAI.Endpoint, + TestConfiguration.AzureOpenAI.ApiKey, + alsoAsTextCompletion: true, + setAsDefault: true); + + maxTokens = ChatMaxTokens ?? (new Microsoft.SemanticKernel.Planning.Stepwise.StepwisePlannerConfig()).MaxTokens; + result.model = model ?? ChatModelOverride ?? TestConfiguration.AzureOpenAI.ChatDeploymentName; + } + else + { + builder.WithAzureTextCompletionService( + model ?? TextModelOverride ?? TestConfiguration.AzureOpenAI.DeploymentName, + TestConfiguration.AzureOpenAI.Endpoint, + TestConfiguration.AzureOpenAI.ApiKey); - builder.WithAzureChatCompletionService( - TestConfiguration.AzureOpenAI.ChatDeploymentName, - TestConfiguration.AzureOpenAI.Endpoint, - TestConfiguration.AzureOpenAI.ApiKey, - alsoAsTextCompletion: true, - setAsDefault: true); + maxTokens = TextMaxTokens ?? (new Microsoft.SemanticKernel.Planning.Stepwise.StepwisePlannerConfig()).MaxTokens; + result.model = model ?? TextModelOverride ?? TestConfiguration.AzureOpenAI.DeploymentName; + } + + Console.WriteLine($"Model: {result.model} ({maxTokens})"); var kernel = builder .WithLoggerFactory(ConsoleLogger.LoggerFactory) @@ -97,32 +239,3 @@ private static IKernel GetKernel() return kernel; } } - -// ***************************************************** -// Question: Who is the current president of the United States? What is his current age divided by 2 -// Result: The current president of the United States is Joe Biden. His current age divided by 2 is 40.5. -// Steps Taken: 9 -// Skills Used: 7 (WebSearch.Search(4), time.Year(1), time.Date(1), advancedCalculator.Calculator(1)) -// Time Taken: 00:01:13.3766860 -// ***************************************************** -// ***************************************************** -// Question: Who is Leo DiCaprio's girlfriend? What is her current age raised to the (his current age)/100 power? -// Result: Leo DiCaprio's girlfriend is Camila Morrone. Her current age raised to the power of (his current age)/100 is approximately 4.94. -// Steps Taken: 9 -// Skills Used: 5 (WebSearch.Search(3), time.Year(1), advancedCalculator.Calculator(1)) -// Time Taken: 00:01:17.6742136 -// ***************************************************** -// ***************************************************** -// Question: What is the capital of France? Who is that cities current mayor? What percentage of their life has been in the 21st century as of today? -// Result: The capital of France is Paris. The current mayor of Paris is Anne Hidalgo, who was born on June 19, 1959. As of today, she has lived for 64 years, with 23 of those years in the 21st century. Therefore, 35.94% of her life has been spent in the 21st century. -// Steps Taken: 14 -// Skills Used: 12 (WebSearch.Search(8), time.Year(1), advancedCalculator.Calculator(3)) -// Time Taken: 00:02:06.6682909 -// ***************************************************** -// ***************************************************** -// Question: What is the current day of the calendar year? Using that as an angle in degrees, what is the area of a unit circle with that angle? -// Result: The current day of the year is 177. Using that as an angle in degrees (approximately 174.58), the area of a unit circle with that angle is approximately 1.523 square units. -// Steps Taken: 11 -// Skills Used: 9 (time.Now(1), time.DayOfYear(1), time.DaysBetween(1), time.MonthNumber(1), time.Day(1), advancedCalculator.Calculator(4)) -// Time Taken: 00:01:41.5585861 -// ***************************************************** diff --git a/dotnet/src/Extensions/Extensions.StepwisePlanner.UnitTests/.editorconfig b/dotnet/src/Extensions/Extensions.StepwisePlanner.UnitTests/.editorconfig new file mode 100644 index 000000000000..8f4c52fa9f51 --- /dev/null +++ b/dotnet/src/Extensions/Extensions.StepwisePlanner.UnitTests/.editorconfig @@ -0,0 +1,5 @@ +# Suppressing errors for Test projects under dotnet folder +[*.cs] +dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task +dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave + diff --git a/dotnet/src/Extensions/Extensions.StepwisePlanner.UnitTests/Extensions.StepwisePlanner.UnitTests.csproj b/dotnet/src/Extensions/Extensions.StepwisePlanner.UnitTests/Extensions.StepwisePlanner.UnitTests.csproj new file mode 100644 index 000000000000..46824c136d83 --- /dev/null +++ b/dotnet/src/Extensions/Extensions.StepwisePlanner.UnitTests/Extensions.StepwisePlanner.UnitTests.csproj @@ -0,0 +1,30 @@ + + + SemanticKernel.Extensions.StepwisePlanner.UnitTests + SemanticKernel.Extensions.StepwisePlanner.UnitTests + net6.0 + LatestMajor + true + enable + disable + false + CA2007,VSTHRD111 + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + \ No newline at end of file diff --git a/dotnet/src/Extensions/Extensions.UnitTests/Planning/StepwisePlanner/ParseResultTests.cs b/dotnet/src/Extensions/Extensions.StepwisePlanner.UnitTests/Planning/StepwisePlanner/ParseResultTests.cs similarity index 53% rename from dotnet/src/Extensions/Extensions.UnitTests/Planning/StepwisePlanner/ParseResultTests.cs rename to dotnet/src/Extensions/Extensions.StepwisePlanner.UnitTests/Planning/StepwisePlanner/ParseResultTests.cs index ff918c0e52c8..b68a1695dec8 100644 --- a/dotnet/src/Extensions/Extensions.UnitTests/Planning/StepwisePlanner/ParseResultTests.cs +++ b/dotnet/src/Extensions/Extensions.StepwisePlanner.UnitTests/Planning/StepwisePlanner/ParseResultTests.cs @@ -7,7 +7,7 @@ using Moq; using Xunit; -namespace SemanticKernel.Extensions.UnitTests.Planning.StepwisePlanner; +namespace SemanticKernel.Extensions.StepwisePlanner.UnitTests.Planning.StepwisePlanner; public sealed class ParseResultTests { @@ -19,6 +19,9 @@ public sealed class ParseResultTests [InlineData("I think I have everything I need.\n[FINAL ANSWER] 42\n\n", "42")] [InlineData("I think I have everything I need.\n[FINAL ANSWER]42\n\n\n", "42")] [InlineData("I think I have everything I need.\n[FINAL ANSWER]\n 42\n\n\n", "42")] + [InlineData("I think I have everything I need.\n\n[FINALANSWER]\n 42\n\n\n", "42")] + [InlineData("I think I have everything I need.\n[FINAL_ANSWER]\n 42\n\n\n", "42")] + [InlineData("I think I have everything I need.\n[FINAL-ANSWER]\n 42\n\n\n", "42")] public void WhenInputIsFinalAnswerReturnsFinalAnswer(string input, string expected) { // Arrange @@ -35,11 +38,36 @@ public void WhenInputIsFinalAnswerReturnsFinalAnswer(string input, string expect } [Theory] - [InlineData("To answer the first part of the question, I need to search for Leo DiCaprio's girlfriend on the web. To answer the second part, I need to find her current age and use a calculator to raise it to the 0.43 power.\n[ACTION]\n{\n \"action\": \"Search\",\n \"action_variables\": {\"input\": \"Leo DiCaprio's girlfriend\"}\n}", "Search", "input", "Leo DiCaprio's girlfriend")] - [InlineData("To answer the first part of the question, I need to search the web for Leo DiCaprio's girlfriend. To answer the second part, I need to find her current age and use the calculator tool to raise it to the 0.43 power.\n[ACTION]\n```\n{\n \"action\": \"Search\",\n \"action_variables\": {\"input\": \"Leo DiCaprio's girlfriend\"}\n}\n```", "Search", "input", "Leo DiCaprio's girlfriend")] - [InlineData("The web search result is a snippet from a Wikipedia article that says Leo DiCaprio's girlfriend is Camila Morrone, an Argentine-American model and actress. I need to find out her current age, which might be in the same article or another source. I can use the WebSearch.Search function again to search for her name and age.\n\n[ACTION] {\n \"action\": \"WebSearch.Search\",\n \"action_variables\": {\"input\": \"Camila Morrone age\", \"count\": \"1\"}\n}", "WebSearch.Search", "input", - "Camila Morrone age", "count", "1")] - public void ParseActionReturnsAction(string input, string expectedAction, params string[] expectedVariables) + [InlineData("To answer the first part of the question, I need to search.\n[ACTION]\n{\n \"action\": \"Search\",\n \"action_variables\": {\"input\": \"something to search\"}\n}", "To answer the first part of the question, I need to search.", "Search", "input", "something to search")] + [InlineData("To answer the first part of the question, I need to search.\n[ACTION]\n```\n{\n \"action\": \"Search\",\n \"action_variables\": {\"input\": \"something to search\"}\n}\n```", "To answer the first part of the question, I need to search.", "Search", "input", "something to search")] + [InlineData("The web search result is a snippet from a Wikipedia article that says something.\n\n[ACTION] {\n \"action\": \"WebSearch.Search\",\n \"action_variables\": {\"input\": \"another search\", \"count\": \"1\"}\n}", "The web search result is a snippet from a Wikipedia article that says something.", "WebSearch.Search", "input", + "another search", "count", "1")] + [InlineData("[ACTION] {\"action\": \"time.Year\", \"action_variables\": {\"input\": \"\"}}", null, "time.Year", "input", "")] + [InlineData(@"[ACTION]{ + ""action"": ""RepositorySkill.PushChangesToBranch"", + ""action_variables"": { + ""branchName"": ""myBranchName"", + ""comment"": ""{MyComment"" + } +} +", null, "RepositorySkill.PushChangesToBranch", "branchName", "myBranchName", "comment", "{MyComment")] + [InlineData(@"[ACTION]{ + ""action"": ""RepositorySkill.PushChangesToBranch"", + ""action_variables"": { + ""branchName"": ""myBranchName"", + ""comment"": ""}MyComment"" + } +} +", null, "RepositorySkill.PushChangesToBranch", "branchName", "myBranchName", "comment", "}MyComment")] + [InlineData(@"[ACTION]{ + ""action"": ""RepositorySkill.PushChangesToBranch"", + ""action_variables"": { + ""branchName"": ""myBranchName"", + ""comment"": ""{MyComment}"" + } +} +", null, "RepositorySkill.PushChangesToBranch", "branchName", "myBranchName", "comment", "{MyComment}")] + public void ParseActionReturnsAction(string input, string expectedThought, string expectedAction, params string[] expectedVariables) { Dictionary? expectedDictionary = null; for (int i = 0; i < expectedVariables.Length; i += 2) @@ -58,8 +86,9 @@ public void ParseActionReturnsAction(string input, string expectedAction, params var result = planner.ParseResult(input); // Assert - Assert.Equal(expectedAction, result.Action); + Assert.Equal(expectedAction ?? string.Empty, result.Action); Assert.Equal(expectedDictionary, result.ActionVariables); + Assert.Equal(expectedThought ?? string.Empty, result.Thought); } // Method to create Mock objects diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/Planning.StepwisePlanner.csproj b/dotnet/src/Extensions/Planning.StepwisePlanner/Planning.StepwisePlanner.csproj index accf224de931..a962e131e112 100644 --- a/dotnet/src/Extensions/Planning.StepwisePlanner/Planning.StepwisePlanner.csproj +++ b/dotnet/src/Extensions/Planning.StepwisePlanner/Planning.StepwisePlanner.csproj @@ -1,31 +1,29 @@ - Microsoft.SemanticKernel.Planning.StepwisePlanner Microsoft.SemanticKernel.Planning.Stepwise netstandard2.0 - - Semantic Kernel - Stepwise Planner Semantic Kernel Stepwise Planner - + - + + + Always - - + \ No newline at end of file diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/RenderFunctionManual/config.json b/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/RenderFunctionManual/config.json new file mode 100644 index 000000000000..a2044c431772 --- /dev/null +++ b/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/RenderFunctionManual/config.json @@ -0,0 +1,14 @@ +{ + "schema": 1, + "description": "Render a function manual text for the agent's functions", + "type": "completion", + "input": { + "parameters": [ + { + "name": "functionDescriptions", + "description": "The manual of the agent's functions", + "defaultValue": "" + } + ] + } +} diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/RenderFunctionManual/skprompt.txt b/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/RenderFunctionManual/skprompt.txt new file mode 100644 index 000000000000..e55ce658979e --- /dev/null +++ b/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/RenderFunctionManual/skprompt.txt @@ -0,0 +1,8 @@ +[AVAILABLE FUNCTIONS] +The function definitions below are in the following format: +: + - : + - ... + +{{$functionDescriptions}} +[END AVAILABLE FUNCTIONS] diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/RenderQuestion/config.json b/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/RenderQuestion/config.json new file mode 100644 index 000000000000..514b474d9515 --- /dev/null +++ b/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/RenderQuestion/config.json @@ -0,0 +1,14 @@ +{ + "schema": 1, + "description": "Render a plan question text for the agent", + "type": "completion", + "input": { + "parameters": [ + { + "name": "question", + "description": "", + "defaultValue": "" + } + ] + } +} diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/RenderQuestion/skprompt.txt b/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/RenderQuestion/skprompt.txt new file mode 100644 index 000000000000..b48a31717560 --- /dev/null +++ b/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/RenderQuestion/skprompt.txt @@ -0,0 +1,2 @@ +[QUESTION] +{{$question}} diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/StepwiseStep/config.json b/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/StepwiseStep/config.json index 51ef104e4597..64bd4ba62bb4 100644 --- a/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/StepwiseStep/config.json +++ b/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/StepwiseStep/config.json @@ -12,20 +12,15 @@ }, "input": { "parameters": [ - { - "name": "question", - "description": "The question to answer", - "defaultValue": "" - }, - { - "name": "agentScratchPad", - "description": "The agent's scratch pad", - "defaultValue": "" - }, { "name": "functionDescriptions", "description": "The manual of the agent's functions", "defaultValue": "" + }, + { + "name": "suffix", + "description": "", + "defaultValue": "Let's break down the problem step by step and think about the best approach. Label steps as they are taken.\n\nContinue the thought process!" } ] } diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/StepwiseStep/skprompt.txt b/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/StepwiseStep/skprompt.txt index 723b68d74c6a..5a0b5005e4c1 100644 --- a/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/StepwiseStep/skprompt.txt +++ b/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/StepwiseStep/skprompt.txt @@ -1,49 +1,43 @@ [INSTRUCTION] Answer the following questions as accurately as possible using the provided functions. -[AVAILABLE FUNCTIONS] -The function definitions below are in the following format: -: - - : - - ... - {{$functionDescriptions}} -[END AVAILABLE FUNCTIONS] - [USAGE INSTRUCTIONS] To use the functions, specify a JSON blob representing an action. The JSON blob should contain an "action" key with the name of the function to use, and an "action_variables" key with a JSON object of string values to use when calling the function. Do not call functions directly; they must be invoked through an action. -The "action_variables" value should always include an "input" key, even if the input value is empty. Additional keys in the "action_variables" value should match the defined [PARAMETERS] of the named "action" in [AVAILABLE FUNCTIONS]. -Dictionary values in "action_variables" must be strings and represent the actual values to be passed to the function. +The keys in "action_variables" value should match the defined [PARAMETERS] of the named "action" in [AVAILABLE FUNCTIONS]. +The values in "action_variables" must be of type string and represent the actual values to be passed to the function. Do not attempt to pass a variable name or other reference to a function. +If a function has no parameters, the "action_variables" key may be omitted. Ensure that the $JSON_BLOB contains only a SINGLE action; do NOT return multiple actions. IMPORTANT: Use only the available functions listed in the [AVAILABLE FUNCTIONS] section. Do not attempt to use any other functions that are not specified. Here is an example of a valid $JSON_BLOB: { "action": "FUNCTION.NAME", - "action_variables": {"INPUT": "some input", "PARAMETER_NAME": "some value", "PARAMETER_NAME_2": "42"} + "action_variables": {"PARAMETER_NAME": "some value", "PARAMETER_NAME_2": "42"} +} + +Here is an example of a valid $JSON_BLOB with no parameters: +{ + "action": "FUNCTION.NAME" } + [END USAGE INSTRUCTIONS] [END INSTRUCTION] -[THOUGHT PROCESS] -[QUESTION] -the input question I must answer -[THOUGHT] -To solve this problem, I should carefully analyze the given question and identify the necessary steps. Any facts I discover earlier in my thought process should be repeated here to keep them readily available. -[ACTION] -$JSON_BLOB -[OBSERVATION] -The result of the action will be provided here. -... (These Thought/Action/Observation can repeat until the final answer is reached.) -[FINAL ANSWER] -Once I have gathered all the necessary observations and performed any required actions, I can provide the final answer in a clear and human-readable format. -[END THOUGHT PROCESS] - -Let's break down the problem step by step and think about the best approach. Questions and observations should be followed by a single thought and an optional single action to take. - -Begin! - -[QUESTION] -{{$question}} -{{$agentScratchPad}} +[VALID STEP LIST] +[QUESTION] - The input question I must answer +[THOUGHT] - A thought I have about the question and how to answer it. +[ACTION] - A single $JSON_BLOB representing a single action to be performed +[OBSERVATION] - The result of the action will be provided here +[FINAL ANSWER] - Once I have gathered all the necessary observations through producing thoughts and actions, I can provide the final answer in a clear and human-readable format. +[END VALID STEP LIST] + +Every Question should be followed by a Thought. +Every Thought should be followed by an Action or Final Answer. +Every Action should be followed by an Observation. +Every Observation should be followed by a Thought or Final Answer. +Produce Thoughts and Actions as necessary until you have a Final Answer. + + +{{$suffix}} diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlanner.cs b/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlanner.cs index 735fb661c612..210a3874d0d3 100644 --- a/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlanner.cs +++ b/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlanner.cs @@ -3,17 +3,23 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Linq; using System.Text.Json; using System.Text.RegularExpressions; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.AI.TextCompletion; using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.Planning.Stepwise; using Microsoft.SemanticKernel.SemanticFunctions; +using Microsoft.SemanticKernel.Services; using Microsoft.SemanticKernel.SkillDefinition; +using Microsoft.SemanticKernel.TemplateEngine.Prompt; #pragma warning disable IDE0130 // ReSharper disable once CheckNamespace - Using NS of Plan @@ -33,39 +39,37 @@ public class StepwisePlanner : IStepwisePlanner /// /// The semantic kernel instance. /// Optional configuration object - /// Optional prompt override - /// Optional prompt config override public StepwisePlanner( IKernel kernel, - StepwisePlannerConfig? config = null, - string? prompt = null, - PromptTemplateConfig? promptUserConfig = null) + StepwisePlannerConfig? config = null) { Verify.NotNull(kernel); this._kernel = kernel; + // Set up Config with default values and excluded skills this.Config = config ?? new(); this.Config.ExcludedSkills.Add(RestrictedSkillName); - var promptConfig = promptUserConfig ?? new PromptTemplateConfig(); - var promptTemplate = prompt ?? EmbeddedResource.Read("Skills.StepwiseStep.skprompt.txt"); + // Set up prompt templates + this._promptTemplate = this.Config.GetPromptTemplate?.Invoke() ?? EmbeddedResource.Read("Skills.StepwiseStep.skprompt.txt"); + this._manualTemplate = EmbeddedResource.Read("Skills.RenderFunctionManual.skprompt.txt"); + this._questionTemplate = EmbeddedResource.Read("Skills.RenderQuestion.skprompt.txt"); - if (promptUserConfig == null) - { - string promptConfigString = EmbeddedResource.Read("Skills.StepwiseStep.config.json"); - if (!string.IsNullOrEmpty(promptConfigString)) - { - promptConfig = PromptTemplateConfig.FromJson(promptConfigString); - } - } + // Load or use default PromptConfig + this._promptConfig = this.Config.PromptUserConfig ?? LoadPromptConfigFromResource(); + + // Set MaxTokens for the prompt config + this._promptConfig.Completion.MaxTokens = this.Config.MaxTokens; - promptConfig.Completion.MaxTokens = this.Config.MaxTokens; + // Initialize prompt renderer + this._promptRenderer = new PromptTemplateEngine(); - this._systemStepFunction = this.ImportSemanticFunction(this._kernel, "StepwiseStep", promptTemplate, promptConfig); + // Import native functions this._nativeFunctions = this._kernel.ImportSkill(this, RestrictedSkillName); + // Create context and logger this._context = this._kernel.CreateNewContext(); - this._logger = this._kernel.LoggerFactory.CreateLogger(typeof(StepwisePlanner)); + this._logger = this._kernel.LoggerFactory.CreateLogger(this.GetType()); } /// @@ -76,16 +80,13 @@ public Plan CreatePlan(string goal) throw new SKException("The goal specified is empty"); } - string functionDescriptions = this.GetFunctionDescriptions(); - Plan planStep = new(this._nativeFunctions["ExecutePlan"]); - planStep.Parameters.Set("functionDescriptions", functionDescriptions); planStep.Parameters.Set("question", goal); - planStep.Outputs.Add("agentScratchPad"); planStep.Outputs.Add("stepCount"); planStep.Outputs.Add("skillCount"); planStep.Outputs.Add("stepsTaken"); + planStep.Outputs.Add("iterations"); Plan plan = new(goal); @@ -94,255 +95,456 @@ public Plan CreatePlan(string goal) return plan; } + /// + /// Execute a plan + /// + /// The question to answer + /// The context to use + /// The cancellation token + /// The context with the result + /// No AIService available for getting completions. [SKFunction, SKName("ExecutePlan"), Description("Execute a plan")] public async Task ExecutePlanAsync( [Description("The question to answer")] string question, - [Description("List of tool descriptions")] - string functionDescriptions, - SKContext context) + SKContext context, + CancellationToken token = default) { + if (string.IsNullOrEmpty(question)) + { + context.Variables.Update("Question not found."); + return context; + } + + ChatHistory chatHistory = await this.InitializeChatHistoryAsync(this.CreateChatHistory(this._kernel, out var aiService), aiService, context).ConfigureAwait(false); + + if (aiService is null) + { + throw new SKException("No AIService available for getting completions."); + } + + if (chatHistory is null) + { + throw new SKException("ChatHistory is null."); + } + + var startingMessageCount = chatHistory.Messages.Count; + var stepsTaken = new List(); - if (!string.IsNullOrEmpty(question)) + SystemStep? lastStep = null; + + var GetNextStepAsync = async () => { - for (int i = 0; i < this.Config.MaxIterations; i++) + var actionText = await this.GetNextStepCompletion(stepsTaken, chatHistory, aiService, startingMessageCount, token).ConfigureAwait(false); + this._logger?.LogDebug("Response: {ActionText}", actionText); + return this.ParseResult(actionText); + }; + + var TryGetFinalAnswer = (SystemStep step, int iterations, SKContext context) => + { + // If a final answer is found, update the context to be returned + if (!string.IsNullOrEmpty(step.FinalAnswer)) { - var scratchPad = this.CreateScratchPad(question, stepsTaken); + this._logger?.LogInformation("Final Answer: {FinalAnswer}", step.FinalAnswer); + + context.Variables.Update(step.FinalAnswer); + + stepsTaken.Add(step); - context.Variables.Set("agentScratchPad", scratchPad); + // Add additional results to the context + AddExecutionStatsToContext(stepsTaken, context, iterations); - var llmResponse = (await this._systemStepFunction.InvokeAsync(context).ConfigureAwait(false)); + return context; + } + + return null; + }; + + var TryGetObservations = (SystemStep step) => + { + // If no Action/Thought is found, return any already available Observation from parsing the response. + // Otherwise, add a message to the chat history to guide LLM into returning the next thought|action. + if (string.IsNullOrEmpty(step.Action) && + string.IsNullOrEmpty(step.Thought)) + { + // If there is an observation, add it to the chat history + if (!string.IsNullOrEmpty(step.Observation)) + { + this._logger?.LogWarning("Invalid response from LLM, observation: {Observation}", step.Observation); + chatHistory.AddUserMessage($"{Observation} {step.Observation}"); + stepsTaken.Add(step); + lastStep = step; + return true; + } - if (llmResponse.ErrorOccurred) + if (lastStep is not null && string.IsNullOrEmpty(lastStep.Action)) { - throw new SKException($"Error occurred while executing stepwise plan: {llmResponse.LastException?.Message}", llmResponse.LastException); + this._logger?.LogWarning("No response from LLM, expected Action"); + chatHistory.AddUserMessage(Action); + } + else + { + this._logger?.LogWarning("No response from LLM, expected Thought"); + chatHistory.AddUserMessage(Thought); } - string actionText = llmResponse.Result.Trim(); - this._logger?.LogTrace("Response: {ActionText}", actionText); + // No action or thought from LLM + return true; + } - var nextStep = this.ParseResult(actionText); - stepsTaken.Add(nextStep); + return false; + }; - if (!string.IsNullOrEmpty(nextStep.FinalAnswer)) + var AddNextStep = (SystemStep step) => + { + // If the thought is empty and the last step had no action, copy action to last step and set as new nextStep + if (string.IsNullOrEmpty(step.Thought) && lastStep is not null && string.IsNullOrEmpty(lastStep.Action)) + { + lastStep.Action = step.Action; + lastStep.ActionVariables = step.ActionVariables; + + lastStep.OriginalResponse += step.OriginalResponse; + step = lastStep; + if (chatHistory.Messages.Count > startingMessageCount) { - this._logger?.LogTrace("Final Answer: {FinalAnswer}", nextStep.FinalAnswer); + chatHistory.Messages.RemoveAt(chatHistory.Messages.Count - 1); + } + } + else + { + this._logger?.LogInformation("Thought: {Thought}", step.Thought); + stepsTaken.Add(step); + lastStep = step; + } - context.Variables.Update(nextStep.FinalAnswer); - var updatedScratchPlan = this.CreateScratchPad(question, stepsTaken); - context.Variables.Set("agentScratchPad", updatedScratchPlan); + return step; + }; - // Add additional results to the context - this.AddExecutionStatsToContext(stepsTaken, context); + var TryGetActionObservationAsync = async (SystemStep step) => + { + if (!string.IsNullOrEmpty(step.Action)) + { + this._logger?.LogInformation("Action: {Action}({ActionVariables}).", + step.Action, JsonSerializer.Serialize(step.ActionVariables)); - return context; - } + // add [thought and] action to chat history + var actionMessage = $"{Action} {{\"action\": \"{step.Action}\",\"action_variables\": {JsonSerializer.Serialize(step.ActionVariables)}}}"; + var message = string.IsNullOrEmpty(step.Thought) ? actionMessage : $"{Thought} {step.Thought}\n{actionMessage}"; - this._logger?.LogTrace("Thought: {Thought}", nextStep.Thought); + chatHistory.AddAssistantMessage(message); - if (!string.IsNullOrEmpty(nextStep!.Action!)) + // Invoke the action + try { - this._logger?.LogInformation("Action: {Action}. Iteration: {Iteration}.", nextStep.Action, i + 1); - this._logger?.LogTrace("Action: {Action}({ActionVariables}). Iteration: {Iteration}.", - nextStep.Action, JsonSerializer.Serialize(nextStep.ActionVariables), i + 1); + var result = await this.InvokeActionAsync(step.Action, step.ActionVariables).ConfigureAwait(false); - try + if (string.IsNullOrEmpty(result)) { - await Task.Delay(this.Config.MinIterationTimeMs).ConfigureAwait(false); - var result = await this.InvokeActionAsync(nextStep.Action!, nextStep!.ActionVariables!).ConfigureAwait(false); - - if (string.IsNullOrEmpty(result)) - { - nextStep.Observation = "Got no result from action"; - } - else - { - nextStep.Observation = result; - } + step.Observation = "Got no result from action"; } - catch (Exception ex) when (!ex.IsCriticalException()) + else { - nextStep.Observation = $"Error invoking action {nextStep.Action} : {ex.Message}"; - this._logger?.LogWarning(ex, "Error invoking action {Action}", nextStep.Action); + step.Observation = result; } - - this._logger?.LogTrace("Observation: {Observation}", nextStep.Observation); } - else + catch (Exception ex) when (!ex.IsCriticalException()) { - this._logger?.LogInformation("Action: No action to take"); + step.Observation = $"Error invoking action {step.Action} : {ex.Message}"; + this._logger?.LogWarning(ex, "Error invoking action {Action}", step.Action); } - // sleep 3 seconds - await Task.Delay(this.Config.MinIterationTimeMs).ConfigureAwait(false); + this._logger?.LogInformation("Observation: {Observation}", step.Observation); + chatHistory.AddUserMessage($"{Observation} {step.Observation}"); + + return true; } - context.Variables.Update($"Result not found, review _stepsTaken to see what happened.\n{JsonSerializer.Serialize(stepsTaken)}"); - } - else + return false; + }; + + var TryGetThought = (SystemStep step) => { - context.Variables.Update("Question not found."); + // Add thought to chat history + if (!string.IsNullOrEmpty(step.Thought)) + { + chatHistory.AddAssistantMessage($"{Thought} {step.Thought}"); + } + + return false; + }; + + for (int i = 0; i < this.Config.MaxIterations; i++) + { + // sleep for a bit to avoid rate limiting + if (i > 0) + { + await Task.Delay(this.Config.MinIterationTimeMs, token).ConfigureAwait(false); + } + + // Get next step from LLM + var nextStep = await GetNextStepAsync().ConfigureAwait(false); + + // If final answer is available, we're done, return the context + var finalContext = TryGetFinalAnswer(nextStep, i + 1, context); + if (finalContext is not null) + { + return finalContext; + } + + // If we have an observation before running the action, continue to the next iteration + if (TryGetObservations(nextStep)) + { + continue; + } + + // Add next step to steps taken, merging with last step if necessary + // (e.g. the LLM gave Thought and Action one at a time, merge to encourage LLM to give both at once in future steps) + nextStep = AddNextStep(nextStep); + + // Execute actions and get observations + if (await TryGetActionObservationAsync(nextStep).ConfigureAwait(false)) + { + continue; + } + + this._logger?.LogInformation("Action: No action to take"); + + // If we have a thought, continue to the next iteration + if (TryGetThought(nextStep)) + { + continue; + } } + AddExecutionStatsToContext(stepsTaken, context, this.Config.MaxIterations); + context.Variables.Update("Result not found, review 'stepsTaken' to see what happened."); + return context; } - public virtual SystemStep ParseResult(string input) + #region setup helpers + + private async Task InitializeChatHistoryAsync(ChatHistory chatHistory, IAIService aiService, SKContext context) { - var result = new SystemStep - { - OriginalResponse = input - }; + string userManual = await this.GetUserManualAsync(context).ConfigureAwait(false); + string userQuestion = await this.GetUserQuestionAsync(context).ConfigureAwait(false); - // Extract final answer - Match finalAnswerMatch = s_finalAnswerRegex.Match(input); + var systemContext = this._kernel.CreateNewContext(); - if (finalAnswerMatch.Success) - { - result.FinalAnswer = finalAnswerMatch.Groups[1].Value.Trim(); - return result; - } + systemContext.Variables.Set("suffix", this.Config.Suffix); + systemContext.Variables.Set("functionDescriptions", userManual); + string systemMessage = await this.GetSystemMessage(systemContext).ConfigureAwait(false); - // Extract thought - Match thoughtMatch = s_thoughtRegex.Match(input); + chatHistory.AddSystemMessage(systemMessage); + chatHistory.AddUserMessage(userQuestion); - if (thoughtMatch.Success) - { - result.Thought = thoughtMatch.Value.Trim(); - } - else if (!input.Contains(Action)) + return chatHistory; + } + + private ChatHistory CreateChatHistory(IKernel kernel, out IAIService aiService) + { + ChatHistory chatHistory; + if (TryGetChatCompletion(this._kernel, out var chatCompletion)) { - result.Thought = input; + chatHistory = chatCompletion.CreateNewChat(); + aiService = chatCompletion; } else { - throw new InvalidOperationException("Unexpected input format"); + var textCompletion = this._kernel.GetService(); + aiService = textCompletion; + chatHistory = new ChatHistory(); } - result.Thought = result.Thought.Replace(Thought, string.Empty).Trim(); + return chatHistory; + } - // Extract action - Match actionMatch = s_actionRegex.Match(input); + private async Task GetUserManualAsync(SKContext context) + { + var descriptions = await this.GetFunctionDescriptionsAsync().ConfigureAwait(false); + context.Variables.Set("functionDescriptions", descriptions); + return await this._promptRenderer.RenderAsync(this._manualTemplate, context).ConfigureAwait(false); + } - if (actionMatch.Success) - { - var json = actionMatch.Groups[1].Value.Trim(); + private Task GetUserQuestionAsync(SKContext context) + { + return this._promptRenderer.RenderAsync(this._questionTemplate, context); + } - try - { - var systemStepResults = JsonSerializer.Deserialize(json); + private Task GetSystemMessage(SKContext context) + { + return this._promptRenderer.RenderAsync(this._promptTemplate, context); + } - if (systemStepResults == null) - { - result.Observation = $"System step parsing error, empty JSON: {json}"; - } - else - { - result.Action = systemStepResults.Action; - result.ActionVariables = systemStepResults.ActionVariables; - } - } - catch (JsonException) + #endregion setup helpers + + #region execution helpers + + private Task GetNextStepCompletion(List stepsTaken, ChatHistory chatHistory, IAIService aiService, int startingMessageCount, CancellationToken token) + { + var tokenCount = this.GetChatHistoryTokens(chatHistory); + + var preserveFirstNSteps = 0; + var removalIndex = (startingMessageCount) + preserveFirstNSteps; + var messagesRemoved = 0; + string? originalThought = null; + while (tokenCount >= this.Config.MaxTokens && chatHistory.Messages.Count > removalIndex) + { + // something needs to be removed. + if (string.IsNullOrEmpty(originalThought)) { - result.Observation = $"System step parsing error, invalid JSON: {json}"; + originalThought = stepsTaken[0].Thought; } - } - if (string.IsNullOrEmpty(result.Thought) && string.IsNullOrEmpty(result.Action)) - { - result.Observation = "System step error, no thought or action found. Please give a valid thought and/or action."; + // Update message history + chatHistory.AddAssistantMessage($"{Thought} {originalThought}"); + preserveFirstNSteps++; + chatHistory.AddAssistantMessage("... I've removed some of my previous work to make room for the new stuff ..."); + preserveFirstNSteps++; + + removalIndex = (startingMessageCount) + preserveFirstNSteps; + + chatHistory.Messages.RemoveAt(removalIndex); + tokenCount = this.GetChatHistoryTokens(chatHistory); + messagesRemoved++; } - return result; + return this.GetCompletionAsync(aiService, chatHistory, stepsTaken.Count == 0, token); } - private void AddExecutionStatsToContext(List stepsTaken, SKContext context) + private async Task GetCompletionAsync(IAIService aiService, ChatHistory chatHistory, bool addThought, CancellationToken token) { - context.Variables.Set("stepCount", stepsTaken.Count.ToString(CultureInfo.InvariantCulture)); - context.Variables.Set("stepsTaken", JsonSerializer.Serialize(stepsTaken)); - - Dictionary actionCounts = new(); - foreach (var step in stepsTaken) + if (aiService is IChatCompletion chatCompletion) { - if (string.IsNullOrEmpty(step.Action)) { continue; } - - _ = actionCounts.TryGetValue(step.Action!, out int currentCount); - actionCounts[step.Action!] = ++currentCount; + var llmResponse = (await chatCompletion.GenerateMessageAsync(chatHistory, ChatRequestSettings.FromCompletionConfig(this._promptConfig.Completion), token).ConfigureAwait(false)); + return llmResponse; } + else if (aiService is ITextCompletion textCompletion) + { + var thoughtProcess = string.Join("\n", chatHistory.Messages.Select(m => m.Content)); - var skillCallListWithCounts = string.Join(", ", actionCounts.Keys.Select(skill => - $"{skill}({actionCounts[skill]})")); + // Add Thought to the thought process at the start of the first iteration + if (addThought) + { + thoughtProcess = $"{thoughtProcess}\n{Thought}"; + addThought = false; + } - var skillCallCountStr = actionCounts.Values.Sum().ToString(CultureInfo.InvariantCulture); + thoughtProcess = $"{thoughtProcess}\n"; + var results = (await textCompletion.GetCompletionsAsync(thoughtProcess, CompleteRequestSettings.FromCompletionConfig(this._promptConfig.Completion), token).ConfigureAwait(false)); - context.Variables.Set("skillCount", $"{skillCallCountStr} ({skillCallListWithCounts})"); + if (results.Count == 0) + { + throw new SKException("No completions returned."); + } + + return await results[0].GetCompletionAsync(token).ConfigureAwait(false); + } + + throw new SKException("No AIService available for getting completions."); } - private string CreateScratchPad(string question, List stepsTaken) + private int GetChatHistoryTokens(ChatHistory chatHistory) { - if (stepsTaken.Count == 0) + var messages = string.Join("\n", chatHistory.Messages); + var tokenCount = messages.Length / 4; + return tokenCount; + } + + /// + /// Parse LLM response into a SystemStep during execution + /// + /// The response from the LLM + /// A SystemStep + protected internal virtual SystemStep ParseResult(string input) + { + var result = new SystemStep { - return string.Empty; - } + OriginalResponse = input + }; - var scratchPadLines = new List(); + // Extract final answer + Match finalAnswerMatch = s_finalAnswerRegex.Match(input); - // Add the original first thought - scratchPadLines.Add(ScratchPadPrefix); - scratchPadLines.Add($"{Thought} {stepsTaken[0].Thought}"); + if (finalAnswerMatch.Success) + { + result.FinalAnswer = finalAnswerMatch.Groups[1].Value.Trim(); + return result; + } - // Keep track of where to insert the next step - var insertPoint = scratchPadLines.Count; + // Extract thought + Match thoughtMatch = s_thoughtRegex.Match(input); - // Keep the most recent steps in the scratch pad. - for (var i = stepsTaken.Count - 1; i >= 0; i--) + if (thoughtMatch.Success) { - if (scratchPadLines.Count / 4.0 > (this.Config.MaxTokens * 0.75)) + // if it contains Action, it was only an action + if (!thoughtMatch.Value.Contains(Action)) { - this._logger.LogDebug("Scratchpad is too long, truncating. Skipping {CountSkipped} steps.", i + 1); - break; + result.Thought = thoughtMatch.Value.Trim(); } + } + else if (!input.Contains(Action)) + { + result.Thought = input; + } + else + { + return result; + } - var s = stepsTaken[i]; - - if (!string.IsNullOrEmpty(s.Observation)) - { - scratchPadLines.Insert(insertPoint, $"{Observation} {s.Observation}"); - } + result.Thought = result.Thought.Replace(Thought, string.Empty).Trim(); - if (!string.IsNullOrEmpty(s.Action)) - { - scratchPadLines.Insert(insertPoint, $"{Action} {{\"action\": \"{s.Action}\",\"action_variables\": {JsonSerializer.Serialize(s.ActionVariables)}}}"); - } + // Extract action + // Using regex is prone to issues with complex action json, so we use a simple string search instead + // This can be less fault tolerant in some scenarios where the LLM tries to call multiple actions, for example. + // TODO -- that could possibly be improved if we allow an action to be a list of actions. + int actionIndex = input.IndexOf(Action, StringComparison.OrdinalIgnoreCase); - if (i != 0) + if (actionIndex != -1) + { + int jsonStartIndex = input.IndexOf("{", actionIndex, StringComparison.OrdinalIgnoreCase); + if (jsonStartIndex != -1) { - scratchPadLines.Insert(insertPoint, $"{Thought} {s.Thought}"); - } - } + int jsonEndIndex = input.Substring(jsonStartIndex).LastIndexOf("}", StringComparison.OrdinalIgnoreCase); + if (jsonEndIndex != -1) + { + string json = input.Substring(jsonStartIndex, jsonEndIndex + 1); - var scratchPad = string.Join("\n", scratchPadLines).Trim(); + try + { + var systemStepResults = JsonSerializer.Deserialize(json); - if (!string.IsNullOrWhiteSpace(scratchPad)) - { - this._logger.LogTrace("Scratchpad: {ScratchPad}", scratchPad); + if (systemStepResults is not null) + { + result.Action = systemStepResults.Action; + result.ActionVariables = systemStepResults.ActionVariables; + } + } + catch (JsonException je) + { + result.Observation = $"Action parsing error: {je.Message}\nInvalid action: {json}"; + } + } + } } - return scratchPad; + return result; } private async Task InvokeActionAsync(string actionName, Dictionary actionVariables) { - var availableFunctions = this.GetAvailableFunctions(); + var availableFunctions = await this.GetAvailableFunctionsAsync().ConfigureAwait(false); var targetFunction = availableFunctions.FirstOrDefault(f => ToFullyQualifiedName(f) == actionName); if (targetFunction == null) { - throw new SKException($"The function '{actionName}' was not found."); + this._logger?.LogDebug("Attempt to invoke action {Action} failed", actionName); + return $"{actionName} is not in [AVAILABLE FUNCTIONS]. Please try again using one of the [AVAILABLE FUNCTIONS]."; } try { - var function = this._kernel.Func(targetFunction.SkillName, targetFunction.Name); + ISKFunction function = this.GetFunction(targetFunction); + var actionContext = this.CreateActionContext(actionVariables); var result = await function.InvokeAsync(actionContext).ConfigureAwait(false); @@ -364,6 +566,48 @@ private async Task InvokeActionAsync(string actionName, Dictionary + { + return this._kernel.Func(skillName, functionName); + }; + var getSkillFunction = this.Config.GetSkillFunction ?? getFunction; + var function = getSkillFunction(targetFunction.SkillName, targetFunction.Name); + return function; + } + + private async Task GetFunctionDescriptionsAsync() + { + // Use configured function provider if available, otherwise use the default SKContext function provider. + var availableFunctions = await this.GetAvailableFunctionsAsync().ConfigureAwait(false); + var functionDescriptions = string.Join("\n\n", availableFunctions.Select(x => ToManualString(x))); + return functionDescriptions; + } + + private Task> GetAvailableFunctionsAsync() + { + if (this.Config.GetAvailableFunctionsAsync is null) + { + FunctionsView functionsView = this._context.Skills!.GetFunctionsView(); + + var excludedSkills = this.Config.ExcludedSkills ?? new(); + var excludedFunctions = this.Config.ExcludedFunctions ?? new(); + + var availableFunctions = + functionsView.NativeFunctions + .Concat(functionsView.SemanticFunctions) + .SelectMany(x => x.Value) + .Where(s => !excludedSkills.Contains(s.SkillName) && !excludedFunctions.Contains(s.Name)) + .OrderBy(x => x.SkillName) + .ThenBy(x => x.Name); + + return Task.FromResult(availableFunctions); + } + + return this.Config.GetAvailableFunctionsAsync(this.Config, null, CancellationToken.None); + } + private SKContext CreateActionContext(Dictionary actionVariables) { var actionContext = this._kernel.CreateNewContext(); @@ -378,37 +622,52 @@ private SKContext CreateActionContext(Dictionary actionVariables return actionContext; } - private IEnumerable GetAvailableFunctions() + #endregion execution helpers + + private static PromptTemplateConfig LoadPromptConfigFromResource() { - FunctionsView functionsView = this._context.Skills!.GetFunctionsView(); - - var excludedSkills = this.Config.ExcludedSkills ?? new(); - var excludedFunctions = this.Config.ExcludedFunctions ?? new(); - - var availableFunctions = - functionsView.NativeFunctions - .Concat(functionsView.SemanticFunctions) - .SelectMany(x => x.Value) - .Where(s => !excludedSkills.Contains(s.SkillName) && !excludedFunctions.Contains(s.Name)) - .OrderBy(x => x.SkillName) - .ThenBy(x => x.Name); - return availableFunctions; + string promptConfigString = EmbeddedResource.Read("Skills.StepwiseStep.config.json"); + return !string.IsNullOrEmpty(promptConfigString) ? PromptTemplateConfig.FromJson(promptConfigString) : new PromptTemplateConfig(); } - private string GetFunctionDescriptions() + private static bool TryGetChatCompletion(IKernel kernel, [NotNullWhen(true)] out IChatCompletion? chatCompletion) { - var availableFunctions = this.GetAvailableFunctions(); + try + { + // Client used to request answers to chat completion models + // TODO #2635 - Using TryGetService would improve cost of this method to avoid exception handling + chatCompletion = kernel.GetService(); + return true; + } + catch (SKException) + { + chatCompletion = null; + } - string functionDescriptions = string.Join("\n", availableFunctions.Select(x => ToManualString(x))); - return functionDescriptions; + return false; } - private ISKFunction ImportSemanticFunction(IKernel kernel, string functionName, string promptTemplate, PromptTemplateConfig config) + private static void AddExecutionStatsToContext(List stepsTaken, SKContext context, int iterations) { - var template = new PromptTemplate(promptTemplate, config, kernel.PromptTemplateEngine); - var functionConfig = new SemanticFunctionConfig(config, template); + context.Variables.Set("stepCount", stepsTaken.Count.ToString(CultureInfo.InvariantCulture)); + context.Variables.Set("stepsTaken", JsonSerializer.Serialize(stepsTaken)); + context.Variables.Set("iterations", iterations.ToString(CultureInfo.InvariantCulture)); - return kernel.RegisterSemanticFunction(RestrictedSkillName, functionName, functionConfig); + Dictionary actionCounts = new(); + foreach (var step in stepsTaken) + { + if (string.IsNullOrEmpty(step.Action)) { continue; } + + _ = actionCounts.TryGetValue(step.Action, out int currentCount); + actionCounts[step.Action!] = ++currentCount; + } + + var skillCallListWithCounts = string.Join(", ", actionCounts.Keys.Select(skill => + $"{skill}({actionCounts[skill]})")); + + var skillCallCountStr = actionCounts.Values.Sum().ToString(CultureInfo.InvariantCulture); + + context.Variables.Set("skillCount", $"{skillCallCountStr} ({skillCallListWithCounts})"); } private static string ToManualString(FunctionView function) @@ -434,6 +693,8 @@ private static string ToFullyQualifiedName(FunctionView function) return $"{function.SkillName}.{function.Name}"; } + #region private + /// /// The configuration for the StepwisePlanner /// @@ -442,7 +703,7 @@ private static string ToFullyQualifiedName(FunctionView function) // Context used to access the list of functions in the kernel private readonly SKContext _context; private readonly IKernel _kernel; - private readonly ILogger _logger; + private readonly ILogger? _logger; /// /// Planner native functions @@ -450,9 +711,29 @@ private static string ToFullyQualifiedName(FunctionView function) private IDictionary _nativeFunctions = new Dictionary(); /// - /// System step function for Plan execution + /// The prompt template to use for the system step + /// + private string _promptTemplate; + + /// + /// The question template to use for the system step /// - private ISKFunction _systemStepFunction; + private string _questionTemplate; + + /// + /// The function manual template to use for the system step + /// + private string _manualTemplate; + + /// + /// The prompt renderer to use for the system step + /// + private PromptTemplateEngine _promptRenderer; + + /// + /// The prompt config to use for the system step + /// + private PromptTemplateConfig _promptConfig; /// /// The name to use when creating semantic functions that are restricted from plan creation @@ -474,23 +755,15 @@ private static string ToFullyQualifiedName(FunctionView function) /// private const string Observation = "[OBSERVATION]"; - /// - /// The prefix used for the scratch pad - /// - private const string ScratchPadPrefix = "This was my previous work (but they haven't seen any of it! They only see what I return as final answer):"; - - /// - /// The regex for parsing the action response - /// - private static readonly Regex s_actionRegex = new(@"\[ACTION\][^{}]*({(?:[^{}]*{[^{}]*})*[^{}]*})", RegexOptions.Singleline); - /// /// The regex for parsing the thought response /// - private static readonly Regex s_thoughtRegex = new(@"(\[THOUGHT\])?(?.+?)(?=\[ACTION\]|$)", RegexOptions.Singleline); + private static readonly Regex s_thoughtRegex = new(@"(\[THOUGHT\])?(?.+?)(?=\[ACTION\]|$)", RegexOptions.Singleline | RegexOptions.IgnoreCase); /// /// The regex for parsing the final answer response /// - private static readonly Regex s_finalAnswerRegex = new(@"\[FINAL ANSWER\](?.+)", RegexOptions.Singleline); + private static readonly Regex s_finalAnswerRegex = new(@"\[FINAL[_\s\-]?ANSWER\](?.+)", RegexOptions.Singleline | RegexOptions.IgnoreCase); + + #endregion private } diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlannerConfig.cs b/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlannerConfig.cs index c50a1dbec2e1..be95384971f8 100644 --- a/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlannerConfig.cs +++ b/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlannerConfig.cs @@ -1,6 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.SemanticFunctions; +using Microsoft.SemanticKernel.SkillDefinition; namespace Microsoft.SemanticKernel.Planning.Stepwise; @@ -9,26 +15,7 @@ namespace Microsoft.SemanticKernel.Planning.Stepwise; /// public sealed class StepwisePlannerConfig { - /// - /// The minimum relevancy score for a function to be considered - /// - /// - /// Depending on the embeddings engine used, the user ask, the step goal - /// and the functions available, this value may need to be adjusted. - /// For default, this is set to null to exhibit previous behavior. - /// - public double? RelevancyThreshold { get; set; } - - /// - /// The maximum number of relevant functions to include in the plan. - /// - /// - /// Limits the number of relevant functions as result of semantic - /// search included in the plan creation request. - /// will be included - /// in the plan regardless of this limit. - /// - public int MaxRelevantFunctions { get; set; } = 100; + #region Use these to configure which functions to include/exclude /// /// A list of skills to exclude from the plan creation request. @@ -45,6 +32,24 @@ public sealed class StepwisePlannerConfig /// public HashSet IncludedFunctions { get; } = new(); + #endregion Use these to configure which functions to include/exclude + + #region Use these to completely override the functions available for planning + + /// + /// Optional callback to get the available functions for planning. + /// + public Func>>? GetAvailableFunctionsAsync { get; set; } + + /// + /// Optional callback to get a function by name. + /// + public Func? GetSkillFunction { get; set; } + + #endregion Use these to completely override the functions available for planning + + #region Execution configuration + /// /// The maximum number of tokens to allow in a plan. /// @@ -53,10 +58,29 @@ public sealed class StepwisePlannerConfig /// /// The maximum number of iterations to allow in a plan. /// - public int MaxIterations { get; set; } = 100; + public int MaxIterations { get; set; } = 15; /// /// The minimum time to wait between iterations in milliseconds. /// public int MinIterationTimeMs { get; set; } = 0; + + /// + /// Delegate to get the prompt template string. + /// + public Func? GetPromptTemplate { get; set; } = null; + + /// + /// The configuration to use for the prompt template. + /// + public PromptTemplateConfig? PromptUserConfig { get; set; } = null; + + /// + /// A suffix to use within the default prompt template. + /// + public string Suffix { get; set; } = @"Let's break down the problem step by step and think about the best approach. Label steps as they are taken. + +Continue the thought process!"; + + #endregion Execution configuration } diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/SystemStep.cs b/dotnet/src/Extensions/Planning.StepwisePlanner/SystemStep.cs index 3fc8ed2dffd3..eae27a1a3cde 100644 --- a/dotnet/src/Extensions/Planning.StepwisePlanner/SystemStep.cs +++ b/dotnet/src/Extensions/Planning.StepwisePlanner/SystemStep.cs @@ -14,35 +14,35 @@ public class SystemStep /// Gets or sets the step number. /// [JsonPropertyName("thought")] - public string? Thought { get; set; } + public string Thought { get; set; } = string.Empty; /// /// Gets or sets the action of the step /// [JsonPropertyName("action")] - public string? Action { get; set; } + public string Action { get; set; } = string.Empty; /// /// Gets or sets the variables for the action /// [JsonPropertyName("action_variables")] - public Dictionary? ActionVariables { get; set; } + public Dictionary ActionVariables { get; set; } = new(); /// /// Gets or sets the output of the action /// [JsonPropertyName("observation")] - public string? Observation { get; set; } + public string Observation { get; set; } = string.Empty; /// /// Gets or sets the output of the system /// [JsonPropertyName("final_answer")] - public string? FinalAnswer { get; set; } + public string FinalAnswer { get; set; } = string.Empty; /// /// The raw response from the action /// [JsonPropertyName("original_response")] - public string? OriginalResponse { get; set; } + public string OriginalResponse { get; set; } = string.Empty; } diff --git a/dotnet/src/IntegrationTests/Planning/StepwisePlanner/StepwisePlannerTests.cs b/dotnet/src/IntegrationTests/Planning/StepwisePlanner/StepwisePlannerTests.cs index 17a0cb63ede0..e8b9badb341e 100644 --- a/dotnet/src/IntegrationTests/Planning/StepwisePlanner/StepwisePlannerTests.cs +++ b/dotnet/src/IntegrationTests/Planning/StepwisePlanner/StepwisePlannerTests.cs @@ -68,8 +68,10 @@ public void CanCreateStepwisePlan(bool useChatModel, string prompt, string expec [Theory] [InlineData(false, "What is the tallest mountain on Earth? How tall is it divided by 2", "Everest")] - // [InlineData(true, "What is the tallest mountain on Earth? How tall is it divided by 2")] // Chat tests take long - public async void CanExecuteStepwisePlan(bool useChatModel, string prompt, string partialExpectedAnswer) + [InlineData(true, "What is the tallest mountain on Earth? How tall is it divided by 2", "Everest")] + [InlineData(false, "What is the weather in Seattle?", "Seattle", 1)] + [InlineData(true, "What is the weather in Seattle?", "Seattle", 1)] + public async void CanExecuteStepwisePlan(bool useChatModel, string prompt, string partialExpectedAnswer, int expectedMinSteps = 1) { // Arrange bool useEmbeddings = false; @@ -91,7 +93,7 @@ public async void CanExecuteStepwisePlan(bool useChatModel, string prompt, strin Assert.True(result.Variables.TryGetValue("stepsTaken", out string? stepsTakenString)); var stepsTaken = JsonSerializer.Deserialize>(stepsTakenString!); Assert.NotNull(stepsTaken); - Assert.True(stepsTaken.Count >= 3 && stepsTaken.Count <= 10, $"Actual: {stepsTaken.Count}. Expected at least 3 steps and at most 10 steps to be taken."); + Assert.True(stepsTaken.Count >= expectedMinSteps && stepsTaken.Count <= 10, $"Actual: {stepsTaken.Count}. Expected at least {expectedMinSteps} steps and at most 10 steps to be taken."); } private IKernel InitializeKernel(bool useEmbeddings = false, bool useChatModel = false) diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatRequestSettings.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatRequestSettings.cs index c74373e2f332..7060bd2f8f9c 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatRequestSettings.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatRequestSettings.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using Microsoft.SemanticKernel.SemanticFunctions; namespace Microsoft.SemanticKernel.AI.ChatCompletion; @@ -58,4 +59,22 @@ public class ChatRequestSettings /// Modify the likelihood of specified tokens appearing in the completion. /// public IDictionary TokenSelectionBiases { get; set; } = new Dictionary(); + + /// + /// Create a new settings object with the values from another settings object. + /// + /// + /// An instance of + public static ChatRequestSettings FromCompletionConfig(PromptTemplateConfig.CompletionConfig config) + { + return new ChatRequestSettings + { + Temperature = config.Temperature, + TopP = config.TopP, + PresencePenalty = config.PresencePenalty, + FrequencyPenalty = config.FrequencyPenalty, + MaxTokens = config.MaxTokens, + StopSequences = config.StopSequences, + }; + } } diff --git a/dotnet/src/SemanticKernel.MetaPackage/SemanticKernel.MetaPackage.csproj b/dotnet/src/SemanticKernel.MetaPackage/SemanticKernel.MetaPackage.csproj index f115ea1f2bf7..f030dcf527cd 100644 --- a/dotnet/src/SemanticKernel.MetaPackage/SemanticKernel.MetaPackage.csproj +++ b/dotnet/src/SemanticKernel.MetaPackage/SemanticKernel.MetaPackage.csproj @@ -1,20 +1,16 @@  - Microsoft.SemanticKernel $(AssemblyName) netstandard2.0 - - Semantic Kernel Semantic Kernel common package collection, including SK Core, OpenAI, Azure OpenAI, DALL-E 2. Empowers app owners to integrate cutting-edge LLM technology quickly and easily into their apps. - @@ -23,6 +19,6 @@ Empowers app owners to integrate cutting-edge LLM technology quickly and easily + - - + \ No newline at end of file diff --git a/dotnet/src/Skills/Skills.Web/WebSearchEngineSkill.cs b/dotnet/src/Skills/Skills.Web/WebSearchEngineSkill.cs index ce55bb5356f3..b655e6c2cd84 100644 --- a/dotnet/src/Skills/Skills.Web/WebSearchEngineSkill.cs +++ b/dotnet/src/Skills/Skills.Web/WebSearchEngineSkill.cs @@ -27,8 +27,8 @@ public WebSearchEngineSkill(IWebSearchEngineConnector connector) [SKFunction, Description("Perform a web search.")] public async Task SearchAsync( - [Description("Text to search for")] string query, - [Description("Number of results")] int count = 1, + [Description("Search query")] string query, + [Description("Number of results")] int count = 10, [Description("Number of results to skip")] int offset = 0, CancellationToken cancellationToken = default) {