Add examples for structured outputs and tool use (#172)

---------

Co-authored-by: Bruce MacDonald <[email protected]>

examples/structured_outputs/structured-outputs-image.ts

@@ -0,0 +1,83 @@
+import ollama from 'ollama';
+
+import { z } from 'zod';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+import { readFileSync } from 'fs';
+import { resolve } from 'path';
+import { createInterface } from 'readline';
+
+/*
+    Ollama vision capabilities with structured outputs
+    It takes an image file as input and returns a structured JSON description of the image contents
+    including detected objects, scene analysis, colors, and any text found in the image
+*/
+
+// Schema for individual objects detected in the image
+const ObjectSchema = z.object({
+    name: z.string().describe('The name of the object'),
+    confidence: z.number().min(0).max(1).describe('The confidence score of the object detection'),
+    attributes: z.record(z.any()).optional().describe('Additional attributes of the object')
+});
+
+// Schema for individual objects detected in the image
+const ImageDescriptionSchema = z.object({
+    summary: z.string().describe('A concise summary of the image'),
+    objects: z.array(ObjectSchema).describe('An array of objects detected in the image'),
+    scene: z.string().describe('The scene of the image'),
+    colors: z.array(z.string()).describe('An array of colors detected in the image'),
+    time_of_day: z.enum(['Morning', 'Afternoon', 'Evening', 'Night']).describe('The time of day the image was taken'),
+    setting: z.enum(['Indoor', 'Outdoor', 'Unknown']).describe('The setting of the image'),
+    text_content: z.string().describe('Any text detected in the image')
+});
+
+async function run(model: string) {
+    // Create readline interface for user input
+    const rl = createInterface({
+        input: process.stdin,
+        output: process.stdout
+    });
+
+    // Get path from user input
+    const path = await new Promise<string>(resolve => {
+        rl.question('Enter the path to your image: ', resolve);
+    });
+    rl.close();
+
+    // Verify the file exists and read it
+    try {
+        const imagePath = resolve(path);
+        const imageBuffer = readFileSync(imagePath);
+        const base64Image = imageBuffer.toString('base64');
+
+        // Convert the Zod schema to JSON Schema format
+        const jsonSchema = zodToJsonSchema(ImageDescriptionSchema);
+
+        const messages = [{
+            role: 'user',
+            content: 'Analyze this image and return a detailed JSON description including objects, scene, colors and any text detected. If you cannot determine certain details, leave those fields empty.',
+            images: [base64Image]
+        }];
+
+        const response = await ollama.chat({
+            model: model,
+            messages: messages,
+            format: jsonSchema,
+            options: {
+                temperature: 0 // Make responses more deterministic
+            }
+        });
+
+        // Parse and validate the response
+        try {
+            const imageAnalysis = ImageDescriptionSchema.parse(JSON.parse(response.message.content));
+            console.log('Image Analysis:', imageAnalysis);
+        } catch (error) {
+            console.error("Generated invalid response:", error);
+        }
+
+    } catch (error) {
+        console.error("Error reading image file:", error);
+    }
+}
+
+run('llama3.2-vision').catch(console.error);

examples/structured_outputs/structured-outputs.ts

@@ -0,0 +1,71 @@
+import ollama from 'ollama';
+
+import { z } from 'zod';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+
+/*
+    Ollama structured outputs capabilities
+    It parses the response from the model into a structured JSON object using Zod
+*/
+
+// Define the schema for friend info
+const FriendInfoSchema = z.object({
+    name: z.string().describe('The name of the friend'),
+    age: z.number().int().describe('The age of the friend'),
+    is_available: z.boolean().describe('Whether the friend is available')
+});
+
+// Define the schema for friend list
+const FriendListSchema = z.object({
+    friends: z.array(FriendInfoSchema).describe('An array of friends')
+});
+
+async function run(model: string) {
+    // Convert the Zod schema to JSON Schema format
+    const jsonSchema = zodToJsonSchema(FriendListSchema);
+
+    /* Can use manually defined schema directly
+    const schema = { 
+        'type': 'object', 
+        'properties': { 
+            'friends': { 
+                'type': 'array', 
+                'items': { 
+                    'type': 'object', 
+                    'properties': { 
+                        'name': { 'type': 'string' }, 
+                        'age': { 'type': 'integer' }, 
+                        'is_available': { 'type': 'boolean' } 
+                    }, 
+                    'required': ['name', 'age', 'is_available'] 
+                } 
+            } 
+        }, 
+        'required': ['friends'] 
+    }
+    */
+
+    const messages = [{
+        role: 'user',
+        content: 'I have two friends. The first is Ollama 22 years old busy saving the world, and the second is Alonso 23 years old and wants to hang out. Return a list of friends in JSON format'
+    }];
+
+    const response = await ollama.chat({
+        model: model,
+        messages: messages,
+        format: jsonSchema, // or format: schema
+        options: {
+            temperature: 0 // Make responses more deterministic
+        }
+    });
+
+    // Parse and validate the response
+    try {
+        const friendsResponse = FriendListSchema.parse(JSON.parse(response.message.content));
+        console.log(friendsResponse);
+    } catch (error) {
+        console.error("Generated invalid response:", error);
+    }
+}
+
+run('llama3.1:8b').catch(console.error);

examples/tools/calculator.ts

@@ -0,0 +1,95 @@
+import ollama from 'ollama';
+
+// Add two numbers function
+function addTwoNumbers(args: { a: number, b: number }): number {
+    return args.a + args.b;
+}
+
+// Subtract two numbers function 
+function subtractTwoNumbers(args: { a: number, b: number }): number {
+    return args.a - args.b;
+}
+
+// Tool definition for add function
+const addTwoNumbersTool = {
+    type: 'function',
+    function: {
+        name: 'addTwoNumbers',
+        description: 'Add two numbers together',
+        parameters: {
+            type: 'object',
+            required: ['a', 'b'],
+            properties: {
+                a: { type: 'number', description: 'The first number' },
+                b: { type: 'number', description: 'The second number' }
+            }
+        }
+    }
+};
+
+// Tool definition for subtract function
+const subtractTwoNumbersTool = {
+    type: 'function',
+    function: {
+        name: 'subtractTwoNumbers',
+        description: 'Subtract two numbers',
+        parameters: {
+            type: 'object',
+            required: ['a', 'b'],
+            properties: {
+                a: { type: 'number', description: 'The first number' },
+                b: { type: 'number', description: 'The second number' }
+            }
+        }
+    }
+};
+
+async function run(model: string) {
+    const messages = [{ role: 'user', content: 'What is three minus one?' }];
+    console.log('Prompt:', messages[0].content);
+
+    const availableFunctions = {
+        addTwoNumbers: addTwoNumbers,
+        subtractTwoNumbers: subtractTwoNumbers
+    };
+
+    const response = await ollama.chat({
+        model: model,
+        messages: messages,
+        tools: [addTwoNumbersTool, subtractTwoNumbersTool]
+    });
+
+    let output: number;
+    if (response.message.tool_calls) {
+        // Process tool calls from the response
+        for (const tool of response.message.tool_calls) {
+            const functionToCall = availableFunctions[tool.function.name];
+            if (functionToCall) {
+                console.log('Calling function:', tool.function.name);
+                console.log('Arguments:', tool.function.arguments);
+                output = functionToCall(tool.function.arguments);
+                console.log('Function output:', output);
+
+                // Add the function response to messages for the model to use
+                messages.push(response.message);
+                messages.push({
+                    role: 'tool',
+                    content: output.toString(),
+                });
+            } else {
+                console.log('Function', tool.function.name, 'not found');
+            }
+        }
+
+        // Get final response from model with function outputs
+        const finalResponse = await ollama.chat({
+            model: model,
+            messages: messages
+        });
+        console.log('Final response:', finalResponse.message.content);
+    } else {
+        console.log('No tool calls returned from model');
+    }
+}
+
+run('llama3.1:8b').catch(error => console.error("An error occurred:", error));

examples/tools/tools.ts → examples/tools/flight-tracker.ts

@@ -70,6 +70,7 @@ async function run(model: string) {
         for (const tool of response.message.tool_calls) {
             const functionToCall = availableFunctions[tool.function.name];
             const functionResponse = functionToCall(tool.function.arguments);
+            console.log('functionResponse', functionResponse)
             // Add function response to the conversation
             messages.push({
                 role: 'tool',