From 55269fbfb03ec0c04fef46fd5961b4b4532f5ec2 Mon Sep 17 00:00:00 2001 From: Parth Sareen Date: Tue, 5 Nov 2024 14:48:08 -0800 Subject: [PATCH 1/3] Clarified aborting examples and readme --- README.md | 6 ++- examples/abort/abort-all-requests.ts | 55 ++++++++++++++++++++++++++++ examples/abort/any-request.ts | 27 -------------- examples/abort/specific-request.ts | 31 ---------------- 4 files changed, 59 insertions(+), 60 deletions(-) create mode 100644 examples/abort/abort-all-requests.ts delete mode 100644 examples/abort/any-request.ts delete mode 100644 examples/abort/specific-request.ts diff --git a/README.md b/README.md index 2aede53..af522cc 100644 --- a/README.md +++ b/README.md @@ -204,8 +204,10 @@ ollama.ps() ollama.abort() ``` -This method will abort all streamed generations currently running. -All asynchronous threads listening to streams (typically the ```for await (const part of response)```) will throw an ```AbortError``` exception +This method will abort **all** streamed generations currently running with the client instance. +If there is a need to manage streams with timeouts, it is recommended to have one Ollama client per stream. + +All asynchronous threads listening to streams (typically the ```for await (const part of response)```) will throw an ```AbortError``` exception. See [examples/abort/abort-all-requests.ts](examples/abort/abort-all-requests.ts) for an example. ## Custom client diff --git a/examples/abort/abort-all-requests.ts b/examples/abort/abort-all-requests.ts new file mode 100644 index 0000000..794f29b --- /dev/null +++ b/examples/abort/abort-all-requests.ts @@ -0,0 +1,55 @@ +import ollama from 'ollama' + +// Set a timeout to abort all requests after 1 second +setTimeout(() => { + console.log('\nAborting all requests...\n') + ollama.abort() +}, 1000) // 1000 milliseconds = 1 second + +// Start multiple concurrent streaming requests +Promise.all([ + ollama.generate({ + model: 'llama3.2', + prompt: 'Write a long story about dragons', + stream: true, + }).then( + async (stream) => { + console.log(' Starting stream for dragons story...') + for await (const chunk of stream) { + process.stdout.write(' 1> ' + chunk.response) + } + } + ), + + ollama.generate({ + model: 'llama3.2', + prompt: 'Write a long story about wizards', + stream: true, + }).then( + async (stream) => { + console.log(' Starting stream for wizards story...') + for await (const chunk of stream) { + process.stdout.write(' 2> ' + chunk.response) + } + } + ), + + ollama.generate({ + model: 'llama3.2', + prompt: 'Write a long story about knights', + stream: true, + }).then( + async (stream) => { + console.log(' Starting stream for knights story...') + for await (const chunk of stream) { + process.stdout.write(' 3>' + chunk.response) + } + } + ) +]).catch(error => { + if (error.name === 'AbortError') { + console.log('All requests have been aborted') + } else { + console.error('An error occurred:', error) + } +}) diff --git a/examples/abort/any-request.ts b/examples/abort/any-request.ts deleted file mode 100644 index a553ffb..0000000 --- a/examples/abort/any-request.ts +++ /dev/null @@ -1,27 +0,0 @@ -import ollama from 'ollama' - -// Set a timeout to abort the request after 1 second -setTimeout(() => { - console.log('\nAborting request...\n') - ollama.abort() -}, 1000) // 1000 milliseconds = 1 second - -ollama.generate({ - model: 'llama3.1', - prompt: 'Write a long story', - stream: true, - }).then( - async (stream) => { - for await (const chunk of stream) { - process.stdout.write(chunk.response) - } - } - ).catch( - (error) => { - if (error.name === 'AbortError') { - console.log('The request has been aborted') - } else { - console.error('An error occurred:', error) - } - } - ) diff --git a/examples/abort/specific-request.ts b/examples/abort/specific-request.ts deleted file mode 100644 index 778c514..0000000 --- a/examples/abort/specific-request.ts +++ /dev/null @@ -1,31 +0,0 @@ -import ollama from 'ollama' -import { AbortableAsyncIterator } from '../../src/utils' - -let stream: AbortableAsyncIterator - -// Set a timeout to abort the request after 1 second -setTimeout(() => { - console.log('\nAborting request...\n') - stream.abort() -}, 1000) // 1000 milliseconds = 1 second - -ollama.generate({ - model: 'llama3.1', - prompt: 'Write a long story', - stream: true, - }).then( - async (_stream) => { - stream = _stream - for await (const chunk of stream) { - process.stdout.write(chunk.response) - } - } - ).catch( - (error) => { - if (error.name === 'AbortError') { - console.log('The request has been aborted') - } else { - console.error('An error occurred:', error) - } - } - ) From e2aef0b11d96aa4312f5fa0ed5af5a24789757fb Mon Sep 17 00:00:00 2001 From: Parth Sareen Date: Tue, 5 Nov 2024 15:02:27 -0800 Subject: [PATCH 2/3] Add multi-client single abort example --- examples/abort/abort-single-request.ts | 50 ++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 examples/abort/abort-single-request.ts diff --git a/examples/abort/abort-single-request.ts b/examples/abort/abort-single-request.ts new file mode 100644 index 0000000..6ef60c4 --- /dev/null +++ b/examples/abort/abort-single-request.ts @@ -0,0 +1,50 @@ +import { Ollama } from 'ollama' + +// Create multiple ollama clients +const client1 = new Ollama() +const client2 = new Ollama() + +// Set a timeout to abort just the first request after 1 second +setTimeout(() => { + console.log('\nAborting dragons story...\n') + // abort the first client + client1.abort() +}, 1000) // 1000 milliseconds = 1 second + +// Start multiple concurrent streaming requests with different clients +Promise.all([ + client1.generate({ + model: 'llama3.2', + prompt: 'Write a long story about dragons', + stream: true, + }).then( + async (stream) => { + console.log(' Starting stream for dragons story...') + for await (const chunk of stream) { + process.stdout.write(' 1> ' + chunk.response) + } + } + ), + + client2.generate({ + model: 'llama3.2', + prompt: 'Write a short story about wizards', + stream: true, + }).then( + async (stream) => { + console.log(' Starting stream for wizards story...') + for await (const chunk of stream) { + process.stdout.write(' 2> ' + chunk.response) + } + } + ), + +]).catch(error => { + if (error.name === 'AbortError') { + console.log('Dragons story request has been aborted') + } else { + console.error('An error occurred:', error) + } +}) + + From 68a071ec8b1f74545a220d0e1b40cecdc52761ff Mon Sep 17 00:00:00 2001 From: Parth Sareen Date: Tue, 5 Nov 2024 15:22:54 -0800 Subject: [PATCH 3/3] Bump timeout in examples to allow loading model into memory Co-authored-by: Bruce MacDonald --- examples/abort/abort-all-requests.ts | 4 ++-- examples/abort/abort-single-request.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/abort/abort-all-requests.ts b/examples/abort/abort-all-requests.ts index 794f29b..3832cd0 100644 --- a/examples/abort/abort-all-requests.ts +++ b/examples/abort/abort-all-requests.ts @@ -1,10 +1,10 @@ import ollama from 'ollama' -// Set a timeout to abort all requests after 1 second +// Set a timeout to abort all requests after 5 seconds setTimeout(() => { console.log('\nAborting all requests...\n') ollama.abort() -}, 1000) // 1000 milliseconds = 1 second +}, 5000) // 5000 milliseconds = 5 seconds // Start multiple concurrent streaming requests Promise.all([ diff --git a/examples/abort/abort-single-request.ts b/examples/abort/abort-single-request.ts index 6ef60c4..214ac87 100644 --- a/examples/abort/abort-single-request.ts +++ b/examples/abort/abort-single-request.ts @@ -4,12 +4,12 @@ import { Ollama } from 'ollama' const client1 = new Ollama() const client2 = new Ollama() -// Set a timeout to abort just the first request after 1 second +// Set a timeout to abort just the first request after 5 seconds setTimeout(() => { console.log('\nAborting dragons story...\n') // abort the first client client1.abort() -}, 1000) // 1000 milliseconds = 1 second +}, 5000) // 5000 milliseconds = 5 seconds // Start multiple concurrent streaming requests with different clients Promise.all([