Skip to content

Commit

Permalink
Tell Ollama to not unload models automatically
Browse files Browse the repository at this point in the history
  • Loading branch information
sedwards2009 committed Feb 24, 2024
1 parent 8446f03 commit 491a993
Showing 1 changed file with 6 additions and 4 deletions.
10 changes: 6 additions & 4 deletions backend/internal/engine/ollama/engine_ollama.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,10 @@ type chatMessage struct {
}

type chatPayload struct {
Model string `json:"model"`
Messages []chatMessage `json:"messages"`
Options optionsPayload `json:"options"`
Model string `json:"model"`
Messages []chatMessage `json:"messages"`
Options optionsPayload `json:"options"`
KeepAlive int `json:"keep_alive"`
}

type chatResponse struct {
Expand Down Expand Up @@ -74,7 +75,8 @@ func (this *OllamaEngineBackend) Process(work *types.Request, model *data.Model,
log.Printf("OllamaEngineBackend Process(): Temperature: %f, TopP: %f\n", preset.Temperature, preset.TopP)
previousMessages := work.Messages[0 : len(work.Messages)-1]
payload := &chatPayload{
Model: model.InternalModelID,
Model: model.InternalModelID,
KeepAlive: -1,
Messages: slices.Map(previousMessages, func(m data.Message) chatMessage {
mRole := "user"
if m.Role == role.Assistant {
Expand Down

0 comments on commit 491a993

Please sign in to comment.