From b427ce52137948eeec575728b059d4fe8473da97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Tue, 4 Jun 2024 19:55:28 +0200 Subject: [PATCH 1/5] Adding initial metrics --- go.mod | 55 ++++--- go.sum | 152 +++++++++---------- server/ai/anthropic/anthropic.go | 31 +++- server/ai/asksage/asksage.go | 41 ++++-- server/ai/openai/openai.go | 42 ++++-- server/api.go | 1 + server/api_channel.go | 4 +- server/api_post.go | 4 +- server/meeting_summarization.go | 14 +- server/metrics.go | 16 ++ server/metrics/metrics.go | 241 +++++++++++++++++++++++++++++++ server/metrics/server.go | 38 +++++ server/middleware.go | 37 +++++ server/plugin.go | 33 +++-- server/post_processing.go | 12 +- server/service.go | 17 ++- 16 files changed, 563 insertions(+), 175 deletions(-) create mode 100644 server/metrics.go create mode 100644 server/metrics/metrics.go create mode 100644 server/metrics/server.go create mode 100644 server/middleware.go diff --git a/go.mod b/go.mod index 13c3cea2..942efe9d 100644 --- a/go.mod +++ b/go.mod @@ -10,35 +10,41 @@ require ( github.com/google/go-github/v41 v41.0.0 github.com/invopop/jsonschema v0.7.0 github.com/jmoiron/sqlx v1.3.5 - github.com/mattermost/mattermost/server/public v0.0.8 + github.com/mattermost/mattermost/server/public v0.1.3 + github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.19.1 github.com/r3labs/sse/v2 v2.10.0 github.com/sashabaranov/go-openai v1.24.0 + github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.4 ) require ( github.com/asticode/go-astikit v0.20.0 // indirect github.com/asticode/go-astits v1.8.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a // indirect - github.com/fatih/color v1.15.0 // indirect + github.com/fatih/color v1.17.0 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect + github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect github.com/go-playground/locales v0.13.0 // indirect github.com/go-playground/universal-translator v0.17.0 // indirect github.com/go-playground/validator/v10 v10.4.1 // indirect github.com/go-sql-driver/mysql v1.7.1 // indirect github.com/golang-jwt/jwt/v4 v4.4.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/uuid v1.3.1 // indirect - github.com/gorilla/websocket v1.5.0 // indirect - github.com/graph-gophers/graphql-go v1.5.1-0.20230110080634-edea822f558a // indirect - github.com/hashicorp/go-hclog v1.5.0 // indirect - github.com/hashicorp/go-plugin v1.4.10 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-plugin v1.6.1 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/iancoleman/orderedmap v0.2.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -48,10 +54,10 @@ require ( github.com/leodido/go-urn v1.2.0 // indirect github.com/lib/pq v1.10.9 // indirect github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404 // indirect - github.com/mattermost/ldap v0.0.0-20201202150706-ee0e6284187d // indirect - github.com/mattermost/logr/v2 v2.0.18 // indirect + github.com/mattermost/ldap v0.0.0-20231116144001-0f480c025956 // indirect + github.com/mattermost/logr/v2 v2.0.21 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -60,24 +66,25 @@ require ( github.com/pborman/uuid v1.2.1 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/philhofer/fwd v1.1.2 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect - github.com/stretchr/objx v0.5.0 // indirect - github.com/tinylib/msgp v1.1.8 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/stretchr/objx v0.5.1 // indirect + github.com/tinylib/msgp v1.1.9 // indirect github.com/trivago/tgo v1.0.7 // indirect github.com/ugorji/go/codec v1.1.7 // indirect - github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/wiggin77/merror v1.0.5 // indirect github.com/wiggin77/srslog v1.0.1 // indirect - golang.org/x/crypto v0.12.0 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/grpc v1.56.1 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/grpc v1.64.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index f6590a3f..fbdf53d0 100644 --- a/go.sum +++ b/go.sum @@ -20,10 +20,16 @@ github.com/asticode/go-astisub v0.26.0/go.mod h1:WTkuSzFB+Bp7wezuSf2Oxulj5A8zu2z github.com/asticode/go-astits v1.8.0 h1:rf6aiiGn/QhlFjNON1n5plqF3Fs025XLUwiQ0NB6oZg= github.com/asticode/go-astits v1.8.0/go.mod h1:DkOWmBNQpnr9mv24KfZjq4JawCFX1FCqjLVGvO0DygQ= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= +github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= +github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -34,8 +40,8 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a h1:etIrTD8BQqzColk9nKRusM9um5+1q0iOEJLqfBMIK64= github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a/go.mod h1:emQhSYTXqB0xxjLITTw4EaWZ+8IIQYw+kx9GqNUKdLg= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -48,13 +54,9 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/go-asn1-ber/asn1-ber v1.3.2-0.20191121212151-29be175fc3a3/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= -github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A= -github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk= +github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= @@ -77,18 +79,15 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-github/v41 v41.0.0 h1:HseJrM2JFf2vfiZJ8anY2hqBjdfY1Vlj/K27ueww4gg= github.com/google/go-github/v41 v41.0.0/go.mod h1:XgmCA5H323A9rtgExdTcnDkcqp6S30AVACCBDOonIxg= @@ -99,21 +98,24 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/graph-gophers/graphql-go v1.5.1-0.20230110080634-edea822f558a h1:i0+Se9S+2zL5CBxJouqn2Ej6UQMwH1c57ZB6DVnqck4= -github.com/graph-gophers/graphql-go v1.5.1-0.20230110080634-edea822f558a/go.mod h1:YtmJZDLbF1YYNrlNAuiO5zAStUWc3XZT07iGsVqe1Os= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= -github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-plugin v1.4.10 h1:xUbmA4jC6Dq163/fWcp8P3JuHilrHHMLNRxzGQJ9hNk= -github.com/hashicorp/go-plugin v1.4.10/go.mod h1:6/1TEzT0eQznvI/gV2CM29DLSkAK/e58mUWKVsPaph0= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-plugin v1.6.1 h1:P7MR2UP6gNKGPp+y7EZw2kOiq4IR9WiqLvp0XOsVdwI= +github.com/hashicorp/go-plugin v1.6.1/go.mod h1:XPHFku2tFo3o3QKFgSYo+cghcUhw1NA1hZyMK0PWAw0= github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= @@ -122,8 +124,8 @@ github.com/iancoleman/orderedmap v0.2.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36 github.com/invopop/jsonschema v0.7.0 h1:2vgQcBz1n256N+FpX3Jq7Y17AjYt46Ig3zIWyy770So= github.com/invopop/jsonschema v0.7.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= -github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= -github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= +github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= +github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -133,8 +135,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -153,12 +155,12 @@ github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404 h1:Khvh6waxG1cHc4Cz5ef9n3XVCxRWpAKUtqg9PJl5+y8= github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404/go.mod h1:RyS7FDNQlzF1PsjbJWHRI35exqaKGSO9qD4iv8QjE34= -github.com/mattermost/ldap v0.0.0-20201202150706-ee0e6284187d h1:/RJ/UV7M5c7L2TQ0KNm4yZxxFvC1nvRz/gY/Daa35aI= -github.com/mattermost/ldap v0.0.0-20201202150706-ee0e6284187d/go.mod h1:HLbgMEI5K131jpxGazJ97AxfPDt31osq36YS1oxFQPQ= -github.com/mattermost/logr/v2 v2.0.18 h1:qiznuwwKckZJoGtBYc4Y9FAY97/oQwV1Pq9oO5qP5nk= -github.com/mattermost/logr/v2 v2.0.18/go.mod h1:1dm/YhTpozsqANXxo5Pi5zYLBsal2xY0pX+JZNbzYJY= -github.com/mattermost/mattermost/server/public v0.0.8 h1:YFgI5zT2U5xOvrYMb7s1YtwuLGNFModi8XJMS1zZHWE= -github.com/mattermost/mattermost/server/public v0.0.8/go.mod h1:sgXQrYzs+IJy51mB8E8OBljagk2u3YwQRoYlBH5goiw= +github.com/mattermost/ldap v0.0.0-20231116144001-0f480c025956 h1:Y1Tu/swM31pVwwb2BTCsOdamENjjWCI6qmfHLbk6OZI= +github.com/mattermost/ldap v0.0.0-20231116144001-0f480c025956/go.mod h1:SRl30Lb7/QoYyohYeVBuqYvvmXSZJxZgiV3Zf6VbxjI= +github.com/mattermost/logr/v2 v2.0.21 h1:CMHsP+nrbRlEC4g7BwOk1GAnMtHkniFhlSQPXy52be4= +github.com/mattermost/logr/v2 v2.0.21/go.mod h1:kZkB/zqKL9e+RY5gB3vGpsyenC+TpuiOenjMkvJJbzc= +github.com/mattermost/mattermost/server/public v0.1.3 h1:A3hQ3rNCwHfKAVxe7Hk3Zd9p2pyUKRmxtRPnkWP5SFM= +github.com/mattermost/mattermost/server/public v0.1.3/go.mod h1:PDPb/iqzJJ5ZvK/m70oDF55AXN/cOvVFj96Yu4e6j+Q= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -166,8 +168,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -186,7 +188,6 @@ github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJE github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= -github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= @@ -202,13 +203,21 @@ github.com/pkg/profile v1.4.0/go.mod h1:NWz/XGvpEW1FyYQ7fCx4dqYBLlfTcE+A9FLAkNKq github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/r3labs/sse/v2 v2.10.0 h1:hFEkLLFY4LDifoHdiCN/LlGBAdVJYsANaLqNYa1l/v0= github.com/r3labs/sse/v2 v2.10.0/go.mod h1:Igau6Whc+F17QUgML1fYe1VPZzTV6EMCnYktEmkNJ7I= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sashabaranov/go-openai v1.24.0 h1:4H4Pg8Bl2RH/YSnU8DYumZbuHnnkfioor/dtNlB20D4= github.com/sashabaranov/go-openai v1.24.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= @@ -241,8 +250,9 @@ github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:Udh github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0= +github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -252,11 +262,12 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= -github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= -github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= +github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU= +github.com/tinylib/msgp v1.1.9/go.mod h1:BCXGB54lDD8qUEPmiG0cQQUANC4IUQyB2ItS2UDlO/k= github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM= github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= @@ -264,18 +275,15 @@ github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= -github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= -github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/wiggin77/merror v1.0.5 h1:P+lzicsn4vPMycAf2mFf7Zk6G9eco5N+jB1qJ2XW3ME= github.com/wiggin77/merror v1.0.5/go.mod h1:H2ETSu7/bPE0Ymf4bEwdUoo73OOEkdClnoRisfw0Nm0= github.com/wiggin77/srslog v1.0.1 h1:gA2XjSMy3DrRdX9UqLuDtuVAAshb8bE1NhX1YK0Qe+8= github.com/wiggin77/srslog v1.0.1/go.mod h1:fehkyYDq1QfuYn60TDPu9YdY2bB85VUW2mvN1WynEls= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= -go.opentelemetry.io/otel v1.6.3/go.mod h1:7BgNga5fNlF/iZjG06hM3yofffp0ofKCDwSXx1GC4dI= -go.opentelemetry.io/otel/trace v1.6.3/go.mod h1:GNJQusJlUgZl9/TQBPKU/Y/ty+0iVB5fjhKeJGZPGFs= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -283,15 +291,12 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -302,15 +307,12 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -320,9 +322,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -339,26 +338,21 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -366,10 +360,6 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= @@ -384,18 +374,16 @@ google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= -google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/server/ai/anthropic/anthropic.go b/server/ai/anthropic/anthropic.go index 484cdcb3..80f8eca3 100644 --- a/server/ai/anthropic/anthropic.go +++ b/server/ai/anthropic/anthropic.go @@ -4,23 +4,28 @@ import ( "fmt" "github.com/mattermost/mattermost-plugin-ai/server/ai" + "github.com/mattermost/mattermost-plugin-ai/server/metrics" ) const DefaultMaxTokens = 4096 type Anthropic struct { - client *Client - defaultModel string - tokenLimit int + client *Client + defaultModel string + tokenLimit int + metricsService metrics.Metrics + name string } -func New(llmService ai.ServiceConfig) *Anthropic { - client := NewClient(llmService.APIKey) +func New(botConfig ai.BotConfig, metricsService metrics.Metrics) *Anthropic { + client := NewClient(botConfig.Service.APIKey) return &Anthropic{ - client: client, - defaultModel: llmService.DefaultModel, - tokenLimit: llmService.TokenLimit, + client: client, + defaultModel: botConfig.Service.DefaultModel, + tokenLimit: botConfig.Service.TokenLimit, + metricsService: metricsService, + name: botConfig.Name, } } @@ -79,6 +84,10 @@ func (a *Anthropic) createCompletionRequest(conversation ai.BotConversation, opt } func (a *Anthropic) ChatCompletion(conversation ai.BotConversation, opts ...ai.LanguageModelOption) (*ai.TextStreamResult, error) { + a.metricsService.ObserveLLMRequest(a.name) + a.metricsService.ObserveLLMTokensSent(a.name, int64(a.CountTokens(conversation.String()))) + a.metricsService.ObserveLLMBytesSent(a.name, int64(len(conversation.String()))) + request := a.createCompletionRequest(conversation, opts) request.Stream = true result, err := a.client.MessageCompletion(request) @@ -90,6 +99,10 @@ func (a *Anthropic) ChatCompletion(conversation ai.BotConversation, opts ...ai.L } func (a *Anthropic) ChatCompletionNoStream(conversation ai.BotConversation, opts ...ai.LanguageModelOption) (string, error) { + a.metricsService.ObserveLLMRequest(a.name) + a.metricsService.ObserveLLMTokensSent(a.name, int64(a.CountTokens(conversation.String()))) + a.metricsService.ObserveLLMBytesSent(a.name, int64(len(conversation.String()))) + request := a.createCompletionRequest(conversation, opts) request.Stream = false result, err := a.client.MessageCompletionNoStream(request) @@ -97,6 +110,8 @@ func (a *Anthropic) ChatCompletionNoStream(conversation ai.BotConversation, opts return "", fmt.Errorf("failed to send query to anthropic: %w", err) } + a.metricsService.ObserveLLMTokensReceived(a.name, int64(a.CountTokens(result))) + a.metricsService.ObserveLLMBytesReceived(a.name, int64(len(result))) return result, nil } diff --git a/server/ai/asksage/asksage.go b/server/ai/asksage/asksage.go index 1467e5e3..756069cb 100644 --- a/server/ai/asksage/asksage.go +++ b/server/ai/asksage/asksage.go @@ -4,24 +4,29 @@ import ( "strings" "github.com/mattermost/mattermost-plugin-ai/server/ai" + "github.com/mattermost/mattermost-plugin-ai/server/metrics" ) type AskSage struct { - client *Client - defaultModel string - maxTokens int + client *Client + defaultModel string + maxTokens int + metricsService metrics.Metrics + name string } -func New(llmService ai.ServiceConfig) *AskSage { +func New(botConfig ai.BotConfig, metricsService metrics.Metrics) *AskSage { client := NewClient("") client.Login(GetTokenParams{ - Email: llmService.Username, - Password: llmService.Password, + Email: botConfig.Service.Username, + Password: botConfig.Service.Password, }) return &AskSage{ - client: client, - defaultModel: llmService.DefaultModel, - maxTokens: llmService.TokenLimit, + client: client, + defaultModel: botConfig.Service.DefaultModel, + maxTokens: botConfig.Service.TokenLimit, + metricsService: metricsService, + name: botConfig.Name, } } @@ -67,14 +72,18 @@ func (s *AskSage) queryParamsFromConfig(cfg ai.LLMConfig) QueryParams { func (s *AskSage) ChatCompletion(conversation ai.BotConversation, opts ...ai.LanguageModelOption) (*ai.TextStreamResult, error) { // Ask Sage does not support streaming. - result, err := s.ChatCompletionNoStream(conversation, opts...) + result, err := s.ChatCompletionBase(conversation, opts...) if err != nil { return nil, err } return ai.NewStreamFromString(result), nil } -func (s *AskSage) ChatCompletionNoStream(conversation ai.BotConversation, opts ...ai.LanguageModelOption) (string, error) { +func (s *AskSage) ChatCompletionBase(conversation ai.BotConversation, opts ...ai.LanguageModelOption) (string, error) { + s.metricsService.ObserveLLMRequest(s.name) + s.metricsService.ObserveLLMTokensSent(s.name, int64(s.CountTokens(conversation.String()))) + s.metricsService.ObserveLLMBytesSent(s.name, int64(len(conversation.String()))) + params := s.queryParamsFromConfig(s.createConfig(opts)) params.Message = conversationToMessagesList(conversation) params.SystemPrompt = conversation.ExtractSystemMessage() @@ -87,6 +96,16 @@ func (s *AskSage) ChatCompletionNoStream(conversation ai.BotConversation, opts . return response.Message, nil } +func (s *AskSage) ChatCompletionNoStream(conversation ai.BotConversation, opts ...ai.LanguageModelOption) (string, error) { + response, err := s.ChatCompletionBase(conversation, opts...) + if err != nil { + return "", err + } + s.metricsService.ObserveLLMTokensReceived(s.name, int64(s.CountTokens(response))) + s.metricsService.ObserveLLMBytesReceived(s.name, int64(len(response))) + return response, nil +} + // TODO: Implement actual token counting. For now just estimated based off OpenAI estimations func (s *AskSage) CountTokens(text string) int { charCount := float64(len(text)) / 4.0 diff --git a/server/ai/openai/openai.go b/server/ai/openai/openai.go index 21f41325..c8b97297 100644 --- a/server/ai/openai/openai.go +++ b/server/ai/openai/openai.go @@ -18,6 +18,7 @@ import ( "github.com/invopop/jsonschema" "github.com/mattermost/mattermost-plugin-ai/server/ai" "github.com/mattermost/mattermost-plugin-ai/server/ai/subtitles" + "github.com/mattermost/mattermost-plugin-ai/server/metrics" openaiClient "github.com/sashabaranov/go-openai" ) @@ -26,6 +27,8 @@ type OpenAI struct { defaultModel string tokenLimit int streamingTimeout time.Duration + metricsService metrics.Metrics + name string } const StreamingTimeoutDefault = 10 * time.Second @@ -36,10 +39,10 @@ const OpenAIMaxImageSize = 20 * 1024 * 1024 // 20 MB var ErrStreamingTimeout = errors.New("timeout streaming") -func NewCompatible(llmService ai.ServiceConfig) *OpenAI { - apiKey := llmService.APIKey - endpointURL := strings.TrimSuffix(llmService.APIURL, "/") - defaultModel := llmService.DefaultModel +func NewCompatible(botConfig ai.BotConfig, metricsService metrics.Metrics) *OpenAI { + apiKey := botConfig.Service.APIKey + endpointURL := strings.TrimSuffix(botConfig.Service.APIURL, "/") + defaultModel := botConfig.Service.DefaultModel config := openaiClient.DefaultConfig(apiKey) config.BaseURL = endpointURL @@ -50,35 +53,39 @@ func NewCompatible(llmService ai.ServiceConfig) *OpenAI { } streamingTimeout := StreamingTimeoutDefault - if llmService.StreamingTimeoutSeconds > 0 { - streamingTimeout = time.Duration(llmService.StreamingTimeoutSeconds) * time.Second + if botConfig.Service.StreamingTimeoutSeconds > 0 { + streamingTimeout = time.Duration(botConfig.Service.StreamingTimeoutSeconds) * time.Second } return &OpenAI{ client: openaiClient.NewClientWithConfig(config), defaultModel: defaultModel, - tokenLimit: llmService.TokenLimit, + tokenLimit: botConfig.Service.TokenLimit, streamingTimeout: streamingTimeout, + metricsService: metricsService, + name: botConfig.Name, } } -func New(llmService ai.ServiceConfig) *OpenAI { - defaultModel := llmService.DefaultModel +func New(botConfig ai.BotConfig, metricsService metrics.Metrics) *OpenAI { + defaultModel := botConfig.Service.DefaultModel if defaultModel == "" { defaultModel = openaiClient.GPT3Dot5Turbo } - config := openaiClient.DefaultConfig(llmService.APIKey) - config.OrgID = llmService.OrgID + config := openaiClient.DefaultConfig(botConfig.Service.APIKey) + config.OrgID = botConfig.Service.OrgID streamingTimeout := StreamingTimeoutDefault - if llmService.StreamingTimeoutSeconds > 0 { - streamingTimeout = time.Duration(llmService.StreamingTimeoutSeconds) * time.Second + if botConfig.Service.StreamingTimeoutSeconds > 0 { + streamingTimeout = time.Duration(botConfig.Service.StreamingTimeoutSeconds) * time.Second } return &OpenAI{ client: openaiClient.NewClientWithConfig(config), defaultModel: defaultModel, - tokenLimit: llmService.TokenLimit, + tokenLimit: botConfig.Service.TokenLimit, streamingTimeout: streamingTimeout, + metricsService: metricsService, + name: botConfig.Name, } } @@ -344,6 +351,10 @@ func (s *OpenAI) completionRequestFromConfig(cfg ai.LLMConfig) openaiClient.Chat } func (s *OpenAI) ChatCompletion(conversation ai.BotConversation, opts ...ai.LanguageModelOption) (*ai.TextStreamResult, error) { + s.metricsService.ObserveLLMRequest(s.name) + s.metricsService.ObserveLLMTokensSent(s.name, int64(s.CountTokens(conversation.String()))) + s.metricsService.ObserveLLMBytesSent(s.name, int64(len(conversation.String()))) + request := s.completionRequestFromConfig(s.createConfig(opts)) request = modifyCompletionRequestWithConversation(request, conversation) request.Stream = true @@ -356,7 +367,8 @@ func (s *OpenAI) ChatCompletionNoStream(conversation ai.BotConversation, opts .. if err != nil { return "", err } - return result.ReadAll(), nil + data := result.ReadAll() + return data, nil } func (s *OpenAI) Transcribe(file io.Reader) (*subtitles.Subtitles, error) { diff --git a/server/api.go b/server/api.go index d2f85abb..7b36d9af 100644 --- a/server/api.go +++ b/server/api.go @@ -22,6 +22,7 @@ func (p *Plugin) ServeHTTP(c *plugin.Context, w http.ResponseWriter, r *http.Req router := gin.Default() router.Use(p.ginlogger) router.Use(p.MattermostAuthorizationRequired) + router.Use(p.metricsMiddleware) router.GET("/ai_threads", p.handleGetAIThreads) router.GET("/ai_bots", p.handleGetAIBots) diff --git a/server/api_channel.go b/server/api_channel.go index 46e84a52..8de12341 100644 --- a/server/api_channel.go +++ b/server/api_channel.go @@ -109,7 +109,7 @@ func (p *Plugin) handleSince(c *gin.Context) { return } - resultStream, err := p.getLLM(bot.cfg.Service).ChatCompletion(prompt) + resultStream, err := p.getLLM(bot.cfg).ChatCompletion(prompt) if err != nil { c.AbortWithError(http.StatusInternalServerError, err) return @@ -117,7 +117,7 @@ func (p *Plugin) handleSince(c *gin.Context) { post := &model.Post{} post.AddProp(NoRegen, "true") - if err := p.streamResultToNewDM(bot.mmBot.UserId, resultStream, user.Id, post); err != nil { + if err := p.streamResultToNewDM(bot.mmBot.UserId, resultStream, user.Id, post, bot.cfg.Name); err != nil { c.AbortWithError(http.StatusInternalServerError, err) return } diff --git a/server/api_post.go b/server/api_post.go index f4f4eeb9..d0a9f4ff 100644 --- a/server/api_post.go +++ b/server/api_post.go @@ -66,7 +66,7 @@ func (p *Plugin) handleReact(c *gin.Context) { return } - emojiName, err := p.getLLM(bot.cfg.Service).ChatCompletionNoStream(prompt, ai.WithMaxGeneratedTokens(25)) + emojiName, err := p.getLLM(bot.cfg).ChatCompletionNoStream(prompt, ai.WithMaxGeneratedTokens(25)) if err != nil { c.AbortWithError(http.StatusInternalServerError, err) return @@ -369,7 +369,7 @@ func (p *Plugin) regeneratePost(bot *Bot, post *model.Post, user *model.User, ch } } - p.streamResultToPost(ctx, result, post) + p.streamResultToPost(ctx, result, post, bot.cfg.Name) return nil } diff --git a/server/meeting_summarization.go b/server/meeting_summarization.go index f7bca1d6..3d6ae5cc 100644 --- a/server/meeting_summarization.go +++ b/server/meeting_summarization.go @@ -179,7 +179,7 @@ func (p *Plugin) newCallTranscriptionSummaryThread(bot *Bot, requestingUser *mod Message: "", } summaryPost.AddProp(ReferencedTranscriptPostID, transcriptionPost.Id) - if err := p.streamResultToNewPost(bot.mmBot.UserId, requestingUser.Id, summaryStream, summaryPost); err != nil { + if err := p.streamResultToNewPost(bot.mmBot.UserId, requestingUser.Id, summaryStream, summaryPost, bot.cfg.Name); err != nil { return fmt.Errorf("unable to stream result to post: %w", err) } @@ -237,7 +237,7 @@ func (p *Plugin) summarizeCallRecording(bot *Bot, rootID string, requestingUser } defer p.finishPostStreaming(transcriptPost.Id) - p.streamResultToPost(ctx, summaryStream, transcriptPost) + p.streamResultToPost(ctx, summaryStream, transcriptPost, bot.cfg.Name) return nil }() @@ -247,8 +247,8 @@ func (p *Plugin) summarizeCallRecording(bot *Bot, rootID string, requestingUser func (p *Plugin) summarizeTranscription(bot *Bot, transcription *subtitles.Subtitles, context ai.ConversationContext) (*ai.TextStreamResult, error) { llmFormattedTranscription := transcription.FormatForLLM() - tokens := p.getLLM(bot.cfg.Service).CountTokens(llmFormattedTranscription) - tokenLimitWithMargin := int(float64(p.getLLM(bot.cfg.Service).TokenLimit())*0.75) - ContextTokenMargin + tokens := p.getLLM(bot.cfg).CountTokens(llmFormattedTranscription) + tokenLimitWithMargin := int(float64(p.getLLM(bot.cfg).TokenLimit())*0.75) - ContextTokenMargin if tokenLimitWithMargin < 0 { tokenLimitWithMargin = ContextTokenMargin / 2 } @@ -265,7 +265,7 @@ func (p *Plugin) summarizeTranscription(bot *Bot, transcription *subtitles.Subti return nil, fmt.Errorf("unable to get summarize chunk prompt: %w", err) } - summarizedChunk, err := p.getLLM(bot.cfg.Service).ChatCompletionNoStream(summarizeChunkPrompt) + summarizedChunk, err := p.getLLM(bot.cfg).ChatCompletionNoStream(summarizeChunkPrompt) if err != nil { return nil, fmt.Errorf("unable to get summarized chunk: %w", err) } @@ -275,7 +275,7 @@ func (p *Plugin) summarizeTranscription(bot *Bot, transcription *subtitles.Subti llmFormattedTranscription = strings.Join(summarizedChunks, "\n\n") isChunked = true - p.pluginAPI.Log.Debug("Completed chunk summarization", "chunks", len(summarizedChunks), "tokens", p.getLLM(bot.cfg.Service).CountTokens(llmFormattedTranscription)) + p.pluginAPI.Log.Debug("Completed chunk summarization", "chunks", len(summarizedChunks), "tokens", p.getLLM(bot.cfg).CountTokens(llmFormattedTranscription)) } context.PromptParameters = map[string]string{"Transcription": llmFormattedTranscription, "IsChunked": fmt.Sprintf("%t", isChunked)} @@ -284,7 +284,7 @@ func (p *Plugin) summarizeTranscription(bot *Bot, transcription *subtitles.Subti return nil, fmt.Errorf("unable to get meeting summary prompt: %w", err) } - summaryStream, err := p.getLLM(bot.cfg.Service).ChatCompletion(summaryPrompt) + summaryStream, err := p.getLLM(bot.cfg).ChatCompletion(summaryPrompt) if err != nil { return nil, fmt.Errorf("unable to get meeting summary: %w", err) } diff --git a/server/metrics.go b/server/metrics.go new file mode 100644 index 00000000..96cd3c4e --- /dev/null +++ b/server/metrics.go @@ -0,0 +1,16 @@ +package main + +import ( + "net/http" + + "github.com/mattermost/mattermost-plugin-ai/server/metrics" + "github.com/mattermost/mattermost/server/public/plugin" +) + +func (p *Plugin) ServeMetrics(_ *plugin.Context, w http.ResponseWriter, r *http.Request) { + p.metricsHandler.ServeHTTP(w, r) +} + +func (p *Plugin) GetMetrics() metrics.Metrics { + return p.metricsService +} diff --git a/server/metrics/metrics.go b/server/metrics/metrics.go new file mode 100644 index 00000000..e89ac1f6 --- /dev/null +++ b/server/metrics/metrics.go @@ -0,0 +1,241 @@ +//go:generate mockery --name=Metrics +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" +) + +const ( + MetricsNamespace = "copilot" + MetricsSubsystemSystem = "system" + MetricsSubsystemApp = "app" + MetricsSubsystemHTTP = "http" + MetricsSubsystemAPI = "api" + MetricsSubsystemEvents = "events" + MetricsSubsystemDB = "db" + MetricsSubsystemLLM = "llm" + + MetricsCloudInstallationLabel = "installationId" + MetricsVersionLabel = "version" + + ActionSourceMSTeams = "msteams" + ActionSourceMattermost = "mattermost" + ActionCreated = "created" + ActionUpdated = "updated" + ActionDeleted = "deleted" + ReactionSetAction = "set" + ReactionUnsetAction = "unset" + SubscriptionRefreshed = "refreshed" + SubscriptionConnected = "connected" + SubscriptionReconnected = "reconnected" +) + +type Metrics interface { + GetRegistry() *prometheus.Registry + + ObserveAPIEndpointDuration(handler, method, statusCode string, elapsed float64) + + IncrementHTTPRequests() + IncrementHTTPErrors() + + ObserveLLMRequest(llmID string) + ObserveLLMTokensSent(llmID string, count int64) + ObserveLLMTokensReceived(llmID string, count int64) + ObserveLLMBytesSent(llmID string, count int64) + ObserveLLMBytesReceived(llmID string, count int64) +} + +type InstanceInfo struct { + InstallationID string + ConnectedUsersLimit int + PluginVersion string +} + +// metrics used to instrumentate metrics in prometheus. +type metrics struct { + registry *prometheus.Registry + + pluginStartTime prometheus.Gauge + pluginInfo prometheus.Gauge + + apiTime *prometheus.HistogramVec + + httpRequestsTotal prometheus.Counter + httpErrorsTotal prometheus.Counter + + llmRequestsTotal *prometheus.CounterVec + llmTokensSent *prometheus.CounterVec + llmTokensReceived *prometheus.CounterVec + llmBytesSent *prometheus.CounterVec + llmBytesReceived *prometheus.CounterVec +} + +// NewMetrics Factory method to create a new metrics collector. +func NewMetrics(info InstanceInfo) Metrics { + m := &metrics{} + + m.registry = prometheus.NewRegistry() + options := collectors.ProcessCollectorOpts{ + Namespace: MetricsNamespace, + } + m.registry.MustRegister(collectors.NewProcessCollector(options)) + m.registry.MustRegister(collectors.NewGoCollector()) + + additionalLabels := map[string]string{} + if info.InstallationID != "" { + additionalLabels[MetricsCloudInstallationLabel] = info.InstallationID + } + + m.pluginStartTime = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: MetricsNamespace, + Subsystem: MetricsSubsystemSystem, + Name: "plugin_start_timestamp_seconds", + Help: "The time the plugin started.", + ConstLabels: additionalLabels, + }) + m.pluginStartTime.SetToCurrentTime() + m.registry.MustRegister(m.pluginStartTime) + + m.pluginInfo = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: MetricsNamespace, + Subsystem: MetricsSubsystemSystem, + Name: "plugin_info", + Help: "The plugin version.", + ConstLabels: map[string]string{ + MetricsCloudInstallationLabel: info.InstallationID, + MetricsVersionLabel: info.PluginVersion, + }, + }) + m.pluginInfo.Set(1) + m.registry.MustRegister(m.pluginInfo) + + m.apiTime = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: MetricsNamespace, + Subsystem: MetricsSubsystemAPI, + Name: "time_seconds", + Help: "Time to execute the api handler", + ConstLabels: additionalLabels, + }, + []string{"handler", "method", "status_code"}, + ) + m.registry.MustRegister(m.apiTime) + + m.httpRequestsTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: MetricsNamespace, + Subsystem: MetricsSubsystemHTTP, + Name: "requests_total", + Help: "The total number of http API requests.", + ConstLabels: additionalLabels, + }) + m.registry.MustRegister(m.httpRequestsTotal) + + m.httpErrorsTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: MetricsNamespace, + Subsystem: MetricsSubsystemHTTP, + Name: "errors_total", + Help: "The total number of http API errors.", + ConstLabels: additionalLabels, + }) + m.registry.MustRegister(m.httpErrorsTotal) + + m.llmRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: MetricsNamespace, + Subsystem: MetricsSubsystemLLM, + Name: "requests_total", + Help: "The total number of LLM requets made.", + ConstLabels: additionalLabels, + }, []string{"llm_name"}) + m.registry.MustRegister(m.llmRequestsTotal) + + m.llmTokensSent = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: MetricsNamespace, + Subsystem: MetricsSubsystemLLM, + Name: "tokens_sent_total", + Help: "The total number of tokens sent.", + ConstLabels: additionalLabels, + }, []string{"llm_name"}) + m.registry.MustRegister(m.llmTokensSent) + + m.llmTokensReceived = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: MetricsNamespace, + Subsystem: MetricsSubsystemLLM, + Name: "tokens_received_total", + Help: "The total number of tokens received.", + ConstLabels: additionalLabels, + }, []string{"llm_name"}) + m.registry.MustRegister(m.llmTokensReceived) + + m.llmBytesSent = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: MetricsNamespace, + Subsystem: MetricsSubsystemLLM, + Name: "bytes_sent_total", + Help: "The total number of bytes sent.", + ConstLabels: additionalLabels, + }, []string{"llm_name"}) + m.registry.MustRegister(m.llmBytesSent) + + m.llmBytesReceived = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: MetricsNamespace, + Subsystem: MetricsSubsystemLLM, + Name: "bytes_received_total", + Help: "The total number of bytes received.", + ConstLabels: additionalLabels, + }, []string{"llm_name"}) + m.registry.MustRegister(m.llmBytesReceived) + + return m +} + +func (m *metrics) GetRegistry() *prometheus.Registry { + return m.registry +} + +func (m *metrics) ObserveAPIEndpointDuration(handler, method, statusCode string, elapsed float64) { + if m != nil { + m.apiTime.With(prometheus.Labels{"handler": handler, "method": method, "status_code": statusCode}).Observe(elapsed) + } +} + +func (m *metrics) ObserveLLMRequest(llmID string) { + if m != nil { + m.llmRequestsTotal.With(prometheus.Labels{"llm_name": llmID}).Inc() + } +} + +func (m *metrics) ObserveLLMTokensSent(llmID string, count int64) { + if m != nil { + m.llmTokensSent.With(prometheus.Labels{"llm_name": llmID}).Add(float64(count)) + } +} + +func (m *metrics) ObserveLLMTokensReceived(llmID string, count int64) { + if m != nil { + m.llmTokensReceived.With(prometheus.Labels{"llm_name": llmID}).Add(float64(count)) + } +} + +func (m *metrics) ObserveLLMBytesSent(llmID string, count int64) { + if m != nil { + m.llmBytesSent.With(prometheus.Labels{"llm_name": llmID}).Add(float64(count)) + } +} + +func (m *metrics) ObserveLLMBytesReceived(llmID string, count int64) { + if m != nil { + m.llmBytesReceived.With(prometheus.Labels{"llm_name": llmID}).Add(float64(count)) + } +} + +func (m *metrics) IncrementHTTPRequests() { + if m != nil { + m.httpRequestsTotal.Inc() + } +} + +func (m *metrics) IncrementHTTPErrors() { + if m != nil { + m.httpErrorsTotal.Inc() + } +} diff --git a/server/metrics/server.go b/server/metrics/server.go new file mode 100644 index 00000000..3e38030e --- /dev/null +++ b/server/metrics/server.go @@ -0,0 +1,38 @@ +package metrics + +import ( + "net/http" + + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" +) + +// Service prometheus to run the server. +type Server struct { + *http.Server +} + +type ErrorLoggerWrapper struct { +} + +func (el *ErrorLoggerWrapper) Println(v ...interface{}) { + logrus.Warn("metric server error", v) +} + +// NewMetricsHandler creates an HTTP handler to expose metrics. +func NewMetricsHandler(metricsService Metrics) http.Handler { + return promhttp.HandlerFor(metricsService.GetRegistry(), promhttp.HandlerOpts{ + ErrorLog: &ErrorLoggerWrapper{}, + }) +} + +// Run will start the prometheus server. +func (h *Server) Run() error { + return errors.Wrap(h.Server.ListenAndServe(), "prometheus ListenAndServe") +} + +// Shutdown will shutdown the prometheus server. +func (h *Server) Shutdown() error { + return errors.Wrap(h.Server.Close(), "prometheus Close") +} diff --git a/server/middleware.go b/server/middleware.go new file mode 100644 index 00000000..9b0bd24e --- /dev/null +++ b/server/middleware.go @@ -0,0 +1,37 @@ +package main + +import ( + "net/http" + "strconv" + "time" + + "github.com/gin-gonic/gin" +) + +type StatusRecorder struct { + http.ResponseWriter + Status int +} + +func (r *StatusRecorder) WriteHeader(status int) { + r.Status = status + r.ResponseWriter.WriteHeader(status) +} + +func (p *Plugin) metricsMiddleware(c *gin.Context) { + p.GetMetrics().IncrementHTTPRequests() + now := time.Now() + + c.Next() + + elapsed := float64(time.Since(now)) / float64(time.Second) + + status := c.Writer.Status() + + if status < 200 || status > 299 { + p.GetMetrics().IncrementHTTPErrors() + } + + endpoint := c.HandlerName() + p.GetMetrics().ObserveAPIEndpointDuration(endpoint, c.Request.Method, strconv.Itoa(status), elapsed) +} diff --git a/server/plugin.go b/server/plugin.go index d85f0106..cc37de72 100644 --- a/server/plugin.go +++ b/server/plugin.go @@ -3,6 +3,8 @@ package main import ( "embed" "fmt" + "net/http" + "os" "os/exec" "sync" @@ -15,6 +17,7 @@ import ( "github.com/mattermost/mattermost-plugin-ai/server/ai/asksage" "github.com/mattermost/mattermost-plugin-ai/server/ai/openai" "github.com/mattermost/mattermost-plugin-ai/server/enterprise" + "github.com/mattermost/mattermost-plugin-ai/server/metrics" "github.com/mattermost/mattermost/server/public/model" "github.com/mattermost/mattermost/server/public/plugin" "github.com/mattermost/mattermost/server/public/pluginapi" @@ -56,6 +59,8 @@ type Plugin struct { streamingContextsMutex sync.Mutex licenseChecker *enterprise.LicenseChecker + metricsService metrics.Metrics + metricsHandler http.Handler botsLock sync.RWMutex bots []*Bot @@ -79,6 +84,12 @@ func (p *Plugin) OnActivate() error { p.licenseChecker = enterprise.NewLicenseChecker(p.pluginAPI) + p.metricsService = metrics.NewMetrics(metrics.InstanceInfo{ + InstallationID: os.Getenv("MM_CLOUD_INSTALLATION_ID"), + PluginVersion: manifest.Version, + }) + p.metricsHandler = metrics.NewMetricsHandler(p.GetMetrics()) + if err := p.MigrateServicesToBots(); err != nil { p.pluginAPI.Log.Error("failed to migrate services to bots", "error", err) // Don't fail on migration errors @@ -110,17 +121,17 @@ func (p *Plugin) OnActivate() error { return nil } -func (p *Plugin) getLLM(llmServiceConfig ai.ServiceConfig) ai.LanguageModel { +func (p *Plugin) getLLM(llmBotConfig ai.BotConfig) ai.LanguageModel { var llm ai.LanguageModel - switch llmServiceConfig.Type { + switch llmBotConfig.Service.Type { case "openai": - llm = openai.New(llmServiceConfig) + llm = openai.New(llmBotConfig, p.metricsService) case "openaicompatible": - llm = openai.NewCompatible(llmServiceConfig) + llm = openai.NewCompatible(llmBotConfig, p.metricsService) case "anthropic": - llm = anthropic.New(llmServiceConfig) + llm = anthropic.New(llmBotConfig, p.metricsService) case "asksage": - llm = asksage.New(llmServiceConfig) + llm = asksage.New(llmBotConfig, p.metricsService) } cfg := p.getConfiguration() @@ -135,18 +146,18 @@ func (p *Plugin) getLLM(llmServiceConfig ai.ServiceConfig) ai.LanguageModel { func (p *Plugin) getTranscribe() ai.Transcriber { cfg := p.getConfiguration() - var transcriptionService ai.ServiceConfig + var botConfig ai.BotConfig for _, bot := range cfg.Bots { if bot.Name == cfg.TranscriptGenerator { - transcriptionService = bot.Service + botConfig = bot break } } - switch transcriptionService.Type { + switch botConfig.Service.Type { case "openai": - return openai.New(transcriptionService) + return openai.New(botConfig, p.metricsService) case "openaicompatible": - return openai.NewCompatible(transcriptionService) + return openai.NewCompatible(botConfig, p.metricsService) } return nil } diff --git a/server/post_processing.go b/server/post_processing.go index 22b38b0e..90823433 100644 --- a/server/post_processing.go +++ b/server/post_processing.go @@ -128,7 +128,7 @@ func (p *Plugin) botDM(botid string, userID string, post *model.Post) error { return nil } -func (p *Plugin) streamResultToNewPost(botid string, requesterUserID string, stream *ai.TextStreamResult, post *model.Post) error { +func (p *Plugin) streamResultToNewPost(botid string, requesterUserID string, stream *ai.TextStreamResult, post *model.Post, llmBotName string) error { if err := p.botCreatePost(botid, requesterUserID, post); err != nil { return fmt.Errorf("unable to create post: %w", err) } @@ -140,13 +140,13 @@ func (p *Plugin) streamResultToNewPost(botid string, requesterUserID string, str go func() { defer p.finishPostStreaming(post.Id) - p.streamResultToPost(ctx, stream, post) + p.streamResultToPost(ctx, stream, post, llmBotName) }() return nil } -func (p *Plugin) streamResultToNewDM(botid string, stream *ai.TextStreamResult, userID string, post *model.Post) error { +func (p *Plugin) streamResultToNewDM(botid string, stream *ai.TextStreamResult, userID string, post *model.Post, llmBotName string) error { if err := p.botDM(botid, userID, post); err != nil { return err } @@ -158,7 +158,7 @@ func (p *Plugin) streamResultToNewDM(botid string, stream *ai.TextStreamResult, go func() { defer p.finishPostStreaming(post.Id) - p.streamResultToPost(ctx, stream, post) + p.streamResultToPost(ctx, stream, post, llmBotName) }() return nil @@ -226,7 +226,7 @@ func (p *Plugin) finishPostStreaming(postID string) { // streamResultToPost streams the result of a TextStreamResult to a post. // it will internally handle logging needs and updating the post. -func (p *Plugin) streamResultToPost(ctx context.Context, stream *ai.TextStreamResult, post *model.Post) { +func (p *Plugin) streamResultToPost(ctx context.Context, stream *ai.TextStreamResult, post *model.Post, llmBotName string) { p.sendPostStreamingControlEvent(post, PostStreamingControlStart) defer func() { p.sendPostStreamingControlEvent(post, PostStreamingControlEnd) @@ -235,6 +235,8 @@ func (p *Plugin) streamResultToPost(ctx context.Context, stream *ai.TextStreamRe for { select { case next := <-stream.Stream: + p.metricsService.ObserveLLMTokensReceived(llmBotName, 1) + p.metricsService.ObserveLLMBytesReceived(llmBotName, int64(len(next))) post.Message += next p.sendPostStreamingUpdateEvent(post, post.Message) case err, ok := <-stream.Err: diff --git a/server/service.go b/server/service.go index 811c00e7..2e35f76c 100644 --- a/server/service.go +++ b/server/service.go @@ -39,7 +39,7 @@ func (p *Plugin) processUserRequestToBot(bot *Bot, context ai.ConversationContex RootId: context.Post.RootId, } responsePost.AddProp(RespondingToProp, context.Post.Id) - if err := p.streamResultToNewPost(bot.mmBot.UserId, context.RequestingUser.Id, result, responsePost); err != nil { + if err := p.streamResultToNewPost(bot.mmBot.UserId, context.RequestingUser.Id, result, responsePost, bot.cfg.Name); err != nil { return err } @@ -53,7 +53,7 @@ func (p *Plugin) newConversation(bot *Bot, context ai.ConversationContext) error } conversation.AddPost(p.PostToAIPost(bot, context.Post)) - result, err := p.getLLM(bot.cfg.Service).ChatCompletion(conversation) + result, err := p.getLLM(bot.cfg).ChatCompletion(conversation) if err != nil { return err } @@ -62,7 +62,7 @@ func (p *Plugin) newConversation(bot *Bot, context ai.ConversationContext) error ChannelId: context.Channel.Id, RootId: context.Post.Id, } - if err := p.streamResultToNewPost(bot.mmBot.UserId, context.RequestingUser.Id, result, responsePost); err != nil { + if err := p.streamResultToNewPost(bot.mmBot.UserId, context.RequestingUser.Id, result, responsePost, bot.cfg.Name); err != nil { return err } @@ -81,7 +81,7 @@ func (p *Plugin) generateTitle(bot *Bot, request string, threadRootID string) er titleRequest := ai.BotConversation{ Posts: []ai.Post{{Role: ai.PostRoleUser, Message: request}}, } - conversationTitle, err := p.getLLM(bot.cfg.Service).ChatCompletionNoStream(titleRequest, ai.WithMaxGeneratedTokens(25)) + conversationTitle, err := p.getLLM(bot.cfg).ChatCompletionNoStream(titleRequest, ai.WithMaxGeneratedTokens(25)) if err != nil { return fmt.Errorf("failed to get title: %w", err) } @@ -133,7 +133,7 @@ func (p *Plugin) continueConversation(bot *Bot, threadData *ThreadData, context } prompt.AppendConversation(p.ThreadToBotConversation(bot, threadData.Posts)) - result, err = p.getLLM(bot.cfg.Service).ChatCompletion(prompt) + result, err = p.getLLM(bot.cfg).ChatCompletion(prompt) if err != nil { return nil, err } @@ -156,7 +156,7 @@ func (p *Plugin) continueThreadConversation(bot *Bot, questionThreadData *Thread } prompt.AppendConversation(p.ThreadToBotConversation(bot, questionThreadData.Posts)) - result, err := p.getLLM(bot.cfg.Service).ChatCompletion(prompt) + result, err := p.getLLM(bot.cfg).ChatCompletion(prompt) if err != nil { return nil, err } @@ -180,7 +180,8 @@ func (p *Plugin) summarizePost(bot *Bot, postIDToSummarize string, context ai.Co if err != nil { return nil, err } - summaryStream, err := p.getLLM(bot.cfg.Service).ChatCompletion(prompt) + + summaryStream, err := p.getLLM(bot.cfg).ChatCompletion(prompt) if err != nil { return nil, err } @@ -209,7 +210,7 @@ func (p *Plugin) startNewSummaryThread(bot *Bot, postIDToSummarize string, conte } post := p.makeSummaryPost(postIDToSummarize) - if err := p.streamResultToNewDM(bot.mmBot.UserId, summaryStream, context.RequestingUser.Id, post); err != nil { + if err := p.streamResultToNewDM(bot.mmBot.UserId, summaryStream, context.RequestingUser.Id, post, bot.cfg.Name); err != nil { return nil, err } From d60781ca96d4057b7fcf8593cc094e61cd3afd6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Wed, 5 Jun 2024 15:02:17 +0200 Subject: [PATCH 2/5] Reduce the metrics to the basic --- server/ai/anthropic/anthropic.go | 31 +++--------- server/ai/asksage/asksage.go | 41 ++++----------- server/ai/openai/openai.go | 42 ++++++--------- server/api_channel.go | 4 +- server/api_post.go | 4 +- server/meeting_summarization.go | 14 ++--- server/metrics/metrics.go | 87 -------------------------------- server/plugin.go | 22 ++++---- server/post_processing.go | 12 ++--- server/service.go | 17 +++---- 10 files changed, 69 insertions(+), 205 deletions(-) diff --git a/server/ai/anthropic/anthropic.go b/server/ai/anthropic/anthropic.go index 80f8eca3..484cdcb3 100644 --- a/server/ai/anthropic/anthropic.go +++ b/server/ai/anthropic/anthropic.go @@ -4,28 +4,23 @@ import ( "fmt" "github.com/mattermost/mattermost-plugin-ai/server/ai" - "github.com/mattermost/mattermost-plugin-ai/server/metrics" ) const DefaultMaxTokens = 4096 type Anthropic struct { - client *Client - defaultModel string - tokenLimit int - metricsService metrics.Metrics - name string + client *Client + defaultModel string + tokenLimit int } -func New(botConfig ai.BotConfig, metricsService metrics.Metrics) *Anthropic { - client := NewClient(botConfig.Service.APIKey) +func New(llmService ai.ServiceConfig) *Anthropic { + client := NewClient(llmService.APIKey) return &Anthropic{ - client: client, - defaultModel: botConfig.Service.DefaultModel, - tokenLimit: botConfig.Service.TokenLimit, - metricsService: metricsService, - name: botConfig.Name, + client: client, + defaultModel: llmService.DefaultModel, + tokenLimit: llmService.TokenLimit, } } @@ -84,10 +79,6 @@ func (a *Anthropic) createCompletionRequest(conversation ai.BotConversation, opt } func (a *Anthropic) ChatCompletion(conversation ai.BotConversation, opts ...ai.LanguageModelOption) (*ai.TextStreamResult, error) { - a.metricsService.ObserveLLMRequest(a.name) - a.metricsService.ObserveLLMTokensSent(a.name, int64(a.CountTokens(conversation.String()))) - a.metricsService.ObserveLLMBytesSent(a.name, int64(len(conversation.String()))) - request := a.createCompletionRequest(conversation, opts) request.Stream = true result, err := a.client.MessageCompletion(request) @@ -99,10 +90,6 @@ func (a *Anthropic) ChatCompletion(conversation ai.BotConversation, opts ...ai.L } func (a *Anthropic) ChatCompletionNoStream(conversation ai.BotConversation, opts ...ai.LanguageModelOption) (string, error) { - a.metricsService.ObserveLLMRequest(a.name) - a.metricsService.ObserveLLMTokensSent(a.name, int64(a.CountTokens(conversation.String()))) - a.metricsService.ObserveLLMBytesSent(a.name, int64(len(conversation.String()))) - request := a.createCompletionRequest(conversation, opts) request.Stream = false result, err := a.client.MessageCompletionNoStream(request) @@ -110,8 +97,6 @@ func (a *Anthropic) ChatCompletionNoStream(conversation ai.BotConversation, opts return "", fmt.Errorf("failed to send query to anthropic: %w", err) } - a.metricsService.ObserveLLMTokensReceived(a.name, int64(a.CountTokens(result))) - a.metricsService.ObserveLLMBytesReceived(a.name, int64(len(result))) return result, nil } diff --git a/server/ai/asksage/asksage.go b/server/ai/asksage/asksage.go index 756069cb..1467e5e3 100644 --- a/server/ai/asksage/asksage.go +++ b/server/ai/asksage/asksage.go @@ -4,29 +4,24 @@ import ( "strings" "github.com/mattermost/mattermost-plugin-ai/server/ai" - "github.com/mattermost/mattermost-plugin-ai/server/metrics" ) type AskSage struct { - client *Client - defaultModel string - maxTokens int - metricsService metrics.Metrics - name string + client *Client + defaultModel string + maxTokens int } -func New(botConfig ai.BotConfig, metricsService metrics.Metrics) *AskSage { +func New(llmService ai.ServiceConfig) *AskSage { client := NewClient("") client.Login(GetTokenParams{ - Email: botConfig.Service.Username, - Password: botConfig.Service.Password, + Email: llmService.Username, + Password: llmService.Password, }) return &AskSage{ - client: client, - defaultModel: botConfig.Service.DefaultModel, - maxTokens: botConfig.Service.TokenLimit, - metricsService: metricsService, - name: botConfig.Name, + client: client, + defaultModel: llmService.DefaultModel, + maxTokens: llmService.TokenLimit, } } @@ -72,18 +67,14 @@ func (s *AskSage) queryParamsFromConfig(cfg ai.LLMConfig) QueryParams { func (s *AskSage) ChatCompletion(conversation ai.BotConversation, opts ...ai.LanguageModelOption) (*ai.TextStreamResult, error) { // Ask Sage does not support streaming. - result, err := s.ChatCompletionBase(conversation, opts...) + result, err := s.ChatCompletionNoStream(conversation, opts...) if err != nil { return nil, err } return ai.NewStreamFromString(result), nil } -func (s *AskSage) ChatCompletionBase(conversation ai.BotConversation, opts ...ai.LanguageModelOption) (string, error) { - s.metricsService.ObserveLLMRequest(s.name) - s.metricsService.ObserveLLMTokensSent(s.name, int64(s.CountTokens(conversation.String()))) - s.metricsService.ObserveLLMBytesSent(s.name, int64(len(conversation.String()))) - +func (s *AskSage) ChatCompletionNoStream(conversation ai.BotConversation, opts ...ai.LanguageModelOption) (string, error) { params := s.queryParamsFromConfig(s.createConfig(opts)) params.Message = conversationToMessagesList(conversation) params.SystemPrompt = conversation.ExtractSystemMessage() @@ -96,16 +87,6 @@ func (s *AskSage) ChatCompletionBase(conversation ai.BotConversation, opts ...ai return response.Message, nil } -func (s *AskSage) ChatCompletionNoStream(conversation ai.BotConversation, opts ...ai.LanguageModelOption) (string, error) { - response, err := s.ChatCompletionBase(conversation, opts...) - if err != nil { - return "", err - } - s.metricsService.ObserveLLMTokensReceived(s.name, int64(s.CountTokens(response))) - s.metricsService.ObserveLLMBytesReceived(s.name, int64(len(response))) - return response, nil -} - // TODO: Implement actual token counting. For now just estimated based off OpenAI estimations func (s *AskSage) CountTokens(text string) int { charCount := float64(len(text)) / 4.0 diff --git a/server/ai/openai/openai.go b/server/ai/openai/openai.go index c8b97297..21f41325 100644 --- a/server/ai/openai/openai.go +++ b/server/ai/openai/openai.go @@ -18,7 +18,6 @@ import ( "github.com/invopop/jsonschema" "github.com/mattermost/mattermost-plugin-ai/server/ai" "github.com/mattermost/mattermost-plugin-ai/server/ai/subtitles" - "github.com/mattermost/mattermost-plugin-ai/server/metrics" openaiClient "github.com/sashabaranov/go-openai" ) @@ -27,8 +26,6 @@ type OpenAI struct { defaultModel string tokenLimit int streamingTimeout time.Duration - metricsService metrics.Metrics - name string } const StreamingTimeoutDefault = 10 * time.Second @@ -39,10 +36,10 @@ const OpenAIMaxImageSize = 20 * 1024 * 1024 // 20 MB var ErrStreamingTimeout = errors.New("timeout streaming") -func NewCompatible(botConfig ai.BotConfig, metricsService metrics.Metrics) *OpenAI { - apiKey := botConfig.Service.APIKey - endpointURL := strings.TrimSuffix(botConfig.Service.APIURL, "/") - defaultModel := botConfig.Service.DefaultModel +func NewCompatible(llmService ai.ServiceConfig) *OpenAI { + apiKey := llmService.APIKey + endpointURL := strings.TrimSuffix(llmService.APIURL, "/") + defaultModel := llmService.DefaultModel config := openaiClient.DefaultConfig(apiKey) config.BaseURL = endpointURL @@ -53,39 +50,35 @@ func NewCompatible(botConfig ai.BotConfig, metricsService metrics.Metrics) *Open } streamingTimeout := StreamingTimeoutDefault - if botConfig.Service.StreamingTimeoutSeconds > 0 { - streamingTimeout = time.Duration(botConfig.Service.StreamingTimeoutSeconds) * time.Second + if llmService.StreamingTimeoutSeconds > 0 { + streamingTimeout = time.Duration(llmService.StreamingTimeoutSeconds) * time.Second } return &OpenAI{ client: openaiClient.NewClientWithConfig(config), defaultModel: defaultModel, - tokenLimit: botConfig.Service.TokenLimit, + tokenLimit: llmService.TokenLimit, streamingTimeout: streamingTimeout, - metricsService: metricsService, - name: botConfig.Name, } } -func New(botConfig ai.BotConfig, metricsService metrics.Metrics) *OpenAI { - defaultModel := botConfig.Service.DefaultModel +func New(llmService ai.ServiceConfig) *OpenAI { + defaultModel := llmService.DefaultModel if defaultModel == "" { defaultModel = openaiClient.GPT3Dot5Turbo } - config := openaiClient.DefaultConfig(botConfig.Service.APIKey) - config.OrgID = botConfig.Service.OrgID + config := openaiClient.DefaultConfig(llmService.APIKey) + config.OrgID = llmService.OrgID streamingTimeout := StreamingTimeoutDefault - if botConfig.Service.StreamingTimeoutSeconds > 0 { - streamingTimeout = time.Duration(botConfig.Service.StreamingTimeoutSeconds) * time.Second + if llmService.StreamingTimeoutSeconds > 0 { + streamingTimeout = time.Duration(llmService.StreamingTimeoutSeconds) * time.Second } return &OpenAI{ client: openaiClient.NewClientWithConfig(config), defaultModel: defaultModel, - tokenLimit: botConfig.Service.TokenLimit, + tokenLimit: llmService.TokenLimit, streamingTimeout: streamingTimeout, - metricsService: metricsService, - name: botConfig.Name, } } @@ -351,10 +344,6 @@ func (s *OpenAI) completionRequestFromConfig(cfg ai.LLMConfig) openaiClient.Chat } func (s *OpenAI) ChatCompletion(conversation ai.BotConversation, opts ...ai.LanguageModelOption) (*ai.TextStreamResult, error) { - s.metricsService.ObserveLLMRequest(s.name) - s.metricsService.ObserveLLMTokensSent(s.name, int64(s.CountTokens(conversation.String()))) - s.metricsService.ObserveLLMBytesSent(s.name, int64(len(conversation.String()))) - request := s.completionRequestFromConfig(s.createConfig(opts)) request = modifyCompletionRequestWithConversation(request, conversation) request.Stream = true @@ -367,8 +356,7 @@ func (s *OpenAI) ChatCompletionNoStream(conversation ai.BotConversation, opts .. if err != nil { return "", err } - data := result.ReadAll() - return data, nil + return result.ReadAll(), nil } func (s *OpenAI) Transcribe(file io.Reader) (*subtitles.Subtitles, error) { diff --git a/server/api_channel.go b/server/api_channel.go index 8de12341..46e84a52 100644 --- a/server/api_channel.go +++ b/server/api_channel.go @@ -109,7 +109,7 @@ func (p *Plugin) handleSince(c *gin.Context) { return } - resultStream, err := p.getLLM(bot.cfg).ChatCompletion(prompt) + resultStream, err := p.getLLM(bot.cfg.Service).ChatCompletion(prompt) if err != nil { c.AbortWithError(http.StatusInternalServerError, err) return @@ -117,7 +117,7 @@ func (p *Plugin) handleSince(c *gin.Context) { post := &model.Post{} post.AddProp(NoRegen, "true") - if err := p.streamResultToNewDM(bot.mmBot.UserId, resultStream, user.Id, post, bot.cfg.Name); err != nil { + if err := p.streamResultToNewDM(bot.mmBot.UserId, resultStream, user.Id, post); err != nil { c.AbortWithError(http.StatusInternalServerError, err) return } diff --git a/server/api_post.go b/server/api_post.go index d0a9f4ff..f4f4eeb9 100644 --- a/server/api_post.go +++ b/server/api_post.go @@ -66,7 +66,7 @@ func (p *Plugin) handleReact(c *gin.Context) { return } - emojiName, err := p.getLLM(bot.cfg).ChatCompletionNoStream(prompt, ai.WithMaxGeneratedTokens(25)) + emojiName, err := p.getLLM(bot.cfg.Service).ChatCompletionNoStream(prompt, ai.WithMaxGeneratedTokens(25)) if err != nil { c.AbortWithError(http.StatusInternalServerError, err) return @@ -369,7 +369,7 @@ func (p *Plugin) regeneratePost(bot *Bot, post *model.Post, user *model.User, ch } } - p.streamResultToPost(ctx, result, post, bot.cfg.Name) + p.streamResultToPost(ctx, result, post) return nil } diff --git a/server/meeting_summarization.go b/server/meeting_summarization.go index 3d6ae5cc..f7bca1d6 100644 --- a/server/meeting_summarization.go +++ b/server/meeting_summarization.go @@ -179,7 +179,7 @@ func (p *Plugin) newCallTranscriptionSummaryThread(bot *Bot, requestingUser *mod Message: "", } summaryPost.AddProp(ReferencedTranscriptPostID, transcriptionPost.Id) - if err := p.streamResultToNewPost(bot.mmBot.UserId, requestingUser.Id, summaryStream, summaryPost, bot.cfg.Name); err != nil { + if err := p.streamResultToNewPost(bot.mmBot.UserId, requestingUser.Id, summaryStream, summaryPost); err != nil { return fmt.Errorf("unable to stream result to post: %w", err) } @@ -237,7 +237,7 @@ func (p *Plugin) summarizeCallRecording(bot *Bot, rootID string, requestingUser } defer p.finishPostStreaming(transcriptPost.Id) - p.streamResultToPost(ctx, summaryStream, transcriptPost, bot.cfg.Name) + p.streamResultToPost(ctx, summaryStream, transcriptPost) return nil }() @@ -247,8 +247,8 @@ func (p *Plugin) summarizeCallRecording(bot *Bot, rootID string, requestingUser func (p *Plugin) summarizeTranscription(bot *Bot, transcription *subtitles.Subtitles, context ai.ConversationContext) (*ai.TextStreamResult, error) { llmFormattedTranscription := transcription.FormatForLLM() - tokens := p.getLLM(bot.cfg).CountTokens(llmFormattedTranscription) - tokenLimitWithMargin := int(float64(p.getLLM(bot.cfg).TokenLimit())*0.75) - ContextTokenMargin + tokens := p.getLLM(bot.cfg.Service).CountTokens(llmFormattedTranscription) + tokenLimitWithMargin := int(float64(p.getLLM(bot.cfg.Service).TokenLimit())*0.75) - ContextTokenMargin if tokenLimitWithMargin < 0 { tokenLimitWithMargin = ContextTokenMargin / 2 } @@ -265,7 +265,7 @@ func (p *Plugin) summarizeTranscription(bot *Bot, transcription *subtitles.Subti return nil, fmt.Errorf("unable to get summarize chunk prompt: %w", err) } - summarizedChunk, err := p.getLLM(bot.cfg).ChatCompletionNoStream(summarizeChunkPrompt) + summarizedChunk, err := p.getLLM(bot.cfg.Service).ChatCompletionNoStream(summarizeChunkPrompt) if err != nil { return nil, fmt.Errorf("unable to get summarized chunk: %w", err) } @@ -275,7 +275,7 @@ func (p *Plugin) summarizeTranscription(bot *Bot, transcription *subtitles.Subti llmFormattedTranscription = strings.Join(summarizedChunks, "\n\n") isChunked = true - p.pluginAPI.Log.Debug("Completed chunk summarization", "chunks", len(summarizedChunks), "tokens", p.getLLM(bot.cfg).CountTokens(llmFormattedTranscription)) + p.pluginAPI.Log.Debug("Completed chunk summarization", "chunks", len(summarizedChunks), "tokens", p.getLLM(bot.cfg.Service).CountTokens(llmFormattedTranscription)) } context.PromptParameters = map[string]string{"Transcription": llmFormattedTranscription, "IsChunked": fmt.Sprintf("%t", isChunked)} @@ -284,7 +284,7 @@ func (p *Plugin) summarizeTranscription(bot *Bot, transcription *subtitles.Subti return nil, fmt.Errorf("unable to get meeting summary prompt: %w", err) } - summaryStream, err := p.getLLM(bot.cfg).ChatCompletion(summaryPrompt) + summaryStream, err := p.getLLM(bot.cfg.Service).ChatCompletion(summaryPrompt) if err != nil { return nil, fmt.Errorf("unable to get meeting summary: %w", err) } diff --git a/server/metrics/metrics.go b/server/metrics/metrics.go index e89ac1f6..d862ef06 100644 --- a/server/metrics/metrics.go +++ b/server/metrics/metrics.go @@ -38,12 +38,6 @@ type Metrics interface { IncrementHTTPRequests() IncrementHTTPErrors() - - ObserveLLMRequest(llmID string) - ObserveLLMTokensSent(llmID string, count int64) - ObserveLLMTokensReceived(llmID string, count int64) - ObserveLLMBytesSent(llmID string, count int64) - ObserveLLMBytesReceived(llmID string, count int64) } type InstanceInfo struct { @@ -63,12 +57,6 @@ type metrics struct { httpRequestsTotal prometheus.Counter httpErrorsTotal prometheus.Counter - - llmRequestsTotal *prometheus.CounterVec - llmTokensSent *prometheus.CounterVec - llmTokensReceived *prometheus.CounterVec - llmBytesSent *prometheus.CounterVec - llmBytesReceived *prometheus.CounterVec } // NewMetrics Factory method to create a new metrics collector. @@ -140,51 +128,6 @@ func NewMetrics(info InstanceInfo) Metrics { }) m.registry.MustRegister(m.httpErrorsTotal) - m.llmRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: MetricsNamespace, - Subsystem: MetricsSubsystemLLM, - Name: "requests_total", - Help: "The total number of LLM requets made.", - ConstLabels: additionalLabels, - }, []string{"llm_name"}) - m.registry.MustRegister(m.llmRequestsTotal) - - m.llmTokensSent = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: MetricsNamespace, - Subsystem: MetricsSubsystemLLM, - Name: "tokens_sent_total", - Help: "The total number of tokens sent.", - ConstLabels: additionalLabels, - }, []string{"llm_name"}) - m.registry.MustRegister(m.llmTokensSent) - - m.llmTokensReceived = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: MetricsNamespace, - Subsystem: MetricsSubsystemLLM, - Name: "tokens_received_total", - Help: "The total number of tokens received.", - ConstLabels: additionalLabels, - }, []string{"llm_name"}) - m.registry.MustRegister(m.llmTokensReceived) - - m.llmBytesSent = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: MetricsNamespace, - Subsystem: MetricsSubsystemLLM, - Name: "bytes_sent_total", - Help: "The total number of bytes sent.", - ConstLabels: additionalLabels, - }, []string{"llm_name"}) - m.registry.MustRegister(m.llmBytesSent) - - m.llmBytesReceived = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: MetricsNamespace, - Subsystem: MetricsSubsystemLLM, - Name: "bytes_received_total", - Help: "The total number of bytes received.", - ConstLabels: additionalLabels, - }, []string{"llm_name"}) - m.registry.MustRegister(m.llmBytesReceived) - return m } @@ -198,36 +141,6 @@ func (m *metrics) ObserveAPIEndpointDuration(handler, method, statusCode string, } } -func (m *metrics) ObserveLLMRequest(llmID string) { - if m != nil { - m.llmRequestsTotal.With(prometheus.Labels{"llm_name": llmID}).Inc() - } -} - -func (m *metrics) ObserveLLMTokensSent(llmID string, count int64) { - if m != nil { - m.llmTokensSent.With(prometheus.Labels{"llm_name": llmID}).Add(float64(count)) - } -} - -func (m *metrics) ObserveLLMTokensReceived(llmID string, count int64) { - if m != nil { - m.llmTokensReceived.With(prometheus.Labels{"llm_name": llmID}).Add(float64(count)) - } -} - -func (m *metrics) ObserveLLMBytesSent(llmID string, count int64) { - if m != nil { - m.llmBytesSent.With(prometheus.Labels{"llm_name": llmID}).Add(float64(count)) - } -} - -func (m *metrics) ObserveLLMBytesReceived(llmID string, count int64) { - if m != nil { - m.llmBytesReceived.With(prometheus.Labels{"llm_name": llmID}).Add(float64(count)) - } -} - func (m *metrics) IncrementHTTPRequests() { if m != nil { m.httpRequestsTotal.Inc() diff --git a/server/plugin.go b/server/plugin.go index cc37de72..af509f56 100644 --- a/server/plugin.go +++ b/server/plugin.go @@ -121,17 +121,17 @@ func (p *Plugin) OnActivate() error { return nil } -func (p *Plugin) getLLM(llmBotConfig ai.BotConfig) ai.LanguageModel { +func (p *Plugin) getLLM(llmServiceConfig ai.ServiceConfig) ai.LanguageModel { var llm ai.LanguageModel - switch llmBotConfig.Service.Type { + switch llmServiceConfig.Type { case "openai": - llm = openai.New(llmBotConfig, p.metricsService) + llm = openai.New(llmServiceConfig) case "openaicompatible": - llm = openai.NewCompatible(llmBotConfig, p.metricsService) + llm = openai.NewCompatible(llmServiceConfig) case "anthropic": - llm = anthropic.New(llmBotConfig, p.metricsService) + llm = anthropic.New(llmServiceConfig) case "asksage": - llm = asksage.New(llmBotConfig, p.metricsService) + llm = asksage.New(llmServiceConfig) } cfg := p.getConfiguration() @@ -146,18 +146,18 @@ func (p *Plugin) getLLM(llmBotConfig ai.BotConfig) ai.LanguageModel { func (p *Plugin) getTranscribe() ai.Transcriber { cfg := p.getConfiguration() - var botConfig ai.BotConfig + var transcriptionService ai.ServiceConfig for _, bot := range cfg.Bots { if bot.Name == cfg.TranscriptGenerator { - botConfig = bot + transcriptionService = bot.Service break } } - switch botConfig.Service.Type { + switch transcriptionService.Type { case "openai": - return openai.New(botConfig, p.metricsService) + return openai.New(transcriptionService) case "openaicompatible": - return openai.NewCompatible(botConfig, p.metricsService) + return openai.NewCompatible(transcriptionService) } return nil } diff --git a/server/post_processing.go b/server/post_processing.go index 90823433..22b38b0e 100644 --- a/server/post_processing.go +++ b/server/post_processing.go @@ -128,7 +128,7 @@ func (p *Plugin) botDM(botid string, userID string, post *model.Post) error { return nil } -func (p *Plugin) streamResultToNewPost(botid string, requesterUserID string, stream *ai.TextStreamResult, post *model.Post, llmBotName string) error { +func (p *Plugin) streamResultToNewPost(botid string, requesterUserID string, stream *ai.TextStreamResult, post *model.Post) error { if err := p.botCreatePost(botid, requesterUserID, post); err != nil { return fmt.Errorf("unable to create post: %w", err) } @@ -140,13 +140,13 @@ func (p *Plugin) streamResultToNewPost(botid string, requesterUserID string, str go func() { defer p.finishPostStreaming(post.Id) - p.streamResultToPost(ctx, stream, post, llmBotName) + p.streamResultToPost(ctx, stream, post) }() return nil } -func (p *Plugin) streamResultToNewDM(botid string, stream *ai.TextStreamResult, userID string, post *model.Post, llmBotName string) error { +func (p *Plugin) streamResultToNewDM(botid string, stream *ai.TextStreamResult, userID string, post *model.Post) error { if err := p.botDM(botid, userID, post); err != nil { return err } @@ -158,7 +158,7 @@ func (p *Plugin) streamResultToNewDM(botid string, stream *ai.TextStreamResult, go func() { defer p.finishPostStreaming(post.Id) - p.streamResultToPost(ctx, stream, post, llmBotName) + p.streamResultToPost(ctx, stream, post) }() return nil @@ -226,7 +226,7 @@ func (p *Plugin) finishPostStreaming(postID string) { // streamResultToPost streams the result of a TextStreamResult to a post. // it will internally handle logging needs and updating the post. -func (p *Plugin) streamResultToPost(ctx context.Context, stream *ai.TextStreamResult, post *model.Post, llmBotName string) { +func (p *Plugin) streamResultToPost(ctx context.Context, stream *ai.TextStreamResult, post *model.Post) { p.sendPostStreamingControlEvent(post, PostStreamingControlStart) defer func() { p.sendPostStreamingControlEvent(post, PostStreamingControlEnd) @@ -235,8 +235,6 @@ func (p *Plugin) streamResultToPost(ctx context.Context, stream *ai.TextStreamRe for { select { case next := <-stream.Stream: - p.metricsService.ObserveLLMTokensReceived(llmBotName, 1) - p.metricsService.ObserveLLMBytesReceived(llmBotName, int64(len(next))) post.Message += next p.sendPostStreamingUpdateEvent(post, post.Message) case err, ok := <-stream.Err: diff --git a/server/service.go b/server/service.go index 2e35f76c..811c00e7 100644 --- a/server/service.go +++ b/server/service.go @@ -39,7 +39,7 @@ func (p *Plugin) processUserRequestToBot(bot *Bot, context ai.ConversationContex RootId: context.Post.RootId, } responsePost.AddProp(RespondingToProp, context.Post.Id) - if err := p.streamResultToNewPost(bot.mmBot.UserId, context.RequestingUser.Id, result, responsePost, bot.cfg.Name); err != nil { + if err := p.streamResultToNewPost(bot.mmBot.UserId, context.RequestingUser.Id, result, responsePost); err != nil { return err } @@ -53,7 +53,7 @@ func (p *Plugin) newConversation(bot *Bot, context ai.ConversationContext) error } conversation.AddPost(p.PostToAIPost(bot, context.Post)) - result, err := p.getLLM(bot.cfg).ChatCompletion(conversation) + result, err := p.getLLM(bot.cfg.Service).ChatCompletion(conversation) if err != nil { return err } @@ -62,7 +62,7 @@ func (p *Plugin) newConversation(bot *Bot, context ai.ConversationContext) error ChannelId: context.Channel.Id, RootId: context.Post.Id, } - if err := p.streamResultToNewPost(bot.mmBot.UserId, context.RequestingUser.Id, result, responsePost, bot.cfg.Name); err != nil { + if err := p.streamResultToNewPost(bot.mmBot.UserId, context.RequestingUser.Id, result, responsePost); err != nil { return err } @@ -81,7 +81,7 @@ func (p *Plugin) generateTitle(bot *Bot, request string, threadRootID string) er titleRequest := ai.BotConversation{ Posts: []ai.Post{{Role: ai.PostRoleUser, Message: request}}, } - conversationTitle, err := p.getLLM(bot.cfg).ChatCompletionNoStream(titleRequest, ai.WithMaxGeneratedTokens(25)) + conversationTitle, err := p.getLLM(bot.cfg.Service).ChatCompletionNoStream(titleRequest, ai.WithMaxGeneratedTokens(25)) if err != nil { return fmt.Errorf("failed to get title: %w", err) } @@ -133,7 +133,7 @@ func (p *Plugin) continueConversation(bot *Bot, threadData *ThreadData, context } prompt.AppendConversation(p.ThreadToBotConversation(bot, threadData.Posts)) - result, err = p.getLLM(bot.cfg).ChatCompletion(prompt) + result, err = p.getLLM(bot.cfg.Service).ChatCompletion(prompt) if err != nil { return nil, err } @@ -156,7 +156,7 @@ func (p *Plugin) continueThreadConversation(bot *Bot, questionThreadData *Thread } prompt.AppendConversation(p.ThreadToBotConversation(bot, questionThreadData.Posts)) - result, err := p.getLLM(bot.cfg).ChatCompletion(prompt) + result, err := p.getLLM(bot.cfg.Service).ChatCompletion(prompt) if err != nil { return nil, err } @@ -180,8 +180,7 @@ func (p *Plugin) summarizePost(bot *Bot, postIDToSummarize string, context ai.Co if err != nil { return nil, err } - - summaryStream, err := p.getLLM(bot.cfg).ChatCompletion(prompt) + summaryStream, err := p.getLLM(bot.cfg.Service).ChatCompletion(prompt) if err != nil { return nil, err } @@ -210,7 +209,7 @@ func (p *Plugin) startNewSummaryThread(bot *Bot, postIDToSummarize string, conte } post := p.makeSummaryPost(postIDToSummarize) - if err := p.streamResultToNewDM(bot.mmBot.UserId, summaryStream, context.RequestingUser.Id, post, bot.cfg.Name); err != nil { + if err := p.streamResultToNewDM(bot.mmBot.UserId, summaryStream, context.RequestingUser.Id, post); err != nil { return nil, err } From b6b93db11d2635a215a4934d43656ada9fd2f6ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Wed, 5 Jun 2024 15:10:50 +0200 Subject: [PATCH 3/5] Simplifiying a bit the code --- server/metrics.go | 21 +++++++++++++++++++++ server/middleware.go | 37 ------------------------------------- 2 files changed, 21 insertions(+), 37 deletions(-) delete mode 100644 server/middleware.go diff --git a/server/metrics.go b/server/metrics.go index 96cd3c4e..68ac8107 100644 --- a/server/metrics.go +++ b/server/metrics.go @@ -2,7 +2,10 @@ package main import ( "net/http" + "strconv" + "time" + "github.com/gin-gonic/gin" "github.com/mattermost/mattermost-plugin-ai/server/metrics" "github.com/mattermost/mattermost/server/public/plugin" ) @@ -14,3 +17,21 @@ func (p *Plugin) ServeMetrics(_ *plugin.Context, w http.ResponseWriter, r *http. func (p *Plugin) GetMetrics() metrics.Metrics { return p.metricsService } + +func (p *Plugin) metricsMiddleware(c *gin.Context) { + p.GetMetrics().IncrementHTTPRequests() + now := time.Now() + + c.Next() + + elapsed := float64(time.Since(now)) / float64(time.Second) + + status := c.Writer.Status() + + if status < 200 || status > 299 { + p.GetMetrics().IncrementHTTPErrors() + } + + endpoint := c.HandlerName() + p.GetMetrics().ObserveAPIEndpointDuration(endpoint, c.Request.Method, strconv.Itoa(status), elapsed) +} diff --git a/server/middleware.go b/server/middleware.go deleted file mode 100644 index 9b0bd24e..00000000 --- a/server/middleware.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "net/http" - "strconv" - "time" - - "github.com/gin-gonic/gin" -) - -type StatusRecorder struct { - http.ResponseWriter - Status int -} - -func (r *StatusRecorder) WriteHeader(status int) { - r.Status = status - r.ResponseWriter.WriteHeader(status) -} - -func (p *Plugin) metricsMiddleware(c *gin.Context) { - p.GetMetrics().IncrementHTTPRequests() - now := time.Now() - - c.Next() - - elapsed := float64(time.Since(now)) / float64(time.Second) - - status := c.Writer.Status() - - if status < 200 || status > 299 { - p.GetMetrics().IncrementHTTPErrors() - } - - endpoint := c.HandlerName() - p.GetMetrics().ObserveAPIEndpointDuration(endpoint, c.Request.Method, strconv.Itoa(status), elapsed) -} From ae61696789c4f11f2b35348a0cec26f1fd7c443c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Wed, 5 Jun 2024 15:11:52 +0200 Subject: [PATCH 4/5] More cleanup --- server/metrics/metrics.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/server/metrics/metrics.go b/server/metrics/metrics.go index d862ef06..7791d184 100644 --- a/server/metrics/metrics.go +++ b/server/metrics/metrics.go @@ -9,26 +9,11 @@ import ( const ( MetricsNamespace = "copilot" MetricsSubsystemSystem = "system" - MetricsSubsystemApp = "app" MetricsSubsystemHTTP = "http" MetricsSubsystemAPI = "api" - MetricsSubsystemEvents = "events" - MetricsSubsystemDB = "db" - MetricsSubsystemLLM = "llm" MetricsCloudInstallationLabel = "installationId" MetricsVersionLabel = "version" - - ActionSourceMSTeams = "msteams" - ActionSourceMattermost = "mattermost" - ActionCreated = "created" - ActionUpdated = "updated" - ActionDeleted = "deleted" - ReactionSetAction = "set" - ReactionUnsetAction = "unset" - SubscriptionRefreshed = "refreshed" - SubscriptionConnected = "connected" - SubscriptionReconnected = "reconnected" ) type Metrics interface { From 07027662eb673dff07de49047a59971ca2ac8204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Thu, 6 Jun 2024 19:34:11 +0200 Subject: [PATCH 5/5] Fixing panic --- server/metrics.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/metrics.go b/server/metrics.go index 68ac8107..cd5c3e48 100644 --- a/server/metrics.go +++ b/server/metrics.go @@ -19,6 +19,11 @@ func (p *Plugin) GetMetrics() metrics.Metrics { } func (p *Plugin) metricsMiddleware(c *gin.Context) { + metrics := p.GetMetrics() + if metrics == nil { + c.Next() + return + } p.GetMetrics().IncrementHTTPRequests() now := time.Now()