From a5353cb0b082b10448d11e0a6cc837efb9ada190 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Fri, 10 Feb 2023 15:22:11 -0500 Subject: [PATCH 01/31] refactor: use db selector --- internal/repo/account.go | 57 ++++++++++++---------------------------- 1 file changed, 17 insertions(+), 40 deletions(-) diff --git a/internal/repo/account.go b/internal/repo/account.go index 09da252c..200f55e3 100644 --- a/internal/repo/account.go +++ b/internal/repo/account.go @@ -2,26 +2,30 @@ package repo import ( "context" - "database/sql" "time" - "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/uptrace/bun" "exusiai.dev/backend-next/internal/model" "exusiai.dev/backend-next/internal/pkg/pgerr" "exusiai.dev/backend-next/internal/pkg/pgid" + "exusiai.dev/backend-next/internal/repo/selector" ) const AccountMaxRetries = 100 type Account struct { + sel selector.S[model.Account] + db *bun.DB } func NewAccount(db *bun.DB) *Account { - return &Account{db: db} + return &Account{ + db: db, + sel: selector.New[model.Account](db), + } } func (c *Account) CreateAccountWithRandomPenguinId(ctx context.Context) (*model.Account, error) { @@ -59,51 +63,24 @@ func (c *Account) CreateAccountWithRandomPenguinId(ctx context.Context) (*model. } func (c *Account) GetAccountById(ctx context.Context, accountId string) (*model.Account, error) { - var account model.Account - - err := c.db.NewSelect(). - Model(&account). - Column("account_id"). - Where("account_id = ?", accountId). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &account, nil + return c.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("account_id = ?", accountId) + }) } func (c *Account) GetAccountByPenguinId(ctx context.Context, penguinId string) (*model.Account, error) { - var account model.Account - - err := c.db.NewSelect(). - Model(&account). - Where("penguin_id = ?", penguinId). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &account, nil + return c.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("penguin_id = ?", penguinId) + }) } func (c *Account) IsAccountExistWithId(ctx context.Context, accountId int) bool { - var account model.Account - - err := c.db.NewSelect(). - Model(&account). - Column("account_id"). - Where("account_id = ?", accountId). - Scan(ctx, &account) + account, err := c.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Column("account_id").Where("account_id = ?", accountId) + }) if err != nil { return false } - return account.AccountID > 0 + return account != nil } From 5741babc70feef1bc8a137a5ce0ef56fac727850 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Fri, 10 Feb 2023 15:23:18 -0500 Subject: [PATCH 02/31] refactor: use db selector --- internal/repo/activity.go | 23 ++++++++--------------- internal/repo/snapshot.go | 8 ++++---- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/internal/repo/activity.go b/internal/repo/activity.go index 9dfc6d29..37d256e2 100644 --- a/internal/repo/activity.go +++ b/internal/repo/activity.go @@ -2,32 +2,25 @@ package repo import ( "context" - "database/sql" - "github.com/pkg/errors" "github.com/uptrace/bun" "exusiai.dev/backend-next/internal/model" + "exusiai.dev/backend-next/internal/repo/selector" ) type Activity struct { - DB *bun.DB + db *bun.DB + + sel selector.S[model.Activity] } func NewActivity(db *bun.DB) *Activity { - return &Activity{DB: db} + return &Activity{db: db, sel: selector.New[model.Activity](db)} } func (c *Activity) GetActivities(ctx context.Context) ([]*model.Activity, error) { - var activities []*model.Activity - err := c.DB.NewSelect(). - Model(&activities). - Order("activity_id ASC"). - Scan(ctx) - - if err != nil && !errors.Is(err, sql.ErrNoRows) { - return nil, err - } - - return activities, nil + return c.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Order("activity_id ASC") + }) } diff --git a/internal/repo/snapshot.go b/internal/repo/snapshot.go index bfda3554..dc426bad 100644 --- a/internal/repo/snapshot.go +++ b/internal/repo/snapshot.go @@ -10,14 +10,14 @@ import ( ) type Snapshot struct { - sel selector.S[model.Snapshot] + db *bun.DB - DB *bun.DB + sel selector.S[model.Snapshot] } func NewSnapshot(db *bun.DB) *Snapshot { return &Snapshot{ - DB: db, + db: db, sel: selector.New[model.Snapshot](db), } } @@ -47,7 +47,7 @@ func (s *Snapshot) GetSnapshotsByVersions(ctx context.Context, key string, versi } func (s *Snapshot) SaveSnapshot(ctx context.Context, snapshot *model.Snapshot) (*model.Snapshot, error) { - _, err := s.DB.NewInsert(). + _, err := s.db.NewInsert(). Model(snapshot). Exec(ctx) if err != nil { From 9d0006ce0f2057a9e594c6f713f91ce9f83a4864 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Fri, 10 Feb 2023 15:32:52 -0500 Subject: [PATCH 03/31] refactor: use db selector --- go.mod | 5 ++- go.sum | 6 ++-- internal/repo/account.go | 3 +- internal/repo/activity.go | 3 +- internal/repo/drop_info.go | 63 ++++++++++---------------------------- internal/repo/snapshot.go | 3 +- 6 files changed, 24 insertions(+), 59 deletions(-) diff --git a/go.mod b/go.mod index db9d65c5..f0fba648 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/ahmetb/go-linq/v3 v3.2.0 github.com/ansrivas/fiberprometheus/v2 v2.5.0 github.com/antonmedv/expr v1.12.0 - github.com/avast/retry-go/v3 v3.1.1 + github.com/avast/retry-go/v4 v4.3.3 github.com/davecgh/go-spew v1.1.1 github.com/dchest/uniuri v1.2.0 github.com/felixge/fgprof v0.9.3 @@ -38,7 +38,7 @@ require ( github.com/stretchr/testify v1.8.1 github.com/swaggo/swag v1.8.10 github.com/tidwall/gjson v1.14.4 - github.com/uptrace/bun v1.1.11 + github.com/uptrace/bun v1.1.10 github.com/uptrace/bun/dialect/pgdialect v1.1.10 github.com/uptrace/bun/driver/pgdriver v1.1.10 github.com/uptrace/bun/extra/bundebug v1.1.10 @@ -67,7 +67,6 @@ require ( github.com/DataDog/datadog-go/v5 v5.0.2 // indirect github.com/DataDog/gostackparse v0.5.0 // indirect github.com/Microsoft/go-winio v0.5.1 // indirect - github.com/avast/retry-go/v4 v4.3.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect diff --git a/go.sum b/go.sum index 6c0377bc..063d36dd 100644 --- a/go.sum +++ b/go.sum @@ -67,8 +67,6 @@ github.com/ansrivas/fiberprometheus/v2 v2.5.0/go.mod h1:j7n4IeswBRIdeQ6Lr8lYqBq6 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antonmedv/expr v1.12.0 h1:hIOn7jjY86E09PXvn9zgdt2FbWVru0ud9Rm5DbNoYNw= github.com/antonmedv/expr v1.12.0/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU= -github.com/avast/retry-go/v3 v3.1.1 h1:49Scxf4v8PmiQ/nY0aY3p0hDueqSmc7++cBbtiDGu2g= -github.com/avast/retry-go/v3 v3.1.1/go.mod h1:6cXRK369RpzFL3UQGqIUp9Q7GDrams+KsYWrfNA1/nQ= github.com/avast/retry-go/v4 v4.3.3 h1:G56Bp6mU0b5HE1SkaoVjscZjlQb0oy4mezwY/cGH19w= github.com/avast/retry-go/v4 v4.3.3/go.mod h1:rg6XFaiuFYII0Xu3RDbZQkxCofFwruZKW8oEF1jpWiU= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= @@ -476,8 +474,8 @@ github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= -github.com/uptrace/bun v1.1.11 h1:PVT+DHdLX13tIJaBoT0tkjh1TOWOybW/pz3TKiOB9lo= -github.com/uptrace/bun v1.1.11/go.mod h1:ysoB7l3gioKLBaZH9wKuJAoeBiOthb/hTyKdbXKcaxg= +github.com/uptrace/bun v1.1.10 h1:wx80lg32mWlOV1gNvmPTbR9znt6A2fYvvURzU5dzSPE= +github.com/uptrace/bun v1.1.10/go.mod h1:pOWJSVU+W0jp3+cbn/TtHdcG/h+QxagECw4EYdKiaSw= github.com/uptrace/bun/dialect/pgdialect v1.1.10 h1:H25ieb5OJV5ywvKvswF++wMTnePRaGRiSNkYCRXrQxc= github.com/uptrace/bun/dialect/pgdialect v1.1.10/go.mod h1:leDSw/IC70/GYPIU3zC8fkOZpJaJ28f51OMT1VnZiY8= github.com/uptrace/bun/driver/pgdriver v1.1.10 h1:xmabc3eG5rz8FadFF3GrQiGs5XvXlz6CJUMZhne+1sM= diff --git a/internal/repo/account.go b/internal/repo/account.go index 200f55e3..35fe293f 100644 --- a/internal/repo/account.go +++ b/internal/repo/account.go @@ -16,9 +16,8 @@ import ( const AccountMaxRetries = 100 type Account struct { + db *bun.DB sel selector.S[model.Account] - - db *bun.DB } func NewAccount(db *bun.DB) *Account { diff --git a/internal/repo/activity.go b/internal/repo/activity.go index 37d256e2..bbd92a76 100644 --- a/internal/repo/activity.go +++ b/internal/repo/activity.go @@ -10,8 +10,7 @@ import ( ) type Activity struct { - db *bun.DB - + db *bun.DB sel selector.S[model.Activity] } diff --git a/internal/repo/drop_info.go b/internal/repo/drop_info.go index f0c8f094..ce95f239 100644 --- a/internal/repo/drop_info.go +++ b/internal/repo/drop_info.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "exusiai.dev/gommon/constant" "github.com/ahmetb/go-linq/v3" "github.com/pkg/errors" "github.com/samber/lo" @@ -17,64 +18,34 @@ import ( "exusiai.dev/backend-next/internal/model" "exusiai.dev/backend-next/internal/pkg/pgerr" "exusiai.dev/backend-next/internal/pkg/pgqry" - "exusiai.dev/gommon/constant" + "exusiai.dev/backend-next/internal/repo/selector" ) type DropInfo struct { - DB *bun.DB + db *bun.DB + sel selector.S[model.DropInfo] } func NewDropInfo(db *bun.DB) *DropInfo { - return &DropInfo{DB: db} + return &DropInfo{db: db, sel: selector.New[model.DropInfo](db)} } func (s *DropInfo) GetDropInfo(ctx context.Context, id int) (*model.DropInfo, error) { - var dropInfo model.DropInfo - err := s.DB.NewSelect(). - Model(&dropInfo). - Where("id = ?", id). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &dropInfo, nil + return s.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("id = ?", id) + }) } func (s *DropInfo) GetDropInfosByServerAndStageId(ctx context.Context, server string, stageId int) ([]*model.DropInfo, error) { - var dropInfo []*model.DropInfo - err := s.DB.NewSelect(). - Model(&dropInfo). - Where("stage_id = ?", stageId). - Where("server = ?", server). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return dropInfo, nil + return s.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("stage_id = ?", stageId).Where("server = ?", server) + }) } func (s *DropInfo) GetDropInfosByServer(ctx context.Context, server string) ([]*model.DropInfo, error) { - var dropInfo []*model.DropInfo - err := s.DB.NewSelect(). - Model(&dropInfo). - Where("server = ?", server). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return dropInfo, nil + return s.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("server = ?", server) + }) } type DropInfoQuery struct { @@ -86,7 +57,7 @@ type DropInfoQuery struct { func (s *DropInfo) GetForCurrentTimeRange(ctx context.Context, query *DropInfoQuery) ([]*model.DropInfo, error) { var dropInfo []*model.DropInfo err := pgqry.New( - s.DB.NewSelect(). + s.db.NewSelect(). Model(&dropInfo). Where("di.server = ?", query.Server). Where("st.ark_stage_id = ?", query.ArkStageId), @@ -108,7 +79,7 @@ func (s *DropInfo) GetForCurrentTimeRange(ctx context.Context, query *DropInfoQu func (s *DropInfo) GetItemDropSetByStageIdAndRangeId(ctx context.Context, server string, stageId int, rangeId int) ([]int, error) { var results []int - err := s.DB.NewSelect(). + err := s.db.NewSelect(). Column("di.item_id"). Model((*model.DropInfo)(nil)). Where("di.server = ?", server). @@ -186,7 +157,7 @@ func (s *DropInfo) GetDropInfosWithFilters(ctx context.Context, server string, t } } } - if err := s.DB.NewSelect().TableExpr("drop_infos as di").Column("di.stage_id", "di.item_id", "di.accumulable"). + if err := s.db.NewSelect().TableExpr("drop_infos as di").Column("di.stage_id", "di.item_id", "di.accumulable"). Where(whereBuilder.String(), server, constant.DropTypeRecognitionOnly, bun.In(stageIdFilter), bun.In(itemIdFilter)). Join("JOIN time_ranges AS tr ON tr.range_id = di.range_id"). Scan(ctx, &results); err != nil { diff --git a/internal/repo/snapshot.go b/internal/repo/snapshot.go index dc426bad..ef027501 100644 --- a/internal/repo/snapshot.go +++ b/internal/repo/snapshot.go @@ -10,8 +10,7 @@ import ( ) type Snapshot struct { - db *bun.DB - + db *bun.DB sel selector.S[model.Snapshot] } From 0db79036491f859d3897f13522931a24da87fed9 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Fri, 10 Feb 2023 15:34:30 -0500 Subject: [PATCH 04/31] refactor: use db selector --- internal/repo/drop_matrix_element.go | 6 ++- internal/repo/drop_pattern_element.go | 69 +++++++-------------------- 2 files changed, 20 insertions(+), 55 deletions(-) diff --git a/internal/repo/drop_matrix_element.go b/internal/repo/drop_matrix_element.go index e04adb5d..7983154c 100644 --- a/internal/repo/drop_matrix_element.go +++ b/internal/repo/drop_matrix_element.go @@ -8,14 +8,16 @@ import ( "github.com/uptrace/bun" "exusiai.dev/backend-next/internal/model" + "exusiai.dev/backend-next/internal/repo/selector" ) type DropMatrixElement struct { - db *bun.DB + db *bun.DB + sel selector.S[model.DropMatrixElement] } func NewDropMatrixElement(db *bun.DB) *DropMatrixElement { - return &DropMatrixElement{db: db} + return &DropMatrixElement{db: db, sel: selector.New[model.DropMatrixElement](db)} } func (s *DropMatrixElement) BatchSaveElements(ctx context.Context, elements []*model.DropMatrixElement, server string) error { diff --git a/internal/repo/drop_pattern_element.go b/internal/repo/drop_pattern_element.go index f73683ff..69861e22 100644 --- a/internal/repo/drop_pattern_element.go +++ b/internal/repo/drop_pattern_element.go @@ -9,62 +9,34 @@ import ( "exusiai.dev/backend-next/internal/model" "exusiai.dev/backend-next/internal/model/types" - "exusiai.dev/backend-next/internal/pkg/pgerr" + "exusiai.dev/backend-next/internal/repo/selector" ) type DropPatternElement struct { - DB *bun.DB + db *bun.DB + sel selector.S[model.DropPatternElement] } func NewDropPatternElement(db *bun.DB) *DropPatternElement { - return &DropPatternElement{DB: db} + return &DropPatternElement{db: db, sel: selector.New[model.DropPatternElement](db)} } func (r *DropPatternElement) GetDropPatternElements(ctx context.Context) ([]*model.DropPatternElement, error) { - var elements []*model.DropPatternElement - err := r.DB.NewSelect(). - Model(&elements). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return elements, nil + return r.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q + }) } func (r *DropPatternElement) GetDropPatternElementById(ctx context.Context, id int) (*model.DropPatternElement, error) { - var DropPatternElement model.DropPatternElement - err := r.DB.NewSelect(). - Model(&DropPatternElement). - Where("id = ?", id). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &DropPatternElement, nil + return r.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("id = ?", id) + }) } func (r *DropPatternElement) GetDropPatternElementByHash(ctx context.Context, hash string) (*model.DropPatternElement, error) { - var DropPatternElement model.DropPatternElement - err := r.DB.NewSelect(). - Model(&DropPatternElement). - Where("hash = ?", hash). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &DropPatternElement, nil + return r.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("hash = ?", hash) + }) } func (r *DropPatternElement) CreateDropPatternElements(ctx context.Context, tx bun.Tx, patternId int, drops []*types.Drop) ([]*model.DropPatternElement, error) { @@ -94,16 +66,7 @@ func (r *DropPatternElement) CreateDropPatternElements(ctx context.Context, tx b } func (r *DropPatternElement) GetDropPatternElementsByPatternId(ctx context.Context, patternId int) ([]*model.DropPatternElement, error) { - var elements []*model.DropPatternElement - err := r.DB.NewSelect(). - Model(&elements). - Where("drop_pattern_id = ?", patternId). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - elements = make([]*model.DropPatternElement, 0) - } else if err != nil { - return nil, err - } - return elements, nil + return r.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("drop_pattern_id = ?", patternId) + }) } From 0773713f50d3ce146373384ea09f5df52af47f08 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Fri, 10 Feb 2023 15:36:25 -0500 Subject: [PATCH 05/31] refactor: use db selector --- internal/repo/drop_pattern.go | 54 +++++++-------------------- internal/repo/drop_pattern_element.go | 2 - internal/repo/drop_report_extra.go | 28 ++++---------- 3 files changed, 21 insertions(+), 63 deletions(-) diff --git a/internal/repo/drop_pattern.go b/internal/repo/drop_pattern.go index 20c7fc39..a05c6d00 100644 --- a/internal/repo/drop_pattern.go +++ b/internal/repo/drop_pattern.go @@ -14,62 +14,34 @@ import ( "exusiai.dev/backend-next/internal/model" "exusiai.dev/backend-next/internal/model/types" - "exusiai.dev/backend-next/internal/pkg/pgerr" + "exusiai.dev/backend-next/internal/repo/selector" ) type DropPattern struct { - DB *bun.DB + db *bun.DB + sel selector.S[model.DropPattern] } func NewDropPattern(db *bun.DB) *DropPattern { - return &DropPattern{DB: db} + return &DropPattern{db: db, sel: selector.New[model.DropPattern](db)} } func (s *DropPattern) GetDropPatterns(ctx context.Context) ([]*model.DropPattern, error) { - dropPatterns := make([]*model.DropPattern, 0) - err := s.DB.NewSelect(). - Model(&dropPatterns). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return dropPatterns, nil + return s.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q + }) } func (s *DropPattern) GetDropPatternById(ctx context.Context, id int) (*model.DropPattern, error) { - var dropPattern model.DropPattern - err := s.DB.NewSelect(). - Model(&dropPattern). - Where("id = ?", id). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &dropPattern, nil + return s.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("id = ?", id) + }) } func (s *DropPattern) GetDropPatternByHash(ctx context.Context, hash string) (*model.DropPattern, error) { - var dropPattern model.DropPattern - err := s.DB.NewSelect(). - Model(&dropPattern). - Where("hash = ?", hash). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &dropPattern, nil + return s.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("hash = ?", hash) + }) } func (s *DropPattern) GetOrCreateDropPatternFromDrops(ctx context.Context, tx bun.Tx, drops []*types.Drop) (*model.DropPattern, bool, error) { diff --git a/internal/repo/drop_pattern_element.go b/internal/repo/drop_pattern_element.go index 69861e22..688ce72b 100644 --- a/internal/repo/drop_pattern_element.go +++ b/internal/repo/drop_pattern_element.go @@ -2,9 +2,7 @@ package repo import ( "context" - "database/sql" - "github.com/pkg/errors" "github.com/uptrace/bun" "exusiai.dev/backend-next/internal/model" diff --git a/internal/repo/drop_report_extra.go b/internal/repo/drop_report_extra.go index f6144d31..f0d80e99 100644 --- a/internal/repo/drop_report_extra.go +++ b/internal/repo/drop_report_extra.go @@ -2,44 +2,32 @@ package repo import ( "context" - "database/sql" - "github.com/pkg/errors" "github.com/uptrace/bun" "exusiai.dev/backend-next/internal/model" - "exusiai.dev/backend-next/internal/pkg/pgerr" + "exusiai.dev/backend-next/internal/repo/selector" ) type DropReportExtra struct { - DB *bun.DB + db *bun.DB + sel selector.S[model.DropReportExtra] } func NewDropReportExtra(db *bun.DB) *DropReportExtra { - return &DropReportExtra{DB: db} + return &DropReportExtra{db: db, sel: selector.New[model.DropReportExtra](db)} } func (c *DropReportExtra) GetDropReportExtraById(ctx context.Context, id int) (*model.DropReportExtra, error) { - var dropReportExtra model.DropReportExtra - - err := c.DB.NewSelect(). - Model(&dropReportExtra). - Where("report_id = ?", id). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &dropReportExtra, nil + return c.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("report_id = ?", id) + }) } func (c *DropReportExtra) IsDropReportExtraMD5Exist(ctx context.Context, md5 string) bool { var dropReportExtra model.DropReportExtra - count, err := c.DB.NewSelect(). + count, err := c.db.NewSelect(). Model(&dropReportExtra). Where("md5 = ?", md5). Count(ctx) From 440ac8d28bf95ca533132f661bfe56c279815c41 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Fri, 10 Feb 2023 15:38:17 -0500 Subject: [PATCH 06/31] refactor: do not use DB in repo --- internal/controller/meta/admin.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/controller/meta/admin.go b/internal/controller/meta/admin.go index e60cd0e1..0cb679fc 100644 --- a/internal/controller/meta/admin.go +++ b/internal/controller/meta/admin.go @@ -34,6 +34,7 @@ import ( type AdminController struct { fx.In + DB *bun.DB PatternRepo *repo.DropPattern PatternElementRepo *repo.DropPatternElement RecognitionDefectRepo *repo.RecognitionDefect @@ -474,7 +475,7 @@ func (c *AdminController) RejectRulesReevaluationApply(ctx *fiber.Ctx) error { changeSet := evaluation.ChangeSet() - err = c.DropReportRepo.DB.RunInTx(ctx.UserContext(), nil, func(ictx context.Context, tx bun.Tx) error { + err = c.DB.RunInTx(ctx.UserContext(), nil, func(ictx context.Context, tx bun.Tx) error { for _, change := range changeSet { log.Debug(). Str("evt.name", "admin.reject_rules.reevaluation.apply"). From fb6d949b06a80ea5eb1a91b96f9512d6c3afc0e1 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Fri, 10 Feb 2023 15:39:28 -0500 Subject: [PATCH 07/31] refactor: do not use DB in repo --- internal/repo/drop_report.go | 45 +++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/internal/repo/drop_report.go b/internal/repo/drop_report.go index 8efcde98..39f6bda3 100644 --- a/internal/repo/drop_report.go +++ b/internal/repo/drop_report.go @@ -15,15 +15,18 @@ import ( modelv2 "exusiai.dev/backend-next/internal/model/v2" "exusiai.dev/backend-next/internal/pkg/gameday" "exusiai.dev/backend-next/internal/pkg/pgqry" + "exusiai.dev/backend-next/internal/repo/selector" ) type DropReport struct { - DB *bun.DB + db *bun.DB + sel selector.S[model.DropReport] } func NewDropReport(db *bun.DB) *DropReport { return &DropReport{ - DB: db, + db: db, + sel: selector.New[model.DropReport](db), } } @@ -35,7 +38,7 @@ func (s *DropReport) CreateDropReport(ctx context.Context, tx bun.Tx, dropReport } func (s *DropReport) DeleteDropReport(ctx context.Context, reportId int) error { - _, err := s.DB.NewUpdate(). + _, err := s.db.NewUpdate(). Model((*model.DropReport)(nil)). Set("reliability = ?", -1). Where("report_id = ?", reportId). @@ -60,7 +63,7 @@ func (s *DropReport) CalcTotalQuantityForDropMatrix( return results, nil } - subq1 := s.DB.NewSelect(). + subq1 := s.db.NewSelect(). TableExpr("drop_reports AS dr"). Column("dr.stage_id", "dr.source_name", "dpe.item_id", "dpe.quantity"). Join("JOIN drop_pattern_elements AS dpe ON dpe.drop_pattern_id = dr.pattern_id") @@ -69,7 +72,7 @@ func (s *DropReport) CalcTotalQuantityForDropMatrix( s.handleServer(subq1, server) s.handleStagesAndItems(subq1, stageIdItemIdMap) - mainq := s.DB.NewSelect(). + mainq := s.db.NewSelect(). TableExpr("(?) AS a", subq1). Column("stage_id", "item_id"). ColumnExpr("SUM(quantity) AS total_quantity") @@ -91,7 +94,7 @@ func (s *DropReport) CalcTotalQuantityForPatternMatrix( return results, nil } - subq1 := s.DB.NewSelect(). + subq1 := s.db.NewSelect(). TableExpr("drop_reports AS dr"). Column("dr.source_name", "dr.stage_id", "dr.pattern_id") s.handleAccountAndReliability(subq1, accountId) @@ -100,7 +103,7 @@ func (s *DropReport) CalcTotalQuantityForPatternMatrix( s.handleStages(subq1, stageIds) s.handleTimes(subq1, 1) - mainq := s.DB.NewSelect(). + mainq := s.db.NewSelect(). TableExpr("(?) AS a", subq1). Column("stage_id", "pattern_id"). ColumnExpr("COUNT(*) AS total_quantity") @@ -122,7 +125,7 @@ func (s *DropReport) CalcTotalTimes( return results, nil } - subq1 := s.DB.NewSelect(). + subq1 := s.db.NewSelect(). TableExpr("drop_reports AS dr"). Column("dr.source_name", "dr.stage_id", "dr.times") s.handleAccountAndReliability(subq1, accountId) @@ -133,7 +136,7 @@ func (s *DropReport) CalcTotalTimes( s.handleServer(subq1, server) s.handleStages(subq1, stageIds) - mainq := s.DB.NewSelect(). + mainq := s.db.NewSelect(). TableExpr("(?) AS a", subq1). Column("stage_id"). ColumnExpr("SUM(times) AS total_times") @@ -155,7 +158,7 @@ func (s *DropReport) CalcQuantityUniqCount( return results, nil } - subq1 := s.DB.NewSelect(). + subq1 := s.db.NewSelect(). TableExpr("drop_reports AS dr"). Column("dr.source_name", "dr.stage_id", "dpe.item_id", "dpe.quantity"). Join("JOIN drop_pattern_elements AS dpe ON dpe.drop_pattern_id = dr.pattern_id") @@ -164,7 +167,7 @@ func (s *DropReport) CalcQuantityUniqCount( s.handleServer(subq1, server) s.handleStagesAndItems(subq1, stageIdItemIdMap) - mainq := s.DB.NewSelect(). + mainq := s.db.NewSelect(). TableExpr("(?) AS a", subq1). Column("stage_id", "item_id", "quantity"). ColumnExpr("COUNT(*) AS count") @@ -189,7 +192,7 @@ func (s *DropReport) CalcTotalQuantityForTrend( gameDayStart := gameday.StartTime(server, *startTime) lastDayEnd := gameDayStart.Add(time.Hour * time.Duration(int(intervalLength.Hours())*(intervalNum+1))) - subq1 := s.DB.NewSelect(). + subq1 := s.db.NewSelect(). With("intervals", s.genSubQueryForTrendSegments(gameDayStart, intervalLength, intervalNum)). TableExpr("drop_reports AS dr"). Column("dr.source_name", "sub.group_id", "sub.interval_start", "sub.interval_end", "dr.stage_id", "dpe.item_id", "dpe.quantity"). @@ -201,7 +204,7 @@ func (s *DropReport) CalcTotalQuantityForTrend( s.handleServer(subq1, server) s.handleStagesAndItems(subq1, stageIdItemIdMap) - mainq := s.DB.NewSelect(). + mainq := s.db.NewSelect(). TableExpr("(?) AS a", subq1). Column("group_id", "interval_start", "interval_end", "stage_id", "item_id"). ColumnExpr("SUM(quantity) AS total_quantity") @@ -226,7 +229,7 @@ func (s *DropReport) CalcTotalTimesForTrend( gameDayStart := gameday.StartTime(server, *startTime) lastDayEnd := gameDayStart.Add(time.Hour * time.Duration(int(intervalLength.Hours())*(intervalNum+1))) - subq1 := s.DB.NewSelect(). + subq1 := s.db.NewSelect(). With("intervals", s.genSubQueryForTrendSegments(gameDayStart, intervalLength, intervalNum)). TableExpr("drop_reports AS dr"). Column("dr.source_name", "sub.group_id", "sub.interval_start", "sub.interval_end", "dr.stage_id", "dr.times"). @@ -237,7 +240,7 @@ func (s *DropReport) CalcTotalTimesForTrend( s.handleServer(subq1, server) s.handleStages(subq1, stageIds) - mainq := s.DB.NewSelect(). + mainq := s.db.NewSelect(). TableExpr("(?) AS a", subq1). Column("group_id", "interval_start", "interval_end", "stage_id"). ColumnExpr("SUM(times) AS total_times") @@ -253,7 +256,7 @@ func (s *DropReport) CalcTotalTimesForTrend( func (s *DropReport) CalcTotalSanityCostForShimSiteStats(ctx context.Context, server string) (sanity int, err error) { err = pgqry.New( - s.DB.NewSelect(). + s.db.NewSelect(). TableExpr("drop_reports AS dr"). ColumnExpr("SUM(st.sanity * dr.times)"). Where("dr.reliability = 0 AND dr.server = ?", server), @@ -267,7 +270,7 @@ func (s *DropReport) CalcTotalStageQuantityForShimSiteStats(ctx context.Context, results := make([]*modelv2.TotalStageTime, 0) err := pgqry.New( - s.DB.NewSelect(). + s.db.NewSelect(). TableExpr("drop_reports AS dr"). Column("st.ark_stage_id"). ColumnExpr("SUM(dr.times) AS total_times"). @@ -294,7 +297,7 @@ func (s *DropReport) CalcTotalItemQuantityForShimSiteStats(ctx context.Context, types := []string{constant.ItemTypeMaterial, constant.ItemTypeFurniture, constant.ItemTypeChip} err := pgqry.New( - s.DB.NewSelect(). + s.db.NewSelect(). TableExpr("drop_reports AS dr"). Column("it.ark_item_id"). ColumnExpr("SUM(dpe.quantity) AS total_quantity"). @@ -312,13 +315,13 @@ func (s *DropReport) CalcTotalItemQuantityForShimSiteStats(ctx context.Context, func (s *DropReport) CalcRecentUniqueUserCountBySource(ctx context.Context, duration time.Duration) ([]*modelv2.UniqueUserCountBySource, error) { results := make([]*modelv2.UniqueUserCountBySource, 0) - subq := s.DB.NewSelect(). + subq := s.db.NewSelect(). TableExpr("drop_reports AS dr"). Column("dr.source_name", "dr.account_id") s.handleCreatedAtWithTime(subq, time.Now().Add(-duration), time.Now()) subq = subq.Group("dr.source_name", "dr.account_id") - mainq := s.DB.NewSelect(). + mainq := s.db.NewSelect(). TableExpr("(?) AS a", subq). Column("source_name"). ColumnExpr("COUNT(*) AS count"). @@ -408,7 +411,7 @@ func (s *DropReport) genSubQueryForTrendSegments(gameDayStart time.Time, interva fmt.Fprintf(&subQueryExprBuilder, "to_timestamp(?) + (n || ' hours')::interval AS interval_start, ") fmt.Fprintf(&subQueryExprBuilder, "to_timestamp(?) + ((n + ?) || ' hours')::interval AS interval_end, ") fmt.Fprintf(&subQueryExprBuilder, "(n / ?) AS group_id") - return s.DB.NewSelect(). + return s.db.NewSelect(). TableExpr("generate_series(?, ? * ?, ?) AS n", 0, int(intervalLength.Hours()), intervalNum, int(intervalLength.Hours())). ColumnExpr(subQueryExprBuilder.String(), gameDayStart.Unix(), From d8aa0e39b622efa55eb492128b1161bf315dec06 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Fri, 10 Feb 2023 15:42:40 -0500 Subject: [PATCH 08/31] refactor: use db selector --- internal/repo/item.go | 130 ++++++++++-------------------------------- 1 file changed, 31 insertions(+), 99 deletions(-) diff --git a/internal/repo/item.go b/internal/repo/item.go index bf3febf6..a54e3958 100644 --- a/internal/repo/item.go +++ b/internal/repo/item.go @@ -2,135 +2,67 @@ package repo import ( "context" - "database/sql" - "github.com/pkg/errors" + "exusiai.dev/gommon/constant" "github.com/uptrace/bun" "exusiai.dev/backend-next/internal/model" modelv2 "exusiai.dev/backend-next/internal/model/v2" - "exusiai.dev/backend-next/internal/pkg/pgerr" - "exusiai.dev/gommon/constant" + "exusiai.dev/backend-next/internal/repo/selector" ) type Item struct { - DB *bun.DB + db *bun.DB + v2sel selector.S[modelv2.Item] + v3sel selector.S[model.Item] } func NewItem(db *bun.DB) *Item { - return &Item{DB: db} + return &Item{ + db: db, + v2sel: selector.New[modelv2.Item](db), + v3sel: selector.New[model.Item](db), + } } func (c *Item) GetItems(ctx context.Context) ([]*model.Item, error) { - var items []*model.Item - err := c.DB.NewSelect(). - Model(&items). - Order("item_id ASC"). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return items, nil + return c.v3sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Order("item_id ASC") + }) } func (c *Item) GetItemById(ctx context.Context, itemId int) (*model.Item, error) { - var item model.Item - err := c.DB.NewSelect(). - Model(&item). - Where("item_id = ?", itemId). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &item, nil + return c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("item_id = ?", itemId) + }) } func (c *Item) GetItemByArkId(ctx context.Context, arkItemId string) (*model.Item, error) { - var item model.Item - err := c.DB.NewSelect(). - Model(&item). - Where("ark_item_id = ?", arkItemId). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &item, nil + return c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("ark_item_id = ?", arkItemId) + }) } func (c *Item) GetShimItems(ctx context.Context) ([]*modelv2.Item, error) { - var items []*modelv2.Item - - err := c.DB.NewSelect(). - Model(&items). - Order("item_id ASC"). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return items, nil + return c.v2sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Order("item_id ASC") + }) } func (c *Item) GetShimItemByArkId(ctx context.Context, itemId string) (*modelv2.Item, error) { - var item modelv2.Item - err := c.DB.NewSelect(). - Model(&item). - Where("ark_item_id = ?", itemId). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &item, nil + return c.v2sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("ark_item_id = ?", itemId) + }) } func (c *Item) SearchItemByName(ctx context.Context, name string) (*model.Item, error) { - var item model.Item - err := c.DB.NewSelect(). - Model(&item). - Where("\"name\"::TEXT ILIKE ?", "%"+name+"%"). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &item, nil + return c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("\"name\"::TEXT ILIKE ?", "%"+name+"%") + }) } func (c *Item) GetRecruitTagItems(ctx context.Context) ([]*model.Item, error) { - var items []*model.Item - err := c.DB.NewSelect(). - Model(&items). - Where("type = ?", constant.RecruitItemType). - Order("item_id ASC"). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return items, nil + return c.v3sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("type = ?", constant.RecruitItemType).Order("item_id ASC") + }) } From fbab51d59579d39e7a63f6c544440b79ff3789e4 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Fri, 10 Feb 2023 15:47:11 -0500 Subject: [PATCH 09/31] refactor: use db selector --- internal/repo/stage.go | 105 +++++++++++------------------------------ 1 file changed, 28 insertions(+), 77 deletions(-) diff --git a/internal/repo/stage.go b/internal/repo/stage.go index 1d390b6a..62ca423e 100644 --- a/internal/repo/stage.go +++ b/internal/repo/stage.go @@ -14,62 +14,39 @@ import ( "exusiai.dev/backend-next/internal/model" modelv2 "exusiai.dev/backend-next/internal/model/v2" "exusiai.dev/backend-next/internal/pkg/pgerr" + "exusiai.dev/backend-next/internal/repo/selector" ) type Stage struct { - db *bun.DB + db *bun.DB + v2sel selector.S[modelv2.Stage] + v3sel selector.S[model.Stage] } func NewStage(db *bun.DB) *Stage { - return &Stage{db: db} + return &Stage{ + db: db, + v2sel: selector.New[modelv2.Stage](db), + v3sel: selector.New[model.Stage](db), + } } func (c *Stage) GetStages(ctx context.Context) ([]*model.Stage, error) { - var stages []*model.Stage - err := c.db.NewSelect(). - Model(&stages). - Order("stage_id ASC"). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return stages, nil + return c.v3sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Order("stage_id ASC") + }) } func (c *Stage) GetStageById(ctx context.Context, stageId int) (*model.Stage, error) { - var stage model.Stage - err := c.db.NewSelect(). - Model(&stage). - Where("stage_id = ?", stageId). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &stage, nil + return c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("stage_id = ?", stageId) + }) } func (c *Stage) GetStageByArkId(ctx context.Context, arkStageId string) (*model.Stage, error) { - var stage model.Stage - err := c.db.NewSelect(). - Model(&stage). - Where("ark_stage_id = ?", arkStageId). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &stage, nil + return c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("ark_stage_id = ?", arkStageId) + }) } func (c *Stage) shimStageQuery(ctx context.Context, server string, stages *[]*modelv2.Stage, t time.Time) error { @@ -160,16 +137,10 @@ func (c *Stage) GetShimStageByArkId(ctx context.Context, arkStageId string, serv } func (c *Stage) GetStageExtraProcessTypeByArkId(ctx context.Context, arkStageId string) (null.String, error) { - var stage model.Stage - err := c.db.NewSelect(). - Model(&stage). - Column("st.extra_process_type"). - Where("st.ark_stage_id = ?", arkStageId). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return null.NewString("", false), pgerr.ErrNotFound - } else if err != nil { + stage, err := c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Column("st.extra_process_type").Where("st.ark_stage_id = ?", arkStageId) + }) + if err != nil { return null.NewString("", false), err } @@ -177,33 +148,13 @@ func (c *Stage) GetStageExtraProcessTypeByArkId(ctx context.Context, arkStageId } func (c *Stage) SearchStageByCode(ctx context.Context, code string) (*model.Stage, error) { - var stage model.Stage - err := c.db.NewSelect(). - Model(&stage). - Where("\"code\"::TEXT ILIKE ?", "%"+code+"%"). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &stage, nil + return c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("\"code\"::TEXT ILIKE ?", "%"+code+"%") + }) } func (c *Stage) GetGachaBoxStages(ctx context.Context) ([]*model.Stage, error) { - var stages []*model.Stage - err := c.db.NewSelect(). - Model(&stages). - Where("extra_process_type = ?", constant.ExtraProcessTypeGachaBox). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return stages, nil + return c.v3sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("st.extra_process_type = ?", constant.ExtraProcessTypeGachaBox) + }) } From 53f5391ea190fb2786eed745c0b76f80a9d6f6e6 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Fri, 10 Feb 2023 15:49:38 -0500 Subject: [PATCH 10/31] refactor: use `r` for repo --- internal/repo/account.go | 16 +++++++-------- internal/repo/activity.go | 4 ++-- internal/repo/drop_info.go | 30 ++++++++++++++-------------- internal/repo/drop_matrix_element.go | 12 +++++------ 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/internal/repo/account.go b/internal/repo/account.go index 35fe293f..c235aaad 100644 --- a/internal/repo/account.go +++ b/internal/repo/account.go @@ -27,7 +27,7 @@ func NewAccount(db *bun.DB) *Account { } } -func (c *Account) CreateAccountWithRandomPenguinId(ctx context.Context) (*model.Account, error) { +func (r *Account) CreateAccountWithRandomPenguinId(ctx context.Context) (*model.Account, error) { // retry if account already exists for i := 0; i < AccountMaxRetries; i++ { account := &model.Account{ @@ -35,7 +35,7 @@ func (c *Account) CreateAccountWithRandomPenguinId(ctx context.Context) (*model. CreatedAt: time.Now(), } - _, err := c.db.NewInsert(). + _, err := r.db.NewInsert(). Model(account). Returning("account_id"). Exec(ctx) @@ -61,20 +61,20 @@ func (c *Account) CreateAccountWithRandomPenguinId(ctx context.Context) (*model. return nil, pgerr.ErrInternalError.Msg("failed to create account") } -func (c *Account) GetAccountById(ctx context.Context, accountId string) (*model.Account, error) { - return c.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Account) GetAccountById(ctx context.Context, accountId string) (*model.Account, error) { + return r.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("account_id = ?", accountId) }) } -func (c *Account) GetAccountByPenguinId(ctx context.Context, penguinId string) (*model.Account, error) { - return c.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Account) GetAccountByPenguinId(ctx context.Context, penguinId string) (*model.Account, error) { + return r.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("penguin_id = ?", penguinId) }) } -func (c *Account) IsAccountExistWithId(ctx context.Context, accountId int) bool { - account, err := c.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Account) IsAccountExistWithId(ctx context.Context, accountId int) bool { + account, err := r.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Column("account_id").Where("account_id = ?", accountId) }) if err != nil { diff --git a/internal/repo/activity.go b/internal/repo/activity.go index bbd92a76..8dbbd629 100644 --- a/internal/repo/activity.go +++ b/internal/repo/activity.go @@ -18,8 +18,8 @@ func NewActivity(db *bun.DB) *Activity { return &Activity{db: db, sel: selector.New[model.Activity](db)} } -func (c *Activity) GetActivities(ctx context.Context) ([]*model.Activity, error) { - return c.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Activity) GetActivities(ctx context.Context) ([]*model.Activity, error) { + return r.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Order("activity_id ASC") }) } diff --git a/internal/repo/drop_info.go b/internal/repo/drop_info.go index ce95f239..06f8278b 100644 --- a/internal/repo/drop_info.go +++ b/internal/repo/drop_info.go @@ -30,20 +30,20 @@ func NewDropInfo(db *bun.DB) *DropInfo { return &DropInfo{db: db, sel: selector.New[model.DropInfo](db)} } -func (s *DropInfo) GetDropInfo(ctx context.Context, id int) (*model.DropInfo, error) { - return s.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *DropInfo) GetDropInfo(ctx context.Context, id int) (*model.DropInfo, error) { + return r.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("id = ?", id) }) } -func (s *DropInfo) GetDropInfosByServerAndStageId(ctx context.Context, server string, stageId int) ([]*model.DropInfo, error) { - return s.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *DropInfo) GetDropInfosByServerAndStageId(ctx context.Context, server string, stageId int) ([]*model.DropInfo, error) { + return r.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("stage_id = ?", stageId).Where("server = ?", server) }) } -func (s *DropInfo) GetDropInfosByServer(ctx context.Context, server string) ([]*model.DropInfo, error) { - return s.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *DropInfo) GetDropInfosByServer(ctx context.Context, server string) ([]*model.DropInfo, error) { + return r.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("server = ?", server) }) } @@ -54,10 +54,10 @@ type DropInfoQuery struct { } // GetDropInfoByArkId returns a drop info by its ark id. -func (s *DropInfo) GetForCurrentTimeRange(ctx context.Context, query *DropInfoQuery) ([]*model.DropInfo, error) { +func (r *DropInfo) GetForCurrentTimeRange(ctx context.Context, query *DropInfoQuery) ([]*model.DropInfo, error) { var dropInfo []*model.DropInfo err := pgqry.New( - s.db.NewSelect(). + r.db.NewSelect(). Model(&dropInfo). Where("di.server = ?", query.Server). Where("st.ark_stage_id = ?", query.ArkStageId), @@ -77,9 +77,9 @@ func (s *DropInfo) GetForCurrentTimeRange(ctx context.Context, query *DropInfoQu return dropInfo, nil } -func (s *DropInfo) GetItemDropSetByStageIdAndRangeId(ctx context.Context, server string, stageId int, rangeId int) ([]int, error) { +func (r *DropInfo) GetItemDropSetByStageIdAndRangeId(ctx context.Context, server string, stageId int, rangeId int) ([]int, error) { var results []int - err := s.db.NewSelect(). + err := r.db.NewSelect(). Column("di.item_id"). Model((*model.DropInfo)(nil)). Where("di.server = ?", server). @@ -95,12 +95,12 @@ func (s *DropInfo) GetItemDropSetByStageIdAndRangeId(ctx context.Context, server } results = lo.Uniq(results) - results = sort.IntSlice(results) + sort.Ints(results) return results, nil } -func (s *DropInfo) GetForCurrentTimeRangeWithDropTypes(ctx context.Context, query *DropInfoQuery) (itemDropInfos, typeDropInfos []*model.DropInfo, err error) { - allDropInfos, err := s.GetForCurrentTimeRange(ctx, query) +func (r *DropInfo) GetForCurrentTimeRangeWithDropTypes(ctx context.Context, query *DropInfoQuery) (itemDropInfos, typeDropInfos []*model.DropInfo, err error) { + allDropInfos, err := r.GetForCurrentTimeRange(ctx, query) if err != nil { return nil, nil, err } @@ -117,7 +117,7 @@ func (s *DropInfo) GetForCurrentTimeRangeWithDropTypes(ctx context.Context, quer return itemDropInfos, typeDropInfos, nil } -func (s *DropInfo) GetDropInfosWithFilters(ctx context.Context, server string, timeRanges []*model.TimeRange, stageIdFilter []int, itemIdFilter []int) ([]*model.DropInfo, error) { +func (r *DropInfo) GetDropInfosWithFilters(ctx context.Context, server string, timeRanges []*model.TimeRange, stageIdFilter []int, itemIdFilter []int) ([]*model.DropInfo, error) { results := make([]*model.DropInfo, 0) var whereBuilder strings.Builder fmt.Fprintf(&whereBuilder, "di.server = ? AND di.drop_type != ? AND di.item_id IS NOT NULL") @@ -157,7 +157,7 @@ func (s *DropInfo) GetDropInfosWithFilters(ctx context.Context, server string, t } } } - if err := s.db.NewSelect().TableExpr("drop_infos as di").Column("di.stage_id", "di.item_id", "di.accumulable"). + if err := r.db.NewSelect().TableExpr("drop_infos as di").Column("di.stage_id", "di.item_id", "di.accumulable"). Where(whereBuilder.String(), server, constant.DropTypeRecognitionOnly, bun.In(stageIdFilter), bun.In(itemIdFilter)). Join("JOIN time_ranges AS tr ON tr.range_id = di.range_id"). Scan(ctx, &results); err != nil { diff --git a/internal/repo/drop_matrix_element.go b/internal/repo/drop_matrix_element.go index 7983154c..26f7d96a 100644 --- a/internal/repo/drop_matrix_element.go +++ b/internal/repo/drop_matrix_element.go @@ -20,8 +20,8 @@ func NewDropMatrixElement(db *bun.DB) *DropMatrixElement { return &DropMatrixElement{db: db, sel: selector.New[model.DropMatrixElement](db)} } -func (s *DropMatrixElement) BatchSaveElements(ctx context.Context, elements []*model.DropMatrixElement, server string) error { - err := s.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { +func (r *DropMatrixElement) BatchSaveElements(ctx context.Context, elements []*model.DropMatrixElement, server string) error { + err := r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { _, err := tx.NewDelete().Model((*model.DropMatrixElement)(nil)).Where("server = ?", server).Exec(ctx) if err != nil { return err @@ -35,14 +35,14 @@ func (s *DropMatrixElement) BatchSaveElements(ctx context.Context, elements []*m return nil } -func (s *DropMatrixElement) DeleteByServer(ctx context.Context, server string) error { - _, err := s.db.NewDelete().Model((*model.DropMatrixElement)(nil)).Where("server = ?", server).Exec(ctx) +func (r *DropMatrixElement) DeleteByServer(ctx context.Context, server string) error { + _, err := r.db.NewDelete().Model((*model.DropMatrixElement)(nil)).Where("server = ?", server).Exec(ctx) return err } -func (s *DropMatrixElement) GetElementsByServerAndSourceCategory(ctx context.Context, server string, sourceCategory string) ([]*model.DropMatrixElement, error) { +func (r *DropMatrixElement) GetElementsByServerAndSourceCategory(ctx context.Context, server string, sourceCategory string) ([]*model.DropMatrixElement, error) { var elements []*model.DropMatrixElement - err := s.db.NewSelect().Model(&elements).Where("server = ?", server).Where("source_category = ?", sourceCategory).Scan(ctx) + err := r.db.NewSelect().Model(&elements).Where("server = ?", server).Where("source_category = ?", sourceCategory).Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return nil, nil } else if err != nil { From 7bb753d233e67e869de5f0609f43ff98500e82aa Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Fri, 10 Feb 2023 15:49:59 -0500 Subject: [PATCH 11/31] refactor: use `r` for repo --- internal/repo/time_range.go | 8 ++++---- internal/repo/trend_element.go | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/repo/time_range.go b/internal/repo/time_range.go index 9adc66f9..07a4fd0a 100644 --- a/internal/repo/time_range.go +++ b/internal/repo/time_range.go @@ -16,9 +16,9 @@ func NewTimeRange(db *bun.DB) *TimeRange { return &TimeRange{db: db} } -func (c *TimeRange) GetTimeRangesByServer(ctx context.Context, server string) ([]*model.TimeRange, error) { +func (r *TimeRange) GetTimeRangesByServer(ctx context.Context, server string) ([]*model.TimeRange, error) { var timeRanges []*model.TimeRange - if err := c.db.NewSelect(). + if err := r.db.NewSelect(). Model(&timeRanges). Where("tr.server = ?", server). Scan(ctx); err != nil { @@ -27,9 +27,9 @@ func (c *TimeRange) GetTimeRangesByServer(ctx context.Context, server string) ([ return timeRanges, nil } -func (c *TimeRange) GetTimeRangeById(ctx context.Context, rangeId int) (*model.TimeRange, error) { +func (r *TimeRange) GetTimeRangeById(ctx context.Context, rangeId int) (*model.TimeRange, error) { var timeRange model.TimeRange - if err := c.db.NewSelect(). + if err := r.db.NewSelect(). Model(&timeRange). Where("tr.range_id = ?", rangeId). Scan(ctx); err != nil { diff --git a/internal/repo/trend_element.go b/internal/repo/trend_element.go index b88f779f..2a636eaa 100644 --- a/internal/repo/trend_element.go +++ b/internal/repo/trend_element.go @@ -18,8 +18,8 @@ func NewTrendElement(db *bun.DB) *TrendElement { return &TrendElement{db: db} } -func (s *TrendElement) BatchSaveElements(ctx context.Context, elements []*model.TrendElement, server string) error { - err := s.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { +func (r *TrendElement) BatchSaveElements(ctx context.Context, elements []*model.TrendElement, server string) error { + err := r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { _, err := tx.NewDelete().Model((*model.TrendElement)(nil)).Where("server = ?", server).Exec(ctx) if err != nil { return err @@ -33,14 +33,14 @@ func (s *TrendElement) BatchSaveElements(ctx context.Context, elements []*model. return nil } -func (s *TrendElement) DeleteByServer(ctx context.Context, server string) error { - _, err := s.db.NewDelete().Model((*model.TrendElement)(nil)).Where("server = ?", server).Exec(ctx) +func (r *TrendElement) DeleteByServer(ctx context.Context, server string) error { + _, err := r.db.NewDelete().Model((*model.TrendElement)(nil)).Where("server = ?", server).Exec(ctx) return err } -func (s *TrendElement) GetElementsByServerAndSourceCategory(ctx context.Context, server string, sourceCategory string) ([]*model.TrendElement, error) { +func (r *TrendElement) GetElementsByServerAndSourceCategory(ctx context.Context, server string, sourceCategory string) ([]*model.TrendElement, error) { var elements []*model.TrendElement - err := s.db.NewSelect().Model(&elements).Where("server = ?", server).Where("source_category = ?", sourceCategory).Scan(ctx) + err := r.db.NewSelect().Model(&elements).Where("server = ?", server).Where("source_category = ?", sourceCategory).Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return nil, nil } else if err != nil { From 1cc9a592170a90a537fc081dcb2f4de2e263a724 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Fri, 10 Feb 2023 15:50:19 -0500 Subject: [PATCH 12/31] refactor: use db selector --- internal/repo/time_range.go | 6 ++++-- internal/repo/trend_element.go | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/repo/time_range.go b/internal/repo/time_range.go index 07a4fd0a..cf8c8593 100644 --- a/internal/repo/time_range.go +++ b/internal/repo/time_range.go @@ -6,14 +6,16 @@ import ( "github.com/uptrace/bun" "exusiai.dev/backend-next/internal/model" + "exusiai.dev/backend-next/internal/repo/selector" ) type TimeRange struct { - db *bun.DB + db *bun.DB + sel selector.S[model.TimeRange] } func NewTimeRange(db *bun.DB) *TimeRange { - return &TimeRange{db: db} + return &TimeRange{db: db, sel: selector.New[model.TimeRange](db)} } func (r *TimeRange) GetTimeRangesByServer(ctx context.Context, server string) ([]*model.TimeRange, error) { diff --git a/internal/repo/trend_element.go b/internal/repo/trend_element.go index 2a636eaa..880a90da 100644 --- a/internal/repo/trend_element.go +++ b/internal/repo/trend_element.go @@ -8,14 +8,16 @@ import ( "github.com/uptrace/bun" "exusiai.dev/backend-next/internal/model" + "exusiai.dev/backend-next/internal/repo/selector" ) type TrendElement struct { - db *bun.DB + db *bun.DB + sel selector.S[model.TrendElement] } func NewTrendElement(db *bun.DB) *TrendElement { - return &TrendElement{db: db} + return &TrendElement{db: db, sel: selector.New[model.TrendElement](db)} } func (r *TrendElement) BatchSaveElements(ctx context.Context, elements []*model.TrendElement, server string) error { From e6c8be0df10758960c754fb760fac9df76d1c2f4 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Fri, 10 Feb 2023 15:51:48 -0500 Subject: [PATCH 13/31] refactor: use db selector --- internal/repo/zone.go | 59 +++++++++++++------------------------------ 1 file changed, 18 insertions(+), 41 deletions(-) diff --git a/internal/repo/zone.go b/internal/repo/zone.go index ffa9e379..43e0771f 100644 --- a/internal/repo/zone.go +++ b/internal/repo/zone.go @@ -10,62 +10,39 @@ import ( "exusiai.dev/backend-next/internal/model" modelv2 "exusiai.dev/backend-next/internal/model/v2" "exusiai.dev/backend-next/internal/pkg/pgerr" + "exusiai.dev/backend-next/internal/repo/selector" ) type Zone struct { - db *bun.DB + db *bun.DB + v2sel selector.S[modelv2.Zone] + v3sel selector.S[model.Zone] } func NewZone(db *bun.DB) *Zone { - return &Zone{db: db} + return &Zone{ + db: db, + v2sel: selector.New[modelv2.Zone](db), + v3sel: selector.New[model.Zone](db), + } } func (c *Zone) GetZones(ctx context.Context) ([]*model.Zone, error) { - var zones []*model.Zone - err := c.db.NewSelect(). - Model(&zones). - Order("zone_id ASC"). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return zones, nil + return c.v3sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Order("zone_id ASC") + }) } func (c *Zone) GetZoneById(ctx context.Context, id int) (*model.Zone, error) { - var zone model.Zone - err := c.db.NewSelect(). - Model(&zone). - Where("zone_id = ?", id). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &zone, nil + return c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("zone_id = ?", id) + }) } func (c *Zone) GetZoneByArkId(ctx context.Context, arkZoneId string) (*model.Zone, error) { - var zone model.Zone - err := c.db.NewSelect(). - Model(&zone). - Where("ark_zone_id = ?", arkZoneId). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &zone, nil + return c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("ark_zone_id = ?", arkZoneId) + }) } func (c *Zone) GetShimZones(ctx context.Context) ([]*modelv2.Zone, error) { From aea01674d51bd7354496e7bca09ec494ccaf3a03 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Fri, 10 Feb 2023 15:54:13 -0500 Subject: [PATCH 14/31] refactor: use db selector --- internal/repo/recognition_defect.go | 49 ++++++++++------------------- internal/repo/reject_rule.go | 8 ++--- internal/repo/snapshot.go | 20 ++++++------ internal/repo/stage.go | 40 +++++++++++------------ internal/repo/zone.go | 16 +++++----- 5 files changed, 58 insertions(+), 75 deletions(-) diff --git a/internal/repo/recognition_defect.go b/internal/repo/recognition_defect.go index 7b9bdaf6..e350bdb0 100644 --- a/internal/repo/recognition_defect.go +++ b/internal/repo/recognition_defect.go @@ -9,22 +9,24 @@ import ( "github.com/uptrace/bun" "exusiai.dev/backend-next/internal/model" + "exusiai.dev/backend-next/internal/repo/selector" ) type RecognitionDefect struct { - DB *bun.DB + db *bun.DB + sel selector.S[model.RecognitionDefect] } func NewRecognitionDefect(db *bun.DB) *RecognitionDefect { - return &RecognitionDefect{DB: db} + return &RecognitionDefect{db: db, sel: selector.New[model.RecognitionDefect](db)} } -func (s *RecognitionDefect) CreateDefectReportDraft(ctx context.Context, defectReport *model.RecognitionDefect) error { +func (r *RecognitionDefect) CreateDefectReportDraft(ctx context.Context, defectReport *model.RecognitionDefect) error { if defectReport.DefectID == "" { defectReport.DefectID = strings.ToLower(ulid.Make().String()) } - _, err := s.DB.NewInsert(). + _, err := r.db.NewInsert(). Model(defectReport). Exec(ctx) if err != nil { @@ -34,8 +36,8 @@ func (s *RecognitionDefect) CreateDefectReportDraft(ctx context.Context, defectR return nil } -func (s *RecognitionDefect) FinalizeDefectReport(ctx context.Context, defectId, imageUri string) error { - _, err := s.DB.NewUpdate(). +func (r *RecognitionDefect) FinalizeDefectReport(ctx context.Context, defectId, imageUri string) error { + _, err := r.db.NewUpdate(). Model((*model.RecognitionDefect)(nil)). Set("image_uri = ?", imageUri). Set("updated_at = ?", time.Now()). @@ -48,33 +50,14 @@ func (s *RecognitionDefect) FinalizeDefectReport(ctx context.Context, defectId, return nil } -func (s *RecognitionDefect) GetDefectReports(ctx context.Context, limit int, page int) ([]*model.RecognitionDefect, error) { - var defectReports []*model.RecognitionDefect - - query := s.DB.NewSelect(). - Model(&defectReports). - Order("created_at DESC"). - Limit(limit). - Offset(page * limit) - - err := query.Scan(ctx) - if err != nil { - return nil, err - } - - return defectReports, nil +func (r *RecognitionDefect) GetDefectReports(ctx context.Context, limit int, page int) ([]*model.RecognitionDefect, error) { + return r.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Order("created_at DESC").Limit(limit).Offset(page * limit) + }) } -func (s *RecognitionDefect) GetDefectReport(ctx context.Context, defectId string) (*model.RecognitionDefect, error) { - var defectReport model.RecognitionDefect - - err := s.DB.NewSelect(). - Model(&defectReport). - Where("defect_id = ?", defectId). - Scan(ctx) - if err != nil { - return nil, err - } - - return &defectReport, nil +func (r *RecognitionDefect) GetDefectReport(ctx context.Context, defectId string) (*model.RecognitionDefect, error) { + return r.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("defect_id = ?", defectId) + }) } diff --git a/internal/repo/reject_rule.go b/internal/repo/reject_rule.go index e163609e..8c2feef5 100644 --- a/internal/repo/reject_rule.go +++ b/internal/repo/reject_rule.go @@ -23,9 +23,9 @@ func NewRejectRule(db *bun.DB) *RejectRule { return &RejectRule{DB: db} } -func (s *RejectRule) GetRejectRule(ctx context.Context, id int) (*model.RejectRule, error) { +func (r *RejectRule) GetRejectRule(ctx context.Context, id int) (*model.RejectRule, error) { var rejectRule model.RejectRule - err := s.DB.NewSelect(). + err := r.DB.NewSelect(). Model(&rejectRule). Where("rule_id = ?", id). Scan(ctx) @@ -39,9 +39,9 @@ func (s *RejectRule) GetRejectRule(ctx context.Context, id int) (*model.RejectRu return &rejectRule, nil } -func (s *RejectRule) GetAllActiveRejectRules(ctx context.Context) ([]*model.RejectRule, error) { +func (r *RejectRule) GetAllActiveRejectRules(ctx context.Context) ([]*model.RejectRule, error) { var rejectRule []*model.RejectRule - err := s.DB.NewSelect(). + err := r.DB.NewSelect(). Model(&rejectRule). Where("status = ?", RejectRuleActiveStatus). Order("rule_id ASC"). diff --git a/internal/repo/snapshot.go b/internal/repo/snapshot.go index ef027501..72303afe 100644 --- a/internal/repo/snapshot.go +++ b/internal/repo/snapshot.go @@ -21,32 +21,32 @@ func NewSnapshot(db *bun.DB) *Snapshot { } } -func (s *Snapshot) GetSnapshotById(ctx context.Context, id int) (*model.Snapshot, error) { - return s.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Snapshot) GetSnapshotById(ctx context.Context, id int) (*model.Snapshot, error) { + return r.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("id = ?", id) }) } -func (s *Snapshot) GetSnapshotsByIds(ctx context.Context, ids []int) ([]*model.Snapshot, error) { - return s.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Snapshot) GetSnapshotsByIds(ctx context.Context, ids []int) ([]*model.Snapshot, error) { + return r.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("id IN (?)", ids) }) } -func (s *Snapshot) GetLatestSnapshotByKey(ctx context.Context, key string) (*model.Snapshot, error) { - return s.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Snapshot) GetLatestSnapshotByKey(ctx context.Context, key string) (*model.Snapshot, error) { + return r.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("key = ?", key).OrderExpr("snapshot_id DESC").Limit(1) }) } -func (s *Snapshot) GetSnapshotsByVersions(ctx context.Context, key string, versions []string) ([]*model.Snapshot, error) { - return s.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Snapshot) GetSnapshotsByVersions(ctx context.Context, key string, versions []string) ([]*model.Snapshot, error) { + return r.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("key = ?", key).Where("version IN (?)", bun.In(versions)) }) } -func (s *Snapshot) SaveSnapshot(ctx context.Context, snapshot *model.Snapshot) (*model.Snapshot, error) { - _, err := s.db.NewInsert(). +func (r *Snapshot) SaveSnapshot(ctx context.Context, snapshot *model.Snapshot) (*model.Snapshot, error) { + _, err := r.db.NewInsert(). Model(snapshot). Exec(ctx) if err != nil { diff --git a/internal/repo/stage.go b/internal/repo/stage.go index 62ca423e..54c5d56e 100644 --- a/internal/repo/stage.go +++ b/internal/repo/stage.go @@ -31,26 +31,26 @@ func NewStage(db *bun.DB) *Stage { } } -func (c *Stage) GetStages(ctx context.Context) ([]*model.Stage, error) { - return c.v3sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Stage) GetStages(ctx context.Context) ([]*model.Stage, error) { + return r.v3sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Order("stage_id ASC") }) } -func (c *Stage) GetStageById(ctx context.Context, stageId int) (*model.Stage, error) { - return c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Stage) GetStageById(ctx context.Context, stageId int) (*model.Stage, error) { + return r.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("stage_id = ?", stageId) }) } -func (c *Stage) GetStageByArkId(ctx context.Context, arkStageId string) (*model.Stage, error) { - return c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Stage) GetStageByArkId(ctx context.Context, arkStageId string) (*model.Stage, error) { + return r.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("ark_stage_id = ?", arkStageId) }) } -func (c *Stage) shimStageQuery(ctx context.Context, server string, stages *[]*modelv2.Stage, t time.Time) error { - return c.db.NewSelect(). +func (r *Stage) shimStageQuery(ctx context.Context, server string, stages *[]*modelv2.Stage, t time.Time) error { + return r.db.NewSelect(). Model(stages). Relation("Zone", func(q *bun.SelectQuery) *bun.SelectQuery { return q.Column("ark_zone_id") @@ -72,10 +72,10 @@ func (c *Stage) shimStageQuery(ctx context.Context, server string, stages *[]*mo Scan(ctx) } -func (c *Stage) GetShimStages(ctx context.Context, server string) ([]*modelv2.Stage, error) { +func (r *Stage) GetShimStages(ctx context.Context, server string) ([]*modelv2.Stage, error) { var stages []*modelv2.Stage - err := c.shimStageQuery(ctx, server, &stages, time.Now()) + err := r.shimStageQuery(ctx, server, &stages, time.Now()) if errors.Is(err, sql.ErrNoRows) { return nil, pgerr.ErrNotFound @@ -86,10 +86,10 @@ func (c *Stage) GetShimStages(ctx context.Context, server string) ([]*modelv2.St return stages, nil } -func (c *Stage) GetShimStagesForFakeTime(ctx context.Context, server string, fakeTime time.Time) ([]*modelv2.Stage, error) { +func (r *Stage) GetShimStagesForFakeTime(ctx context.Context, server string, fakeTime time.Time) ([]*modelv2.Stage, error) { var stages []*modelv2.Stage - err := c.shimStageQuery(ctx, server, &stages, fakeTime) + err := r.shimStageQuery(ctx, server, &stages, fakeTime) if errors.Is(err, sql.ErrNoRows) { return nil, pgerr.ErrNotFound @@ -100,9 +100,9 @@ func (c *Stage) GetShimStagesForFakeTime(ctx context.Context, server string, fak return stages, nil } -func (c *Stage) GetShimStageByArkId(ctx context.Context, arkStageId string, server string) (*modelv2.Stage, error) { +func (r *Stage) GetShimStageByArkId(ctx context.Context, arkStageId string, server string) (*modelv2.Stage, error) { var stage modelv2.Stage - err := c.db.NewSelect(). + err := r.db.NewSelect(). Model(&stage). Relation("Zone", func(q *bun.SelectQuery) *bun.SelectQuery { return q.Column("ark_zone_id") @@ -136,8 +136,8 @@ func (c *Stage) GetShimStageByArkId(ctx context.Context, arkStageId string, serv return &stage, nil } -func (c *Stage) GetStageExtraProcessTypeByArkId(ctx context.Context, arkStageId string) (null.String, error) { - stage, err := c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Stage) GetStageExtraProcessTypeByArkId(ctx context.Context, arkStageId string) (null.String, error) { + stage, err := r.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Column("st.extra_process_type").Where("st.ark_stage_id = ?", arkStageId) }) if err != nil { @@ -147,14 +147,14 @@ func (c *Stage) GetStageExtraProcessTypeByArkId(ctx context.Context, arkStageId return stage.ExtraProcessType, nil } -func (c *Stage) SearchStageByCode(ctx context.Context, code string) (*model.Stage, error) { - return c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Stage) SearchStageByCode(ctx context.Context, code string) (*model.Stage, error) { + return r.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("\"code\"::TEXT ILIKE ?", "%"+code+"%") }) } -func (c *Stage) GetGachaBoxStages(ctx context.Context) ([]*model.Stage, error) { - return c.v3sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Stage) GetGachaBoxStages(ctx context.Context) ([]*model.Stage, error) { + return r.v3sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("st.extra_process_type = ?", constant.ExtraProcessTypeGachaBox) }) } diff --git a/internal/repo/zone.go b/internal/repo/zone.go index 43e0771f..bacc6a4b 100644 --- a/internal/repo/zone.go +++ b/internal/repo/zone.go @@ -27,14 +27,14 @@ func NewZone(db *bun.DB) *Zone { } } -func (c *Zone) GetZones(ctx context.Context) ([]*model.Zone, error) { - return c.v3sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Zone) GetZones(ctx context.Context) ([]*model.Zone, error) { + return r.v3sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Order("zone_id ASC") }) } -func (c *Zone) GetZoneById(ctx context.Context, id int) (*model.Zone, error) { - return c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Zone) GetZoneById(ctx context.Context, id int) (*model.Zone, error) { + return r.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("zone_id = ?", id) }) } @@ -45,10 +45,10 @@ func (c *Zone) GetZoneByArkId(ctx context.Context, arkZoneId string) (*model.Zon }) } -func (c *Zone) GetShimZones(ctx context.Context) ([]*modelv2.Zone, error) { +func (r *Zone) GetShimZones(ctx context.Context) ([]*modelv2.Zone, error) { var zones []*modelv2.Zone - err := c.db.NewSelect(). + err := r.db.NewSelect(). Model(&zones). // `Stages` shall match the model's `Stages` field name, rather for any else references Relation("Stages", func(q *bun.SelectQuery) *bun.SelectQuery { @@ -68,9 +68,9 @@ func (c *Zone) GetShimZones(ctx context.Context) ([]*modelv2.Zone, error) { return zones, nil } -func (c *Zone) GetShimZoneByArkId(ctx context.Context, arkZoneId string) (*modelv2.Zone, error) { +func (r *Zone) GetShimZoneByArkId(ctx context.Context, arkZoneId string) (*modelv2.Zone, error) { var zone modelv2.Zone - err := c.db.NewSelect(). + err := r.db.NewSelect(). Model(&zone). Relation("Stages", func(q *bun.SelectQuery) *bun.SelectQuery { // zone_id is for go-pg/pg's internal joining for has-many relationship. removing it will cause an error From 8c066e9ee5ead7cb163ccb9c6423c92a5c8c362b Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Fri, 10 Feb 2023 15:58:13 -0500 Subject: [PATCH 15/31] refactor: use db selector --- internal/repo/drop_pattern.go | 18 +-- internal/repo/drop_report.go | 156 ++++++++++++------------ internal/repo/drop_report_extra.go | 10 +- internal/repo/pattern_matrix_element.go | 18 +-- internal/repo/property.go | 45 ++----- internal/repo/reject_rule.go | 8 +- 6 files changed, 118 insertions(+), 137 deletions(-) diff --git a/internal/repo/drop_pattern.go b/internal/repo/drop_pattern.go index a05c6d00..613b12b2 100644 --- a/internal/repo/drop_pattern.go +++ b/internal/repo/drop_pattern.go @@ -26,26 +26,26 @@ func NewDropPattern(db *bun.DB) *DropPattern { return &DropPattern{db: db, sel: selector.New[model.DropPattern](db)} } -func (s *DropPattern) GetDropPatterns(ctx context.Context) ([]*model.DropPattern, error) { - return s.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *DropPattern) GetDropPatterns(ctx context.Context) ([]*model.DropPattern, error) { + return r.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q }) } -func (s *DropPattern) GetDropPatternById(ctx context.Context, id int) (*model.DropPattern, error) { - return s.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *DropPattern) GetDropPatternById(ctx context.Context, id int) (*model.DropPattern, error) { + return r.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("id = ?", id) }) } -func (s *DropPattern) GetDropPatternByHash(ctx context.Context, hash string) (*model.DropPattern, error) { - return s.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *DropPattern) GetDropPatternByHash(ctx context.Context, hash string) (*model.DropPattern, error) { + return r.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("hash = ?", hash) }) } -func (s *DropPattern) GetOrCreateDropPatternFromDrops(ctx context.Context, tx bun.Tx, drops []*types.Drop) (*model.DropPattern, bool, error) { - originalFingerprint, hash := s.calculateDropPatternHash(drops) +func (r *DropPattern) GetOrCreateDropPatternFromDrops(ctx context.Context, tx bun.Tx, drops []*types.Drop) (*model.DropPattern, bool, error) { + originalFingerprint, hash := r.calculateDropPatternHash(drops) dropPattern := &model.DropPattern{ Hash: hash, OriginalFingerprint: originalFingerprint, @@ -71,7 +71,7 @@ func (s *DropPattern) GetOrCreateDropPatternFromDrops(ctx context.Context, tx bu return dropPattern, true, nil } -func (s *DropPattern) calculateDropPatternHash(drops []*types.Drop) (originalFingerprint, hexHash string) { +func (r *DropPattern) calculateDropPatternHash(drops []*types.Drop) (originalFingerprint, hexHash string) { segments := make([]string, len(drops)) for i, drop := range drops { diff --git a/internal/repo/drop_report.go b/internal/repo/drop_report.go index 39f6bda3..fcf7a1ae 100644 --- a/internal/repo/drop_report.go +++ b/internal/repo/drop_report.go @@ -19,26 +19,26 @@ import ( ) type DropReport struct { - db *bun.DB + db *bun.DB sel selector.S[model.DropReport] } func NewDropReport(db *bun.DB) *DropReport { return &DropReport{ - db: db, + db: db, sel: selector.New[model.DropReport](db), } } -func (s *DropReport) CreateDropReport(ctx context.Context, tx bun.Tx, dropReport *model.DropReport) error { +func (r *DropReport) CreateDropReport(ctx context.Context, tx bun.Tx, dropReport *model.DropReport) error { _, err := tx.NewInsert(). Model(dropReport). Exec(ctx) return err } -func (s *DropReport) DeleteDropReport(ctx context.Context, reportId int) error { - _, err := s.db.NewUpdate(). +func (r *DropReport) DeleteDropReport(ctx context.Context, reportId int) error { + _, err := r.db.NewUpdate(). Model((*model.DropReport)(nil)). Set("reliability = ?", -1). Where("report_id = ?", reportId). @@ -46,7 +46,7 @@ func (s *DropReport) DeleteDropReport(ctx context.Context, reportId int) error { return err } -func (s *DropReport) UpdateDropReportReliability(ctx context.Context, tx bun.Tx, reportId int, reliability int) error { +func (r *DropReport) UpdateDropReportReliability(ctx context.Context, tx bun.Tx, reportId int, reliability int) error { _, err := tx.NewUpdate(). Model((*model.DropReport)(nil)). Set("reliability = ?", reliability). @@ -55,7 +55,7 @@ func (s *DropReport) UpdateDropReportReliability(ctx context.Context, tx bun.Tx, return err } -func (s *DropReport) CalcTotalQuantityForDropMatrix( +func (r *DropReport) CalcTotalQuantityForDropMatrix( ctx context.Context, server string, timeRange *model.TimeRange, stageIdItemIdMap map[int][]int, accountId null.Int, sourceCategory string, ) ([]*model.TotalQuantityResultForDropMatrix, error) { results := make([]*model.TotalQuantityResultForDropMatrix, 0) @@ -63,20 +63,20 @@ func (s *DropReport) CalcTotalQuantityForDropMatrix( return results, nil } - subq1 := s.db.NewSelect(). + subq1 := r.db.NewSelect(). TableExpr("drop_reports AS dr"). Column("dr.stage_id", "dr.source_name", "dpe.item_id", "dpe.quantity"). Join("JOIN drop_pattern_elements AS dpe ON dpe.drop_pattern_id = dr.pattern_id") - s.handleAccountAndReliability(subq1, accountId) - s.handleCreatedAtWithTimeRange(subq1, timeRange) - s.handleServer(subq1, server) - s.handleStagesAndItems(subq1, stageIdItemIdMap) + r.handleAccountAndReliability(subq1, accountId) + r.handleCreatedAtWithTimeRange(subq1, timeRange) + r.handleServer(subq1, server) + r.handleStagesAndItems(subq1, stageIdItemIdMap) - mainq := s.db.NewSelect(). + mainq := r.db.NewSelect(). TableExpr("(?) AS a", subq1). Column("stage_id", "item_id"). ColumnExpr("SUM(quantity) AS total_quantity") - s.handleSourceName(mainq, sourceCategory) + r.handleSourceName(mainq, sourceCategory) if err := mainq. Group("stage_id", "item_id"). @@ -86,7 +86,7 @@ func (s *DropReport) CalcTotalQuantityForDropMatrix( return results, nil } -func (s *DropReport) CalcTotalQuantityForPatternMatrix( +func (r *DropReport) CalcTotalQuantityForPatternMatrix( ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, accountId null.Int, sourceCategory string, ) ([]*model.TotalQuantityResultForPatternMatrix, error) { results := make([]*model.TotalQuantityResultForPatternMatrix, 0) @@ -94,20 +94,20 @@ func (s *DropReport) CalcTotalQuantityForPatternMatrix( return results, nil } - subq1 := s.db.NewSelect(). + subq1 := r.db.NewSelect(). TableExpr("drop_reports AS dr"). Column("dr.source_name", "dr.stage_id", "dr.pattern_id") - s.handleAccountAndReliability(subq1, accountId) - s.handleCreatedAtWithTimeRange(subq1, timeRange) - s.handleServer(subq1, server) - s.handleStages(subq1, stageIds) - s.handleTimes(subq1, 1) + r.handleAccountAndReliability(subq1, accountId) + r.handleCreatedAtWithTimeRange(subq1, timeRange) + r.handleServer(subq1, server) + r.handleStages(subq1, stageIds) + r.handleTimes(subq1, 1) - mainq := s.db.NewSelect(). + mainq := r.db.NewSelect(). TableExpr("(?) AS a", subq1). Column("stage_id", "pattern_id"). ColumnExpr("COUNT(*) AS total_quantity") - s.handleSourceName(mainq, sourceCategory) + r.handleSourceName(mainq, sourceCategory) if err := mainq. Group("stage_id", "pattern_id"). @@ -117,7 +117,7 @@ func (s *DropReport) CalcTotalQuantityForPatternMatrix( return results, nil } -func (s *DropReport) CalcTotalTimes( +func (r *DropReport) CalcTotalTimes( ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, accountId null.Int, excludeNonOneTimes bool, sourceCategory string, ) ([]*model.TotalTimesResult, error) { results := make([]*model.TotalTimesResult, 0) @@ -125,22 +125,22 @@ func (s *DropReport) CalcTotalTimes( return results, nil } - subq1 := s.db.NewSelect(). + subq1 := r.db.NewSelect(). TableExpr("drop_reports AS dr"). Column("dr.source_name", "dr.stage_id", "dr.times") - s.handleAccountAndReliability(subq1, accountId) + r.handleAccountAndReliability(subq1, accountId) if excludeNonOneTimes { - s.handleTimes(subq1, 1) + r.handleTimes(subq1, 1) } - s.handleCreatedAtWithTimeRange(subq1, timeRange) - s.handleServer(subq1, server) - s.handleStages(subq1, stageIds) + r.handleCreatedAtWithTimeRange(subq1, timeRange) + r.handleServer(subq1, server) + r.handleStages(subq1, stageIds) - mainq := s.db.NewSelect(). + mainq := r.db.NewSelect(). TableExpr("(?) AS a", subq1). Column("stage_id"). ColumnExpr("SUM(times) AS total_times") - s.handleSourceName(mainq, sourceCategory) + r.handleSourceName(mainq, sourceCategory) if err := mainq. Group("stage_id"). @@ -150,7 +150,7 @@ func (s *DropReport) CalcTotalTimes( return results, nil } -func (s *DropReport) CalcQuantityUniqCount( +func (r *DropReport) CalcQuantityUniqCount( ctx context.Context, server string, timeRange *model.TimeRange, stageIdItemIdMap map[int][]int, accountId null.Int, sourceCategory string, ) ([]*model.QuantityUniqCountResultForDropMatrix, error) { results := make([]*model.QuantityUniqCountResultForDropMatrix, 0) @@ -158,20 +158,20 @@ func (s *DropReport) CalcQuantityUniqCount( return results, nil } - subq1 := s.db.NewSelect(). + subq1 := r.db.NewSelect(). TableExpr("drop_reports AS dr"). Column("dr.source_name", "dr.stage_id", "dpe.item_id", "dpe.quantity"). Join("JOIN drop_pattern_elements AS dpe ON dpe.drop_pattern_id = dr.pattern_id") - s.handleAccountAndReliability(subq1, accountId) - s.handleCreatedAtWithTimeRange(subq1, timeRange) - s.handleServer(subq1, server) - s.handleStagesAndItems(subq1, stageIdItemIdMap) + r.handleAccountAndReliability(subq1, accountId) + r.handleCreatedAtWithTimeRange(subq1, timeRange) + r.handleServer(subq1, server) + r.handleStagesAndItems(subq1, stageIdItemIdMap) - mainq := s.db.NewSelect(). + mainq := r.db.NewSelect(). TableExpr("(?) AS a", subq1). Column("stage_id", "item_id", "quantity"). ColumnExpr("COUNT(*) AS count") - s.handleSourceName(mainq, sourceCategory) + r.handleSourceName(mainq, sourceCategory) if err := mainq. Group("stage_id", "item_id", "quantity"). @@ -181,7 +181,7 @@ func (s *DropReport) CalcQuantityUniqCount( return results, nil } -func (s *DropReport) CalcTotalQuantityForTrend( +func (r *DropReport) CalcTotalQuantityForTrend( ctx context.Context, server string, startTime *time.Time, intervalLength time.Duration, intervalNum int, stageIdItemIdMap map[int][]int, accountId null.Int, sourceCategory string, ) ([]*model.TotalQuantityResultForTrend, error) { results := make([]*model.TotalQuantityResultForTrend, 0) @@ -192,23 +192,23 @@ func (s *DropReport) CalcTotalQuantityForTrend( gameDayStart := gameday.StartTime(server, *startTime) lastDayEnd := gameDayStart.Add(time.Hour * time.Duration(int(intervalLength.Hours())*(intervalNum+1))) - subq1 := s.db.NewSelect(). - With("intervals", s.genSubQueryForTrendSegments(gameDayStart, intervalLength, intervalNum)). + subq1 := r.db.NewSelect(). + With("intervals", r.genSubQueryForTrendSegments(gameDayStart, intervalLength, intervalNum)). TableExpr("drop_reports AS dr"). Column("dr.source_name", "sub.group_id", "sub.interval_start", "sub.interval_end", "dr.stage_id", "dpe.item_id", "dpe.quantity"). Join("JOIN drop_pattern_elements AS dpe ON dpe.drop_pattern_id = dr.pattern_id"). Join("RIGHT JOIN intervals AS sub"). JoinOn("dr.created_at >= sub.interval_start AND dr.created_at < sub.interval_end") - s.handleAccountAndReliability(subq1, accountId) - s.handleCreatedAtWithTime(subq1, gameDayStart, lastDayEnd) - s.handleServer(subq1, server) - s.handleStagesAndItems(subq1, stageIdItemIdMap) + r.handleAccountAndReliability(subq1, accountId) + r.handleCreatedAtWithTime(subq1, gameDayStart, lastDayEnd) + r.handleServer(subq1, server) + r.handleStagesAndItems(subq1, stageIdItemIdMap) - mainq := s.db.NewSelect(). + mainq := r.db.NewSelect(). TableExpr("(?) AS a", subq1). Column("group_id", "interval_start", "interval_end", "stage_id", "item_id"). ColumnExpr("SUM(quantity) AS total_quantity") - s.handleSourceName(mainq, sourceCategory) + r.handleSourceName(mainq, sourceCategory) if err := mainq. Group("group_id", "interval_start", "interval_end", "stage_id", "item_id"). @@ -218,7 +218,7 @@ func (s *DropReport) CalcTotalQuantityForTrend( return results, nil } -func (s *DropReport) CalcTotalTimesForTrend( +func (r *DropReport) CalcTotalTimesForTrend( ctx context.Context, server string, startTime *time.Time, intervalLength time.Duration, intervalNum int, stageIds []int, accountId null.Int, sourceCategory string, ) ([]*model.TotalTimesResultForTrend, error) { results := make([]*model.TotalTimesResultForTrend, 0) @@ -229,22 +229,22 @@ func (s *DropReport) CalcTotalTimesForTrend( gameDayStart := gameday.StartTime(server, *startTime) lastDayEnd := gameDayStart.Add(time.Hour * time.Duration(int(intervalLength.Hours())*(intervalNum+1))) - subq1 := s.db.NewSelect(). - With("intervals", s.genSubQueryForTrendSegments(gameDayStart, intervalLength, intervalNum)). + subq1 := r.db.NewSelect(). + With("intervals", r.genSubQueryForTrendSegments(gameDayStart, intervalLength, intervalNum)). TableExpr("drop_reports AS dr"). Column("dr.source_name", "sub.group_id", "sub.interval_start", "sub.interval_end", "dr.stage_id", "dr.times"). Join("RIGHT JOIN intervals AS sub"). JoinOn("dr.created_at >= sub.interval_start AND dr.created_at < sub.interval_end") - s.handleAccountAndReliability(subq1, accountId) - s.handleCreatedAtWithTime(subq1, gameDayStart, lastDayEnd) - s.handleServer(subq1, server) - s.handleStages(subq1, stageIds) + r.handleAccountAndReliability(subq1, accountId) + r.handleCreatedAtWithTime(subq1, gameDayStart, lastDayEnd) + r.handleServer(subq1, server) + r.handleStages(subq1, stageIds) - mainq := s.db.NewSelect(). + mainq := r.db.NewSelect(). TableExpr("(?) AS a", subq1). Column("group_id", "interval_start", "interval_end", "stage_id"). ColumnExpr("SUM(times) AS total_times") - s.handleSourceName(mainq, sourceCategory) + r.handleSourceName(mainq, sourceCategory) if err := mainq. Group("group_id", "interval_start", "interval_end", "stage_id"). @@ -254,9 +254,9 @@ func (s *DropReport) CalcTotalTimesForTrend( return results, nil } -func (s *DropReport) CalcTotalSanityCostForShimSiteStats(ctx context.Context, server string) (sanity int, err error) { +func (r *DropReport) CalcTotalSanityCostForShimSiteStats(ctx context.Context, server string) (sanity int, err error) { err = pgqry.New( - s.db.NewSelect(). + r.db.NewSelect(). TableExpr("drop_reports AS dr"). ColumnExpr("SUM(st.sanity * dr.times)"). Where("dr.reliability = 0 AND dr.server = ?", server), @@ -266,11 +266,11 @@ func (s *DropReport) CalcTotalSanityCostForShimSiteStats(ctx context.Context, se return sanity, err } -func (s *DropReport) CalcTotalStageQuantityForShimSiteStats(ctx context.Context, server string, isRecent24h bool) ([]*modelv2.TotalStageTime, error) { +func (r *DropReport) CalcTotalStageQuantityForShimSiteStats(ctx context.Context, server string, isRecent24h bool) ([]*modelv2.TotalStageTime, error) { results := make([]*modelv2.TotalStageTime, 0) err := pgqry.New( - s.db.NewSelect(). + r.db.NewSelect(). TableExpr("drop_reports AS dr"). Column("st.ark_stage_id"). ColumnExpr("SUM(dr.times) AS total_times"). @@ -292,12 +292,12 @@ func (s *DropReport) CalcTotalStageQuantityForShimSiteStats(ctx context.Context, return results, nil } -func (s *DropReport) CalcTotalItemQuantityForShimSiteStats(ctx context.Context, server string) ([]*modelv2.TotalItemQuantity, error) { +func (r *DropReport) CalcTotalItemQuantityForShimSiteStats(ctx context.Context, server string) ([]*modelv2.TotalItemQuantity, error) { results := make([]*modelv2.TotalItemQuantity, 0) types := []string{constant.ItemTypeMaterial, constant.ItemTypeFurniture, constant.ItemTypeChip} err := pgqry.New( - s.db.NewSelect(). + r.db.NewSelect(). TableExpr("drop_reports AS dr"). Column("it.ark_item_id"). ColumnExpr("SUM(dpe.quantity) AS total_quantity"). @@ -313,15 +313,15 @@ func (s *DropReport) CalcTotalItemQuantityForShimSiteStats(ctx context.Context, return results, nil } -func (s *DropReport) CalcRecentUniqueUserCountBySource(ctx context.Context, duration time.Duration) ([]*modelv2.UniqueUserCountBySource, error) { +func (r *DropReport) CalcRecentUniqueUserCountBySource(ctx context.Context, duration time.Duration) ([]*modelv2.UniqueUserCountBySource, error) { results := make([]*modelv2.UniqueUserCountBySource, 0) - subq := s.db.NewSelect(). + subq := r.db.NewSelect(). TableExpr("drop_reports AS dr"). Column("dr.source_name", "dr.account_id") - s.handleCreatedAtWithTime(subq, time.Now().Add(-duration), time.Now()) + r.handleCreatedAtWithTime(subq, time.Now().Add(-duration), time.Now()) subq = subq.Group("dr.source_name", "dr.account_id") - mainq := s.db.NewSelect(). + mainq := r.db.NewSelect(). TableExpr("(?) AS a", subq). Column("source_name"). ColumnExpr("COUNT(*) AS count"). @@ -334,7 +334,7 @@ func (s *DropReport) CalcRecentUniqueUserCountBySource(ctx context.Context, dura return results, nil } -func (s *DropReport) handleStagesAndItems(query *bun.SelectQuery, stageIdItemIdMap map[int][]int) { +func (r *DropReport) handleStagesAndItems(query *bun.SelectQuery, stageIdItemIdMap map[int][]int) { stageConditions := make([]string, 0) for stageId, itemIds := range stageIdItemIdMap { var stageB strings.Builder @@ -353,7 +353,7 @@ func (s *DropReport) handleStagesAndItems(query *bun.SelectQuery, stageIdItemIdM query.Where(strings.Join(stageConditions, " OR ")) } -func (s *DropReport) handleStages(query *bun.SelectQuery, stageIds []int) { +func (r *DropReport) handleStages(query *bun.SelectQuery, stageIds []int) { var b strings.Builder b.WriteString("dr.stage_id") if len(stageIds) == 1 { @@ -368,7 +368,7 @@ func (s *DropReport) handleStages(query *bun.SelectQuery, stageIds []int) { query.Where(b.String()) } -func (s *DropReport) handleAccountAndReliability(query *bun.SelectQuery, accountId null.Int) { +func (r *DropReport) handleAccountAndReliability(query *bun.SelectQuery, accountId null.Int) { if accountId.Valid { query = query.Where("dr.reliability >= 0 AND dr.account_id = ?", accountId.Int64) } else { @@ -376,7 +376,7 @@ func (s *DropReport) handleAccountAndReliability(query *bun.SelectQuery, account } } -func (s *DropReport) handleCreatedAtWithTimeRange(query *bun.SelectQuery, timeRange *model.TimeRange) { +func (r *DropReport) handleCreatedAtWithTimeRange(query *bun.SelectQuery, timeRange *model.TimeRange) { if timeRange.StartTime != nil { query = query.Where("dr.created_at >= timestamp with time zone ?", timeRange.StartTime.Format(time.RFC3339)) } @@ -385,20 +385,20 @@ func (s *DropReport) handleCreatedAtWithTimeRange(query *bun.SelectQuery, timeRa } } -func (s *DropReport) handleCreatedAtWithTime(query *bun.SelectQuery, start time.Time, end time.Time) { +func (r *DropReport) handleCreatedAtWithTime(query *bun.SelectQuery, start time.Time, end time.Time) { query = query.Where("dr.created_at >= to_timestamp(?)", start.Unix()) query = query.Where("dr.created_at < to_timestamp(?)", end.Unix()) } -func (s *DropReport) handleServer(query *bun.SelectQuery, server string) { +func (r *DropReport) handleServer(query *bun.SelectQuery, server string) { query = query.Where("dr.server = ?", server) } -func (s *DropReport) handleTimes(query *bun.SelectQuery, times int) { +func (r *DropReport) handleTimes(query *bun.SelectQuery, times int) { query = query.Where("dr.times = ?", times) } -func (s *DropReport) handleSourceName(query *bun.SelectQuery, sourceCategory string) { +func (r *DropReport) handleSourceName(query *bun.SelectQuery, sourceCategory string) { if sourceCategory == constant.SourceCategoryManual { query = query.Where("source_name IN (?)", bun.In(constant.ManualSources)) } else if sourceCategory == constant.SourceCategoryAutomated { @@ -406,12 +406,12 @@ func (s *DropReport) handleSourceName(query *bun.SelectQuery, sourceCategory str } } -func (s *DropReport) genSubQueryForTrendSegments(gameDayStart time.Time, intervalLength time.Duration, intervalNum int) *bun.SelectQuery { +func (r *DropReport) genSubQueryForTrendSegments(gameDayStart time.Time, intervalLength time.Duration, intervalNum int) *bun.SelectQuery { var subQueryExprBuilder strings.Builder fmt.Fprintf(&subQueryExprBuilder, "to_timestamp(?) + (n || ' hours')::interval AS interval_start, ") fmt.Fprintf(&subQueryExprBuilder, "to_timestamp(?) + ((n + ?) || ' hours')::interval AS interval_end, ") fmt.Fprintf(&subQueryExprBuilder, "(n / ?) AS group_id") - return s.db.NewSelect(). + return r.db.NewSelect(). TableExpr("generate_series(?, ? * ?, ?) AS n", 0, int(intervalLength.Hours()), intervalNum, int(intervalLength.Hours())). ColumnExpr(subQueryExprBuilder.String(), gameDayStart.Unix(), diff --git a/internal/repo/drop_report_extra.go b/internal/repo/drop_report_extra.go index f0d80e99..13b33b1b 100644 --- a/internal/repo/drop_report_extra.go +++ b/internal/repo/drop_report_extra.go @@ -18,16 +18,16 @@ func NewDropReportExtra(db *bun.DB) *DropReportExtra { return &DropReportExtra{db: db, sel: selector.New[model.DropReportExtra](db)} } -func (c *DropReportExtra) GetDropReportExtraById(ctx context.Context, id int) (*model.DropReportExtra, error) { - return c.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *DropReportExtra) GetDropReportExtraById(ctx context.Context, id int) (*model.DropReportExtra, error) { + return r.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("report_id = ?", id) }) } -func (c *DropReportExtra) IsDropReportExtraMD5Exist(ctx context.Context, md5 string) bool { +func (r *DropReportExtra) IsDropReportExtraMD5Exist(ctx context.Context, md5 string) bool { var dropReportExtra model.DropReportExtra - count, err := c.db.NewSelect(). + count, err := r.db.NewSelect(). Model(&dropReportExtra). Where("md5 = ?", md5). Count(ctx) @@ -38,7 +38,7 @@ func (c *DropReportExtra) IsDropReportExtraMD5Exist(ctx context.Context, md5 str return count > 0 } -func (c *DropReportExtra) CreateDropReportExtra(ctx context.Context, tx bun.Tx, report *model.DropReportExtra) error { +func (r *DropReportExtra) CreateDropReportExtra(ctx context.Context, tx bun.Tx, report *model.DropReportExtra) error { _, err := tx.NewInsert(). Model(report). Exec(ctx) diff --git a/internal/repo/pattern_matrix_element.go b/internal/repo/pattern_matrix_element.go index 2296200d..afef4b0c 100644 --- a/internal/repo/pattern_matrix_element.go +++ b/internal/repo/pattern_matrix_element.go @@ -8,18 +8,20 @@ import ( "github.com/uptrace/bun" "exusiai.dev/backend-next/internal/model" + "exusiai.dev/backend-next/internal/repo/selector" ) type PatternMatrixElement struct { - db *bun.DB + db *bun.DB + sel selector.S[model.PatternMatrixElement] } func NewPatternMatrixElement(db *bun.DB) *PatternMatrixElement { - return &PatternMatrixElement{db: db} + return &PatternMatrixElement{db: db, sel: selector.New[model.PatternMatrixElement](db)} } -func (s *PatternMatrixElement) BatchSaveElements(ctx context.Context, elements []*model.PatternMatrixElement, server string) error { - err := s.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { +func (r *PatternMatrixElement) BatchSaveElements(ctx context.Context, elements []*model.PatternMatrixElement, server string) error { + err := r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { _, err := tx.NewDelete().Model((*model.PatternMatrixElement)(nil)).Where("server = ?", server).Exec(ctx) if err != nil { return err @@ -33,14 +35,14 @@ func (s *PatternMatrixElement) BatchSaveElements(ctx context.Context, elements [ return nil } -func (s *PatternMatrixElement) DeleteByServer(ctx context.Context, server string) error { - _, err := s.db.NewDelete().Model((*model.PatternMatrixElement)(nil)).Where("server = ?", server).Exec(ctx) +func (r *PatternMatrixElement) DeleteByServer(ctx context.Context, server string) error { + _, err := r.db.NewDelete().Model((*model.PatternMatrixElement)(nil)).Where("server = ?", server).Exec(ctx) return err } -func (s *PatternMatrixElement) GetElementsByServerAndSourceCategory(ctx context.Context, server string, sourceCategory string) ([]*model.PatternMatrixElement, error) { +func (r *PatternMatrixElement) GetElementsByServerAndSourceCategory(ctx context.Context, server string, sourceCategory string) ([]*model.PatternMatrixElement, error) { var elements []*model.PatternMatrixElement - err := s.db.NewSelect().Model(&elements).Where("server = ?", server).Where("source_category = ?", sourceCategory).Scan(ctx) + err := r.db.NewSelect().Model(&elements).Where("server = ?", server).Where("source_category = ?", sourceCategory).Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return nil, nil } else if err != nil { diff --git a/internal/repo/property.go b/internal/repo/property.go index e32cf3bb..96fc3b98 100644 --- a/internal/repo/property.go +++ b/internal/repo/property.go @@ -2,51 +2,30 @@ package repo import ( "context" - "database/sql" - "github.com/pkg/errors" "github.com/uptrace/bun" "exusiai.dev/backend-next/internal/model" - "exusiai.dev/backend-next/internal/pkg/pgerr" + "exusiai.dev/backend-next/internal/repo/selector" ) type Property struct { - db *bun.DB + db *bun.DB + sel selector.S[model.Property] } func NewProperty(db *bun.DB) *Property { - return &Property{db: db} + return &Property{db: db, sel: selector.New[model.Property](db)} } -func (c *Property) GetProperties(ctx context.Context) ([]*model.Property, error) { - var properties []*model.Property - err := c.db.NewSelect(). - Model(&properties). - Order("property_id ASC"). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return properties, nil +func (r *Property) GetProperties(ctx context.Context) ([]*model.Property, error) { + return r.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Order("property_id ASC") + }) } -func (c *Property) GetPropertyByKey(ctx context.Context, key string) (*model.Property, error) { - var property model.Property - err := c.db.NewSelect(). - Model(&property). - Where("key = ?", key). - Scan(ctx) - - if errors.Is(err, sql.ErrNoRows) { - return nil, pgerr.ErrNotFound - } else if err != nil { - return nil, err - } - - return &property, nil +func (r *Property) GetPropertyByKey(ctx context.Context, key string) (*model.Property, error) { + return r.sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Where("key = ?", key) + }) } diff --git a/internal/repo/reject_rule.go b/internal/repo/reject_rule.go index 8c2feef5..1a6c0cad 100644 --- a/internal/repo/reject_rule.go +++ b/internal/repo/reject_rule.go @@ -16,16 +16,16 @@ const ( ) type RejectRule struct { - DB *bun.DB + db *bun.DB } func NewRejectRule(db *bun.DB) *RejectRule { - return &RejectRule{DB: db} + return &RejectRule{db: db} } func (r *RejectRule) GetRejectRule(ctx context.Context, id int) (*model.RejectRule, error) { var rejectRule model.RejectRule - err := r.DB.NewSelect(). + err := r.db.NewSelect(). Model(&rejectRule). Where("rule_id = ?", id). Scan(ctx) @@ -41,7 +41,7 @@ func (r *RejectRule) GetRejectRule(ctx context.Context, id int) (*model.RejectRu func (r *RejectRule) GetAllActiveRejectRules(ctx context.Context) ([]*model.RejectRule, error) { var rejectRule []*model.RejectRule - err := r.DB.NewSelect(). + err := r.db.NewSelect(). Model(&rejectRule). Where("status = ?", RejectRuleActiveStatus). Order("rule_id ASC"). From 542035922a52c488e10f2be698bf01bdff2442aa Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Fri, 10 Feb 2023 15:59:33 -0500 Subject: [PATCH 16/31] refactor: use db selector --- internal/repo/item.go | 28 ++++++++++++++-------------- internal/repo/notice.go | 24 ++++++++---------------- 2 files changed, 22 insertions(+), 30 deletions(-) diff --git a/internal/repo/item.go b/internal/repo/item.go index a54e3958..9540e56c 100644 --- a/internal/repo/item.go +++ b/internal/repo/item.go @@ -25,44 +25,44 @@ func NewItem(db *bun.DB) *Item { } } -func (c *Item) GetItems(ctx context.Context) ([]*model.Item, error) { - return c.v3sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Item) GetItems(ctx context.Context) ([]*model.Item, error) { + return r.v3sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Order("item_id ASC") }) } -func (c *Item) GetItemById(ctx context.Context, itemId int) (*model.Item, error) { - return c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Item) GetItemById(ctx context.Context, itemId int) (*model.Item, error) { + return r.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("item_id = ?", itemId) }) } -func (c *Item) GetItemByArkId(ctx context.Context, arkItemId string) (*model.Item, error) { - return c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Item) GetItemByArkId(ctx context.Context, arkItemId string) (*model.Item, error) { + return r.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("ark_item_id = ?", arkItemId) }) } -func (c *Item) GetShimItems(ctx context.Context) ([]*modelv2.Item, error) { - return c.v2sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Item) GetShimItems(ctx context.Context) ([]*modelv2.Item, error) { + return r.v2sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Order("item_id ASC") }) } -func (c *Item) GetShimItemByArkId(ctx context.Context, itemId string) (*modelv2.Item, error) { - return c.v2sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Item) GetShimItemByArkId(ctx context.Context, itemId string) (*modelv2.Item, error) { + return r.v2sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("ark_item_id = ?", itemId) }) } -func (c *Item) SearchItemByName(ctx context.Context, name string) (*model.Item, error) { - return c.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Item) SearchItemByName(ctx context.Context, name string) (*model.Item, error) { + return r.v3sel.SelectOne(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("\"name\"::TEXT ILIKE ?", "%"+name+"%") }) } -func (c *Item) GetRecruitTagItems(ctx context.Context) ([]*model.Item, error) { - return c.v3sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { +func (r *Item) GetRecruitTagItems(ctx context.Context) ([]*model.Item, error) { + return r.v3sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("type = ?", constant.RecruitItemType).Order("item_id ASC") }) } diff --git a/internal/repo/notice.go b/internal/repo/notice.go index 6351af82..f162cc56 100644 --- a/internal/repo/notice.go +++ b/internal/repo/notice.go @@ -2,32 +2,24 @@ package repo import ( "context" - "database/sql" - "github.com/pkg/errors" "github.com/uptrace/bun" "exusiai.dev/backend-next/internal/model" + "exusiai.dev/backend-next/internal/repo/selector" ) type Notice struct { - DB *bun.DB + db *bun.DB + sel selector.S[model.Notice] } func NewNotice(db *bun.DB) *Notice { - return &Notice{DB: db} + return &Notice{db: db, sel: selector.New[model.Notice](db)} } -func (c *Notice) GetNotices(ctx context.Context) ([]*model.Notice, error) { - var notice []*model.Notice - err := c.DB.NewSelect(). - Model(¬ice). - Order("notice_id ASC"). - Scan(ctx) - - if err != nil && !errors.Is(err, sql.ErrNoRows) { - return nil, err - } - - return notice, nil +func (r *Notice) GetNotices(ctx context.Context) ([]*model.Notice, error) { + return r.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { + return q.Order("notice_id ASC") + }) } From 3c15676ef477cf0c8a8c3bc2450bd8e6bf1d7e59 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 23 Nov 2023 02:48:59 -0800 Subject: [PATCH 17/31] chore: fix variable --- internal/repo/drop_report_extra.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/repo/drop_report_extra.go b/internal/repo/drop_report_extra.go index cf84a8af..6ee64493 100644 --- a/internal/repo/drop_report_extra.go +++ b/internal/repo/drop_report_extra.go @@ -27,7 +27,7 @@ func (r *DropReportExtra) GetDropReportExtraById(ctx context.Context, id int) (* func (c *DropReportExtra) GetDropReportExtraForArchive(ctx context.Context, cursor *model.Cursor, idInclusiveStart int, idInclusiveEnd int, limit int) ([]*model.DropReportExtra, model.Cursor, error) { dropReportExtras := make([]*model.DropReportExtra, 0) - query := c.DB.NewSelect(). + query := c.db.NewSelect(). Model(&dropReportExtras). Where("report_id >= ?", idInclusiveStart). Where("report_id <= ?", idInclusiveEnd). From a6feddcc74df46ad4443a8282c3ae56271ccdadf Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 23 Nov 2023 02:57:59 -0800 Subject: [PATCH 18/31] fix: build error --- internal/controller/meta/admin.go | 4 +++- internal/repo/activity.go | 6 ++++-- internal/repo/drop_info.go | 6 +++--- internal/repo/drop_pattern_element.go | 4 +++- internal/repo/property.go | 9 ++++++--- internal/repo/recognition_defect.go | 2 +- 6 files changed, 20 insertions(+), 11 deletions(-) diff --git a/internal/controller/meta/admin.go b/internal/controller/meta/admin.go index 9ab5b1e3..5e811916 100644 --- a/internal/controller/meta/admin.go +++ b/internal/controller/meta/admin.go @@ -37,6 +37,7 @@ import ( type AdminController struct { fx.In + DB *bun.DB PatternRepo *repo.DropPattern PatternElementRepo *repo.DropPatternElement RecognitionDefectRepo *repo.RecognitionDefect @@ -475,7 +476,8 @@ func (c *AdminController) RejectRulesReevaluationApply(ctx *fiber.Ctx) error { changeSet := evaluation.ChangeSet() err = c.DB.RunInTx(ctx.UserContext(), nil, func(ictx context.Context, tx bun.Tx) error { - for _, change := range changeSet { + chunks := lo.Chunk(changeSet, 100) + for _, changeChunk := range chunks { log.Debug(). Str("evt.name", "admin.reject_rules.reevaluation.apply_chunk"). Int("chunk_size", len(changeChunk)). diff --git a/internal/repo/activity.go b/internal/repo/activity.go index 145033ca..63d5586e 100644 --- a/internal/repo/activity.go +++ b/internal/repo/activity.go @@ -2,7 +2,9 @@ package repo import ( "context" + "database/sql" + "github.com/pkg/errors" "github.com/uptrace/bun" "exusiai.dev/backend-next/internal/model" @@ -24,9 +26,9 @@ func (r *Activity) GetActivities(ctx context.Context) ([]*model.Activity, error) }) } -func (c *Activity) GetActivityById(ctx context.Context, activityId int) (*model.Activity, error) { +func (r *Activity) GetActivityById(ctx context.Context, activityId int) (*model.Activity, error) { var activity model.Activity - err := c.DB.NewSelect(). + err := r.db.NewSelect(). Model(&activity). Where("activity_id = ?", activityId). Scan(ctx) diff --git a/internal/repo/drop_info.go b/internal/repo/drop_info.go index 202a0968..07119e19 100644 --- a/internal/repo/drop_info.go +++ b/internal/repo/drop_info.go @@ -166,9 +166,9 @@ func (r *DropInfo) GetDropInfosWithFilters(ctx context.Context, server string, t return results, nil } -func (s *DropInfo) GetDropInfosByServerAndRangeId(ctx context.Context, server string, rangeId int) ([]*model.DropInfo, error) { +func (r *DropInfo) GetDropInfosByServerAndRangeId(ctx context.Context, server string, rangeId int) ([]*model.DropInfo, error) { var dropInfo []*model.DropInfo - err := s.DB.NewSelect(). + err := r.db.NewSelect(). Model(&dropInfo). Where("server = ?", server). Where("range_id = ?", rangeId). @@ -193,7 +193,7 @@ func (s *DropInfo) CloneDropInfosFromCN(ctx context.Context, originRangeId int, dropInfo.RangeID = destRangeId dropInfo.Server = server } - _, err = s.DB.NewInsert().Model(&dropInfos).Exec(ctx) + _, err = s.db.NewInsert().Model(&dropInfos).Exec(ctx) if err != nil { return err } diff --git a/internal/repo/drop_pattern_element.go b/internal/repo/drop_pattern_element.go index 1cac656f..9587f5cc 100644 --- a/internal/repo/drop_pattern_element.go +++ b/internal/repo/drop_pattern_element.go @@ -2,7 +2,9 @@ package repo import ( "context" + "database/sql" + "github.com/pkg/errors" "github.com/uptrace/bun" "exusiai.dev/backend-next/internal/model" @@ -71,7 +73,7 @@ func (r *DropPatternElement) GetDropPatternElementsByPatternId(ctx context.Conte func (r *DropPatternElement) GetDropPatternElementsByPatternIds(ctx context.Context, patternIds []int) ([]*model.DropPatternElement, error) { var elements []*model.DropPatternElement - err := r.DB.NewSelect(). + err := r.db.NewSelect(). Model(&elements). Where("drop_pattern_id IN (?)", bun.In(patternIds)). Order("drop_pattern_id", "quantity DESC", "item_id"). diff --git a/internal/repo/property.go b/internal/repo/property.go index c81f4de3..2d1e8265 100644 --- a/internal/repo/property.go +++ b/internal/repo/property.go @@ -2,10 +2,13 @@ package repo import ( "context" + "database/sql" + "github.com/pkg/errors" "github.com/uptrace/bun" "exusiai.dev/backend-next/internal/model" + "exusiai.dev/backend-next/internal/pkg/pgerr" "exusiai.dev/backend-next/internal/repo/selector" ) @@ -30,9 +33,9 @@ func (r *Property) GetPropertyByKey(ctx context.Context, key string) (*model.Pro }) } -func (c *Property) UpdatePropertyByKey(ctx context.Context, key string, value string) (*model.Property, error) { +func (r *Property) UpdatePropertyByKey(ctx context.Context, key string, value string) (*model.Property, error) { var property model.Property - err := c.db.NewSelect(). + err := r.db.NewSelect(). Model(&property). Where("key = ?", key). Scan(ctx) @@ -44,7 +47,7 @@ func (c *Property) UpdatePropertyByKey(ctx context.Context, key string, value st } property.Value = value - _, err = c.db.NewUpdate(). + _, err = r.db.NewUpdate(). Model(&property). Where("key = ?", key). Exec(ctx) diff --git a/internal/repo/recognition_defect.go b/internal/repo/recognition_defect.go index 1f65a6ad..018039f9 100644 --- a/internal/repo/recognition_defect.go +++ b/internal/repo/recognition_defect.go @@ -21,7 +21,7 @@ func NewRecognitionDefect(db *bun.DB) *RecognitionDefect { return &RecognitionDefect{db: db, sel: selector.New[model.RecognitionDefect](db)} } -func (s *RecognitionDefect) CreateDefectReportDraft(ctx context.Context, defectReport *model.RecognitionDefect) error { +func (r *RecognitionDefect) CreateDefectReportDraft(ctx context.Context, defectReport *model.RecognitionDefect) error { defectReport.DefectID = strings.ToLower(ulid.Make().String()) _, err := r.db.NewInsert(). From 23a5eeb2dfd7b399c0f5742b40ffb6104dc38ac1 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 23 Nov 2023 03:27:17 -0800 Subject: [PATCH 19/31] fix: merge artifacts --- internal/repo/drop_info.go | 2 +- internal/repo/drop_pattern_element.go | 2 +- internal/repo/notice.go | 2 +- internal/repo/selector/selector.go | 24 +++++++++++++++++++++++- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/internal/repo/drop_info.go b/internal/repo/drop_info.go index 07119e19..35eedfcf 100644 --- a/internal/repo/drop_info.go +++ b/internal/repo/drop_info.go @@ -95,7 +95,7 @@ func (r *DropInfo) GetItemDropSetByStageIdAndRangeId(ctx context.Context, server } results = lo.Uniq(results) - sort.Ints(results) + results = sort.IntSlice(results) return results, nil } diff --git a/internal/repo/drop_pattern_element.go b/internal/repo/drop_pattern_element.go index 9587f5cc..cd93a876 100644 --- a/internal/repo/drop_pattern_element.go +++ b/internal/repo/drop_pattern_element.go @@ -68,7 +68,7 @@ func (r *DropPatternElement) CreateDropPatternElements(ctx context.Context, tx b func (r *DropPatternElement) GetDropPatternElementsByPatternId(ctx context.Context, patternId int) ([]*model.DropPatternElement, error) { return r.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Where("drop_pattern_id = ?", patternId) - }) + }, selector.OptionUseZeroLenSliceOnNull) } func (r *DropPatternElement) GetDropPatternElementsByPatternIds(ctx context.Context, patternIds []int) ([]*model.DropPatternElement, error) { diff --git a/internal/repo/notice.go b/internal/repo/notice.go index f162cc56..aa3c9925 100644 --- a/internal/repo/notice.go +++ b/internal/repo/notice.go @@ -21,5 +21,5 @@ func NewNotice(db *bun.DB) *Notice { func (r *Notice) GetNotices(ctx context.Context) ([]*model.Notice, error) { return r.sel.SelectMany(ctx, func(q *bun.SelectQuery) *bun.SelectQuery { return q.Order("notice_id ASC") - }) + }, selector.OptionUseZeroLenSliceOnNull) } diff --git a/internal/repo/selector/selector.go b/internal/repo/selector/selector.go index 3a78580e..609c2147 100644 --- a/internal/repo/selector/selector.go +++ b/internal/repo/selector/selector.go @@ -20,6 +20,22 @@ func New[T any](db *bun.DB) S[T] { } } +type Option int + +const OptionUseZeroLenSliceOnNull = iota + +type Options []Option + +func (o Options) Contains(option Option) bool { + for _, v := range o { + if v == option { + return true + } + } + + return false +} + func (r S[T]) SelectOne(ctx context.Context, fn func(q *bun.SelectQuery) *bun.SelectQuery) (*T, error) { var model T err := fn(r.DB.NewSelect().Model(&model)).Scan(ctx) @@ -32,10 +48,16 @@ func (r S[T]) SelectOne(ctx context.Context, fn func(q *bun.SelectQuery) *bun.Se return &model, nil } -func (r S[T]) SelectMany(ctx context.Context, fn func(q *bun.SelectQuery) *bun.SelectQuery) ([]*T, error) { +func (r S[T]) SelectMany(ctx context.Context, fn func(q *bun.SelectQuery) *bun.SelectQuery, options ...Option) ([]*T, error) { + o := Options(options) var model []*T err := fn(r.DB.NewSelect().Model(&model)).Scan(ctx) if errors.Is(err, sql.ErrNoRows) { + if o.Contains(OptionUseZeroLenSliceOnNull) { + model = make([]*T, 0) + return model, nil + } + return nil, pgerr.ErrNotFound } else if err != nil { return nil, err From 93d10cc6c8cc98b29f4cd90c747c591cc9dc395b Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 23 Nov 2023 03:41:43 -0800 Subject: [PATCH 20/31] build: add set -e --- .github/workflows/pr-auto-upsert.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr-auto-upsert.yml b/.github/workflows/pr-auto-upsert.yml index ad7290b9..cff70eb8 100644 --- a/.github/workflows/pr-auto-upsert.yml +++ b/.github/workflows/pr-auto-upsert.yml @@ -27,6 +27,8 @@ jobs: name: Get summary from GPT continue-on-error: true run: | + set -e + # get all commit messages from last release tag to HEAD git log $(git describe --tags --abbrev=0)..HEAD --pretty=format:%s > /tmp/commit_messages.txt From 7fc82be25f1dcd40109ea1cdb3c881dc71f57be8 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 23 Nov 2023 03:46:10 -0800 Subject: [PATCH 21/31] build: use bash variables instead of tmp files --- .github/workflows/pr-auto-upsert.yml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pr-auto-upsert.yml b/.github/workflows/pr-auto-upsert.yml index cff70eb8..ed749b80 100644 --- a/.github/workflows/pr-auto-upsert.yml +++ b/.github/workflows/pr-auto-upsert.yml @@ -28,16 +28,16 @@ jobs: continue-on-error: true run: | set -e - + # get all commit messages from last release tag to HEAD - git log $(git describe --tags --abbrev=0)..HEAD --pretty=format:%s > /tmp/commit_messages.txt + COMMIT_MESSAGES=$(git log $(git describe --tags --abbrev=0)..HEAD --pretty=format:%s) # replace newlines with \n - sed -i ':a;N;$!ba;s/\n/\\n/g' /tmp/commit_messages.txt + COMMIT_MESSAGES=$(echo $COMMIT_MESSAGES | sed ':a;N;$!ba;s/\n/\\n/g') - echo "commit messages: $(cat /tmp/commit_messages.txt)" + echo "commit messages: $COMMIT_MESSAGES" - curl https://api.openai.com/v1/chat/completions \ + RESULT=$(curl https://api.openai.com/v1/chat/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer ${{ secrets.OPENAI_API_KEY }}" \ -d '{ @@ -65,17 +65,14 @@ jobs: "top_p": 1, "frequency_penalty": 0.1, "presence_penalty": 0.2 - }' | jq -r '.choices[0].message.content' > /tmp/summary.txt - - # replace newlines with spaces - sed -i ':a;N;$!ba;s/\n/ /g' /tmp/summary.txt + }' | jq -r '.choices[0].message.content') # replace " with \" - sed -i 's/"/\\"/g' /tmp/summary.txt + RESULT=$(echo $RESULT | sed 's/"/\\"/g') - echo "summary: $(cat /tmp/summary.txt)" + echo "summary: $RESULT" - echo "summary=$(cat /tmp/summary.txt)" >> $GITHUB_OUTPUT + echo "summary=$RESULT" >> $GITHUB_OUTPUT # create a PR from dev to main, with title in form: Release # where, is the next version number to be released, based on the last release in git tag From 00593bdef686848806f6872ea0a4955654cdf6b6 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 23 Nov 2023 03:46:20 -0800 Subject: [PATCH 22/31] build: use bash variables instead of tmp files --- .github/workflows/pr-auto-upsert.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-auto-upsert.yml b/.github/workflows/pr-auto-upsert.yml index ed749b80..30efccad 100644 --- a/.github/workflows/pr-auto-upsert.yml +++ b/.github/workflows/pr-auto-upsert.yml @@ -57,7 +57,7 @@ jobs: }, { "role": "user", - "content": "$(cat /tmp/commit_messages.txt)" + "content": "$COMMIT_MESSAGES" } ], "temperature": 1, From fe9990d6f048a81d046d2f67acfae732512761a6 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 23 Nov 2023 03:49:16 -0800 Subject: [PATCH 23/31] build: use bash variables instead of tmp files --- .github/workflows/pr-auto-upsert.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-auto-upsert.yml b/.github/workflows/pr-auto-upsert.yml index 30efccad..94bbf430 100644 --- a/.github/workflows/pr-auto-upsert.yml +++ b/.github/workflows/pr-auto-upsert.yml @@ -57,7 +57,7 @@ jobs: }, { "role": "user", - "content": "$COMMIT_MESSAGES" + "content": "'$COMMIT_MESSAGES'" } ], "temperature": 1, From 476f0110e06479c10e5924bd35848681d5540d83 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 23 Nov 2023 03:57:04 -0800 Subject: [PATCH 24/31] build: use bash variables instead of tmp files --- .github/workflows/pr-auto-upsert.yml | 141 ++++++++++++++++++--------- 1 file changed, 96 insertions(+), 45 deletions(-) diff --git a/.github/workflows/pr-auto-upsert.yml b/.github/workflows/pr-auto-upsert.yml index 94bbf430..a48de6fe 100644 --- a/.github/workflows/pr-auto-upsert.yml +++ b/.github/workflows/pr-auto-upsert.yml @@ -23,56 +23,56 @@ jobs: # fetch all history so that git log can get all commit messages fetch-depth: 0 - - id: get_summary - name: Get summary from GPT - continue-on-error: true - run: | - set -e + # - id: get_summary + # name: Get summary from GPT + # continue-on-error: true + # run: | + # set -e - # get all commit messages from last release tag to HEAD - COMMIT_MESSAGES=$(git log $(git describe --tags --abbrev=0)..HEAD --pretty=format:%s) + # # get all commit messages from last release tag to HEAD + # COMMIT_MESSAGES=$(git log $(git describe --tags --abbrev=0)..HEAD --pretty=format:%s) - # replace newlines with \n - COMMIT_MESSAGES=$(echo $COMMIT_MESSAGES | sed ':a;N;$!ba;s/\n/\\n/g') + # # replace newlines with \n + # COMMIT_MESSAGES=$(echo $COMMIT_MESSAGES | sed ':a;N;$!ba;s/\n/\\n/g') - echo "commit messages: $COMMIT_MESSAGES" + # echo "commit messages: $COMMIT_MESSAGES" - RESULT=$(curl https://api.openai.com/v1/chat/completions \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer ${{ secrets.OPENAI_API_KEY }}" \ - -d '{ - "model": "gpt-3.5-turbo", - "messages": [ - { - "role": "system", - "content": "You are now a git commit message summarizer. By analyzing commit messages, you should provide a concise summary, highlighting key changes and updates made in each commit. This can save time and improve clarity for developers working in collaborative projects.\n\nCommits are typically constructed using the Conventional Commit standard.\n\nYour summarized message should consist of 1-3 sentences that are compacted into sentences for each of the categories (e.g. Added, Fixed, Removed). You should NOT use bullet points." - }, - { - "role": "user", - "content": "feat: remove deleted required flag\nbuild: add workflow_dispatch\nfeat: add rowsAffected & fix json struct tag\nchore: add logs" - }, - { - "role": "assistant", - "content": "Added: `workflow_dispatch`, `rowsAffected`, logs.\nFixed: JSON struct tag.\nRemoved: required flag of `deleted`." - }, - { - "role": "user", - "content": "'$COMMIT_MESSAGES'" - } - ], - "temperature": 1, - "max_tokens": 256, - "top_p": 1, - "frequency_penalty": 0.1, - "presence_penalty": 0.2 - }' | jq -r '.choices[0].message.content') + # RESULT=$(curl https://api.openai.com/v1/chat/completions \ + # -H "Content-Type: application/json" \ + # -H "Authorization: Bearer ${{ secrets.OPENAI_API_KEY }}" \ + # -d '{ + # "model": "gpt-3.5-turbo", + # "messages": [ + # { + # "role": "system", + # "content": "You are now a git commit message summarizer. By analyzing commit messages, you should provide a concise summary, highlighting key changes and updates made in each commit. This can save time and improve clarity for developers working in collaborative projects.\n\nCommits are typically constructed using the Conventional Commit standard.\n\nYour summarized message should consist of 1-3 sentences that are compacted into sentences for each of the categories (e.g. Added, Fixed, Removed). You should NOT use bullet points." + # }, + # { + # "role": "user", + # "content": "feat: remove deleted required flag\nbuild: add workflow_dispatch\nfeat: add rowsAffected & fix json struct tag\nchore: add logs" + # }, + # { + # "role": "assistant", + # "content": "Added: `workflow_dispatch`, `rowsAffected`, logs.\nFixed: JSON struct tag.\nRemoved: required flag of `deleted`." + # }, + # { + # "role": "user", + # "content": "'$COMMIT_MESSAGES'" + # } + # ], + # "temperature": 1, + # "max_tokens": 256, + # "top_p": 1, + # "frequency_penalty": 0.1, + # "presence_penalty": 0.2 + # }' | jq -r '.choices[0].message.content') - # replace " with \" - RESULT=$(echo $RESULT | sed 's/"/\\"/g') + # # replace " with \" + # RESULT=$(echo $RESULT | sed 's/"/\\"/g') - echo "summary: $RESULT" + # echo "summary: $RESULT" - echo "summary=$RESULT" >> $GITHUB_OUTPUT + # echo "summary=$RESULT" >> $GITHUB_OUTPUT # create a PR from dev to main, with title in form: Release # where, is the next version number to be released, based on the last release in git tag @@ -81,14 +81,65 @@ jobs: with: github-token: ${{ secrets.PAT_FOR_RELEASE_TAGGER }} script: | + const https = require('https') + + const commitMessages = await github.rest.repos.compareCommits({ + owner: context.repo.owner, + repo: context.repo.repo, + base: 'main', + head: 'dev' + }).then(res => res.data.commits.map(commit => commit.commit.message)) + + const summary = await new Promise((resolve, reject) => { + const req = https.request({ + hostname: 'api.openai.com', + path: '/v1/chat/completions', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${{ secrets.OPENAI_API_KEY }}` + } + }, res => { + let data = '' + res.on('data', chunk => data += chunk) + res.on('end', () => resolve(JSON.parse(data).choices[0].text)) + }) + req.on('error', reject) + req.write(JSON.stringify({ + "messages": [ + { + "role": "system", + "content": "You are now a git commit message summarizer. By analyzing commit messages, you should provide a concise summary, highlighting key changes and updates made in each commit. This can save time and improve clarity for developers working in collaborative projects.\n\nCommits are typically constructed using the Conventional Commit standard.\n\nYour summarized message should consist of 1-3 sentences that are compacted into sentences for each of the categories (e.g. Added, Fixed, Removed). You should NOT use bullet points." + }, + { + "role": "user", + "content": "feat: remove deleted required flag\nbuild: add workflow_dispatch\nfeat: add rowsAffected & fix json struct tag\nchore: add logs" + }, + { + "role": "assistant", + "content": "Added: `workflow_dispatch`, `rowsAffected`, logs.\nFixed: JSON struct tag.\nRemoved: required flag of `deleted`." + }, + { + "role": "user", + "content": commitMessages.join('\n') + } + ], + temperature: 1, + max_tokens: 256, + top_p: 1, + frequency_penalty: 0.1, + presence_penalty: 0.2 + })) + req.end() + }) + const { data: { tag_name: lastRelease } } = await github.rest.repos.getLatestRelease({ owner: context.repo.owner, repo: context.repo.repo }) const nextRelease = lastRelease.replace(/(\d+)$/, (match, p1) => Number(p1) + 1) const prTitle = `Release ${nextRelease}` - let body = `> *This PR is automatically created by actions defined in this repository. To see the run log of this action, please click [here](/${{ github.repository }}/actions/runs/${{ github.run_id }})*\n\n---\n\n## Summary\n\n` - body += "${{ steps.get_summary.outputs.summary || '(no summary)' }}" + let body = `> *This PR is automatically created by actions defined in this repository. To see the run log of this action, please click [here](/${{ github.repository }}/actions/runs/${{ github.run_id }})*\n\n---\n\n## Summary\n\n${summary}` const existedPR = await github.rest.pulls.list({ owner: context.repo.owner, repo: context.repo.repo, From d63863b6f0241c7a976e8fd85f5915989eafcd39 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 23 Nov 2023 03:58:57 -0800 Subject: [PATCH 25/31] build: use github-script --- .github/workflows/pr-auto-upsert.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-auto-upsert.yml b/.github/workflows/pr-auto-upsert.yml index a48de6fe..7a9fc3a8 100644 --- a/.github/workflows/pr-auto-upsert.yml +++ b/.github/workflows/pr-auto-upsert.yml @@ -102,7 +102,10 @@ jobs: }, res => { let data = '' res.on('data', chunk => data += chunk) - res.on('end', () => resolve(JSON.parse(data).choices[0].text)) + res.on('end', () => { + console.log(data) + resolve(JSON.parse(data).choices[0].message.text) + }) }) req.on('error', reject) req.write(JSON.stringify({ From 61f42aff33035442e822de498d39d90f9684a770 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 23 Nov 2023 04:00:02 -0800 Subject: [PATCH 26/31] build: use github-script --- .github/workflows/pr-auto-upsert.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-auto-upsert.yml b/.github/workflows/pr-auto-upsert.yml index 7a9fc3a8..de8e19e2 100644 --- a/.github/workflows/pr-auto-upsert.yml +++ b/.github/workflows/pr-auto-upsert.yml @@ -109,6 +109,7 @@ jobs: }) req.on('error', reject) req.write(JSON.stringify({ + "model": "gpt-3.5-turbo", "messages": [ { "role": "system", From c0349169c71bf786f17b7aaae923c8f9fbf07fe1 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 23 Nov 2023 04:02:47 -0800 Subject: [PATCH 27/31] build: use gpt-4 --- .github/workflows/pr-auto-upsert.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pr-auto-upsert.yml b/.github/workflows/pr-auto-upsert.yml index de8e19e2..daca1f3c 100644 --- a/.github/workflows/pr-auto-upsert.yml +++ b/.github/workflows/pr-auto-upsert.yml @@ -103,13 +103,12 @@ jobs: let data = '' res.on('data', chunk => data += chunk) res.on('end', () => { - console.log(data) resolve(JSON.parse(data).choices[0].message.text) }) }) req.on('error', reject) req.write(JSON.stringify({ - "model": "gpt-3.5-turbo", + "model": "gpt-4", "messages": [ { "role": "system", From 865e17a14518d8809b22c5c38005d6e730c462ee Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 23 Nov 2023 04:03:31 -0800 Subject: [PATCH 28/31] build: fix resolve content --- .github/workflows/pr-auto-upsert.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-auto-upsert.yml b/.github/workflows/pr-auto-upsert.yml index daca1f3c..4c133d8a 100644 --- a/.github/workflows/pr-auto-upsert.yml +++ b/.github/workflows/pr-auto-upsert.yml @@ -103,7 +103,7 @@ jobs: let data = '' res.on('data', chunk => data += chunk) res.on('end', () => { - resolve(JSON.parse(data).choices[0].message.text) + resolve(JSON.parse(data).choices[0].message.content) }) }) req.on('error', reject) From bb4320bd8ef535fd268ee4da5fe971fdf589f89f Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 23 Nov 2023 04:04:22 -0800 Subject: [PATCH 29/31] build: use name `Upsert PR` --- .github/workflows/pr-auto-upsert.yml | 53 +--------------------------- 1 file changed, 1 insertion(+), 52 deletions(-) diff --git a/.github/workflows/pr-auto-upsert.yml b/.github/workflows/pr-auto-upsert.yml index 4c133d8a..d4779c73 100644 --- a/.github/workflows/pr-auto-upsert.yml +++ b/.github/workflows/pr-auto-upsert.yml @@ -23,60 +23,9 @@ jobs: # fetch all history so that git log can get all commit messages fetch-depth: 0 - # - id: get_summary - # name: Get summary from GPT - # continue-on-error: true - # run: | - # set -e - - # # get all commit messages from last release tag to HEAD - # COMMIT_MESSAGES=$(git log $(git describe --tags --abbrev=0)..HEAD --pretty=format:%s) - - # # replace newlines with \n - # COMMIT_MESSAGES=$(echo $COMMIT_MESSAGES | sed ':a;N;$!ba;s/\n/\\n/g') - - # echo "commit messages: $COMMIT_MESSAGES" - - # RESULT=$(curl https://api.openai.com/v1/chat/completions \ - # -H "Content-Type: application/json" \ - # -H "Authorization: Bearer ${{ secrets.OPENAI_API_KEY }}" \ - # -d '{ - # "model": "gpt-3.5-turbo", - # "messages": [ - # { - # "role": "system", - # "content": "You are now a git commit message summarizer. By analyzing commit messages, you should provide a concise summary, highlighting key changes and updates made in each commit. This can save time and improve clarity for developers working in collaborative projects.\n\nCommits are typically constructed using the Conventional Commit standard.\n\nYour summarized message should consist of 1-3 sentences that are compacted into sentences for each of the categories (e.g. Added, Fixed, Removed). You should NOT use bullet points." - # }, - # { - # "role": "user", - # "content": "feat: remove deleted required flag\nbuild: add workflow_dispatch\nfeat: add rowsAffected & fix json struct tag\nchore: add logs" - # }, - # { - # "role": "assistant", - # "content": "Added: `workflow_dispatch`, `rowsAffected`, logs.\nFixed: JSON struct tag.\nRemoved: required flag of `deleted`." - # }, - # { - # "role": "user", - # "content": "'$COMMIT_MESSAGES'" - # } - # ], - # "temperature": 1, - # "max_tokens": 256, - # "top_p": 1, - # "frequency_penalty": 0.1, - # "presence_penalty": 0.2 - # }' | jq -r '.choices[0].message.content') - - # # replace " with \" - # RESULT=$(echo $RESULT | sed 's/"/\\"/g') - - # echo "summary: $RESULT" - - # echo "summary=$RESULT" >> $GITHUB_OUTPUT - # create a PR from dev to main, with title in form: Release # where, is the next version number to be released, based on the last release in git tag - - name: Create PR + - name: Upsert PR Content uses: actions/github-script@v6 with: github-token: ${{ secrets.PAT_FOR_RELEASE_TAGGER }} From a082ec8e8a0d4d6c243797a98a050bc15e456cb4 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 23 Nov 2023 04:07:02 -0800 Subject: [PATCH 30/31] build: improve summary prompt --- .github/workflows/pr-auto-upsert.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-auto-upsert.yml b/.github/workflows/pr-auto-upsert.yml index d4779c73..654adee5 100644 --- a/.github/workflows/pr-auto-upsert.yml +++ b/.github/workflows/pr-auto-upsert.yml @@ -61,7 +61,7 @@ jobs: "messages": [ { "role": "system", - "content": "You are now a git commit message summarizer. By analyzing commit messages, you should provide a concise summary, highlighting key changes and updates made in each commit. This can save time and improve clarity for developers working in collaborative projects.\n\nCommits are typically constructed using the Conventional Commit standard.\n\nYour summarized message should consist of 1-3 sentences that are compacted into sentences for each of the categories (e.g. Added, Fixed, Removed). You should NOT use bullet points." + "content": "You are now a git commit message summarizer. By analyzing commit messages, you should provide a concise summary, highlighting key changes and updates made in each commit. This can save time and improve clarity for developers working in collaborative projects.\n\nCommits are typically constructed using the Conventional Commit standard.\n\nYour summarized message should consist of 1-3 sentences that are short, concise and includes the overall intent of the PR. You should NOT use bullet points. You MAY omit merge commits in your response." }, { "role": "user", @@ -91,7 +91,7 @@ jobs: }) const nextRelease = lastRelease.replace(/(\d+)$/, (match, p1) => Number(p1) + 1) const prTitle = `Release ${nextRelease}` - let body = `> *This PR is automatically created by actions defined in this repository. To see the run log of this action, please click [here](/${{ github.repository }}/actions/runs/${{ github.run_id }})*\n\n---\n\n## Summary\n\n${summary}` + let body = `> *This PR is automatically created by actions defined in this repository. To see the run log of this action, please click [here](/${{ github.repository }}/actions/runs/${{ github.run_id }})*\n\n---\n\n## Summary\n\n${summary.replace(\n/g, '\n\n')}` const existedPR = await github.rest.pulls.list({ owner: context.repo.owner, repo: context.repo.repo, From 7f21565739b7301df425e5ec08ca1a78d125b94a Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 23 Nov 2023 04:09:42 -0800 Subject: [PATCH 31/31] build: fix summarizer --- .github/workflows/pr-auto-upsert.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-auto-upsert.yml b/.github/workflows/pr-auto-upsert.yml index 654adee5..f83441d6 100644 --- a/.github/workflows/pr-auto-upsert.yml +++ b/.github/workflows/pr-auto-upsert.yml @@ -91,7 +91,7 @@ jobs: }) const nextRelease = lastRelease.replace(/(\d+)$/, (match, p1) => Number(p1) + 1) const prTitle = `Release ${nextRelease}` - let body = `> *This PR is automatically created by actions defined in this repository. To see the run log of this action, please click [here](/${{ github.repository }}/actions/runs/${{ github.run_id }})*\n\n---\n\n## Summary\n\n${summary.replace(\n/g, '\n\n')}` + let body = `> *This PR is automatically created by actions defined in this repository. To see the run log of this action, please click [here](/${{ github.repository }}/actions/runs/${{ github.run_id }})*\n\n---\n\n## Summary\n\n${summary.replace(/\n/g, '\n\n')}` const existedPR = await github.rest.pulls.list({ owner: context.repo.owner, repo: context.repo.repo,