diff --git a/go.mod b/go.mod index 2ad40d55..22738f56 100644 --- a/go.mod +++ b/go.mod @@ -25,10 +25,10 @@ require ( github.com/go-sql-driver/mysql v1.8.1 github.com/gobuffalo/fizz v1.14.4 github.com/gobuffalo/httptest v1.5.2 - github.com/gobuffalo/pop/v6 v6.0.8 + github.com/gobuffalo/pop/v6 v6.1.1 github.com/gobwas/glob v0.2.3 github.com/goccy/go-yaml v1.9.6 - github.com/gofrs/uuid v4.3.0+incompatible + github.com/gofrs/uuid v4.3.1+incompatible github.com/golang-jwt/jwt/v5 v5.2.1 github.com/golang/mock v1.6.0 github.com/google/go-jsonnet v0.20.0 @@ -138,12 +138,12 @@ require ( github.com/go-openapi/swag v0.22.3 // indirect github.com/go-pdf/fpdf v0.6.0 // indirect github.com/gobuffalo/envy v1.10.2 // indirect - github.com/gobuffalo/flect v0.3.0 // indirect + github.com/gobuffalo/flect v1.0.0 // indirect github.com/gobuffalo/github_flavored_markdown v1.1.3 // indirect github.com/gobuffalo/helpers v0.6.7 // indirect github.com/gobuffalo/here v0.6.7 // indirect github.com/gobuffalo/nulls v0.4.2 // indirect - github.com/gobuffalo/plush/v4 v4.1.16 // indirect + github.com/gobuffalo/plush/v4 v4.1.18 // indirect github.com/gobuffalo/tags/v3 v3.1.4 // indirect github.com/gobuffalo/validate/v3 v3.3.3 // indirect github.com/goccy/go-json v0.10.2 // indirect diff --git a/go.sum b/go.sum index 542089ee..40646e27 100644 --- a/go.sum +++ b/go.sum @@ -236,6 +236,7 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -253,8 +254,9 @@ github.com/gobuffalo/fizz v1.14.4/go.mod h1:9/2fGNXNeIFOXEEgTPJwiK63e44RjG+Nc4hf github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.3.0 h1:erfPWM+K1rFNIQeRPdeEXxo8yFr/PO17lhRnS8FUrtk= github.com/gobuffalo/flect v0.3.0/go.mod h1:5pf3aGnsvqvCj50AVni7mJJF8ICxGZ8HomberC3pXLE= +github.com/gobuffalo/flect v1.0.0 h1:eBFmskjXZgAOagiTXJH25Nt5sdFwNRcb8DKZsIsAUQI= +github.com/gobuffalo/flect v1.0.0/go.mod h1:l9V6xSb4BlXwsxEMj3FVEub2nkdQjWhPvD8XTTlHPQc= github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= @@ -284,10 +286,11 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packd v1.0.2/go.mod h1:sUc61tDqGMXON80zpKGp92lDb86Km28jfvX7IAyxFT8= github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/plush/v4 v4.1.16 h1:Y6jVVTLdg1BxRXDIbTJz+J8QRzEAtv5ZwYpGdIFR7VU= github.com/gobuffalo/plush/v4 v4.1.16/go.mod h1:6t7swVsarJ8qSLw1qyAH/KbrcSTwdun2ASEQkOznakg= -github.com/gobuffalo/pop/v6 v6.0.8 h1:9+5ShHYh3x9NDFCITfm/gtKDDRSgOwiY7kA0Hf7N9aQ= -github.com/gobuffalo/pop/v6 v6.0.8/go.mod h1:f4JQ4Zvkffcevz+t+XAwBLStD7IQs19DiIGIDFYw1eA= +github.com/gobuffalo/plush/v4 v4.1.18 h1:bnPjdMTEUQHqj9TNX2Ck3mxEXYZa+0nrFMNM07kpX9g= +github.com/gobuffalo/plush/v4 v4.1.18/go.mod h1:xi2tJIhFI4UdzIL8sxZtzGYOd2xbBpcFbLZlIPGGZhU= +github.com/gobuffalo/pop/v6 v6.1.1 h1:eUDBaZcb0gYrmFnKwpuTEUA7t5ZHqNfvS4POqJYXDZY= +github.com/gobuffalo/pop/v6 v6.1.1/go.mod h1:1n7jAmI1i7fxuXPZjZb0VBPQDbksRtCoFnrDV5IsvaI= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gobuffalo/tags/v3 v3.1.4 h1:X/ydLLPhgXV4h04Hp2xlbI2oc5MDaa7eub6zw8oHjsM= github.com/gobuffalo/tags/v3 v3.1.4/go.mod h1:ArRNo3ErlHO8BtdA0REaZxijuWnWzF6PUXngmMXd2I0= @@ -303,8 +306,8 @@ github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gofrs/uuid v4.3.0+incompatible h1:CaSVZxm5B+7o45rtab4jC2G37WGYX1zQfuU2i6DSvnc= -github.com/gofrs/uuid v4.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.3.1+incompatible h1:0/KbAdpx3UXAx1kEOWHJeOkpbgRFGHVgv+CFIY7dBJI= +github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -416,6 +419,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s= @@ -745,7 +749,7 @@ github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/ github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= diff --git a/popx/match.go b/popx/match.go new file mode 100644 index 00000000..1c99e42e --- /dev/null +++ b/popx/match.go @@ -0,0 +1,70 @@ +// Copyright © 2024 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package popx + +import ( + "fmt" + "regexp" + + "github.com/gobuffalo/pop/v6" +) + +var mrx = regexp.MustCompile( + `^(\d+)_([^.]+)(\.[a-z0-9]+)?(\.autocommit)?\.(up|down)\.(sql)$`, +) + +// Match holds the information parsed from a migration filename. +type Match struct { + Version string + Name string + DBType string + Direction string + Type string + Autocommit bool +} + +// ParseMigrationFilename parses a migration filename. +func ParseMigrationFilename(filename string) (*Match, error) { + matches := mrx.FindAllStringSubmatch(filename, -1) + if len(matches) == 0 { + return nil, nil + } + m := matches[0] + + var autocommit bool + var dbType string + if m[3] == ".autocommit" { + // A special case where autocommit group moves forward to the 3rd index. + autocommit = true + dbType = "all" + } else if m[3] == "" { + dbType = "all" + } else { + dbType = pop.CanonicalDialect(m[3][1:]) + if !pop.DialectSupported(dbType) { + return nil, fmt.Errorf("unsupported dialect %s", dbType) + } + } + + if m[6] == "fizz" && dbType != "all" { + return nil, fmt.Errorf("invalid database type %q, expected \"all\" because fizz is database type independent", dbType) + } + + if m[4] == ".autocommit" { + autocommit = true + } else if m[4] != "" { + return nil, fmt.Errorf("invalid autocommit flag %q", m[4]) + } + + match := &Match{ + Version: m[1], + Name: m[2], + DBType: dbType, + Autocommit: autocommit, + Direction: m[5], + Type: m[6], + } + + return match, nil +} diff --git a/popx/match_test.go b/popx/match_test.go new file mode 100644 index 00000000..cc088f66 --- /dev/null +++ b/popx/match_test.go @@ -0,0 +1,66 @@ +// Copyright © 2024 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package popx + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_ParseMigrationFilenameSQLUp(t *testing.T) { + r := require.New(t) + + m, err := ParseMigrationFilename("20190611004000_create_providers.up.sql") + r.NoError(err) + r.NotNil(m) + r.Equal(m.Version, "20190611004000") + r.Equal(m.Name, "create_providers") + r.Equal(m.DBType, "all") + r.Equal(m.Direction, "up") + r.Equal(m.Type, "sql") + r.Equal(m.Autocommit, false) +} + +func Test_ParseMigrationFilenameSQLUpPostgres(t *testing.T) { + r := require.New(t) + + m, err := ParseMigrationFilename("20190611004000_create_providers.pg.up.sql") + r.NoError(err) + r.NotNil(m) + r.Equal(m.Version, "20190611004000") + r.Equal(m.Name, "create_providers") + r.Equal(m.DBType, "postgres") + r.Equal(m.Direction, "up") + r.Equal(m.Type, "sql") + r.Equal(m.Autocommit, false) +} + +func Test_ParseMigrationFilenameSQLUpAutocommit(t *testing.T) { + r := require.New(t) + + m, err := ParseMigrationFilename("20190611004000_create_providers.autocommit.up.sql") + r.NoError(err) + r.NotNil(m) + r.Equal(m.Version, "20190611004000") + r.Equal(m.Name, "create_providers") + r.Equal(m.DBType, "all") + r.Equal(m.Direction, "up") + r.Equal(m.Type, "sql") + r.Equal(m.Autocommit, true) +} + +func Test_ParseMigrationFilenameSQLDownAutocommit(t *testing.T) { + r := require.New(t) + + m, err := ParseMigrationFilename("20190611004000_create_providers.mysql.autocommit.down.sql") + r.NoError(err) + r.NotNil(m) + r.Equal(m.Version, "20190611004000") + r.Equal(m.Name, "create_providers") + r.Equal(m.DBType, "mysql") + r.Equal(m.Direction, "down") + r.Equal(m.Type, "sql") + r.Equal(m.Autocommit, true) +} diff --git a/popx/migration_box.go b/popx/migration_box.go index f5919f6d..74490358 100644 --- a/popx/migration_box.go +++ b/popx/migration_box.go @@ -146,7 +146,7 @@ func NewMigrationBox(dir fs.FS, m *Migrator, opts ...MigrationBoxOption) (*Migra mb = o(mb) } - runner := func(b []byte) func(Migration, *pop.Connection, *pop.Tx) error { + txRunner := func(b []byte) func(Migration, *pop.Connection, *pop.Tx) error { return func(mf Migration, c *pop.Connection, tx *pop.Tx) error { content, err := mb.migrationContent(mf, c, b, true) if err != nil { @@ -163,7 +163,24 @@ func NewMigrationBox(dir fs.FS, m *Migrator, opts ...MigrationBoxOption) (*Migra } } - err := mb.findMigrations(runner) + autoCommitRunner := func(b []byte) func(Migration, *pop.Connection) error { + return func(mf Migration, c *pop.Connection) error { + content, err := mb.migrationContent(mf, c, b, true) + if err != nil { + return errors.Wrapf(err, "error processing %s", mf.Path) + } + if isMigrationEmpty(content) { + m.l.WithField("migration", mf.Path).Trace("This is usually ok - ignoring migration because content is empty. This is ok!") + return nil + } + if _, err = c.RawQuery(content).ExecWithCount(); err != nil { + return errors.Wrapf(err, "error executing %s, sql: %s", mf.Path, content) + } + return nil + } + } + + err := mb.findMigrations(txRunner, autoCommitRunner) if err != nil { return mb, err } @@ -178,7 +195,10 @@ func NewMigrationBox(dir fs.FS, m *Migrator, opts ...MigrationBoxOption) (*Migra return mb, nil } -func (fm *MigrationBox) findMigrations(runner func([]byte) func(mf Migration, c *pop.Connection, tx *pop.Tx) error) error { +func (fm *MigrationBox) findMigrations( + runner func([]byte) func(mf Migration, c *pop.Connection, tx *pop.Tx) error, + runnerNoTx func([]byte) func(mf Migration, c *pop.Connection) error, +) error { return fs.WalkDir(fm.Dir, ".", func(p string, info fs.DirEntry, err error) error { if err != nil { return errors.WithStack(err) @@ -188,7 +208,7 @@ func (fm *MigrationBox) findMigrations(runner func([]byte) func(mf Migration, c return nil } - match, err := pop.ParseMigrationFilename(info.Name()) + match, err := ParseMigrationFilename(info.Name()) if err != nil { if strings.HasPrefix(err.Error(), "unsupported dialect") { fm.l.Tracef("This is usually ok - ignoring migration file %s because dialect is not supported: %s", info.Name(), err.Error()) @@ -219,8 +239,14 @@ func (fm *MigrationBox) findMigrations(runner func([]byte) func(mf Migration, c DBType: match.DBType, Direction: match.Direction, Type: match.Type, - Runner: runner(content), } + + if match.Autocommit { + mf.RunnerNoTx = runnerNoTx(content) + } else { + mf.Runner = runner(content) + } + fm.Migrations[mf.Direction] = append(fm.Migrations[mf.Direction], mf) mod := sort.Interface(fm.Migrations[mf.Direction]) if mf.Direction == "down" { diff --git a/popx/migration_box_testdata_test.go b/popx/migration_box_testdata_test.go index fe6860a7..fa4c8945 100644 --- a/popx/migration_box_testdata_test.go +++ b/popx/migration_box_testdata_test.go @@ -22,6 +22,9 @@ var testData embed.FS //go:embed stub/migrations/testdata_migrations/* var empty embed.FS +//go:embed stub/migrations/notx/* +var notx embed.FS + //go:embed stub/migrations/check/valid/* var checkValidFS embed.FS @@ -54,6 +57,25 @@ func TestMigrationBoxWithTestdata(t *testing.T) { assert.Equal(t, "testdata", data.Data) } +func TestMigrationBoxWithoutTransaction(t *testing.T) { + c, err := pop.NewConnection(&pop.ConnectionDetails{ + URL: "sqlite://file::memory:?_fk=true", + }) + require.NoError(t, err) + require.NoError(t, c.Open()) + + mb, err := popx.NewMigrationBox( + notx, + popx.NewMigrator(c, logrusx.New("", ""), nil, 0), + ) + + require.NoError(t, err) + assert.Len(t, mb.Migrations["up"], 1) + assert.Len(t, mb.Migrations["down"], 1) + + require.NoError(t, mb.Up(context.Background()), "should not fail even though we are creating a transaction in the migration") +} + func TestMigrationBox_CheckNoErr(t *testing.T) { c, err := pop.NewConnection(&pop.ConnectionDetails{ URL: "sqlite://file::memory:?_fk=true", diff --git a/popx/stub/migrations/notx/20241031_notx.autocommit.down.sql b/popx/stub/migrations/notx/20241031_notx.autocommit.down.sql new file mode 100644 index 00000000..8ea7e8fb --- /dev/null +++ b/popx/stub/migrations/notx/20241031_notx.autocommit.down.sql @@ -0,0 +1 @@ +BEGIN;ROLLBACK; \ No newline at end of file diff --git a/popx/stub/migrations/notx/20241031_notx.autocommit.up.sql b/popx/stub/migrations/notx/20241031_notx.autocommit.up.sql new file mode 100644 index 00000000..8ea7e8fb --- /dev/null +++ b/popx/stub/migrations/notx/20241031_notx.autocommit.up.sql @@ -0,0 +1 @@ +BEGIN;ROLLBACK; \ No newline at end of file