From fc21c82dbf3ffe8be613fc05ebd934eb66e4854d Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Tue, 24 Dec 2024 08:57:58 +0200 Subject: [PATCH] schemadiff: range partition analysis, rotation and retention Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/schemadiff/analysis.go | 68 -- go/vt/schemadiff/analysis_test.go | 83 -- go/vt/schemadiff/diff_test.go | 34 +- go/vt/schemadiff/env.go | 7 + go/vt/schemadiff/errors.go | 10 + go/vt/schemadiff/mysql.go | 21 +- go/vt/schemadiff/partitioning_analysis.go | 593 +++++++++++ .../schemadiff/partitioning_analysis_test.go | 978 ++++++++++++++++++ go/vt/schemadiff/schema.go | 4 +- go/vt/schemadiff/schema_test.go | 23 + go/vt/schemadiff/table.go | 43 +- 11 files changed, 1692 insertions(+), 172 deletions(-) delete mode 100644 go/vt/schemadiff/analysis.go delete mode 100644 go/vt/schemadiff/analysis_test.go create mode 100644 go/vt/schemadiff/partitioning_analysis.go create mode 100644 go/vt/schemadiff/partitioning_analysis_test.go diff --git a/go/vt/schemadiff/analysis.go b/go/vt/schemadiff/analysis.go deleted file mode 100644 index ae0f22559f2..00000000000 --- a/go/vt/schemadiff/analysis.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2024 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package schemadiff - -import ( - "vitess.io/vitess/go/vt/sqlparser" -) - -// AlterTableRotatesRangePartition answers `true` when the given ALTER TABLE statement performs any sort -// of range partition rotation, that is applicable immediately and without moving data. -// Such would be: -// - Dropping any partition(s) -// - Adding a new partition (empty, at the end of the list) -func AlterTableRotatesRangePartition(createTable *sqlparser.CreateTable, alterTable *sqlparser.AlterTable) (bool, error) { - // Validate original table is partitioned by RANGE - if createTable.TableSpec.PartitionOption == nil { - return false, nil - } - if createTable.TableSpec.PartitionOption.Type != sqlparser.RangeType { - return false, nil - } - - spec := alterTable.PartitionSpec - if spec == nil { - return false, nil - } - errorResult := func(conflictingNode sqlparser.SQLNode) error { - return &PartitionSpecNonExclusiveError{ - Table: alterTable.Table.Name.String(), - PartitionSpec: spec, - ConflictingStatement: sqlparser.CanonicalString(conflictingNode), - } - } - if len(alterTable.AlterOptions) > 0 { - // This should never happen, unless someone programmatically tampered with the AlterTable AST. - return false, errorResult(alterTable.AlterOptions[0]) - } - if alterTable.PartitionOption != nil { - // This should never happen, unless someone programmatically tampered with the AlterTable AST. - return false, errorResult(alterTable.PartitionOption) - } - switch spec.Action { - case sqlparser.AddAction: - if len(spec.Definitions) > 1 { - // This should never happen, unless someone programmatically tampered with the AlterTable AST. - return false, errorResult(spec.Definitions[1]) - } - return true, nil - case sqlparser.DropAction: - return true, nil - default: - return false, nil - } -} diff --git a/go/vt/schemadiff/analysis_test.go b/go/vt/schemadiff/analysis_test.go deleted file mode 100644 index b0092fb7aac..00000000000 --- a/go/vt/schemadiff/analysis_test.go +++ /dev/null @@ -1,83 +0,0 @@ -/* -Copyright 2024 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package schemadiff - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "vitess.io/vitess/go/vt/sqlparser" -) - -// AnalyzePartitionRotation analyzes a given AlterTable statement to see whether it has partition rotation -// commands, and if so, is the ALTER TABLE statement valid in MySQL. In MySQL, a single ALTER TABLE statement -// cannot apply multiple rotation commands, nor can it mix rotation commands with other types of changes. -func TestAlterTableRotatesRangePartition(t *testing.T) { - tcases := []struct { - create string - alter string - expect bool - }{ - { - alter: "ALTER TABLE t ADD PARTITION (PARTITION p1 VALUES LESS THAN (10))", - expect: true, - }, - { - alter: "ALTER TABLE t DROP PARTITION p1", - expect: true, - }, - { - alter: "ALTER TABLE t DROP PARTITION p1, p2", - expect: true, - }, - { - alter: "ALTER TABLE t TRUNCATE PARTITION p3", - }, - { - alter: "ALTER TABLE t COALESCE PARTITION 3", - }, - { - alter: "ALTER TABLE t partition by range (id) (partition p1 values less than (10), partition p2 values less than (20), partition p3 values less than (30))", - }, - { - alter: "ALTER TABLE t ADD COLUMN c1 INT, DROP COLUMN c2", - }, - } - - for _, tcase := range tcases { - t.Run(tcase.alter, func(t *testing.T) { - if tcase.create == "" { - tcase.create = "CREATE TABLE t (id int PRIMARY KEY) PARTITION BY RANGE (id) (PARTITION p0 VALUES LESS THAN (10))" - } - stmt, err := sqlparser.NewTestParser().ParseStrictDDL(tcase.create) - require.NoError(t, err) - createTable, ok := stmt.(*sqlparser.CreateTable) - require.True(t, ok) - - stmt, err = sqlparser.NewTestParser().ParseStrictDDL(tcase.alter) - require.NoError(t, err) - alterTable, ok := stmt.(*sqlparser.AlterTable) - require.True(t, ok) - - result, err := AlterTableRotatesRangePartition(createTable, alterTable) - require.NoError(t, err) - assert.Equal(t, tcase.expect, result) - }) - } -} diff --git a/go/vt/schemadiff/diff_test.go b/go/vt/schemadiff/diff_test.go index 185d233ef20..5278f11885d 100644 --- a/go/vt/schemadiff/diff_test.go +++ b/go/vt/schemadiff/diff_test.go @@ -587,9 +587,11 @@ func TestDiffSchemas(t *testing.T) { diffs []string cdiffs []string expectError string - tableRename int annotated []string + // hints: + tableRename int fkStrategy int + rotation int }{ { name: "identical tables", @@ -922,6 +924,35 @@ func TestDiffSchemas(t *testing.T) { "ALTER TABLE `t` RENAME INDEX `i_idx` TO `i_alternative`", }, }, + // Partitions + { + name: "change partitioning range: statements, add", + from: "create table t1 (id int primary key) partition by range (id) (partition p1 values less than (10), partition p2 values less than (20))", + to: "create table t1 (id int primary key) partition by range (id) (partition p1 values less than (10), partition p2 values less than (20), partition p3 values less than (30))", + rotation: RangeRotationDistinctStatements, + diffs: []string{ + "alter table t1 add partition (partition p3 values less than (30))", + }, + cdiffs: []string{ + "ALTER TABLE `t1` ADD PARTITION (PARTITION `p3` VALUES LESS THAN (30))", + }, + }, + { + name: "change partitioning range: statements, multiple drops, distinct", + from: "create table t1 (id int primary key) partition by range (id) (partition p1 values less than (10), partition p2 values less than (20), partition p3 values less than (30))", + to: "create table t1 (id int primary key) partition by range (id) (partition p3 values less than (30))", + rotation: RangeRotationDistinctStatements, + diffs: []string{"alter table t1 drop partition p1, p2"}, + cdiffs: []string{"ALTER TABLE `t1` DROP PARTITION `p1`, `p2`"}, + }, + { + name: "change partitioning range: statements, multiple, assorted", + from: "create table t1 (id int primary key) partition by range (id) (partition p1 values less than (10), partition p2 values less than (20), partition p3 values less than (30))", + to: "create table t1 (id int primary key) partition by range (id) (partition p2 values less than (20), partition p3 values less than (30), partition p4 values less than (40))", + rotation: RangeRotationDistinctStatements, + diffs: []string{"alter table t1 drop partition p1", "alter table t1 add partition (partition p4 values less than (40))"}, + cdiffs: []string{"ALTER TABLE `t1` DROP PARTITION `p1`", "ALTER TABLE `t1` ADD PARTITION (PARTITION `p4` VALUES LESS THAN (40))"}, + }, // Views { name: "identical views", @@ -1043,6 +1074,7 @@ func TestDiffSchemas(t *testing.T) { hints := &DiffHints{ TableRenameStrategy: ts.tableRename, ForeignKeyCheckStrategy: ts.fkStrategy, + RangeRotationStrategy: ts.rotation, } diff, err := DiffSchemasSQL(env, ts.from, ts.to, hints) if ts.expectError != "" { diff --git a/go/vt/schemadiff/env.go b/go/vt/schemadiff/env.go index 9037de40b01..01912f3fa3b 100644 --- a/go/vt/schemadiff/env.go +++ b/go/vt/schemadiff/env.go @@ -17,6 +17,13 @@ func NewTestEnv() *Environment { } } +func New84TestEnv() *Environment { + return &Environment{ + Environment: vtenv.New84TestEnv(), + DefaultColl: collations.MySQL8().DefaultConnectionCharset(), + } +} + func NewEnv(env *vtenv.Environment, defaultColl collations.ID) *Environment { return &Environment{ Environment: env, diff --git a/go/vt/schemadiff/errors.go b/go/vt/schemadiff/errors.go index c938e736206..09ca7829e0a 100644 --- a/go/vt/schemadiff/errors.go +++ b/go/vt/schemadiff/errors.go @@ -256,6 +256,16 @@ func (e *InvalidColumnInPartitionError) Error() string { sqlescape.EscapeID(e.Column), sqlescape.EscapeID(e.Table)) } +type UnsupportedRangeColumnsTypeError struct { + Table string + Column string + Type string +} + +func (e *UnsupportedRangeColumnsTypeError) Error() string { + return fmt.Sprintf("unsupported column type %s for column %s indicated by RANGE COLUMNS in table %s", e.Type, e.Column, e.Table) +} + type MissingPartitionColumnInUniqueKeyError struct { Table string Column string diff --git a/go/vt/schemadiff/mysql.go b/go/vt/schemadiff/mysql.go index 65adcc1b7a1..9c3c97429d4 100644 --- a/go/vt/schemadiff/mysql.go +++ b/go/vt/schemadiff/mysql.go @@ -16,6 +16,15 @@ limitations under the License. package schemadiff +import "strings" + +const ( + DateFormat = "2006-01-02" + TimestampFormat = "2006-01-02 15:04:05" + TimestampFormatPrecision3 = "2006-01-02 15:04:05.000" + TimestampFormatPrecision6 = "2006-01-02 15:04:05.000000" +) + var engineCasing = map[string]string{ "INNODB": "InnoDB", "MYISAM": "MyISAM", @@ -66,29 +75,29 @@ var blobStorageExponent = map[string]int{ } func IsFloatingPointType(columnType string) bool { - _, ok := floatTypes[columnType] + _, ok := floatTypes[strings.ToLower(columnType)] return ok } func FloatingPointTypeStorage(columnType string) int { - return floatTypes[columnType] + return floatTypes[strings.ToLower(columnType)] } func IsIntegralType(columnType string) bool { - _, ok := integralTypes[columnType] + _, ok := integralTypes[strings.ToLower(columnType)] return ok } func IntegralTypeStorage(columnType string) int { - return integralTypes[columnType] + return integralTypes[strings.ToLower(columnType)] } func IsDecimalType(columnType string) bool { - return decimalTypes[columnType] + return decimalTypes[strings.ToLower(columnType)] } func BlobTypeStorage(columnType string) int { - return blobStorageExponent[columnType] + return blobStorageExponent[strings.ToLower(columnType)] } // expandedDataTypes maps some known and difficult-to-compute by INFORMATION_SCHEMA data types which expand other data types. diff --git a/go/vt/schemadiff/partitioning_analysis.go b/go/vt/schemadiff/partitioning_analysis.go new file mode 100644 index 00000000000..844e707d849 --- /dev/null +++ b/go/vt/schemadiff/partitioning_analysis.go @@ -0,0 +1,593 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schemadiff + +import ( + "errors" + "fmt" + "strconv" + "strings" + "time" + + "vitess.io/vitess/go/mysql/capabilities" + "vitess.io/vitess/go/mysql/datetime" + "vitess.io/vitess/go/mysql/decimal" + "vitess.io/vitess/go/vt/sqlparser" +) + +// IsRangePartitioned returns `true` when the given CREATE TABLE statement is partitioned by RANGE. +func IsRangePartitioned(createTable *sqlparser.CreateTable) bool { + if createTable.TableSpec.PartitionOption == nil { + return false + } + if createTable.TableSpec.PartitionOption.Type != sqlparser.RangeType { + return false + } + return true +} + +// AlterTableRotatesRangePartition answers `true` when the given ALTER TABLE statement performs any sort +// of range partition rotation, that is applicable immediately and without moving data. +// Such would be: +// - Dropping any partition(s) +// - Adding a new partition (empty, at the end of the list) +func AlterTableRotatesRangePartition(createTable *sqlparser.CreateTable, alterTable *sqlparser.AlterTable) (bool, error) { + // Validate original table is partitioned by RANGE + if !IsRangePartitioned(createTable) { + return false, nil + } + + spec := alterTable.PartitionSpec + if spec == nil { + return false, nil + } + errorResult := func(conflictingNode sqlparser.SQLNode) error { + return &PartitionSpecNonExclusiveError{ + Table: alterTable.Table.Name.String(), + PartitionSpec: spec, + ConflictingStatement: sqlparser.CanonicalString(conflictingNode), + } + } + if len(alterTable.AlterOptions) > 0 { + // This should never happen, unless someone programmatically tampered with the AlterTable AST. + return false, errorResult(alterTable.AlterOptions[0]) + } + if alterTable.PartitionOption != nil { + // This should never happen, unless someone programmatically tampered with the AlterTable AST. + return false, errorResult(alterTable.PartitionOption) + } + switch spec.Action { + case sqlparser.AddAction: + if len(spec.Definitions) > 1 { + // This should never happen, unless someone programmatically tampered with the AlterTable AST. + return false, errorResult(spec.Definitions[1]) + } + return true, nil + case sqlparser.DropAction: + return true, nil + default: + return false, nil + } +} + +// TemporalRangePartitioningAnalysis is the result of analyzing a table for temporal range partitioning. +type TemporalRangePartitioningAnalysis struct { + IsRangePartitioned bool // Is the table at all partitioned by RANGE? + IsTemporalRangePartitioned bool // Is the table range partitioned using temporal values? + IsRangeColumns bool // Is RANGE COLUMNS used? + MinimalInterval datetime.IntervalType // The minimal interval that the table is partitioned by (e.g. if partitioned by TO_DAYS, the minimal interval is 1 day) + Col *ColumnDefinitionEntity // The column used in the RANGE expression + FuncExpr *sqlparser.FuncExpr // The function used in the RANGE expression, if any + MaxvaluePartition *sqlparser.PartitionDefinition // The partition that has MAXVALUE, if any + HighestValue datetime.DateTime // The datetime value of the highest partition (excluding MAXVALUE) + Reason string // Why IsTemporalRangePartitioned is false + Error error // If there was an error during analysis +} + +// AnalyzeTemporalRangePartitioning analyzes a table for temporal range partitioning. +func AnalyzeTemporalRangePartitioning(createTableEntity *CreateTableEntity) (*TemporalRangePartitioningAnalysis, error) { + analysis := &TemporalRangePartitioningAnalysis{} + withReason := func(msg string, args ...any) (*TemporalRangePartitioningAnalysis, error) { + analysis.Reason = fmt.Sprintf(msg, args...) + return analysis, nil + } + getColumn := func(lowered string) *ColumnDefinitionEntity { + // The column will exist, because we invoke Validate() before this. + return createTableEntity.ColumnDefinitionEntitiesMap()[lowered] + } + if err := createTableEntity.validate(); err != nil { + return nil, err + } + if !IsRangePartitioned(createTableEntity.CreateTable) { + return withReason("Table does not use PARTITION BY RANGE") + } + analysis.IsRangePartitioned = true + is84, err := capabilities.ServerVersionAtLeast(createTableEntity.Env.MySQLVersion(), 8, 4) + if err != nil { + return nil, err + } + + partitionOption := createTableEntity.CreateTable.TableSpec.PartitionOption + if partitionOption.SubPartition != nil { + return withReason("Table uses sub-partitions") + } + + analysis.IsRangeColumns = len(partitionOption.ColList) > 0 + switch len(partitionOption.ColList) { + case 0: + // This is a PARTITION BY RANGE(expr), where "expr" can be just column name, or a complex expression. + // Of all the options, we only support the following: + // - column_name + // - to_seconds(column_name) + // - to_days(column_name) + // - year(column_name) + // - unix_timestamp(column_name) (MySQL 8.4+) + // Instead of programmatically validating that the expression is one of the above (a complex task), + // we create dummy statements with all supported variations, and check for equality. + var col *ColumnDefinitionEntity + expr := sqlparser.CloneExpr(partitionOption.Expr) + _ = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { + switch node := node.(type) { + case *sqlparser.ColName: + col = getColumn(node.Name.Lowered()) // known to be not-nil thanks to validate() + analysis.Col = col + case *sqlparser.FuncExpr: + analysis.FuncExpr = sqlparser.CloneRefOfFuncExpr(node) + node.Name = sqlparser.NewIdentifierCI(node.Name.Lowered()) + } + return true, nil + }, expr) + supportedVariations := []string{ + "create table %s (id int) PARTITION BY RANGE (%s)", + "create table %s (id int) PARTITION BY RANGE (to_seconds(%s))", + "create table %s (id int) PARTITION BY RANGE (to_days(%s))", + "create table %s (id int) PARTITION BY RANGE (year(%s))", + } + if is84 { + supportedVariations = append(supportedVariations, "create table %s (id int) PARTITION BY RANGE (unix_timestamp(%s))") + } + supportedVariationsEntities := []*CreateTableEntity{} + for _, supportedVariation := range supportedVariations { + query := fmt.Sprintf(supportedVariation, + sqlparser.CanonicalString(createTableEntity.CreateTable.GetTable().Name), + sqlparser.CanonicalString(col.ColumnDefinition.Name), + ) + cte, err := NewCreateTableEntityFromSQL(createTableEntity.Env, query) + if err != nil { + return nil, err + } + supportedVariationsEntities = append(supportedVariationsEntities, cte) + } + matchFound := false + for i, cte := range supportedVariationsEntities { + if !sqlparser.Equals.Expr(expr, cte.CreateTable.TableSpec.PartitionOption.Expr) { + continue + } + matchFound = true + if i == 0 { + // First variation: no function, just the column + if col.IsIntegralType() { + return withReason("column %s of type %s in table %s is not a temporal type for temporal range partitioning", col.Name(), col.Type(), createTableEntity.Name()) + } + return nil, fmt.Errorf("column type %s is unsupported in column %s in table %s indicated by RANGE expression", col.Type(), col.Name(), createTableEntity.Name()) + } + // Has a function expression. + // function must operate on a TIME, DATE, DATETIME column type. See https://dev.mysql.com/doc/refman/8.0/en/partitioning-range.html + // And we only support DATE, DATETIME (because range rotation over a TIME column is not meaningful). + // In MySQL 8.4 it is also possible to use a TIMESTAMP column specifically with UNIX_TIMESTAMP() function. + // See https://dev.mysql.com/doc/refman/8.4/en/partitioning-range.html + switch strings.ToLower(col.Type()) { + case "date", "datetime": + // OK + case "timestamp": + if is84 && analysis.FuncExpr != nil && analysis.FuncExpr.Name.Lowered() == "unix_timestamp" { + analysis.MinimalInterval = datetime.IntervalSecond + } else { + return nil, fmt.Errorf("column type %s is unsupported in temporal range partitioning analysis for column %s in table %s", col.Type(), col.Name(), createTableEntity.Name()) + } + default: + return nil, fmt.Errorf("column type %s is unsupported in temporal range partitioning analysis for column %s in table %s", col.Type(), col.Name(), createTableEntity.Name()) + } + } + if !matchFound { + return nil, fmt.Errorf("expression: %v is unsupported in temporal range partitioning analysis in table %s", sqlparser.CanonicalString(partitionOption.Expr), createTableEntity.Name()) + } + case 1: + // PARTITION BY RANGE COLUMNS (single_column). For temporal range rotation, we only + // support DATE and DATETIME. + col := getColumn(partitionOption.ColList[0].Lowered()) + analysis.Col = col + switch strings.ToLower(col.Type()) { + case "date": + analysis.MinimalInterval = datetime.IntervalDay + case "datetime": + // Good! + default: + // Generally allowed by MySQL, but not considered as "temporal" for our purposes. + return withReason("%s type in column %s is not temporal for temporal range partitioning purposes", col.Type(), col.Name()) + } + default: + // PARTITION BY RANGE COLUMNS (col1, col2, ...) + // Multiple columns do not depict a temporal range. + return withReason("Table uses multiple columns in RANGE COLUMNS") + } + analysis.IsTemporalRangePartitioned = true + if analysis.FuncExpr != nil { + switch analysis.FuncExpr.Name.Lowered() { + case "unix_timestamp": + analysis.MinimalInterval = datetime.IntervalSecond + case "to_seconds": + analysis.MinimalInterval = datetime.IntervalSecond + case "to_days": + analysis.MinimalInterval = datetime.IntervalDay + case "year": + analysis.MinimalInterval = datetime.IntervalYear + } + } + for _, partition := range partitionOption.Definitions { + if partition.Options == nil { + continue + } + if partition.Options.ValueRange == nil { + continue + } + if partition.Options.ValueRange.Maxvalue { + analysis.MaxvaluePartition = partition + } else { + highestValue, err := computeDateTime(partition.Options.ValueRange.Range[0], analysis.Col.Type(), analysis.FuncExpr) + if err != nil { + return analysis, err + } + highestValue, err = truncateDateTime(highestValue, analysis.MinimalInterval) + if err != nil { + return analysis, err + } + analysis.HighestValue = highestValue + } + } + return analysis, nil +} + +func parseDateTime(s string, colType string) (result datetime.DateTime, err error) { + switch strings.ToLower(colType) { + case "date": + d, ok := datetime.ParseDate(s) + if ok { + return datetime.DateTime{Date: d}, nil + } + return result, fmt.Errorf("invalid date literal %s", s) + case "datetime": + result, _, ok := datetime.ParseDateTime(s, -1) + if ok { + return result, err + } + // It's also OK to parse a datetime out of a date. + d, ok := datetime.ParseDate(s) + if ok { + return datetime.DateTime{Date: d}, nil + } + return result, fmt.Errorf("invalid datetime literal %s", s) + default: + return result, fmt.Errorf("unsupported column type %s", colType) + } +} + +// computeDateTime computes a datetime value from a given expression. +// We assume AnalyzeTemporalRangePartitioning has already executed, which means we've validated the expression +// to be one of supported variations. +func computeDateTime(expr sqlparser.Expr, colType string, funcExpr *sqlparser.FuncExpr) (result datetime.DateTime, err error) { + if funcExpr == nil { + // This is a simple column name, and we only support DATE and DATETIME types. So the value + // must be a literal date or datetime representation, e.g. '2021-01-05' or '2021-01-05 17:00:00'. + literal, ok := expr.(*sqlparser.Literal) + if !ok { + return result, fmt.Errorf("expected literal value in %s", sqlparser.CanonicalString(expr)) + } + if literal.Type != sqlparser.StrVal { + return result, fmt.Errorf("expected string literal value in %s", sqlparser.CanonicalString(expr)) + } + return parseDateTime(literal.Val, colType) + } + // The table is partitioned using a function. + // The function may or may not appear in the expression. Normally it will not, since MySQL computes a literal out of a + // partition definition with function, e.g. + // `PARTITION p0 VALUES LESS THAN (TO_DAYS('2021-01-01'))` computes to + // `PARTITION p0 VALUES LESS THAN (738156)`. + // However, schemadiff supports declarative statements, and those may include the function. + + var literal *sqlparser.Literal + var hasFuncExpr bool + err = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { + switch node := node.(type) { + case *sqlparser.FuncExpr: + if node.Name.Lowered() != funcExpr.Name.Lowered() { + return false, fmt.Errorf("expected function %s, got %s", funcExpr.Name.Lowered(), node.Name.Lowered()) + } + hasFuncExpr = true + case *sqlparser.Literal: + literal = node + } + return true, nil + }, expr) + if err != nil { + return result, err + } + if literal == nil { + return result, fmt.Errorf("expected literal value in %s", sqlparser.CanonicalString(expr)) + } + if hasFuncExpr { + // e.g. `PARTITION p0 VALUES LESS THAN (TO_DAYS('2021-01-01'))` + // The literal must be a DATE or DATETIME, on which the function operates. + if literal.Type != sqlparser.StrVal { + return result, fmt.Errorf("expected string literal value in %s", sqlparser.CanonicalString(expr)) + } + // The literal is the the value we're looking for. + return parseDateTime(literal.Val, colType) + } + // No function expression + // e.g. `PARTITION p0 VALUES LESS THAN (738156)` + // The literal must be an integer, because the function is not present in the expression. + if literal.Type != sqlparser.IntVal { + return result, fmt.Errorf("expected integer literal value in %s", sqlparser.CanonicalString(expr)) + } + intval, err := strconv.ParseInt(literal.Val, 0, 64) + if err != nil { + return result, err + } + // We now want to produce the original DATE or DATETIME value by reversing function on the literal. + switch funcExpr.Name.Lowered() { + case "unix_timestamp": + t := time.Unix(intval, 0) + return datetime.NewDateTimeFromStd(t), nil + case "to_seconds": + return datetime.NewDateTimeFromSeconds(decimal.NewFromInt(intval)), nil + case "to_days": + d := datetime.DateFromDayNumber(int(intval)) + return datetime.DateTime{Date: d}, nil + case "year": + return parseDateTime(fmt.Sprintf("%d-01-01", intval), "date") + default: + return result, fmt.Errorf("unsupported function %s in RANGE expression", funcExpr.Name.Lowered()) + } +} + +// temporalPartitionName returns a name for a partition, based on a given DATETIME and resolution. +// It either returns a short name or a long name, depending on the resolution. +// The name is prepended with "p". +// It's noteworthy that this is naming system is purely conventional. MySQL does not enforce any naming convention. +func temporalPartitionName(dt datetime.DateTime, resolution datetime.IntervalType) (string, error) { + switch resolution { + case datetime.IntervalYear, + datetime.IntervalMonth, + datetime.IntervalDay: + return "p" + string(datetime.Date_YYYYMMDD.Format(dt, 0)), nil + case datetime.IntervalHour, + datetime.IntervalMinute, + datetime.IntervalSecond: + return "p" + string(datetime.DateTime_YYYYMMDDhhmmss.Format(dt, 0)), nil + } + return "", fmt.Errorf("unsupported resolution %s", resolution.ToString()) +} + +// truncateDateTime truncates a datetime to a given resolution. +// e.g. if resolution is IntervalDay, the time part is removed. +// If resolution is IntervalMonth, the day part is set to 1. +// etc. +func truncateDateTime(dt datetime.DateTime, interval datetime.IntervalType) (datetime.DateTime, error) { + if interval >= datetime.IntervalHour { + // Remove the minutes, seconds, subseconds parts + hourInterval := datetime.ParseIntervalInt64(int64(dt.Time.Hour()), datetime.IntervalHour, false) + dt = datetime.DateTime{Date: dt.Date} // Remove the Time part + var ok bool + dt, _, ok = dt.AddInterval(hourInterval, 0, false) + if !ok { + return dt, fmt.Errorf("failed to add interval %v to reference time %v", hourInterval, dt.Format(0)) + } + } + if interval >= datetime.IntervalDay { + // Remove the Time part: + dt = datetime.DateTime{Date: dt.Date} + } + if interval >= datetime.IntervalMonth { + // Get back to the first day of the month: + dayInterval := datetime.ParseIntervalInt64(int64(-(dt.Date.Day() - 1)), datetime.IntervalDay, false) + var ok bool + dt, _, ok = dt.AddInterval(dayInterval, 0, false) + if !ok { + return dt, fmt.Errorf("failed to add interval %v to reference time %v", dayInterval, dt.Format(0)) + } + } + if interval >= datetime.IntervalYear { + // Get back to the first day of the year: + monthInterval := datetime.ParseIntervalInt64(int64(-(dt.Date.Month() - 1)), datetime.IntervalMonth, false) + var ok bool + dt, _, ok = dt.AddInterval(monthInterval, 0, false) + if !ok { + return dt, fmt.Errorf("failed to add interval %v to reference time %v", monthInterval, dt.Format(0)) + } + } + return dt, nil +} + +// TemporalRangePartitioningNextRotation returns a (possibly empty) diffs that, if run sequentially, +// will rotate forward a range partitioned table. +// The table must be range partitioned by temporal values, otherwise an error is returned. +// The input indicates how many rotations to prepare ahead, and the reference time to start from, +// e.g. "prepare 7 days ahead, starting from today". +// The function computes values of existing partitions to determine how many new partitions are actually +// required to satisfy the terms. +func TemporalRangePartitioningNextRotation(createTableEntity *CreateTableEntity, interval datetime.IntervalType, prepareAheadCount int, reference time.Time) (diffs []*AlterTableEntityDiff, err error) { + analysis, err := AnalyzeTemporalRangePartitioning(createTableEntity) + if err != nil { + return nil, err + } + if !analysis.IsTemporalRangePartitioned { + return nil, errors.New(analysis.Reason) + } + if interval < analysis.MinimalInterval { + return nil, fmt.Errorf("interval %s is less than the minimal interval %s for table %s", interval.ToString(), analysis.MinimalInterval.ToString(), createTableEntity.Name()) + } + referenceDatetime, err := truncateDateTime(datetime.NewDateTimeFromStd(reference), interval) + if err != nil { + return nil, err + } + + for i := range prepareAheadCount { + aheadInterval := datetime.ParseIntervalInt64(int64(i+1), interval, false) + aheadDatetime, _, ok := referenceDatetime.AddInterval(aheadInterval, 0, false) + if !ok { + return nil, fmt.Errorf("failed to add interval %v to reference time %v", aheadInterval, reference) + } + if aheadDatetime.Compare(analysis.HighestValue) <= 0 { + // This `LESS THAN` value is already covered by an existing partition. + continue + } + + var partitionExpr sqlparser.Literal + switch { + case analysis.IsRangeColumns: + // PARTITION BY RANGE COLUMNS. The column could be DATE or DATETIME + partitionExpr.Type = sqlparser.StrVal + switch strings.ToLower(analysis.Col.Type()) { + case "date": + partitionExpr.Val = string(aheadDatetime.Date.Format()) + case "datetime": + partitionExpr.Val = string(aheadDatetime.Format(0)) + default: + return nil, fmt.Errorf("unsupported partitioning rotation in table %s", createTableEntity.Name()) + } + case analysis.FuncExpr != nil: + partitionExpr.Type = sqlparser.IntVal + var intval int64 + switch analysis.FuncExpr.Name.Lowered() { + case "unix_timestamp": + intval = aheadDatetime.ToStdTime(reference).UTC().Unix() + case "to_seconds": + intval = aheadDatetime.ToSeconds() + case "to_days": + intval = int64(datetime.MysqlDayNumber(aheadDatetime.Date.Year(), aheadDatetime.Date.Month(), aheadDatetime.Date.Day())) + case "year": + intval = int64(aheadDatetime.Date.Year()) + default: + return nil, fmt.Errorf("unsupported partitioning rotation in table %s", createTableEntity.Name()) + } + partitionExpr.Val = fmt.Sprintf("%d", intval) + default: + return nil, fmt.Errorf("unsupported partitioning rotation in table %s", createTableEntity.Name()) + } + // Compute new partition: + partitionNameAheadInterval := datetime.ParseIntervalInt64(int64(i), interval, false) + partitionNameAheadDatetime, _, ok := referenceDatetime.AddInterval(partitionNameAheadInterval, 0, false) + if !ok { + return nil, fmt.Errorf("failed to add interval %v to reference time %v", partitionNameAheadInterval, reference) + } + partitionName, err := temporalPartitionName(partitionNameAheadDatetime, interval) + if err != nil { + return nil, err + } + newPartition := &sqlparser.PartitionDefinition{ + Name: sqlparser.NewIdentifierCI(partitionName), + Options: &sqlparser.PartitionDefinitionOptions{ + ValueRange: &sqlparser.PartitionValueRange{ + Type: sqlparser.LessThanType, + Maxvalue: false, + Range: []sqlparser.Expr{ + &partitionExpr, + }, + }, + }, + } + // Build the diff + var partitionSpec *sqlparser.PartitionSpec + switch analysis.MaxvaluePartition { + case nil: + // ADD PARTITION + partitionSpec = &sqlparser.PartitionSpec{ + Action: sqlparser.AddAction, + Definitions: []*sqlparser.PartitionDefinition{newPartition}, + } + default: + // REORGANIZE PARTITION + // alter table `test`.`quarterly_report_status` reorganize partition `p6` into (partition `p20090701000000` values less than (1246395600) /* 2009-07-01 00:00:00 */ , partition p_maxvalue values less than MAXVALUE) + partitionSpec = &sqlparser.PartitionSpec{ + Action: sqlparser.ReorganizeAction, + Names: sqlparser.Partitions{analysis.MaxvaluePartition.Name}, + Definitions: []*sqlparser.PartitionDefinition{newPartition, analysis.MaxvaluePartition}, + } + } + alterTable := &sqlparser.AlterTable{ + Table: createTableEntity.Table, + PartitionSpec: partitionSpec, + } + diff := &AlterTableEntityDiff{alterTable: alterTable, from: createTableEntity} + diffs = append(diffs, diff) + } + return diffs, nil +} + +// TemporalRangePartitioningRetention generates a ALTER TABLE ... DROP PARTITION diff that drops all temporal partitions +// expired by given time. The function returns nil if no partitions are expired. The functions returns an error if +// all partitions were to be dropped, as this is highly unlikely to be the intended action. +func TemporalRangePartitioningRetention(createTableEntity *CreateTableEntity, expire time.Time) (diff *AlterTableEntityDiff, err error) { + analysis, err := AnalyzeTemporalRangePartitioning(createTableEntity) + if err != nil { + return nil, err + } + if !analysis.IsTemporalRangePartitioned { + return nil, errors.New(analysis.Reason) + } + expireDatetime := datetime.NewDateTimeFromStd(expire) + if err != nil { + return nil, err + } + alterTable := &sqlparser.AlterTable{ + Table: createTableEntity.Table, + PartitionSpec: &sqlparser.PartitionSpec{ + Action: sqlparser.DropAction, + }, + } + countValueRangePartitions := 0 + for _, partition := range createTableEntity.TableSpec.PartitionOption.Definitions { + if partition.Options == nil { + continue + } + if partition.Options.ValueRange == nil { + continue + } + if partition.Options.ValueRange.Maxvalue { + break + } + countValueRangePartitions++ + value, err := computeDateTime(partition.Options.ValueRange.Range[0], analysis.Col.Type(), analysis.FuncExpr) + if err != nil { + return nil, err + } + if value.Compare(expireDatetime) <= 0 { + alterTable.PartitionSpec.Names = append(alterTable.PartitionSpec.Names, partition.Name) + } + } + if len(alterTable.PartitionSpec.Names) == 0 { + return nil, nil + } + if len(alterTable.PartitionSpec.Names) == countValueRangePartitions { + // This would DROP all partitions, which is highly unlikely to be the intended action. + // We reject this operation. + return nil, fmt.Errorf("retention at %s would drop all partitions in table %s", expireDatetime.Format(0), createTableEntity.Name()) + } + diff = &AlterTableEntityDiff{alterTable: alterTable, from: createTableEntity} + return diff, nil +} diff --git a/go/vt/schemadiff/partitioning_analysis_test.go b/go/vt/schemadiff/partitioning_analysis_test.go new file mode 100644 index 00000000000..8d925c166b4 --- /dev/null +++ b/go/vt/schemadiff/partitioning_analysis_test.go @@ -0,0 +1,978 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schemadiff + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql/datetime" + "vitess.io/vitess/go/vt/sqlparser" +) + +func TestIsRangePartitioned(t *testing.T) { + tcases := []struct { + create string + expect bool + }{ + { + create: "CREATE TABLE t (id int PRIMARY KEY) PARTITION BY RANGE (id) (PARTITION p0 VALUES LESS THAN (10))", + expect: true, + }, + { + create: "CREATE TABLE t (id int PRIMARY KEY) PARTITION BY RANGE COLUMNS (id) (PARTITION p0 VALUES LESS THAN (10))", + expect: true, + }, + { + create: "CREATE TABLE t (id int PRIMARY KEY)", + }, + { + create: "CREATE TABLE t (id int PRIMARY KEY) PARTITION BY LIST (id) (PARTITION p0 VALUES IN (1))", + }, + } + env := NewTestEnv() + + for _, tcase := range tcases { + t.Run(tcase.create, func(t *testing.T) { + entity, err := NewCreateTableEntityFromSQL(env, tcase.create) + require.NoError(t, err) + + result := IsRangePartitioned(entity.CreateTable) + assert.Equal(t, tcase.expect, result) + }) + } +} + +// AnalyzePartitionRotation analyzes a given AlterTable statement to see whether it has partition rotation +// commands, and if so, is the ALTER TABLE statement valid in MySQL. In MySQL, a single ALTER TABLE statement +// cannot apply multiple rotation commands, nor can it mix rotation commands with other types of changes. +func TestAlterTableRotatesRangePartition(t *testing.T) { + tcases := []struct { + create string + alter string + expect bool + }{ + { + alter: "ALTER TABLE t ADD PARTITION (PARTITION p1 VALUES LESS THAN (10))", + expect: true, + }, + { + alter: "ALTER TABLE t DROP PARTITION p1", + expect: true, + }, + { + alter: "ALTER TABLE t DROP PARTITION p1, p2", + expect: true, + }, + { + alter: "ALTER TABLE t TRUNCATE PARTITION p3", + }, + { + alter: "ALTER TABLE t COALESCE PARTITION 3", + }, + { + alter: "ALTER TABLE t partition by range (id) (partition p1 values less than (10), partition p2 values less than (20), partition p3 values less than (30))", + }, + { + alter: "ALTER TABLE t ADD COLUMN c1 INT, DROP COLUMN c2", + }, + } + + for _, tcase := range tcases { + t.Run(tcase.alter, func(t *testing.T) { + if tcase.create == "" { + tcase.create = "CREATE TABLE t (id int PRIMARY KEY) PARTITION BY RANGE (id) (PARTITION p0 VALUES LESS THAN (10))" + } + stmt, err := sqlparser.NewTestParser().ParseStrictDDL(tcase.create) + require.NoError(t, err) + createTable, ok := stmt.(*sqlparser.CreateTable) + require.True(t, ok) + + stmt, err = sqlparser.NewTestParser().ParseStrictDDL(tcase.alter) + require.NoError(t, err) + alterTable, ok := stmt.(*sqlparser.AlterTable) + require.True(t, ok) + + result, err := AlterTableRotatesRangePartition(createTable, alterTable) + require.NoError(t, err) + assert.Equal(t, tcase.expect, result) + }) + } +} + +func TestTemporalFunctions(t *testing.T) { + tm, err := time.Parse(TimestampFormat, "2024-12-19 09:56:32") + require.NoError(t, err) + dt := datetime.NewDateTimeFromStd(tm) + { + // Compatible with `select YEAR('2024-12-19 09:56:32');`: + assert.EqualValues(t, 2024, dt.Date.Year()) + numDays := datetime.MysqlDayNumber(dt.Date.Year(), dt.Date.Month(), dt.Date.Day()) + // Compatible with `select to_days('2024-12-19 09:56:32');`: + assert.EqualValues(t, 739604, numDays) + // Compatible with `select to_seconds('2024-12-19 09:56:32');`: + assert.EqualValues(t, 63901821392, dt.ToSeconds()) + } + { + interval := datetime.ParseIntervalInt64(3, datetime.IntervalMonth, false) + // Should be `2025-03-19 09:56:32` + dt, _, ok := dt.AddInterval(interval, 0, false) + require.True(t, ok) + // Compatible with `select YEAR('2025-03-19 09:56:32');`: + assert.EqualValues(t, 2025, dt.Date.Year()) + assert.EqualValues(t, 3, dt.Date.Month()) + assert.EqualValues(t, 19, dt.Date.Day()) + + numDays := datetime.MysqlDayNumber(dt.Date.Year(), dt.Date.Month(), dt.Date.Day()) + // Compatible with `select to_days('2024-12-19 09:56:32' + INTERVAL 3 MONTH);`: + assert.EqualValues(t, 739694, numDays) + // Compatible with `select to_seconds('2024-12-19 09:56:32' + INTERVAL 3 MONTH);`: + assert.EqualValues(t, 63909597392, dt.ToSeconds()) + } +} + +func TestTruncateDateTime(t *testing.T) { + tm, err := time.Parse(TimestampFormat, "2024-12-19 09:56:32") + require.NoError(t, err) + dt := datetime.NewDateTimeFromStd(tm) + + tcases := []struct { + interval datetime.IntervalType + expect string + }{ + { + interval: datetime.IntervalYear, + expect: "2024-01-01 00:00:00", + }, + { + interval: datetime.IntervalMonth, + expect: "2024-12-01 00:00:00", + }, + { + interval: datetime.IntervalDay, + expect: "2024-12-19 00:00:00", + }, + { + interval: datetime.IntervalHour, + expect: "2024-12-19 09:00:00", + }, + { + // No truncating for this resolution + interval: datetime.IntervalMinute, + expect: "2024-12-19 09:56:32", + }, + { + // No truncating for this resolution + interval: datetime.IntervalSecond, + expect: "2024-12-19 09:56:32", + }, + } + for _, tcase := range tcases { + t.Run(tcase.interval.ToString(), func(t *testing.T) { + truncated, err := truncateDateTime(dt, tcase.interval) + require.NoError(t, err) + assert.Equal(t, tcase.expect, string(truncated.Format(0))) + }) + } +} + +func TestAnalyzeTemporalRangePartitioning(t *testing.T) { + tcases := []struct { + name string + create string + env *Environment + expect *TemporalRangePartitioningAnalysis + expectHighestValue string + expectErr error + }{ + { + name: "not partitioned", + create: "CREATE TABLE t (id int PRIMARY KEY)", + expect: &TemporalRangePartitioningAnalysis{ + Reason: "Table does not use PARTITION BY RANGE", + }, + }, + { + name: "unknown column", + create: "CREATE TABLE t (id int PRIMARY KEY) PARTITION BY RANGE (TO_DAYS(created_at)) (PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19 09:56:32')))", + // Error created by validate() + expectErr: &InvalidColumnInPartitionError{Table: "t", Column: "created_at"}, + }, + { + name: "partition by INT column", + create: "CREATE TABLE t (id int PRIMARY KEY) PARTITION BY RANGE (id) (PARTITION p0 VALUES LESS THAN (1))", + // Error created by validate() + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: false, + Reason: "column id of type int in table t is not a temporal type for temporal range partitioning", + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("id")}, + }, + }, + }, + { + name: "range by unknown column", + create: "CREATE TABLE t (id int, i INT, PRIMARY KEY (id, i)) PARTITION BY RANGE COLUMNS (i2) (PARTITION p0 VALUES LESS THAN (1))", + // Error created by validate() + expectErr: &InvalidColumnInPartitionError{Table: "t", Column: "i2"}, + }, + { + name: "range by TIME column", + create: "CREATE TABLE t (id int, tm TIME, PRIMARY KEY (id, tm)) PARTITION BY RANGE COLUMNS (tm) (PARTITION p0 VALUES LESS THAN (1))", + // Error created by AnalyzeTemporalRangePartitioning() + expectErr: &UnsupportedRangeColumnsTypeError{Table: "t", Column: "tm", Type: "time"}, + }, + { + name: "range by DATE column", + create: "CREATE TABLE t (id int, dt DATE, PRIMARY KEY (id, dt)) PARTITION BY RANGE COLUMNS (dt) (PARTITION p0 VALUES LESS THAN ('2025-01-01'))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + IsRangeColumns: true, + MinimalInterval: datetime.IntervalDay, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("dt")}, + }, + }, + expectHighestValue: "2025-01-01 00:00:00", + }, + { + name: "range by DATE column with MAXVALUE", + create: "CREATE TABLE t (id int, dt DATE, PRIMARY KEY (id, dt)) PARTITION BY RANGE COLUMNS (dt) (PARTITION p0 VALUES LESS THAN ('2025-01-01'), PARTITION pmax VALUES LESS THAN MAXVALUE)", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + IsRangeColumns: true, + MinimalInterval: datetime.IntervalDay, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("dt")}, + }, + MaxvaluePartition: &sqlparser.PartitionDefinition{Name: sqlparser.NewIdentifierCI("pmax")}, + }, + expectHighestValue: "2025-01-01 00:00:00", + }, + { + name: "range by date (lower case) column", + create: "CREATE TABLE t (id int, dt date, PRIMARY KEY (id, dt)) PARTITION BY RANGE COLUMNS (dt) (PARTITION p0 VALUES LESS THAN ('2025-01-01'))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + IsRangeColumns: true, + MinimalInterval: datetime.IntervalDay, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("dt")}, + }, + }, + expectHighestValue: "2025-01-01 00:00:00", + }, + { + name: "range by DATETIME column", + create: "CREATE TABLE t (id int, dt datetime, PRIMARY KEY (id, dt)) PARTITION BY RANGE COLUMNS (dt) (PARTITION p0 VALUES LESS THAN ('2025-01-01'))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + IsRangeColumns: true, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("dt")}, + }, + }, + expectHighestValue: "2025-01-01 00:00:00", + }, + { + name: "range by nonstandard named DATETIME column", + create: "CREATE TABLE t (id int, `d-t` datetime, PRIMARY KEY (id, `d-t`)) PARTITION BY RANGE (TO_DAYS(`d-t`)) (PARTITION p0 VALUES LESS THAN (TO_DAYS('2025-01-01')))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalDay, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("d-t")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("TO_DAYS"), + }, + }, + expectHighestValue: "2025-01-01 00:00:00", + }, + { + name: "range by DATETIME column, literal value", + create: "CREATE TABLE t (id int, dt datetime, PRIMARY KEY (id, dt)) PARTITION BY RANGE (TO_DAYS(dt)) (PARTITION p0 VALUES LESS THAN (739617))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalDay, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("dt")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("TO_DAYS"), + }, + }, + expectHighestValue: "2025-01-01 00:00:00", + }, + { + name: "range by TO_DAYS(TIMESTAMP)", + create: "CREATE TABLE t (id int, created_at TIMESTAMP, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (TO_DAYS(created_at)) (PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19 09:56:32')))", + expectErr: fmt.Errorf("column type timestamp is unsupported in temporal range partitioning analysis for column created_at in table t"), + }, + { + name: "range by TO_DAYS(DATETIME)", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (TO_DAYS(created_at)) (PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19 09:56:32')))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalDay, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("created_at")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("TO_DAYS"), + }, + }, + expectHighestValue: "2024-12-19 00:00:00", + }, + { + name: "range by TO_DAYS(DATETIME) with MAXVALUE", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (TO_DAYS(created_at)) (PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19 09:56:32')), PARTITION pmax VALUES LESS THAN MAXVALUE)", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalDay, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("created_at")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("TO_DAYS"), + }, + MaxvaluePartition: &sqlparser.PartitionDefinition{Name: sqlparser.NewIdentifierCI("pmax")}, + }, + expectHighestValue: "2024-12-19 00:00:00", + }, + { + name: "range by to_days(DATETIME) (lower case)", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (to_days(created_at)) (PARTITION p0 VALUES LESS THAN (to_days('2024-12-19 09:56:32')))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalDay, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("created_at")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("to_days"), + }, + }, + expectHighestValue: "2024-12-19 00:00:00", + }, + { + name: "range by to_seconds(DATETIME)", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (to_seconds(created_at)) (PARTITION p0 VALUES LESS THAN (to_seconds('2024-12-19 09:56:32')))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalSecond, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("created_at")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("to_seconds"), + }, + }, + expectHighestValue: "2024-12-19 09:56:32", + }, { + name: "range by year(DATETIME)", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (year(created_at)) (PARTITION p0 VALUES LESS THAN (year('2024-12-19 09:56:32')))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalYear, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("created_at")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("year"), + }, + }, + expectHighestValue: "2024-01-01 00:00:00", + }, + { + name: "unsupported function expression", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (ABS(created_at)) (PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19 09:56:32')))", + expectErr: fmt.Errorf("expression: ABS(`created_at`) is unsupported in temporal range partitioning analysis in table t"), + }, + { + name: "range by compound expression to_days(DATETIME)+1", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (to_days(created_at)+1) (PARTITION p0 VALUES LESS THAN (to_days('2024-12-19 09:56:32')+1))", + expectErr: fmt.Errorf("expression: to_days(`created_at`) + 1 is unsupported in temporal range partitioning analysis in table t"), + }, + { + name: "range by UNIX_TIMESTAMP(TIMESTAMP)", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (UNIX_TIMESTAMP(created_at)) (PARTITION p0 VALUES LESS THAN (UNIX_TIMESTAMP('2024-12-19 09:56:32')))", + expectErr: fmt.Errorf("expression: UNIX_TIMESTAMP(`created_at`) is unsupported in temporal range partitioning analysis in table t"), + }, + { + name: "range by UNIX_TIMESTAMP(TIMESTAMP), 8.4", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (UNIX_TIMESTAMP(created_at)) (PARTITION p0 VALUES LESS THAN (UNIX_TIMESTAMP('2024-12-19 09:56:32')))", + env: New84TestEnv(), + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalSecond, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("created_at")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("UNIX_TIMESTAMP"), + }, + }, + expectHighestValue: "2024-12-19 09:56:32", + }, + } + for _, tcase := range tcases { + t.Run(tcase.name, func(t *testing.T) { + env := NewTestEnv() + if tcase.env != nil { + env = tcase.env + } + + entity, err := NewCreateTableEntityFromSQL(env, tcase.create) + require.NoError(t, err) + + result, err := AnalyzeTemporalRangePartitioning(entity) + if tcase.expectErr != nil { + require.Error(t, err) + assert.EqualError(t, err, tcase.expectErr.Error()) + return + } + assert.NoError(t, result.Error) + require.NotNil(t, tcase.expect) + assert.Equal(t, tcase.expect.Reason, result.Reason) + require.NotNil(t, result) + + if tcase.expect.IsTemporalRangePartitioned { + require.NotEmpty(t, tcase.expectHighestValue) + dt, _, ok := datetime.ParseDateTime(tcase.expectHighestValue, -1) + require.True(t, ok) + tcase.expect.HighestValue = dt + } + + assert.Equal(t, tcase.expect.IsRangePartitioned, result.IsRangePartitioned, "IsRangePartitioned") + assert.Equal(t, tcase.expect.IsTemporalRangePartitioned, result.IsTemporalRangePartitioned, "IsTemporalRangePartitioned") + assert.Equal(t, tcase.expect.IsRangeColumns, result.IsRangeColumns, "IsRangeColumns") + assert.Equal(t, tcase.expect.MinimalInterval, result.MinimalInterval, "MinimalInterval") + if tcase.expect.Col != nil { + require.NotNil(t, result.Col, "column") + assert.Equal(t, tcase.expect.Col.Name(), result.Col.Name()) + } else { + assert.Nil(t, result.Col, "column") + } + if tcase.expect.FuncExpr != nil { + require.NotNil(t, result.FuncExpr) + assert.Equal(t, tcase.expect.FuncExpr.Name.String(), result.FuncExpr.Name.String()) + } else { + assert.Nil(t, result.FuncExpr, "funcExpr") + } + if tcase.expect.MaxvaluePartition != nil { + require.NotNil(t, result.MaxvaluePartition) + assert.Equal(t, tcase.expect.MaxvaluePartition.Name.String(), result.MaxvaluePartition.Name.String()) + } else { + assert.Nil(t, result.MaxvaluePartition, "maxvaluePartition") + } + assert.Equal(t, tcase.expect.HighestValue, result.HighestValue) + }) + } +} + +func TestTemporalRangePartitioningNextRotation(t *testing.T) { + tcases := []struct { + name string + create string + interval datetime.IntervalType + prepareAheadCount int + expactMaxValue bool + expectStatements []string + expectErr error + }{ + { + name: "not partitioned", + create: "CREATE TABLE t (id int)", + interval: datetime.IntervalHour, + prepareAheadCount: 7, + expectErr: fmt.Errorf("Table does not use PARTITION BY RANGE"), + }, + { + name: "interval too short", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (TO_DAYS(created_at)) (PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19 09:56:32')))", + interval: datetime.IntervalHour, + prepareAheadCount: 7, + expectErr: fmt.Errorf("interval hour is less than the minimal interval day for table t"), + }, + { + name: "day interval with 7 days, DATE", + create: "CREATE TABLE t (id int, created_at DATE, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (TO_DAYS(created_at)) (PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19')))", + interval: datetime.IntervalDay, + prepareAheadCount: 7, + expectStatements: []string{ + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241219` VALUES LESS THAN (739605))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241220` VALUES LESS THAN (739606))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241221` VALUES LESS THAN (739607))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241222` VALUES LESS THAN (739608))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241223` VALUES LESS THAN (739609))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241224` VALUES LESS THAN (739610))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241225` VALUES LESS THAN (739611))", + }, + }, + { + name: "day interval with 7 days, DATETIME", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (TO_DAYS(created_at)) (PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19 09:56:32')))", + interval: datetime.IntervalDay, + prepareAheadCount: 7, + expectStatements: []string{ + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241219` VALUES LESS THAN (739605))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241220` VALUES LESS THAN (739606))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241221` VALUES LESS THAN (739607))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241222` VALUES LESS THAN (739608))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241223` VALUES LESS THAN (739609))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241224` VALUES LESS THAN (739610))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241225` VALUES LESS THAN (739611))", + }, + }, + { + name: "day interval with 7 days, DATETIME, 2 days covered", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) + PARTITION BY RANGE (TO_DAYS(created_at)) + ( + PARTITION p0 VALUES LESS THAN (739604), + PARTITION p20241219 VALUES LESS THAN (739605), + PARTITION p20241220 VALUES LESS THAN (739606) + )`, + interval: datetime.IntervalDay, + prepareAheadCount: 7, + expectStatements: []string{ + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241221` VALUES LESS THAN (739607))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241222` VALUES LESS THAN (739608))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241223` VALUES LESS THAN (739609))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241224` VALUES LESS THAN (739610))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241225` VALUES LESS THAN (739611))", + }, + }, + { + name: "range columns over datetime, day interval with 7 days", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) (PARTITION p0 VALUES LESS THAN ('2024-12-19 00:00:00'))", + interval: datetime.IntervalDay, + prepareAheadCount: 7, + expectStatements: []string{ + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241219` VALUES LESS THAN ('2024-12-20 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241220` VALUES LESS THAN ('2024-12-21 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241221` VALUES LESS THAN ('2024-12-22 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241222` VALUES LESS THAN ('2024-12-23 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241223` VALUES LESS THAN ('2024-12-24 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241224` VALUES LESS THAN ('2024-12-25 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241225` VALUES LESS THAN ('2024-12-26 00:00:00'))", + }, + }, + { + name: "range columns over datetime, day interval with 7 days and MAXVALUE", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) (PARTITION p0 VALUES LESS THAN ('2024-12-19 00:00:00'), PARTITION pmax VALUES LESS THAN MAXVALUE)", + interval: datetime.IntervalDay, + prepareAheadCount: 7, + expactMaxValue: true, + expectStatements: []string{ + "ALTER TABLE `t` REORGANIZE PARTITION `pmax` INTO (PARTITION `p20241219` VALUES LESS THAN ('2024-12-20 00:00:00'), PARTITION `pmax` VALUES LESS THAN MAXVALUE)", + "ALTER TABLE `t` REORGANIZE PARTITION `pmax` INTO (PARTITION `p20241220` VALUES LESS THAN ('2024-12-21 00:00:00'), PARTITION `pmax` VALUES LESS THAN MAXVALUE)", + "ALTER TABLE `t` REORGANIZE PARTITION `pmax` INTO (PARTITION `p20241221` VALUES LESS THAN ('2024-12-22 00:00:00'), PARTITION `pmax` VALUES LESS THAN MAXVALUE)", + "ALTER TABLE `t` REORGANIZE PARTITION `pmax` INTO (PARTITION `p20241222` VALUES LESS THAN ('2024-12-23 00:00:00'), PARTITION `pmax` VALUES LESS THAN MAXVALUE)", + "ALTER TABLE `t` REORGANIZE PARTITION `pmax` INTO (PARTITION `p20241223` VALUES LESS THAN ('2024-12-24 00:00:00'), PARTITION `pmax` VALUES LESS THAN MAXVALUE)", + "ALTER TABLE `t` REORGANIZE PARTITION `pmax` INTO (PARTITION `p20241224` VALUES LESS THAN ('2024-12-25 00:00:00'), PARTITION `pmax` VALUES LESS THAN MAXVALUE)", + "ALTER TABLE `t` REORGANIZE PARTITION `pmax` INTO (PARTITION `p20241225` VALUES LESS THAN ('2024-12-26 00:00:00'), PARTITION `pmax` VALUES LESS THAN MAXVALUE)", + }, + }, + { + name: "range columns over datetime, day interval with 7 days and MAXVALUE, 2 days covered", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) + ( + PARTITION p0 VALUES LESS THAN ('2024-12-19 00:00:00'), + PARTITION p20241220 VALUES LESS THAN ('2024-12-20 00:00:00'), + PARTITION p_anyname VALUES LESS THAN ('2024-12-21 00:00:00'), + PARTITION pmax VALUES LESS THAN MAXVALUE + )`, + interval: datetime.IntervalDay, + prepareAheadCount: 7, + expactMaxValue: true, + expectStatements: []string{ + "ALTER TABLE `t` REORGANIZE PARTITION `pmax` INTO (PARTITION `p20241221` VALUES LESS THAN ('2024-12-22 00:00:00'), PARTITION `pmax` VALUES LESS THAN MAXVALUE)", + "ALTER TABLE `t` REORGANIZE PARTITION `pmax` INTO (PARTITION `p20241222` VALUES LESS THAN ('2024-12-23 00:00:00'), PARTITION `pmax` VALUES LESS THAN MAXVALUE)", + "ALTER TABLE `t` REORGANIZE PARTITION `pmax` INTO (PARTITION `p20241223` VALUES LESS THAN ('2024-12-24 00:00:00'), PARTITION `pmax` VALUES LESS THAN MAXVALUE)", + "ALTER TABLE `t` REORGANIZE PARTITION `pmax` INTO (PARTITION `p20241224` VALUES LESS THAN ('2024-12-25 00:00:00'), PARTITION `pmax` VALUES LESS THAN MAXVALUE)", + "ALTER TABLE `t` REORGANIZE PARTITION `pmax` INTO (PARTITION `p20241225` VALUES LESS THAN ('2024-12-26 00:00:00'), PARTITION `pmax` VALUES LESS THAN MAXVALUE)", + }, + }, + { + name: "hour interval with 4 hours", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) (PARTITION p0 VALUES LESS THAN ('2024-12-19 09:00:00'))", + interval: datetime.IntervalHour, + prepareAheadCount: 4, + expectStatements: []string{ + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241219090000` VALUES LESS THAN ('2024-12-19 10:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241219100000` VALUES LESS THAN ('2024-12-19 11:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241219110000` VALUES LESS THAN ('2024-12-19 12:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241219120000` VALUES LESS THAN ('2024-12-19 13:00:00'))", + }, + }, + { + name: "hour interval with 4 hours, 2 of which are covered", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) (PARTITION p0 VALUES LESS THAN ('2024-12-19 09:00:00'), PARTITION p20241219100000 VALUES LESS THAN ('2024-12-19 10:00:00'), PARTITION p20241219110000 VALUES LESS THAN ('2024-12-19 11:00:00'))", + interval: datetime.IntervalHour, + prepareAheadCount: 4, + expectStatements: []string{ + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241219110000` VALUES LESS THAN ('2024-12-19 12:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241219120000` VALUES LESS THAN ('2024-12-19 13:00:00'))", + }, + }, + { + name: "month interval with 3 months", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) (PARTITION p0 VALUES LESS THAN ('2024-12-19 09:00:00'))", + interval: datetime.IntervalMonth, + prepareAheadCount: 3, + expectStatements: []string{ + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241201` VALUES LESS THAN ('2025-01-01 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20250101` VALUES LESS THAN ('2025-02-01 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20250201` VALUES LESS THAN ('2025-03-01 00:00:00'))", + }, + }, + { + name: "month interval with 3 months, 1 covered", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) (PARTITION p0 VALUES LESS THAN ('2024-12-19 09:00:00'), PARTITION p20250101 VALUES LESS THAN ('2025-01-01 00:00:00'))", + interval: datetime.IntervalMonth, + prepareAheadCount: 3, + expectStatements: []string{ + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20250101` VALUES LESS THAN ('2025-02-01 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20250201` VALUES LESS THAN ('2025-03-01 00:00:00'))", + }, + }, { + name: "month interval with 3 months, all covered", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) + ( + PARTITION p0 VALUES LESS THAN ('2024-12-19 09:00:00'), + PARTITION p20250101 VALUES LESS THAN ('2025-01-01 00:00:00'), + PARTITION p20250201 VALUES LESS THAN ('2025-02-01 00:00:00'), + PARTITION p20250301 VALUES LESS THAN ('2025-03-01 00:00:00'), + PARTITION p20250401 VALUES LESS THAN ('2025-04-01 00:00:00') + )`, + interval: datetime.IntervalMonth, + prepareAheadCount: 3, + expectStatements: []string{}, + }, + { + name: "year interval with 3 years", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) (PARTITION p0 VALUES LESS THAN ('2024-12-19 09:00:00'))", + interval: datetime.IntervalYear, + prepareAheadCount: 3, + expectStatements: []string{ + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20240101` VALUES LESS THAN ('2025-01-01 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20250101` VALUES LESS THAN ('2026-01-01 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20260101` VALUES LESS THAN ('2027-01-01 00:00:00'))", + }, + }, + { + name: "partition by INT column", + create: "CREATE TABLE t (id int PRIMARY KEY) PARTITION BY RANGE (id) (PARTITION p0 VALUES LESS THAN (1))", + interval: datetime.IntervalDay, + prepareAheadCount: 3, + expectErr: fmt.Errorf("column id of type int in table t is not a temporal type for temporal range partitioning"), + }, + } + reference, err := time.Parse(TimestampFormat, "2024-12-19 09:56:32") + require.NoError(t, err) + env := NewTestEnv() + for _, tcase := range tcases { + t.Run(tcase.name, func(t *testing.T) { + entity, err := NewCreateTableEntityFromSQL(env, tcase.create) + require.NoError(t, err) + + diffs, err := TemporalRangePartitioningNextRotation(entity, tcase.interval, tcase.prepareAheadCount, reference) + if tcase.expectErr != nil { + require.Error(t, err) + assert.EqualError(t, err, tcase.expectErr.Error()) + return + } + require.NoError(t, err) + + require.Len(t, diffs, len(tcase.expectStatements)) + for i, diff := range diffs { + if tcase.expactMaxValue { + assert.Len(t, diff.alterTable.PartitionSpec.Definitions, 2) + } else { + assert.Len(t, diff.alterTable.PartitionSpec.Definitions, 1) + } + assert.Equal(t, tcase.expectStatements[i], diff.CanonicalStatementString()) + } + }) + } +} + +func TestTemporalRangePartitioningRetention(t *testing.T) { + tcases := []struct { + name string + create string + expire string + expectStatement string + expectErr error + }{ + { + name: "not partitioned", + create: "CREATE TABLE t (id int)", + expire: "2024-12-19 09:00:00", + expectErr: fmt.Errorf("Table does not use PARTITION BY RANGE"), + }, + { + name: "day interval, no impact", + create: "CREATE TABLE t (id int, created_at DATE, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (TO_DAYS(created_at)) (PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19')))", + expire: "2024-12-07 09:00:00", + }, + { + name: "day interval, all partitions impacted", + create: "CREATE TABLE t (id int, created_at DATE, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (TO_DAYS(created_at)) (PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19')))", + expire: "2024-12-19 09:00:00", + expectErr: fmt.Errorf("retention at 2024-12-19 09:00:00 would drop all partitions in table t"), + }, + { + name: "day interval with MAXVALUE, no impact", + create: "CREATE TABLE t (id int, created_at DATE, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (TO_DAYS(created_at)) (PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19')), PARTITION pmax VALUES LESS THAN MAXVALUE)", + expire: "2024-12-07 09:00:00", + }, + { + name: "day interval with MAXVALUE, all partitions impacted", + create: "CREATE TABLE t (id int, created_at DATE, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (TO_DAYS(created_at)) (PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19')), PARTITION pmax VALUES LESS THAN MAXVALUE)", + expire: "2024-12-19 09:00:00", + expectErr: fmt.Errorf("retention at 2024-12-19 09:00:00 would drop all partitions in table t"), + }, + { + name: "range columns over datetime, day interval with 7 days and MAXVALUE, no impact", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) + ( + PARTITION p0 VALUES LESS THAN ('2024-12-19 00:00:00'), + PARTITION p20241220 VALUES LESS THAN ('2024-12-20 00:00:00'), + PARTITION p_anyname VALUES LESS THAN ('2024-12-21 00:00:00'), + PARTITION pmax VALUES LESS THAN MAXVALUE + )`, + expire: "2024-12-18 09:00:00", + }, + { + name: "range columns over datetime, day interval with 7 days and MAXVALUE, single partition dropped", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) + ( + PARTITION p0 VALUES LESS THAN ('2024-12-19 00:00:00'), + PARTITION p20241220 VALUES LESS THAN ('2024-12-20 00:00:00'), + PARTITION p_anyname VALUES LESS THAN ('2024-12-21 00:00:00'), + PARTITION pmax VALUES LESS THAN MAXVALUE + )`, + expire: "2024-12-19 00:00:00", + expectStatement: "ALTER TABLE `t` DROP PARTITION `p0`", + }, + { + name: "range columns over datetime, day interval with 7 days and MAXVALUE, single partition dropped, passed threshold", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) + ( + PARTITION p0 VALUES LESS THAN ('2024-12-19 00:00:00'), + PARTITION p20241220 VALUES LESS THAN ('2024-12-20 00:00:00'), + PARTITION p_anyname VALUES LESS THAN ('2024-12-21 00:00:00'), + PARTITION pmax VALUES LESS THAN MAXVALUE + )`, + expire: "2024-12-19 01:02:03", + expectStatement: "ALTER TABLE `t` DROP PARTITION `p0`", + }, + { + name: "range columns over datetime, day interval with 7 days and MAXVALUE, two partitions dropped", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) + ( + PARTITION p0 VALUES LESS THAN ('2024-12-19 00:00:00'), + PARTITION p20241220 VALUES LESS THAN ('2024-12-20 00:00:00'), + PARTITION p_anyname VALUES LESS THAN ('2024-12-21 00:00:00'), + PARTITION pmax VALUES LESS THAN MAXVALUE + )`, + expire: "2024-12-20 00:00:00", + expectStatement: "ALTER TABLE `t` DROP PARTITION `p0`, `p20241220`", + }, + { + name: "range columns over datetime, day interval with 7 days and MAXVALUE, two partitions dropped, passed threshold", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) + ( + PARTITION p0 VALUES LESS THAN ('2024-12-19 00:00:00'), + PARTITION p20241220 VALUES LESS THAN ('2024-12-20 00:00:00'), + PARTITION p_anyname VALUES LESS THAN ('2024-12-21 00:00:00'), + PARTITION pmax VALUES LESS THAN MAXVALUE + )`, + expire: "2024-12-20 23:59:59", + expectStatement: "ALTER TABLE `t` DROP PARTITION `p0`, `p20241220`", + }, + { + name: "range columns over datetime, day interval with 7 days and MAXVALUE, error dropping all partitions", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) + ( + PARTITION p0 VALUES LESS THAN ('2024-12-19 00:00:00'), + PARTITION p20241220 VALUES LESS THAN ('2024-12-20 00:00:00'), + PARTITION p_anyname VALUES LESS THAN ('2024-12-21 00:00:00'), + PARTITION pmax VALUES LESS THAN MAXVALUE + )`, + expire: "2024-12-21 00:00:00", + expectErr: fmt.Errorf("retention at 2024-12-21 00:00:00 would drop all partitions in table t"), + }, + { + name: "range columns over datetime, day interval with 7 days and MAXVALUE, error dropping all partitions, futuristic", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) + ( + PARTITION p0 VALUES LESS THAN ('2024-12-19 00:00:00'), + PARTITION p20241220 VALUES LESS THAN ('2024-12-20 00:00:00'), + PARTITION p_anyname VALUES LESS THAN ('2024-12-21 00:00:00'), + PARTITION pmax VALUES LESS THAN MAXVALUE + )`, + expire: "2025-01-01 00:00:00", + expectErr: fmt.Errorf("retention at 2025-01-01 00:00:00 would drop all partitions in table t"), + }, + { + name: "day interval using TO_DAYS, DATETIME, no impact", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) + PARTITION BY RANGE (TO_DAYS(created_at)) + ( + PARTITION p0 VALUES LESS THAN (739604), + PARTITION p20241219 VALUES LESS THAN (739605), + PARTITION p20241220 VALUES LESS THAN (739606) + )`, + expire: "2024-12-18 00:00:00", + }, + { + name: "day interval using TO_DAYS, DATETIME, drop 1 partition", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) + PARTITION BY RANGE (TO_DAYS(created_at)) + ( + PARTITION p0 VALUES LESS THAN (739604), + PARTITION p20241219 VALUES LESS THAN (739605), + PARTITION p20241220 VALUES LESS THAN (739606) + )`, + expire: "2024-12-19 00:00:00", + expectStatement: "ALTER TABLE `t` DROP PARTITION `p0`", + }, + { + name: "day interval using TO_DAYS, DATETIME, drop 2 partitions", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) + PARTITION BY RANGE (TO_DAYS(created_at)) + ( + PARTITION p0 VALUES LESS THAN (739604), + PARTITION p20241219 VALUES LESS THAN (739605), + PARTITION p20241220 VALUES LESS THAN (739606) + )`, + expire: "2024-12-20 00:00:00", + expectStatement: "ALTER TABLE `t` DROP PARTITION `p0`, `p20241219`", + }, + { + name: "day interval using TO_DAYS, DATETIME, error dropping all partitions", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) + PARTITION BY RANGE (TO_DAYS(created_at)) + ( + PARTITION p0 VALUES LESS THAN (739604), + PARTITION p20241219 VALUES LESS THAN (739605), + PARTITION p20241220 VALUES LESS THAN (739606) + )`, + expire: "2024-12-21 00:00:00", + expectErr: fmt.Errorf("retention at 2024-12-21 00:00:00 would drop all partitions in table t"), + }, + { + name: "day interval using TO_DAYS in expression, DATETIME, no impact", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) + PARTITION BY RANGE (TO_DAYS(created_at)) + ( + PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19 00:00:00')), + PARTITION p20241219 VALUES LESS THAN (TO_DAYS('2024-12-20 00:00:00')), + PARTITION p20241220 VALUES LESS THAN (TO_DAYS('2024-12-21 00:00:00')) + )`, + expire: "2024-12-18 00:00:00", + }, + { + name: "day interval using TO_DAYS in expression, DATETIME, drop 1 partition", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) + PARTITION BY RANGE (TO_DAYS(created_at)) + ( + PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19 00:00:00')), + PARTITION p20241219 VALUES LESS THAN (TO_DAYS('2024-12-20 00:00:00')), + PARTITION p20241220 VALUES LESS THAN (TO_DAYS('2024-12-21 00:00:00')) + )`, + expire: "2024-12-19 00:00:00", + expectStatement: "ALTER TABLE `t` DROP PARTITION `p0`", + }, + { + name: "day interval using TO_DAYS in expression, DATETIME, drop 2 partitions", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) + PARTITION BY RANGE (TO_DAYS(created_at)) + ( + PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19 00:00:00')), + PARTITION p20241219 VALUES LESS THAN (TO_DAYS('2024-12-20 00:00:00')), + PARTITION p20241220 VALUES LESS THAN (TO_DAYS('2024-12-21 00:00:00')) + )`, + expire: "2024-12-20 00:00:00", + expectStatement: "ALTER TABLE `t` DROP PARTITION `p0`, `p20241219`", + }, + { + name: "day interval using TO_DAYS in expression, DATETIME, error dropping all partitions", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) + PARTITION BY RANGE (TO_DAYS(created_at)) + ( + PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19 00:00:00')), + PARTITION p20241219 VALUES LESS THAN (TO_DAYS('2024-12-20 00:00:00')), + PARTITION p20241220 VALUES LESS THAN (TO_DAYS('2024-12-21 00:00:00')) + )`, + expire: "2024-12-21 00:00:00", + expectErr: fmt.Errorf("retention at 2024-12-21 00:00:00 would drop all partitions in table t"), + }, + { + name: "partition by INT column", + create: "CREATE TABLE t (id int PRIMARY KEY) PARTITION BY RANGE (id) (PARTITION p0 VALUES LESS THAN (1))", + expire: "2024-12-21 00:00:00", + expectErr: fmt.Errorf("column id of type int in table t is not a temporal type for temporal range partitioning"), + }, + } + env := NewTestEnv() + for _, tcase := range tcases { + t.Run(tcase.name, func(t *testing.T) { + expire, err := time.Parse(TimestampFormat, tcase.expire) + require.NoError(t, err) + + entity, err := NewCreateTableEntityFromSQL(env, tcase.create) + require.NoError(t, err) + + diff, err := TemporalRangePartitioningRetention(entity, expire) + if tcase.expectErr != nil { + require.Error(t, err) + assert.EqualError(t, err, tcase.expectErr.Error()) + return + } + require.NoError(t, err) + if tcase.expectStatement == "" { + assert.Nil(t, diff) + } else { + require.NotNil(t, diff) + assert.Equal(t, tcase.expectStatement, diff.CanonicalStatementString()) + } + }) + } + +} diff --git a/go/vt/schemadiff/schema.go b/go/vt/schemadiff/schema.go index 3b42d6cf42d..7dad618507a 100644 --- a/go/vt/schemadiff/schema.go +++ b/go/vt/schemadiff/schema.go @@ -734,7 +734,7 @@ func (s *Schema) apply(diffs []EntityDiff, hints *DiffHints) error { if _, ok := s.named[name]; ok { return &ApplyDuplicateEntityError{Entity: name} } - s.tables = append(s.tables, &CreateTableEntity{CreateTable: diff.createTable}) + s.tables = append(s.tables, &CreateTableEntity{CreateTable: diff.createTable, Env: s.env}) _, s.named[name] = diff.Entities() case *CreateViewEntityDiff: // We expect the view to not exist @@ -742,7 +742,7 @@ func (s *Schema) apply(diffs []EntityDiff, hints *DiffHints) error { if _, ok := s.named[name]; ok { return &ApplyDuplicateEntityError{Entity: name} } - s.views = append(s.views, &CreateViewEntity{CreateView: diff.createView}) + s.views = append(s.views, &CreateViewEntity{CreateView: diff.createView, env: s.env}) _, s.named[name] = diff.Entities() case *DropTableEntityDiff: // We expect the table to exist diff --git a/go/vt/schemadiff/schema_test.go b/go/vt/schemadiff/schema_test.go index 8a4f54269cd..5d5b0c459ab 100644 --- a/go/vt/schemadiff/schema_test.go +++ b/go/vt/schemadiff/schema_test.go @@ -522,6 +522,29 @@ func TestInvalidSchema(t *testing.T) { { schema: "create table post (id varchar(191) charset utf8mb4 not null, `title` text, primary key (`id`)); create table post_fks (id varchar(191) not null, `post_id` varchar(191) collate utf8mb4_0900_ai_ci, primary key (id), constraint post_fk foreign key (post_id) references post (id)) charset utf8mb4, collate utf8mb4_0900_as_ci;", }, + // Partitioning: + { + // RANGE COLUMNS uses valid column + schema: "CREATE TABLE t (id int, dt DATE, PRIMARY KEY (id, dt)) PARTITION BY RANGE COLUMNS (dt) (PARTITION p0 VALUES LESS THAN (1))", + }, + { + // RANGE COLUMNS uses valid column + schema: "CREATE TABLE t (id int, i BIGINT, PRIMARY KEY (id, i)) PARTITION BY RANGE COLUMNS (i) (PARTITION p0 VALUES LESS THAN (1))", + }, + { + // RANGE COLUMNS uses valid column + schema: "CREATE TABLE t (id int, v VARCHAR(100), PRIMARY KEY (id, v)) PARTITION BY RANGE COLUMNS (v) (PARTITION p0 VALUES LESS THAN (1))", + }, + { + // RANGE COLUMNS uses non-existent column + schema: "CREATE TABLE t (id int, i INT, PRIMARY KEY (id, i)) PARTITION BY RANGE COLUMNS (i2) (PARTITION p0 VALUES LESS THAN (1))", + expectErr: &InvalidColumnInPartitionError{Table: "t", Column: "i2"}, + }, + { + // RANGE COLUMNS uses unsupported column type + schema: "CREATE TABLE t (id int, tm TIME, PRIMARY KEY (id, tm)) PARTITION BY RANGE COLUMNS (tm) (PARTITION p0 VALUES LESS THAN (1))", + expectErr: &UnsupportedRangeColumnsTypeError{Table: "t", Column: "tm", Type: "time"}, + }, } for _, ts := range tt { t.Run(ts.schema, func(t *testing.T) { diff --git a/go/vt/schemadiff/table.go b/go/vt/schemadiff/table.go index e002ef18e15..165dc4c5c17 100644 --- a/go/vt/schemadiff/table.go +++ b/go/vt/schemadiff/table.go @@ -2686,13 +2686,14 @@ func (c *CreateTableEntity) validateDuplicateKeyNameError() error { // validate checks that the table structure is valid: // - all columns referenced by keys exist func (c *CreateTableEntity) validate() error { - columnExists := map[string]bool{} - for _, col := range c.CreateTable.TableSpec.Columns { - colName := col.Name.Lowered() - if columnExists[colName] { - return &ApplyDuplicateColumnError{Table: c.Name(), Column: col.Name.String()} + entities := c.ColumnDefinitionEntities() + columnsMap := make(map[string]*ColumnDefinitionEntity, len(entities)) + for _, entity := range entities { + lowered := entity.NameLowered() + if _, ok := columnsMap[lowered]; ok { + return &ApplyDuplicateColumnError{Table: c.Name(), Column: entity.Name()} } - columnExists[colName] = true + columnsMap[lowered] = entity } // validate all columns used by foreign key constraints do in fact exist, // and that there exists an index over those columns @@ -2705,7 +2706,7 @@ func (c *CreateTableEntity) validate() error { return &ForeignKeyColumnCountMismatchError{Table: c.Name(), Constraint: cs.Name.String(), ColumnCount: len(fk.Source), ReferencedTable: fk.ReferenceDefinition.ReferencedTable.Name.String(), ReferencedColumnCount: len(fk.ReferenceDefinition.ReferencedColumns)} } for _, col := range fk.Source { - if !columnExists[col.Lowered()] { + if _, ok := columnsMap[col.Lowered()]; !ok { return &InvalidColumnInForeignKeyConstraintError{Table: c.Name(), Constraint: cs.Name.String(), Column: col.String()} } } @@ -2713,7 +2714,7 @@ func (c *CreateTableEntity) validate() error { // validate all columns referenced by indexes do in fact exist for _, key := range c.CreateTable.TableSpec.Indexes { for colName := range getKeyColumnNames(key) { - if !columnExists[colName] { + if _, ok := columnsMap[colName]; !ok { return &InvalidColumnInKeyError{Table: c.Name(), Column: colName, Key: key.Info.Name.String()} } } @@ -2733,7 +2734,7 @@ func (c *CreateTableEntity) validate() error { return err } for _, referencedColName := range referencedColumns { - if !columnExists[strings.ToLower(referencedColName)] { + if _, ok := columnsMap[strings.ToLower(referencedColName)]; !ok { return &InvalidColumnInGeneratedColumnError{Table: c.Name(), Column: referencedColName, GeneratedColumn: col.Name.String()} } } @@ -2754,7 +2755,7 @@ func (c *CreateTableEntity) validate() error { return err } for _, referencedColName := range referencedColumns { - if !columnExists[strings.ToLower(referencedColName)] { + if _, ok := columnsMap[strings.ToLower(referencedColName)]; !ok { return &InvalidColumnInKeyError{Table: c.Name(), Column: referencedColName, Key: idx.Info.Name.String()} } } @@ -2778,11 +2779,12 @@ func (c *CreateTableEntity) validate() error { return err } for _, referencedColName := range referencedColumns { - if !columnExists[strings.ToLower(referencedColName)] { + if _, ok := columnsMap[strings.ToLower(referencedColName)]; !ok { return &InvalidColumnInCheckConstraintError{Table: c.Name(), Constraint: cs.Name.String(), Column: referencedColName} } } } + // validate no two keys have same name if err := c.validateDuplicateKeyNameError(); err != nil { return err @@ -2810,10 +2812,27 @@ func (c *CreateTableEntity) validate() error { if err != nil { return err } + for _, colName := range partition.ColList { + // PARTITION BY RANGE COLUMNS + col, ok := columnsMap[strings.ToLower(colName.String())] + if !ok { + return &InvalidColumnInPartitionError{Table: c.Name(), Column: colName.String()} + } + // Validate column type + // See https://dev.mysql.com/doc/refman/8.0/en/partitioning-columns.html + if !IsIntegralType(col.Type()) { + switch strings.ToLower(col.Type()) { + case "date", "datetime": + case "char", "varchar", "binary", "varbinary": + default: + return &UnsupportedRangeColumnsTypeError{Table: c.Name(), Column: colName.String(), Type: col.Type()} + } + } + } for _, partitionColName := range partitionColNames { // Validate columns exists in table: - if !columnExists[strings.ToLower(partitionColName)] { + if _, ok := columnsMap[strings.ToLower(partitionColName)]; !ok { return &InvalidColumnInPartitionError{Table: c.Name(), Column: partitionColName} }