From f7291d8779dabc7d8b383edc0136cc3f421a76b3 Mon Sep 17 00:00:00 2001 From: Leszek Zalewski Date: Wed, 14 Oct 2026 22:19:44 +0200 Subject: [PATCH 1/3] Add repro for rows at or below the minimum pagination key The first batch selects pagination keys strictly greater than the minimum (0 for integers, empty for binary keys), so rows keyed 0 or '' are never copied, a table holding only key 0 is skipped, and rows with negative keys are silently dropped. --- sharding/test/copy_filter_test.go | 18 ++++++++++ test/go/data_iterator_test.go | 59 +++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/sharding/test/copy_filter_test.go b/sharding/test/copy_filter_test.go index dab0dfed4..94ec09b78 100644 --- a/sharding/test/copy_filter_test.go +++ b/sharding/test/copy_filter_test.go @@ -285,6 +285,24 @@ func (t *CopyFilterTestSuite) TestSelectsPrimaryKeyTables() { t.Require().Equal([]interface{}{t.shardingValue, t.paginationKeyCursor.SQLValue()}, args) } +func (t *CopyFilterTestSuite) TestIncludesMinimumPaginationKey() { + minKey := ghostferry.NewUint64Key(0) + expected := map[*ghostferry.TableSchema]string{ + t.normalTable: "SELECT * FROM `shard_1`.`normaltable` JOIN (SELECT `id` FROM `shard_1`.`normaltable` USE INDEX (`good_sharding_index`) WHERE `tenant_id` = ? AND `id` >= ? ORDER BY `id` LIMIT 1024) AS `batch` USING(`id`)", + t.joinedTable: "SELECT * FROM `shard_1`.`joinedtable` WHERE `joined_paginationKey` IN (SELECT * FROM (SELECT `joined_paginationKey1` AS sharding_join_alias FROM `shard_1`.`join1` WHERE `tenant_id` = ? AND `joined_paginationKey1` >= ? UNION DISTINCT SELECT `joined_paginationKey2` AS sharding_join_alias FROM `shard_1`.`join2` WHERE `tenant_id` = ? AND `joined_paginationKey2` >= ? ORDER BY sharding_join_alias LIMIT 1024) AS sharding_join_table) ORDER BY `joined_paginationKey`", + t.primaryKeyTable: "SELECT * FROM `shard_1`.`pkTable` USE INDEX (PRIMARY) WHERE `tenant_id` = ? AND `tenant_id` >= ?", + } + + for table, expectedSQL := range expected { + selectBuilder, err := t.filter.BuildSelect([]string{"*"}, table, minKey, 1024) + t.Require().Nil(err) + + sql, _, err := selectBuilder.ToSql() + t.Require().Nil(err) + t.Require().Equal(expectedSQL, sql) + } +} + func (t *CopyFilterTestSuite) TestShardingValueTypes() { tenantIds := []interface{}{ uint64(1), uint32(1), uint16(1), uint8(1), uint(1), diff --git a/test/go/data_iterator_test.go b/test/go/data_iterator_test.go index f30d0cfb2..a9ef77237 100644 --- a/test/go/data_iterator_test.go +++ b/test/go/data_iterator_test.go @@ -153,6 +153,65 @@ func (this *DataIteratorTestSuite) TestExistingRowsAreIterated() { ) } +func (this *DataIteratorTestSuite) loadTablesAfterExecuting(statements []string, names ...string) []*ghostferry.TableSchema { + for _, statement := range statements { + _, err := this.Ferry.SourceDB.Exec(statement) + this.Require().Nil(err) + } + + tableFilter := &testhelpers.TestTableFilter{ + DbsFunc: testhelpers.DbApplicabilityFilter([]string{testhelpers.TestSchemaName}), + } + cache, err := ghostferry.LoadTables(this.Ferry.SourceDB, tableFilter, nil, nil, nil, nil) + this.Require().Nil(err) + + tables := make([]*ghostferry.TableSchema, len(names)) + for i, name := range names { + tables[i] = cache.Get(testhelpers.TestSchemaName, name) + this.Require().NotNil(tables[i], name) + } + return tables +} + +func (this *DataIteratorTestSuite) receivedIds(table string) []interface{} { + ids := []interface{}{} + for _, row := range this.receivedRows[table] { + ids = append(ids, row[0]) + } + return ids +} + +func (this *DataIteratorTestSuite) TestRowsAtTheMinimumPaginationKeyAreIterated() { + tables := this.loadTablesAfterExecuting([]string{ + "CREATE TABLE gftest.zero_and_up (id bigint NOT NULL, data TEXT, PRIMARY KEY (id))", + "INSERT INTO gftest.zero_and_up VALUES (0, 'a'), (1, 'b'), (2, 'c')", + "CREATE TABLE gftest.only_zero (id bigint unsigned NOT NULL, data TEXT, PRIMARY KEY (id))", + "INSERT INTO gftest.only_zero VALUES (0, 'a')", + "CREATE TABLE gftest.empty_binary_key (id varbinary(16) NOT NULL, data TEXT, PRIMARY KEY (id))", + "INSERT INTO gftest.empty_binary_key VALUES ('', 'a'), ('x', 'b')", + }, "zero_and_up", "only_zero", "empty_binary_key") + + this.di.Run(tables) + + this.Equal([]interface{}{int64(0), int64(1), int64(2)}, this.receivedIds("zero_and_up")) + this.Equal([]interface{}{int64(0)}, this.receivedIds("only_zero")) + this.Equal([]interface{}{[]byte(""), []byte("x")}, this.receivedIds("empty_binary_key")) +} + +func (this *DataIteratorTestSuite) TestNegativePaginationKeyIsRejected() { + tables := this.loadTablesAfterExecuting([]string{ + "CREATE TABLE gftest.negative_ids (id bigint NOT NULL, data TEXT, PRIMARY KEY (id))", + "INSERT INTO gftest.negative_ids VALUES (-1, 'a'), (1, 'b')", + }, "negative_ids") + + errorHandler := &testhelpers.ErrorHandler{} + this.di.ErrorHandler = errorHandler + this.di.Run(tables) + + this.Require().NotNil(errorHandler.LastError) + this.Require().Contains(errorHandler.LastError.Error(), "negative pagination key") +} + func (this *DataIteratorTestSuite) TestDoneListenerGetsNotifiedWhenDone() { wasNotified := false From efeaff351b40b86e8f4e236d7e67e64a23b777a5 Mon Sep 17 00:00:00 2001 From: Leszek Zalewski Date: Wed, 14 Oct 2026 22:25:55 +0200 Subject: [PATCH 2/3] Include the minimum pagination key and reject negative keys Cursors starting from the minimum key (0, or empty for binary keys) now select rows >= that key on the first batch, so those rows are copied and a table holding only such a row is no longer skipped. DefaultBuildSelect and ShardedCopyFilter use the new PaginationKey.IsMin(). Negative values in a signed integer pagination column cannot be reached by integer pagination, so the data iterator now aborts instead of silently skipping them. Fixes #305. --- CHANGELOG.md | 3 +++ cursor.go | 17 ++++++++++++++--- filter.go | 6 ++++-- pagination_key.go | 11 +++++++++++ sharding/filter.go | 15 +++++++++++---- table_schema_cache.go | 40 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 83 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4b59edfb..5004e3192 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ All notable changes to this project will be documented in this file. - Extend `DMLEvent` with `BinlogCoordinate()` and `ResumableBinlogCoordinate()`. `DMLEventBase` implements both; custom implementations must provide these methods. - Replace `BinlogCoordinate.Compare` with `HasReached`, using file/position ordering or GTID-set containment. Comparing different coordinate types now returns an error instead of panicking. - Compare parsed GTID sets directly during cutover, eliminating per-event GTID serialization and parsing. +- Add `IsMin()` to `PaginationKey`; custom implementations must provide it. A `CopyFilter.BuildSelect` that builds its own pagination condition must include the key itself when `IsMin()` is true. `DefaultBuildSelect` and `sharding.ShardedCopyFilter` do. - Updated golang to 1.27.2 - Updated go packages: - `github.com/sirupsen/logrus` to v1.10.2 @@ -40,6 +41,8 @@ All notable changes to this project will be documented in this file. - Publish binlog stop requests atomically after recording the stop coordinate, including reads from the control server. - Decode unsigned `MEDIUMINT` binlog values of 2^23 and above correctly. Previously they were sign-extended, so inserts were clamped to 16777215 on the target and updates of already-copied rows failed with an out-of-range error ([#299](https://github.com/Shopify/ghostferry/issues/299)). - Resume from the binlog writer coordinate when the InlineVerifier is disabled. Previously a coordinate left in the state by an earlier verifier-enabled run still held back the resume point, so resuming failed once the source purged those binlogs ([#184](https://github.com/Shopify/ghostferry/issues/184)). +- Copy rows whose pagination key is the minimum value: 0 for integer keys, empty for binary and string keys. The first batch used to select keys strictly greater than the minimum, so these rows were silently skipped, and a table holding only such a row was not copied at all ([#305](https://github.com/Shopify/ghostferry/issues/305)). +- Abort when a signed integer pagination column holds negative values instead of silently skipping those rows ([#305](https://github.com/Shopify/ghostferry/issues/305)). ## [1.3.1 - 2026-04-15] diff --git a/cursor.go b/cursor.go index 3c83a1eae..679ad45d5 100644 --- a/cursor.go +++ b/cursor.go @@ -95,7 +95,11 @@ func (c *Cursor) Each(f func(*RowBatch) error) error { c.ColumnsToSelect = []string{"*"} } - for c.lastSuccessfulPaginationKey.Compare(c.MaxPaginationKey) < 0 { + // Starting from the minimum key, the first query includes rows equal to it + // (DefaultBuildSelect uses >=), so that batch may end at the start key and + // must run even when the minimum is also the maximum. + inclusive := c.lastSuccessfulPaginationKey.IsMin() + for inclusive || c.lastSuccessfulPaginationKey.Compare(c.MaxPaginationKey) < 0 { var tx SqlPreparerAndRollbacker var batch *RowBatch var paginationKeypos PaginationKey @@ -136,7 +140,8 @@ func (c *Cursor) Each(f func(*RowBatch) error) error { break } - if paginationKeypos.Compare(c.lastSuccessfulPaginationKey) <= 0 { + advance := paginationKeypos.Compare(c.lastSuccessfulPaginationKey) + if advance < 0 || (advance == 0 && !inclusive) { tx.Rollback() err = fmt.Errorf("new paginationKeypos %s <= lastSuccessfulPaginationKey %s", paginationKeypos.String(), c.lastSuccessfulPaginationKey.String()) c.logger.WithError(err).Errorf("last successful paginationKey position did not advance") @@ -153,6 +158,7 @@ func (c *Cursor) Each(f func(*RowBatch) error) error { tx.Rollback() c.lastSuccessfulPaginationKey = paginationKeypos + inclusive = false } return nil @@ -307,9 +313,14 @@ func ScanByteRow(rows *sqlorig.Rows, columnCount int) ([][]byte, error) { func DefaultBuildSelect(columns []string, table *TableSchema, lastPaginationKey PaginationKey, batchSize uint64) squirrel.SelectBuilder { quotedPaginationKey := QuoteField(table.GetPaginationColumn().Name) + var after squirrel.Sqlizer = squirrel.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()} + if lastPaginationKey.IsMin() { + after = squirrel.GtOrEq{quotedPaginationKey: lastPaginationKey.SQLValue()} + } + return squirrel.Select(columns...). From(QuotedTableName(table)). - Where(squirrel.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()}). + Where(after). Limit(batchSize). OrderBy(quotedPaginationKey) } diff --git a/filter.go b/filter.go index e9a23f1e3..a8c328c0c 100644 --- a/filter.go +++ b/filter.go @@ -13,8 +13,10 @@ type CopyFilter interface { // here will cause the query to be retried, until the retry limit is // reached, at which point the ferry will be aborted. BuildSelect is passed // the columns to be selected, table being copied, the last pagination key value - // from the previous batch, and the batch size. Call DefaultBuildSelect to - // generate the default query, which may be used as a starting point. + // from the previous batch, and the batch size. Select rows with a greater + // pagination key, or a greater or equal one when the key's IsMin() is true + // (nothing copied yet). Call DefaultBuildSelect to generate the default + // query, which may be used as a starting point. BuildSelect([]string, *TableSchema, PaginationKey, uint64) (sq.SelectBuilder, error) // ApplicableEvent is used to filter events for rows that have been diff --git a/pagination_key.go b/pagination_key.go index 5a70e68f1..a83724a6a 100644 --- a/pagination_key.go +++ b/pagination_key.go @@ -29,6 +29,9 @@ type PaginationKey interface { MarshalJSON() ([]byte, error) // IsMax returns true if this key represents the maximum possible value for its type. IsMax() bool + // IsMin returns true if this key represents the minimum possible value for + // its type. Cursors start here, so queries must include rows equal to it. + IsMin() bool } type Uint64Key struct { @@ -84,6 +87,10 @@ func (k Uint64Key) IsMax() bool { return k.Value == math.MaxUint64 } +func (k Uint64Key) IsMin() bool { + return k.Value == 0 +} + func (k Uint64Key) MarshalJSON() ([]byte, error) { valBytes, err := json.Marshal(k.Value) @@ -170,6 +177,10 @@ func (k BinaryKey) IsMax() bool { return true } +func (k BinaryKey) IsMin() bool { + return len(k.Value) == 0 +} + func (k BinaryKey) MarshalJSON() ([]byte, error) { valBytes, err := json.Marshal(hex.EncodeToString(k.Value)) diff --git a/sharding/filter.go b/sharding/filter.go index a29145b49..2ed0a83f7 100644 --- a/sharding/filter.go +++ b/sharding/filter.go @@ -37,6 +37,13 @@ func (f *ShardedCopyFilter) BuildSelect(columns []string, table *ghostferry.Tabl quotedShardingKey := "`" + f.ShardingKey + "`" quotedTable := ghostferry.QuotedTableName(table) + // A cursor starting from the minimum key has not copied anything yet, so + // rows equal to it (e.g. id 0) must be included. + after := ">" + if lastPaginationKey.IsMin() { + after = ">=" + } + if _, exists := f.PrimaryKeyTables[table.Name]; exists { // This table uses the sharding key as its primary key, and thus contains // a single row per sharding key. This can commonly occur when representing @@ -48,7 +55,7 @@ func (f *ShardedCopyFilter) BuildSelect(columns []string, table *ghostferry.Tabl return sq.Select(columns...). From(quotedTable + " USE INDEX (PRIMARY)"). Where(sq.Eq{quotedPaginationKey: f.ShardingValue}). - Where(sq.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()}), nil + Where(sq.Expr(quotedPaginationKey+" "+after+" ?", lastPaginationKey.SQLValue())), nil } joinTables, exists := f.JoinedTables[table.Name] @@ -84,7 +91,7 @@ func (f *ShardedCopyFilter) BuildSelect(columns []string, table *ghostferry.Tabl selectPaginationKeys += " " + f.shardingKeyIndexHint(table, indexHint) } - selectPaginationKeys += " WHERE " + quotedShardingKey + " = ? AND " + quotedPaginationKey + " > ?" + + selectPaginationKeys += " WHERE " + quotedShardingKey + " = ? AND " + quotedPaginationKey + " " + after + " ?" + " ORDER BY " + quotedPaginationKey + " LIMIT " + strconv.Itoa(int(batchSize)) return sq.Select(columns...). @@ -122,8 +129,8 @@ func (f *ShardedCopyFilter) BuildSelect(columns []string, table *ghostferry.Tabl var args []interface{} for _, joinTable := range joinTables { - pattern := "SELECT `%s` AS sharding_join_alias FROM `%s`.`%s` WHERE `%s` = ? AND `%s` > ?" - sql := fmt.Sprintf(pattern, joinTable.JoinColumn, table.Schema, joinTable.TableName, f.ShardingKey, joinTable.JoinColumn) + pattern := "SELECT `%s` AS sharding_join_alias FROM `%s`.`%s` WHERE `%s` = ? AND `%s` %s ?" + sql := fmt.Sprintf(pattern, joinTable.JoinColumn, table.Schema, joinTable.TableName, f.ShardingKey, joinTable.JoinColumn, after) clauses = append(clauses, sql) args = append(args, f.ShardingValue, lastPaginationKey.SQLValue()) } diff --git a/table_schema_cache.go b/table_schema_cache.go index a5e2f389b..7212613ec 100644 --- a/table_schema_cache.go +++ b/table_schema_cache.go @@ -132,6 +132,15 @@ func MaxPaginationKeys(db *sql.DB, tables []*TableSchema, logger Logger) (map[*T for _, table := range tables { logger := logger.WithField("table", table.String()) + hasNegative, err := hasNegativePaginationKey(db, table) + if err != nil { + logger.WithError(err).Errorf("failed to check for negative values in pagination key %s", table.GetPaginationColumn().Name) + return tablesWithData, emptyTables, err + } + if hasNegative { + return tablesWithData, emptyTables, fmt.Errorf("table %s has a negative pagination key in column %s, which Ghostferry cannot copy", table.String(), table.GetPaginationColumn().Name) + } + maxPaginationKey, maxPaginationKeyExists, err := maxPaginationKey(db, table) if err != nil { logger.WithError(err).Errorf("failed to get max primary key %s", table.GetPaginationColumn().Name) @@ -419,6 +428,37 @@ func showTablesFrom(c *sql.DB, dbname string) ([]string, error) { return tables, nil } +// hasNegativePaginationKey reports whether a signed integer pagination column +// holds negative values. Integer pagination starts at 0, so such rows would +// never be copied. +func hasNegativePaginationKey(db *sql.DB, table *TableSchema) (bool, error) { + column := table.GetPaginationColumn() + if column.IsUnsigned || (column.Type != schema.TYPE_NUMBER && column.Type != schema.TYPE_MEDIUM_INT) { + return false, nil + } + + query, args, err := sq. + Select("1"). + From(QuotedTableName(table)). + Where(sq.Lt{QuoteField(column.Name): 0}). + Limit(1). + ToSql() + if err != nil { + return false, err + } + + var found int + err = db.QueryRow(query, args...).Scan(&found) + switch { + case err == sqlorig.ErrNoRows: + return false, nil + case err != nil: + return false, err + default: + return true, nil + } +} + func maxPaginationKey(db *sql.DB, table *TableSchema) (PaginationKey, bool, error) { primaryKeyColumn := table.GetPaginationColumn() paginationKeyName := QuoteField(primaryKeyColumn.Name) From 16975d13c0c997659a0a68663c658255c1b4de39 Mon Sep 17 00:00:00 2001 From: Leszek Zalewski Date: Mon, 19 Oct 2026 19:33:42 +0200 Subject: [PATCH 3/3] Separate pagination start from checkpoints and scope negative-key checks --- CHANGELOG.md | 5 +- cursor.go | 34 +++++---- data_iterator.go | 5 +- filter.go | 8 +- iterative_verifier.go | 2 +- pagination_key.go | 11 --- sharding/filter.go | 42 +++++----- sharding/test/copy_filter_test.go | 18 ----- sharding/test/trivial_integration_test.go | 72 ++++++++++++++++++ state_tracker.go | 8 +- table_schema_cache.go | 69 +++++++++-------- test/go/data_iterator_test.go | 93 ++++++++++++++++++++++- 12 files changed, 256 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5004e3192..a7681c2db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,8 @@ All notable changes to this project will be documented in this file. - Extend `DMLEvent` with `BinlogCoordinate()` and `ResumableBinlogCoordinate()`. `DMLEventBase` implements both; custom implementations must provide these methods. - Replace `BinlogCoordinate.Compare` with `HasReached`, using file/position ordering or GTID-set containment. Comparing different coordinate types now returns an error instead of panicking. - Compare parsed GTID sets directly during cutover, eliminating per-event GTID serialization and parsing. -- Add `IsMin()` to `PaginationKey`; custom implementations must provide it. A `CopyFilter.BuildSelect` that builds its own pagination condition must include the key itself when `IsMin()` is true. `DefaultBuildSelect` and `sharding.ShardedCopyFilter` do. +- Cursors and `CopyFilter.BuildSelect` use a nil starting key to mean no lower bound; every non-nil checkpoint is exclusive, including 0 and empty binary keys. `StateTracker.LastSuccessfulPaginationKey` now returns nil until a batch has been copied. Custom query builders must handle nil and return rows in ascending pagination-key order. +- `MaxPaginationKeys` now accepts a `*CursorConfig` instead of a database connection so startup validation honors the configured copy filter. - Updated golang to 1.27.2 - Updated go packages: - `github.com/sirupsen/logrus` to v1.10.2 @@ -42,7 +43,7 @@ All notable changes to this project will be documented in this file. - Decode unsigned `MEDIUMINT` binlog values of 2^23 and above correctly. Previously they were sign-extended, so inserts were clamped to 16777215 on the target and updates of already-copied rows failed with an out-of-range error ([#299](https://github.com/Shopify/ghostferry/issues/299)). - Resume from the binlog writer coordinate when the InlineVerifier is disabled. Previously a coordinate left in the state by an earlier verifier-enabled run still held back the resume point, so resuming failed once the source purged those binlogs ([#184](https://github.com/Shopify/ghostferry/issues/184)). - Copy rows whose pagination key is the minimum value: 0 for integer keys, empty for binary and string keys. The first batch used to select keys strictly greater than the minimum, so these rows were silently skipped, and a table holding only such a row was not copied at all ([#305](https://github.com/Shopify/ghostferry/issues/305)). -- Abort when a signed integer pagination column holds negative values instead of silently skipping those rows ([#305](https://github.com/Shopify/ghostferry/issues/305)). +- Abort when selected rows have negative signed integer pagination keys instead of silently skipping them. Startup validation uses the configured copy filter, so negative keys belonging to excluded tenants do not block a migration ([#305](https://github.com/Shopify/ghostferry/issues/305)). ## [1.3.1 - 2026-04-15] diff --git a/cursor.go b/cursor.go index 679ad45d5..67259f9ea 100644 --- a/cursor.go +++ b/cursor.go @@ -45,7 +45,7 @@ type CursorConfig struct { ReadRetries int } -// returns a new Cursor with an embedded copy of itself +// NewCursor resumes after startPaginationKey, or starts at the beginning when it is nil. func (c *CursorConfig) NewCursor(table *TableSchema, startPaginationKey, maxPaginationKey PaginationKey) *Cursor { return &Cursor{ CursorConfig: *c, @@ -95,11 +95,7 @@ func (c *Cursor) Each(f func(*RowBatch) error) error { c.ColumnsToSelect = []string{"*"} } - // Starting from the minimum key, the first query includes rows equal to it - // (DefaultBuildSelect uses >=), so that batch may end at the start key and - // must run even when the minimum is also the maximum. - inclusive := c.lastSuccessfulPaginationKey.IsMin() - for inclusive || c.lastSuccessfulPaginationKey.Compare(c.MaxPaginationKey) < 0 { + for c.lastSuccessfulPaginationKey == nil || c.lastSuccessfulPaginationKey.Compare(c.MaxPaginationKey) < 0 { var tx SqlPreparerAndRollbacker var batch *RowBatch var paginationKeypos PaginationKey @@ -140,8 +136,7 @@ func (c *Cursor) Each(f func(*RowBatch) error) error { break } - advance := paginationKeypos.Compare(c.lastSuccessfulPaginationKey) - if advance < 0 || (advance == 0 && !inclusive) { + if c.lastSuccessfulPaginationKey != nil && paginationKeypos.Compare(c.lastSuccessfulPaginationKey) <= 0 { tx.Rollback() err = fmt.Errorf("new paginationKeypos %s <= lastSuccessfulPaginationKey %s", paginationKeypos.String(), c.lastSuccessfulPaginationKey.String()) c.logger.WithError(err).Errorf("last successful paginationKey position did not advance") @@ -158,7 +153,6 @@ func (c *Cursor) Each(f func(*RowBatch) error) error { tx.Rollback() c.lastSuccessfulPaginationKey = paginationKeypos - inclusive = false } return nil @@ -257,6 +251,16 @@ func (c *Cursor) Fetch(db SqlPreparer) (batch *RowBatch, paginationKeypos Pagina return } + // The first query has no lower bound. Recheck its first row in case a + // negative key was inserted after startup validation. + if len(batchData) == 0 && !c.paginationKeyColumn.IsUnsigned && + (c.paginationKeyColumn.Type == schema.TYPE_NUMBER || c.paginationKeyColumn.Type == schema.TYPE_MEDIUM_INT) { + if value, ok := Int64Value(rowData[paginationKeyIndex]); ok && value < 0 { + err = fmt.Errorf("table %s has a negative pagination key in column %s, which Ghostferry cannot copy", c.Table.String(), c.paginationKeyColumn.Name) + return + } + } + batchData = append(batchData, rowData) } @@ -313,14 +317,12 @@ func ScanByteRow(rows *sqlorig.Rows, columnCount int) ([][]byte, error) { func DefaultBuildSelect(columns []string, table *TableSchema, lastPaginationKey PaginationKey, batchSize uint64) squirrel.SelectBuilder { quotedPaginationKey := QuoteField(table.GetPaginationColumn().Name) - var after squirrel.Sqlizer = squirrel.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()} - if lastPaginationKey.IsMin() { - after = squirrel.GtOrEq{quotedPaginationKey: lastPaginationKey.SQLValue()} - } - - return squirrel.Select(columns...). + query := squirrel.Select(columns...). From(QuotedTableName(table)). - Where(after). Limit(batchSize). OrderBy(quotedPaginationKey) + if lastPaginationKey != nil { + query = query.Where(squirrel.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()}) + } + return query } diff --git a/data_iterator.go b/data_iterator.go index 0586fd8a6..567a73a22 100644 --- a/data_iterator.go +++ b/data_iterator.go @@ -39,9 +39,10 @@ func (d *DataIterator) Run(tables []*TableSchema) { } d.logger.WithField("tablesCount", len(tables)).Info("starting data iterator run") - tablesWithData, emptyTables, err := MaxPaginationKeys(d.DB, tables, d.logger) + tablesWithData, emptyTables, err := MaxPaginationKeys(d.CursorConfig, tables, d.logger) if err != nil { d.ErrorHandler.Fatal("data_iterator", err) + return } for _, table := range emptyTables { @@ -84,7 +85,7 @@ func (d *DataIterator) Run(tables []*TableSchema) { } startPaginationKey := d.StateTracker.LastSuccessfulPaginationKey(table.String(), table) - if startPaginationKey.IsMax() { + if startPaginationKey != nil && startPaginationKey.IsMax() { err := fmt.Errorf("%v has been marked as completed but a table iterator has been spawned, this is likely a programmer error which resulted in the inconsistent starting state", table.String()) logger.WithError(err).Error("this is definitely a bug") d.ErrorHandler.Fatal("data_iterator", err) diff --git a/filter.go b/filter.go index a8c328c0c..a14f0c8bd 100644 --- a/filter.go +++ b/filter.go @@ -13,10 +13,10 @@ type CopyFilter interface { // here will cause the query to be retried, until the retry limit is // reached, at which point the ferry will be aborted. BuildSelect is passed // the columns to be selected, table being copied, the last pagination key value - // from the previous batch, and the batch size. Select rows with a greater - // pagination key, or a greater or equal one when the key's IsMin() is true - // (nothing copied yet). Call DefaultBuildSelect to generate the default - // query, which may be used as a starting point. + // from the previous batch, and the batch size. Select rows strictly after + // that key in ascending pagination-key order. A nil key means no lower + // bound; it is used for the first batch and startup validation. Call + // DefaultBuildSelect to generate the default query as a starting point. BuildSelect([]string, *TableSchema, PaginationKey, uint64) (sq.SelectBuilder, error) // ApplicableEvent is used to filter events for rows that have been diff --git a/iterative_verifier.go b/iterative_verifier.go index 0ba1b07ce..345008b40 100644 --- a/iterative_verifier.go +++ b/iterative_verifier.go @@ -388,7 +388,7 @@ func (v *IterativeVerifier) iterateAllTables(mismatchedPaginationKeyFunc func(st func (v *IterativeVerifier) iterateTableFingerprints(table *TableSchema, mismatchedPaginationKeyFunc func(string, *TableSchema) error) error { // The cursor will stop iterating when it cannot find anymore rows, // so it will not iterate until MaxPaginationKey. - cursor := v.CursorConfig.NewCursorWithoutRowLock(table, MinPaginationKey(table.GetPaginationColumn()), MaxPaginationKey(table.GetPaginationColumn())) + cursor := v.CursorConfig.NewCursorWithoutRowLock(table, nil, MaxPaginationKey(table.GetPaginationColumn())) // It only needs the PaginationKeys, not the entire row. cursor.ColumnsToSelect = []string{fmt.Sprintf("`%s`", table.GetPaginationColumn().Name)} diff --git a/pagination_key.go b/pagination_key.go index a83724a6a..5a70e68f1 100644 --- a/pagination_key.go +++ b/pagination_key.go @@ -29,9 +29,6 @@ type PaginationKey interface { MarshalJSON() ([]byte, error) // IsMax returns true if this key represents the maximum possible value for its type. IsMax() bool - // IsMin returns true if this key represents the minimum possible value for - // its type. Cursors start here, so queries must include rows equal to it. - IsMin() bool } type Uint64Key struct { @@ -87,10 +84,6 @@ func (k Uint64Key) IsMax() bool { return k.Value == math.MaxUint64 } -func (k Uint64Key) IsMin() bool { - return k.Value == 0 -} - func (k Uint64Key) MarshalJSON() ([]byte, error) { valBytes, err := json.Marshal(k.Value) @@ -177,10 +170,6 @@ func (k BinaryKey) IsMax() bool { return true } -func (k BinaryKey) IsMin() bool { - return len(k.Value) == 0 -} - func (k BinaryKey) MarshalJSON() ([]byte, error) { valBytes, err := json.Marshal(hex.EncodeToString(k.Value)) diff --git a/sharding/filter.go b/sharding/filter.go index 2ed0a83f7..3fe2026d7 100644 --- a/sharding/filter.go +++ b/sharding/filter.go @@ -37,25 +37,20 @@ func (f *ShardedCopyFilter) BuildSelect(columns []string, table *ghostferry.Tabl quotedShardingKey := "`" + f.ShardingKey + "`" quotedTable := ghostferry.QuotedTableName(table) - // A cursor starting from the minimum key has not copied anything yet, so - // rows equal to it (e.g. id 0) must be included. - after := ">" - if lastPaginationKey.IsMin() { - after = ">=" - } - if _, exists := f.PrimaryKeyTables[table.Name]; exists { // This table uses the sharding key as its primary key, and thus contains // a single row per sharding key. This can commonly occur when representing // the model for a single tenant in a multitenant database, e.g. a shop on // a multitenant commerce platform. // - // It is necessary to use two WHERE conditions on quotedPaginationKey so the second batch will be empty. // No LIMIT clause is necessary since at most one row is present. - return sq.Select(columns...). + query := sq.Select(columns...). From(quotedTable + " USE INDEX (PRIMARY)"). - Where(sq.Eq{quotedPaginationKey: f.ShardingValue}). - Where(sq.Expr(quotedPaginationKey+" "+after+" ?", lastPaginationKey.SQLValue())), nil + Where(sq.Eq{quotedPaginationKey: f.ShardingValue}) + if lastPaginationKey != nil { + query = query.Where(sq.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()}) + } + return query, nil } joinTables, exists := f.JoinedTables[table.Name] @@ -91,12 +86,18 @@ func (f *ShardedCopyFilter) BuildSelect(columns []string, table *ghostferry.Tabl selectPaginationKeys += " " + f.shardingKeyIndexHint(table, indexHint) } - selectPaginationKeys += " WHERE " + quotedShardingKey + " = ? AND " + quotedPaginationKey + " " + after + " ?" + - " ORDER BY " + quotedPaginationKey + " LIMIT " + strconv.Itoa(int(batchSize)) + selectPaginationKeys += " WHERE " + quotedShardingKey + " = ?" + args := make([]interface{}, 0, 2) + args = append(args, f.ShardingValue) + if lastPaginationKey != nil { + selectPaginationKeys += " AND " + quotedPaginationKey + " > ?" + args = append(args, lastPaginationKey.SQLValue()) + } + selectPaginationKeys += " ORDER BY " + quotedPaginationKey + " LIMIT " + strconv.Itoa(int(batchSize)) return sq.Select(columns...). From(quotedTable). - Join("("+selectPaginationKeys+") AS `batch` USING("+quotedPaginationKey+")", f.ShardingValue, lastPaginationKey.SQLValue()), nil + Join("("+selectPaginationKeys+") AS `batch` USING("+quotedPaginationKey+")", args...), nil } // This is a "joined table". It is the only supported type of table that @@ -127,12 +128,19 @@ func (f *ShardedCopyFilter) BuildSelect(columns []string, table *ghostferry.Tabl // it to a batch, then select the rest of the columns. var clauses []string var args []interface{} + const firstPage = "SELECT `%[1]s` AS sharding_join_alias FROM `%[2]s`.`%[3]s` WHERE `%[4]s` = ?" + pattern := firstPage + if lastPaginationKey != nil { + pattern = firstPage + " AND `%[1]s` > ?" + } for _, joinTable := range joinTables { - pattern := "SELECT `%s` AS sharding_join_alias FROM `%s`.`%s` WHERE `%s` = ? AND `%s` %s ?" - sql := fmt.Sprintf(pattern, joinTable.JoinColumn, table.Schema, joinTable.TableName, f.ShardingKey, joinTable.JoinColumn, after) + sql := fmt.Sprintf(pattern, joinTable.JoinColumn, table.Schema, joinTable.TableName, f.ShardingKey) clauses = append(clauses, sql) - args = append(args, f.ShardingValue, lastPaginationKey.SQLValue()) + args = append(args, f.ShardingValue) + if lastPaginationKey != nil { + args = append(args, lastPaginationKey.SQLValue()) + } } subquery := strings.Join(clauses, " UNION DISTINCT ") diff --git a/sharding/test/copy_filter_test.go b/sharding/test/copy_filter_test.go index 94ec09b78..dab0dfed4 100644 --- a/sharding/test/copy_filter_test.go +++ b/sharding/test/copy_filter_test.go @@ -285,24 +285,6 @@ func (t *CopyFilterTestSuite) TestSelectsPrimaryKeyTables() { t.Require().Equal([]interface{}{t.shardingValue, t.paginationKeyCursor.SQLValue()}, args) } -func (t *CopyFilterTestSuite) TestIncludesMinimumPaginationKey() { - minKey := ghostferry.NewUint64Key(0) - expected := map[*ghostferry.TableSchema]string{ - t.normalTable: "SELECT * FROM `shard_1`.`normaltable` JOIN (SELECT `id` FROM `shard_1`.`normaltable` USE INDEX (`good_sharding_index`) WHERE `tenant_id` = ? AND `id` >= ? ORDER BY `id` LIMIT 1024) AS `batch` USING(`id`)", - t.joinedTable: "SELECT * FROM `shard_1`.`joinedtable` WHERE `joined_paginationKey` IN (SELECT * FROM (SELECT `joined_paginationKey1` AS sharding_join_alias FROM `shard_1`.`join1` WHERE `tenant_id` = ? AND `joined_paginationKey1` >= ? UNION DISTINCT SELECT `joined_paginationKey2` AS sharding_join_alias FROM `shard_1`.`join2` WHERE `tenant_id` = ? AND `joined_paginationKey2` >= ? ORDER BY sharding_join_alias LIMIT 1024) AS sharding_join_table) ORDER BY `joined_paginationKey`", - t.primaryKeyTable: "SELECT * FROM `shard_1`.`pkTable` USE INDEX (PRIMARY) WHERE `tenant_id` = ? AND `tenant_id` >= ?", - } - - for table, expectedSQL := range expected { - selectBuilder, err := t.filter.BuildSelect([]string{"*"}, table, minKey, 1024) - t.Require().Nil(err) - - sql, _, err := selectBuilder.ToSql() - t.Require().Nil(err) - t.Require().Equal(expectedSQL, sql) - } -} - func (t *CopyFilterTestSuite) TestShardingValueTypes() { tenantIds := []interface{}{ uint64(1), uint32(1), uint16(1), uint8(1), uint(1), diff --git a/sharding/test/trivial_integration_test.go b/sharding/test/trivial_integration_test.go index bd4c28982..6e7b3431b 100644 --- a/sharding/test/trivial_integration_test.go +++ b/sharding/test/trivial_integration_test.go @@ -5,11 +5,13 @@ import ( "sync/atomic" "testing" + "github.com/Shopify/ghostferry" sql "github.com/Shopify/ghostferry/sqlwrapper" "github.com/Shopify/ghostferry/sharding" "github.com/Shopify/ghostferry/testhelpers" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func setupSingleTableDatabase(f *testhelpers.TestFerry, sourceDB, targetDB *sql.DB) { @@ -55,6 +57,76 @@ func TestSelectiveCopyDataWithoutAnyWritesToSource(t *testing.T) { assert.Equal(t, 333, len(rows)) } +func TestSelectiveCopyIncludesMinimumKeysAndIgnoresExcludedNegativeKeys(t *testing.T) { + ferry := testhelpers.NewTestFerry() + ferry.Config.UpdatableConfig.DataIterationBatchSize = 1 + ferry.Config.DataIterationConcurrency = 1 + ferry.Config.VerifierType = ghostferry.VerifierTypeInline + ferry.Config.TableFilter = &testhelpers.TestTableFilter{ + DbsFunc: testhelpers.DbApplicabilityFilter([]string{"gftest"}), + } + ferry.Config.CopyFilter = &sharding.ShardedCopyFilter{ + ShardingKey: "tenant_id", + ShardingValue: int64(0), + PrimaryKeyTables: map[string]struct{}{ + "tenants": {}, + }, + JoinedTables: map[string][]sharding.JoinTable{ + "shared_rows": {{TableName: "links", JoinColumn: "shared_id"}}, + }, + } + require.NoError(t, ferry.Config.ValidateConfig()) + errorHandler := &testhelpers.ErrorHandler{} + ferry.ErrorHandler = errorHandler + testcase := &testhelpers.IntegrationTestCase{ + T: t, + Ferry: ferry, + SetupAction: func(_ *testhelpers.TestFerry, sourceDB, targetDB *sql.DB) { + for _, db := range []*sql.DB{sourceDB, targetDB} { + for _, statement := range []string{ + "CREATE DATABASE gftest", + "CREATE TABLE gftest.normal_rows (id bigint PRIMARY KEY, tenant_id bigint NOT NULL, data TEXT, KEY tenant_idx (tenant_id, id))", + "CREATE TABLE gftest.tenants (tenant_id bigint PRIMARY KEY, data TEXT)", + "CREATE TABLE gftest.links (id bigint PRIMARY KEY, tenant_id bigint NOT NULL, shared_id bigint NOT NULL, KEY tenant_idx (tenant_id, id))", + "CREATE TABLE gftest.shared_rows (id bigint PRIMARY KEY, data TEXT)", + "CREATE TABLE gftest.excluded_rows (id bigint PRIMARY KEY, tenant_id bigint NOT NULL, KEY tenant_idx (tenant_id, id))", + } { + _, err := db.Exec(statement) + require.NoError(t, err) + } + } + for _, statement := range []string{ + "INSERT INTO gftest.normal_rows VALUES (-1, 2, 'excluded'), (0, 0, 'zero'), (1, 0, 'one')", + "INSERT INTO gftest.tenants VALUES (-1, 'excluded'), (0, 'zero'), (2, 'excluded')", + "INSERT INTO gftest.links VALUES (-1, 2, -1), (0, 0, 0), (1, 0, 1)", + "INSERT INTO gftest.shared_rows VALUES (-1, 'excluded'), (0, 'zero'), (1, 'one')", + "INSERT INTO gftest.excluded_rows VALUES (-2, 2), (-1, 2)", + } { + _, err := sourceDB.Exec(statement) + require.NoError(t, err) + } + }, + } + defer testcase.Teardown() + testcase.CopyData() + require.NoError(t, errorHandler.LastError) + + for _, check := range []struct { + query string + want string + }{ + {"SELECT GROUP_CONCAT(CONCAT(id, ':', data) ORDER BY id) FROM gftest.normal_rows", "0:zero,1:one"}, + {"SELECT GROUP_CONCAT(CONCAT(tenant_id, ':', data) ORDER BY tenant_id) FROM gftest.tenants", "0:zero"}, + {"SELECT GROUP_CONCAT(CONCAT(id, ':', shared_id) ORDER BY id) FROM gftest.links", "0:0,1:1"}, + {"SELECT GROUP_CONCAT(CONCAT(id, ':', data) ORDER BY id) FROM gftest.shared_rows", "0:zero,1:one"}, + {"SELECT COUNT(*) FROM gftest.excluded_rows", "0"}, + } { + var got string + require.NoError(t, testcase.TargetDB.QueryRow(check.query).Scan(&got)) + assert.Equal(t, check.want, got, check.query) + } +} + func TestSelectiveCopyDataWithInsertLoadOnOtherTenants(t *testing.T) { testcase := &testhelpers.IntegrationTestCase{ T: t, diff --git a/state_tracker.go b/state_tracker.go index ecb33b474..13aee1314 100644 --- a/state_tracker.go +++ b/state_tracker.go @@ -371,6 +371,7 @@ func (s *StateTracker) RowStatsWrittenPerTable() map[string]RowStats { return d } +// LastSuccessfulPaginationKey returns nil until a batch has been copied. func (s *StateTracker) LastSuccessfulPaginationKey(table string, tableSchema *TableSchema) PaginationKey { s.CopyRWMutex.RLock() defer s.CopyRWMutex.RUnlock() @@ -380,12 +381,7 @@ func (s *StateTracker) LastSuccessfulPaginationKey(table string, tableSchema *Ta return MaxPaginationKey(tableSchema.GetPaginationColumn()) } - paginationKey, found := s.lastSuccessfulPaginationKeys[table] - if !found { - return MinPaginationKey(tableSchema.GetPaginationColumn()) - } - - return paginationKey + return s.lastSuccessfulPaginationKeys[table] } func (s *StateTracker) MarkTableAsCompleted(table string) { diff --git a/table_schema_cache.go b/table_schema_cache.go index 7212613ec..e0e0baad9 100644 --- a/table_schema_cache.go +++ b/table_schema_cache.go @@ -125,23 +125,32 @@ func QuotedTableNameFromString(database, table string) string { return fmt.Sprintf("`%s`.`%s`", database, table) } -func MaxPaginationKeys(db *sql.DB, tables []*TableSchema, logger Logger) (map[*TableSchema]PaginationKey, []*TableSchema, error) { +// MaxPaginationKeys validates selected signed keys and returns table-wide upper +// bounds for iteration. The first-key probe uses the same filter as the cursor. +func MaxPaginationKeys(cursorConfig *CursorConfig, tables []*TableSchema, logger Logger) (map[*TableSchema]PaginationKey, []*TableSchema, error) { tablesWithData := make(map[*TableSchema]PaginationKey) emptyTables := make([]*TableSchema, 0, len(tables)) for _, table := range tables { logger := logger.WithField("table", table.String()) - hasNegative, err := hasNegativePaginationKey(db, table) - if err != nil { - logger.WithError(err).Errorf("failed to check for negative values in pagination key %s", table.GetPaginationColumn().Name) - return tablesWithData, emptyTables, err - } - if hasNegative { - return tablesWithData, emptyTables, fmt.Errorf("table %s has a negative pagination key in column %s, which Ghostferry cannot copy", table.String(), table.GetPaginationColumn().Name) + column := table.GetPaginationColumn() + if !column.IsUnsigned && (column.Type == schema.TYPE_NUMBER || column.Type == schema.TYPE_MEDIUM_INT) { + firstKey, found, err := firstSignedPaginationKey(cursorConfig, table) + if err != nil { + logger.WithError(err).Errorf("failed to check first selected pagination key %s", column.Name) + return tablesWithData, emptyTables, err + } + if !found { + emptyTables = append(emptyTables, table) + continue + } + if firstKey < 0 { + return tablesWithData, emptyTables, fmt.Errorf("table %s has a negative pagination key in column %s, which Ghostferry cannot copy", table.String(), column.Name) + } } - maxPaginationKey, maxPaginationKeyExists, err := maxPaginationKey(db, table) + maxPaginationKey, maxPaginationKeyExists, err := maxPaginationKey(cursorConfig.DB, table) if err != nil { logger.WithError(err).Errorf("failed to get max primary key %s", table.GetPaginationColumn().Name) return tablesWithData, emptyTables, err @@ -428,35 +437,29 @@ func showTablesFrom(c *sql.DB, dbname string) ([]string, error) { return tables, nil } -// hasNegativePaginationKey reports whether a signed integer pagination column -// holds negative values. Integer pagination starts at 0, so such rows would -// never be copied. -func hasNegativePaginationKey(db *sql.DB, table *TableSchema) (bool, error) { - column := table.GetPaginationColumn() - if column.IsUnsigned || (column.Type != schema.TYPE_NUMBER && column.Type != schema.TYPE_MEDIUM_INT) { - return false, nil +func firstSignedPaginationKey(cursorConfig *CursorConfig, table *TableSchema) (int64, bool, error) { + columns := []string{QuoteField(table.GetPaginationColumn().Name)} + var builder sq.SelectBuilder + if cursorConfig.BuildSelect == nil { + builder = DefaultBuildSelect(columns, table, nil, 1) + } else { + var err error + builder, err = cursorConfig.BuildSelect(columns, table, nil, 1) + if err != nil { + return 0, false, err + } } - - query, args, err := sq. - Select("1"). - From(QuotedTableName(table)). - Where(sq.Lt{QuoteField(column.Name): 0}). - Limit(1). - ToSql() + query, args, err := builder.ToSql() if err != nil { - return false, err + return 0, false, err } - var found int - err = db.QueryRow(query, args...).Scan(&found) - switch { - case err == sqlorig.ErrNoRows: - return false, nil - case err != nil: - return false, err - default: - return true, nil + var value int64 + err = cursorConfig.DB.QueryRow(query, args...).Scan(&value) + if err == sqlorig.ErrNoRows { + return 0, false, nil } + return value, err == nil, err } func maxPaginationKey(db *sql.DB, table *TableSchema) (PaginationKey, bool, error) { diff --git a/test/go/data_iterator_test.go b/test/go/data_iterator_test.go index a9ef77237..8710320c2 100644 --- a/test/go/data_iterator_test.go +++ b/test/go/data_iterator_test.go @@ -5,6 +5,7 @@ import ( "sync" "testing" + "github.com/Masterminds/squirrel" "github.com/Shopify/ghostferry" "github.com/Shopify/ghostferry/testhelpers" "github.com/stretchr/testify/suite" @@ -56,7 +57,7 @@ func (this *DataIteratorTestSuite) SetupTest() { BatchSizePerTableOverride: config.DataIterationBatchSizePerTableOverride, ReadRetries: config.DBReadRetries, }, - StateTracker: ghostferry.NewStateTracker(config.DataIterationConcurrency * 10), + StateTracker: ghostferry.NewStateTracker(config.DataIterationConcurrency * 10), TargetPaginationKeys: &sync.Map{}, } @@ -182,6 +183,11 @@ func (this *DataIteratorTestSuite) receivedIds(table string) []interface{} { } func (this *DataIteratorTestSuite) TestRowsAtTheMinimumPaginationKeyAreIterated() { + this.Ferry.Config.UpdatableConfig.DataIterationBatchSize = 1 + this.di.Concurrency = 1 + errorHandler := &testhelpers.ErrorHandler{} + this.di.ErrorHandler = errorHandler + tables := this.loadTablesAfterExecuting([]string{ "CREATE TABLE gftest.zero_and_up (id bigint NOT NULL, data TEXT, PRIMARY KEY (id))", "INSERT INTO gftest.zero_and_up VALUES (0, 'a'), (1, 'b'), (2, 'c')", @@ -192,12 +198,72 @@ func (this *DataIteratorTestSuite) TestRowsAtTheMinimumPaginationKeyAreIterated( }, "zero_and_up", "only_zero", "empty_binary_key") this.di.Run(tables) + this.Require().NoError(errorHandler.LastError) this.Equal([]interface{}{int64(0), int64(1), int64(2)}, this.receivedIds("zero_and_up")) this.Equal([]interface{}{int64(0)}, this.receivedIds("only_zero")) this.Equal([]interface{}{[]byte(""), []byte("x")}, this.receivedIds("empty_binary_key")) } +func (this *DataIteratorTestSuite) TestResumeAfterMinimumPaginationKey() { + this.Ferry.Config.UpdatableConfig.DataIterationBatchSize = 1 + this.di.Concurrency = 1 + errorHandler := &testhelpers.ErrorHandler{} + this.di.ErrorHandler = errorHandler + tables := this.loadTablesAfterExecuting([]string{ + "CREATE TABLE gftest.resume_zero (id bigint NOT NULL PRIMARY KEY)", + "INSERT INTO gftest.resume_zero VALUES (0), (1)", + "CREATE TABLE gftest.resume_empty (id varbinary(16) NOT NULL PRIMARY KEY)", + "INSERT INTO gftest.resume_empty VALUES (''), ('x')", + }, "resume_zero", "resume_empty") + + this.di.StateTracker.UpdateLastSuccessfulPaginationKey("gftest.resume_zero", ghostferry.NewUint64Key(0), ghostferry.RowStats{}) + this.di.StateTracker.UpdateLastSuccessfulPaginationKey("gftest.resume_empty", ghostferry.NewBinaryKey(nil), ghostferry.RowStats{}) + state := this.di.StateTracker.Serialize(nil, nil) + this.di.StateTracker = ghostferry.NewStateTrackerFromSerializedState(0, state) + this.di.Run(tables) + + this.Require().NoError(errorHandler.LastError) + this.Equal([]interface{}{int64(1)}, this.receivedIds("resume_zero")) + this.Equal([]interface{}{[]byte("x")}, this.receivedIds("resume_empty")) +} + +func (this *DataIteratorTestSuite) TestNegativePaginationKeyCheckUsesCopyFilter() { + tables := this.loadTablesAfterExecuting([]string{ + "CREATE TABLE gftest.filtered_ids (id bigint NOT NULL PRIMARY KEY, tenant_id bigint NOT NULL)", + "INSERT INTO gftest.filtered_ids VALUES (-1, 2), (0, 1), (1, 1)", + "CREATE TABLE gftest.excluded_ids (id bigint NOT NULL PRIMARY KEY, tenant_id bigint NOT NULL)", + "INSERT INTO gftest.excluded_ids VALUES (-2, 2), (-1, 2)", + }, "filtered_ids", "excluded_ids") + this.di.CursorConfig.BuildSelect = func(columns []string, table *ghostferry.TableSchema, lastKey ghostferry.PaginationKey, batchSize uint64) (squirrel.SelectBuilder, error) { + return ghostferry.DefaultBuildSelect(columns, table, lastKey, batchSize).Where(squirrel.Eq{"tenant_id": 1}), nil + } + errorHandler := &testhelpers.ErrorHandler{} + this.di.ErrorHandler = errorHandler + this.di.Run(tables) + + this.Require().NoError(errorHandler.LastError) + this.Equal([]interface{}{int64(0), int64(1)}, this.receivedIds("filtered_ids")) + this.Empty(this.receivedIds("excluded_ids")) + this.True(this.completedTables()["gftest.excluded_ids"]) +} + +func (this *DataIteratorTestSuite) TestSelectedNegativePaginationKeyIsRejected() { + tables := this.loadTablesAfterExecuting([]string{ + "CREATE TABLE gftest.filtered_negative (id mediumint NOT NULL PRIMARY KEY, tenant_id bigint NOT NULL)", + "INSERT INTO gftest.filtered_negative VALUES (-1, 1), (1, 2)", + }, "filtered_negative") + this.di.CursorConfig.BuildSelect = func(columns []string, table *ghostferry.TableSchema, lastKey ghostferry.PaginationKey, batchSize uint64) (squirrel.SelectBuilder, error) { + return ghostferry.DefaultBuildSelect(columns, table, lastKey, batchSize).Where(squirrel.Eq{"tenant_id": 1}), nil + } + errorHandler := &testhelpers.ErrorHandler{} + this.di.ErrorHandler = errorHandler + this.di.Run(tables) + + this.Require().ErrorContains(errorHandler.LastError, "negative pagination key") + this.Empty(this.receivedIds("filtered_negative")) +} + func (this *DataIteratorTestSuite) TestNegativePaginationKeyIsRejected() { tables := this.loadTablesAfterExecuting([]string{ "CREATE TABLE gftest.negative_ids (id bigint NOT NULL, data TEXT, PRIMARY KEY (id))", @@ -212,6 +278,31 @@ func (this *DataIteratorTestSuite) TestNegativePaginationKeyIsRejected() { this.Require().Contains(errorHandler.LastError.Error(), "negative pagination key") } +func (this *DataIteratorTestSuite) TestNegativePaginationKeyInsertedAfterValidation() { + this.Ferry.Config.UpdatableConfig.DataIterationBatchSize = 1 + this.di.CursorConfig.ReadRetries = 1 + tables := this.loadTablesAfterExecuting([]string{ + "CREATE TABLE gftest.late_negative (id bigint NOT NULL PRIMARY KEY)", + "INSERT INTO gftest.late_negative VALUES (0), (1)", + }, "late_negative") + queries := 0 + this.di.CursorConfig.BuildSelect = func(columns []string, table *ghostferry.TableSchema, lastKey ghostferry.PaginationKey, batchSize uint64) (squirrel.SelectBuilder, error) { + queries++ + if queries == 2 { + if _, err := this.Ferry.SourceDB.Exec("INSERT INTO gftest.late_negative VALUES (-1)"); err != nil { + return squirrel.SelectBuilder{}, err + } + } + return ghostferry.DefaultBuildSelect(columns, table, lastKey, batchSize), nil + } + errorHandler := &testhelpers.ErrorHandler{} + this.di.ErrorHandler = errorHandler + this.di.Run(tables) + + this.Require().ErrorContains(errorHandler.LastError, "negative pagination key") + this.Empty(this.receivedIds("late_negative")) +} + func (this *DataIteratorTestSuite) TestDoneListenerGetsNotifiedWhenDone() { wasNotified := false