diff --git a/CHANGELOG.md b/CHANGELOG.md index f4b59edf..a7681c2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ All notable changes to this project will be documented in this file. - Extend `DMLEvent` with `BinlogCoordinate()` and `ResumableBinlogCoordinate()`. `DMLEventBase` implements both; custom implementations must provide these methods. - Replace `BinlogCoordinate.Compare` with `HasReached`, using file/position ordering or GTID-set containment. Comparing different coordinate types now returns an error instead of panicking. - Compare parsed GTID sets directly during cutover, eliminating per-event GTID serialization and parsing. +- Cursors and `CopyFilter.BuildSelect` use a nil starting key to mean no lower bound; every non-nil checkpoint is exclusive, including 0 and empty binary keys. `StateTracker.LastSuccessfulPaginationKey` now returns nil until a batch has been copied. Custom query builders must handle nil and return rows in ascending pagination-key order. +- `MaxPaginationKeys` now accepts a `*CursorConfig` instead of a database connection so startup validation honors the configured copy filter. - Updated golang to 1.27.2 - Updated go packages: - `github.com/sirupsen/logrus` to v1.10.2 @@ -40,6 +42,8 @@ All notable changes to this project will be documented in this file. - Publish binlog stop requests atomically after recording the stop coordinate, including reads from the control server. - Decode unsigned `MEDIUMINT` binlog values of 2^23 and above correctly. Previously they were sign-extended, so inserts were clamped to 16777215 on the target and updates of already-copied rows failed with an out-of-range error ([#299](https://github.com/Shopify/ghostferry/issues/299)). - Resume from the binlog writer coordinate when the InlineVerifier is disabled. Previously a coordinate left in the state by an earlier verifier-enabled run still held back the resume point, so resuming failed once the source purged those binlogs ([#184](https://github.com/Shopify/ghostferry/issues/184)). +- Copy rows whose pagination key is the minimum value: 0 for integer keys, empty for binary and string keys. The first batch used to select keys strictly greater than the minimum, so these rows were silently skipped, and a table holding only such a row was not copied at all ([#305](https://github.com/Shopify/ghostferry/issues/305)). +- Abort when selected rows have negative signed integer pagination keys instead of silently skipping them. Startup validation uses the configured copy filter, so negative keys belonging to excluded tenants do not block a migration ([#305](https://github.com/Shopify/ghostferry/issues/305)). ## [1.3.1 - 2026-04-15] diff --git a/cursor.go b/cursor.go index 3c83a1ea..67259f9e 100644 --- a/cursor.go +++ b/cursor.go @@ -45,7 +45,7 @@ type CursorConfig struct { ReadRetries int } -// returns a new Cursor with an embedded copy of itself +// NewCursor resumes after startPaginationKey, or starts at the beginning when it is nil. func (c *CursorConfig) NewCursor(table *TableSchema, startPaginationKey, maxPaginationKey PaginationKey) *Cursor { return &Cursor{ CursorConfig: *c, @@ -95,7 +95,7 @@ func (c *Cursor) Each(f func(*RowBatch) error) error { c.ColumnsToSelect = []string{"*"} } - for c.lastSuccessfulPaginationKey.Compare(c.MaxPaginationKey) < 0 { + for c.lastSuccessfulPaginationKey == nil || c.lastSuccessfulPaginationKey.Compare(c.MaxPaginationKey) < 0 { var tx SqlPreparerAndRollbacker var batch *RowBatch var paginationKeypos PaginationKey @@ -136,7 +136,7 @@ func (c *Cursor) Each(f func(*RowBatch) error) error { break } - if paginationKeypos.Compare(c.lastSuccessfulPaginationKey) <= 0 { + if c.lastSuccessfulPaginationKey != nil && paginationKeypos.Compare(c.lastSuccessfulPaginationKey) <= 0 { tx.Rollback() err = fmt.Errorf("new paginationKeypos %s <= lastSuccessfulPaginationKey %s", paginationKeypos.String(), c.lastSuccessfulPaginationKey.String()) c.logger.WithError(err).Errorf("last successful paginationKey position did not advance") @@ -251,6 +251,16 @@ func (c *Cursor) Fetch(db SqlPreparer) (batch *RowBatch, paginationKeypos Pagina return } + // The first query has no lower bound. Recheck its first row in case a + // negative key was inserted after startup validation. + if len(batchData) == 0 && !c.paginationKeyColumn.IsUnsigned && + (c.paginationKeyColumn.Type == schema.TYPE_NUMBER || c.paginationKeyColumn.Type == schema.TYPE_MEDIUM_INT) { + if value, ok := Int64Value(rowData[paginationKeyIndex]); ok && value < 0 { + err = fmt.Errorf("table %s has a negative pagination key in column %s, which Ghostferry cannot copy", c.Table.String(), c.paginationKeyColumn.Name) + return + } + } + batchData = append(batchData, rowData) } @@ -307,9 +317,12 @@ func ScanByteRow(rows *sqlorig.Rows, columnCount int) ([][]byte, error) { func DefaultBuildSelect(columns []string, table *TableSchema, lastPaginationKey PaginationKey, batchSize uint64) squirrel.SelectBuilder { quotedPaginationKey := QuoteField(table.GetPaginationColumn().Name) - return squirrel.Select(columns...). + query := squirrel.Select(columns...). From(QuotedTableName(table)). - Where(squirrel.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()}). Limit(batchSize). OrderBy(quotedPaginationKey) + if lastPaginationKey != nil { + query = query.Where(squirrel.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()}) + } + return query } diff --git a/data_iterator.go b/data_iterator.go index 0586fd8a..567a73a2 100644 --- a/data_iterator.go +++ b/data_iterator.go @@ -39,9 +39,10 @@ func (d *DataIterator) Run(tables []*TableSchema) { } d.logger.WithField("tablesCount", len(tables)).Info("starting data iterator run") - tablesWithData, emptyTables, err := MaxPaginationKeys(d.DB, tables, d.logger) + tablesWithData, emptyTables, err := MaxPaginationKeys(d.CursorConfig, tables, d.logger) if err != nil { d.ErrorHandler.Fatal("data_iterator", err) + return } for _, table := range emptyTables { @@ -84,7 +85,7 @@ func (d *DataIterator) Run(tables []*TableSchema) { } startPaginationKey := d.StateTracker.LastSuccessfulPaginationKey(table.String(), table) - if startPaginationKey.IsMax() { + if startPaginationKey != nil && startPaginationKey.IsMax() { err := fmt.Errorf("%v has been marked as completed but a table iterator has been spawned, this is likely a programmer error which resulted in the inconsistent starting state", table.String()) logger.WithError(err).Error("this is definitely a bug") d.ErrorHandler.Fatal("data_iterator", err) diff --git a/filter.go b/filter.go index e9a23f1e..a14f0c8b 100644 --- a/filter.go +++ b/filter.go @@ -13,8 +13,10 @@ type CopyFilter interface { // here will cause the query to be retried, until the retry limit is // reached, at which point the ferry will be aborted. BuildSelect is passed // the columns to be selected, table being copied, the last pagination key value - // from the previous batch, and the batch size. Call DefaultBuildSelect to - // generate the default query, which may be used as a starting point. + // from the previous batch, and the batch size. Select rows strictly after + // that key in ascending pagination-key order. A nil key means no lower + // bound; it is used for the first batch and startup validation. Call + // DefaultBuildSelect to generate the default query as a starting point. BuildSelect([]string, *TableSchema, PaginationKey, uint64) (sq.SelectBuilder, error) // ApplicableEvent is used to filter events for rows that have been diff --git a/iterative_verifier.go b/iterative_verifier.go index 0ba1b07c..345008b4 100644 --- a/iterative_verifier.go +++ b/iterative_verifier.go @@ -388,7 +388,7 @@ func (v *IterativeVerifier) iterateAllTables(mismatchedPaginationKeyFunc func(st func (v *IterativeVerifier) iterateTableFingerprints(table *TableSchema, mismatchedPaginationKeyFunc func(string, *TableSchema) error) error { // The cursor will stop iterating when it cannot find anymore rows, // so it will not iterate until MaxPaginationKey. - cursor := v.CursorConfig.NewCursorWithoutRowLock(table, MinPaginationKey(table.GetPaginationColumn()), MaxPaginationKey(table.GetPaginationColumn())) + cursor := v.CursorConfig.NewCursorWithoutRowLock(table, nil, MaxPaginationKey(table.GetPaginationColumn())) // It only needs the PaginationKeys, not the entire row. cursor.ColumnsToSelect = []string{fmt.Sprintf("`%s`", table.GetPaginationColumn().Name)} diff --git a/sharding/filter.go b/sharding/filter.go index a29145b4..3fe2026d 100644 --- a/sharding/filter.go +++ b/sharding/filter.go @@ -43,12 +43,14 @@ func (f *ShardedCopyFilter) BuildSelect(columns []string, table *ghostferry.Tabl // the model for a single tenant in a multitenant database, e.g. a shop on // a multitenant commerce platform. // - // It is necessary to use two WHERE conditions on quotedPaginationKey so the second batch will be empty. // No LIMIT clause is necessary since at most one row is present. - return sq.Select(columns...). + query := sq.Select(columns...). From(quotedTable + " USE INDEX (PRIMARY)"). - Where(sq.Eq{quotedPaginationKey: f.ShardingValue}). - Where(sq.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()}), nil + Where(sq.Eq{quotedPaginationKey: f.ShardingValue}) + if lastPaginationKey != nil { + query = query.Where(sq.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()}) + } + return query, nil } joinTables, exists := f.JoinedTables[table.Name] @@ -84,12 +86,18 @@ func (f *ShardedCopyFilter) BuildSelect(columns []string, table *ghostferry.Tabl selectPaginationKeys += " " + f.shardingKeyIndexHint(table, indexHint) } - selectPaginationKeys += " WHERE " + quotedShardingKey + " = ? AND " + quotedPaginationKey + " > ?" + - " ORDER BY " + quotedPaginationKey + " LIMIT " + strconv.Itoa(int(batchSize)) + selectPaginationKeys += " WHERE " + quotedShardingKey + " = ?" + args := make([]interface{}, 0, 2) + args = append(args, f.ShardingValue) + if lastPaginationKey != nil { + selectPaginationKeys += " AND " + quotedPaginationKey + " > ?" + args = append(args, lastPaginationKey.SQLValue()) + } + selectPaginationKeys += " ORDER BY " + quotedPaginationKey + " LIMIT " + strconv.Itoa(int(batchSize)) return sq.Select(columns...). From(quotedTable). - Join("("+selectPaginationKeys+") AS `batch` USING("+quotedPaginationKey+")", f.ShardingValue, lastPaginationKey.SQLValue()), nil + Join("("+selectPaginationKeys+") AS `batch` USING("+quotedPaginationKey+")", args...), nil } // This is a "joined table". It is the only supported type of table that @@ -120,12 +128,19 @@ func (f *ShardedCopyFilter) BuildSelect(columns []string, table *ghostferry.Tabl // it to a batch, then select the rest of the columns. var clauses []string var args []interface{} + const firstPage = "SELECT `%[1]s` AS sharding_join_alias FROM `%[2]s`.`%[3]s` WHERE `%[4]s` = ?" + pattern := firstPage + if lastPaginationKey != nil { + pattern = firstPage + " AND `%[1]s` > ?" + } for _, joinTable := range joinTables { - pattern := "SELECT `%s` AS sharding_join_alias FROM `%s`.`%s` WHERE `%s` = ? AND `%s` > ?" - sql := fmt.Sprintf(pattern, joinTable.JoinColumn, table.Schema, joinTable.TableName, f.ShardingKey, joinTable.JoinColumn) + sql := fmt.Sprintf(pattern, joinTable.JoinColumn, table.Schema, joinTable.TableName, f.ShardingKey) clauses = append(clauses, sql) - args = append(args, f.ShardingValue, lastPaginationKey.SQLValue()) + args = append(args, f.ShardingValue) + if lastPaginationKey != nil { + args = append(args, lastPaginationKey.SQLValue()) + } } subquery := strings.Join(clauses, " UNION DISTINCT ") diff --git a/sharding/test/trivial_integration_test.go b/sharding/test/trivial_integration_test.go index bd4c2898..6e7b3431 100644 --- a/sharding/test/trivial_integration_test.go +++ b/sharding/test/trivial_integration_test.go @@ -5,11 +5,13 @@ import ( "sync/atomic" "testing" + "github.com/Shopify/ghostferry" sql "github.com/Shopify/ghostferry/sqlwrapper" "github.com/Shopify/ghostferry/sharding" "github.com/Shopify/ghostferry/testhelpers" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func setupSingleTableDatabase(f *testhelpers.TestFerry, sourceDB, targetDB *sql.DB) { @@ -55,6 +57,76 @@ func TestSelectiveCopyDataWithoutAnyWritesToSource(t *testing.T) { assert.Equal(t, 333, len(rows)) } +func TestSelectiveCopyIncludesMinimumKeysAndIgnoresExcludedNegativeKeys(t *testing.T) { + ferry := testhelpers.NewTestFerry() + ferry.Config.UpdatableConfig.DataIterationBatchSize = 1 + ferry.Config.DataIterationConcurrency = 1 + ferry.Config.VerifierType = ghostferry.VerifierTypeInline + ferry.Config.TableFilter = &testhelpers.TestTableFilter{ + DbsFunc: testhelpers.DbApplicabilityFilter([]string{"gftest"}), + } + ferry.Config.CopyFilter = &sharding.ShardedCopyFilter{ + ShardingKey: "tenant_id", + ShardingValue: int64(0), + PrimaryKeyTables: map[string]struct{}{ + "tenants": {}, + }, + JoinedTables: map[string][]sharding.JoinTable{ + "shared_rows": {{TableName: "links", JoinColumn: "shared_id"}}, + }, + } + require.NoError(t, ferry.Config.ValidateConfig()) + errorHandler := &testhelpers.ErrorHandler{} + ferry.ErrorHandler = errorHandler + testcase := &testhelpers.IntegrationTestCase{ + T: t, + Ferry: ferry, + SetupAction: func(_ *testhelpers.TestFerry, sourceDB, targetDB *sql.DB) { + for _, db := range []*sql.DB{sourceDB, targetDB} { + for _, statement := range []string{ + "CREATE DATABASE gftest", + "CREATE TABLE gftest.normal_rows (id bigint PRIMARY KEY, tenant_id bigint NOT NULL, data TEXT, KEY tenant_idx (tenant_id, id))", + "CREATE TABLE gftest.tenants (tenant_id bigint PRIMARY KEY, data TEXT)", + "CREATE TABLE gftest.links (id bigint PRIMARY KEY, tenant_id bigint NOT NULL, shared_id bigint NOT NULL, KEY tenant_idx (tenant_id, id))", + "CREATE TABLE gftest.shared_rows (id bigint PRIMARY KEY, data TEXT)", + "CREATE TABLE gftest.excluded_rows (id bigint PRIMARY KEY, tenant_id bigint NOT NULL, KEY tenant_idx (tenant_id, id))", + } { + _, err := db.Exec(statement) + require.NoError(t, err) + } + } + for _, statement := range []string{ + "INSERT INTO gftest.normal_rows VALUES (-1, 2, 'excluded'), (0, 0, 'zero'), (1, 0, 'one')", + "INSERT INTO gftest.tenants VALUES (-1, 'excluded'), (0, 'zero'), (2, 'excluded')", + "INSERT INTO gftest.links VALUES (-1, 2, -1), (0, 0, 0), (1, 0, 1)", + "INSERT INTO gftest.shared_rows VALUES (-1, 'excluded'), (0, 'zero'), (1, 'one')", + "INSERT INTO gftest.excluded_rows VALUES (-2, 2), (-1, 2)", + } { + _, err := sourceDB.Exec(statement) + require.NoError(t, err) + } + }, + } + defer testcase.Teardown() + testcase.CopyData() + require.NoError(t, errorHandler.LastError) + + for _, check := range []struct { + query string + want string + }{ + {"SELECT GROUP_CONCAT(CONCAT(id, ':', data) ORDER BY id) FROM gftest.normal_rows", "0:zero,1:one"}, + {"SELECT GROUP_CONCAT(CONCAT(tenant_id, ':', data) ORDER BY tenant_id) FROM gftest.tenants", "0:zero"}, + {"SELECT GROUP_CONCAT(CONCAT(id, ':', shared_id) ORDER BY id) FROM gftest.links", "0:0,1:1"}, + {"SELECT GROUP_CONCAT(CONCAT(id, ':', data) ORDER BY id) FROM gftest.shared_rows", "0:zero,1:one"}, + {"SELECT COUNT(*) FROM gftest.excluded_rows", "0"}, + } { + var got string + require.NoError(t, testcase.TargetDB.QueryRow(check.query).Scan(&got)) + assert.Equal(t, check.want, got, check.query) + } +} + func TestSelectiveCopyDataWithInsertLoadOnOtherTenants(t *testing.T) { testcase := &testhelpers.IntegrationTestCase{ T: t, diff --git a/state_tracker.go b/state_tracker.go index ecb33b47..13aee131 100644 --- a/state_tracker.go +++ b/state_tracker.go @@ -371,6 +371,7 @@ func (s *StateTracker) RowStatsWrittenPerTable() map[string]RowStats { return d } +// LastSuccessfulPaginationKey returns nil until a batch has been copied. func (s *StateTracker) LastSuccessfulPaginationKey(table string, tableSchema *TableSchema) PaginationKey { s.CopyRWMutex.RLock() defer s.CopyRWMutex.RUnlock() @@ -380,12 +381,7 @@ func (s *StateTracker) LastSuccessfulPaginationKey(table string, tableSchema *Ta return MaxPaginationKey(tableSchema.GetPaginationColumn()) } - paginationKey, found := s.lastSuccessfulPaginationKeys[table] - if !found { - return MinPaginationKey(tableSchema.GetPaginationColumn()) - } - - return paginationKey + return s.lastSuccessfulPaginationKeys[table] } func (s *StateTracker) MarkTableAsCompleted(table string) { diff --git a/table_schema_cache.go b/table_schema_cache.go index a5e2f389..e0e0baad 100644 --- a/table_schema_cache.go +++ b/table_schema_cache.go @@ -125,14 +125,32 @@ func QuotedTableNameFromString(database, table string) string { return fmt.Sprintf("`%s`.`%s`", database, table) } -func MaxPaginationKeys(db *sql.DB, tables []*TableSchema, logger Logger) (map[*TableSchema]PaginationKey, []*TableSchema, error) { +// MaxPaginationKeys validates selected signed keys and returns table-wide upper +// bounds for iteration. The first-key probe uses the same filter as the cursor. +func MaxPaginationKeys(cursorConfig *CursorConfig, tables []*TableSchema, logger Logger) (map[*TableSchema]PaginationKey, []*TableSchema, error) { tablesWithData := make(map[*TableSchema]PaginationKey) emptyTables := make([]*TableSchema, 0, len(tables)) for _, table := range tables { logger := logger.WithField("table", table.String()) - maxPaginationKey, maxPaginationKeyExists, err := maxPaginationKey(db, table) + column := table.GetPaginationColumn() + if !column.IsUnsigned && (column.Type == schema.TYPE_NUMBER || column.Type == schema.TYPE_MEDIUM_INT) { + firstKey, found, err := firstSignedPaginationKey(cursorConfig, table) + if err != nil { + logger.WithError(err).Errorf("failed to check first selected pagination key %s", column.Name) + return tablesWithData, emptyTables, err + } + if !found { + emptyTables = append(emptyTables, table) + continue + } + if firstKey < 0 { + return tablesWithData, emptyTables, fmt.Errorf("table %s has a negative pagination key in column %s, which Ghostferry cannot copy", table.String(), column.Name) + } + } + + maxPaginationKey, maxPaginationKeyExists, err := maxPaginationKey(cursorConfig.DB, table) if err != nil { logger.WithError(err).Errorf("failed to get max primary key %s", table.GetPaginationColumn().Name) return tablesWithData, emptyTables, err @@ -419,6 +437,31 @@ func showTablesFrom(c *sql.DB, dbname string) ([]string, error) { return tables, nil } +func firstSignedPaginationKey(cursorConfig *CursorConfig, table *TableSchema) (int64, bool, error) { + columns := []string{QuoteField(table.GetPaginationColumn().Name)} + var builder sq.SelectBuilder + if cursorConfig.BuildSelect == nil { + builder = DefaultBuildSelect(columns, table, nil, 1) + } else { + var err error + builder, err = cursorConfig.BuildSelect(columns, table, nil, 1) + if err != nil { + return 0, false, err + } + } + query, args, err := builder.ToSql() + if err != nil { + return 0, false, err + } + + var value int64 + err = cursorConfig.DB.QueryRow(query, args...).Scan(&value) + if err == sqlorig.ErrNoRows { + return 0, false, nil + } + return value, err == nil, err +} + func maxPaginationKey(db *sql.DB, table *TableSchema) (PaginationKey, bool, error) { primaryKeyColumn := table.GetPaginationColumn() paginationKeyName := QuoteField(primaryKeyColumn.Name) diff --git a/test/go/data_iterator_test.go b/test/go/data_iterator_test.go index f30d0cfb..8710320c 100644 --- a/test/go/data_iterator_test.go +++ b/test/go/data_iterator_test.go @@ -5,6 +5,7 @@ import ( "sync" "testing" + "github.com/Masterminds/squirrel" "github.com/Shopify/ghostferry" "github.com/Shopify/ghostferry/testhelpers" "github.com/stretchr/testify/suite" @@ -56,7 +57,7 @@ func (this *DataIteratorTestSuite) SetupTest() { BatchSizePerTableOverride: config.DataIterationBatchSizePerTableOverride, ReadRetries: config.DBReadRetries, }, - StateTracker: ghostferry.NewStateTracker(config.DataIterationConcurrency * 10), + StateTracker: ghostferry.NewStateTracker(config.DataIterationConcurrency * 10), TargetPaginationKeys: &sync.Map{}, } @@ -153,6 +154,155 @@ func (this *DataIteratorTestSuite) TestExistingRowsAreIterated() { ) } +func (this *DataIteratorTestSuite) loadTablesAfterExecuting(statements []string, names ...string) []*ghostferry.TableSchema { + for _, statement := range statements { + _, err := this.Ferry.SourceDB.Exec(statement) + this.Require().Nil(err) + } + + tableFilter := &testhelpers.TestTableFilter{ + DbsFunc: testhelpers.DbApplicabilityFilter([]string{testhelpers.TestSchemaName}), + } + cache, err := ghostferry.LoadTables(this.Ferry.SourceDB, tableFilter, nil, nil, nil, nil) + this.Require().Nil(err) + + tables := make([]*ghostferry.TableSchema, len(names)) + for i, name := range names { + tables[i] = cache.Get(testhelpers.TestSchemaName, name) + this.Require().NotNil(tables[i], name) + } + return tables +} + +func (this *DataIteratorTestSuite) receivedIds(table string) []interface{} { + ids := []interface{}{} + for _, row := range this.receivedRows[table] { + ids = append(ids, row[0]) + } + return ids +} + +func (this *DataIteratorTestSuite) TestRowsAtTheMinimumPaginationKeyAreIterated() { + this.Ferry.Config.UpdatableConfig.DataIterationBatchSize = 1 + this.di.Concurrency = 1 + errorHandler := &testhelpers.ErrorHandler{} + this.di.ErrorHandler = errorHandler + + tables := this.loadTablesAfterExecuting([]string{ + "CREATE TABLE gftest.zero_and_up (id bigint NOT NULL, data TEXT, PRIMARY KEY (id))", + "INSERT INTO gftest.zero_and_up VALUES (0, 'a'), (1, 'b'), (2, 'c')", + "CREATE TABLE gftest.only_zero (id bigint unsigned NOT NULL, data TEXT, PRIMARY KEY (id))", + "INSERT INTO gftest.only_zero VALUES (0, 'a')", + "CREATE TABLE gftest.empty_binary_key (id varbinary(16) NOT NULL, data TEXT, PRIMARY KEY (id))", + "INSERT INTO gftest.empty_binary_key VALUES ('', 'a'), ('x', 'b')", + }, "zero_and_up", "only_zero", "empty_binary_key") + + this.di.Run(tables) + this.Require().NoError(errorHandler.LastError) + + this.Equal([]interface{}{int64(0), int64(1), int64(2)}, this.receivedIds("zero_and_up")) + this.Equal([]interface{}{int64(0)}, this.receivedIds("only_zero")) + this.Equal([]interface{}{[]byte(""), []byte("x")}, this.receivedIds("empty_binary_key")) +} + +func (this *DataIteratorTestSuite) TestResumeAfterMinimumPaginationKey() { + this.Ferry.Config.UpdatableConfig.DataIterationBatchSize = 1 + this.di.Concurrency = 1 + errorHandler := &testhelpers.ErrorHandler{} + this.di.ErrorHandler = errorHandler + tables := this.loadTablesAfterExecuting([]string{ + "CREATE TABLE gftest.resume_zero (id bigint NOT NULL PRIMARY KEY)", + "INSERT INTO gftest.resume_zero VALUES (0), (1)", + "CREATE TABLE gftest.resume_empty (id varbinary(16) NOT NULL PRIMARY KEY)", + "INSERT INTO gftest.resume_empty VALUES (''), ('x')", + }, "resume_zero", "resume_empty") + + this.di.StateTracker.UpdateLastSuccessfulPaginationKey("gftest.resume_zero", ghostferry.NewUint64Key(0), ghostferry.RowStats{}) + this.di.StateTracker.UpdateLastSuccessfulPaginationKey("gftest.resume_empty", ghostferry.NewBinaryKey(nil), ghostferry.RowStats{}) + state := this.di.StateTracker.Serialize(nil, nil) + this.di.StateTracker = ghostferry.NewStateTrackerFromSerializedState(0, state) + this.di.Run(tables) + + this.Require().NoError(errorHandler.LastError) + this.Equal([]interface{}{int64(1)}, this.receivedIds("resume_zero")) + this.Equal([]interface{}{[]byte("x")}, this.receivedIds("resume_empty")) +} + +func (this *DataIteratorTestSuite) TestNegativePaginationKeyCheckUsesCopyFilter() { + tables := this.loadTablesAfterExecuting([]string{ + "CREATE TABLE gftest.filtered_ids (id bigint NOT NULL PRIMARY KEY, tenant_id bigint NOT NULL)", + "INSERT INTO gftest.filtered_ids VALUES (-1, 2), (0, 1), (1, 1)", + "CREATE TABLE gftest.excluded_ids (id bigint NOT NULL PRIMARY KEY, tenant_id bigint NOT NULL)", + "INSERT INTO gftest.excluded_ids VALUES (-2, 2), (-1, 2)", + }, "filtered_ids", "excluded_ids") + this.di.CursorConfig.BuildSelect = func(columns []string, table *ghostferry.TableSchema, lastKey ghostferry.PaginationKey, batchSize uint64) (squirrel.SelectBuilder, error) { + return ghostferry.DefaultBuildSelect(columns, table, lastKey, batchSize).Where(squirrel.Eq{"tenant_id": 1}), nil + } + errorHandler := &testhelpers.ErrorHandler{} + this.di.ErrorHandler = errorHandler + this.di.Run(tables) + + this.Require().NoError(errorHandler.LastError) + this.Equal([]interface{}{int64(0), int64(1)}, this.receivedIds("filtered_ids")) + this.Empty(this.receivedIds("excluded_ids")) + this.True(this.completedTables()["gftest.excluded_ids"]) +} + +func (this *DataIteratorTestSuite) TestSelectedNegativePaginationKeyIsRejected() { + tables := this.loadTablesAfterExecuting([]string{ + "CREATE TABLE gftest.filtered_negative (id mediumint NOT NULL PRIMARY KEY, tenant_id bigint NOT NULL)", + "INSERT INTO gftest.filtered_negative VALUES (-1, 1), (1, 2)", + }, "filtered_negative") + this.di.CursorConfig.BuildSelect = func(columns []string, table *ghostferry.TableSchema, lastKey ghostferry.PaginationKey, batchSize uint64) (squirrel.SelectBuilder, error) { + return ghostferry.DefaultBuildSelect(columns, table, lastKey, batchSize).Where(squirrel.Eq{"tenant_id": 1}), nil + } + errorHandler := &testhelpers.ErrorHandler{} + this.di.ErrorHandler = errorHandler + this.di.Run(tables) + + this.Require().ErrorContains(errorHandler.LastError, "negative pagination key") + this.Empty(this.receivedIds("filtered_negative")) +} + +func (this *DataIteratorTestSuite) TestNegativePaginationKeyIsRejected() { + tables := this.loadTablesAfterExecuting([]string{ + "CREATE TABLE gftest.negative_ids (id bigint NOT NULL, data TEXT, PRIMARY KEY (id))", + "INSERT INTO gftest.negative_ids VALUES (-1, 'a'), (1, 'b')", + }, "negative_ids") + + errorHandler := &testhelpers.ErrorHandler{} + this.di.ErrorHandler = errorHandler + this.di.Run(tables) + + this.Require().NotNil(errorHandler.LastError) + this.Require().Contains(errorHandler.LastError.Error(), "negative pagination key") +} + +func (this *DataIteratorTestSuite) TestNegativePaginationKeyInsertedAfterValidation() { + this.Ferry.Config.UpdatableConfig.DataIterationBatchSize = 1 + this.di.CursorConfig.ReadRetries = 1 + tables := this.loadTablesAfterExecuting([]string{ + "CREATE TABLE gftest.late_negative (id bigint NOT NULL PRIMARY KEY)", + "INSERT INTO gftest.late_negative VALUES (0), (1)", + }, "late_negative") + queries := 0 + this.di.CursorConfig.BuildSelect = func(columns []string, table *ghostferry.TableSchema, lastKey ghostferry.PaginationKey, batchSize uint64) (squirrel.SelectBuilder, error) { + queries++ + if queries == 2 { + if _, err := this.Ferry.SourceDB.Exec("INSERT INTO gftest.late_negative VALUES (-1)"); err != nil { + return squirrel.SelectBuilder{}, err + } + } + return ghostferry.DefaultBuildSelect(columns, table, lastKey, batchSize), nil + } + errorHandler := &testhelpers.ErrorHandler{} + this.di.ErrorHandler = errorHandler + this.di.Run(tables) + + this.Require().ErrorContains(errorHandler.LastError, "negative pagination key") + this.Empty(this.receivedIds("late_negative")) +} + func (this *DataIteratorTestSuite) TestDoneListenerGetsNotifiedWhenDone() { wasNotified := false