Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ All notable changes to this project will be documented in this file.
- Extend `DMLEvent` with `BinlogCoordinate()` and `ResumableBinlogCoordinate()`. `DMLEventBase` implements both; custom implementations must provide these methods.
- Replace `BinlogCoordinate.Compare` with `HasReached`, using file/position ordering or GTID-set containment. Comparing different coordinate types now returns an error instead of panicking.
- Compare parsed GTID sets directly during cutover, eliminating per-event GTID serialization and parsing.
- Cursors and `CopyFilter.BuildSelect` use a nil starting key to mean no lower bound; every non-nil checkpoint is exclusive, including 0 and empty binary keys. `StateTracker.LastSuccessfulPaginationKey` now returns nil until a batch has been copied. Custom query builders must handle nil and return rows in ascending pagination-key order.
- `MaxPaginationKeys` now accepts a `*CursorConfig` instead of a database connection so startup validation honors the configured copy filter.
- Updated golang to 1.27.2
- Updated go packages:
- `github.com/sirupsen/logrus` to v1.10.2
Expand All @@ -40,6 +42,8 @@ All notable changes to this project will be documented in this file.
- Publish binlog stop requests atomically after recording the stop coordinate, including reads from the control server.
- Decode unsigned `MEDIUMINT` binlog values of 2^23 and above correctly. Previously they were sign-extended, so inserts were clamped to 16777215 on the target and updates of already-copied rows failed with an out-of-range error ([#299](https://github.com/Shopify/ghostferry/issues/299)).
- Resume from the binlog writer coordinate when the InlineVerifier is disabled. Previously a coordinate left in the state by an earlier verifier-enabled run still held back the resume point, so resuming failed once the source purged those binlogs ([#184](https://github.com/Shopify/ghostferry/issues/184)).
- Copy rows whose pagination key is the minimum value: 0 for integer keys, empty for binary and string keys. The first batch used to select keys strictly greater than the minimum, so these rows were silently skipped, and a table holding only such a row was not copied at all ([#305](https://github.com/Shopify/ghostferry/issues/305)).
- Abort when selected rows have negative signed integer pagination keys instead of silently skipping them. Startup validation uses the configured copy filter, so negative keys belonging to excluded tenants do not block a migration ([#305](https://github.com/Shopify/ghostferry/issues/305)).

## [1.3.1 - 2026-04-15]

Expand Down
23 changes: 18 additions & 5 deletions cursor.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ type CursorConfig struct {
ReadRetries int
}

// returns a new Cursor with an embedded copy of itself
// NewCursor resumes after startPaginationKey, or starts at the beginning when it is nil.
func (c *CursorConfig) NewCursor(table *TableSchema, startPaginationKey, maxPaginationKey PaginationKey) *Cursor {
return &Cursor{
CursorConfig: *c,
Expand Down Expand Up @@ -95,7 +95,7 @@ func (c *Cursor) Each(f func(*RowBatch) error) error {
c.ColumnsToSelect = []string{"*"}
}

for c.lastSuccessfulPaginationKey.Compare(c.MaxPaginationKey) < 0 {
for c.lastSuccessfulPaginationKey == nil || c.lastSuccessfulPaginationKey.Compare(c.MaxPaginationKey) < 0 {
var tx SqlPreparerAndRollbacker
var batch *RowBatch
var paginationKeypos PaginationKey
Expand Down Expand Up @@ -136,7 +136,7 @@ func (c *Cursor) Each(f func(*RowBatch) error) error {
break
}

if paginationKeypos.Compare(c.lastSuccessfulPaginationKey) <= 0 {
if c.lastSuccessfulPaginationKey != nil && paginationKeypos.Compare(c.lastSuccessfulPaginationKey) <= 0 {
tx.Rollback()
err = fmt.Errorf("new paginationKeypos %s <= lastSuccessfulPaginationKey %s", paginationKeypos.String(), c.lastSuccessfulPaginationKey.String())
c.logger.WithError(err).Errorf("last successful paginationKey position did not advance")
Expand Down Expand Up @@ -251,6 +251,16 @@ func (c *Cursor) Fetch(db SqlPreparer) (batch *RowBatch, paginationKeypos Pagina
return
}

// The first query has no lower bound. Recheck its first row in case a
// negative key was inserted after startup validation.
if len(batchData) == 0 && !c.paginationKeyColumn.IsUnsigned &&

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My bot found an edge case (quite unlikely?) where a negative pagination key can cause silent data loss here

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we need this check if c.lastSuccessfulPaginationKey is not nil? - it means the following batches are not negative right?

And even though this is not a terminal error, this will still retry 5 times because the whole logic is wrapped in WithRetries

(c.paginationKeyColumn.Type == schema.TYPE_NUMBER || c.paginationKeyColumn.Type == schema.TYPE_MEDIUM_INT) {
if value, ok := Int64Value(rowData[paginationKeyIndex]); ok && value < 0 {
err = fmt.Errorf("table %s has a negative pagination key in column %s, which Ghostferry cannot copy", c.Table.String(), c.paginationKeyColumn.Name)
return
}
}

batchData = append(batchData, rowData)
}

Expand Down Expand Up @@ -307,9 +317,12 @@ func ScanByteRow(rows *sqlorig.Rows, columnCount int) ([][]byte, error) {
func DefaultBuildSelect(columns []string, table *TableSchema, lastPaginationKey PaginationKey, batchSize uint64) squirrel.SelectBuilder {
quotedPaginationKey := QuoteField(table.GetPaginationColumn().Name)

return squirrel.Select(columns...).
query := squirrel.Select(columns...).
From(QuotedTableName(table)).
Where(squirrel.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()}).
Limit(batchSize).
OrderBy(quotedPaginationKey)
if lastPaginationKey != nil {
query = query.Where(squirrel.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()})
}
return query
}
5 changes: 3 additions & 2 deletions data_iterator.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,10 @@ func (d *DataIterator) Run(tables []*TableSchema) {
}

d.logger.WithField("tablesCount", len(tables)).Info("starting data iterator run")
tablesWithData, emptyTables, err := MaxPaginationKeys(d.DB, tables, d.logger)
tablesWithData, emptyTables, err := MaxPaginationKeys(d.CursorConfig, tables, d.logger)
if err != nil {
d.ErrorHandler.Fatal("data_iterator", err)
return
}

for _, table := range emptyTables {
Expand Down Expand Up @@ -84,7 +85,7 @@ func (d *DataIterator) Run(tables []*TableSchema) {
}

startPaginationKey := d.StateTracker.LastSuccessfulPaginationKey(table.String(), table)
if startPaginationKey.IsMax() {
if startPaginationKey != nil && startPaginationKey.IsMax() {
err := fmt.Errorf("%v has been marked as completed but a table iterator has been spawned, this is likely a programmer error which resulted in the inconsistent starting state", table.String())
logger.WithError(err).Error("this is definitely a bug")
d.ErrorHandler.Fatal("data_iterator", err)
Expand Down
6 changes: 4 additions & 2 deletions filter.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@ type CopyFilter interface {
// here will cause the query to be retried, until the retry limit is
// reached, at which point the ferry will be aborted. BuildSelect is passed
// the columns to be selected, table being copied, the last pagination key value
// from the previous batch, and the batch size. Call DefaultBuildSelect to
// generate the default query, which may be used as a starting point.
// from the previous batch, and the batch size. Select rows strictly after
// that key in ascending pagination-key order. A nil key means no lower
// bound; it is used for the first batch and startup validation. Call
// DefaultBuildSelect to generate the default query as a starting point.
BuildSelect([]string, *TableSchema, PaginationKey, uint64) (sq.SelectBuilder, error)

// ApplicableEvent is used to filter events for rows that have been
Expand Down
2 changes: 1 addition & 1 deletion iterative_verifier.go
Original file line number Diff line number Diff line change
Expand Up @@ -388,7 +388,7 @@ func (v *IterativeVerifier) iterateAllTables(mismatchedPaginationKeyFunc func(st
func (v *IterativeVerifier) iterateTableFingerprints(table *TableSchema, mismatchedPaginationKeyFunc func(string, *TableSchema) error) error {
// The cursor will stop iterating when it cannot find anymore rows,
// so it will not iterate until MaxPaginationKey.
cursor := v.CursorConfig.NewCursorWithoutRowLock(table, MinPaginationKey(table.GetPaginationColumn()), MaxPaginationKey(table.GetPaginationColumn()))
cursor := v.CursorConfig.NewCursorWithoutRowLock(table, nil, MaxPaginationKey(table.GetPaginationColumn()))

// It only needs the PaginationKeys, not the entire row.
cursor.ColumnsToSelect = []string{fmt.Sprintf("`%s`", table.GetPaginationColumn().Name)}
Expand Down
35 changes: 25 additions & 10 deletions sharding/filter.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,14 @@ func (f *ShardedCopyFilter) BuildSelect(columns []string, table *ghostferry.Tabl
// the model for a single tenant in a multitenant database, e.g. a shop on
// a multitenant commerce platform.
//
// It is necessary to use two WHERE conditions on quotedPaginationKey so the second batch will be empty.
// No LIMIT clause is necessary since at most one row is present.
return sq.Select(columns...).
query := sq.Select(columns...).
From(quotedTable + " USE INDEX (PRIMARY)").
Where(sq.Eq{quotedPaginationKey: f.ShardingValue}).
Where(sq.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()}), nil
Where(sq.Eq{quotedPaginationKey: f.ShardingValue})
if lastPaginationKey != nil {
query = query.Where(sq.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()})
}
return query, nil
}

joinTables, exists := f.JoinedTables[table.Name]
Expand Down Expand Up @@ -84,12 +86,18 @@ func (f *ShardedCopyFilter) BuildSelect(columns []string, table *ghostferry.Tabl
selectPaginationKeys += " " + f.shardingKeyIndexHint(table, indexHint)
}

selectPaginationKeys += " WHERE " + quotedShardingKey + " = ? AND " + quotedPaginationKey + " > ?" +
" ORDER BY " + quotedPaginationKey + " LIMIT " + strconv.Itoa(int(batchSize))
selectPaginationKeys += " WHERE " + quotedShardingKey + " = ?"
args := make([]interface{}, 0, 2)
args = append(args, f.ShardingValue)
if lastPaginationKey != nil {
selectPaginationKeys += " AND " + quotedPaginationKey + " > ?"
args = append(args, lastPaginationKey.SQLValue())
}
selectPaginationKeys += " ORDER BY " + quotedPaginationKey + " LIMIT " + strconv.Itoa(int(batchSize))

return sq.Select(columns...).
From(quotedTable).
Join("("+selectPaginationKeys+") AS `batch` USING("+quotedPaginationKey+")", f.ShardingValue, lastPaginationKey.SQLValue()), nil
Join("("+selectPaginationKeys+") AS `batch` USING("+quotedPaginationKey+")", args...), nil

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we add .OrderBy(quotedPaginationKey) after this .Join(...)?

selectPaginationKeys already sorts the keys, but this JOIN fetches the full rows for those keys and can return them in a different order. The cursor depends on the returned rows being sorted: it checks the first row for a negative key and uses the last row’s key as the next checkpoint.

I reproduced this on MySQL 8.0.32 with batch size 4. Startup sees keys 10, 20, 30, 40. If -1 is inserted before the first batch, this query returns [10, 20, 30, -1]. The negative-key check sees only 10 and passes. The checkpoint becomes -1, which wraps to the largest unsigned integer, so the cursor reports success without copying 40.

Adding ordering to this outer query makes it return [-1, 10, 20, 30], so the existing check rejects the negative key. Could we cover this with a regression test using a batch size greater than one?

}

// This is a "joined table". It is the only supported type of table that
Expand Down Expand Up @@ -120,12 +128,19 @@ func (f *ShardedCopyFilter) BuildSelect(columns []string, table *ghostferry.Tabl
// it to a batch, then select the rest of the columns.
var clauses []string
var args []interface{}
const firstPage = "SELECT `%[1]s` AS sharding_join_alias FROM `%[2]s`.`%[3]s` WHERE `%[4]s` = ?"
pattern := firstPage
if lastPaginationKey != nil {
pattern = firstPage + " AND `%[1]s` > ?"
}

for _, joinTable := range joinTables {
pattern := "SELECT `%s` AS sharding_join_alias FROM `%s`.`%s` WHERE `%s` = ? AND `%s` > ?"
sql := fmt.Sprintf(pattern, joinTable.JoinColumn, table.Schema, joinTable.TableName, f.ShardingKey, joinTable.JoinColumn)
sql := fmt.Sprintf(pattern, joinTable.JoinColumn, table.Schema, joinTable.TableName, f.ShardingKey)
clauses = append(clauses, sql)
args = append(args, f.ShardingValue, lastPaginationKey.SQLValue())
args = append(args, f.ShardingValue)
if lastPaginationKey != nil {
args = append(args, lastPaginationKey.SQLValue())
}
}

subquery := strings.Join(clauses, " UNION DISTINCT ")
Expand Down
72 changes: 72 additions & 0 deletions sharding/test/trivial_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@ import (
"sync/atomic"
"testing"

"github.com/Shopify/ghostferry"
sql "github.com/Shopify/ghostferry/sqlwrapper"

"github.com/Shopify/ghostferry/sharding"
"github.com/Shopify/ghostferry/testhelpers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func setupSingleTableDatabase(f *testhelpers.TestFerry, sourceDB, targetDB *sql.DB) {
Expand Down Expand Up @@ -55,6 +57,76 @@ func TestSelectiveCopyDataWithoutAnyWritesToSource(t *testing.T) {
assert.Equal(t, 333, len(rows))
}

func TestSelectiveCopyIncludesMinimumKeysAndIgnoresExcludedNegativeKeys(t *testing.T) {
ferry := testhelpers.NewTestFerry()
ferry.Config.UpdatableConfig.DataIterationBatchSize = 1
ferry.Config.DataIterationConcurrency = 1
ferry.Config.VerifierType = ghostferry.VerifierTypeInline
ferry.Config.TableFilter = &testhelpers.TestTableFilter{
DbsFunc: testhelpers.DbApplicabilityFilter([]string{"gftest"}),
}
ferry.Config.CopyFilter = &sharding.ShardedCopyFilter{
ShardingKey: "tenant_id",
ShardingValue: int64(0),
PrimaryKeyTables: map[string]struct{}{
"tenants": {},
},
JoinedTables: map[string][]sharding.JoinTable{
"shared_rows": {{TableName: "links", JoinColumn: "shared_id"}},
},
}
require.NoError(t, ferry.Config.ValidateConfig())
errorHandler := &testhelpers.ErrorHandler{}
ferry.ErrorHandler = errorHandler
testcase := &testhelpers.IntegrationTestCase{
T: t,
Ferry: ferry,
SetupAction: func(_ *testhelpers.TestFerry, sourceDB, targetDB *sql.DB) {
for _, db := range []*sql.DB{sourceDB, targetDB} {
for _, statement := range []string{
"CREATE DATABASE gftest",
"CREATE TABLE gftest.normal_rows (id bigint PRIMARY KEY, tenant_id bigint NOT NULL, data TEXT, KEY tenant_idx (tenant_id, id))",
"CREATE TABLE gftest.tenants (tenant_id bigint PRIMARY KEY, data TEXT)",
"CREATE TABLE gftest.links (id bigint PRIMARY KEY, tenant_id bigint NOT NULL, shared_id bigint NOT NULL, KEY tenant_idx (tenant_id, id))",
"CREATE TABLE gftest.shared_rows (id bigint PRIMARY KEY, data TEXT)",
"CREATE TABLE gftest.excluded_rows (id bigint PRIMARY KEY, tenant_id bigint NOT NULL, KEY tenant_idx (tenant_id, id))",
} {
_, err := db.Exec(statement)
require.NoError(t, err)
}
}
for _, statement := range []string{
"INSERT INTO gftest.normal_rows VALUES (-1, 2, 'excluded'), (0, 0, 'zero'), (1, 0, 'one')",
"INSERT INTO gftest.tenants VALUES (-1, 'excluded'), (0, 'zero'), (2, 'excluded')",
"INSERT INTO gftest.links VALUES (-1, 2, -1), (0, 0, 0), (1, 0, 1)",
"INSERT INTO gftest.shared_rows VALUES (-1, 'excluded'), (0, 'zero'), (1, 'one')",
"INSERT INTO gftest.excluded_rows VALUES (-2, 2), (-1, 2)",
} {
_, err := sourceDB.Exec(statement)
require.NoError(t, err)
}
},
}
defer testcase.Teardown()
testcase.CopyData()
require.NoError(t, errorHandler.LastError)

for _, check := range []struct {
query string
want string
}{
{"SELECT GROUP_CONCAT(CONCAT(id, ':', data) ORDER BY id) FROM gftest.normal_rows", "0:zero,1:one"},
{"SELECT GROUP_CONCAT(CONCAT(tenant_id, ':', data) ORDER BY tenant_id) FROM gftest.tenants", "0:zero"},
{"SELECT GROUP_CONCAT(CONCAT(id, ':', shared_id) ORDER BY id) FROM gftest.links", "0:0,1:1"},
{"SELECT GROUP_CONCAT(CONCAT(id, ':', data) ORDER BY id) FROM gftest.shared_rows", "0:zero,1:one"},
{"SELECT COUNT(*) FROM gftest.excluded_rows", "0"},
} {
var got string
require.NoError(t, testcase.TargetDB.QueryRow(check.query).Scan(&got))
assert.Equal(t, check.want, got, check.query)
}
}

func TestSelectiveCopyDataWithInsertLoadOnOtherTenants(t *testing.T) {
testcase := &testhelpers.IntegrationTestCase{
T: t,
Expand Down
8 changes: 2 additions & 6 deletions state_tracker.go
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,7 @@ func (s *StateTracker) RowStatsWrittenPerTable() map[string]RowStats {
return d
}

// LastSuccessfulPaginationKey returns nil until a batch has been copied.
func (s *StateTracker) LastSuccessfulPaginationKey(table string, tableSchema *TableSchema) PaginationKey {
s.CopyRWMutex.RLock()
defer s.CopyRWMutex.RUnlock()
Expand All @@ -380,12 +381,7 @@ func (s *StateTracker) LastSuccessfulPaginationKey(table string, tableSchema *Ta
return MaxPaginationKey(tableSchema.GetPaginationColumn())
}

paginationKey, found := s.lastSuccessfulPaginationKeys[table]
if !found {
return MinPaginationKey(tableSchema.GetPaginationColumn())
}

return paginationKey
return s.lastSuccessfulPaginationKeys[table]
}

func (s *StateTracker) MarkTableAsCompleted(table string) {
Expand Down
47 changes: 45 additions & 2 deletions table_schema_cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,14 +125,32 @@ func QuotedTableNameFromString(database, table string) string {
return fmt.Sprintf("`%s`.`%s`", database, table)
}

func MaxPaginationKeys(db *sql.DB, tables []*TableSchema, logger Logger) (map[*TableSchema]PaginationKey, []*TableSchema, error) {
// MaxPaginationKeys validates selected signed keys and returns table-wide upper
// bounds for iteration. The first-key probe uses the same filter as the cursor.
func MaxPaginationKeys(cursorConfig *CursorConfig, tables []*TableSchema, logger Logger) (map[*TableSchema]PaginationKey, []*TableSchema, error) {
tablesWithData := make(map[*TableSchema]PaginationKey)
emptyTables := make([]*TableSchema, 0, len(tables))

for _, table := range tables {
logger := logger.WithField("table", table.String())

maxPaginationKey, maxPaginationKeyExists, err := maxPaginationKey(db, table)
column := table.GetPaginationColumn()
if !column.IsUnsigned && (column.Type == schema.TYPE_NUMBER || column.Type == schema.TYPE_MEDIUM_INT) {
firstKey, found, err := firstSignedPaginationKey(cursorConfig, table)
if err != nil {
logger.WithError(err).Errorf("failed to check first selected pagination key %s", column.Name)
return tablesWithData, emptyTables, err
}
if !found {
emptyTables = append(emptyTables, table)
continue
}
if firstKey < 0 {
return tablesWithData, emptyTables, fmt.Errorf("table %s has a negative pagination key in column %s, which Ghostferry cannot copy", table.String(), column.Name)
}
}

maxPaginationKey, maxPaginationKeyExists, err := maxPaginationKey(cursorConfig.DB, table)
if err != nil {
logger.WithError(err).Errorf("failed to get max primary key %s", table.GetPaginationColumn().Name)
return tablesWithData, emptyTables, err
Expand Down Expand Up @@ -419,6 +437,31 @@ func showTablesFrom(c *sql.DB, dbname string) ([]string, error) {
return tables, nil
}

func firstSignedPaginationKey(cursorConfig *CursorConfig, table *TableSchema) (int64, bool, error) {
columns := []string{QuoteField(table.GetPaginationColumn().Name)}
var builder sq.SelectBuilder
if cursorConfig.BuildSelect == nil {
builder = DefaultBuildSelect(columns, table, nil, 1)
} else {
var err error
builder, err = cursorConfig.BuildSelect(columns, table, nil, 1)
if err != nil {
return 0, false, err
}
}
query, args, err := builder.ToSql()
if err != nil {
return 0, false, err
}

var value int64
err = cursorConfig.DB.QueryRow(query, args...).Scan(&value)
if err == sqlorig.ErrNoRows {
return 0, false, nil
}
return value, err == nil, err
}

func maxPaginationKey(db *sql.DB, table *TableSchema) (PaginationKey, bool, error) {
primaryKeyColumn := table.GetPaginationColumn()
paginationKeyName := QuoteField(primaryKeyColumn.Name)
Expand Down
Loading
Loading