Repository navigation
Copy rows at the minimum pagination key and reject negative keys #465
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -45,7 +45,7 @@ type CursorConfig struct { | |
| ReadRetries int | ||
| } | ||
|
|
||
| // returns a new Cursor with an embedded copy of itself | ||
| // NewCursor resumes after startPaginationKey, or starts at the beginning when it is nil. | ||
| func (c *CursorConfig) NewCursor(table *TableSchema, startPaginationKey, maxPaginationKey PaginationKey) *Cursor { | ||
| return &Cursor{ | ||
| CursorConfig: *c, | ||
|
|
@@ -95,7 +95,7 @@ func (c *Cursor) Each(f func(*RowBatch) error) error { | |
| c.ColumnsToSelect = []string{"*"} | ||
| } | ||
|
|
||
| for c.lastSuccessfulPaginationKey.Compare(c.MaxPaginationKey) < 0 { | ||
| for c.lastSuccessfulPaginationKey == nil || c.lastSuccessfulPaginationKey.Compare(c.MaxPaginationKey) < 0 { | ||
| var tx SqlPreparerAndRollbacker | ||
| var batch *RowBatch | ||
| var paginationKeypos PaginationKey | ||
|
|
@@ -136,7 +136,7 @@ func (c *Cursor) Each(f func(*RowBatch) error) error { | |
| break | ||
| } | ||
|
|
||
| if paginationKeypos.Compare(c.lastSuccessfulPaginationKey) <= 0 { | ||
| if c.lastSuccessfulPaginationKey != nil && paginationKeypos.Compare(c.lastSuccessfulPaginationKey) <= 0 { | ||
| tx.Rollback() | ||
| err = fmt.Errorf("new paginationKeypos %s <= lastSuccessfulPaginationKey %s", paginationKeypos.String(), c.lastSuccessfulPaginationKey.String()) | ||
| c.logger.WithError(err).Errorf("last successful paginationKey position did not advance") | ||
|
|
@@ -251,6 +251,16 @@ func (c *Cursor) Fetch(db SqlPreparer) (batch *RowBatch, paginationKeypos Pagina | |
| return | ||
| } | ||
|
|
||
| // The first query has no lower bound. Recheck its first row in case a | ||
| // negative key was inserted after startup validation. | ||
| if len(batchData) == 0 && !c.paginationKeyColumn.IsUnsigned && | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. do we need this check if c.lastSuccessfulPaginationKey is not nil? - it means the following batches are not negative right? And even though this is not a terminal error, this will still retry 5 times because the whole logic is wrapped in |
||
| (c.paginationKeyColumn.Type == schema.TYPE_NUMBER || c.paginationKeyColumn.Type == schema.TYPE_MEDIUM_INT) { | ||
| if value, ok := Int64Value(rowData[paginationKeyIndex]); ok && value < 0 { | ||
| err = fmt.Errorf("table %s has a negative pagination key in column %s, which Ghostferry cannot copy", c.Table.String(), c.paginationKeyColumn.Name) | ||
| return | ||
| } | ||
| } | ||
|
|
||
| batchData = append(batchData, rowData) | ||
| } | ||
|
|
||
|
|
@@ -307,9 +317,12 @@ func ScanByteRow(rows *sqlorig.Rows, columnCount int) ([][]byte, error) { | |
| func DefaultBuildSelect(columns []string, table *TableSchema, lastPaginationKey PaginationKey, batchSize uint64) squirrel.SelectBuilder { | ||
| quotedPaginationKey := QuoteField(table.GetPaginationColumn().Name) | ||
|
|
||
| return squirrel.Select(columns...). | ||
| query := squirrel.Select(columns...). | ||
| From(QuotedTableName(table)). | ||
| Where(squirrel.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()}). | ||
| Limit(batchSize). | ||
| OrderBy(quotedPaginationKey) | ||
| if lastPaginationKey != nil { | ||
| query = query.Where(squirrel.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()}) | ||
| } | ||
| return query | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -43,12 +43,14 @@ func (f *ShardedCopyFilter) BuildSelect(columns []string, table *ghostferry.Tabl | |
| // the model for a single tenant in a multitenant database, e.g. a shop on | ||
| // a multitenant commerce platform. | ||
| // | ||
| // It is necessary to use two WHERE conditions on quotedPaginationKey so the second batch will be empty. | ||
| // No LIMIT clause is necessary since at most one row is present. | ||
| return sq.Select(columns...). | ||
| query := sq.Select(columns...). | ||
| From(quotedTable + " USE INDEX (PRIMARY)"). | ||
| Where(sq.Eq{quotedPaginationKey: f.ShardingValue}). | ||
| Where(sq.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()}), nil | ||
| Where(sq.Eq{quotedPaginationKey: f.ShardingValue}) | ||
| if lastPaginationKey != nil { | ||
| query = query.Where(sq.Gt{quotedPaginationKey: lastPaginationKey.SQLValue()}) | ||
| } | ||
| return query, nil | ||
| } | ||
|
|
||
| joinTables, exists := f.JoinedTables[table.Name] | ||
|
|
@@ -84,12 +86,18 @@ func (f *ShardedCopyFilter) BuildSelect(columns []string, table *ghostferry.Tabl | |
| selectPaginationKeys += " " + f.shardingKeyIndexHint(table, indexHint) | ||
| } | ||
|
|
||
| selectPaginationKeys += " WHERE " + quotedShardingKey + " = ? AND " + quotedPaginationKey + " > ?" + | ||
| " ORDER BY " + quotedPaginationKey + " LIMIT " + strconv.Itoa(int(batchSize)) | ||
| selectPaginationKeys += " WHERE " + quotedShardingKey + " = ?" | ||
| args := make([]interface{}, 0, 2) | ||
| args = append(args, f.ShardingValue) | ||
| if lastPaginationKey != nil { | ||
| selectPaginationKeys += " AND " + quotedPaginationKey + " > ?" | ||
| args = append(args, lastPaginationKey.SQLValue()) | ||
| } | ||
| selectPaginationKeys += " ORDER BY " + quotedPaginationKey + " LIMIT " + strconv.Itoa(int(batchSize)) | ||
|
|
||
| return sq.Select(columns...). | ||
| From(quotedTable). | ||
| Join("("+selectPaginationKeys+") AS `batch` USING("+quotedPaginationKey+")", f.ShardingValue, lastPaginationKey.SQLValue()), nil | ||
| Join("("+selectPaginationKeys+") AS `batch` USING("+quotedPaginationKey+")", args...), nil | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we add
I reproduced this on MySQL 8.0.32 with batch size 4. Startup sees keys Adding ordering to this outer query makes it return |
||
| } | ||
|
|
||
| // This is a "joined table". It is the only supported type of table that | ||
|
|
@@ -120,12 +128,19 @@ func (f *ShardedCopyFilter) BuildSelect(columns []string, table *ghostferry.Tabl | |
| // it to a batch, then select the rest of the columns. | ||
| var clauses []string | ||
| var args []interface{} | ||
| const firstPage = "SELECT `%[1]s` AS sharding_join_alias FROM `%[2]s`.`%[3]s` WHERE `%[4]s` = ?" | ||
| pattern := firstPage | ||
| if lastPaginationKey != nil { | ||
| pattern = firstPage + " AND `%[1]s` > ?" | ||
| } | ||
|
|
||
| for _, joinTable := range joinTables { | ||
| pattern := "SELECT `%s` AS sharding_join_alias FROM `%s`.`%s` WHERE `%s` = ? AND `%s` > ?" | ||
| sql := fmt.Sprintf(pattern, joinTable.JoinColumn, table.Schema, joinTable.TableName, f.ShardingKey, joinTable.JoinColumn) | ||
| sql := fmt.Sprintf(pattern, joinTable.JoinColumn, table.Schema, joinTable.TableName, f.ShardingKey) | ||
| clauses = append(clauses, sql) | ||
| args = append(args, f.ShardingValue, lastPaginationKey.SQLValue()) | ||
| args = append(args, f.ShardingValue) | ||
| if lastPaginationKey != nil { | ||
| args = append(args, lastPaginationKey.SQLValue()) | ||
| } | ||
| } | ||
|
|
||
| subquery := strings.Join(clauses, " UNION DISTINCT ") | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
My bot found an edge case (quite unlikely?) where a negative pagination key can cause silent data loss here