Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ All notable changes to this project will be documented in this file.
- `Progress.TimeTaken`, and the control server's "Time Taken", stop counting when the data copy completes instead of also including the wait for cutover and the cutover itself ([#314](https://github.com/Shopify/ghostferry/issues/314)).
- ghostferry-copydb keeps top-level `DatabaseRewrites` and `TableRewrites` instead of overwriting them with the (often empty) `Databases.Rewrites` and `Tables.Rewrites`. When both forms contain rewrites, the maps must be identical ([#216](https://github.com/Shopify/ghostferry/issues/216)).
**Migration safety:** Finish active migrations before upgrading: top-level rewrites ignored by older binaries now take effect. Resume interrupted migrations with the original binary and unchanged effective database/table mappings.
- Synchronize live `UpdatableConfig.DataIterationBatchSize` updates with cursor, iterative verifier and control server reads, and `LagThrottler` lag updates with `Throttled` callers ([#471](https://github.com/Shopify/ghostferry/issues/471)).

## [1.3.1 - 2026-04-15]

Expand Down
5 changes: 4 additions & 1 deletion config.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"fmt"
"io/ioutil"
"strings"
"sync/atomic"
"time"

sql "github.com/Shopify/ghostferry/sqlwrapper"
Expand Down Expand Up @@ -972,9 +973,11 @@ func (c *Config) logDeprecated(deprecatedConfig string, newConfig string) {
LogWithField("tag", "config").Warnf("Config.%s is deprecated in favour of Config.%s", deprecatedConfig, newConfig)
}

// Update applies a live configuration change. DataIterationBatchSize is read
// concurrently by running cursors, so it is stored atomically.
func (c *Config) Update(updatedConfig UpdatableConfig) {
if updatedConfig.DataIterationBatchSize > 0 {
c.UpdatableConfig.DataIterationBatchSize = updatedConfig.DataIterationBatchSize
atomic.StoreUint64(&c.UpdatableConfig.DataIterationBatchSize, updatedConfig.DataIterationBatchSize)
}
}

Expand Down
6 changes: 5 additions & 1 deletion control_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"path/filepath"
"sort"
"sync"
"sync/atomic"
"time"

"github.com/gorilla/mux"
Expand Down Expand Up @@ -369,7 +370,10 @@ func (this *ControlServer) HandleScript(w http.ResponseWriter, r *http.Request)

func (this *ControlServer) HandleConfigGet(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
err := json.NewEncoder(w).Encode(this.F.UpdatableConfig)
snapshot := UpdatableConfig{
DataIterationBatchSize: atomic.LoadUint64(&this.F.UpdatableConfig.DataIterationBatchSize),
}
err := json.NewEncoder(w).Encode(snapshot)
if err != nil {
this.logger.WithError(err).Error("failed to send updatable config")
}
Expand Down
4 changes: 3 additions & 1 deletion cursor.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
sqlorig "database/sql"
"fmt"
"strings"
"sync/atomic"
"time"

sql "github.com/Shopify/ghostferry/sqlwrapper"
Expand Down Expand Up @@ -40,6 +41,7 @@ type CursorConfig struct {
BuildSelect func([]string, *TableSchema, PaginationKey, uint64) (squirrel.SelectBuilder, error)
// BatchSize is a pointer to the BatchSize in Config.UpdatableConfig which can be independently updated from this code.
// Having it as a pointer allows the updated value to be read without needing additional code to copy the batch size value into the cursor config for each cursor we create.
// Runtime reads and writes of the pointed-to value must be atomic.
BatchSize *uint64
BatchSizePerTableOverride *DataIterationBatchSizePerTableOverride
ReadRetries int
Expand Down Expand Up @@ -69,7 +71,7 @@ func (c CursorConfig) GetBatchSize(schemaName string, tableName string) uint64 {
return batchSize
}
}
return *c.BatchSize
return atomic.LoadUint64(c.BatchSize)
}

type Cursor struct {
Expand Down
3 changes: 2 additions & 1 deletion iterative_verifier.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"time"

sql "github.com/Shopify/ghostferry/sqlwrapper"
Expand Down Expand Up @@ -434,7 +435,7 @@ func (v *IterativeVerifier) iterateTableFingerprints(table *TableSchema, mismatc
}

func (v *IterativeVerifier) verifyStore(sourceTag string, additionalTags []MetricTag) (VerificationResult, error) {
allBatches := v.reverifyStore.FlushAndBatchByTable(int(*v.CursorConfig.BatchSize))
allBatches := v.reverifyStore.FlushAndBatchByTable(int(atomic.LoadUint64(v.CursorConfig.BatchSize)))
v.logger.WithField("batches", len(allBatches)).Debug("reverifying")

if len(allBatches) == 0 {
Expand Down
136 changes: 136 additions & 0 deletions test/go/control_server_test.go
Original file line number Diff line number Diff line change
@@ -1 +1,137 @@
package test

import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"sync"
"testing"

"github.com/Shopify/ghostferry"
"github.com/Shopify/ghostferry/testhelpers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func postBatchSize(client *http.Client, url string, batchSize uint64) error {
body, err := json.Marshal(ghostferry.UpdatableConfig{DataIterationBatchSize: batchSize})
if err != nil {
return err
}

resp, err := client.Post(url+"/api/config", "application/json", bytes.NewReader(body))
if err != nil {
return err
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return fmt.Errorf("POST /api/config returned %d", resp.StatusCode)
}
return nil
}

func getBatchSize(client *http.Client, url string) (uint64, error) {
req, err := http.NewRequest(http.MethodGet, url+"/api/config", nil)
if err != nil {
return 0, err
}
req.Header.Set("Content-Type", "application/json")

resp, err := client.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return 0, fmt.Errorf("GET /api/config returned %d", resp.StatusCode)
}

var config ghostferry.UpdatableConfig
if err := json.NewDecoder(resp.Body).Decode(&config); err != nil {
return 0, err
}
return config.DataIterationBatchSize, nil
}

func TestConcurrentBatchSizeUpdates(t *testing.T) {
const (
writers = 4
readers = 4
iterations = 50
overrideBatchSize = uint64(80)
)

config := testhelpers.NewTestConfig()
config.UpdatableConfig.DataIterationBatchSize = 200

server := &ghostferry.ControlServer{
Config: &ghostferry.ControlServerConfig{WebBasedir: "../.."},
F: &ghostferry.Ferry{Config: config},
}
require.NoError(t, server.Initialize())

httpServer := httptest.NewServer(server)
defer httpServer.Close()
client := httpServer.Client()

cursorConfig := &ghostferry.CursorConfig{
BatchSize: &config.UpdatableConfig.DataIterationBatchSize,
BatchSizePerTableOverride: &ghostferry.DataIterationBatchSizePerTableOverride{
TableOverride: map[string]map[string]uint64{
testhelpers.TestSchemaName: {"overridden": overrideBatchSize},
},
},
}

allowed := map[uint64]bool{104: true, 200: true}
start := make(chan struct{})
wg := &sync.WaitGroup{}

for w := range writers {
wg.Add(1)
go func() {
defer wg.Done()
<-start
for i := range iterations {
batchSize := uint64(104)
if (w+i)%2 == 1 {
batchSize = 200
}
assert.NoError(t, postBatchSize(client, httpServer.URL, batchSize))
}
}()
}

for range readers {
wg.Add(1)
go func() {
defer wg.Done()
<-start
for range iterations {
base := cursorConfig.GetBatchSize(testhelpers.TestSchemaName, "not_overridden")
assert.True(t, allowed[base], "cursor observed unexpected batch size %d", base)
assert.Equal(t, overrideBatchSize, cursorConfig.GetBatchSize(testhelpers.TestSchemaName, "overridden"))

snapshot, err := getBatchSize(client, httpServer.URL)
if assert.NoError(t, err) {
assert.True(t, allowed[snapshot], "HTTP snapshot observed unexpected batch size %d", snapshot)
}
}
}()
}

close(start)
wg.Wait()

require.NoError(t, postBatchSize(client, httpServer.URL, 104))

snapshot, err := getBatchSize(client, httpServer.URL)
require.NoError(t, err)
require.Equal(t, uint64(104), snapshot)
require.Equal(t, uint64(104), cursorConfig.GetBatchSize(testhelpers.TestSchemaName, "not_overridden"))
require.Equal(t, overrideBatchSize, cursorConfig.GetBatchSize(testhelpers.TestSchemaName, "overridden"))
}
16 changes: 14 additions & 2 deletions throttler.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
sqlorig "database/sql"
"fmt"
sql "github.com/Shopify/ghostferry/sqlwrapper"
"sync"
"sync/atomic"
"time"
)
Expand Down Expand Up @@ -83,9 +84,12 @@ type LagThrottler struct {
config *LagThrottlerConfig

DB *sql.DB
lag int
logger Logger
interval time.Duration

// lag is written by Run and read by every Throttled caller.
lagLock sync.Mutex
lag int
}

func NewLagThrottler(config *LagThrottlerConfig) (*LagThrottler, error) {
Expand Down Expand Up @@ -125,7 +129,13 @@ func NewLagThrottler(config *LagThrottlerConfig) (*LagThrottler, error) {
}

func (t *LagThrottler) Throttled() bool {
return t.PauserThrottler.Throttled() || t.lag > t.config.MaxLag
if t.PauserThrottler.Throttled() {
return true
}

t.lagLock.Lock()
defer t.lagLock.Unlock()
return t.lag > t.config.MaxLag
}

func (t *LagThrottler) Run(ctx context.Context) error {
Expand Down Expand Up @@ -159,6 +169,8 @@ func (t *LagThrottler) updateLag(ctx context.Context) error {
return nil
}

t.lagLock.Lock()
t.lag = int(newLag.Int64)
t.lagLock.Unlock()
return nil
}
Loading