Skip to content

(WIP) Parallel syncing #591

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ggreer wants to merge 9 commits into
base: main
Choose a base branch
from
Open

(WIP) Parallel syncing #591

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
8c394b1
Checkpoint
jirwin Aug 29, 2025
eb21e93
make protogen after rebase.
jirwin Oct 20, 2025
5bbad14
Use context in QueryRow when checking for wal mode
jirwin Oct 22, 2025
878da44
Remove WAL checkpointing. Use builtin min() function.
ggreer Dec 4, 2025
b510385
Move ProgressCounts into its own package. Rename it to ProgressLog, s…
ggreer Dec 4, 2025
b0f488f
Add options to progress logger constructor. Remove sequentialMode att…
ggreer Dec 5, 2025
73100a6
Fix bad rebase. Move sequential sync function into its own file.
ggreer Dec 6, 2025
d31ec7a
Progress log: make data private. Use methods (which are thread safe) …
ggreer Dec 6, 2025
d939bfa
WIP parallel sync
ggreer Dec 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 28 additions & 4 deletions pb/c1/connector/v2/resource.pb.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions pb/c1/connector/v2/resource.pb.validate.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 22 additions & 2 deletions pb/c1/connector/v2/resource_protoopaque.pb.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions pkg/cli/commands.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,10 @@ func MakeMainCommand[T field.Configurable](
}
}

if v.GetBool("parallel-sync") {
opts = append(opts, connectorrunner.WithParallelSyncEnabled())
}

if v.GetString("c1z-temp-dir") != "" {
c1zTmpDir := v.GetString("c1z-temp-dir")
if _, err := os.Stat(c1zTmpDir); os.IsNotExist(err) {
Expand Down
13 changes: 12 additions & 1 deletion pkg/connectorrunner/runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,7 @@ type runnerConfig struct {
syncDifferConfig *syncDifferConfig
syncCompactorConfig *syncCompactorConfig
skipFullSync bool
parallelSync bool
targetedSyncResourceIDs []string
externalResourceC1Z string
externalResourceEntitlementIdFilter string
Expand Down Expand Up @@ -554,6 +555,13 @@ func WithFullSyncDisabled() Option {
}
}

func WithParallelSyncEnabled() Option {
return func(ctx context.Context, cfg *runnerConfig) error {
cfg.parallelSync = true
return nil
}
}

func WithTargetedSyncResources(resourceIDs []string) Option {
return func(ctx context.Context, cfg *runnerConfig) error {
cfg.targetedSyncResourceIDs = resourceIDs
Expand Down Expand Up @@ -814,6 +822,7 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op
local.WithSkipEntitlementsAndGrants(cfg.skipEntitlementsAndGrants),
local.WithSkipGrants(cfg.skipGrants),
local.WithSyncResourceTypeIDs(cfg.syncResourceTypeIDs),
local.WithParallelSyncEnabled(cfg.parallelSync),
)
if err != nil {
return nil, err
Expand All @@ -826,7 +835,8 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op
return runner, nil
}

tm, err := c1api.NewC1TaskManager(ctx,
tm, err := c1api.NewC1TaskManager(
ctx,
cfg.clientID,
cfg.clientSecret,
cfg.tempDir,
Expand All @@ -835,6 +845,7 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op
cfg.externalResourceEntitlementIdFilter,
resources,
cfg.syncResourceTypeIDs,
cfg.parallelSync,
)
if err != nil {
return nil, err
Expand Down
5 changes: 1 addition & 4 deletions pkg/dotc1z/sql_helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -459,10 +459,7 @@ func executeChunkedInsert(

for i := 0; i < chunks; i++ {
start := i * chunkSize
end := (i + 1) * chunkSize
if end > len(rows) {
end = len(rows)
}
end := min((i+1)*chunkSize, len(rows))
chunkedRows := rows[start:end]

// Create the base insert dataset
Expand Down
2 changes: 2 additions & 0 deletions pkg/field/defaults.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ var (
WithPersistent(true),
WithExportTarget(ExportTargetOps))
skipFullSync = BoolField("skip-full-sync", WithDescription("This must be set to skip a full sync"), WithPersistent(true), WithExportTarget(ExportTargetNone))
parallelSync = BoolField("parallel-sync", WithDescription("This must be set to enable parallel sync"), WithPersistent(true), WithExportTarget(ExportTargetNone))
targetedSyncResourceIDs = StringSliceField("sync-resources", WithDescription("The resource IDs to sync"), WithPersistent(true), WithExportTarget(ExportTargetNone))
skipEntitlementsAndGrants = BoolField("skip-entitlements-and-grants",
WithDescription("This must be set to skip syncing of entitlements and grants"),
Expand Down Expand Up @@ -304,6 +305,7 @@ var DefaultFields = []SchemaField{
invokeActionField,
invokeActionArgsField,
ServerSessionStoreMaximumSizeField,
parallelSync,

otelCollectorEndpoint,
otelCollectorEndpointTLSCertPath,
Expand Down
Loading
Loading