package syncapi import ( "context" "crypto/tls" "errors" "fmt" "net" "net/http" "slices" "sync" "time" "github.com/garethgeorge/backrest/gen/go/types" v1 "github.com/garethgeorge/backrest/gen/go/v1" "github.com/garethgeorge/backrest/gen/go/v1/v1connect" "github.com/garethgeorge/backrest/internal/api/syncapi/permissions" "github.com/garethgeorge/backrest/internal/env" "github.com/garethgeorge/backrest/internal/oplog" "github.com/garethgeorge/backrest/internal/protoutil" "go.uber.org/zap" "golang.org/x/net/http2" "google.golang.org/protobuf/proto" ) type SyncClient struct { mgr *SyncManager syncConfigSnapshot syncConfigSnapshot localInstanceID string peer *v1.Multihost_Peer oplog *oplog.OpLog client v1connect.BackrestSyncServiceClient reconnectDelay time.Duration l *zap.Logger reconnectAttempts int } func newInsecureClient() *http.Client { return &http.Client{ Transport: &http2.Transport{ AllowHTTP: true, DialTLS: func(network, addr string, _ *tls.Config) (net.Conn, error) { return net.Dial(network, addr) }, IdleConnTimeout: 300 * time.Second, ReadIdleTimeout: 60 * time.Second, }, } } func NewSyncClient( mgr *SyncManager, snapshot syncConfigSnapshot, peer *v1.Multihost_Peer, oplog *oplog.OpLog, ) (*SyncClient, error) { if peer.GetInstanceUrl() == "" { return nil, errors.New("peer instance URL is required") } client := v1connect.NewBackrestSyncServiceClient( newInsecureClient(), peer.GetInstanceUrl(), ) c := &SyncClient{ mgr: mgr, syncConfigSnapshot: snapshot, localInstanceID: snapshot.config.Instance, peer: peer, reconnectDelay: mgr.syncClientRetryDelay, client: client, oplog: oplog, l: zap.L().Named(fmt.Sprintf("syncclient for %q", peer.GetInstanceId())), } c.mgr.peerStateManager.SetPeerState(peer.Keyid, newPeerState(peer.InstanceId, peer.Keyid)) return c, nil } func (c *SyncClient) RunSync(ctx context.Context) { for { if ctx.Err() != nil { return } lastConnect := time.Now() syncSessionHandler := newSyncHandlerClient( c.l, c.mgr, c.syncConfigSnapshot, c.oplog, c.peer, ) cmdStream := newBidiSyncCommandStream() c.l.Sugar().Infof("connecting to peer %q (%s) at %s", c.peer.InstanceId, c.peer.Keyid, c.peer.GetInstanceUrl()) var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() err := runSync( ctx, c.localInstanceID, c.syncConfigSnapshot.identityKey, cmdStream, syncSessionHandler, c.syncConfigSnapshot.config.GetMultihost().GetKnownHosts(), ) cmdStream.SendErrorAndTerminate(err) }() if err := cmdStream.ConnectStream(ctx, c.client.Sync(ctx)); err != nil { c.l.Sugar().Infof("lost stream connection to peer %q (%s): %v", c.peer.InstanceId, c.peer.Keyid, err) var syncErr *SyncError state := c.mgr.peerStateManager.GetPeerState(c.peer.Keyid).Clone() if state == nil { state = newPeerState(c.peer.InstanceId, c.peer.Keyid) } state.LastHeartbeat = time.Now() if errors.As(err, &syncErr) { state.ConnectionState = syncErr.State state.ConnectionStateMessage = syncErr.Message.Error() } else { state.ConnectionState = v1.SyncConnectionState_CONNECTION_STATE_ERROR_INTERNAL state.ConnectionStateMessage = err.Error() } c.mgr.peerStateManager.SetPeerState(c.peer.Keyid, state) } else { c.reconnectAttempts = 0 } wg.Wait() delay := c.reconnectDelay - time.Since(lastConnect) if c.reconnectAttempts > 0 { backoff := time.Duration(1<= 256 { if err := sendOpsFunc(); err != nil { return err } } } if len(sendOps) > 0 { if err := sendOpsFunc(); err != nil { return err } } if len(deletedIDs) > 0 { stream.Send(&v1.SyncStreamItem{ Action: &v1.SyncStreamItem_SendOperations{ SendOperations: &v1.SyncStreamItem_SyncActionSendOperations{ Event: &v1.OperationEvent{ Event: &v1.OperationEvent_DeletedOperations{ DeletedOperations: &types.Int64List{Values: deletedIDs}, }, }, }, }, }) } c.l.Debug("replied to an operations request", zap.Int("num_ops_requested", len(requestedOperations)), zap.Int("num_ops_sent", sentOps), zap.Int("num_ops_deleted", len(deletedIDs))) return nil } func (c *syncSessionHandlerClient) HandleSendOperations(ctx context.Context, stream *bidiSyncCommandStream, item *v1.SyncStreamItem_SyncActionSendOperations) error { return NewSyncErrorProtocol(errors.New("client should not receive SendOperations messages, this is a host-only message")) } func (c *syncSessionHandlerClient) HandleSendConfig(ctx context.Context, stream *bidiSyncCommandStream, item *v1.SyncStreamItem_SyncActionSendConfig) error { c.l.Sugar().Debugf("received remote config update") peerState := c.mgr.peerStateManager.GetPeerState(c.peer.Keyid).Clone() if peerState == nil { return NewSyncErrorInternal(fmt.Errorf("peer state for %q not found", c.peer.Keyid)) } newRemoteConfig := item.Config if newRemoteConfig == nil { return NewSyncErrorProtocol(fmt.Errorf("received nil remote config")) } peerState.Config = newRemoteConfig c.mgr.peerStateManager.SetPeerState(c.peer.Keyid, peerState) return nil } func (c *syncSessionHandlerClient) HandleSetConfig(ctx context.Context, stream *bidiSyncCommandStream, item *v1.SyncStreamItem_SyncActionSetConfig) error { // Log the received config updates c.l.Sugar().Debugf("received SetConfig request from peer %q") // Fetch latest config from the config manager latestConfig, err := c.mgr.configMgr.Get() if err != nil { return fmt.Errorf("fetch latest config: %w", err) } latestConfig = proto.Clone(latestConfig).(*v1.Config) // Clone to avoid modifying the original config for _, plan := range item.GetPlans() { c.l.Sugar().Debugf("received plan update: %s", plan.Id) if !c.permissions.CheckPermissionForPlan(plan.Id, v1.Multihost_Permission_PERMISSION_READ_WRITE_CONFIG) { return NewSyncErrorAuth(fmt.Errorf("peer %q is not allowed to update plan %q", c.peer.InstanceId, plan.Id)) } // Update the plan in the local config idx := slices.IndexFunc(latestConfig.Plans, func(p *v1.Plan) bool { return p.Id == plan.Id }) if idx >= 0 { latestConfig.Plans[idx] = plan } else { latestConfig.Plans = append(latestConfig.Plans, plan) } } for _, repo := range item.GetRepos() { c.l.Sugar().Debugf("received repo update: %s", repo.Guid) if !c.permissions.CheckPermissionForRepo(repo.Id, v1.Multihost_Permission_PERMISSION_READ_WRITE_CONFIG) { return NewSyncErrorAuth(fmt.Errorf("peer %q is not allowed to update repo %q", c.peer.InstanceId, repo.Id)) } // Update the repo in the local config idx := slices.IndexFunc(latestConfig.Repos, func(r *v1.Repo) bool { return r.Guid == repo.Guid }) if idx >= 0 { latestConfig.Repos[idx] = repo } else { latestConfig.Repos = append(latestConfig.Repos, repo) } } for _, plan := range item.GetPlansToDelete() { c.l.Sugar().Debugf("received plan deletion request: %s", plan) if !c.permissions.CheckPermissionForPlan(plan, v1.Multihost_Permission_PERMISSION_READ_WRITE_CONFIG) { return NewSyncErrorAuth(fmt.Errorf("peer %q is not allowed to delete plan %q", c.peer.InstanceId, plan)) } // Remove the plan from the local config idx := slices.IndexFunc(latestConfig.Plans, func(p *v1.Plan) bool { return p.Id == plan }) if idx >= 0 { latestConfig.Plans = append(latestConfig.Plans[:idx], latestConfig.Plans[idx+1:]...) } else { c.l.Sugar().Warnf("received plan deletion request for non-existent plan %q, ignoring", plan) } } for _, repo := range item.GetReposToDelete() { c.l.Sugar().Debugf("received repo deletion request: %s", repo) if !c.permissions.CheckPermissionForRepo(repo, v1.Multihost_Permission_PERMISSION_READ_WRITE_CONFIG) { return NewSyncErrorAuth(fmt.Errorf("peer %q is not allowed to delete repo %q", c.peer.InstanceId, repo)) } // Remove the repo from the local config idx := slices.IndexFunc(latestConfig.Repos, func(r *v1.Repo) bool { return r.Id == repo }) if idx >= 0 { latestConfig.Repos = append(latestConfig.Repos[:idx], latestConfig.Repos[idx+1:]...) } else { c.l.Sugar().Warnf("received repo deletion request for non-existent repo %q, ignoring", repo) } } // Update the local config with the new changes latestConfig.Modno++ if err := c.mgr.configMgr.Update(latestConfig); err != nil { return fmt.Errorf("set updated config: %w", err) } return nil } func (c *syncSessionHandlerClient) HandleListResources(ctx context.Context, stream *bidiSyncCommandStream, item *v1.SyncStreamItem_SyncActionListResources) error { c.l.Sugar().Debugf("received ListResources request from peer %q", c.peer.InstanceId) return nil }