lee/backrest
1
0
Fork 0
mirror of https://github.com/garethgeorge/backrest.git synced 2026-09-27 10:25:40 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat: experimental multihost sync (#1204)

Browse Source
This commit is contained in:
Gareth
2026-05-02 22:29:39 -07:00
committed by GitHub
parent 87d76231bc
commit b98b1dc3cc
96 changed files with 8089 additions and 2790 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/syncapi/authmiddleware.go
-245
View File
@@ -1,245 +0,0 @@
package syncapi
import (
"context"
"encoding/base64"
"encoding/binary"
"errors"
"fmt"
"net/http"
"slices"
"time"
"connectrpc.com/connect"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/gen/go/v1sync"
"github.com/garethgeorge/backrest/internal/config"
"github.com/garethgeorge/backrest/internal/cryptoutil"
"google.golang.org/protobuf/proto"
)
var authTokenHeader = "Authorization"
var maxSignatureAge = 5 * time.Minute // Maximum age of a signature before it is considered invalid
type peerContextKey string
const PeerContextKey peerContextKey = "peer"
func ContextWithPeer(ctx context.Context, peer *v1.Multihost_Peer) context.Context {
return context.WithValue(ctx, PeerContextKey, peer)
}
func PeerFromContext(ctx context.Context) *v1.Multihost_Peer {
peer, ok := ctx.Value(PeerContextKey).(*v1.Multihost_Peer)
if !ok {
return nil
}
return peer
}
func newAuthHandler(config *config.ConfigManager, next http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
config, err := config.Get()
if err != nil {
http.Error(rw, "internal error", http.StatusInternalServerError)
return
}
authHeaderValue, err := createAuthHeader(config)
if err != nil {
http.Error(rw, fmt.Sprintf("internal error: %v", err), http.StatusInternalServerError)
return
}
rw.Header().Set(authTokenHeader, authHeaderValue)
peer, err := decodeAndVerifyAuthHeader(r, config.Instance, config.GetMultihost().GetAuthorizedClients())
if err != nil {
http.Error(rw, fmt.Sprintf("unauthorized: %v", err), http.StatusUnauthorized)
return
}
next.ServeHTTP(rw, r.WithContext(context.WithValue(r.Context(), PeerContextKey, peer)))
})
}
func createAuthHeader(config *v1.Config) (string, error) {
if config == nil || config.GetMultihost().GetIdentity() == nil {
return "", errors.New("config missing multihost.identity")
}
privKey, err := cryptoutil.NewPrivateKey(config.GetMultihost().GetIdentity())
if err != nil {
return "", fmt.Errorf("load private key: %w", err)
}
signedMessage, err := createSignedMessage([]byte(config.Instance), privKey)
if err != nil {
return "", fmt.Errorf("create signed message: %w", err)
}
authToken := &v1sync.AuthorizationToken{
InstanceId: signedMessage,
PublicKey: privKey.PublicKeyProto(),
}
tokenBytes, err := proto.Marshal(authToken)
if err != nil {
return "", fmt.Errorf("marshal auth token: %w", err)
}
return base64.StdEncoding.EncodeToString(tokenBytes), nil
}
type authHeaderClient struct {
configManager *config.ConfigManager
delegate connect.HTTPClient
wantPeer *v1.Multihost_Peer
}
func (c *authHeaderClient) Do(req *http.Request) (*http.Response, error) {
// create the header
cfg, err := c.configManager.Get()
if err != nil {
return nil, fmt.Errorf("get config: %w", err)
}
authHeaderValue, err := createAuthHeader(cfg)
if err != nil {
return nil, fmt.Errorf("create auth header: %w", err)
}
req.Header.Set(authTokenHeader, authHeaderValue)
resp, err := c.delegate.Do(req)
// verify the response header
if err != nil {
return nil, fmt.Errorf("HTTP request failed: %w", err)
}
if resp.StatusCode != http.StatusOK {
return resp, fmt.Errorf("HTTP request failed with status %d: %s", resp.StatusCode, resp.Status)
}
peer, err := decodeAndVerifyAuthHeader(req, cfg.Instance, cfg.GetMultihost().GetAuthorizedClients())
if err != nil {
return resp, fmt.Errorf("verify auth header: %w", err)
}
// Check the peer matches the expected one.
if c.wantPeer == nil || c.wantPeer.GetInstanceId() != peer.GetInstanceId() {
return resp, fmt.Errorf("peer instance ID mismatch: expected %s, got %s", c.wantPeer.GetInstanceId(), peer.GetInstanceId())
}
if c.wantPeer.GetKeyid() != peer.GetKeyid() {
return resp, fmt.Errorf("peer key ID mismatch: expected %s, got %s", c.wantPeer.GetKeyid(), peer.GetKeyid())
}
return resp, nil
}
func newHTTPClientWithConfig(cfg *config.ConfigManager, delegate connect.HTTPClient) (connect.HTTPClient, error) {
return &authHeaderClient{
configManager: cfg,
delegate: delegate,
}, nil
}
func decodeAndVerifyAuthHeader(r *http.Request, localInstanceID string, peers []*v1.Multihost_Peer) (*v1.Multihost_Peer, error) {
authHeader := r.Header.Get(authTokenHeader)
if len(authHeader) == 0 {
return nil, errors.New("missing authorization header")
}
// Decode the auth token from the header
tokenBytes, err := base64.StdEncoding.DecodeString(authHeader)
if err != nil {
return nil, errors.New("invalid authorization header format")
}
var token v1sync.AuthorizationToken
if err := proto.Unmarshal(tokenBytes, &token); err != nil {
return nil, fmt.Errorf("unmarshal authorization token: %w", err)
}
// Load the public key from the token
publicKey, err := cryptoutil.NewPublicKey(token.GetPublicKey())
if err != nil {
return nil, fmt.Errorf("load public key: %w", err)
}
if publicKey.KeyID() != token.InstanceId.GetKeyid() {
return nil, fmt.Errorf("instance ID must be signed with public key in token: expected %s, got %s", token.InstanceId.GetKeyid(), publicKey.KeyID())
}
// Verify the signed message
if err := verifySignedMessage(token.GetInstanceId(), publicKey); err != nil {
return nil, fmt.Errorf("verify signed message: %w", err)
}
// Now that we've validated that the peer was able to sign the message, we can look it up in the config
peerIdx := slices.IndexFunc(peers, func(peer *v1.Multihost_Peer) bool {
return peer.Keyid == publicKey.KeyID()
})
if peerIdx == -1 {
return nil, fmt.Errorf("peer with key ID %s not found in authorized clients", publicKey.KeyID())
}
// Finally check that the instance ID in the token matches the one in the config
peer := peers[peerIdx]
tokenInstanceID := string(token.GetInstanceId().GetPayload())
if peer.InstanceId != tokenInstanceID {
return nil, fmt.Errorf("instance ID mismatch: expected %s, got %s", peer.InstanceId, tokenInstanceID)
}
return peer, nil
}
func createSignedMessage(payload []byte, identity *cryptoutil.PrivateKey) (*v1.SignedMessage, error) {
if len(payload) == 0 {
return nil, errors.New("payload must not be empty")
}
timestampMillis := time.Now().UnixMilli()
payloadWithTimestamp := make([]byte, 0, len(payload)+8)
binary.BigEndian.AppendUint64(payloadWithTimestamp, uint64(timestampMillis))
payloadWithTimestamp = append(payloadWithTimestamp, payload...)
signature, err := identity.Sign(payloadWithTimestamp)
if err != nil {
return nil, fmt.Errorf("signing payload: %w", err)
}
return &v1.SignedMessage{
Payload: payload,
Signature: signature,
Keyid: identity.KeyID(),
TimestampMillis: timestampMillis,
}, nil
}
func verifySignedMessage(msg *v1.SignedMessage, publicKey *cryptoutil.PublicKey) error {
if msg == nil {
return errors.New("signed message must not be nil")
}
if len(msg.GetPayload()) == 0 {
return errors.New("signed message payload must not be empty")
}
if len(msg.GetSignature()) == 0 {
return errors.New("signed message signature must not be empty")
}
if len(msg.GetKeyid()) == 0 {
return errors.New("signed message key ID must not be empty")
}
if publicKey.KeyID() != msg.GetKeyid() {
return fmt.Errorf("public key ID mismatch: expected %s, got %s", publicKey.KeyID(), msg.GetKeyid())
}
payloadWithTimestamp := make([]byte, 0, len(msg.GetPayload())+8)
binary.BigEndian.AppendUint64(payloadWithTimestamp, uint64(msg.GetTimestampMillis()))
payloadWithTimestamp = append(payloadWithTimestamp, msg.GetPayload()...)
if err := publicKey.Verify(payloadWithTimestamp, msg.GetSignature()); err != nil {
return fmt.Errorf("verifying signed message: %w", err)
}
if time.Since(time.UnixMilli(msg.GetTimestampMillis())) > maxSignatureAge {
return fmt.Errorf("signature is too old, max age is %s. Is the clock out of sync?", maxSignatureAge)
}
return nil
}
internal/api/syncapi/authmiddleware_test.go
-200
View File
@@ -1,200 +0,0 @@
package syncapi
import (
"encoding/base64"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/gen/go/v1sync"
"github.com/garethgeorge/backrest/internal/config"
"github.com/garethgeorge/backrest/internal/config/migrations"
"github.com/garethgeorge/backrest/internal/cryptoutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto"
)
func TestAuthMiddleware(t *testing.T) {
serverPrivKey, err := cryptoutil.GeneratePrivateKey()
require.NoError(t, err)
clientPrivKey, err := cryptoutil.GeneratePrivateKey()
require.NoError(t, err)
// Create a mock config manager
cfgManager := &config.ConfigManager{
Store: &config.MemoryStore{
Config: &v1.Config{
Version: migrations.CurrentVersion,
Instance: "test-instance",
Multihost: &v1.Multihost{
Identity: serverPrivKey,
AuthorizedClients: []*v1.Multihost_Peer{
{
InstanceId: "client-instance",
Keyid: clientPrivKey.Keyid,
},
},
},
},
},
}
// Create a mock handler
mockHandler := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
peer := PeerFromContext(r.Context())
require.NotNil(t, peer)
assert.Equal(t, "client-instance", peer.InstanceId)
rw.WriteHeader(http.StatusOK)
})
// Create the auth handler
authHandler := newAuthHandler(cfgManager, mockHandler)
// Create a test server
server := httptest.NewServer(authHandler)
defer server.Close()
t.Run("valid auth header", func(t *testing.T) {
// Create a request with a valid auth header
req, err := http.NewRequest("GET", server.URL, nil)
require.NoError(t, err)
// Create a valid auth header
clientCfg := &v1.Config{
Instance: "client-instance",
Multihost: &v1.Multihost{
Identity: clientPrivKey,
},
}
authHeader, err := createAuthHeader(clientCfg)
require.NoError(t, err)
req.Header.Set(authTokenHeader, authHeader)
// Make the request
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
// Check the response
assert.Equal(t, http.StatusOK, resp.StatusCode)
})
t.Run("missing auth header", func(t *testing.T) {
req, err := http.NewRequest("GET", server.URL, nil)
require.NoError(t, err)
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
})
t.Run("invalid auth header", func(t *testing.T) {
req, err := http.NewRequest("GET", server.URL, nil)
require.NoError(t, err)
req.Header.Set(authTokenHeader, "invalid")
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
})
t.Run("unauthorized peer", func(t *testing.T) {
unauthorizedPrivKey, err := cryptoutil.GeneratePrivateKey()
require.NoError(t, err)
req, err := http.NewRequest("GET", server.URL, nil)
require.NoError(t, err)
clientCfg := &v1.Config{
Instance: "unauthorized-instance",
Multihost: &v1.Multihost{
Identity: unauthorizedPrivKey,
},
}
authHeader, err := createAuthHeader(clientCfg)
require.NoError(t, err)
req.Header.Set(authTokenHeader, authHeader)
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
})
t.Run("instance id mismatch", func(t *testing.T) {
req, err := http.NewRequest("GET", server.URL, nil)
require.NoError(t, err)
clientCfg := &v1.Config{
Instance: "wrong-instance",
Multihost: &v1.Multihost{
Identity: clientPrivKey,
},
}
authHeader, err := createAuthHeader(clientCfg)
require.NoError(t, err)
req.Header.Set(authTokenHeader, authHeader)
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
})
t.Run("signature too old", func(t *testing.T) {
req, err := http.NewRequest("GET", server.URL, nil)
require.NoError(t, err)
clientCfg := &v1.Config{
Instance: "client-instance",
Multihost: &v1.Multihost{
Identity: clientPrivKey,
},
}
privKey, err := cryptoutil.NewPrivateKey(clientPrivKey)
require.NoError(t, err)
// create a signed message with an old timestamp
signedMessage, err := createSignedMessage([]byte(clientCfg.Instance), privKey)
require.NoError(t, err)
signedMessage.TimestampMillis = time.Now().Add(-2 * maxSignatureAge).UnixMilli()
// create the auth token
authToken, err := createAuthToken(signedMessage, privKey)
require.NoError(t, err)
req.Header.Set(authTokenHeader, authToken)
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
})
}
func createAuthToken(signedMessage *v1.SignedMessage, privKey *cryptoutil.PrivateKey) (string, error) {
authToken := &v1sync.AuthorizationToken{
InstanceId: signedMessage,
PublicKey: privKey.PublicKeyProto(),
}
tokenBytes, err := proto.Marshal(authToken)
if err != nil {
return "", fmt.Errorf("marshal auth token: %w", err)
}
return base64.StdEncoding.EncodeToString(tokenBytes), nil
}
internal/api/syncapi/cmdstreamutil.go
+11 -7
View File
@@ -27,7 +27,7 @@ type bidiSyncCommandStream struct {
func newBidiSyncCommandStream() *bidiSyncCommandStream {
return &bidiSyncCommandStream{
sendChan: make(chan *v1sync.SyncStreamItem, 64), // Buffered channel to allow sending items without blocking
sendChan: make(chan *v1sync.SyncStreamItem, 256), // Buffered channel to allow sending items without blocking
recvChan: make(chan *v1sync.SyncStreamItem, 1),
terminateWithErrChan: make(chan error, 1),
}
@@ -74,15 +74,19 @@ func (s *bidiSyncCommandStream) ConnectStream(ctx context.Context, stream syncCo
ctx, cancel := context.WithCancel(ctx)
defer cancel()
go func() {
for ctx.Err() == nil {
if val, err := stream.Receive(); err != nil {
defer close(s.recvChan)
for {
val, err := stream.Receive()
if err != nil {
s.SendErrorAndTerminate(NewSyncErrorDisconnected(fmt.Errorf("receiving item: %w", err)))
break
} else {
s.recvChan <- val
return
}
select {
case s.recvChan <- val:
case <-ctx.Done():
return
}
}
close(s.recvChan)
}()
for {
internal/api/syncapi/errors.go
-28
View File
@@ -28,34 +28,6 @@ func NewSyncErrorDisconnected(message error) *SyncError {
}
}
func NewSyncErrorUnknown(message error) *SyncError {
return &SyncError{
State: v1sync.ConnectionState_CONNECTION_STATE_UNKNOWN,
Message: message,
}
}
func NewSyncErrorPending(message error) *SyncError {
return &SyncError{
State: v1sync.ConnectionState_CONNECTION_STATE_PENDING,
Message: message,
}
}
func NewSyncErrorConnected(message error) *SyncError {
return &SyncError{
State: v1sync.ConnectionState_CONNECTION_STATE_CONNECTED,
Message: message,
}
}
func NewSyncErrorRetryWait(message error) *SyncError {
return &SyncError{
State: v1sync.ConnectionState_CONNECTION_STATE_RETRY_WAIT,
Message: message,
}
}
func NewSyncErrorAuth(message error) *SyncError {
return &SyncError{
State: v1sync.ConnectionState_CONNECTION_STATE_ERROR_AUTH,
internal/api/syncapi/pairing_test.go
+416
View File
@@ -0,0 +1,416 @@
package syncapi
import (
"context"
"fmt"
"sync"
"testing"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/gen/go/v1sync"
"github.com/garethgeorge/backrest/internal/config/migrations"
"github.com/garethgeorge/backrest/internal/cryptoutil"
"github.com/garethgeorge/backrest/internal/testutil"
)
func TestValidatePairingSecret(t *testing.T) {
now := time.Unix(1000, 0)
tokens := []*v1.Multihost_PairingToken{
{
Secret: "valid-secret",
Label: "test-token",
CreatedAtUnix: 900,
ExpiresAtUnix: 2000,
MaxUses: 3,
Uses: 1,
},
{
Secret: "unlimited-token",
Label: "unlimited",
CreatedAtUnix: 900,
ExpiresAtUnix: 0, // no expiry
MaxUses: 0, // unlimited uses
Uses: 100,
},
}
tests := []struct {
name string
secret string
tokens []*v1.Multihost_PairingToken
now time.Time
wantLabel string
wantErr bool
}{
{
name: "valid secret",
secret: "valid-secret",
tokens: tokens,
now: now,
wantLabel: "test-token",
},
{
name: "unlimited token",
secret: "unlimited-token",
tokens: tokens,
now: now,
wantLabel: "unlimited",
},
{
name: "empty secret",
secret: "",
tokens: tokens,
now: now,
wantErr: true,
},
{
name: "wrong secret",
secret: "wrong-secret",
tokens: tokens,
now: now,
wantErr: true,
},
{
name: "expired token",
secret: "valid-secret",
tokens: []*v1.Multihost_PairingToken{
{
Secret: "valid-secret",
Label: "expired",
ExpiresAtUnix: 500,
MaxUses: 0,
},
},
now: now,
wantErr: true,
},
{
name: "max uses reached",
secret: "valid-secret",
tokens: []*v1.Multihost_PairingToken{
{
Secret: "valid-secret",
Label: "exhausted",
MaxUses: 2,
Uses: 2,
},
},
now: now,
wantErr: true,
},
{
name: "nil tokens",
secret: "anything",
tokens: nil,
now: now,
wantErr: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
token, err := ValidatePairingSecret(tc.secret, tc.tokens, tc.now)
if tc.wantErr {
if err == nil {
t.Fatal("expected error, got nil")
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if token.Label != tc.wantLabel {
t.Errorf("label = %q, want %q", token.Label, tc.wantLabel)
}
})
}
}
func TestPairingTokenFlow(t *testing.T) {
testutil.InstallZapLogger(t)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
peerHostAddr := testutil.AllocOpenBindAddr(t)
peerClientAddr := testutil.AllocOpenBindAddr(t)
pairingSecret, err := cryptoutil.GeneratePairingSecret()
if err != nil {
t.Fatalf("failed to generate pairing secret: %v", err)
}
// Host has a pairing token but NO authorized clients yet.
peerHostConfig := &v1.Config{
Version: migrations.CurrentVersion,
Instance: defaultHostID,
Repos: []*v1.Repo{},
Multihost: &v1.Multihost{
Identity: identity1,
AuthorizedClients: []*v1.Multihost_Peer{}, // empty — client not pre-authorized
PairingTokens: []*v1.Multihost_PairingToken{
{
Secret: pairingSecret,
Label: "test-pairing",
CreatedAtUnix: time.Now().Unix(),
ExpiresAtUnix: time.Now().Add(1 * time.Hour).Unix(),
MaxUses: 1,
Uses: 0,
},
},
},
}
// Client knows about the host and has the pairing secret.
peerClientConfig := &v1.Config{
Version: migrations.CurrentVersion,
Instance: defaultClientID,
Repos: []*v1.Repo{},
Multihost: &v1.Multihost{
Identity: identity2,
KnownHosts: []*v1.Multihost_Peer{
{
Keyid: identity1.Keyid,
InstanceId: defaultHostID,
InstanceUrl: fmt.Sprintf("http://%s", peerHostAddr),
InitialPairingSecret: pairingSecret,
},
},
},
}
peerHost := newPeerUnderTest(t, peerHostConfig)
peerClient := newPeerUnderTest(t, peerClientConfig)
startRunningSyncAPI(t, peerHost, peerHostAddr)
startRunningSyncAPI(t, peerClient, peerClientAddr)
// The client should successfully connect via the pairing token.
tryConnect(t, ctx, peerClient, peerClientConfig.Multihost.KnownHosts[0])
// Verify the host now has the client in its authorized_clients.
hostConfig, err := peerHost.configMgr.Get()
if err != nil {
t.Fatalf("failed to get host config: %v", err)
}
if len(hostConfig.Multihost.AuthorizedClients) != 1 {
t.Fatalf("expected 1 authorized client, got %d", len(hostConfig.Multihost.AuthorizedClients))
}
ac := hostConfig.Multihost.AuthorizedClients[0]
if ac.Keyid != identity2.Keyid {
t.Errorf("authorized client keyid = %q, want %q", ac.Keyid, identity2.Keyid)
}
if ac.InstanceId != defaultClientID {
t.Errorf("authorized client instance id = %q, want %q", ac.InstanceId, defaultClientID)
}
// Verify the pairing token was consumed (max_uses=1, so it should be removed).
if len(hostConfig.Multihost.PairingTokens) != 0 {
t.Errorf("expected 0 pairing tokens after consumption, got %d", len(hostConfig.Multihost.PairingTokens))
}
}
func TestPairingTokenExpiredRejected(t *testing.T) {
testutil.InstallZapLogger(t)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
peerHostAddr := testutil.AllocOpenBindAddr(t)
peerClientAddr := testutil.AllocOpenBindAddr(t)
pairingSecret, _ := cryptoutil.GeneratePairingSecret()
// Host has an expired pairing token.
peerHostConfig := &v1.Config{
Version: migrations.CurrentVersion,
Instance: defaultHostID,
Repos: []*v1.Repo{},
Multihost: &v1.Multihost{
Identity: identity1,
AuthorizedClients: []*v1.Multihost_Peer{},
PairingTokens: []*v1.Multihost_PairingToken{
{
Secret: pairingSecret,
Label: "expired-token",
CreatedAtUnix: time.Now().Add(-2 * time.Hour).Unix(),
ExpiresAtUnix: time.Now().Add(-1 * time.Hour).Unix(), // expired 1 hour ago
MaxUses: 0,
},
},
},
}
peerClientConfig := &v1.Config{
Version: migrations.CurrentVersion,
Instance: defaultClientID,
Repos: []*v1.Repo{},
Multihost: &v1.Multihost{
Identity: identity2,
KnownHosts: []*v1.Multihost_Peer{
{
Keyid: identity1.Keyid,
InstanceId: defaultHostID,
InstanceUrl: fmt.Sprintf("http://%s", peerHostAddr),
InitialPairingSecret: pairingSecret,
},
},
},
}
peerHost := newPeerUnderTest(t, peerHostConfig)
peerClient := newPeerUnderTest(t, peerClientConfig)
startRunningSyncAPI(t, peerHost, peerHostAddr)
startRunningSyncAPI(t, peerClient, peerClientAddr)
// Connection should fail with auth error.
waitForConnectionState(t, ctx, peerClient, peerClientConfig.Multihost.KnownHosts[0], v1sync.ConnectionState_CONNECTION_STATE_ERROR_AUTH)
// Host should still have no authorized clients.
hostConfig, _ := peerHost.configMgr.Get()
if len(hostConfig.Multihost.AuthorizedClients) != 0 {
t.Errorf("expected 0 authorized clients, got %d", len(hostConfig.Multihost.AuthorizedClients))
}
}
func TestPairingTokenMaxUsesEnforced(t *testing.T) {
testutil.InstallZapLogger(t)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
peerHostAddr := testutil.AllocOpenBindAddr(t)
pairingSecret, _ := cryptoutil.GeneratePairingSecret()
identity3, _ := cryptoutil.GeneratePrivateKey()
// Host has a pairing token with max_uses=1.
peerHostConfig := &v1.Config{
Version: migrations.CurrentVersion,
Instance: defaultHostID,
Repos: []*v1.Repo{},
Multihost: &v1.Multihost{
Identity: identity1,
AuthorizedClients: []*v1.Multihost_Peer{},
PairingTokens: []*v1.Multihost_PairingToken{
{
Secret: pairingSecret,
Label: "single-use",
CreatedAtUnix: time.Now().Unix(),
ExpiresAtUnix: time.Now().Add(1 * time.Hour).Unix(),
MaxUses: 1,
Uses: 0,
},
},
},
}
// First client pairs successfully.
peerClient1Config := &v1.Config{
Version: migrations.CurrentVersion,
Instance: "client-1",
Repos: []*v1.Repo{},
Multihost: &v1.Multihost{
Identity: identity2,
KnownHosts: []*v1.Multihost_Peer{
{
Keyid: identity1.Keyid,
InstanceId: defaultHostID,
InstanceUrl: fmt.Sprintf("http://%s", peerHostAddr),
InitialPairingSecret: pairingSecret,
},
},
},
}
peerHost := newPeerUnderTest(t, peerHostConfig)
peerClient1 := newPeerUnderTest(t, peerClient1Config)
var wg sync.WaitGroup
syncCtx, cancelSync := context.WithCancel(ctx)
wg.Add(2)
go func() { defer wg.Done(); runSyncAPIWithCtx(syncCtx, peerHost, peerHostAddr) }()
go func() {
defer wg.Done()
peerClient1Addr := testutil.AllocOpenBindAddr(t)
runSyncAPIWithCtx(syncCtx, peerClient1, peerClient1Addr)
}()
tryConnect(t, ctx, peerClient1, peerClient1Config.Multihost.KnownHosts[0])
// Stop first client, start second client with same pairing secret.
cancelSync()
wg.Wait()
peerClient2Config := &v1.Config{
Version: migrations.CurrentVersion,
Instance: "client-2",
Repos: []*v1.Repo{},
Multihost: &v1.Multihost{
Identity: identity3,
KnownHosts: []*v1.Multihost_Peer{
{
Keyid: identity1.Keyid,
InstanceId: defaultHostID,
InstanceUrl: fmt.Sprintf("http://%s", peerHostAddr),
InitialPairingSecret: pairingSecret,
},
},
},
}
peerClient2 := newPeerUnderTest(t, peerClient2Config)
startRunningSyncAPI(t, peerHost, peerHostAddr)
startRunningSyncAPI(t, peerClient2, testutil.AllocOpenBindAddr(t))
// Second client should fail — token is consumed.
waitForConnectionState(t, ctx, peerClient2, peerClient2Config.Multihost.KnownHosts[0], v1sync.ConnectionState_CONNECTION_STATE_ERROR_AUTH)
}
func TestNoPairingSecretRejected(t *testing.T) {
testutil.InstallZapLogger(t)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
peerHostAddr := testutil.AllocOpenBindAddr(t)
peerClientAddr := testutil.AllocOpenBindAddr(t)
// Host has NO pairing tokens and NO authorized clients.
peerHostConfig := &v1.Config{
Version: migrations.CurrentVersion,
Instance: defaultHostID,
Repos: []*v1.Repo{},
Multihost: &v1.Multihost{
Identity: identity1,
AuthorizedClients: []*v1.Multihost_Peer{},
},
}
// Client tries to connect without any pairing secret.
peerClientConfig := &v1.Config{
Version: migrations.CurrentVersion,
Instance: defaultClientID,
Repos: []*v1.Repo{},
Multihost: &v1.Multihost{
Identity: identity2,
KnownHosts: []*v1.Multihost_Peer{
{
Keyid: identity1.Keyid,
InstanceId: defaultHostID,
InstanceUrl: fmt.Sprintf("http://%s", peerHostAddr),
},
},
},
}
peerHost := newPeerUnderTest(t, peerHostConfig)
peerClient := newPeerUnderTest(t, peerClientConfig)
startRunningSyncAPI(t, peerHost, peerHostAddr)
startRunningSyncAPI(t, peerClient, peerClientAddr)
waitForConnectionState(t, ctx, peerClient, peerClientConfig.Multihost.KnownHosts[0], v1sync.ConnectionState_CONNECTION_STATE_ERROR_AUTH)
}
internal/api/syncapi/permissions/groups.go
+4
View File
@@ -23,4 +23,8 @@ var (
PermsCanViewOperations = []v1.Multihost_Permission_Type{
v1.Multihost_Permission_PERMISSION_READ_OPERATIONS,
}
PermsCanReceiveSharedRepos = []v1.Multihost_Permission_Type{
v1.Multihost_Permission_PERMISSION_RECEIVE_SHARED_REPOS,
}
)
internal/api/syncapi/permissions/permissions.go
+22 -3
View File
@@ -96,6 +96,9 @@ type PermissionSet struct {
// immutable after construction
perms map[v1.Multihost_Permission_Type]ScopeSet
// scopelessPerms tracks permission types that were granted without scopes (e.g. PERMISSION_RECEIVE_SHARED_REPOS)
scopelessPerms map[v1.Multihost_Permission_Type]bool
// caches store computed permission checks per scope id and permission type
// cache is best-effort and not bounded; PermissionSet is expected to be short-lived (per connection/request)
mu sync.RWMutex
@@ -105,13 +108,15 @@ type PermissionSet struct {
func NewPermissionSet(perms []*v1.Multihost_Permission) (*PermissionSet, error) {
permSet := &PermissionSet{
perms: make(map[v1.Multihost_Permission_Type]ScopeSet),
planCache: make(map[string]map[v1.Multihost_Permission_Type]bool),
repoCache: make(map[string]map[v1.Multihost_Permission_Type]bool),
perms: make(map[v1.Multihost_Permission_Type]ScopeSet),
scopelessPerms: make(map[v1.Multihost_Permission_Type]bool),
planCache: make(map[string]map[v1.Multihost_Permission_Type]bool),
repoCache: make(map[string]map[v1.Multihost_Permission_Type]bool),
}
for _, perm := range perms {
if perm.Scopes == nil {
permSet.scopelessPerms[perm.Type] = true
continue
}
scopeSet, err := NewScopeSet(perm.Scopes)
@@ -124,6 +129,20 @@ func NewPermissionSet(perms []*v1.Multihost_Permission) (*PermissionSet, error)
return permSet, nil
}
// HasPermissionType checks if any of the given permission types are granted, regardless of scopes.
// Use this for scope-less permissions like PERMISSION_RECEIVE_SHARED_REPOS.
func (p *PermissionSet) HasPermissionType(permTypes ...v1.Multihost_Permission_Type) bool {
for _, permType := range permTypes {
if _, ok := p.scopelessPerms[permType]; ok {
return true
}
if _, ok := p.perms[permType]; ok {
return true
}
}
return false
}
func (p *PermissionSet) CheckPermissionForPlan(planID string, permType ...v1.Multihost_Permission_Type) bool {
for _, pt := range permType {
if p.checkPlanSingle(planID, pt) {
internal/api/syncapi/signing.go
+70
View File
@@ -0,0 +1,70 @@
package syncapi
import (
"encoding/binary"
"errors"
"fmt"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/cryptoutil"
)
const maxSignatureAge = 5 * time.Minute
func createSignedMessage(payload []byte, identity *cryptoutil.PrivateKey) (*v1.SignedMessage, error) {
if len(payload) == 0 {
return nil, errors.New("payload must not be empty")
}
timestampMillis := time.Now().UnixMilli()
payloadWithTimestamp := make([]byte, 0, len(payload)+8)
payloadWithTimestamp = binary.BigEndian.AppendUint64(payloadWithTimestamp, uint64(timestampMillis))
payloadWithTimestamp = append(payloadWithTimestamp, payload...)
signature, err := identity.Sign(payloadWithTimestamp)
if err != nil {
return nil, fmt.Errorf("signing payload: %w", err)
}
return &v1.SignedMessage{
Payload: payload,
Signature: signature,
Keyid: identity.KeyID(),
TimestampMillis: timestampMillis,
}, nil
}
func verifySignedMessage(msg *v1.SignedMessage, publicKey *cryptoutil.PublicKey) error {
if msg == nil {
return errors.New("signed message must not be nil")
}
if len(msg.GetPayload()) == 0 {
return errors.New("signed message payload must not be empty")
}
if len(msg.GetSignature()) == 0 {
return errors.New("signed message signature must not be empty")
}
if len(msg.GetKeyid()) == 0 {
return errors.New("signed message key ID must not be empty")
}
if publicKey.KeyID() != msg.GetKeyid() {
return fmt.Errorf("public key ID mismatch: expected %s, got %s", publicKey.KeyID(), msg.GetKeyid())
}
payloadWithTimestamp := make([]byte, 0, len(msg.GetPayload())+8)
payloadWithTimestamp = binary.BigEndian.AppendUint64(payloadWithTimestamp, uint64(msg.GetTimestampMillis()))
payloadWithTimestamp = append(payloadWithTimestamp, msg.GetPayload()...)
if err := publicKey.Verify(payloadWithTimestamp, msg.GetSignature()); err != nil {
return fmt.Errorf("verifying signed message: %w", err)
}
if time.Since(time.UnixMilli(msg.GetTimestampMillis())) > maxSignatureAge {
return fmt.Errorf("signature is too old, max age is %s. Is the clock out of sync?", maxSignatureAge)
}
return nil
}
internal/api/syncapi/syncapi_test.go
+4 -8
View File
@@ -88,8 +88,7 @@ func TestConnectionSucceeds(t *testing.T) {
AuthorizedClients: []*v1.Multihost_Peer{
{
Keyid: identity2.Keyid,
KeyidVerified: true,
InstanceId: defaultClientID,
InstanceId: defaultClientID,
},
},
},
@@ -192,8 +191,7 @@ func TestSyncConfigChange(t *testing.T) {
AuthorizedClients: []*v1.Multihost_Peer{
{
Keyid: identity2.Keyid,
KeyidVerified: true,
InstanceId: defaultClientID,
InstanceId: defaultClientID,
Permissions: []*v1.Multihost_Permission{
{
Type: v1.Multihost_Permission_PERMISSION_READ_CONFIG,
@@ -292,8 +290,7 @@ func TestSimpleOperationSync(t *testing.T) {
AuthorizedClients: []*v1.Multihost_Peer{
{
Keyid: identity2.Keyid,
KeyidVerified: true,
InstanceId: defaultClientID,
InstanceId: defaultClientID,
},
},
},
@@ -419,8 +416,7 @@ func TestSyncMutations(t *testing.T) {
AuthorizedClients: []*v1.Multihost_Peer{
{
Keyid: identity2.Keyid,
KeyidVerified: true,
InstanceId: defaultClientID,
InstanceId: defaultClientID,
},
},
},
internal/api/syncapi/syncclient.go
+252 -163
View File
@@ -109,37 +109,48 @@ func (c *SyncClient) RunSync(ctx context.Context) {
cmdStream,
syncSessionHandler,
c.syncConfigSnapshot.config.GetMultihost().GetKnownHosts(),
c.peer.GetInitialPairingSecret(),
nil, // client never handles unknown peers
)
cmdStream.SendErrorAndTerminate(err)
}()
if err := cmdStream.ConnectStream(ctx, c.client.Sync(ctx)); err != nil {
c.l.Sugar().Infof("lost stream connection to peer %q (%s): %v", c.peer.InstanceId, c.peer.Keyid, err)
connectErr := cmdStream.ConnectStream(ctx, c.client.Sync(ctx))
if connectErr != nil {
c.l.Sugar().Infof("lost stream connection to peer %q (%s): %v", c.peer.InstanceId, c.peer.Keyid, connectErr)
var syncErr *SyncError
state := c.mgr.peerStateManager.GetPeerState(c.peer.Keyid).Clone()
if state == nil {
state = newPeerState(c.peer.InstanceId, c.peer.Keyid)
}
state.LastHeartbeat = time.Now()
if errors.As(err, &syncErr) {
if errors.As(connectErr, &syncErr) {
state.ConnectionState = syncErr.State
state.ConnectionStateMessage = syncErr.Message.Error()
} else {
state.ConnectionState = v1sync.ConnectionState_CONNECTION_STATE_ERROR_INTERNAL
state.ConnectionStateMessage = err.Error()
state.ConnectionStateMessage = connectErr.Error()
}
c.mgr.peerStateManager.SetPeerState(c.peer.Keyid, state)
} else {
c.reconnectAttempts = 0
}
wg.Wait()
// Reset reconnect backoff if the session lasted long enough to be considered a real success,
// rather than a handshake that failed immediately. Using reconnectDelay as the threshold means
// any session that ran at least one full retry window counts as stable.
if time.Since(lastConnect) >= c.reconnectDelay {
c.reconnectAttempts = 0
}
delay := c.reconnectDelay - time.Since(lastConnect)
if c.reconnectAttempts > 0 {
backoff := time.Duration(1<<min(c.reconnectAttempts, 5)) * c.reconnectDelay // 2^reconnectAttempts, max 32
delay += backoff
}
if delay < 0 {
delay = 0
}
c.l.Sugar().Infof("disconnected, will retry after %v (attempt %d)", delay, c.reconnectAttempts)
c.reconnectAttempts++
select {
@@ -165,6 +176,8 @@ type syncSessionHandlerClient struct {
canForwardReposSet map[string]struct{}
canForwardPlansSet map[string]struct{}
oplogSubscription *oplog.Subscription // set while subscribed; unsubscribed in OnConnectionDisconnected.
}
func newSyncHandlerClient(
@@ -217,6 +230,43 @@ func (c *syncSessionHandlerClient) canForwardOperation(op *v1.Operation) bool {
return false
}
func (c *syncSessionHandlerClient) canForwardMeta(meta oplog.OpMetadata) bool {
if meta.OriginalID != 0 {
return false // don't forward ops received from other peers
}
if _, ok := c.canForwardReposSet[meta.RepoGUID]; ok {
return true
}
if meta.PlanID != "" {
if _, ok := c.canForwardPlansSet[meta.PlanID]; ok {
return true
}
}
return false
}
func (c *syncSessionHandlerClient) sendManifest(stream *bidiSyncCommandStream) (int, error) {
var opIDs, modnos []int64
if err := c.oplog.QueryMetadata(oplog.Query{}, func(meta oplog.OpMetadata) error {
if c.canForwardMeta(meta) {
opIDs = append(opIDs, meta.ID)
modnos = append(modnos, meta.Modno)
}
return nil
}); err != nil {
return 0, fmt.Errorf("querying operation metadata for manifest: %w", err)
}
stream.Send(&v1sync.SyncStreamItem{
Action: &v1sync.SyncStreamItem_OperationManifest{
OperationManifest: &v1sync.SyncStreamItem_SyncActionOperationManifest{
OpIds: opIDs,
Modnos: modnos,
},
},
})
return len(opIDs), nil
}
func (c *syncSessionHandlerClient) OnConnectionEstablished(ctx context.Context, stream *bidiSyncCommandStream, peer *v1.Multihost_Peer) error {
// A client expects to connect to a specific peer, so we check that the peer we connected to matches the one we expect.
if !proto.Equal(c.peer, peer) {
@@ -242,16 +292,40 @@ func (c *syncSessionHandlerClient) OnConnectionEstablished(ctx context.Context,
peerState.LastHeartbeat = time.Now()
c.mgr.peerStateManager.SetPeerState(peer.Keyid, peerState)
// Clear the pairing secret from the known host entry now that pairing has succeeded.
snapshotHosts := c.syncConfigSnapshot.config.GetMultihost().GetKnownHosts()
khIdx := slices.IndexFunc(snapshotHosts, func(kh *v1.Multihost_Peer) bool {
return kh.GetKeyid() == peer.GetKeyid()
})
if khIdx >= 0 && snapshotHosts[khIdx].GetInitialPairingSecret() != "" {
if err := c.mgr.configMgr.Transform(func(cfg *v1.Config) (*v1.Config, error) {
idx := slices.IndexFunc(cfg.GetMultihost().GetKnownHosts(), func(kh *v1.Multihost_Peer) bool {
return kh.GetKeyid() == peer.GetKeyid()
})
if idx >= 0 {
cfg.GetMultihost().GetKnownHosts()[idx].InitialPairingSecret = ""
}
cfg.Modno++
return cfg, nil
}); err != nil {
c.l.Sugar().Warnf("failed to clear pairing secret after successful pairing: %v", err)
} else {
c.l.Sugar().Infof("cleared pairing secret for peer %q after successful connection", peer.InstanceId)
}
}
// Send a heartbeat every interval to keep the connection alive.
go sendHeartbeats(ctx, stream, env.MultihostHeartbeatInterval())
// Forward a view of our config (if the peer is allowed to see it).
if err := c.sendConfig(ctx, stream); err != nil {
repoCount, planCount, err := c.sendConfig(ctx, stream)
if err != nil {
return fmt.Errorf("send config to peer %q: %w", peer.InstanceId, err)
}
// Forward a list of the resources we're making available to the peer
if err := c.sendResourceList(ctx, stream); err != nil {
resRepoCount, resPlanCount, err := c.sendResourceList(ctx, stream)
if err != nil {
return fmt.Errorf("send resource list to peer %q: %w", peer.InstanceId, err)
}
@@ -305,16 +379,31 @@ func (c *syncSessionHandlerClient) OnConnectionEstablished(ctx context.Context,
},
})
}
c.oplog.Subscribe(oplog.Query{}, &oplogSubscription)
go func() {
<-ctx.Done()
c.oplog.Unsubscribe(&oplogSubscription)
}()
c.oplogSubscription = &oplogSubscription
c.oplog.Subscribe(oplog.Query{}, c.oplogSubscription)
// Send initial operation manifest to the server for reconciliation.
opCount, err := c.sendManifest(stream)
if err != nil {
return fmt.Errorf("send manifest to peer %q: %w", peer.InstanceId, err)
}
c.l.Sugar().Infof("sent initial state to server: %d operations, %d repos, %d plans (config); %d repos, %d plans (resources)",
opCount, repoCount, planCount, resRepoCount, resPlanCount)
return nil
}
func (c *syncSessionHandlerClient) OnConnectionDisconnected() {
if c.oplogSubscription != nil {
c.oplog.Unsubscribe(c.oplogSubscription)
c.oplogSubscription = nil
}
}
func (c *syncSessionHandlerClient) HandleRequestResources(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionRequestResources) error {
return c.sendResourceList(ctx, stream)
_, _, err := c.sendResourceList(ctx, stream)
return err
}
func (c *syncSessionHandlerClient) HandleHeartbeat(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionHeartbeat) error {
@@ -327,79 +416,46 @@ func (c *syncSessionHandlerClient) HandleHeartbeat(ctx context.Context, stream *
return nil
}
func (c *syncSessionHandlerClient) HandleRequestOperations(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionRequestOperations) error {
highModno := item.GetHighModno()
highOpid := item.GetHighOpid()
c.l.Sugar().Debugf("received operation request for high_modno: %d, high_opid: %d", highModno, highOpid)
var batch []*v1.Operation
send := func() error {
if len(batch) == 0 {
return nil
}
// find new and updated operations
var newOps []*v1.Operation
var updatedOps []*v1.Operation
for _, op := range batch {
if op.GetId() > highOpid {
newOps = append(newOps, op)
} else {
updatedOps = append(updatedOps, op)
}
}
// send new and updated operations
if len(newOps) > 0 {
stream.Send(&v1sync.SyncStreamItem{
Action: &v1sync.SyncStreamItem_ReceiveOperations{
ReceiveOperations: &v1sync.SyncStreamItem_SyncActionReceiveOperations{
Event: &v1.OperationEvent{
Event: &v1.OperationEvent_CreatedOperations{
CreatedOperations: &v1.OperationList{Operations: batch},
},
},
},
},
})
}
if len(updatedOps) > 0 {
stream.Send(&v1sync.SyncStreamItem{
Action: &v1sync.SyncStreamItem_ReceiveOperations{
ReceiveOperations: &v1sync.SyncStreamItem_SyncActionReceiveOperations{
Event: &v1.OperationEvent{
Event: &v1.OperationEvent_UpdatedOperations{
UpdatedOperations: &v1.OperationList{Operations: updatedOps},
},
},
},
},
})
}
batch = batch[:0]
return nil
}
c.oplog.Query(oplog.Query{}.SetModnoGte(highModno), func(op *v1.Operation) error {
if !c.canForwardOperation(op) {
return nil // skip operations that the peer is not allowed to read
}
batch = append(batch, op)
if len(batch) >= 256 {
if err := send(); err != nil {
return err
}
}
return nil
})
if err := send(); err != nil {
func (c *syncSessionHandlerClient) HandleOperationManifest(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionOperationManifest) error {
// Server re-requested a manifest (e.g. after reconnect). Respond with a fresh one.
opCount, err := c.sendManifest(stream)
if err != nil {
return err
}
c.l.Sugar().Debugf("re-sent operation manifest with %d operations", opCount)
return nil
}
func (c *syncSessionHandlerClient) HandleRequestOperationData(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionRequestOperationData) error {
var batch []*v1.Operation
send := func() {
if len(batch) == 0 {
return
}
stream.Send(&v1sync.SyncStreamItem{
Action: &v1sync.SyncStreamItem_ReceiveOperations{
ReceiveOperations: &v1sync.SyncStreamItem_SyncActionReceiveOperations{
Event: &v1.OperationEvent{
Event: &v1.OperationEvent_UpdatedOperations{
UpdatedOperations: &v1.OperationList{Operations: batch},
},
},
},
},
})
batch = batch[:0]
}
for _, id := range item.GetOpIds() {
op, err := c.oplog.Get(id)
if err != nil {
continue // may have been deleted between manifest and request
}
batch = append(batch, op)
if len(batch) >= 256 {
send()
}
}
send()
return nil
}
@@ -408,9 +464,7 @@ func (c *syncSessionHandlerClient) HandleReceiveOperations(ctx context.Context,
}
func (c *syncSessionHandlerClient) HandleReceiveResources(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionReceiveResources) error {
c.l.Debug("received resource list from server",
zap.Any("repos", item.GetRepos()),
zap.Any("plans", item.GetPlans()))
c.l.Sugar().Debugf("received resource list from server: %d repos, %d plans", len(item.GetRepos()), len(item.GetPlans()))
peerState := c.mgr.peerStateManager.GetPeerState(c.peer.Keyid).Clone()
if peerState == nil {
return NewSyncErrorInternal(fmt.Errorf("peer state for %q not found", c.peer.Keyid))
@@ -429,7 +483,8 @@ func (c *syncSessionHandlerClient) HandleReceiveResources(ctx context.Context, s
// Note unused: there isn't a situation where the host would send its config for information, the host will only call 'SetConfig' to update the config.
func (c *syncSessionHandlerClient) HandleReceiveConfig(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionReceiveConfig) error {
c.l.Sugar().Debugf("received remote config update")
c.l.Sugar().Debugf("received remote config: %d repos, %d plans, modno=%d",
len(item.GetConfig().GetRepos()), len(item.GetConfig().GetPlans()), item.GetConfig().GetModno())
peerState := c.mgr.peerStateManager.GetPeerState(c.peer.Keyid).Clone()
if peerState == nil {
return NewSyncErrorInternal(fmt.Errorf("peer state for %q not found", c.peer.Keyid))
@@ -444,95 +499,129 @@ func (c *syncSessionHandlerClient) HandleReceiveConfig(ctx context.Context, stre
}
func (c *syncSessionHandlerClient) HandleSetConfig(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionSetConfig) error {
// Log the received config updates
c.l.Sugar().Debugf("received SetConfig request from peer %q")
return c.mgr.configMgr.Transform(func(cfg *v1.Config) (*v1.Config, error) {
snapshot := proto.Clone(cfg).(*v1.Config) // snapshot for change detection
// Fetch latest config from the config manager
latestConfig, err := c.mgr.configMgr.Get()
if err != nil {
return fmt.Errorf("fetch latest config: %w", err)
}
var plansNew, plansUpdated, plansUnchanged int
for _, plan := range item.GetPlans() {
if !c.permissions.CheckPermissionForPlan(plan.Id, permissions.PermsCanWriteConfiguration...) {
return nil, NewSyncErrorAuth(fmt.Errorf("peer %q is not allowed to update plan %q", c.peer.InstanceId, plan.Id))
}
latestConfig = proto.Clone(latestConfig).(*v1.Config) // Clone to avoid modifying the original config
for _, plan := range item.GetPlans() {
c.l.Sugar().Debugf("received plan update: %s", plan.Id)
if !c.permissions.CheckPermissionForPlan(plan.Id, permissions.PermsCanWriteConfiguration...) {
return NewSyncErrorAuth(fmt.Errorf("peer %q is not allowed to update plan %q", c.peer.InstanceId, plan.Id))
idx := slices.IndexFunc(cfg.Plans, func(p *v1.Plan) bool {
return p.Id == plan.Id
})
if idx >= 0 {
if proto.Equal(cfg.Plans[idx], plan) {
c.l.Sugar().Debugf("received plan %s (unchanged)", plan.Id)
plansUnchanged++
} else {
c.l.Sugar().Debugf("received plan %s (updated)", plan.Id)
plansUpdated++
}
cfg.Plans[idx] = plan
} else {
c.l.Sugar().Debugf("received plan %s (new)", plan.Id)
plansNew++
cfg.Plans = append(cfg.Plans, plan)
}
}
// Update the plan in the local config
idx := slices.IndexFunc(latestConfig.Plans, func(p *v1.Plan) bool {
return p.Id == plan.Id
})
if idx >= 0 {
latestConfig.Plans[idx] = plan
} else {
latestConfig.Plans = append(latestConfig.Plans, plan)
}
}
var reposNew, reposUpdated, reposUnchanged, reposSkipped int
for _, repo := range item.GetRepos() {
idx := slices.IndexFunc(cfg.Repos, func(r *v1.Repo) bool {
return r.Guid == repo.Guid
})
for _, repo := range item.GetRepos() {
c.l.Sugar().Debugf("received repo update: %s", repo.Guid)
if !c.permissions.CheckPermissionForRepo(repo.Id, permissions.PermsCanWriteConfiguration...) {
return NewSyncErrorAuth(fmt.Errorf("peer %q is not allowed to update repo %q", c.peer.InstanceId, repo.Id))
// Permission check: accept if we have RECEIVE_SHARED_REPOS and the repo
// is either new or already owned by this peer; otherwise require scoped write perms.
isNewOrOwnedByPeer := idx < 0 || cfg.Repos[idx].GetOriginInstanceId() == c.peer.InstanceId
allowed := (isNewOrOwnedByPeer && c.permissions.HasPermissionType(permissions.PermsCanReceiveSharedRepos...)) ||
c.permissions.CheckPermissionForRepo(repo.Id, permissions.PermsCanWriteConfiguration...)
if !allowed {
return nil, NewSyncErrorAuth(fmt.Errorf("peer %q is not allowed to update repo %q", c.peer.InstanceId, repo.Id))
}
if idx >= 0 {
if proto.Equal(cfg.Repos[idx], repo) {
c.l.Sugar().Debugf("received repo %s (unchanged)", repo.Id)
reposUnchanged++
} else {
c.l.Sugar().Debugf("received repo %s (updated)", repo.Id)
reposUpdated++
}
cfg.Repos[idx] = repo
} else {
conflictIdx := slices.IndexFunc(cfg.Repos, func(r *v1.Repo) bool {
return r.Id == repo.Id || r.Uri == repo.Uri
})
if conflictIdx >= 0 {
c.l.Sugar().Warnf("received shared repo %q (guid %s) conflicts with existing local repo %q (guid %s), skipping", repo.Id, repo.Guid, cfg.Repos[conflictIdx].Id, cfg.Repos[conflictIdx].Guid)
reposSkipped++
continue
}
c.l.Sugar().Debugf("received repo %s (new)", repo.Id)
reposNew++
cfg.Repos = append(cfg.Repos, repo)
}
}
// Update the repo in the local config
idx := slices.IndexFunc(latestConfig.Repos, func(r *v1.Repo) bool {
return r.Guid == repo.Guid
})
if idx >= 0 {
latestConfig.Repos[idx] = repo
} else {
latestConfig.Repos = append(latestConfig.Repos, repo)
}
}
var plansDeleted int
for _, plan := range item.GetPlansToDelete() {
if !c.permissions.CheckPermissionForPlan(plan, permissions.PermsCanWriteConfiguration...) {
return nil, NewSyncErrorAuth(fmt.Errorf("peer %q is not allowed to delete plan %q", c.peer.InstanceId, plan))
}
for _, plan := range item.GetPlansToDelete() {
c.l.Sugar().Debugf("received plan deletion request: %s", plan)
if !c.permissions.CheckPermissionForPlan(plan, permissions.PermsCanWriteConfiguration...) {
return NewSyncErrorAuth(fmt.Errorf("peer %q is not allowed to delete plan %q", c.peer.InstanceId, plan))
idx := slices.IndexFunc(cfg.Plans, func(p *v1.Plan) bool {
return p.Id == plan
})
if idx >= 0 {
c.l.Sugar().Debugf("received plan deletion: %s", plan)
plansDeleted++
cfg.Plans = append(cfg.Plans[:idx], cfg.Plans[idx+1:]...)
} else {
c.l.Sugar().Warnf("received plan deletion request for non-existent plan %q, ignoring", plan)
}
}
// Remove the plan from the local config
idx := slices.IndexFunc(latestConfig.Plans, func(p *v1.Plan) bool {
return p.Id == plan
})
if idx >= 0 {
latestConfig.Plans = append(latestConfig.Plans[:idx], latestConfig.Plans[idx+1:]...)
} else {
c.l.Sugar().Warnf("received plan deletion request for non-existent plan %q, ignoring", plan)
}
}
var reposDeleted int
for _, repoID := range item.GetReposToDelete() {
if !c.permissions.CheckPermissionForRepo(repoID, permissions.PermsCanWriteConfiguration...) {
return nil, NewSyncErrorAuth(fmt.Errorf("peer %q is not allowed to delete repo %q", c.peer.InstanceId, repoID))
}
for _, repoID := range item.GetReposToDelete() {
c.l.Sugar().Debugf("received repo deletion request: %s", repoID)
if !c.permissions.CheckPermissionForRepo(repoID, permissions.PermsCanWriteConfiguration...) {
return NewSyncErrorAuth(fmt.Errorf("peer %q is not allowed to delete repo %q", c.peer.InstanceId, repoID))
idx := slices.IndexFunc(cfg.Repos, func(r *v1.Repo) bool {
return r.Id == repoID
})
if idx >= 0 {
c.l.Sugar().Debugf("received repo deletion: %s", repoID)
reposDeleted++
cfg.Repos = append(cfg.Repos[:idx], cfg.Repos[idx+1:]...)
} else {
c.l.Sugar().Warnf("received repo deletion request for non-existent repo %q, ignoring", repoID)
}
}
// Remove the repo from the local config
idx := slices.IndexFunc(latestConfig.Repos, func(r *v1.Repo) bool {
return r.Id == repoID
})
if idx >= 0 {
latestConfig.Repos = append(latestConfig.Repos[:idx], latestConfig.Repos[idx+1:]...)
} else {
c.l.Sugar().Warnf("received repo deletion request for non-existent repo %q, ignoring", repoID)
// Skip the update if nothing actually changed to avoid triggering a reconnect loop.
hasChanges := !proto.Equal(cfg, snapshot)
if hasChanges {
cfg.Modno++
}
}
// Update the local config with the new changes
latestConfig.Modno++
if err := c.mgr.configMgr.Update(latestConfig); err != nil {
return fmt.Errorf("set updated config: %w", err)
}
c.l.Sugar().Debugf("SetConfig from peer %q: repos(%d new, %d updated, %d unchanged, %d skipped, %d deleted) plans(%d new, %d updated, %d unchanged, %d deleted) — config %s",
c.peer.GetInstanceId(),
reposNew, reposUpdated, reposUnchanged, reposSkipped, reposDeleted,
plansNew, plansUpdated, plansUnchanged, plansDeleted,
map[bool]string{true: "updated", false: "unchanged"}[hasChanges])
return nil
if !hasChanges {
return nil, nil
}
return cfg, nil
})
}
func (c *syncSessionHandlerClient) sendConfig(ctx context.Context, stream *bidiSyncCommandStream) error {
func (c *syncSessionHandlerClient) sendConfig(ctx context.Context, stream *bidiSyncCommandStream) (int, int, error) {
localConfig := c.syncConfigSnapshot.config
remoteConfig := &v1sync.RemoteConfig{
Version: localConfig.Version,
@@ -540,7 +629,7 @@ func (c *syncSessionHandlerClient) sendConfig(ctx context.Context, stream *bidiS
}
for _, repo := range localConfig.Repos {
if c.permissions.CheckPermissionForRepo(repo.Guid, permissions.PermsCanViewConfiguration...) {
if c.permissions.CheckPermissionForRepo(repo.Id, permissions.PermsCanViewConfiguration...) {
remoteConfig.Repos = append(remoteConfig.Repos, repo)
}
}
@@ -559,10 +648,10 @@ func (c *syncSessionHandlerClient) sendConfig(ctx context.Context, stream *bidiS
},
})
return nil
return len(remoteConfig.Repos), len(remoteConfig.Plans), nil
}
func (c *syncSessionHandlerClient) sendResourceList(ctx context.Context, stream *bidiSyncCommandStream) error {
func (c *syncSessionHandlerClient) sendResourceList(ctx context.Context, stream *bidiSyncCommandStream) (int, int, error) {
repoMetadatas := []*v1sync.RepoMetadata{}
planMetadatas := []*v1sync.PlanMetadata{}
@@ -592,5 +681,5 @@ func (c *syncSessionHandlerClient) sendResourceList(ctx context.Context, stream
},
})
return nil
return len(repoMetadatas), len(planMetadatas), nil
}
internal/api/syncapi/synccommon.go
+74 -23
View File
@@ -16,6 +16,12 @@ import (
lru "github.com/hashicorp/golang-lru/v2"
)
// onUnknownPeerFunc is called when a peer is not found in the known peers list during handshake.
// It receives the handshake item and may return a peer definition to authorize the connection
// (e.g. by validating a pairing token and adding the peer to the config).
// If it returns nil, the connection is rejected.
type onUnknownPeerFunc func(handshake *v1sync.SyncStreamItem) (*v1.Multihost_Peer, error)
func runSync(
ctx context.Context,
localInstanceID string,
@@ -23,9 +29,17 @@ func runSync(
commandStream *bidiSyncCommandStream,
handler syncSessionHandler,
knownPeers []*v1.Multihost_Peer, // could be known hosts or authorized clients, doesn't matter. This is used to verify the handshake packet, authorization comes later.
pairingSecret string, // optional one-time pairing secret to send during the handshake
onUnknownPeer onUnknownPeerFunc, // optional callback for handling unknown peers (e.g. pairing), nil to reject all unknown peers
) error {
// Session-scoped context: cancelled when this runSync invocation returns. Any per-session
// goroutines the handler spawns (heartbeats, watchers, etc.) should use this ctx so they
// die with the session rather than outliving it into the next reconnect cycle.
ctx, cancel := context.WithCancel(ctx)
defer cancel()
// send the initial handshake packet to the peer to establish the connection.
handshakePacket, err := createHandshakePacket(localInstanceID, localKey)
handshakePacket, err := createHandshakePacket(localInstanceID, localKey, pairingSecret)
if err != nil {
return NewSyncErrorAuth(fmt.Errorf("creating handshake packet: %w", err))
}
@@ -47,7 +61,14 @@ func runSync(
})
if peerIdx >= 0 {
peer = knownPeers[peerIdx]
} else {
} else if onUnknownPeer != nil {
// Peer not in known list — try the onUnknownPeer callback (e.g. pairing token validation).
peer, err = onUnknownPeer(handshake)
if err != nil {
return NewSyncErrorAuth(fmt.Errorf("pairing failed: %w", err))
}
}
if peer == nil {
return NewSyncErrorAuth(fmt.Errorf("peer public key ID %s (instance ID %s) not found in known peers", handshake.GetHandshake().GetPublicKey().GetKeyid(), string(handshake.GetHandshake().GetInstanceId().GetPayload())))
}
@@ -55,6 +76,8 @@ func runSync(
return NewSyncErrorAuth(fmt.Errorf("authorizing handshake as peer: %w", err))
}
defer handler.OnConnectionDisconnected()
if err := handler.OnConnectionEstablished(ctx, commandStream, peer); err != nil {
return err
}
@@ -65,9 +88,13 @@ func runSync(
if err := handler.HandleHeartbeat(ctx, commandStream, item.GetHeartbeat()); err != nil {
return fmt.Errorf("handling heartbeat: %w", err)
}
case *v1sync.SyncStreamItem_RequestOperations:
if err := handler.HandleRequestOperations(ctx, commandStream, item.GetRequestOperations()); err != nil {
return fmt.Errorf("handling request operations: %w", err)
case *v1sync.SyncStreamItem_OperationManifest:
if err := handler.HandleOperationManifest(ctx, commandStream, item.GetOperationManifest()); err != nil {
return fmt.Errorf("handling operation manifest: %w", err)
}
case *v1sync.SyncStreamItem_RequestOperationData:
if err := handler.HandleRequestOperationData(ctx, commandStream, item.GetRequestOperationData()); err != nil {
return fmt.Errorf("handling request operation data: %w", err)
}
case *v1sync.SyncStreamItem_ReceiveOperations:
if err := handler.HandleReceiveOperations(ctx, commandStream, item.GetReceiveOperations()); err != nil {
@@ -108,7 +135,7 @@ func runSync(
return nil
}
func createHandshakePacket(instanceID string, identity *cryptoutil.PrivateKey) (*v1sync.SyncStreamItem, error) {
func createHandshakePacket(instanceID string, identity *cryptoutil.PrivateKey, pairingSecret string) (*v1sync.SyncStreamItem, error) {
signedMessage, err := createSignedMessage([]byte(instanceID), identity)
if err != nil {
return nil, fmt.Errorf("signing instance ID: %w", err)
@@ -120,6 +147,7 @@ func createHandshakePacket(instanceID string, identity *cryptoutil.PrivateKey) (
ProtocolVersion: SyncProtocolVersion,
InstanceId: signedMessage,
PublicKey: identity.PublicKeyProto(),
PairingSecret: pairingSecret,
},
},
}, nil
@@ -198,10 +226,15 @@ func sendHeartbeats(ctx context.Context, stream *bidiSyncCommandStream, interval
// syncSessionHandler is a stateful handler for the messages within the context of a sync stream session.
// the handler does not need to be thread safe as it is guaranteed to be called from a single thread.
//
// The ctx passed to every method is scoped to the session: it is cancelled when runSync returns.
// Goroutines spawned by the handler should use this ctx so they don't leak across reconnect cycles.
type syncSessionHandler interface {
OnConnectionEstablished(ctx context.Context, stream *bidiSyncCommandStream, peer *v1.Multihost_Peer) error
OnConnectionDisconnected()
HandleHeartbeat(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionHeartbeat) error
HandleRequestOperations(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionRequestOperations) error
HandleOperationManifest(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionOperationManifest) error
HandleRequestOperationData(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionRequestOperationData) error
HandleReceiveOperations(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionReceiveOperations) error
HandleReceiveConfig(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionReceiveConfig) error
HandleSetConfig(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionSetConfig) error
@@ -220,12 +253,18 @@ func (h *unimplementedSyncSessionHandler) OnConnectionEstablished(ctx context.Co
return NewSyncErrorProtocol(fmt.Errorf("OnConnectionEstablished not implemented"))
}
func (h *unimplementedSyncSessionHandler) OnConnectionDisconnected() {}
func (h *unimplementedSyncSessionHandler) HandleHeartbeat(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionHeartbeat) error {
return NewSyncErrorProtocol(fmt.Errorf("HandleHeartbeat not implemented"))
}
func (h *unimplementedSyncSessionHandler) HandleRequestOperations(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionRequestOperations) error {
return NewSyncErrorProtocol(fmt.Errorf("HandleRequestOperations not implemented"))
func (h *unimplementedSyncSessionHandler) HandleOperationManifest(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionOperationManifest) error {
return NewSyncErrorProtocol(fmt.Errorf("HandleOperationManifest not implemented"))
}
func (h *unimplementedSyncSessionHandler) HandleRequestOperationData(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionRequestOperationData) error {
return NewSyncErrorProtocol(fmt.Errorf("HandleRequestOperationData not implemented"))
}
func (h *unimplementedSyncSessionHandler) HandleReceiveOperations(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionReceiveOperations) error {
@@ -306,20 +345,26 @@ func (om *remoteOpIDMapper) translateOpID(originalInstanceKeyid string, original
return translatedID, nil
}
// Cache miss - query the database
op, err := om.oplog.FindOneMetadata(oplog.Query{
// Cache miss - query the database. Use QueryMetadata directly to handle
// the case where duplicates already exist (return the first match).
var translatedID int64
err := om.oplog.QueryMetadata(oplog.Query{
OriginalInstanceKeyid: &originalInstanceKeyid,
OriginalID: &originalOpId,
}, func(op oplog.OpMetadata) error {
if translatedID == 0 {
translatedID = op.ID
}
return oplog.ErrStopIteration
})
if err != nil {
if errors.Is(err, oplog.ErrNoResults) {
return 0, nil // No results means the ID is not found
}
return 0, err // Other errors should be propagated
return 0, err
}
if translatedID == 0 {
return 0, nil // No results means the ID is not found
}
// Cache the result and return
translatedID := op.ID
om.opIDLru.Add(cacheKey, translatedID)
return translatedID, nil
}
@@ -340,20 +385,26 @@ func (om *remoteOpIDMapper) translateFlowID(originalInstanceKeyid string, origin
return translatedID, nil
}
// Cache miss - query the database
op, err := om.oplog.FindOneMetadata(oplog.Query{
// Cache miss - query the database. Use QueryMetadata directly to handle
// the case where duplicates already exist (return the first match).
var translatedID int64
err := om.oplog.QueryMetadata(oplog.Query{
OriginalInstanceKeyid: &originalInstanceKeyid,
OriginalFlowID: &originalFlowId,
}, func(op oplog.OpMetadata) error {
if translatedID == 0 {
translatedID = op.FlowID
}
return oplog.ErrStopIteration
})
if err != nil {
if errors.Is(err, oplog.ErrNoResults) {
return 0, nil // No results means the ID is not found
}
return 0, err // Other errors should be propagated
return 0, err
}
if translatedID == 0 {
return 0, nil // No results means the ID is not found
}
// Cache the result and return
translatedID := op.FlowID
om.flowIDLru.Add(cacheKey, translatedID)
return translatedID, nil
}
internal/api/syncapi/syncmanager.go
+109 -11
View File
@@ -10,13 +10,32 @@ import (
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/gen/go/v1sync"
"github.com/garethgeorge/backrest/internal/api/syncapi/permissions"
"github.com/garethgeorge/backrest/internal/config"
"github.com/garethgeorge/backrest/internal/cryptoutil"
"github.com/garethgeorge/backrest/internal/oplog"
"github.com/garethgeorge/backrest/internal/orchestrator"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"
)
// connectedPeerHandle represents a connected peer's stream and metadata.
// It allows the API layer to send messages to a specific connected peer.
type connectedPeerHandle struct {
stream *bidiSyncCommandStream
peer *v1.Multihost_Peer
permissions *permissions.PermissionSet
}
// SendSetConfig sends a SyncActionSetConfig message to the connected peer.
func (h *connectedPeerHandle) SendSetConfig(setConfig *v1sync.SyncStreamItem_SyncActionSetConfig) {
h.stream.Send(&v1sync.SyncStreamItem{
Action: &v1sync.SyncStreamItem_SetConfig{
SetConfig: setConfig,
},
})
}
type SyncManager struct {
configMgr *config.ConfigManager
orchestrator *orchestrator.Orchestrator
@@ -31,6 +50,10 @@ type SyncManager struct {
syncClients map[string]*SyncClient // current sync clients, protected by mu
// connectedPeers tracks connected authorized client peers by key ID.
// This allows the API layer to send messages to specific connected peers.
connectedPeers map[string]*connectedPeerHandle
peerStateManager PeerStateManager
}
@@ -66,6 +89,7 @@ func NewSyncManager(configMgr *config.ConfigManager, oplog *oplog.OpLog, orchest
syncClientRetryDelay: 60 * time.Second,
syncClients: make(map[string]*SyncClient),
connectedPeers: make(map[string]*connectedPeerHandle),
peerStateManager: peerStateManager,
}
@@ -89,7 +113,7 @@ func (m *SyncManager) RunSync(ctx context.Context) {
zap.L().Info("syncmanager exited")
}()
runSyncWithNewConfig := func() {
startSync := func(config *v1.Config) {
m.mu.Lock()
defer m.mu.Unlock()
@@ -103,12 +127,6 @@ func (m *SyncManager) RunSync(ctx context.Context) {
syncCtx, cancel := context.WithCancel(ctx)
cancelLastSync = cancel
config, err := m.configMgr.Get()
if err != nil {
zap.S().Errorf("syncmanager failed to refresh config with latest changes so sync is stopped: %v", err)
return
}
if config.Multihost.GetIdentity() == nil {
zap.S().Info("syncmanager no identity key configured, sync feature is disabled.")
m.snapshot = nil // Clear the snapshot to indicate sync is disabled
@@ -152,14 +170,68 @@ func (m *SyncManager) RunSync(ctx context.Context) {
}
}
runSyncWithNewConfig()
// lastConfig tracks the config that sync is currently running with.
// We only restart sync when the config changes in a meaningful way
// (i.e. ignoring the Modno field which increments on every write).
var lastConfig *v1.Config
syncConfigEqual := func(a, b *v1.Config) bool {
if a == nil || b == nil {
return a == b
}
// Compare ignoring Modno which changes on every config write.
ac := proto.Clone(a).(*v1.Config)
bc := proto.Clone(b).(*v1.Config)
ac.Modno = 0
bc.Modno = 0
return proto.Equal(ac, bc)
}
restartSyncIfChanged := func() {
config, err := m.configMgr.Get()
if err != nil {
zap.S().Errorf("syncmanager failed to refresh config with latest changes so sync is stopped: %v", err)
return
}
if syncConfigEqual(config, lastConfig) {
zap.L().Debug("syncmanager config changed but sync-relevant config is unchanged, skipping restart")
return
}
lastConfig = proto.Clone(config).(*v1.Config)
startSync(config)
}
restartSyncIfChanged()
// Clock jump detection: if the ticker fires much later than expected
// (e.g. after system sleep), force a reconnect to recover dead streams.
clockJumpInterval := 1 * time.Minute
clockJumpGrace := 30 * time.Second
clockJumpTicker := time.NewTicker(clockJumpInterval)
defer clockJumpTicker.Stop()
lastTickTime := time.Now()
for {
select {
case <-ctx.Done():
return
case <-configWatchCh:
runSyncWithNewConfig()
restartSyncIfChanged()
case <-clockJumpTicker.C:
delta := time.Since(lastTickTime) - clockJumpInterval
lastTickTime = time.Now()
if delta < 0 {
delta = -delta
}
if delta > clockJumpGrace {
zap.S().Warnf("syncmanager detected clock jump of %v, forcing reconnection", delta)
config, err := m.configMgr.Get()
if err != nil {
zap.S().Errorf("syncmanager failed to get config after clock jump: %v", err)
continue
}
startSync(config)
}
}
}
}
@@ -178,19 +250,45 @@ func (m *SyncManager) runSyncWithPeerInternal(ctx context.Context, config *v1.Co
return fmt.Errorf("creating sync client: %w", err)
}
m.mu.Lock()
m.syncClients[knownHostPeer.InstanceId] = newClient
m.syncClients[knownHostPeer.Keyid] = newClient
m.mu.Unlock()
go func() {
newClient.RunSync(ctx)
m.mu.Lock()
delete(m.syncClients, knownHostPeer.InstanceId)
// Only remove the entry if it still points at us. On reconfiguration the new
// client may have already inserted itself under the same key; deleting blindly
// would wipe the replacement.
if m.syncClients[knownHostPeer.Keyid] == newClient {
delete(m.syncClients, knownHostPeer.Keyid)
}
m.mu.Unlock()
}()
return nil
}
// registerConnectedPeer registers a connected peer's stream handle.
func (m *SyncManager) registerConnectedPeer(keyID string, handle *connectedPeerHandle) {
m.mu.Lock()
defer m.mu.Unlock()
m.connectedPeers[keyID] = handle
}
// unregisterConnectedPeer removes a connected peer's stream handle.
func (m *SyncManager) unregisterConnectedPeer(keyID string) {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.connectedPeers, keyID)
}
// GetConnectedPeer returns the handle for a connected peer, or nil if not connected.
func (m *SyncManager) GetConnectedPeer(keyID string) *connectedPeerHandle {
m.mu.Lock()
defer m.mu.Unlock()
return m.connectedPeers[keyID]
}
type syncConfigSnapshot struct {
config *v1.Config
identityKey *cryptoutil.PrivateKey // the local instance's identity key, used for signing sync messages
internal/api/syncapi/syncoperations_test.go
+397
View File
@@ -0,0 +1,397 @@
package syncapi
import (
"context"
"fmt"
"math/rand"
"sync"
"testing"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/config/migrations"
"github.com/garethgeorge/backrest/internal/cryptoutil"
"github.com/garethgeorge/backrest/internal/oplog"
"github.com/garethgeorge/backrest/internal/testutil"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/testing/protocmp"
"github.com/google/go-cmp/cmp"
)
// TestFuzzOperationSync exercises the sync protocol with randomized operation
// mutations (add, update, delete) interleaved with connection drops and
// reconnections. After every round the test asserts that the host's view of
// the client's operations exactly matches the client's local state.
func TestFuzzOperationSync(t *testing.T) {
testutil.InstallZapLogger(t)
const (
numRounds = 10 // rounds of mutations
opsPerRound = 20 // mutations per round
reconnectEvery = 3 // force reconnect every N rounds
testTimeout = 60 * time.Second
)
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
defer cancel()
peerHostAddr := testutil.AllocOpenBindAddr(t)
peerClientAddr := testutil.AllocOpenBindAddr(t)
repoGUID := cryptoutil.MustRandomID(cryptoutil.DefaultIDBits)
peerHostConfig := &v1.Config{
Version: migrations.CurrentVersion,
Instance: defaultHostID,
Repos: []*v1.Repo{
{Id: defaultRepoID, Guid: repoGUID, Uri: "test-uri"},
},
Multihost: &v1.Multihost{
Identity: identity1,
AuthorizedClients: []*v1.Multihost_Peer{
{Keyid: identity2.Keyid, InstanceId: defaultClientID},
},
},
}
peerClientConfig := &v1.Config{
Version: migrations.CurrentVersion,
Instance: defaultClientID,
Repos: []*v1.Repo{
{Id: defaultRepoID, Guid: repoGUID, Uri: "backrest://" + defaultHostID},
},
Multihost: &v1.Multihost{
Identity: identity2,
KnownHosts: []*v1.Multihost_Peer{
{
Keyid: identity1.Keyid,
InstanceId: defaultHostID,
InstanceUrl: fmt.Sprintf("http://%s", peerHostAddr),
Permissions: []*v1.Multihost_Permission{
{
Type: v1.Multihost_Permission_PERMISSION_READ_OPERATIONS,
Scopes: []string{"repo:" + defaultRepoID},
},
},
},
},
},
}
peerHost := newPeerUnderTest(t, peerHostConfig)
peerClient := newPeerUnderTest(t, peerClientConfig)
opTempl := &v1.Operation{
InstanceId: defaultClientID,
RepoId: defaultRepoID,
RepoGuid: repoGUID,
PlanId: defaultPlanID,
UnixTimeStartMs: time.Now().UnixMilli() - 1000,
UnixTimeEndMs: time.Now().UnixMilli(),
Status: v1.OperationStatus_STATUS_SUCCESS,
Op: &v1.Operation_OperationBackup{},
}
rng := rand.New(rand.NewSource(time.Now().UnixNano()))
query := oplog.Query{}.SetInstanceID(defaultClientID).SetRepoGUID(repoGUID)
// Track live operations on the client by their ID.
liveOps := map[int64]*v1.Operation{}
// Start sync infrastructure.
syncCtx, cancelSync := context.WithCancel(ctx)
var syncWg sync.WaitGroup
startSync := func() {
syncCtx, cancelSync = context.WithCancel(ctx)
syncWg.Add(2)
go func() { defer syncWg.Done(); runSyncAPIWithCtx(syncCtx, peerHost, peerHostAddr) }()
go func() { defer syncWg.Done(); runSyncAPIWithCtx(syncCtx, peerClient, peerClientAddr) }()
tryConnect(t, ctx, peerClient, peerClientConfig.Multihost.KnownHosts[0])
}
stopSync := func() {
cancelSync()
syncWg.Wait()
}
startSync()
for round := 0; round < numRounds; round++ {
t.Logf("=== Round %d: %d live ops ===", round, len(liveOps))
// Reconnect periodically to exercise the RequestOperations catch-up path.
if round > 0 && round%reconnectEvery == 0 {
t.Logf("--- reconnecting ---")
stopSync()
startSync()
}
for i := 0; i < opsPerRound; i++ {
action := rng.Intn(10)
switch {
case action < 5: // 50%: add a new operation
op := proto.Clone(opTempl).(*v1.Operation)
op.DisplayMessage = fmt.Sprintf("r%d-op%d", round, i)
op.UnixTimeStartMs = time.Now().UnixMilli() - int64(rng.Intn(10000))
op.UnixTimeEndMs = op.UnixTimeStartMs + int64(rng.Intn(5000))
statuses := []v1.OperationStatus{
v1.OperationStatus_STATUS_PENDING,
v1.OperationStatus_STATUS_INPROGRESS,
v1.OperationStatus_STATUS_SUCCESS,
v1.OperationStatus_STATUS_ERROR,
}
op.Status = statuses[rng.Intn(len(statuses))]
if err := peerClient.oplog.Add(op); err != nil {
t.Fatalf("round %d: add: %v", round, err)
}
liveOps[op.Id] = op
case action < 8 && len(liveOps) > 0: // 30%: update a random op
op := pickRandom(rng, liveOps)
op = proto.Clone(op).(*v1.Operation)
op.DisplayMessage = fmt.Sprintf("r%d-op%d-updated", round, i)
op.Status = v1.OperationStatus_STATUS_SUCCESS
if err := peerClient.oplog.Update(op); err != nil {
t.Fatalf("round %d: update: %v", round, err)
}
liveOps[op.Id] = op
case len(liveOps) > 0: // 20%: delete a random op
op := pickRandom(rng, liveOps)
if err := peerClient.oplog.Delete(op.Id); err != nil {
t.Fatalf("round %d: delete: %v", round, err)
}
delete(liveOps, op.Id)
}
}
// Wait for the host to converge with the client.
assertOpsConverge(t, ctx, peerClient, peerHost, query,
fmt.Sprintf("round %d: ops should converge", round))
}
// Final reconnect to exercise one more catch-up after all mutations.
t.Logf("=== Final reconnect ===")
stopSync()
startSync()
assertOpsConverge(t, ctx, peerClient, peerHost, query, "final: ops should converge after reconnect")
// Assert no duplicates on the host.
assertNoDuplicateOriginalIDs(t, peerHost, query)
stopSync()
}
// TestFuzzOperationSyncOfflineMutations creates operations, syncs, disconnects,
// mutates heavily offline, then reconnects and verifies convergence.
func TestFuzzOperationSyncOfflineMutations(t *testing.T) {
testutil.InstallZapLogger(t)
const (
initialOps = 20
offlineOps = 40
testTimeout = 15 * time.Second
)
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
defer cancel()
peerHostAddr := testutil.AllocOpenBindAddr(t)
peerClientAddr := testutil.AllocOpenBindAddr(t)
repoGUID := cryptoutil.MustRandomID(cryptoutil.DefaultIDBits)
peerHostConfig := &v1.Config{
Version: migrations.CurrentVersion,
Instance: defaultHostID,
Repos: []*v1.Repo{{Id: defaultRepoID, Guid: repoGUID, Uri: "test-uri"}},
Multihost: &v1.Multihost{
Identity: identity1,
AuthorizedClients: []*v1.Multihost_Peer{{Keyid: identity2.Keyid, InstanceId: defaultClientID}},
},
}
peerClientConfig := &v1.Config{
Version: migrations.CurrentVersion,
Instance: defaultClientID,
Repos: []*v1.Repo{{Id: defaultRepoID, Guid: repoGUID, Uri: "backrest://" + defaultHostID}},
Multihost: &v1.Multihost{
Identity: identity2,
KnownHosts: []*v1.Multihost_Peer{{
Keyid: identity1.Keyid,
InstanceId: defaultHostID,
InstanceUrl: fmt.Sprintf("http://%s", peerHostAddr),
Permissions: []*v1.Multihost_Permission{{
Type: v1.Multihost_Permission_PERMISSION_READ_OPERATIONS,
Scopes: []string{"repo:" + defaultRepoID},
}},
}},
},
}
peerHost := newPeerUnderTest(t, peerHostConfig)
peerClient := newPeerUnderTest(t, peerClientConfig)
opTempl := &v1.Operation{
InstanceId: defaultClientID,
RepoId: defaultRepoID,
RepoGuid: repoGUID,
PlanId: defaultPlanID,
UnixTimeStartMs: time.Now().UnixMilli(),
UnixTimeEndMs: time.Now().UnixMilli(),
Status: v1.OperationStatus_STATUS_SUCCESS,
Op: &v1.Operation_OperationBackup{},
}
rng := rand.New(rand.NewSource(42)) // deterministic seed
query := oplog.Query{}.SetInstanceID(defaultClientID).SetRepoGUID(repoGUID)
liveOps := map[int64]*v1.Operation{}
// Phase 1: add initial operations while connected
syncCtx, cancelSync := context.WithCancel(ctx)
var syncWg sync.WaitGroup
syncWg.Add(2)
go func() { defer syncWg.Done(); runSyncAPIWithCtx(syncCtx, peerHost, peerHostAddr) }()
go func() { defer syncWg.Done(); runSyncAPIWithCtx(syncCtx, peerClient, peerClientAddr) }()
tryConnect(t, ctx, peerClient, peerClientConfig.Multihost.KnownHosts[0])
for i := 0; i < initialOps; i++ {
op := proto.Clone(opTempl).(*v1.Operation)
op.DisplayMessage = fmt.Sprintf("init-%d", i)
if err := peerClient.oplog.Add(op); err != nil {
t.Fatalf("init add: %v", err)
}
liveOps[op.Id] = op
}
assertOpsConverge(t, ctx, peerClient, peerHost, query, "initial sync")
// Phase 2: disconnect and mutate heavily
cancelSync()
syncWg.Wait()
for i := 0; i < offlineOps; i++ {
action := rng.Intn(10)
switch {
case action < 5: // 50%: add
op := proto.Clone(opTempl).(*v1.Operation)
op.DisplayMessage = fmt.Sprintf("offline-add-%d", i)
if err := peerClient.oplog.Add(op); err != nil {
t.Fatalf("offline add: %v", err)
}
liveOps[op.Id] = op
case action < 8 && len(liveOps) > 0: // 30%: update
op := pickRandom(rng, liveOps)
op = proto.Clone(op).(*v1.Operation)
op.DisplayMessage = fmt.Sprintf("offline-upd-%d", i)
if err := peerClient.oplog.Update(op); err != nil {
t.Fatalf("offline update: %v", err)
}
liveOps[op.Id] = op
case len(liveOps) > 0: // 20%: delete
op := pickRandom(rng, liveOps)
if err := peerClient.oplog.Delete(op.Id); err != nil {
t.Fatalf("offline delete: %v", err)
}
delete(liveOps, op.Id)
}
}
// Phase 3: reconnect and verify convergence
syncCtx, cancelSync = context.WithCancel(ctx)
syncWg.Add(2)
go func() { defer syncWg.Done(); runSyncAPIWithCtx(syncCtx, peerHost, peerHostAddr) }()
go func() { defer syncWg.Done(); runSyncAPIWithCtx(syncCtx, peerClient, peerClientAddr) }()
tryConnect(t, ctx, peerClient, peerClientConfig.Multihost.KnownHosts[0])
assertOpsConverge(t, ctx, peerClient, peerHost, query, "after offline mutations")
assertNoDuplicateOriginalIDs(t, peerHost, query)
cancelSync()
syncWg.Wait()
}
// --- helpers ---
func pickRandom(rng *rand.Rand, m map[int64]*v1.Operation) *v1.Operation {
keys := make([]int64, 0, len(m))
for k := range m {
keys = append(keys, k)
}
return m[keys[rng.Intn(len(keys))]]
}
func assertOpsConverge(t *testing.T, ctx context.Context, client, host *peerUnderTest, query oplog.Query, msg string) {
t.Helper()
err := testutil.Retry(t, ctx, func() error {
clientOps := getOperations(t, client.oplog, query)
hostOps := getOperations(t, host.oplog, query)
// Normalize: clear locally-assigned fields that differ between peers.
normalize := func(ops []*v1.Operation) []*v1.Operation {
out := make([]*v1.Operation, len(ops))
for i, op := range ops {
c := proto.Clone(op).(*v1.Operation)
c.Id = 0
c.FlowId = 0
c.OriginalId = 0
c.OriginalFlowId = 0
c.OriginalInstanceKeyid = ""
c.Modno = 0
out[i] = c
}
return out
}
cn := normalize(clientOps)
hn := normalize(hostOps)
sortByMessage := func(a, b *v1.Operation) int {
if a.DisplayMessage < b.DisplayMessage {
return -1
}
if a.DisplayMessage > b.DisplayMessage {
return 1
}
return 0
}
sortByMessageStable(cn, sortByMessage)
sortByMessageStable(hn, sortByMessage)
if len(cn) == 0 && len(hn) == 0 {
return nil // both empty is fine
}
if diff := cmp.Diff(cn, hn, protocmp.Transform()); diff != "" {
return fmt.Errorf("not converged (client has %d, host has %d): %s", len(clientOps), len(hostOps), diff)
}
return nil
})
if err != nil {
t.Fatalf("%s: %v", msg, err)
}
}
func sortByMessageStable(ops []*v1.Operation, cmp func(a, b *v1.Operation) int) {
for i := 1; i < len(ops); i++ {
for j := i; j > 0 && cmp(ops[j-1], ops[j]) > 0; j-- {
ops[j-1], ops[j] = ops[j], ops[j-1]
}
}
}
func assertNoDuplicateOriginalIDs(t *testing.T, peer *peerUnderTest, query oplog.Query) {
t.Helper()
ops := getOperations(t, peer.oplog, query)
seen := map[int64]bool{}
for _, op := range ops {
origID := op.OriginalId
if origID == 0 {
continue
}
if seen[origID] {
t.Errorf("duplicate original_id %d found on host", origID)
}
seen[origID] = true
}
}
internal/api/syncapi/syncserver.go
+237 -41
View File
@@ -15,6 +15,7 @@ import (
"github.com/garethgeorge/backrest/internal/env"
"github.com/garethgeorge/backrest/internal/oplog"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"
)
const SyncProtocolVersion = 1
@@ -28,7 +29,10 @@ type BackrestSyncHandler struct {
var _ v1syncconnect.BackrestSyncServiceHandler = &BackrestSyncHandler{}
func NewBackrestSyncHandler(mgr *SyncManager) *BackrestSyncHandler {
mapper, _ := newRemoteOpIDMapper(mgr.oplog, 4096) // error can be ignored, it just checks for valid size
mapper, err := newRemoteOpIDMapper(mgr.oplog, 4096)
if err != nil {
panic(fmt.Errorf("syncapi: constructing remote op ID mapper: %w", err))
}
return &BackrestSyncHandler{
mgr: mgr,
mapper: mapper,
@@ -54,6 +58,8 @@ func (h *BackrestSyncHandler) Sync(ctx context.Context, stream *connect.BidiStre
cmdStream,
sessionHandler,
snapshot.config.GetMultihost().GetAuthorizedClients(),
"", // server never sends a pairing secret
h.handleUnknownPeerPairing(snapshot),
)
cmdStream.SendErrorAndTerminate(err)
}()
@@ -141,10 +147,6 @@ func (h *syncSessionHandlerServer) OnConnectionEstablished(ctx context.Context,
return NewSyncErrorInternal(fmt.Errorf("failed to create permission set for client %q: %w", peer.InstanceId, err))
}
if !h.peer.KeyidVerified {
return NewSyncErrorAuth(fmt.Errorf("client %q is not visually verified, please verify the key ID %q", peer.InstanceId, h.peer.Keyid))
}
// Configure the state for the connected peer.
peerState := newPeerState(peer.InstanceId, h.peer.Keyid)
peerState.ConnectionStateMessage = "connected"
@@ -154,6 +156,13 @@ func (h *syncSessionHandlerServer) OnConnectionEstablished(ctx context.Context,
h.l.Sugar().Infof("accepted a connection from client instance ID %q", h.peer.InstanceId)
// Register this peer's stream handle so the API layer can send messages to it.
h.mgr.registerConnectedPeer(h.peer.Keyid, &connectedPeerHandle{
stream: stream,
peer: h.peer,
permissions: h.permissions,
})
// start a heartbeat thread
go sendHeartbeats(ctx, stream, env.MultihostHeartbeatInterval())
@@ -164,27 +173,66 @@ func (h *syncSessionHandlerServer) OnConnectionEstablished(ctx context.Context,
for {
select {
case <-configWatchCh:
h.l.Sugar().Infof("disconnecting client due to configuration change")
stream.SendErrorAndTerminate(nil) // terminate so client reconnects and gets new config
return
newConfig, err := h.mgr.configMgr.Get()
if err != nil {
h.l.Sugar().Warnf("failed to get config on change: %v, disconnecting client", err)
stream.SendErrorAndTerminate(nil)
return
}
// Check if this peer is still authorized
peerIdx := slices.IndexFunc(newConfig.Multihost.GetAuthorizedClients(), func(p *v1.Multihost_Peer) bool {
return p.InstanceId == h.peer.InstanceId && p.Keyid == h.peer.Keyid
})
if peerIdx == -1 {
h.l.Sugar().Infof("disconnecting client %q: no longer authorized", h.peer.InstanceId)
stream.SendErrorAndTerminate(nil)
return
}
// Check if permissions changed by comparing the proto peer definition
updatedPeer := newConfig.Multihost.AuthorizedClients[peerIdx]
if !proto.Equal(h.peer, updatedPeer) {
h.l.Sugar().Infof("disconnecting client %q: peer configuration changed", h.peer.InstanceId)
stream.SendErrorAndTerminate(nil)
return
}
// Permissions unchanged — send updated config and shared repos to client
configRepos, configPlans, err := h.sendConfigToClient(stream, newConfig)
if err != nil {
h.l.Sugar().Warnf("failed to send updated config to client %q: %v", h.peer.InstanceId, err)
} else {
sharedRepos := h.sendSharedReposToClient(stream, newConfig)
h.l.Sugar().Debugf("config changed, sent update to client %q: %d repos, %d plans (config); %d shared repos pushed",
h.peer.InstanceId, configRepos, configPlans, sharedRepos)
}
case <-ctx.Done():
return
}
}
}()
if err := h.sendConfigToClient(stream, h.snapshot.config); err != nil {
configRepos, configPlans, err := h.sendConfigToClient(stream, h.snapshot.config)
if err != nil {
return NewSyncErrorInternal(fmt.Errorf("sending initial config to client: %w", err))
}
// send initial request for operation sync
if err := h.sendOperationSyncRequest(stream); err != nil {
return NewSyncErrorInternal(fmt.Errorf("sending initial operation sync request: %w", err))
}
// Push shared repos to the client
sharedRepoCount := h.sendSharedReposToClient(stream, h.snapshot.config)
h.l.Sugar().Infof("sent initial state to client %q: %d repos, %d plans (config); %d shared repos pushed",
h.peer.InstanceId, configRepos, configPlans, sharedRepoCount)
return nil
}
func (h *syncSessionHandlerServer) OnConnectionDisconnected() {
if h.peer != nil {
h.mgr.unregisterConnectedPeer(h.peer.Keyid)
}
}
func (h *syncSessionHandlerServer) HandleHeartbeat(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionHeartbeat) error {
peerState := h.mgr.peerStateManager.GetPeerState(h.peer.Keyid).Clone()
if peerState == nil {
@@ -266,18 +314,9 @@ func (h *syncSessionHandlerServer) insertOrUpdate(op *v1.Operation, isUpdate boo
op.OriginalFlowId = op.FlowId
op.Id = localOpID
op.FlowId = localFlowID
if op.Id == 0 {
if isUpdate {
h.l.Sugar().Warnf("received update for non-existent operation %+v, inserting instead", op)
}
op.Modno = 0
return h.mgr.oplog.Add(op)
} else {
if !isUpdate {
h.l.Sugar().Warnf("received insert for existing operation %+v, updating instead", op)
}
return h.mgr.oplog.Update(op)
}
// Use Set which handles both insert (Id==0) and update (Id!=0),
// preserving the operation's Modno from the client.
return h.mgr.oplog.Set(oplog.SetOptions{}, op)
}
func (h *syncSessionHandlerServer) deleteByOriginalID(originalID int64) error {
@@ -294,14 +333,12 @@ func (h *syncSessionHandlerServer) deleteByOriginalID(originalID int64) error {
return h.mgr.oplog.Delete(foundOp.ID)
}
func (h *syncSessionHandlerServer) sendConfigToClient(stream *bidiSyncCommandStream, config *v1.Config) error {
func (h *syncSessionHandlerServer) sendConfigToClient(stream *bidiSyncCommandStream, config *v1.Config) (int, int, error) {
remoteConfig := &v1sync.RemoteConfig{
Version: config.Version,
Modno: config.Modno,
}
resourceListMsg := &v1sync.SyncStreamItem_SyncActionReceiveResources{}
var allowedRepoIDs []string
var allowedPlanIDs []string
for _, repo := range config.Repos {
if h.permissions.CheckPermissionForRepo(repo.Id, permissions.PermsCanViewConfiguration...) {
remoteConfig.Repos = append(remoteConfig.Repos, repo)
@@ -309,7 +346,6 @@ func (h *syncSessionHandlerServer) sendConfigToClient(stream *bidiSyncCommandStr
Id: repo.Id,
Guid: repo.Guid,
})
allowedRepoIDs = append(allowedRepoIDs, repo.Id)
}
}
for _, plan := range config.Plans {
@@ -318,10 +354,8 @@ func (h *syncSessionHandlerServer) sendConfigToClient(stream *bidiSyncCommandStr
resourceListMsg.Plans = append(resourceListMsg.Plans, &v1sync.PlanMetadata{
Id: plan.Id,
})
allowedPlanIDs = append(allowedPlanIDs, plan.Id)
}
}
h.l.Sugar().Debugf("determined client %v is allowlisted to read configs for repos %v and plans %v", h.peer.InstanceId, allowedRepoIDs, allowedPlanIDs)
// Send the config, this is the first meaningful packet the client will receive.
stream.Send(&v1sync.SyncStreamItem{
@@ -338,22 +372,184 @@ func (h *syncSessionHandlerServer) sendConfigToClient(stream *bidiSyncCommandStr
ReceiveResources: resourceListMsg,
},
})
return nil
return len(remoteConfig.Repos), len(remoteConfig.Plans), nil
}
func (h *syncSessionHandlerServer) sendOperationSyncRequest(stream *bidiSyncCommandStream) error {
highestID, highestModno, err := h.mgr.oplog.GetHighestOpIDAndModno(oplog.Query{}.SetOriginalInstanceKeyid(h.peer.Keyid))
if err != nil {
return fmt.Errorf("getting highest opid and modno: %w", err)
// sendSharedReposToClient sends repos marked as shared to the client via SetConfig.
// This pushes repo configurations to the client so they are added to the client's local config.
// Returns the number of shared repos sent.
func (h *syncSessionHandlerServer) sendSharedReposToClient(stream *bidiSyncCommandStream, config *v1.Config) int {
var sharedRepos []*v1.Repo
for _, repo := range config.Repos {
if repo.GetShared() {
repoCopy := proto.Clone(repo).(*v1.Repo)
repoCopy.OriginInstanceId = config.Instance
sharedRepos = append(sharedRepos, repoCopy)
}
}
if len(sharedRepos) == 0 {
return 0
}
stream.Send(&v1sync.SyncStreamItem{
Action: &v1sync.SyncStreamItem_RequestOperations{
RequestOperations: &v1sync.SyncStreamItem_SyncActionRequestOperations{
HighOpid: highestID,
HighModno: highestModno,
Action: &v1sync.SyncStreamItem_SetConfig{
SetConfig: &v1sync.SyncStreamItem_SyncActionSetConfig{
Repos: sharedRepos,
},
},
})
h.l.Sugar().Debugf("requested operations from client starting at opID %d and modno %d", highestID, highestModno)
return len(sharedRepos)
}
// ValidatePairingSecret checks a pairing secret against a list of pairing tokens.
// Returns the matching token if valid, or an error explaining why validation failed.
// This is a pure function with no side effects, making it easy to test exhaustively.
func ValidatePairingSecret(secret string, tokens []*v1.Multihost_PairingToken, now time.Time) (*v1.Multihost_PairingToken, error) {
if secret == "" {
return nil, fmt.Errorf("empty pairing secret")
}
for _, token := range tokens {
if token.Secret != secret {
continue
}
if token.ExpiresAtUnix > 0 && now.Unix() > token.ExpiresAtUnix {
return nil, fmt.Errorf("pairing token %q has expired", token.Label)
}
if token.MaxUses > 0 && token.Uses >= token.MaxUses {
return nil, fmt.Errorf("pairing token %q has reached its maximum number of uses (%d)", token.Label, token.MaxUses)
}
return token, nil
}
return nil, fmt.Errorf("no matching pairing token found")
}
// handleUnknownPeerPairing returns an onUnknownPeerFunc that validates a pairing secret
// from the handshake, adds the client to authorized_clients in the config, and consumes the token.
// The peer is added to the config BEFORE runSync proceeds with its normal authorization check,
// ensuring that runSync's hard gate (peer must be in authorized_clients) is never bypassed.
func (h *BackrestSyncHandler) handleUnknownPeerPairing(snapshot *syncConfigSnapshot) onUnknownPeerFunc {
return func(handshake *v1sync.SyncStreamItem) (*v1.Multihost_Peer, error) {
pairingSecret := handshake.GetHandshake().GetPairingSecret()
if pairingSecret == "" {
return nil, fmt.Errorf("unknown peer and no pairing secret provided")
}
// Defense-in-depth: re-verify the handshake signature to ensure the client
// holds the private key for the public key it presents. This is already checked
// by verifyHandshakePacket in runSync, but we verify again here since this is
// a security-critical path that adds a new authorized client.
if _, err := verifyHandshakePacket(handshake); err != nil {
return nil, fmt.Errorf("handshake signature verification failed: %w", err)
}
peerKeyID := handshake.GetHandshake().GetPublicKey().GetKeyid()
peerInstanceID := string(handshake.GetHandshake().GetInstanceId().GetPayload())
// Atomically validate the pairing secret and add the client.
var newPeer *v1.Multihost_Peer
if err := h.mgr.configMgr.Transform(func(cfg *v1.Config) (*v1.Config, error) {
token, err := ValidatePairingSecret(pairingSecret, cfg.GetMultihost().GetPairingTokens(), time.Now())
if err != nil {
zap.S().Warnf("rejected pairing attempt from %q (%s): %v", peerInstanceID, peerKeyID, err)
return nil, err
}
newPeer = &v1.Multihost_Peer{
InstanceId: peerInstanceID,
Keyid: peerKeyID,
Permissions: token.Permissions,
}
cfg.Multihost.AuthorizedClients = append(cfg.Multihost.AuthorizedClients, newPeer)
// Consume the token: increment uses, remove if exhausted.
token.Uses++
if token.MaxUses > 0 && token.Uses >= token.MaxUses {
cfg.Multihost.PairingTokens = slices.DeleteFunc(cfg.Multihost.PairingTokens, func(t *v1.Multihost_PairingToken) bool {
return t.Secret == token.Secret
})
}
cfg.Modno++
return cfg, nil
}); err != nil {
return nil, fmt.Errorf("failed to save paired client: %w", err)
}
zap.S().Infof("successfully paired client %q (%s)", peerInstanceID, peerKeyID)
return newPeer, nil
}
}
func (h *syncSessionHandlerServer) HandleOperationManifest(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionOperationManifest) error {
h.l.Sugar().Debugf("received operation manifest with %d operations", len(item.GetOpIds()))
// Build local state: original_id → {localID, modno}
type localOp struct {
localID int64
modno int64
}
localState := map[int64]localOp{}
if err := h.mgr.oplog.QueryMetadata(
oplog.Query{}.SetOriginalInstanceKeyid(h.peer.Keyid),
func(meta oplog.OpMetadata) error {
localState[meta.OriginalID] = localOp{localID: meta.ID, modno: meta.Modno}
return nil
},
); err != nil {
return fmt.Errorf("querying local operation metadata: %w", err)
}
h.l.Sugar().Debugf("local state has %d operations from this peer", len(localState))
// Build remote set from manifest
if len(item.GetOpIds()) != len(item.GetModnos()) {
return NewSyncErrorProtocol(fmt.Errorf("operation manifest has mismatched OpIds (%d) and Modnos (%d) lengths", len(item.GetOpIds()), len(item.GetModnos())))
}
remoteSet := make(map[int64]int64, len(item.GetOpIds()))
for i, id := range item.GetOpIds() {
remoteSet[id] = item.GetModnos()[i]
}
// Delete ops not in manifest
var toDelete []int64
for origID, local := range localState {
if _, exists := remoteSet[origID]; !exists {
toDelete = append(toDelete, local.localID)
}
}
if len(toDelete) > 0 {
h.l.Sugar().Debugf("deleting %d stale operations", len(toDelete))
if err := h.mgr.oplog.Delete(toDelete...); err != nil {
h.l.Sugar().Warnf("failed to delete stale operations: %v", err)
}
}
// Find ops we need (new or changed modno), preserving manifest order
opIDs := item.GetOpIds()
modnos := item.GetModnos()
var needIDs []int64
for i, id := range opIDs {
modno := modnos[i]
local, exists := localState[id]
if !exists || local.modno != modno {
needIDs = append(needIDs, id)
}
}
h.l.Sugar().Debugf("need %d operations (new or changed), local state comparison: remoteSet=%v localState=%v", len(needIDs), remoteSet, localState)
// Request the ops we need
if len(needIDs) > 0 {
stream.Send(&v1sync.SyncStreamItem{
Action: &v1sync.SyncStreamItem_RequestOperationData{
RequestOperationData: &v1sync.SyncStreamItem_SyncActionRequestOperationData{
OpIds: needIDs,
},
},
})
}
return nil
}
func (h *syncSessionHandlerServer) HandleRequestOperationData(ctx context.Context, stream *bidiSyncCommandStream, item *v1sync.SyncStreamItem_SyncActionRequestOperationData) error {
return NewSyncErrorProtocol(fmt.Errorf("server should not receive RequestOperationData"))
}
internal/api/syncapi/syncstatehandler.go
+22
View File
@@ -2,6 +2,7 @@ package syncapi
import (
"context"
"fmt"
"time"
"connectrpc.com/connect"
@@ -23,6 +24,27 @@ func NewBackrestSyncStateHandler(mgr *SyncManager) *BackrestSyncStateHandler {
}
}
func (h *BackrestSyncStateHandler) SetRemoteClientConfig(ctx context.Context, req *connect.Request[v1sync.SetRemoteClientConfigRequest]) (*connect.Response[v1sync.SetRemoteClientConfigResponse], error) {
peerKeyID := req.Msg.GetPeerKeyid()
if peerKeyID == "" {
return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("peer_keyid is required"))
}
handle := h.mgr.GetConnectedPeer(peerKeyID)
if handle == nil {
return nil, connect.NewError(connect.CodeNotFound, fmt.Errorf("peer %q is not connected", peerKeyID))
}
handle.SendSetConfig(&v1sync.SyncStreamItem_SyncActionSetConfig{
Repos: req.Msg.GetRepos(),
Plans: req.Msg.GetPlans(),
ReposToDelete: req.Msg.GetReposToDelete(),
PlansToDelete: req.Msg.GetPlansToDelete(),
})
return connect.NewResponse(&v1sync.SetRemoteClientConfigResponse{}), nil
}
func (h *BackrestSyncStateHandler) GetPeerSyncStatesStream(ctx context.Context, req *connect.Request[v1sync.SyncStateStreamRequest], stream *connect.ServerStream[v1sync.PeerState]) error {
ctx, cancel := context.WithCancelCause(ctx)
defer cancel(nil)
internal/api/syncapi/uriutil.go
-60
View File
@@ -1,60 +0,0 @@
package syncapi
import (
"errors"
"net/url"
)
var ErrNotBackrestURI = errors.New("not a backrest URI")
func CreateRemoteRepoURI(instanceUrl string) (string, error) {
u, err := url.Parse(instanceUrl)
if err != nil {
return "", err
}
if u.Scheme == "http" {
u.Scheme = "backrest"
} else if u.Scheme == "https" {
u.Scheme = "sbackrest"
} else {
return "", errors.New("unsupported scheme")
}
return u.String(), nil
}
func IsBackrestRemoteRepoURI(repoUri string) bool {
u, err := url.Parse(repoUri)
if err != nil {
return false
}
return u.Scheme == "backrest"
}
func InstanceForBackrestURI(repoUri string) (string, error) {
u, err := url.Parse(repoUri)
if err != nil {
return "", err
}
if u.Scheme != "backrest" {
return "", errors.New("not a backrest URI")
}
return u.Hostname(), nil
}
func RepoForBackrestURI(repoUri string) (string, error) {
u, err := url.Parse(repoUri)
if err != nil {
return "", err
}
if u.Scheme != "backrest" {
return "", errors.New("not a backrest URI")
}
return u.Path, nil
}