feat: sync api creates and uses cryptographic identity of local instance (#780)

internal/api/syncapi/identity.go

@@ -1,123 +0,0 @@
-package syncapi
-
-import (
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
-	"crypto/sha256"
-	"crypto/x509"
-	"encoding/pem"
-	"errors"
-	"fmt"
-	"os"
-)
-
-var (
-	curve = elliptic.P256() // ed25519
-)
-
-type Identity struct {
-	InstanceID     string
-	credentialFile string
-
-	privateKey *ecdsa.PrivateKey
-	publicKey  *ecdsa.PublicKey
-}
-
-func NewIdentity(instanceID, credentialFile string) (*Identity, error) {
-	i := &Identity{
-		InstanceID:     instanceID,
-		credentialFile: credentialFile,
-	}
-	if err := i.loadOrGenerateKey(); err != nil {
-		return nil, err
-	}
-	return i, nil
-}
-
-func (i *Identity) loadOrGenerateKey() error {
-	privKeyBytes, errpriv := os.ReadFile(i.credentialFile)
-	pubKeyBytes, errpub := os.ReadFile(i.credentialFile + ".pub")
-	if errpriv != nil || errpub != nil {
-		if os.IsNotExist(errpriv) || os.IsNotExist(errpub) {
-			return i.generateKeys()
-		}
-		if errpriv != nil {
-			return fmt.Errorf("open private key: %w", errpriv)
-		}
-		if errpub != nil {
-			return fmt.Errorf("open public key: %w", errpub)
-		}
-	}
-
-	privKeyBlock, _ := pem.Decode(privKeyBytes)
-	if privKeyBlock == nil {
-		return errors.New("no private key found in pem")
-	}
-	privKey, err := x509.ParseECPrivateKey(privKeyBlock.Bytes)
-	if err != nil {
-		return fmt.Errorf("parse private key: %w", err)
-	}
-
-	pubKeyBlock, _ := pem.Decode(pubKeyBytes)
-	if pubKeyBlock == nil {
-		return errors.New("no public key found in pem")
-	}
-	pubKey, err := x509.ParsePKIXPublicKey(pubKeyBytes)
-	if err != nil {
-		return fmt.Errorf("parse public key: %w", err)
-	}
-
-	i.privateKey = privKey
-	i.publicKey = pubKey.(*ecdsa.PublicKey)
-
-	return nil
-}
-
-func (i *Identity) generateKeys() error {
-	privKey, err := ecdsa.GenerateKey(curve, rand.Reader)
-	if err != nil {
-		return err
-	}
-
-	i.privateKey = privKey
-	i.publicKey = &privKey.PublicKey
-
-	privateKeyBytes, err := x509.MarshalECPrivateKey(i.privateKey)
-	if err != nil {
-		return fmt.Errorf("marshal private key: %w", err)
-	}
-	pemPrivateKeyBytes := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE", Bytes: privateKeyBytes})
-	if err := os.WriteFile(i.credentialFile, pemPrivateKeyBytes, 0600); err != nil {
-		return fmt.Errorf("write private key: %w", err)
-	}
-
-	publicKeyBytes, err := x509.MarshalPKIXPublicKey(&i.privateKey.PublicKey)
-	if err != nil {
-		return fmt.Errorf("marshal public key: %w", err)
-	}
-	pemPublicKeyBytes := pem.EncodeToMemory(&pem.Block{Type: "EC PUBLIC", Bytes: publicKeyBytes})
-	if err := os.WriteFile(i.credentialFile+".pub", pemPublicKeyBytes, 0600); err != nil {
-		return fmt.Errorf("write public key: %w", err)
-	}
-
-	return nil
-}
-
-func (i *Identity) SignMessage(message []byte) ([]byte, error) {
-	hash := sha256.Sum256(message)
-
-	sig, err := ecdsa.SignASN1(rand.Reader, i.privateKey, hash[:])
-	if err != nil {
-		return nil, err
-	}
-	return sig, nil
-}
-
-func (i *Identity) VerifySignature(message, sig []byte) error {
-	hash := sha256.Sum256(message)
-	if !ecdsa.VerifyASN1(i.publicKey, hash[:], sig) {
-		return errors.New("signature verification failed")
-	}
-	return nil
-}

internal/api/syncapi/identity_test.go

@@ -1,29 +0,0 @@
-package syncapi
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"testing"
-)
-
-func TestIdentity(t *testing.T) {
-	dir := t.TempDir()
-
-	// Create a new identity
-	ident, err := NewIdentity("test-instance", filepath.Join(dir, "myidentity.pem"))
-	if err != nil {
-		t.Fatalf("failed to create identity: %v", err)
-	}
-
-	signature, err := ident.SignMessage([]byte("hello world!"))
-	fmt.Printf("signed message: %x\n", signature)
-
-	// Load and print identity file
-	bytes, _ := os.ReadFile(filepath.Join(dir, "myidentity.pem"))
-	t.Log(string(bytes))
-
-	// Load and print public key file
-	bytes, _ = os.ReadFile(filepath.Join(dir, "myidentity.pem.pub"))
-	t.Log(string(bytes))
-}

internal/api/syncapi/syncapi_test.go

@@ -8,10 +8,12 @@ import (
 	"net/http"
 	"path/filepath"
 	"slices"
+	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	connect "connectrpc.com/connect"
 	v1 "github.com/garethgeorge/backrest/gen/go/v1"
 	"github.com/garethgeorge/backrest/gen/go/v1/v1connect"
 	"github.com/garethgeorge/backrest/internal/config"
@@ -39,6 +41,9 @@ const (
 
 var (
 	defaultRepoGUID = cryptoutil.MustRandomID(cryptoutil.DefaultIDBits)
+
+	identity1, _ = cryptoutil.GeneratePrivateKey()
+	identity2, _ = cryptoutil.GeneratePrivateKey()
 )
 
 var (
@@ -76,9 +81,12 @@ func TestConnectionSucceeds(t *testing.T) {
 		Instance: defaultHostID,
 		Repos:    []*v1.Repo{},
 		Multihost: &v1.Multihost{
+			Identity: identity1,
 			AuthorizedClients: []*v1.Multihost_Peer{
 				{
-					InstanceId: defaultClientID,
+					Keyid:         identity2.Keyid,
+					KeyidVerified: true,
+					InstanceId:    defaultClientID,
 				},
 			},
 		},
@@ -88,8 +96,10 @@ func TestConnectionSucceeds(t *testing.T) {
 		Instance: defaultClientID,
 		Repos:    []*v1.Repo{},
 		Multihost: &v1.Multihost{
+			Identity: identity2,
 			KnownHosts: []*v1.Multihost_Peer{
 				{
+					Keyid:       identity1.Keyid,
 					InstanceId:  defaultHostID,
 					InstanceUrl: fmt.Sprintf("http://%s", peerHostAddr),
 				},
@@ -106,6 +116,48 @@ func TestConnectionSucceeds(t *testing.T) {
 	tryConnect(t, ctx, peerClient, defaultHostID)
 }
 
+func TestConnectionBadKeyRejected(t *testing.T) {
+	testutil.InstallZapLogger(t)
+	ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
+
+	peerHostAddr := allocBindAddrForTest(t)
+	peerClientAddr := allocBindAddrForTest(t)
+
+	// Host has identity1, and authorizes no one.
+	peerHostConfig := &v1.Config{
+		Instance: defaultHostID,
+		Repos:    []*v1.Repo{},
+		Multihost: &v1.Multihost{
+			Identity:          identity1,
+			AuthorizedClients: []*v1.Multihost_Peer{}, // No authorized clients
+		},
+	}
+
+	// Client has identity2 and tries to connect to host.
+	peerClientConfig := &v1.Config{
+		Instance: defaultClientID,
+		Repos:    []*v1.Repo{},
+		Multihost: &v1.Multihost{
+			Identity: identity2,
+			KnownHosts: []*v1.Multihost_Peer{
+				{
+					Keyid:       identity1.Keyid,
+					InstanceId:  defaultHostID,
+					InstanceUrl: fmt.Sprintf("http://%s", peerHostAddr),
+				},
+			},
+		},
+	}
+
+	peerHost := newPeerUnderTest(t, peerHostConfig)
+	peerClient := newPeerUnderTest(t, peerClientConfig)
+
+	startRunningSyncAPI(t, peerHost, peerHostAddr)
+	startRunningSyncAPI(t, peerClient, peerClientAddr)
+
+	tryExpectConnectionFailure(t, ctx, peerClient, defaultHostID, connect.CodePermissionDenied)
+}
+
 func TestSyncConfigChange(t *testing.T) {
 	testutil.InstallZapLogger(t)
 	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
@@ -128,9 +180,12 @@ func TestSyncConfigChange(t *testing.T) {
 			},
 		},
 		Multihost: &v1.Multihost{
+			Identity: identity1,
 			AuthorizedClients: []*v1.Multihost_Peer{
 				{
-					InstanceId: defaultClientID,
+					Keyid:         identity2.Keyid,
+					KeyidVerified: true,
+					InstanceId:    defaultClientID,
 				},
 			},
 		},
@@ -146,8 +201,10 @@ func TestSyncConfigChange(t *testing.T) {
 			},
 		},
 		Multihost: &v1.Multihost{
+			Identity: identity2,
 			KnownHosts: []*v1.Multihost_Peer{
 				{
+					Keyid:       identity1.Keyid,
 					InstanceId:  defaultHostID,
 					InstanceUrl: fmt.Sprintf("http://%s", peerHostAddr),
 				},
@@ -204,9 +261,12 @@ func TestSimpleOperationSync(t *testing.T) {
 			},
 		},
 		Multihost: &v1.Multihost{
+			Identity: identity1,
 			AuthorizedClients: []*v1.Multihost_Peer{
 				{
-					InstanceId: defaultClientID,
+					Keyid:         identity2.Keyid,
+					KeyidVerified: true,
+					InstanceId:    defaultClientID,
 				},
 			},
 		},
@@ -222,8 +282,10 @@ func TestSimpleOperationSync(t *testing.T) {
 			},
 		},
 		Multihost: &v1.Multihost{
+			Identity: identity2,
 			KnownHosts: []*v1.Multihost_Peer{
 				{
+					Keyid:       identity1.Keyid,
 					InstanceId:  defaultHostID,
 					InstanceUrl: fmt.Sprintf("http://%s", peerHostAddr),
 				},
@@ -272,25 +334,28 @@ func TestSimpleOperationSync(t *testing.T) {
 	tryExpectExactOperations(t, ctx, peerHost, oplog.Query{}.SetInstanceID(defaultClientID).SetRepoGUID(defaultRepoGUID),
 		testutil.OperationsWithDefaults(basicClientOperationTempl, []*v1.Operation{
 			{
-				Id:             3, // b/c of the already inserted host ops the sync'd ops start at 3
-				FlowId:         3,
-				OriginalId:     1,
-				OriginalFlowId: 1,
-				DisplayMessage: "clientop1",
+				Id:                    3, // b/c of the already inserted host ops the sync'd ops start at 3
+				FlowId:                3,
+				OriginalId:            1,
+				OriginalFlowId:        1,
+				OriginalInstanceKeyid: identity2.Keyid,
+				DisplayMessage:        "clientop1",
 			},
 			{
-				Id:             4,
-				FlowId:         3,
-				OriginalId:     2,
-				OriginalFlowId: 1,
-				DisplayMessage: "clientop2",
+				Id:                    4,
+				FlowId:                3,
+				OriginalId:            2,
+				OriginalFlowId:        1,
+				OriginalInstanceKeyid: identity2.Keyid,
+				DisplayMessage:        "clientop2",
 			},
 			{
-				Id:             5,
-				FlowId:         5,
-				OriginalId:     3,
-				OriginalFlowId: 2,
-				DisplayMessage: "clientop3",
+				Id:                    5,
+				FlowId:                5,
+				OriginalId:            3,
+				OriginalFlowId:        2,
+				OriginalInstanceKeyid: identity2.Keyid,
+				DisplayMessage:        "clientop3",
 			},
 		}), "host and client should be synced")
 }
@@ -313,9 +378,12 @@ func TestSyncMutations(t *testing.T) {
 			},
 		},
 		Multihost: &v1.Multihost{
+			Identity: identity1,
 			AuthorizedClients: []*v1.Multihost_Peer{
 				{
-					InstanceId: defaultClientID,
+					Keyid:         identity2.Keyid,
+					KeyidVerified: true,
+					InstanceId:    defaultClientID,
 				},
 			},
 		},
@@ -331,8 +399,10 @@ func TestSyncMutations(t *testing.T) {
 			},
 		},
 		Multihost: &v1.Multihost{
+			Identity: identity2,
 			KnownHosts: []*v1.Multihost_Peer{
 				{
+					Keyid:       identity1.Keyid,
 					InstanceId:  defaultHostID,
 					InstanceUrl: fmt.Sprintf("http://%s", peerHostAddr),
 				},
@@ -377,11 +447,12 @@ func TestSyncMutations(t *testing.T) {
 	tryExpectExactOperations(t, ctx, peerHost, oplog.Query{}.SetRepoGUID(defaultRepoGUID),
 		testutil.OperationsWithDefaults(basicClientOperationTempl, []*v1.Operation{
 			{
-				Id:             1,
-				DisplayMessage: "clientop1-mod-while-online",
-				OriginalFlowId: 1,
-				OriginalId:     1,
-				FlowId:         1,
+				Id:                    1,
+				DisplayMessage:        "clientop1-mod-while-online",
+				OriginalFlowId:        1,
+				OriginalId:            1,
+				FlowId:                1,
+				OriginalInstanceKeyid: identity2.Keyid,
 			},
 		}), "host and client should sync online edits")
 
@@ -413,11 +484,12 @@ func TestSyncMutations(t *testing.T) {
 	tryExpectExactOperations(t, ctx, peerHost, oplog.Query{}.SetRepoGUID(defaultRepoGUID),
 		testutil.OperationsWithDefaults(basicClientOperationTempl, []*v1.Operation{
 			{
-				Id:             1,
-				DisplayMessage: "clientop1-mod-while-offline",
-				OriginalFlowId: 1,
-				OriginalId:     1,
-				FlowId:         1,
+				Id:                    1,
+				DisplayMessage:        "clientop1-mod-while-offline",
+				OriginalFlowId:        1,
+				OriginalId:            1,
+				FlowId:                1,
+				OriginalInstanceKeyid: identity2.Keyid,
 			},
 		}), "host and client should sync offline edits")
 
@@ -465,12 +537,14 @@ func tryExpectOperationsSynced(t *testing.T, ctx context.Context, peer1 *peerUnd
 			op.FlowId = 0
 			op.OriginalId = 0
 			op.OriginalFlowId = 0
+			op.OriginalInstanceKeyid = ""
 		}
 		for _, op := range peer2Ops {
 			op.Id = 0
 			op.FlowId = 0
 			op.OriginalId = 0
 			op.OriginalFlowId = 0
+			op.OriginalInstanceKeyid = ""
 		}
 
 		sortFn := func(a, b *v1.Operation) int {
@@ -532,6 +606,33 @@ func tryConnect(t *testing.T, ctx context.Context, peer *peerUnderTest, instance
 	})
 }
 
+func tryExpectConnectionFailure(t *testing.T, ctx context.Context, peer *peerUnderTest, instanceID string, wantCode connect.Code) {
+	t.Helper()
+	testutil.Try(t, ctx, func() error {
+		allClients := peer.manager.GetSyncClients()
+		client, ok := allClients[instanceID]
+		if !ok {
+			// It might take a moment for the client to be created.
+			return fmt.Errorf("client for instance %q not found yet", instanceID)
+		}
+
+		state, reason := client.GetConnectionState()
+		// The state can be either ERROR_AUTH or DISCONNECTED, since there's a race.
+		// The important part is that the reason contains the permission denied error.
+		if state != v1.SyncConnectionState_CONNECTION_STATE_ERROR_AUTH && state != v1.SyncConnectionState_CONNECTION_STATE_DISCONNECTED {
+			return fmt.Errorf("expected connection state to be ERROR_AUTH or DISCONNECTED, got %v (reason: %q)", state, reason)
+		}
+
+		// The reason is the error string. For connect errors, it's "<code>: <message>".
+		// e.g. "permission_denied: peer ... not authorized"
+		if !strings.Contains(reason, wantCode.String()) {
+			return fmt.Errorf("expected reason to contain %q, but got %q", wantCode.String(), reason)
+		}
+
+		return nil
+	})
+}
+
 func allocBindAddrForTest(t *testing.T) string {
 	t.Helper()
 

internal/api/syncapi/syncclient.go

@@ -24,13 +24,15 @@ import (
 )
 
 type SyncClient struct {
-	mgr             *SyncManager
-	localInstanceID string
-	peer            *v1.Multihost_Peer
-	oplog           *oplog.OpLog
-	client          v1connect.BackrestSyncServiceClient
-	reconnectDelay  time.Duration
-	l               *zap.Logger
+	mgr *SyncManager
+
+	syncConfigSnapshot syncConfigSnapshot
+	localInstanceID    string
+	peer               *v1.Multihost_Peer
+	oplog              *oplog.OpLog
+	client             v1connect.BackrestSyncServiceClient
+	reconnectDelay     time.Duration
+	l                  *zap.Logger
 
 	// mutable properties
 	mu                      sync.Mutex
@@ -52,7 +54,12 @@ func newInsecureClient() *http.Client {
 	}
 }
 
-func NewSyncClient(mgr *SyncManager, localInstanceID string, peer *v1.Multihost_Peer, oplog *oplog.OpLog) (*SyncClient, error) {
+func NewSyncClient(
+	mgr *SyncManager,
+	snapshot syncConfigSnapshot,
+	peer *v1.Multihost_Peer,
+	oplog *oplog.OpLog,
+) (*SyncClient, error) {
 	if peer.GetInstanceUrl() == "" {
 		return nil, errors.New("peer instance URL is required")
 	}
@@ -63,13 +70,14 @@ func NewSyncClient(mgr *SyncManager, localInstanceID string, peer *v1.Multihost_
 	)
 
 	c := &SyncClient{
-		mgr:             mgr,
-		localInstanceID: localInstanceID,
-		peer:            peer,
-		reconnectDelay:  mgr.syncClientRetryDelay,
-		client:          client,
-		oplog:           oplog,
-		l:               zap.L().Named(fmt.Sprintf("syncclient for %q", peer.GetInstanceId())),
+		mgr:                mgr,
+		syncConfigSnapshot: snapshot,
+		localInstanceID:    snapshot.config.Instance,
+		peer:               peer,
+		reconnectDelay:     mgr.syncClientRetryDelay,
+		client:             client,
+		oplog:              oplog,
+		l:                  zap.L().Named(fmt.Sprintf("syncclient for %q", peer.GetInstanceId())),
 	}
 	c.setConnectionState(v1.SyncConnectionState_CONNECTION_STATE_DISCONNECTED, "starting up")
 	return c, nil
@@ -117,11 +125,14 @@ func (c *SyncClient) runSyncInternal(ctx context.Context) error {
 	c.l.Info("connecting to sync server")
 	stream := c.client.Sync(ctx)
 
+	localConfig := c.syncConfigSnapshot.config
+	localIdentityKey := c.syncConfigSnapshot.identityKey
+
 	ctx, cancelWithError := context.WithCancelCause(ctx)
 	defer cancelWithError(nil)
 
-	receiveError := make(chan error, 1)
-	receive := make(chan *v1.SyncStreamItem, 1)
+	receiveError := make(chan error)
+	receive := make(chan *v1.SyncStreamItem)
 	send := make(chan *v1.SyncStreamItem, 100)
 
 	go func() {
@@ -129,26 +140,21 @@ func (c *SyncClient) runSyncInternal(ctx context.Context) error {
 			item, err := stream.Receive()
 			if err != nil {
 				receiveError <- err
-				return
+				break
 			}
 			receive <- item
 		}
+		close(receive)
 	}()
 
 	// Broadcast initial packet containing the protocol version and instance ID.
 	// TODO: do this in a header instead of as a part of the stream.
-	if err := stream.Send(&v1.SyncStreamItem{
-		Action: &v1.SyncStreamItem_Handshake{
-			Handshake: &v1.SyncStreamItem_SyncActionHandshake{
-				ProtocolVersion: SyncProtocolVersion,
-				InstanceId: &v1.SignedMessage{
-					Payload:   []byte(c.localInstanceID),
-					Signature: []byte("TOOD: inject a valid signature"),
-					Keyid:     "TODO: inject a valid key ID",
-				},
-			},
-		},
-	}); err != nil {
+	handshakePacket, err := createHandshakePacket(c.localInstanceID, localIdentityKey)
+	if err != nil {
+		return fmt.Errorf("create handshake packet: %w", err)
+	}
+
+	if err := stream.Send(handshakePacket); err != nil {
 		// note: the error checking w/streams in connectrpc is fairly awkward.
 		// If write returns an EOF error, we are expected to call stream.Receive()
 		// to get the unmarshalled network failure.
@@ -163,29 +169,22 @@ func (c *SyncClient) runSyncInternal(ctx context.Context) error {
 	}
 	c.setConnectionState(v1.SyncConnectionState_CONNECTION_STATE_CONNECTED, "connected")
 
+	c.l.Debug("sent handshake packet, now waiting for server handshake", zap.String("local_instance_id", c.localInstanceID), zap.String("host_instance_id", c.peer.InstanceId))
+
 	// Wait for the handshake packet from the server.
-	serverInstanceID := ""
-	if msg, ok := <-receive; ok {
-		handshake := msg.GetHandshake()
-		if handshake == nil {
-			return connect.NewError(connect.CodeInvalidArgument, errors.New("handshake packet must be sent first"))
-		}
-
-		serverInstanceID = string(handshake.GetInstanceId().GetPayload())
-		if serverInstanceID == "" {
-			return connect.NewError(connect.CodeInvalidArgument, errors.New("instance ID is required"))
-		}
-
-		if handshake.GetProtocolVersion() != SyncProtocolVersion {
-			return connect.NewError(connect.CodeFailedPrecondition, fmt.Errorf("unsupported peer protocol version, got %d, expected %d", handshake.GetProtocolVersion(), SyncProtocolVersion))
-		}
-	} else {
-		return connect.NewError(connect.CodeInvalidArgument, errors.New("no packets received"))
+	handshakeMsg, err := tryReceiveWithinDuration(ctx, receive, receiveError, 5*time.Second)
+	if err != nil {
+		return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("read error before handshake packet: %v", err))
 	}
-
-	if serverInstanceID != c.peer.InstanceId {
-		return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("server instance ID %q does not match expected peer instance ID %q", serverInstanceID, c.peer.InstanceId))
+	if _, err := verifyHandshakePacket(handshakeMsg); err != nil {
+		return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("verify handshake packet: %v", err))
 	}
+	if err := authorizeHandshakeAsPeer(handshakeMsg, c.peer); err != nil {
+		return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("authorize handshake packet: %v", err))
+	}
+	serverInstanceID := c.peer.InstanceId
+
+	c.l.Debug("received handshake packet from server", zap.String("server_instance_id", serverInstanceID))
 
 	// haveRunSync tracks which repo GUIDs we've initiated a sync for with the server.
 	// operation requests (from the server) are ignored if the GUID is not allowlisted in this map.
@@ -267,13 +266,6 @@ func (c *SyncClient) runSyncInternal(ctx context.Context) error {
 				}
 			}
 
-			// load the local config so that we can index the remote repos into any local repos that reference their URIs
-			// e.g. backrest:<instance-id> format URI.
-			localConfig, err := c.mgr.configMgr.Get()
-			if err != nil {
-				return fmt.Errorf("get local config: %w", err)
-			}
-
 			for _, repo := range newRemoteConfig.Repos {
 				_, ok := haveRunSync[repo.GetGuid()]
 				if ok {
@@ -291,8 +283,9 @@ func (c *SyncClient) runSyncInternal(ctx context.Context) error {
 				}
 
 				diffSel := &v1.OpSelector{
-					InstanceId: proto.String(c.localInstanceID),
-					RepoGuid:   proto.String(repo.GetGuid()),
+					InstanceId:            proto.String(c.localInstanceID),
+					OriginalInstanceKeyid: proto.String(c.syncConfigSnapshot.identityKey.KeyID()),
+					RepoGuid:              proto.String(repo.GetGuid()),
 				}
 
 				diffQuery, err := protoutil.OpSelectorToQuery(diffSel)

internal/api/syncapi/synccommon.go

@@ -0,0 +1,101 @@
+package syncapi
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	v1 "github.com/garethgeorge/backrest/gen/go/v1"
+	"github.com/garethgeorge/backrest/internal/cryptoutil"
+)
+
+func tryReceiveWithinDuration(ctx context.Context, receiveChan chan *v1.SyncStreamItem, receiveErrChan chan error, timeout time.Duration) (*v1.SyncStreamItem, error) {
+	if timeout > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, timeout)
+		defer cancel()
+	}
+
+	select {
+	case item := <-receiveChan:
+		return item, nil
+	case err := <-receiveErrChan:
+		return nil, err
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
+
+func createHandshakePacket(instanceID string, identity *cryptoutil.PrivateKey) (*v1.SyncStreamItem, error) {
+	instanceIDBytes := []byte(instanceID)
+	instanceIDBytesSignature, err := identity.Sign(instanceIDBytes)
+	if err != nil {
+		return nil, fmt.Errorf("signing instance ID: %w", err)
+	}
+
+	return &v1.SyncStreamItem{
+		Action: &v1.SyncStreamItem_Handshake{
+			Handshake: &v1.SyncStreamItem_SyncActionHandshake{
+				ProtocolVersion: SyncProtocolVersion,
+				InstanceId: &v1.SignedMessage{
+					Payload:   instanceIDBytes,
+					Signature: instanceIDBytesSignature,
+					Keyid:     identity.KeyID(),
+				},
+				PublicKey: identity.PublicKeyProto(),
+			},
+		},
+	}, nil
+}
+
+// verifyHandshakePacket verifies that
+//   - the signature on the instance ID is valid against the public key provided in the handshake
+//   - that the public key's ID is as attested in the handshake packet e.g. matches handshake.PublicKey.Keyid
+//
+// To authenticate, the caller must then check that the public key is trusted by checking the key ID against a local list.
+func verifyHandshakePacket(item *v1.SyncStreamItem) (*cryptoutil.PublicKey, error) {
+	handshake := item.GetHandshake()
+	if handshake == nil {
+		return nil, fmt.Errorf("empty or nil handshake, handshake packet must be sent first")
+	}
+
+	if handshake.ProtocolVersion != SyncProtocolVersion {
+		return nil, fmt.Errorf("protocol version mismatch: expected %d, got %d", SyncProtocolVersion, handshake.ProtocolVersion)
+	}
+
+	if len(handshake.InstanceId.GetPayload()) == 0 || len(handshake.InstanceId.GetSignature()) == 0 {
+		return nil, errors.New("instance ID payload and signature must not be empty")
+	}
+
+	if len(handshake.PublicKey.Keyid) == 0 {
+		return nil, errors.New("public key ID must not be empty")
+	}
+
+	peerKey, err := cryptoutil.NewPublicKey(handshake.PublicKey)
+	if err != nil {
+		return nil, fmt.Errorf("loading peer public key: %w", err)
+	}
+
+	if err := peerKey.Verify(handshake.InstanceId.GetPayload(), handshake.InstanceId.GetSignature()); err != nil {
+		return nil, fmt.Errorf("verifying instance ID: %w", err)
+	}
+
+	return peerKey, nil
+}
+
+// authorizeHandshakeAsPeer checks that the handshake packet has the expected key ID and instance ID.
+// If this succeeds and the handshake is verified, then it is safe to assume the identity we are talking to.
+func authorizeHandshakeAsPeer(item *v1.SyncStreamItem, peer *v1.Multihost_Peer) error {
+	handshake := item.GetHandshake()
+	if handshake == nil {
+		return fmt.Errorf("empty or nil handshake, handshake packet must be sent first")
+	}
+	if handshake.GetPublicKey().GetKeyid() != peer.Keyid {
+		return fmt.Errorf("public key ID mismatch: expected %s, got %s", peer.Keyid, handshake.PublicKey.Keyid)
+	}
+	if string(handshake.GetInstanceId().GetPayload()) != peer.InstanceId {
+		return fmt.Errorf("instance ID mismatch: expected %s, got %s", peer.InstanceId, string(handshake.InstanceId.GetPayload()))
+	}
+	return nil
+}

internal/api/syncapi/synchandler.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"slices"
 	"sort"
+	"time"
 
 	"connectrpc.com/connect"
 	v1 "github.com/garethgeorge/backrest/gen/go/v1"
@@ -35,17 +36,22 @@ func NewBackrestSyncHandler(mgr *SyncManager) *BackrestSyncHandler {
 func (h *BackrestSyncHandler) Sync(ctx context.Context, stream *connect.BidiStream[v1.SyncStreamItem, v1.SyncStreamItem]) error {
 	// TODO: this request can be very long lived, we must periodically refresh the config
 	// e.g. to disconnect a client if its access is revoked.
-	initialConfig, err := h.mgr.configMgr.Get()
-	if err != nil {
-		return err
+	snapshot := h.mgr.getSyncConfigSnapshot()
+	if snapshot == nil {
+		return connect.NewError(connect.CodePermissionDenied, errors.New("sync server is not configured"))
 	}
 
-	receive := make(chan *v1.SyncStreamItem, 1)
+	initialConfig := snapshot.config
+	identityKey := snapshot.identityKey
+
+	receiveError := make(chan error)
+	receive := make(chan *v1.SyncStreamItem)
 	send := make(chan *v1.SyncStreamItem, 1)
 	go func() {
 		for {
 			item, err := stream.Receive()
 			if err != nil {
+				receiveError <- err
 				break
 			}
 			receive <- item
@@ -55,37 +61,26 @@ func (h *BackrestSyncHandler) Sync(ctx context.Context, stream *connect.BidiStre
 
 	// Broadcast initial packet containing the protocol version and instance ID.
 	zap.S().Debugf("syncserver a client connected, broadcast handshake as %v", initialConfig.Instance)
-	if err := stream.Send(&v1.SyncStreamItem{
-		Action: &v1.SyncStreamItem_Handshake{
-			Handshake: &v1.SyncStreamItem_SyncActionHandshake{
-				ProtocolVersion: SyncProtocolVersion,
-				InstanceId: &v1.SignedMessage{
-					Payload:   []byte(initialConfig.Instance),
-					Signature: []byte("TODO: inject a valid signature"),
-					Keyid:     "TODO: inject a valid key ID",
-				},
-			},
-		},
-	}); err != nil {
+	handshakePacket, err := createHandshakePacket(initialConfig.Instance, identityKey)
+	if err != nil {
+		zap.S().Warnf("syncserver failed to create handshake packet: %v", err)
+		return connect.NewError(connect.CodeInternal, errors.New("couldn't build handshake packet, check server logs"))
+	}
+	if err := stream.Send(handshakePacket); err != nil {
 		return err
 	}
 
 	// Try to read the handshake packet from the client.
 	// TODO: perform this handshake in a header as a pre-flight before opening the stream.
-	clientInstanceID := ""
-	if msg, ok := <-receive; ok {
-		handshake := msg.GetHandshake()
-		if handshake == nil {
-			return connect.NewError(connect.CodeInvalidArgument, errors.New("handshake packet must be sent first"))
-		}
-
-		clientInstanceID = string(handshake.GetInstanceId().GetPayload())
-		if clientInstanceID == "" {
-			return connect.NewError(connect.CodeInvalidArgument, errors.New("instance ID is required"))
-		}
-	} else {
-		return connect.NewError(connect.CodeInvalidArgument, errors.New("no packets received"))
+	handshakeMsg, err := tryReceiveWithinDuration(ctx, receive, receiveError, 5*time.Second)
+	if err != nil {
+		return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("handshake packet not received: %w", err))
 	}
+	handshake := handshakeMsg.GetHandshake()
+	if _, err := verifyHandshakePacket(handshakeMsg); err != nil {
+		return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("verify handshake packet: %w", err))
+	}
+	clientInstanceID := string(handshake.GetInstanceId().Payload)
 
 	var authorizedClientPeer *v1.Multihost_Peer
 	authorizedClientPeerIdx := slices.IndexFunc(initialConfig.Multihost.GetAuthorizedClients(), func(peer *v1.Multihost_Peer) bool {
@@ -98,20 +93,36 @@ func (h *BackrestSyncHandler) Sync(ctx context.Context, stream *connect.BidiStre
 	} else {
 		authorizedClientPeer = initialConfig.Multihost.AuthorizedClients[authorizedClientPeerIdx]
 	}
-	zap.S().Infof("syncserver accepted a connection from client instance ID %q", authorizedClientPeer.InstanceId)
 
-	opIDLru, _ := lru.New[int64, int64](128)   // original ID -> local ID
-	flowIDLru, _ := lru.New[int64, int64](128) // original flow ID -> local flow ID
+	if !authorizedClientPeer.KeyidVerified {
+		return errors.New("authorized keyid must be verified prior to establishing connection")
+	} else if err := authorizeHandshakeAsPeer(handshakeMsg, authorizedClientPeer); err != nil {
+		return connect.NewError(connect.CodePermissionDenied, fmt.Errorf("rejected authorization as peer %v: %w", authorizedClientPeer.InstanceId, err))
+	}
+
+	// TODO: implement key handshake and verification
+	// key handshake flow is
+	// 1. both ends send their public keys and key ids
+	// 2. key ids are checked against values stored in config and against the public key exchanged. E.g. it must match the hash of the key.
+	// 3. start communicating.
+
+	zap.S().Infof("syncserver accepted a connection from client instance ID %q", authorizedClientPeer.InstanceId)
+	opIDLru, _ := lru.New[int64, int64](4096)   // original ID -> local ID
+	flowIDLru, _ := lru.New[int64, int64](1024) // original flow ID -> local flow ID
 
 	insertOrUpdate := func(op *v1.Operation) error {
+		op.OriginalInstanceKeyid = authorizedClientPeer.Keyid
 		op.OriginalId = op.Id
 		op.OriginalFlowId = op.FlowId
+		op.Id = 0
+		op.FlowId = 0
+
 		var ok bool
 		if op.Id, ok = opIDLru.Get(op.OriginalId); !ok {
 			var foundOp *v1.Operation
 			if err := h.mgr.oplog.Query(oplog.Query{}.
-				SetOriginalID(op.OriginalId).
-				SetInstanceID(op.InstanceId), func(o *v1.Operation) error {
+				SetOriginalInstanceKeyid(op.OriginalInstanceKeyid).
+				SetOriginalID(op.OriginalId), func(o *v1.Operation) error {
 				foundOp = o
 				return nil
 			}); err != nil {
@@ -125,8 +136,8 @@ func (h *BackrestSyncHandler) Sync(ctx context.Context, stream *connect.BidiStre
 		if op.FlowId, ok = flowIDLru.Get(op.OriginalFlowId); !ok {
 			var flowOp *v1.Operation
 			if err := h.mgr.oplog.Query(oplog.Query{}.
-				SetOriginalFlowID(op.OriginalFlowId).
-				SetInstanceID(op.InstanceId), func(o *v1.Operation) error {
+				SetOriginalInstanceKeyid(op.OriginalInstanceKeyid).
+				SetOriginalFlowID(op.OriginalFlowId), func(o *v1.Operation) error {
 				flowOp = o
 				return nil
 			}); err != nil {
@@ -143,7 +154,9 @@ func (h *BackrestSyncHandler) Sync(ctx context.Context, stream *connect.BidiStre
 
 	deleteByOriginalID := func(originalID int64) error {
 		var foundOp *v1.Operation
-		if err := h.mgr.oplog.Query(oplog.Query{}.SetOriginalID(originalID), func(o *v1.Operation) error {
+		if err := h.mgr.oplog.Query(oplog.Query{}.
+			SetOriginalInstanceKeyid(authorizedClientPeer.Keyid).
+			SetOriginalID(originalID), func(o *v1.Operation) error {
 			foundOp = o
 			return nil
 		}); err != nil {
@@ -190,7 +203,6 @@ func (h *BackrestSyncHandler) Sync(ctx context.Context, stream *connect.BidiStre
 			return errors.New("clients can not push configs to server")
 		case *v1.SyncStreamItem_DiffOperations:
 			diffSel := action.DiffOperations.GetHaveOperationsSelector()
-
 			if diffSel == nil {
 				return connect.NewError(connect.CodeInvalidArgument, errors.New("action DiffOperations: selector is required"))
 			}
@@ -317,14 +329,14 @@ func (h *BackrestSyncHandler) Sync(ctx context.Context, stream *connect.BidiStre
 				zap.L().Debug("syncserver received created operations", zap.Any("operations", event.CreatedOperations.GetOperations()))
 				for _, op := range event.CreatedOperations.GetOperations() {
 					if err := insertOrUpdate(op); err != nil {
-						return fmt.Errorf("action SendOperations: operation event create: %w", err)
+						return fmt.Errorf("action SendOperations: operation event create %+v: %w", op, err)
 					}
 				}
 			case *v1.OperationEvent_UpdatedOperations:
 				zap.L().Debug("syncserver received update operations", zap.Any("operations", event.UpdatedOperations.GetOperations()))
 				for _, op := range event.UpdatedOperations.GetOperations() {
 					if err := insertOrUpdate(op); err != nil {
-						return fmt.Errorf("action SendOperations: operation event update: %w", err)
+						return fmt.Errorf("action SendOperations: operation event update %+v: %w", op, err)
 					}
 				}
 			case *v1.OperationEvent_DeletedOperations:
@@ -352,22 +364,23 @@ func (h *BackrestSyncHandler) Sync(ctx context.Context, stream *connect.BidiStre
 
 	for {
 		select {
-		case item, ok := <-receive:
-			if !ok {
-				return nil
-			}
-
-			if err := handleSyncCommand(item); err != nil {
-				return err
-			}
+		case err := <-receiveError:
+			zap.S().Debugf("syncserver receive error from client %q: %v", authorizedClientPeer.InstanceId, err)
+			return err
 		case sendItem, ok := <-send: // note: send channel should only be used when sending from a different goroutine than the main loop
 			if !ok {
 				return nil
 			}
-
 			if err := stream.Send(sendItem); err != nil {
 				return err
 			}
+		case item, ok := <-receive:
+			if !ok {
+				return nil
+			}
+			if err := handleSyncCommand(item); err != nil {
+				return err
+			}
 		case <-configWatchCh:
 			newConfig, err := h.mgr.configMgr.Get()
 			if err != nil {

internal/api/syncapi/syncmanager.go

@@ -10,6 +10,7 @@ import (
 
 	v1 "github.com/garethgeorge/backrest/gen/go/v1"
 	"github.com/garethgeorge/backrest/internal/config"
+	"github.com/garethgeorge/backrest/internal/cryptoutil"
 	"github.com/garethgeorge/backrest/internal/oplog"
 	"github.com/garethgeorge/backrest/internal/orchestrator"
 	"go.uber.org/zap"
@@ -24,9 +25,11 @@ type SyncManager struct {
 	// mutable properties
 	mu sync.Mutex
 
-	syncClientRetryDelay time.Duration // the default retry delay for sync clients
+	snapshot *syncConfigSnapshot // the current snapshot of the sync context, protected by mu
 
-	syncClients map[string]*SyncClient
+	syncClientRetryDelay time.Duration // the default retry delay for sync clients, protected by mu
+
+	syncClients map[string]*SyncClient // current sync clients, protected by mu
 }
 
 func NewSyncManager(configMgr *config.ConfigManager, remoteConfigStore RemoteConfigStore, oplog *oplog.OpLog, orchestrator *orchestrator.Orchestrator) *SyncManager {
@@ -75,12 +78,26 @@ func (m *SyncManager) RunSync(ctx context.Context) {
 			return
 		}
 
-		if len(config.Multihost.GetKnownHosts()) == 0 {
-			zap.L().Debug("syncmanager no known host peers declared, sync client exiting early")
+		// Pull out configuration from the new config and cache it for sync handler e.g. the config and identity key.
+		identityKey, err := cryptoutil.NewPrivateKey(config.Multihost.GetIdentity())
+		if err != nil {
+			zap.S().Warnf("syncmanager failed to load local instance identity key, synchandler will reject requests: %v", err)
 			return
 		}
 
-		zap.S().Infof("syncmanager applying new config, starting sync goroutines for %d known peers", len(config.Multihost.GetKnownHosts()))
+		m.snapshot = &syncConfigSnapshot{
+			config:      config,
+			identityKey: identityKey,
+		}
+
+		// Past this point, determine if sync clients are configured and start threads for any.
+		if len(config.Multihost.GetKnownHosts()) == 0 {
+			zap.L().Info("syncmanager no known host peers declared, sync client exiting early")
+			return
+		}
+
+		zap.S().Infof("syncmanager applying new config, starting sync with identity %v, spawning goroutines for %d known peers",
+			config.Multihost.GetIdentity().GetKeyid(), len(config.Multihost.GetKnownHosts()))
 		for _, knownHostPeer := range config.Multihost.KnownHosts {
 			if knownHostPeer.InstanceId == "" {
 				continue
@@ -115,9 +132,11 @@ func (m *SyncManager) RunSync(ctx context.Context) {
 func (m *SyncManager) runSyncWithPeerInternal(ctx context.Context, config *v1.Config, knownHostPeer *v1.Multihost_Peer) error {
 	if config.Instance == "" {
 		return errors.New("local instance must set instance name before peersync can be enabled")
+	} else if config.Multihost == nil {
+		return errors.New("multihost config must be set before peersync can be enabled")
 	}
 
-	newClient, err := NewSyncClient(m, config.Instance, knownHostPeer, m.oplog)
+	newClient, err := NewSyncClient(m, *m.snapshot, knownHostPeer, m.oplog)
 	if err != nil {
 		return fmt.Errorf("creating sync client: %w", err)
 	}
@@ -134,3 +153,21 @@ func (m *SyncManager) runSyncWithPeerInternal(ctx context.Context, config *v1.Co
 
 	return nil
 }
+
+type syncConfigSnapshot struct {
+	config      *v1.Config
+	identityKey *cryptoutil.PrivateKey
+}
+
+func (m *SyncManager) getSyncConfigSnapshot() *syncConfigSnapshot {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if m.snapshot == nil {
+		return nil
+	}
+
+	// defensive copy
+	copy := *m.snapshot
+	return &copy
+}