fix: redis startup (#2571)

add event logging and handling to deal with redis startup

src/backend/src/clients/redis/redisSingleton.test.ts

@@ -0,0 +1,102 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+const redisMocks = vi.hoisted(() => {
+    const redisClusterInstances: Array<{
+        on: ReturnType<typeof vi.fn>;
+        once: ReturnType<typeof vi.fn>;
+    }> = [];
+
+    return {
+        redisClusterInstances,
+        redisClusterConstructorMock: vi.fn(),
+        mockRedisClusterConstructorMock: vi.fn(),
+    };
+});
+
+vi.mock('ioredis', () => {
+    class RedisClusterMock {
+        on = vi.fn().mockReturnThis();
+        once = vi.fn().mockReturnThis();
+
+        constructor (...args: unknown[]) {
+            redisMocks.redisClusterConstructorMock(...args);
+            redisMocks.redisClusterInstances.push(this);
+        }
+    }
+
+    return {
+        default: {
+            Cluster: RedisClusterMock,
+        },
+    };
+});
+
+vi.mock('ioredis-mock', () => {
+    class MockRedisClusterMock {
+        constructor (...args: unknown[]) {
+            redisMocks.mockRedisClusterConstructorMock(...args);
+        }
+    }
+
+    return {
+        default: {
+            Cluster: MockRedisClusterMock,
+        },
+    };
+});
+
+describe('redisSingleton', () => {
+    const initialRedisConfig = process.env.REDIS_CONFIG;
+
+    beforeEach(() => {
+        vi.resetModules();
+        redisMocks.redisClusterInstances.length = 0;
+        redisMocks.redisClusterConstructorMock.mockReset();
+        redisMocks.mockRedisClusterConstructorMock.mockReset();
+        process.env.REDIS_CONFIG = JSON.stringify([{ host: '127.0.0.1', port: 6379 }]);
+        vi.spyOn(console, 'log').mockImplementation(() => undefined);
+        vi.spyOn(console, 'warn').mockImplementation(() => undefined);
+        vi.spyOn(console, 'error').mockImplementation(() => undefined);
+    });
+
+    afterEach(() => {
+        if ( initialRedisConfig === undefined ) {
+            delete process.env.REDIS_CONFIG;
+        } else {
+            process.env.REDIS_CONFIG = initialRedisConfig;
+        }
+        vi.restoreAllMocks();
+    });
+
+    it('uses resilient cluster options and registers startup-safe listeners', async () => {
+        const singletonModule = await import('./redisSingleton.ts');
+
+        expect(redisMocks.redisClusterConstructorMock).toHaveBeenCalledTimes(1);
+        const [startupNodes, clusterOptions] = redisMocks.redisClusterConstructorMock.mock.calls[0];
+
+        expect(startupNodes).toEqual([{ host: '127.0.0.1', port: 6379 }]);
+        expect(clusterOptions).toEqual(expect.objectContaining({
+            enableOfflineQueue: true,
+            retryDelayOnFailover: 500,
+            retryDelayOnClusterDown: 1000,
+            retryDelayOnTryAgain: 300,
+            slotsRefreshTimeout: 5000,
+            clusterRetryStrategy: expect.any(Function),
+            dnsLookup: expect.any(Function),
+            redisOptions: expect.objectContaining({
+                connectTimeout: 10000,
+                maxRetriesPerRequest: null,
+                tls: {},
+            }),
+        }));
+        expect(clusterOptions.clusterRetryStrategy(1)).toBe(200);
+        expect(clusterOptions.clusterRetryStrategy(100)).toBe(2000);
+
+        const clusterInstance = redisMocks.redisClusterInstances[0];
+        expect(singletonModule.redisClient).toBe(clusterInstance);
+        expect(clusterInstance.once).toHaveBeenCalledWith('connect', expect.any(Function));
+        expect(clusterInstance.once).toHaveBeenCalledWith('ready', expect.any(Function));
+        expect(clusterInstance.on).toHaveBeenCalledWith('error', expect.any(Function));
+        expect(clusterInstance.on).toHaveBeenCalledWith('node error', expect.any(Function));
+    });
+});

src/backend/src/clients/redis/redisSingleton.ts

@@ -1,20 +1,69 @@
 import Redis, { Cluster } from 'ioredis';
 import MockRedis from 'ioredis-mock';
 
+const redisStartupRetryMaxDelayMs = 2000;
+const redisSlotsRefreshTimeoutMs = 5000;
+const redisConnectTimeoutMs = 10000;
+const redisBootRetryRegex = /Cluster(All)?FailedError|None of startup nodes is available/i;
+
+const formatRedisError = (error: unknown): string => {
+    if ( error instanceof Error ) {
+        return `${error.name}: ${error.message}`;
+    }
+    return String(error);
+};
+
+const attachClusterEventHandlers = (clusterClient: Cluster): void => {
+    clusterClient.once('connect', () => {
+        console.log('[redis] cluster transport connected');
+    });
+
+    clusterClient.once('ready', () => {
+        console.log('[redis] cluster ready');
+    });
+
+    clusterClient.on('error', (error: unknown) => {
+        const errorText = formatRedisError(error);
+        if ( redisBootRetryRegex.test(errorText) ) {
+            console.warn(`[redis] startup issue while connecting to cluster; retrying automatically (${errorText})`);
+            return;
+        }
+        console.error('[redis] cluster error', error);
+    });
+
+    clusterClient.on('node error', (error: unknown, nodeKey: string) => {
+        const errorText = formatRedisError(error);
+        if ( redisBootRetryRegex.test(errorText) ) {
+            console.warn(`[redis] startup issue for cluster node ${nodeKey}; retrying automatically (${errorText})`);
+            return;
+        }
+        console.error(`[redis] cluster node error (${nodeKey})`, error);
+    });
+};
+
 let redisOpt: Cluster;
 
 if ( process.env.REDIS_CONFIG ) {
     const redisConfig = JSON.parse(process.env.REDIS_CONFIG);
     redisOpt = new Redis.Cluster(redisConfig, {
         dnsLookup: (address, callback) => callback(null, address),
+        clusterRetryStrategy: (attempts) => Math.min(100 + (attempts * 100), redisStartupRetryMaxDelayMs),
+        retryDelayOnFailover: 500,
+        retryDelayOnClusterDown: 1000,
+        retryDelayOnTryAgain: 300,
+        slotsRefreshTimeout: redisSlotsRefreshTimeoutMs,
+        enableOfflineQueue: true,
         redisOptions: {
             tls: {},
+            connectTimeout: redisConnectTimeoutMs,
+            maxRetriesPerRequest: null,
         },
     });
-    console.log('connected to redis from config');
+    attachClusterEventHandlers(redisOpt);
+    console.log('connecting to redis from config');
 } else {
-    redisOpt = new MockRedis.Cluster(['PuterS3Service._get_clientredis://localhost:7001']);
+    redisOpt = new MockRedis.Cluster(['redis://localhost:7001']);
     console.log('connected to local redis mock');
 }
 
-export const redisClient = redisOpt;
+export const redisClient = redisOpt;