From 6ebac9ea1b57d102057b502b1c20d14687950772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Garapich?= Date: Wed, 13 May 2026 13:59:25 +0200 Subject: [PATCH 1/7] Retry mumble socket errors before notifying Discord --- src/mumble/client.ts | 107 +++++++++++++++++++++++++++++++++---------- 1 file changed, 82 insertions(+), 25 deletions(-) diff --git a/src/mumble/client.ts b/src/mumble/client.ts index f8f35b309..5d3324fd2 100644 --- a/src/mumble/client.ts +++ b/src/mumble/client.ts @@ -1,4 +1,6 @@ import { Client } from '@tf2pickup-org/mumble-client' +import { secondsToMilliseconds } from 'date-fns' +import { delay } from 'es-toolkit' import { configuration } from '../configuration' import { VoiceServerType } from '../shared/types/voice-server-type' import { logger } from '../logger' @@ -12,6 +14,9 @@ import { errors } from '../errors' export let client: Client | undefined +const maxReconnectAttempts = 3 +const reconnectDelay = secondsToMilliseconds(1) + export async function tryConnect() { client?.disconnect() setStatus(MumbleClientStatus.disconnected) @@ -49,34 +54,86 @@ export async function tryConnect() { rejectUnauthorized: false, }) - await client.connect() - assertClientIsConnected(client) - logger.info( - { - mumbleUser: { - name: client.user.name, - }, - welcomeText: client.welcomeText, - }, - `connected to the mumble server`, - ) - - await client.user.setSelfDeaf(true) - await moveToTargetChannel() - - const permissions = await client.user.channel.getPermissions() - if (!permissions.canCreateChannel) { - logger.warn(`bot ${client.user.name} does not have permissions to create new channels`) - } - setStatus(MumbleClientStatus.connected) - - client.on('error', (error: unknown) => { - logger.error(error, 'mumble client error') - setStatus(MumbleClientStatus.error) - events.emit('mumble/error', { error }) + let attempt = 0 + client.on('error', async (error: unknown) => { + if (!isSocketError(error)) { + logger.error(error, 'mumble client error') + setStatus(MumbleClientStatus.error) + events.emit('mumble/error', { error }) + return + } + + attempt += 1 + if (attempt >= maxReconnectAttempts) { + logger.error(error, 'mumble socket error') + setStatus(MumbleClientStatus.error) + events.emit('mumble/error', { error }) + return + } + + logger.warn( + error, + `mumble socket error, reconnect attempt ${attempt + 1}/${maxReconnectAttempts}...`, + ) + await delay(reconnectDelay) + await client?.connect() + await afterConnect() }) + + await client.connect() + await afterConnect() } catch (error) { setStatus(MumbleClientStatus.error) + events.emit('mumble/error', { error }) throw error } } + +async function afterConnect() { + assertClientIsConnected(client) + logger.info( + { + mumbleUser: { + name: client.user.name, + }, + welcomeText: client.welcomeText, + }, + `connected to the mumble server`, + ) + + await client.user.setSelfDeaf(true) + await moveToTargetChannel() + + const permissions = await client.user.channel.getPermissions() + if (!permissions.canCreateChannel) { + logger.warn(`bot ${client.user.name} does not have permissions to create new channels`) + } + setStatus(MumbleClientStatus.connected) +} + +function isSocketError(error: unknown) { + if (!(error instanceof Error)) { + return false + } + + if (error.message === 'socket not writable') { + return true + } + + return ( + 'code' in error && + typeof error.code === 'string' && + [ + 'ECONNRESET', + 'ECONNREFUSED', + 'EHOSTDOWN', + 'EHOSTUNREACH', + 'ENETDOWN', + 'ENETRESET', + 'ENETUNREACH', + 'ENOTFOUND', + 'EPIPE', + 'ETIMEDOUT', + ].includes(error.code) + ) +} From 48325db2f1f2d22801a60e7c1222974fe7da41bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Garapich?= Date: Fri, 22 May 2026 18:06:11 +0200 Subject: [PATCH 2/7] fixes --- src/mumble/client.ts | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/src/mumble/client.ts b/src/mumble/client.ts index 5d3324fd2..74da58663 100644 --- a/src/mumble/client.ts +++ b/src/mumble/client.ts @@ -55,6 +55,7 @@ export async function tryConnect() { }) let attempt = 0 + let isReconnecting = false client.on('error', async (error: unknown) => { if (!isSocketError(error)) { logger.error(error, 'mumble client error') @@ -63,21 +64,32 @@ export async function tryConnect() { return } - attempt += 1 - if (attempt >= maxReconnectAttempts) { - logger.error(error, 'mumble socket error') + if (isReconnecting) return + isReconnecting = true + + try { + attempt += 1 + if (attempt >= maxReconnectAttempts) { + logger.error(error, 'mumble socket error') + setStatus(MumbleClientStatus.error) + events.emit('mumble/error', { error }) + return + } + + logger.warn( + error, + `mumble socket error, reconnect attempt ${attempt}/${maxReconnectAttempts}...`, + ) + await delay(reconnectDelay) + await client?.connect() + await afterConnect() + attempt = 0 + } catch (reconnectError) { setStatus(MumbleClientStatus.error) - events.emit('mumble/error', { error }) - return + events.emit('mumble/error', { error: reconnectError }) + } finally { + isReconnecting = false } - - logger.warn( - error, - `mumble socket error, reconnect attempt ${attempt + 1}/${maxReconnectAttempts}...`, - ) - await delay(reconnectDelay) - await client?.connect() - await afterConnect() }) await client.connect() From 0e9ad778889b337fb097ef619de05f80145526f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Garapich?= Date: Fri, 22 May 2026 18:16:27 +0200 Subject: [PATCH 3/7] cleanup --- src/mumble/client.ts | 49 +++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/src/mumble/client.ts b/src/mumble/client.ts index 74da58663..1784c6dce 100644 --- a/src/mumble/client.ts +++ b/src/mumble/client.ts @@ -1,6 +1,6 @@ import { Client } from '@tf2pickup-org/mumble-client' import { secondsToMilliseconds } from 'date-fns' -import { delay } from 'es-toolkit' +import { retry } from 'es-toolkit' import { configuration } from '../configuration' import { VoiceServerType } from '../shared/types/voice-server-type' import { logger } from '../logger' @@ -17,6 +17,11 @@ export let client: Client | undefined const maxReconnectAttempts = 3 const reconnectDelay = secondsToMilliseconds(1) +function reportError(error: unknown) { + setStatus(MumbleClientStatus.error) + events.emit('mumble/error', { error }) +} + export async function tryConnect() { client?.disconnect() setStatus(MumbleClientStatus.disconnected) @@ -54,49 +59,37 @@ export async function tryConnect() { rejectUnauthorized: false, }) - let attempt = 0 - let isReconnecting = false + let reconnecting = false + const localClient = client client.on('error', async (error: unknown) => { if (!isSocketError(error)) { logger.error(error, 'mumble client error') - setStatus(MumbleClientStatus.error) - events.emit('mumble/error', { error }) + reportError(error) return } - if (isReconnecting) return - isReconnecting = true + if (reconnecting) return + reconnecting = true + logger.warn(error, 'mumble socket error, attempting to reconnect...') try { - attempt += 1 - if (attempt >= maxReconnectAttempts) { - logger.error(error, 'mumble socket error') - setStatus(MumbleClientStatus.error) - events.emit('mumble/error', { error }) - return - } - - logger.warn( - error, - `mumble socket error, reconnect attempt ${attempt}/${maxReconnectAttempts}...`, + await retry( + async () => { + await localClient.connect() + await afterConnect() + }, + { retries: maxReconnectAttempts - 1, delay: reconnectDelay, shouldRetry: isSocketError }, ) - await delay(reconnectDelay) - await client?.connect() - await afterConnect() - attempt = 0 + reconnecting = false } catch (reconnectError) { - setStatus(MumbleClientStatus.error) - events.emit('mumble/error', { error: reconnectError }) - } finally { - isReconnecting = false + reportError(reconnectError) } }) await client.connect() await afterConnect() } catch (error) { - setStatus(MumbleClientStatus.error) - events.emit('mumble/error', { error }) + reportError(error) throw error } } From b46f5a5e92549c91a58a568d0fb276474f2e8c6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Garapich?= Date: Fri, 22 May 2026 18:25:32 +0200 Subject: [PATCH 4/7] clenaup --- src/mumble/client.ts | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/src/mumble/client.ts b/src/mumble/client.ts index 1784c6dce..f42b7fcbd 100644 --- a/src/mumble/client.ts +++ b/src/mumble/client.ts @@ -1,6 +1,6 @@ import { Client } from '@tf2pickup-org/mumble-client' import { secondsToMilliseconds } from 'date-fns' -import { retry } from 'es-toolkit' +import { delay } from 'es-toolkit' import { configuration } from '../configuration' import { VoiceServerType } from '../shared/types/voice-server-type' import { logger } from '../logger' @@ -17,11 +17,6 @@ export let client: Client | undefined const maxReconnectAttempts = 3 const reconnectDelay = secondsToMilliseconds(1) -function reportError(error: unknown) { - setStatus(MumbleClientStatus.error) - events.emit('mumble/error', { error }) -} - export async function tryConnect() { client?.disconnect() setStatus(MumbleClientStatus.disconnected) @@ -59,6 +54,7 @@ export async function tryConnect() { rejectUnauthorized: false, }) + let attempt = 0 let reconnecting = false const localClient = client client.on('error', async (error: unknown) => { @@ -71,18 +67,28 @@ export async function tryConnect() { if (reconnecting) return reconnecting = true - logger.warn(error, 'mumble socket error, attempting to reconnect...') + attempt += 1 + if (attempt > maxReconnectAttempts) { + logger.error(error, 'mumble socket error') + reportError(error) + return // reconnecting stays true — suppresses further events after exhaustion + } + + logger.warn( + error, + `mumble socket error, reconnect attempt ${attempt}/${maxReconnectAttempts}...`, + ) + await delay(reconnectDelay) + reconnecting = false // release before connecting so the next error event can trigger a new retry + try { - await retry( - async () => { - await localClient.connect() - await afterConnect() - }, - { retries: maxReconnectAttempts - 1, delay: reconnectDelay, shouldRetry: isSocketError }, - ) - reconnecting = false + await localClient.connect() + await afterConnect() + attempt = 0 } catch (reconnectError) { + logger.error(reconnectError, 'mumble reconnect error') reportError(reconnectError) + reconnecting = true // suppress further events on non-socket error } }) @@ -94,6 +100,11 @@ export async function tryConnect() { } } +function reportError(error: unknown) { + setStatus(MumbleClientStatus.error) + events.emit('mumble/error', { error }) +} + async function afterConnect() { assertClientIsConnected(client) logger.info( From 22f51dc08384796e8e68e6440b2230624f0db19f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Garapich?= Date: Fri, 22 May 2026 18:46:40 +0200 Subject: [PATCH 5/7] fix: thread client instance through afterConnect and moveToTargetChannel Eliminates two related issues: - afterConnect/moveToTargetChannel used the module-level `client` variable which could be stale if tryConnect() was called again during a retry - reconnecting flag was released before localClient.connect(), opening a window for parallel concurrent retry attempts Co-Authored-By: Claude Sonnet 4.6 --- src/mumble/client.ts | 24 ++++++++++++------------ src/mumble/move-to-target-channel.ts | 13 ++++++------- src/mumble/plugins/remove-channels.ts | 2 +- src/mumble/setup-game-channels.ts | 2 +- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/mumble/client.ts b/src/mumble/client.ts index f42b7fcbd..b5f3e2d70 100644 --- a/src/mumble/client.ts +++ b/src/mumble/client.ts @@ -79,21 +79,21 @@ export async function tryConnect() { `mumble socket error, reconnect attempt ${attempt}/${maxReconnectAttempts}...`, ) await delay(reconnectDelay) - reconnecting = false // release before connecting so the next error event can trigger a new retry try { await localClient.connect() - await afterConnect() + await afterConnect(localClient) attempt = 0 + reconnecting = false } catch (reconnectError) { logger.error(reconnectError, 'mumble reconnect error') reportError(reconnectError) - reconnecting = true // suppress further events on non-socket error + reconnecting = false } }) await client.connect() - await afterConnect() + await afterConnect(localClient) } catch (error) { reportError(error) throw error @@ -105,24 +105,24 @@ function reportError(error: unknown) { events.emit('mumble/error', { error }) } -async function afterConnect() { - assertClientIsConnected(client) +async function afterConnect(c: Client) { + assertClientIsConnected(c) logger.info( { mumbleUser: { - name: client.user.name, + name: c.user.name, }, - welcomeText: client.welcomeText, + welcomeText: c.welcomeText, }, `connected to the mumble server`, ) - await client.user.setSelfDeaf(true) - await moveToTargetChannel() + await c.user.setSelfDeaf(true) + await moveToTargetChannel(c) - const permissions = await client.user.channel.getPermissions() + const permissions = await c.user.channel.getPermissions() if (!permissions.canCreateChannel) { - logger.warn(`bot ${client.user.name} does not have permissions to create new channels`) + logger.warn(`bot ${c.user.name} does not have permissions to create new channels`) } setStatus(MumbleClientStatus.connected) } diff --git a/src/mumble/move-to-target-channel.ts b/src/mumble/move-to-target-channel.ts index ebef3f7ce..b35d31a5f 100644 --- a/src/mumble/move-to-target-channel.ts +++ b/src/mumble/move-to-target-channel.ts @@ -1,19 +1,18 @@ -import type { Channel } from '@tf2pickup-org/mumble-client' +import type { Channel, Client } from '@tf2pickup-org/mumble-client' import { configuration } from '../configuration' import { logger } from '../logger' import { assertClientIsConnected } from './assert-client-is-connected' -import { client } from './client' import { errors } from '../errors' -export async function moveToTargetChannel() { - assertClientIsConnected(client) +export async function moveToTargetChannel(c: Client | undefined) { + assertClientIsConnected(c) let channel: Channel | undefined const channelName = await configuration.get('games.voice_server.mumble.channel_name') if (!channelName) { - channel = client.channels.byId(0) // 0 is the root channel + channel = c.channels.byId(0) // 0 is the root channel } else { - channel = client.channels.byName(channelName) + channel = c.channels.byName(channelName) } if (!channel) { @@ -21,5 +20,5 @@ export async function moveToTargetChannel() { } logger.trace({ channel: { id: channel.id, name: channel.name } }, 'mumble channel found') - await client.user.moveToChannel(channel.id) + await c.user.moveToChannel(channel.id) } diff --git a/src/mumble/plugins/remove-channels.ts b/src/mumble/plugins/remove-channels.ts index 7339bf8fd..007bc809a 100644 --- a/src/mumble/plugins/remove-channels.ts +++ b/src/mumble/plugins/remove-channels.ts @@ -16,7 +16,7 @@ export default fp( // eslint-disable-next-line @typescript-eslint/require-await async () => { tasks.register('mumble.cleanupChannel', async ({ gameNumber }) => { - await moveToTargetChannel() + await moveToTargetChannel(client) assertClientIsConnected(client) const channel = client.user.channel.subChannels.find(({ name }) => name === `${gameNumber}`) diff --git a/src/mumble/setup-game-channels.ts b/src/mumble/setup-game-channels.ts index ccc2e5f1e..691784d5a 100644 --- a/src/mumble/setup-game-channels.ts +++ b/src/mumble/setup-game-channels.ts @@ -12,7 +12,7 @@ const subChannelNames = [Tf2Team.blu.toUpperCase(), Tf2Team.red.toUpperCase()] export async function setupGameChannels(game: GameModel) { assertClientIsConnected(client) - await moveToTargetChannel() + await moveToTargetChannel(client) const channelName = `${game.number}` const channel = await client.user.channel.createSubChannel(channelName) await Promise.all( From 71adb33bbf79d5741c0dfd72a9e467c34c4ad26e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Garapich?= Date: Fri, 22 May 2026 22:33:32 +0200 Subject: [PATCH 6/7] cleanup --- src/mumble/client.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/mumble/client.ts b/src/mumble/client.ts index b5f3e2d70..e400a8495 100644 --- a/src/mumble/client.ts +++ b/src/mumble/client.ts @@ -54,24 +54,27 @@ export async function tryConnect() { rejectUnauthorized: false, }) + const localClient = client let attempt = 0 let reconnecting = false - const localClient = client - client.on('error', async (error: unknown) => { + localClient.on('error', async (error: unknown) => { if (!isSocketError(error)) { logger.error(error, 'mumble client error') reportError(error) return } - if (reconnecting) return - reconnecting = true + if (reconnecting) { + return + } + reconnecting = true attempt += 1 + if (attempt > maxReconnectAttempts) { logger.error(error, 'mumble socket error') reportError(error) - return // reconnecting stays true — suppresses further events after exhaustion + return } logger.warn( @@ -84,15 +87,14 @@ export async function tryConnect() { await localClient.connect() await afterConnect(localClient) attempt = 0 - reconnecting = false } catch (reconnectError) { logger.error(reconnectError, 'mumble reconnect error') reportError(reconnectError) - reconnecting = false } + reconnecting = false }) - await client.connect() + await localClient.connect() await afterConnect(localClient) } catch (error) { reportError(error) From b6708e9ffa3696a15b4834c7ea1c9843dab9c059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Garapich?= Date: Fri, 22 May 2026 23:20:28 +0200 Subject: [PATCH 7/7] fix --- src/mumble/client.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mumble/client.ts b/src/mumble/client.ts index e400a8495..892095c44 100644 --- a/src/mumble/client.ts +++ b/src/mumble/client.ts @@ -74,6 +74,7 @@ export async function tryConnect() { if (attempt > maxReconnectAttempts) { logger.error(error, 'mumble socket error') reportError(error) + reconnecting = false return }