fix: retry mumble client on socket errors

src/mumble/client.ts

@@ -1,4 +1,6 @@
 import { Client } from '@tf2pickup-org/mumble-client'
+import { secondsToMilliseconds } from 'date-fns'
+import { delay } from 'es-toolkit'
 import { configuration } from '../configuration'
 import { VoiceServerType } from '../shared/types/voice-server-type'
 import { logger } from '../logger'
@@ -12,6 +14,9 @@ import { errors } from '../errors'
 
 export let client: Client | undefined
 
+const maxReconnectAttempts = 3
+const reconnectDelay = secondsToMilliseconds(1)
+
 export async function tryConnect() {
   client?.disconnect()
   setStatus(MumbleClientStatus.disconnected)
@@ -49,34 +54,105 @@ export async function tryConnect() {
       rejectUnauthorized: false,
     })
 
-    await client.connect()
-    assertClientIsConnected(client)
-    logger.info(
-      {
-        mumbleUser: {
-          name: client.user.name,
-        },
-        welcomeText: client.welcomeText,
-      },
-      `connected to the mumble server`,
-    )
-
-    await client.user.setSelfDeaf(true)
-    await moveToTargetChannel()
-
-    const permissions = await client.user.channel.getPermissions()
-    if (!permissions.canCreateChannel) {
-      logger.warn(`bot ${client.user.name} does not have permissions to create new channels`)
-    }
-    setStatus(MumbleClientStatus.connected)
-
-    client.on('error', (error: unknown) => {
-      logger.error(error, 'mumble client error')
-      setStatus(MumbleClientStatus.error)
-      events.emit('mumble/error', { error })
+    const localClient = client
+    let attempt = 0
+    let reconnecting = false
+    localClient.on('error', async (error: unknown) => {
+      if (!isSocketError(error)) {
+        logger.error(error, 'mumble client error')
+        reportError(error)
+        return
+      }
+
+      if (reconnecting) {
+        return
+      }
+
+      reconnecting = true
+      attempt += 1
+
+      if (attempt > maxReconnectAttempts) {
+        logger.error(error, 'mumble socket error')
+        reportError(error)
+        reconnecting = false
+        return
+      }
+
+      logger.warn(
+        error,
+        `mumble socket error, reconnect attempt ${attempt}/${maxReconnectAttempts}...`,
+      )
+      await delay(reconnectDelay)
+
+      try {
+        await localClient.connect()
+        await afterConnect(localClient)
+        attempt = 0
+      } catch (reconnectError) {
+        logger.error(reconnectError, 'mumble reconnect error')
+        reportError(reconnectError)
+      }
+      reconnecting = false
     })
+
+    await localClient.connect()
+    await afterConnect(localClient)
   } catch (error) {
-    setStatus(MumbleClientStatus.error)
+    reportError(error)
     throw error
   }
 }
+
+function reportError(error: unknown) {
+  setStatus(MumbleClientStatus.error)
+  events.emit('mumble/error', { error })
+}
+
+async function afterConnect(c: Client) {
+  assertClientIsConnected(c)
+  logger.info(
+    {
+      mumbleUser: {
+        name: c.user.name,
+      },
+      welcomeText: c.welcomeText,
+    },
+    `connected to the mumble server`,
+  )
+
+  await c.user.setSelfDeaf(true)
+  await moveToTargetChannel(c)
+
+  const permissions = await c.user.channel.getPermissions()
+  if (!permissions.canCreateChannel) {
+    logger.warn(`bot ${c.user.name} does not have permissions to create new channels`)
+  }
+  setStatus(MumbleClientStatus.connected)
+}
+
+function isSocketError(error: unknown) {
+  if (!(error instanceof Error)) {
+    return false
+  }
+
+  if (error.message === 'socket not writable') {
+    return true
+  }
+
+  return (
+    'code' in error &&
+    typeof error.code === 'string' &&
+    [
+      'ECONNRESET',
+      'ECONNREFUSED',
+      'EHOSTDOWN',
+      'EHOSTUNREACH',
+      'ENETDOWN',
+      'ENETRESET',
+      'ENETUNREACH',
+      'ENOTFOUND',
+      'EPIPE',
+      'ETIMEDOUT',
+    ].includes(error.code)
+  )
+}

src/mumble/move-to-target-channel.ts

@@ -1,25 +1,24 @@
-import type { Channel } from '@tf2pickup-org/mumble-client'
+import type { Channel, Client } from '@tf2pickup-org/mumble-client'
 import { configuration } from '../configuration'
 import { logger } from '../logger'
 import { assertClientIsConnected } from './assert-client-is-connected'
-import { client } from './client'
 import { errors } from '../errors'
 
-export async function moveToTargetChannel() {
-  assertClientIsConnected(client)
+export async function moveToTargetChannel(c: Client | undefined) {
+  assertClientIsConnected(c)
   let channel: Channel | undefined
 
   const channelName = await configuration.get('games.voice_server.mumble.channel_name')
   if (!channelName) {
-    channel = client.channels.byId(0) // 0 is the root channel
+    channel = c.channels.byId(0) // 0 is the root channel
   } else {
-    channel = client.channels.byName(channelName)
+    channel = c.channels.byName(channelName)
   }
 
   if (!channel) {
     throw errors.badRequest(`channel does not exist: ${channelName}`)
   }
 
   logger.trace({ channel: { id: channel.id, name: channel.name } }, 'mumble channel found')
-  await client.user.moveToChannel(channel.id)
+  await c.user.moveToChannel(channel.id)
 }

src/mumble/plugins/remove-channels.ts

@@ -16,7 +16,7 @@ export default fp(
   // eslint-disable-next-line @typescript-eslint/require-await
   async () => {
     tasks.register('mumble.cleanupChannel', async ({ gameNumber }) => {
-      await moveToTargetChannel()
+      await moveToTargetChannel(client)
       assertClientIsConnected(client)
 
       const channel = client.user.channel.subChannels.find(({ name }) => name === `${gameNumber}`)

src/mumble/setup-game-channels.ts

@@ -12,7 +12,7 @@ const subChannelNames = [Tf2Team.blu.toUpperCase(), Tf2Team.red.toUpperCase()]
 
 export async function setupGameChannels(game: GameModel) {
   assertClientIsConnected(client)
-  await moveToTargetChannel()
+  await moveToTargetChannel(client)
   const channelName = `${game.number}`
   const channel = await client.user.channel.createSubChannel(channelName)
   await Promise.all(