Live sockets back off until a connection holds: one the server accepts and drops at once (a Worker being replaced) no longer reconnects every second from every open tab
5 files+38−30/5 viewed
| 3 | 3 | import type { ChatLiveEvent } from "@g1t/contracts"; | |
| 4 | 4 | ||
| 5 | 5 | import { backoff } from "../../lib/chat"; | |
| 6 | + | import { heldOpen } from "../../lib/notify-store"; | |
| 6 | 7 | ||
| 7 | 8 | /** How long a conversation's socket stays open after leaving it, while the next one opens. */ | |
| 8 | 9 | const HANDOFF_MS = 1500; | |
| ⋯ | |||
| 44 | 45 | return; | |
| 45 | 46 | } | |
| 46 | 47 | socket.current = ws; | |
| 48 | + | let openedAt: number | null = null; | |
| 47 | 49 | ws.onopen = () => { | |
| 48 | 50 | const again = opened; | |
| 49 | 51 | opened = true; | |
| 50 | − | attempt = 0; | |
| 52 | + | openedAt = Date.now(); | |
| 51 | 53 | setState("open"); | |
| 52 | 54 | if (again) handlers.current.onReconnect(); | |
| 53 | 55 | }; | |
| ⋯ | |||
| 61 | 63 | ws.onclose = () => { | |
| 62 | 64 | if (socket.current === ws) socket.current = null; | |
| 63 | 65 | if (closed) return; | |
| 66 | + | // Only a connection that held starts the backoff over. | |
| 67 | + | if (heldOpen(openedAt)) attempt = 0; | |
| 64 | 68 | setState(opened ? "reconnecting" : "connecting"); | |
| 65 | 69 | schedule(); | |
| 66 | 70 | }; | |
| 17 | 17 | import * as syncProtocol from "y-protocols/sync"; | |
| 18 | 18 | import * as Y from "yjs"; | |
| 19 | 19 | ||
| 20 | + | import { heldOpen } from "../../lib/notify-store"; | |
| 21 | + | ||
| 20 | 22 | const MESSAGE_SYNC = 0; | |
| 21 | 23 | const MESSAGE_AWARENESS = 1; | |
| 22 | 24 | const MESSAGE_QUERY_AWARENESS = 3; | |
| ⋯ | |||
| 29 | 31 | status: LiveStatus = "connecting"; | |
| 30 | 32 | private socket: WebSocket | null = null; | |
| 31 | 33 | private attempts = 0; | |
| 34 | + | /** When the current connection opened; the backoff starts over only once one holds. */ | |
| 35 | + | private openedAt: number | null = null; | |
| 32 | 36 | private timer: ReturnType<typeof setTimeout> | null = null; | |
| 33 | 37 | private keepalive: ReturnType<typeof setInterval> | null = null; | |
| 34 | 38 | private stopped = false; | |
| ⋯ | |||
| 74 | 78 | socket.binaryType = "arraybuffer"; | |
| 75 | 79 | this.socket = socket; | |
| 76 | 80 | socket.onopen = () => { | |
| 77 | − | this.attempts = 0; | |
| 81 | + | this.openedAt = Date.now(); | |
| 78 | 82 | // Our state vector: the room answers with what we lack, and asks for what it lacks. | |
| 79 | 83 | const encoder = encoding.createEncoder(); | |
| 80 | 84 | encoding.writeVarUint(encoder, MESSAGE_SYNC); | |
| ⋯ | |||
| 121 | 125 | return; | |
| 122 | 126 | } | |
| 123 | 127 | this.setStatus("offline"); | |
| 128 | + | // Only a connection that held starts the backoff over. | |
| 129 | + | if (heldOpen(this.openedAt)) this.attempts = 0; | |
| 130 | + | this.openedAt = null; | |
| 124 | 131 | const delay = Math.min(30_000, 500 * 2 ** this.attempts) + Math.random() * 500; | |
| 125 | 132 | this.attempts++; | |
| 126 | 133 | this.timer = setTimeout(() => this.connect(), delay); | |
| 30 | 30 | dismissToast, | |
| 31 | 31 | markReadLocally, | |
| 32 | 32 | offerPush, | |
| 33 | + | heldOpen, | |
| 33 | 34 | reconnectDelay, | |
| 34 | 35 | titleWith, | |
| 35 | 36 | waitingCards, | |
| ⋯ | |||
| 409 | 410 | return; | |
| 410 | 411 | } | |
| 411 | 412 | socket = ws; | |
| 413 | + | let openedAt: number | null = null; | |
| 412 | 414 | ws.onopen = () => { | |
| 413 | − | attempt = 0; | |
| 415 | + | openedAt = Date.now(); | |
| 414 | 416 | set({ connected: true }); | |
| 415 | 417 | sendState(); | |
| 416 | 418 | }; | |
| ⋯ | |||
| 424 | 426 | }; | |
| 425 | 427 | ws.onclose = () => { | |
| 426 | 428 | if (socket === ws) socket = null; | |
| 429 | + | // Only a connection that held starts the backoff over. | |
| 430 | + | if (heldOpen(openedAt)) attempt = 0; | |
| 427 | 431 | set({ connected: false }); | |
| 428 | 432 | schedule(); | |
| 429 | 433 | }; | |
| 4 | 4 | import type { ChatSidebarEntry, FeedCounts, FeedNotification } from "@g1t/contracts"; | |
| 5 | 5 | ||
| 6 | 6 | import { | |
| 7 | + | heldOpen, | |
| 7 | 8 | MAX_TOASTS, | |
| 8 | 9 | RECENT_KEPT, | |
| 9 | 10 | WAITING_MS, | |
| ⋯ | |||
| 216 | 217 | // Acting on the newest puts the card away; the older one about it does not come back. | |
| 217 | 218 | assert.deepEqual(waitingCards(recent, new Set(["approval:ses_1:2000000"]), now, "acme"), []); | |
| 218 | 219 | }); | |
| 220 | + | ||
| 221 | + | test("the backoff starts over only after a connection held for a while", () => { | |
| 222 | + | assert.equal(heldOpen(null, 50_000), false); | |
| 223 | + | assert.equal(heldOpen(45_000, 50_000), false, "dropped within seconds: keep backing off"); | |
| 224 | + | assert.equal(heldOpen(40_000, 50_000), true); | |
| 225 | + | }); | |
| 259 | 259 | } | |
| 260 | 260 | ||
| 261 | 261 | /** Full jitter: a wait anywhere up to the doubled step, from a second up to half a minute. */ | |
| 262 | + | /** | |
| 263 | + | * How long a connection must stay open before its backoff starts over. A | |
| 264 | + | * socket the server accepts and drops at once (a Worker being replaced, a | |
| 265 | + | * room that can't start) would otherwise reconnect every second from every | |
| 266 | + | * open tab, and a busy network runs into the rate limit. | |
| 267 | + | */ | |
| 268 | + | export const STABLE_MS = 10_000; | |
| 269 | + | ||
| 270 | + | /** Whether a connection that opened at `openedAt` held long enough to start the backoff over. */ | |
| 271 | + | export function heldOpen(openedAt: number | null, now: number = Date.now()): boolean { | |
| 272 | + | return openedAt !== null && now - openedAt >= STABLE_MS; | |
| 273 | + | } | |
| 274 | + | ||
| 262 | 275 | export function reconnectDelay(attempt: number, random: () => number = Math.random): number { | |
| 263 | 276 | const ceiling = Math.min(30_000, 1_000 * 2 ** Math.max(0, Math.min(attempt, 10))); | |
| 264 | 277 | return Math.round(500 + random() * (ceiling - 500)); |