| 1 | /** |
| 2 | * Taking and keeping production screenshots with Browser Rendering. |
| 3 | * |
| 4 | * A screenshot is a few seconds of browser time (at most `SETTLE_MS` and |
| 5 | * the shot itself): Workers Paid includes 10 browser hours a month and |
| 6 | * charges $0.09 an hour after that, so one costs about $0.0001-$0.0003. |
| 7 | * Each production deploy on g1t.page takes at most one, plus a retry after |
| 8 | * `RETRY_AFTER_MS` if it failed. A production deployed elsewhere takes one |
| 9 | * when its address is set or changed and when a production deployment is |
| 10 | * reported for it, and otherwise at most one a day, from the daily cron |
| 11 | * (at most `EXTERNAL_PER_RUN` a run): a hundred such projects come to |
| 12 | * about 3,000 shots and 4 to 10 browser hours a month. A project that |
| 13 | * shows an uploaded image instead is not visited at all. That is small |
| 14 | * enough not to meter. |
| 15 | */ |
| 16 | import puppeteer from "@cloudflare/puppeteer"; |
| 17 | |
| 18 | import { |
| 19 | type ExternalShot, |
| 20 | SETTLE_MS, |
| 21 | type ShotRequest, |
| 22 | VIEWPORT, |
| 23 | attemptKey, |
| 24 | dayOf, |
| 25 | dueForRefresh, |
| 26 | expired, |
| 27 | externalAttemptKey, |
| 28 | externalShotKey, |
| 29 | isCurrent, |
| 30 | isFresh, |
| 31 | parseExternalShot, |
| 32 | shotKey, |
| 33 | shouldAttempt, |
| 34 | } from "./screenshot.ts"; |
| 35 | |
| 36 | export type ShotEnv = { |
| 37 | BROWSER: Fetcher; |
| 38 | SCREENSHOTS: R2Bucket; |
| 39 | }; |
| 40 | |
| 41 | export type Shot = { body: ArrayBuffer; contentType: string; commit: string; capturedAt: string }; |
| 42 | |
| 43 | /** Where one screenshot is kept and its attempts noted, and the page it shows. */ |
| 44 | type Target = { key: string; attempt: string; url: string; host: string; commit: string }; |
| 45 | |
| 46 | async function read(env: ShotEnv, key: string): Promise<Shot | null> { |
| 47 | const object = await env.SCREENSHOTS.get(key); |
| 48 | if (!object) return null; |
| 49 | return { |
| 50 | body: await object.arrayBuffer(), |
| 51 | contentType: object.httpMetadata?.contentType ?? "image/jpeg", |
| 52 | commit: object.customMetadata?.commit ?? "", |
| 53 | capturedAt: object.customMetadata?.capturedAt ?? object.uploaded.toISOString(), |
| 54 | }; |
| 55 | } |
| 56 | |
| 57 | /** The kept screenshot, whatever commit it shows. */ |
| 58 | export function stored(env: ShotEnv, host: string): Promise<Shot | null> { |
| 59 | return read(env, shotKey(host)); |
| 60 | } |
| 61 | |
| 62 | /** The kept screenshot of a production deployed elsewhere, by its host. */ |
| 63 | export function storedExternal(env: ShotEnv, host: string): Promise<Shot | null> { |
| 64 | return read(env, externalShotKey(host)); |
| 65 | } |
| 66 | |
| 67 | /** Takes the target's screenshot, unless one was tried for its commit moments ago. */ |
| 68 | async function shoot(env: ShotEnv, target: Target): Promise<Shot | null> { |
| 69 | const marker = await env.SCREENSHOTS.head(target.attempt); |
| 70 | if (!shouldAttempt(marker?.customMetadata, target.commit, Date.now())) return null; |
| 71 | const at = new Date().toISOString(); |
| 72 | await env.SCREENSHOTS.put(target.attempt, "", { customMetadata: { commit: target.commit, at } }); |
| 73 | |
| 74 | let browser: Awaited<ReturnType<typeof puppeteer.launch>> | null = null; |
| 75 | try { |
| 76 | browser = await puppeteer.launch(env.BROWSER as unknown as Parameters<typeof puppeteer.launch>[0]); |
| 77 | const page = await browser.newPage(); |
| 78 | await page.setViewport(VIEWPORT); |
| 79 | // A page that keeps the network busy is taken as it is when time is up. |
| 80 | await page.goto(target.url, { waitUntil: "networkidle2", timeout: SETTLE_MS }).catch((error: unknown) => { |
| 81 | console.warn("og: page did not settle", target.host, String(error)); |
| 82 | }); |
| 83 | const image = (await page.screenshot({ type: "jpeg", quality: 80 })) as Uint8Array; |
| 84 | const body = image.buffer.slice(image.byteOffset, image.byteOffset + image.byteLength) as ArrayBuffer; |
| 85 | const capturedAt = new Date().toISOString(); |
| 86 | await env.SCREENSHOTS.put(target.key, body, { |
| 87 | httpMetadata: { contentType: "image/jpeg" }, |
| 88 | customMetadata: { commit: target.commit, capturedAt }, |
| 89 | }); |
| 90 | return { body, contentType: "image/jpeg", commit: target.commit, capturedAt }; |
| 91 | } catch (error) { |
| 92 | console.error("og: screenshot failed", target.host, error); |
| 93 | return null; |
| 94 | } finally { |
| 95 | await browser?.close().catch(() => undefined); |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | /** |
| 100 | * Takes the screenshot of `commit` of an app on g1t.page, unless one was |
| 101 | * tried for it moments ago. Returns it, or null when it was not taken. |
| 102 | */ |
| 103 | export function take(env: ShotEnv, { host, commit }: ShotRequest): Promise<Shot | null> { |
| 104 | return shoot(env, { key: shotKey(host), attempt: attemptKey(host), url: `https://${host}/`, host, commit }); |
| 105 | } |
| 106 | |
| 107 | /** Takes the screenshot of a production deployed elsewhere, at its address. */ |
| 108 | export function takeExternal(env: ShotEnv, { host, url, commit }: ExternalShot): Promise<Shot | null> { |
| 109 | return shoot(env, { key: externalShotKey(host), attempt: externalAttemptKey(host), url, host, commit }); |
| 110 | } |
| 111 | |
| 112 | /** The screenshot of the request's deploy, taken now if it has not been; else the last one kept. */ |
| 113 | export async function screenshotOf(env: ShotEnv, request: ShotRequest): Promise<Shot | null> { |
| 114 | const kept = await stored(env, request.host); |
| 115 | if (kept && isCurrent(kept, request)) return kept; |
| 116 | return (await take(env, request)) ?? kept; |
| 117 | } |
| 118 | |
| 119 | /** |
| 120 | * The screenshot of a production deployed elsewhere: the one kept, with |
| 121 | * `stale` when it should be taken again (older than a day, or than the |
| 122 | * deploy the request names), which the caller does in the background. With |
| 123 | * none kept, it is taken now. |
| 124 | */ |
| 125 | export async function externalOf(env: ShotEnv, request: ExternalShot, now = Date.now()): Promise<{ shot: Shot | null; stale: boolean }> { |
| 126 | const kept = await storedExternal(env, request.host); |
| 127 | if (kept) return { shot: kept, stale: !isFresh(kept, now, request.since) }; |
| 128 | return { shot: await takeExternal(env, request), stale: false }; |
| 129 | } |
| 130 | |
| 131 | /** |
| 132 | * The daily refresh of productions deployed elsewhere: each address whose |
| 133 | * screenshot is missing or more than a day old is taken again, the |
| 134 | * stalest first, one at a time, at most `EXTERNAL_PER_RUN`. Addresses that |
| 135 | * are not public are skipped. Returns how many were taken. |
| 136 | */ |
| 137 | export async function refreshExternal(env: ShotEnv, urls: readonly string[], now = Date.now()): Promise<number> { |
| 138 | const day = dayOf(now); |
| 139 | const byHost = new Map<string, ExternalShot>(); |
| 140 | for (const url of urls) { |
| 141 | const request = parseExternalShot({ url, commit: day }); |
| 142 | if (request && !byHost.has(request.host)) byHost.set(request.host, request); |
| 143 | } |
| 144 | const targets = await Promise.all( |
| 145 | [...byHost.values()].map(async (request) => { |
| 146 | const head = await env.SCREENSHOTS.head(externalShotKey(request.host)).catch(() => null); |
| 147 | return { request, capturedAt: head ? (head.customMetadata?.capturedAt ?? head.uploaded.toISOString()) : null }; |
| 148 | }), |
| 149 | ); |
| 150 | let taken = 0; |
| 151 | for (const { request } of dueForRefresh(targets, now)) { |
| 152 | if (await takeExternal(env, request)) taken += 1; |
| 153 | } |
| 154 | return taken; |
| 155 | } |
| 156 | |
| 157 | /** |
| 158 | * Deletes screenshots and attempt notes older than `KEEP_DAYS`: apps that |
| 159 | * moved or were removed leave theirs behind, and a live app's is taken |
| 160 | * again the next time its page asks. Returns how many went. |
| 161 | */ |
| 162 | export async function sweep(env: ShotEnv, now = Date.now()): Promise<number> { |
| 163 | let removed = 0; |
| 164 | for (const prefix of ["production/", "external/", "attempts/"]) { |
| 165 | let cursor: string | undefined; |
| 166 | do { |
| 167 | const page = await env.SCREENSHOTS.list({ prefix, cursor, limit: 500 }); |
| 168 | const old = page.objects.filter((object) => expired(object.uploaded, now)).map((object) => object.key); |
| 169 | if (old.length) { |
| 170 | await env.SCREENSHOTS.delete(old); |
| 171 | removed += old.length; |
| 172 | } |
| 173 | cursor = page.truncated ? page.cursor : undefined; |
| 174 | } while (cursor); |
| 175 | } |
| 176 | return removed; |
| 177 | } |