Commit

Self-hosting: RustFS for S3-compatible storage, buckets and the packs' lifecycle rule made with the AWS CLI

The compose file's object store is RustFS (rustfs/rustfs:1.0.1, pinned), on a new g1t-objects volume, health-checked at /health/ready. storage-setup (amazon/aws-cli:2.37.10) makes the packages, backups and clone pack buckets and puts the packs' bucket's lifecycle configuration: packs/ expires after 7 days, multipart uploads unfinished after a day are aborted. RustFS's scanner applies both, so no sweep of our own is needed. clone-check.mjs --s3 runs against RustFS with the AWS CLI on its network, and also checks the packs are multipart objects that read back whole and that both lifecycle rules are in place. smoke.sh publishes an npm package to the installation's registry and installs it back (PACKAGES=off skips).

syntaqxcommitted Parentd4dee47Browse files
4 files+117−740/4 viewed
+4−4
99 // - ARTIFACTS (git storage) becomes a service binding to workers/artifacts,
1010 // which keeps repositories in the git store (gitstore/server.mjs);
1111 // - EMAIL (Email Sending) becomes a service binding to workers/mail;
12−// - the packages service keeps files in S3-compatible storage (MinIO)
12+// - the packages service keeps files in S3-compatible storage (RustFS)
1313 // instead of R2, and the repos service its nightly backups (a bucket of
1414 // their own, BACKUP_S3_BUCKET);
1515 // - services that are off in this phase (agents, the context hub, the
216216 // Access requests are summarised to your own address, not g1t.sh's.
217217 config.vars.WAITLIST_NOTIFY_EMAIL = process.env.WAITLIST_NOTIFY_EMAIL ?? "";
218218 }
219− // Packages' files go to the compose file's MinIO (or any S3-compatible
219+ // Packages' files go to the compose file's RustFS (or any S3-compatible
220220 // store) instead of R2, with no request size limit, and package
221221 // addresses start with this installation's host.
222222 if (hosted.name === "g1t-packages") {
223223 Object.assign(config.vars, {
224224 BLOB_STORE: "s3",
225− S3_ENDPOINT: process.env.S3_ENDPOINT ?? "http://minio:9000",
225+ S3_ENDPOINT: process.env.S3_ENDPOINT ?? "http://rustfs:9000",
226226 S3_BUCKET: process.env.S3_BUCKET ?? "g1t-packages",
227227 S3_REGION: process.env.S3_REGION ?? "us-east-1",
228228 S3_ACCESS_KEY_ID: process.env.S3_ACCESS_KEY_ID ?? "",
252252 BACKUP_S3_BUCKET: process.env.BACKUP_S3_BUCKET ?? "g1t-backups",
253253 PACK_STORE: "s3",
254254 PACK_S3_BUCKET: process.env.PACK_S3_BUCKET ?? "g1t-git-packs",
255− S3_ENDPOINT: process.env.S3_ENDPOINT ?? "http://minio:9000",
255+ S3_ENDPOINT: process.env.S3_ENDPOINT ?? "http://rustfs:9000",
256256 S3_REGION: process.env.S3_REGION ?? "us-east-1",
257257 S3_ACCESS_KEY_ID: process.env.S3_ACCESS_KEY_ID ?? "",
258258 S3_SECRET_ACCESS_KEY: process.env.S3_SECRET_ACCESS_KEY ?? "",
+32−34
99 # What runs: the site and every core service in one workerd (g1t), git
1010 # repositories as bare repos on a volume (gitstore), the API in a second
1111 # workerd beside it, packages' files, backups and the clone pack cache in
12−# MinIO, and Mailpit for mail.
12+# RustFS (S3-compatible storage), and Mailpit for mail.
1313 # Agents, deployments, context search and billing are off. See
1414 # docs/SELF_HOSTING.md.
1515 name: g1t
5151 INVITES_PER_USER: ${INVITES_PER_USER:-}
5252 # Where summaries of new access requests go; empty sends none.
5353 WAITLIST_NOTIFY_EMAIL: ${WAITLIST_NOTIFY_EMAIL:-}
54− # Packages' files (container images and the rest), in MinIO below or
54+ # Packages' files (container images and the rest), in RustFS below or
5555 # any S3-compatible store. S3_PUBLIC_ENDPOINT, when clients can reach
5656 # the store, sends large downloads there directly.
57− S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
57+ S3_ENDPOINT: ${S3_ENDPOINT:-http://rustfs:9000}
5858 S3_BUCKET: ${S3_BUCKET:-g1t-packages}
5959 S3_REGION: ${S3_REGION:-us-east-1}
6060 S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-g1t}
6464 # the same store (docs/SELF_HOSTING.md, "Backups").
6565 BACKUP_S3_BUCKET: ${BACKUP_S3_BUCKET:-g1t-backups}
6666 # Packs kept for fresh clones, so the next clone of the same commit
67− # does not rebuild one; minio-setup expires them after 7 days.
67+ # does not rebuild one; storage-setup expires them after 7 days.
6868 PACK_S3_BUCKET: ${PACK_S3_BUCKET:-g1t-git-packs}
6969 volumes:
7070 - g1t-data:/data
7474 condition: service_healthy
7575 mailpit:
7676 condition: service_started
77− minio-setup:
77+ storage-setup:
7878 condition: service_completed_successfully
7979 restart: unless-stopped
8080
110110 # Not published: only the g1t container reaches it.
111111 restart: unless-stopped
112112
113− # Packages' files, backups and clone packs. Not published: only the g1t
114− # container reaches it, unless you publish 9000 and set
115− # S3_PUBLIC_ENDPOINT. MinIO no longer publishes images of its own;
116− # MINIO_IMAGE is a community build of the same server, with `mc` in it.
117− # Any S3-compatible store works in its place (S3_ENDPOINT).
118− minio:
119− image: ${MINIO_IMAGE:-pgsty/minio:latest}
120− command: ["server", "/data"]
113+ # Packages' files, backups and clone packs, in RustFS (S3-compatible).
114+ # Not published: only the g1t container reaches it, unless you publish
115+ # 9000 and set S3_PUBLIC_ENDPOINT. Any S3-compatible store works in its
116+ # place (S3_ENDPOINT).
117+ rustfs:
118+ image: ${RUSTFS_IMAGE:-rustfs/rustfs:1.0.1}
121119 environment:
122− MINIO_ROOT_USER: ${S3_ACCESS_KEY_ID:-g1t}
123− MINIO_ROOT_PASSWORD: ${S3_SECRET_ACCESS_KEY:-g1t-packages-secret}
124− # What the health check's `mc ready local` asks.
125− MC_HOST_local: http://localhost:9000
120+ RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY_ID:-g1t}
121+ RUSTFS_SECRET_KEY: ${S3_SECRET_ACCESS_KEY:-g1t-packages-secret}
122+ RUSTFS_CONSOLE_ENABLE: "false"
126123 volumes:
127− - g1t-packages:/data
124+ - g1t-objects:/data
128125 healthcheck:
129− test: ["CMD", "mc", "ready", "local"]
126+ test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://localhost:9000/health/ready"]
130127 interval: 5s
131128 retries: 20
132129 restart: unless-stopped
133130
134131 # Makes the buckets once, then exits: packages' files, backups, and
135− # clone packs with a rule that deletes packs 7 days old. (MinIO itself
136− # removes uploads left unfinished after 24 hours.)
137− minio-setup:
138− image: ${MINIO_IMAGE:-pgsty/minio:latest}
132+ # clone packs, with a lifecycle rule on the packs' bucket that deletes
133+ # packs 7 days old and uploads left unfinished after a day.
134+ storage-setup:
135+ image: ${AWS_CLI_IMAGE:-amazon/aws-cli:2.37.10}
139136 depends_on:
140− minio:
137+ rustfs:
141138 condition: service_healthy
142139 entrypoint:
143140 - sh
144141 - -c
145142 - >-
146143 set -e;
147− mc alias set local http://minio:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD";
148− mc mb --ignore-existing "local/$$S3_BUCKET";
149− mc mb --ignore-existing "local/$$BACKUP_S3_BUCKET";
150− mc mb --ignore-existing "local/$$PACK_S3_BUCKET";
151− if ! mc ilm rule ls "local/$$PACK_S3_BUCKET" >/dev/null 2>&1; then
152− mc ilm rule add --prefix packs/ --expire-days 7 "local/$$PACK_S3_BUCKET";
153− fi
144+ for bucket in "$$S3_BUCKET" "$$BACKUP_S3_BUCKET" "$$PACK_S3_BUCKET"; do
145+ aws s3api head-bucket --bucket "$$bucket" >/dev/null 2>&1 || aws s3api create-bucket --bucket "$$bucket" >/dev/null;
146+ done;
147+ aws s3api put-bucket-lifecycle-configuration --bucket "$$PACK_S3_BUCKET" --lifecycle-configuration
148+ '{"Rules":[{"ID":"expire-packs","Status":"Enabled","Filter":{"Prefix":"packs/"},"Expiration":{"Days":7}},{"ID":"abort-unfinished-uploads","Status":"Enabled","Filter":{"Prefix":""},"AbortIncompleteMultipartUpload":{"DaysAfterInitiation":1}}]}';
149+ echo "buckets ready"
154150 environment:
155− MINIO_ROOT_USER: ${S3_ACCESS_KEY_ID:-g1t}
156− MINIO_ROOT_PASSWORD: ${S3_SECRET_ACCESS_KEY:-g1t-packages-secret}
151+ AWS_ENDPOINT_URL: http://rustfs:9000
152+ AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-g1t}
153+ AWS_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-g1t-packages-secret}
154+ AWS_DEFAULT_REGION: ${S3_REGION:-us-east-1}
157155 S3_BUCKET: ${S3_BUCKET:-g1t-packages}
158156 BACKUP_S3_BUCKET: ${BACKUP_S3_BUCKET:-g1t-backups}
159157 PACK_S3_BUCKET: ${PACK_S3_BUCKET:-g1t-git-packs}
173171
174172 volumes:
175173 g1t-data:
176− g1t-packages:
174+ g1t-objects:
177175 g1t-status:
178176 g1t-git:
179177 g1t-secrets:
+25−3
22 # End-to-end check of a self-hosted g1t: sign up, confirm the email, make a
33 # workspace and a repository, push and clone over HTTP, open an issue, and
44 # read the code back through the site. Then the API on its own port (REST,
5−# OAuth metadata and MCP, with an access token), pull requests from a branch
5+# OAuth metadata and MCP, with an access token), an npm package published to
6+# the installation's registry and installed back, pull requests from a branch
67 # and from a fork merged onto main, the merge queue taking a pull request
78 # and giving it back, and every cron handler the scheduler runs.
89 #
2122 # (In Git Bash on Windows, start the command with `env MSYS_NO_PATHCONV=1`
2223 # so /data is not rewritten into a Windows path.)
2324 # PACK_CACHE=off skips the check that a second clone is served from the
24−# clone pack cache.
25+# clone pack cache. PACKAGES=off skips publishing an npm package to the
26+# installation's registry and installing it back.
2527 #
26−# Needs curl, git and node (to read JSON).
28+# Needs curl, git and node (to read JSON), and npm for the package.
2729 set -euo pipefail
2830
2931 G1T_URL="${G1T_URL:-http://localhost:8787}"
3537 # off: this installation keeps no clone packs (no PACK_STORE), so a second
3638 # clone is not checked for a kept one.
3739 PACK_CACHE="${PACK_CACHE:-on}"
40+# off: skip publishing and installing an npm package (which needs npm).
41+PACKAGES="${PACKAGES:-on}"
3842 RUN="$(date +%s)"
3943 USER_NAME="smoke${RUN}"
4044 EMAIL="${USER_NAME}@example.com"
209213 [ "$code" = 200 ] && grep -q '"tools"' "$WORK/api" || fail "MCP tools/list: $code $(head -c 300 "$WORK/api")"
210214 echo "MCP lists its tools at $MCP_URL"
211215
216+if [ "$PACKAGES" != off ]; then
217+ step "publish an npm package and install it"
218+ # The tarball is kept in the packages store (S3), and read back from it.
219+ npmrc="@$WORKSPACE:registry=$G1T_URL/-/npm/
220+//${G1T_URL#*://}/-/npm/:_authToken=$TOKEN"
221+ mkdir -p "$WORK/pkg" "$WORK/app"
222+ printf '%s\n' "$npmrc" > "$WORK/pkg/.npmrc"
223+ printf '%s\n' "$npmrc" > "$WORK/app/.npmrc"
224+ printf '{"name":"@%s/%s","version":"1.0.0","repository":"%s/%s/%s","main":"index.js"}\n' \
225+ "$WORKSPACE" "$REPO" "$G1T_URL" "$WORKSPACE" "$REPO" > "$WORK/pkg/package.json"
226+ printf 'module.exports = "published by smoke.sh %s";\n' "$RUN" > "$WORK/pkg/index.js"
227+ (cd "$WORK/pkg" && npm publish --silent) || fail "npm publish"
228+ (cd "$WORK/app" && npm init -y >/dev/null && npm install --silent --no-audit --no-fund "@$WORKSPACE/$REPO@1.0.0") || fail "npm install"
229+ got="$(cd "$WORK/app" && node -p "require('@$WORKSPACE/$REPO')")"
230+ [ "$got" = "published by smoke.sh $RUN" ] || fail "the installed package says: $got"
231+ echo "installed @$WORKSPACE/$REPO@1.0.0"
232+fi
233+
212234 step "a pull request from a branch, merged"
213235 git -C "$WORK/clone" config credential.helper ""
214236 commit_file "$WORK/clone" "$remote" from-branch docs/branch.md
+56−33
1111 // cd services/repos && node ../../scripts/build-rust-worker.mjs
1212 // node dev/clone-check.mjs
1313 //
14−// With `--s3`, packs are kept in MinIO instead of local R2, as a
15−// self-hosted installation keeps them (PACK_STORE=s3): it starts MinIO in
16−// Docker, makes the `g1t-git-packs` bucket with the same expiry rule the
17−// compose file gives it, and checks that what was kept is there, whole,
18−// with no upload left unfinished.
14+// With `--s3`, packs are kept in RustFS instead of local R2, as a
15+// self-hosted installation keeps them (PACK_STORE=s3): it starts RustFS in
16+// Docker, makes the `g1t-git-packs` bucket with the same lifecycle rule the
17+// compose file gives it (with the AWS CLI, as the compose file does), and
18+// checks that what was kept is there, whole, with no upload left
19+// unfinished.
1920 //
2021 // node dev/clone-check.mjs --s3
2122 //
22−// GITSTORE_PORT, REPOS_PORT and MINIO_PORT move it off 8799, 8791 and 9010.
23+// GITSTORE_PORT, REPOS_PORT and S3_PORT move it off 8799, 8791 and 9010.
24+// RUSTFS_IMAGE and AWS_CLI_IMAGE choose other images.
2325 //
2426 // Needs node, git (with git-http-backend) and the repository's npm
2527 // packages; with `--s3`, Docker too.
5658 );
5759 const KEY = "acme--rocket";
5860 const children = [];
59−// --s3: packs in MinIO (see the top).
61+// --s3: packs in RustFS (see the top). The images are the compose file's.
6062 const S3 = process.argv.includes("--s3");
61−const MINIO = "g1t-clone-check-minio";
62−const MINIO_PORT = process.env.MINIO_PORT ?? "9010";
63−const MINIO_USER = "g1t";
64−const MINIO_PASSWORD = "g1t-clone-check-secret";
63+const STORAGE = "g1t-clone-check-rustfs";
64+const S3_PORT = process.env.S3_PORT ?? "9010";
65+const S3_USER = "g1t";
66+const S3_PASSWORD = "g1t-clone-check-secret";
67+const RUSTFS_IMAGE = process.env.RUSTFS_IMAGE ?? "rustfs/rustfs:1.0.1";
68+const AWS_CLI_IMAGE = process.env.AWS_CLI_IMAGE ?? "amazon/aws-cli:2.37.10";
6569 const PACKS_BUCKET = "g1t-git-packs";
66−/** `mc` inside the MinIO container, against itself. */
67−const mc = (args, options = {}) => run("docker", ["exec", MINIO, "mc", ...args], options);
70+/** The AWS CLI's `s3api`, in a container on the store's network, against it. */
71+const s3api = (args, options = {}) =>
72+ run("docker", [
73+ "run", "--rm", "--network", `container:${STORAGE}`,
74+ "-e", `AWS_ACCESS_KEY_ID=${S3_USER}`, "-e", `AWS_SECRET_ACCESS_KEY=${S3_PASSWORD}`, "-e", "AWS_DEFAULT_REGION=us-east-1",
75+ AWS_CLI_IMAGE, "--endpoint-url", "http://localhost:9000", "--output", "json", "s3api", ...args,
76+ ], options);
77+/** The same, its answer parsed. */
78+const s3json = (args) => JSON.parse(s3api(args).stdout.trim() || "{}");
6879 // Wrangler from the repository's packages, run with node: no shell to quote for.
6980 const WRANGLER = join(dirname(createRequire(join(service, "package.json")).resolve("wrangler/package.json")), "bin/wrangler.js");
7081
163174 });
164175 sql("INSERT INTO repos (id, namespace, name, is_private, owner_id, default_branch, refs_version) VALUES ('rep_rocket', 'acme', 'rocket', 0, 'usr_dev', 'main', 1)");
165176
166− // MinIO, with the bucket and expiry rule deploy/self-host/docker-compose.yml makes.
177+ // RustFS, with the bucket and lifecycle rule deploy/self-host/docker-compose.yml makes.
167178 const s3Vars = [];
168179 if (S3) {
169− run("docker", ["rm", "-f", MINIO], { allowFail: true });
180+ run("docker", ["rm", "-f", STORAGE], { allowFail: true });
170181 run("docker", [
171− "run", "-d", "--rm", "--name", MINIO, "-p", `${MINIO_PORT}:9000`,
172− "-e", `MINIO_ROOT_USER=${MINIO_USER}`, "-e", `MINIO_ROOT_PASSWORD=${MINIO_PASSWORD}`,
173− process.env.MINIO_IMAGE ?? "pgsty/minio:latest", "server", "/data",
182+ "run", "-d", "--rm", "--name", STORAGE, "-p", `${S3_PORT}:9000`,
183+ "-e", `RUSTFS_ACCESS_KEY=${S3_USER}`, "-e", `RUSTFS_SECRET_KEY=${S3_PASSWORD}`, "-e", "RUSTFS_CONSOLE_ENABLE=false",
184+ RUSTFS_IMAGE,
185+ ]);
186+ await waitFor(`http://localhost:${S3_PORT}/health/ready`, "RustFS");
187+ s3api(["create-bucket", "--bucket", PACKS_BUCKET]);
188+ s3api([
189+ "put-bucket-lifecycle-configuration", "--bucket", PACKS_BUCKET, "--lifecycle-configuration",
190+ JSON.stringify({
191+ Rules: [
192+ { ID: "expire-packs", Status: "Enabled", Filter: { Prefix: "packs/" }, Expiration: { Days: 7 } },
193+ { ID: "abort-unfinished-uploads", Status: "Enabled", Filter: { Prefix: "" }, AbortIncompleteMultipartUpload: { DaysAfterInitiation: 1 } },
194+ ],
195+ }),
174196 ]);
175− await waitFor(`http://localhost:${MINIO_PORT}/minio/health/ready`, "MinIO");
176− mc(["alias", "set", "local", "http://localhost:9000", MINIO_USER, MINIO_PASSWORD]);
177− mc(["mb", "--ignore-existing", `local/${PACKS_BUCKET}`]);
178− mc(["ilm", "rule", "add", "--prefix", "packs/", "--expire-days", "7", `local/${PACKS_BUCKET}`]);
179197 for (const [name, value] of Object.entries({
180198 PACK_STORE: "s3",
181199 PACK_S3_BUCKET: PACKS_BUCKET,
182− S3_ENDPOINT: `http://localhost:${MINIO_PORT}`,
200+ S3_ENDPOINT: `http://localhost:${S3_PORT}`,
183201 S3_REGION: "us-east-1",
184− S3_ACCESS_KEY_ID: MINIO_USER,
185− S3_SECRET_ACCESS_KEY: MINIO_PASSWORD,
202+ S3_ACCESS_KEY_ID: S3_USER,
203+ S3_SECRET_ACCESS_KEY: S3_PASSWORD,
186204 })) {
187205 s3Vars.push("--var", `${name}:${value}`);
188206 }
209227 if (S3) {
210228 // Fills finish after git has its answer: give the last one a moment.
211229 await new Promise((resolve) => setTimeout(resolve, 3000));
212− const listed = mc(["ls", "--recursive", "--json", `local/${PACKS_BUCKET}/packs/`]).stdout.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line));
213− const sizes = listed.map((entry) => entry.size);
230+ const listed = s3json(["list-objects-v2", "--bucket", PACKS_BUCKET, "--prefix", "packs/"]).Contents ?? [];
231+ const sizes = listed.map((entry) => entry.Size);
214232 // Nine clones: four kinds twice, each kept once, and one more after the refs moved.
215− check("the packs are in MinIO, one per distinct clone", listed.length === 5, `${listed.length}: ${sizes.join(", ")}`);
233+ check("the packs are in RustFS, one per distinct clone", listed.length === 5, `${listed.length}: ${sizes.join(", ")}`);
216234 check("the full clone's pack went up in parts", sizes.some((size) => size > 5 * 1024 * 1024), `largest ${Math.max(...sizes)}`);
217− const incomplete = mc(["ls", "--recursive", "--incomplete", `local/${PACKS_BUCKET}`]).stdout.trim();
218− check("no upload is left unfinished", incomplete === "", incomplete);
219− const rules = mc(["ilm", "rule", "ls", "--json", `local/${PACKS_BUCKET}`]).stdout;
220− check("the bucket expires packs after 7 days", /"Days":\s*7/.test(rules) && rules.includes("packs/"));
235+ // A multipart object's ETag ends in -<number of parts>.
236+ check("the large pack is one multipart object", listed.some((entry) => /-\d+"?$/.test(entry.ETag ?? "")), listed.map((entry) => entry.ETag).join(", "));
237+ const short = listed.filter((entry) => s3json(["head-object", "--bucket", PACKS_BUCKET, "--key", entry.Key]).ContentLength !== entry.Size);
238+ check("every pack reads back whole", short.length === 0, short.map((entry) => entry.Key).join(", "));
239+ const incomplete = s3json(["list-multipart-uploads", "--bucket", PACKS_BUCKET]).Uploads ?? [];
240+ check("no upload is left unfinished", incomplete.length === 0, incomplete.map((upload) => upload.Key).join(", "));
241+ const rules = s3json(["get-bucket-lifecycle-configuration", "--bucket", PACKS_BUCKET]).Rules ?? [];
242+ check("the bucket expires packs after 7 days", rules.some((rule) => rule.Expiration?.Days === 7 && rule.Filter?.Prefix === "packs/"), JSON.stringify(rules));
243+ check("the bucket aborts uploads unfinished after a day", rules.some((rule) => rule.AbortIncompleteMultipartUpload?.DaysAfterInitiation === 1));
221244 }
222245 } catch (error) {
223246 console.error(error);
227250 if (process.platform === "win32") spawnSync("taskkill", ["/pid", String(child.pid), "/t", "/f"], { stdio: "ignore" });
228251 else child.kill();
229252 }
230− if (S3) run("docker", ["rm", "-f", MINIO], { allowFail: true });
253+ if (S3) run("docker", ["rm", "-f", STORAGE], { allowFail: true });
231254 try {
232255 rmSync(work, { recursive: true, force: true });
233256 } catch {}