Self-hosting: RustFS for S3-compatible storage, buckets and the packs' lifecycle rule made with the AWS CLI
The compose file's object store is RustFS (rustfs/rustfs:1.0.1, pinned), on a new g1t-objects volume, health-checked at /health/ready. storage-setup (amazon/aws-cli:2.37.10) makes the packages, backups and clone pack buckets and puts the packs' bucket's lifecycle configuration: packs/ expires after 7 days, multipart uploads unfinished after a day are aborted. RustFS's scanner applies both, so no sweep of our own is needed. clone-check.mjs --s3 runs against RustFS with the AWS CLI on its network, and also checks the packs are multipart objects that read back whole and that both lifecycle rules are in place. smoke.sh publishes an npm package to the installation's registry and installs it back (PACKAGES=off skips).
4 files+117−740/4 viewed
| 9 | 9 | // - ARTIFACTS (git storage) becomes a service binding to workers/artifacts, | |
| 10 | 10 | // which keeps repositories in the git store (gitstore/server.mjs); | |
| 11 | 11 | // - EMAIL (Email Sending) becomes a service binding to workers/mail; | |
| 12 | − | // - the packages service keeps files in S3-compatible storage (MinIO) | |
| 12 | + | // - the packages service keeps files in S3-compatible storage (RustFS) | |
| 13 | 13 | // instead of R2, and the repos service its nightly backups (a bucket of | |
| 14 | 14 | // their own, BACKUP_S3_BUCKET); | |
| 15 | 15 | // - services that are off in this phase (agents, the context hub, the | |
| 216 | 216 | // Access requests are summarised to your own address, not g1t.sh's. | |
| 217 | 217 | config.vars.WAITLIST_NOTIFY_EMAIL = process.env.WAITLIST_NOTIFY_EMAIL ?? ""; | |
| 218 | 218 | } | |
| 219 | − | // Packages' files go to the compose file's MinIO (or any S3-compatible | |
| 219 | + | // Packages' files go to the compose file's RustFS (or any S3-compatible | |
| 220 | 220 | // store) instead of R2, with no request size limit, and package | |
| 221 | 221 | // addresses start with this installation's host. | |
| 222 | 222 | if (hosted.name === "g1t-packages") { | |
| 223 | 223 | Object.assign(config.vars, { | |
| 224 | 224 | BLOB_STORE: "s3", | |
| 225 | − | S3_ENDPOINT: process.env.S3_ENDPOINT ?? "http://minio:9000", | |
| 225 | + | S3_ENDPOINT: process.env.S3_ENDPOINT ?? "http://rustfs:9000", | |
| 226 | 226 | S3_BUCKET: process.env.S3_BUCKET ?? "g1t-packages", | |
| 227 | 227 | S3_REGION: process.env.S3_REGION ?? "us-east-1", | |
| 228 | 228 | S3_ACCESS_KEY_ID: process.env.S3_ACCESS_KEY_ID ?? "", | |
| 252 | 252 | BACKUP_S3_BUCKET: process.env.BACKUP_S3_BUCKET ?? "g1t-backups", | |
| 253 | 253 | PACK_STORE: "s3", | |
| 254 | 254 | PACK_S3_BUCKET: process.env.PACK_S3_BUCKET ?? "g1t-git-packs", | |
| 255 | − | S3_ENDPOINT: process.env.S3_ENDPOINT ?? "http://minio:9000", | |
| 255 | + | S3_ENDPOINT: process.env.S3_ENDPOINT ?? "http://rustfs:9000", | |
| 256 | 256 | S3_REGION: process.env.S3_REGION ?? "us-east-1", | |
| 257 | 257 | S3_ACCESS_KEY_ID: process.env.S3_ACCESS_KEY_ID ?? "", | |
| 258 | 258 | S3_SECRET_ACCESS_KEY: process.env.S3_SECRET_ACCESS_KEY ?? "", |
| 9 | 9 | # What runs: the site and every core service in one workerd (g1t), git | |
| 10 | 10 | # repositories as bare repos on a volume (gitstore), the API in a second | |
| 11 | 11 | # workerd beside it, packages' files, backups and the clone pack cache in | |
| 12 | − | # MinIO, and Mailpit for mail. | |
| 12 | + | # RustFS (S3-compatible storage), and Mailpit for mail. | |
| 13 | 13 | # Agents, deployments, context search and billing are off. See | |
| 14 | 14 | # docs/SELF_HOSTING.md. | |
| 15 | 15 | name: g1t | |
| 51 | 51 | INVITES_PER_USER: ${INVITES_PER_USER:-} | |
| 52 | 52 | # Where summaries of new access requests go; empty sends none. | |
| 53 | 53 | WAITLIST_NOTIFY_EMAIL: ${WAITLIST_NOTIFY_EMAIL:-} | |
| 54 | − | # Packages' files (container images and the rest), in MinIO below or | |
| 54 | + | # Packages' files (container images and the rest), in RustFS below or | |
| 55 | 55 | # any S3-compatible store. S3_PUBLIC_ENDPOINT, when clients can reach | |
| 56 | 56 | # the store, sends large downloads there directly. | |
| 57 | − | S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000} | |
| 57 | + | S3_ENDPOINT: ${S3_ENDPOINT:-http://rustfs:9000} | |
| 58 | 58 | S3_BUCKET: ${S3_BUCKET:-g1t-packages} | |
| 59 | 59 | S3_REGION: ${S3_REGION:-us-east-1} | |
| 60 | 60 | S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-g1t} | |
| 64 | 64 | # the same store (docs/SELF_HOSTING.md, "Backups"). | |
| 65 | 65 | BACKUP_S3_BUCKET: ${BACKUP_S3_BUCKET:-g1t-backups} | |
| 66 | 66 | # Packs kept for fresh clones, so the next clone of the same commit | |
| 67 | − | # does not rebuild one; minio-setup expires them after 7 days. | |
| 67 | + | # does not rebuild one; storage-setup expires them after 7 days. | |
| 68 | 68 | PACK_S3_BUCKET: ${PACK_S3_BUCKET:-g1t-git-packs} | |
| 69 | 69 | volumes: | |
| 70 | 70 | - g1t-data:/data | |
| 74 | 74 | condition: service_healthy | |
| 75 | 75 | mailpit: | |
| 76 | 76 | condition: service_started | |
| 77 | − | minio-setup: | |
| 77 | + | storage-setup: | |
| 78 | 78 | condition: service_completed_successfully | |
| 79 | 79 | restart: unless-stopped | |
| 80 | 80 | ||
| 110 | 110 | # Not published: only the g1t container reaches it. | |
| 111 | 111 | restart: unless-stopped | |
| 112 | 112 | ||
| 113 | − | # Packages' files, backups and clone packs. Not published: only the g1t | |
| 114 | − | # container reaches it, unless you publish 9000 and set | |
| 115 | − | # S3_PUBLIC_ENDPOINT. MinIO no longer publishes images of its own; | |
| 116 | − | # MINIO_IMAGE is a community build of the same server, with `mc` in it. | |
| 117 | − | # Any S3-compatible store works in its place (S3_ENDPOINT). | |
| 118 | − | minio: | |
| 119 | − | image: ${MINIO_IMAGE:-pgsty/minio:latest} | |
| 120 | − | command: ["server", "/data"] | |
| 113 | + | # Packages' files, backups and clone packs, in RustFS (S3-compatible). | |
| 114 | + | # Not published: only the g1t container reaches it, unless you publish | |
| 115 | + | # 9000 and set S3_PUBLIC_ENDPOINT. Any S3-compatible store works in its | |
| 116 | + | # place (S3_ENDPOINT). | |
| 117 | + | rustfs: | |
| 118 | + | image: ${RUSTFS_IMAGE:-rustfs/rustfs:1.0.1} | |
| 121 | 119 | environment: | |
| 122 | − | MINIO_ROOT_USER: ${S3_ACCESS_KEY_ID:-g1t} | |
| 123 | − | MINIO_ROOT_PASSWORD: ${S3_SECRET_ACCESS_KEY:-g1t-packages-secret} | |
| 124 | − | # What the health check's `mc ready local` asks. | |
| 125 | − | MC_HOST_local: http://localhost:9000 | |
| 120 | + | RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY_ID:-g1t} | |
| 121 | + | RUSTFS_SECRET_KEY: ${S3_SECRET_ACCESS_KEY:-g1t-packages-secret} | |
| 122 | + | RUSTFS_CONSOLE_ENABLE: "false" | |
| 126 | 123 | volumes: | |
| 127 | − | - g1t-packages:/data | |
| 124 | + | - g1t-objects:/data | |
| 128 | 125 | healthcheck: | |
| 129 | − | test: ["CMD", "mc", "ready", "local"] | |
| 126 | + | test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://localhost:9000/health/ready"] | |
| 130 | 127 | interval: 5s | |
| 131 | 128 | retries: 20 | |
| 132 | 129 | restart: unless-stopped | |
| 133 | 130 | ||
| 134 | 131 | # Makes the buckets once, then exits: packages' files, backups, and | |
| 135 | − | # clone packs with a rule that deletes packs 7 days old. (MinIO itself | |
| 136 | − | # removes uploads left unfinished after 24 hours.) | |
| 137 | − | minio-setup: | |
| 138 | − | image: ${MINIO_IMAGE:-pgsty/minio:latest} | |
| 132 | + | # clone packs, with a lifecycle rule on the packs' bucket that deletes | |
| 133 | + | # packs 7 days old and uploads left unfinished after a day. | |
| 134 | + | storage-setup: | |
| 135 | + | image: ${AWS_CLI_IMAGE:-amazon/aws-cli:2.37.10} | |
| 139 | 136 | depends_on: | |
| 140 | − | minio: | |
| 137 | + | rustfs: | |
| 141 | 138 | condition: service_healthy | |
| 142 | 139 | entrypoint: | |
| 143 | 140 | - sh | |
| 144 | 141 | - -c | |
| 145 | 142 | - >- | |
| 146 | 143 | set -e; | |
| 147 | − | mc alias set local http://minio:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"; | |
| 148 | − | mc mb --ignore-existing "local/$$S3_BUCKET"; | |
| 149 | − | mc mb --ignore-existing "local/$$BACKUP_S3_BUCKET"; | |
| 150 | − | mc mb --ignore-existing "local/$$PACK_S3_BUCKET"; | |
| 151 | − | if ! mc ilm rule ls "local/$$PACK_S3_BUCKET" >/dev/null 2>&1; then | |
| 152 | − | mc ilm rule add --prefix packs/ --expire-days 7 "local/$$PACK_S3_BUCKET"; | |
| 153 | − | fi | |
| 144 | + | for bucket in "$$S3_BUCKET" "$$BACKUP_S3_BUCKET" "$$PACK_S3_BUCKET"; do | |
| 145 | + | aws s3api head-bucket --bucket "$$bucket" >/dev/null 2>&1 || aws s3api create-bucket --bucket "$$bucket" >/dev/null; | |
| 146 | + | done; | |
| 147 | + | aws s3api put-bucket-lifecycle-configuration --bucket "$$PACK_S3_BUCKET" --lifecycle-configuration | |
| 148 | + | '{"Rules":[{"ID":"expire-packs","Status":"Enabled","Filter":{"Prefix":"packs/"},"Expiration":{"Days":7}},{"ID":"abort-unfinished-uploads","Status":"Enabled","Filter":{"Prefix":""},"AbortIncompleteMultipartUpload":{"DaysAfterInitiation":1}}]}'; | |
| 149 | + | echo "buckets ready" | |
| 154 | 150 | environment: | |
| 155 | − | MINIO_ROOT_USER: ${S3_ACCESS_KEY_ID:-g1t} | |
| 156 | − | MINIO_ROOT_PASSWORD: ${S3_SECRET_ACCESS_KEY:-g1t-packages-secret} | |
| 151 | + | AWS_ENDPOINT_URL: http://rustfs:9000 | |
| 152 | + | AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-g1t} | |
| 153 | + | AWS_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-g1t-packages-secret} | |
| 154 | + | AWS_DEFAULT_REGION: ${S3_REGION:-us-east-1} | |
| 157 | 155 | S3_BUCKET: ${S3_BUCKET:-g1t-packages} | |
| 158 | 156 | BACKUP_S3_BUCKET: ${BACKUP_S3_BUCKET:-g1t-backups} | |
| 159 | 157 | PACK_S3_BUCKET: ${PACK_S3_BUCKET:-g1t-git-packs} | |
| 173 | 171 | ||
| 174 | 172 | volumes: | |
| 175 | 173 | g1t-data: | |
| 176 | − | g1t-packages: | |
| 174 | + | g1t-objects: | |
| 177 | 175 | g1t-status: | |
| 178 | 176 | g1t-git: | |
| 179 | 177 | g1t-secrets: |
| 2 | 2 | # End-to-end check of a self-hosted g1t: sign up, confirm the email, make a | |
| 3 | 3 | # workspace and a repository, push and clone over HTTP, open an issue, and | |
| 4 | 4 | # read the code back through the site. Then the API on its own port (REST, | |
| 5 | − | # OAuth metadata and MCP, with an access token), pull requests from a branch | |
| 5 | + | # OAuth metadata and MCP, with an access token), an npm package published to | |
| 6 | + | # the installation's registry and installed back, pull requests from a branch | |
| 6 | 7 | # and from a fork merged onto main, the merge queue taking a pull request | |
| 7 | 8 | # and giving it back, and every cron handler the scheduler runs. | |
| 8 | 9 | # | |
| 21 | 22 | # (In Git Bash on Windows, start the command with `env MSYS_NO_PATHCONV=1` | |
| 22 | 23 | # so /data is not rewritten into a Windows path.) | |
| 23 | 24 | # PACK_CACHE=off skips the check that a second clone is served from the | |
| 24 | − | # clone pack cache. | |
| 25 | + | # clone pack cache. PACKAGES=off skips publishing an npm package to the | |
| 26 | + | # installation's registry and installing it back. | |
| 25 | 27 | # | |
| 26 | − | # Needs curl, git and node (to read JSON). | |
| 28 | + | # Needs curl, git and node (to read JSON), and npm for the package. | |
| 27 | 29 | set -euo pipefail | |
| 28 | 30 | ||
| 29 | 31 | G1T_URL="${G1T_URL:-http://localhost:8787}" | |
| 35 | 37 | # off: this installation keeps no clone packs (no PACK_STORE), so a second | |
| 36 | 38 | # clone is not checked for a kept one. | |
| 37 | 39 | PACK_CACHE="${PACK_CACHE:-on}" | |
| 40 | + | # off: skip publishing and installing an npm package (which needs npm). | |
| 41 | + | PACKAGES="${PACKAGES:-on}" | |
| 38 | 42 | RUN="$(date +%s)" | |
| 39 | 43 | USER_NAME="smoke${RUN}" | |
| 40 | 44 | EMAIL="${USER_NAME}@example.com" | |
| 209 | 213 | [ "$code" = 200 ] && grep -q '"tools"' "$WORK/api" || fail "MCP tools/list: $code $(head -c 300 "$WORK/api")" | |
| 210 | 214 | echo "MCP lists its tools at $MCP_URL" | |
| 211 | 215 | ||
| 216 | + | if [ "$PACKAGES" != off ]; then | |
| 217 | + | step "publish an npm package and install it" | |
| 218 | + | # The tarball is kept in the packages store (S3), and read back from it. | |
| 219 | + | npmrc="@$WORKSPACE:registry=$G1T_URL/-/npm/ | |
| 220 | + | //${G1T_URL#*://}/-/npm/:_authToken=$TOKEN" | |
| 221 | + | mkdir -p "$WORK/pkg" "$WORK/app" | |
| 222 | + | printf '%s\n' "$npmrc" > "$WORK/pkg/.npmrc" | |
| 223 | + | printf '%s\n' "$npmrc" > "$WORK/app/.npmrc" | |
| 224 | + | printf '{"name":"@%s/%s","version":"1.0.0","repository":"%s/%s/%s","main":"index.js"}\n' \ | |
| 225 | + | "$WORKSPACE" "$REPO" "$G1T_URL" "$WORKSPACE" "$REPO" > "$WORK/pkg/package.json" | |
| 226 | + | printf 'module.exports = "published by smoke.sh %s";\n' "$RUN" > "$WORK/pkg/index.js" | |
| 227 | + | (cd "$WORK/pkg" && npm publish --silent) || fail "npm publish" | |
| 228 | + | (cd "$WORK/app" && npm init -y >/dev/null && npm install --silent --no-audit --no-fund "@$WORKSPACE/$REPO@1.0.0") || fail "npm install" | |
| 229 | + | got="$(cd "$WORK/app" && node -p "require('@$WORKSPACE/$REPO')")" | |
| 230 | + | [ "$got" = "published by smoke.sh $RUN" ] || fail "the installed package says: $got" | |
| 231 | + | echo "installed @$WORKSPACE/$REPO@1.0.0" | |
| 232 | + | fi | |
| 233 | + | ||
| 212 | 234 | step "a pull request from a branch, merged" | |
| 213 | 235 | git -C "$WORK/clone" config credential.helper "" | |
| 214 | 236 | commit_file "$WORK/clone" "$remote" from-branch docs/branch.md |
| 11 | 11 | // cd services/repos && node ../../scripts/build-rust-worker.mjs | |
| 12 | 12 | // node dev/clone-check.mjs | |
| 13 | 13 | // | |
| 14 | − | // With `--s3`, packs are kept in MinIO instead of local R2, as a | |
| 15 | − | // self-hosted installation keeps them (PACK_STORE=s3): it starts MinIO in | |
| 16 | − | // Docker, makes the `g1t-git-packs` bucket with the same expiry rule the | |
| 17 | − | // compose file gives it, and checks that what was kept is there, whole, | |
| 18 | − | // with no upload left unfinished. | |
| 14 | + | // With `--s3`, packs are kept in RustFS instead of local R2, as a | |
| 15 | + | // self-hosted installation keeps them (PACK_STORE=s3): it starts RustFS in | |
| 16 | + | // Docker, makes the `g1t-git-packs` bucket with the same lifecycle rule the | |
| 17 | + | // compose file gives it (with the AWS CLI, as the compose file does), and | |
| 18 | + | // checks that what was kept is there, whole, with no upload left | |
| 19 | + | // unfinished. | |
| 19 | 20 | // | |
| 20 | 21 | // node dev/clone-check.mjs --s3 | |
| 21 | 22 | // | |
| 22 | − | // GITSTORE_PORT, REPOS_PORT and MINIO_PORT move it off 8799, 8791 and 9010. | |
| 23 | + | // GITSTORE_PORT, REPOS_PORT and S3_PORT move it off 8799, 8791 and 9010. | |
| 24 | + | // RUSTFS_IMAGE and AWS_CLI_IMAGE choose other images. | |
| 23 | 25 | // | |
| 24 | 26 | // Needs node, git (with git-http-backend) and the repository's npm | |
| 25 | 27 | // packages; with `--s3`, Docker too. | |
| 56 | 58 | ); | |
| 57 | 59 | const KEY = "acme--rocket"; | |
| 58 | 60 | const children = []; | |
| 59 | − | // --s3: packs in MinIO (see the top). | |
| 61 | + | // --s3: packs in RustFS (see the top). The images are the compose file's. | |
| 60 | 62 | const S3 = process.argv.includes("--s3"); | |
| 61 | − | const MINIO = "g1t-clone-check-minio"; | |
| 62 | − | const MINIO_PORT = process.env.MINIO_PORT ?? "9010"; | |
| 63 | − | const MINIO_USER = "g1t"; | |
| 64 | − | const MINIO_PASSWORD = "g1t-clone-check-secret"; | |
| 63 | + | const STORAGE = "g1t-clone-check-rustfs"; | |
| 64 | + | const S3_PORT = process.env.S3_PORT ?? "9010"; | |
| 65 | + | const S3_USER = "g1t"; | |
| 66 | + | const S3_PASSWORD = "g1t-clone-check-secret"; | |
| 67 | + | const RUSTFS_IMAGE = process.env.RUSTFS_IMAGE ?? "rustfs/rustfs:1.0.1"; | |
| 68 | + | const AWS_CLI_IMAGE = process.env.AWS_CLI_IMAGE ?? "amazon/aws-cli:2.37.10"; | |
| 65 | 69 | const PACKS_BUCKET = "g1t-git-packs"; | |
| 66 | − | /** `mc` inside the MinIO container, against itself. */ | |
| 67 | − | const mc = (args, options = {}) => run("docker", ["exec", MINIO, "mc", ...args], options); | |
| 70 | + | /** The AWS CLI's `s3api`, in a container on the store's network, against it. */ | |
| 71 | + | const s3api = (args, options = {}) => | |
| 72 | + | run("docker", [ | |
| 73 | + | "run", "--rm", "--network", `container:${STORAGE}`, | |
| 74 | + | "-e", `AWS_ACCESS_KEY_ID=${S3_USER}`, "-e", `AWS_SECRET_ACCESS_KEY=${S3_PASSWORD}`, "-e", "AWS_DEFAULT_REGION=us-east-1", | |
| 75 | + | AWS_CLI_IMAGE, "--endpoint-url", "http://localhost:9000", "--output", "json", "s3api", ...args, | |
| 76 | + | ], options); | |
| 77 | + | /** The same, its answer parsed. */ | |
| 78 | + | const s3json = (args) => JSON.parse(s3api(args).stdout.trim() || "{}"); | |
| 68 | 79 | // Wrangler from the repository's packages, run with node: no shell to quote for. | |
| 69 | 80 | const WRANGLER = join(dirname(createRequire(join(service, "package.json")).resolve("wrangler/package.json")), "bin/wrangler.js"); | |
| 70 | 81 | ||
| 163 | 174 | }); | |
| 164 | 175 | sql("INSERT INTO repos (id, namespace, name, is_private, owner_id, default_branch, refs_version) VALUES ('rep_rocket', 'acme', 'rocket', 0, 'usr_dev', 'main', 1)"); | |
| 165 | 176 | ||
| 166 | − | // MinIO, with the bucket and expiry rule deploy/self-host/docker-compose.yml makes. | |
| 177 | + | // RustFS, with the bucket and lifecycle rule deploy/self-host/docker-compose.yml makes. | |
| 167 | 178 | const s3Vars = []; | |
| 168 | 179 | if (S3) { | |
| 169 | − | run("docker", ["rm", "-f", MINIO], { allowFail: true }); | |
| 180 | + | run("docker", ["rm", "-f", STORAGE], { allowFail: true }); | |
| 170 | 181 | run("docker", [ | |
| 171 | − | "run", "-d", "--rm", "--name", MINIO, "-p", `${MINIO_PORT}:9000`, | |
| 172 | − | "-e", `MINIO_ROOT_USER=${MINIO_USER}`, "-e", `MINIO_ROOT_PASSWORD=${MINIO_PASSWORD}`, | |
| 173 | − | process.env.MINIO_IMAGE ?? "pgsty/minio:latest", "server", "/data", | |
| 182 | + | "run", "-d", "--rm", "--name", STORAGE, "-p", `${S3_PORT}:9000`, | |
| 183 | + | "-e", `RUSTFS_ACCESS_KEY=${S3_USER}`, "-e", `RUSTFS_SECRET_KEY=${S3_PASSWORD}`, "-e", "RUSTFS_CONSOLE_ENABLE=false", | |
| 184 | + | RUSTFS_IMAGE, | |
| 185 | + | ]); | |
| 186 | + | await waitFor(`http://localhost:${S3_PORT}/health/ready`, "RustFS"); | |
| 187 | + | s3api(["create-bucket", "--bucket", PACKS_BUCKET]); | |
| 188 | + | s3api([ | |
| 189 | + | "put-bucket-lifecycle-configuration", "--bucket", PACKS_BUCKET, "--lifecycle-configuration", | |
| 190 | + | JSON.stringify({ | |
| 191 | + | Rules: [ | |
| 192 | + | { ID: "expire-packs", Status: "Enabled", Filter: { Prefix: "packs/" }, Expiration: { Days: 7 } }, | |
| 193 | + | { ID: "abort-unfinished-uploads", Status: "Enabled", Filter: { Prefix: "" }, AbortIncompleteMultipartUpload: { DaysAfterInitiation: 1 } }, | |
| 194 | + | ], | |
| 195 | + | }), | |
| 174 | 196 | ]); | |
| 175 | − | await waitFor(`http://localhost:${MINIO_PORT}/minio/health/ready`, "MinIO"); | |
| 176 | − | mc(["alias", "set", "local", "http://localhost:9000", MINIO_USER, MINIO_PASSWORD]); | |
| 177 | − | mc(["mb", "--ignore-existing", `local/${PACKS_BUCKET}`]); | |
| 178 | − | mc(["ilm", "rule", "add", "--prefix", "packs/", "--expire-days", "7", `local/${PACKS_BUCKET}`]); | |
| 179 | 197 | for (const [name, value] of Object.entries({ | |
| 180 | 198 | PACK_STORE: "s3", | |
| 181 | 199 | PACK_S3_BUCKET: PACKS_BUCKET, | |
| 182 | − | S3_ENDPOINT: `http://localhost:${MINIO_PORT}`, | |
| 200 | + | S3_ENDPOINT: `http://localhost:${S3_PORT}`, | |
| 183 | 201 | S3_REGION: "us-east-1", | |
| 184 | − | S3_ACCESS_KEY_ID: MINIO_USER, | |
| 185 | − | S3_SECRET_ACCESS_KEY: MINIO_PASSWORD, | |
| 202 | + | S3_ACCESS_KEY_ID: S3_USER, | |
| 203 | + | S3_SECRET_ACCESS_KEY: S3_PASSWORD, | |
| 186 | 204 | })) { | |
| 187 | 205 | s3Vars.push("--var", `${name}:${value}`); | |
| 188 | 206 | } | |
| 209 | 227 | if (S3) { | |
| 210 | 228 | // Fills finish after git has its answer: give the last one a moment. | |
| 211 | 229 | await new Promise((resolve) => setTimeout(resolve, 3000)); | |
| 212 | − | const listed = mc(["ls", "--recursive", "--json", `local/${PACKS_BUCKET}/packs/`]).stdout.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)); | |
| 213 | − | const sizes = listed.map((entry) => entry.size); | |
| 230 | + | const listed = s3json(["list-objects-v2", "--bucket", PACKS_BUCKET, "--prefix", "packs/"]).Contents ?? []; | |
| 231 | + | const sizes = listed.map((entry) => entry.Size); | |
| 214 | 232 | // Nine clones: four kinds twice, each kept once, and one more after the refs moved. | |
| 215 | − | check("the packs are in MinIO, one per distinct clone", listed.length === 5, `${listed.length}: ${sizes.join(", ")}`); | |
| 233 | + | check("the packs are in RustFS, one per distinct clone", listed.length === 5, `${listed.length}: ${sizes.join(", ")}`); | |
| 216 | 234 | check("the full clone's pack went up in parts", sizes.some((size) => size > 5 * 1024 * 1024), `largest ${Math.max(...sizes)}`); | |
| 217 | − | const incomplete = mc(["ls", "--recursive", "--incomplete", `local/${PACKS_BUCKET}`]).stdout.trim(); | |
| 218 | − | check("no upload is left unfinished", incomplete === "", incomplete); | |
| 219 | − | const rules = mc(["ilm", "rule", "ls", "--json", `local/${PACKS_BUCKET}`]).stdout; | |
| 220 | − | check("the bucket expires packs after 7 days", /"Days":\s*7/.test(rules) && rules.includes("packs/")); | |
| 235 | + | // A multipart object's ETag ends in -<number of parts>. | |
| 236 | + | check("the large pack is one multipart object", listed.some((entry) => /-\d+"?$/.test(entry.ETag ?? "")), listed.map((entry) => entry.ETag).join(", ")); | |
| 237 | + | const short = listed.filter((entry) => s3json(["head-object", "--bucket", PACKS_BUCKET, "--key", entry.Key]).ContentLength !== entry.Size); | |
| 238 | + | check("every pack reads back whole", short.length === 0, short.map((entry) => entry.Key).join(", ")); | |
| 239 | + | const incomplete = s3json(["list-multipart-uploads", "--bucket", PACKS_BUCKET]).Uploads ?? []; | |
| 240 | + | check("no upload is left unfinished", incomplete.length === 0, incomplete.map((upload) => upload.Key).join(", ")); | |
| 241 | + | const rules = s3json(["get-bucket-lifecycle-configuration", "--bucket", PACKS_BUCKET]).Rules ?? []; | |
| 242 | + | check("the bucket expires packs after 7 days", rules.some((rule) => rule.Expiration?.Days === 7 && rule.Filter?.Prefix === "packs/"), JSON.stringify(rules)); | |
| 243 | + | check("the bucket aborts uploads unfinished after a day", rules.some((rule) => rule.AbortIncompleteMultipartUpload?.DaysAfterInitiation === 1)); | |
| 221 | 244 | } | |
| 222 | 245 | } catch (error) { | |
| 223 | 246 | console.error(error); | |
| 227 | 250 | if (process.platform === "win32") spawnSync("taskkill", ["/pid", String(child.pid), "/t", "/f"], { stdio: "ignore" }); | |
| 228 | 251 | else child.kill(); | |
| 229 | 252 | } | |
| 230 | − | if (S3) run("docker", ["rm", "-f", MINIO], { allowFail: true }); | |
| 253 | + | if (S3) run("docker", ["rm", "-f", STORAGE], { allowFail: true }); | |
| 231 | 254 | try { | |
| 232 | 255 | rmSync(work, { recursive: true, force: true }); | |
| 233 | 256 | } catch {} |