Skip to content
Sign up

flagon-io/g1t

public
0
250 linesCodeBlameRaw
1/**
2 * Datasets: the safe query layer behind dashboards in Artifacts. Mirrors
3 * `crates/contracts/src/datasets.rs`; a Rust test keeps the catalog the
4 * same and runs both validators over `datasets.fixtures.json`.
5 *
6 * Not SQL. A query names a dataset from a declared catalog, one measure,
7 * at most one dimension to group by, a time interval, filters on declared
8 * fields and a range. The service that owns the data (`service` in the
9 * catalog) runs it for the viewer, over only what the viewer can read, with
10 * a fixed query per measure and dimension, and caps the rows.
11 *
12 * Values never live in a folio: not in its Yjs document, versions, text,
13 * preview, index or templates. Only the query does.
14 *
15 * Wire shapes are snake_case.
16 */
17
18export type DatasetId = "issues" | "pull_requests" | "workflow_runs" | "deployments" | "spend" | "agent_sessions";
19
20export const DATASET_IDS: readonly DatasetId[] = ["issues", "pull_requests", "workflow_runs", "deployments", "spend", "agent_sessions"];
21
22/** How rows are summed up. `count` takes no field; `rate` takes a rate field; the rest a measure field. */
23export type DatasetMeasureOp = "count" | "sum" | "avg" | "p50" | "p95" | "rate";
24export const DATASET_MEASURE_OPS: readonly DatasetMeasureOp[] = ["count", "sum", "avg", "p50", "p95", "rate"];
25
26export type DatasetInterval = "day" | "week" | "month";
27export const DATASET_INTERVALS: readonly DatasetInterval[] = ["day", "week", "month"];
28
29export type DatasetFilterOp = "eq" | "neq" | "in" | "gte" | "lte";
30export const DATASET_FILTER_OPS: readonly DatasetFilterOp[] = ["eq", "neq", "in", "gte", "lte"];
31
32export type DatasetRangePreset = "7d" | "30d" | "90d";
33export const DATASET_RANGE_PRESETS: readonly DatasetRangePreset[] = ["7d", "30d", "90d"];
34
35/**
36 * A time range: a preset counted back from now, or between two times.
37 * `from` and `to` are dates (`2026-10-01`) or RFC 3339 UTC times
38 * (`2026-10-01T09:00:00Z`); `to` is exclusive.
39 */
40export type DatasetRange = DatasetRangePreset | { from: string; to: string };
41
42export type DatasetFilter = {
43 field: string;
44 op: DatasetFilterOp;
45 /** Text for `eq`/`neq` on a dimension, a list for `in`, a number on a measure. */
46 value: string | number | string[];
47};
48
49export type DatasetQuery = {
50 dataset: DatasetId;
51 measure: { op: DatasetMeasureOp; field?: string | null };
52 /** A declared dimension only. */
53 group_by?: string | null;
54 /** A time series, over the dataset's time field. */
55 interval?: DatasetInterval | null;
56 /** Which of the dataset's time fields the range and interval use; its first when left out. */
57 time?: string | null;
58 filters?: DatasetFilter[] | null;
59 /** The tile's own range; else the dashboard's. */
60 range?: DatasetRange | null;
61 /** At most `DATASET_MAX_ROWS`. */
62 limit?: number | null;
63};
64
65export type DatasetColumnType = "string" | "number" | "time" | "money";
66
67export type DatasetResult = {
68 columns: { name: string; type: DatasetColumnType }[];
69 rows: (string | number | null)[][];
70 /** More rows matched than were returned. */
71 truncated: boolean;
72 /** The viewer cannot read everything the query covers: shown as "Based on what you can see". */
73 partial: boolean;
74 /** When the numbers were computed, RFC 3339. */
75 as_of: string;
76};
77
78/** The services that own datasets, each answering `query_dataset` for its own. */
79export type DatasetService = "work" | "actions" | "deployments" | "billing" | "agents";
80
81/** What a viewer needs to query a dataset: membership, or the workspace's billing role. */
82export type DatasetNeeds = "member" | "billing";
83
84export type DatasetSpec = {
85 label: string;
86 service: DatasetService;
87 needs: DatasetNeeds;
88 /** Time fields, the default first. */
89 times: string[];
90 /** Fields to group and filter by (text). */
91 dimensions: string[];
92 /** Number fields for sum, avg, p50, p95, and number filters. */
93 measures: string[];
94 /** Yes-or-no fields `rate` gives the share of. */
95 rates: string[];
96};
97
98/** The catalog. One dataset per line: the Rust mirror test reads it that way. */
99export const DATASETS: Record<DatasetId, DatasetSpec> = {
100 issues: { label: "Issues", service: "work", needs: "member", times: ["created_at", "closed_at"], dimensions: ["repo", "label", "state", "author_kind", "assignee_kind", "milestone"], measures: ["time_to_close_hours", "comments"], rates: ["closed"] },
101 pull_requests: { label: "Pull requests", service: "work", needs: "member", times: ["created_at", "merged_at", "closed_at"], dimensions: ["repo", "label", "state", "author_kind", "base_branch"], measures: ["cycle_time_hours", "time_to_first_review_hours", "review_count", "additions", "deletions", "changed_files"], rates: ["merged"] },
102 workflow_runs: { label: "Workflow runs", service: "actions", needs: "member", times: ["started_at", "completed_at"], dimensions: ["repo", "workflow", "branch", "event", "conclusion", "runner_kind"], measures: ["duration_seconds", "queue_seconds"], rates: ["succeeded"] },
103 deployments: { label: "Deployments", service: "deployments", needs: "member", times: ["created_at"], dimensions: ["repo", "project", "environment", "state"], measures: ["duration_seconds", "time_to_restore_hours"], rates: ["failed"] },
104 spend: { label: "Spend", service: "billing", needs: "billing", times: ["day"], dimensions: ["product", "project", "person", "model"], measures: ["amount_micros"], rates: [] },
105 agent_sessions: { label: "Agent sessions", service: "agents", needs: "member", times: ["started_at"], dimensions: ["agent", "repo", "outcome", "model", "trigger"], measures: ["duration_seconds", "cost_micros", "tokens"], rates: ["succeeded"] },
106};
107
108/** The most rows a query returns. */
109export const DATASET_MAX_ROWS = 100;
110/** The most filters on one query. */
111export const DATASET_MAX_FILTERS = 10;
112/** The most values in an `in` filter. */
113export const DATASET_MAX_IN_VALUES = 50;
114/** The longest `{ from, to }` range, in days. */
115export const DATASET_MAX_RANGE_DAYS = 366;
116
117const DAY_MS = 86_400_000;
118const TIME = /^(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,3}))?Z)?$/;
119
120/** A range end as milliseconds since the epoch, or null when it is not a real date or RFC 3339 UTC time. */
121export function datasetTime(text: string): number | null {
122 const match = TIME.exec(text);
123 if (!match) return null;
124 const [year, month, day] = [Number(match[1]), Number(match[2]), Number(match[3])];
125 const [hour, minute, second] = [Number(match[4] ?? 0), Number(match[5] ?? 0), Number(match[6] ?? 0)];
126 const millis = Number((match[7] ?? "").padEnd(3, "0") || 0);
127 if (month < 1 || month > 12 || day < 1 || day > daysInMonth(year, month)) return null;
128 if (hour > 23 || minute > 59 || second > 59) return null;
129 return Date.UTC(year, month - 1, day, hour, minute, second, millis);
130}
131
132function daysInMonth(year: number, month: number): number {
133 if (month === 2) return (year % 4 === 0 && year % 100 !== 0) || year % 400 === 0 ? 29 : 28;
134 return [4, 6, 9, 11].includes(month) ? 30 : 31;
135}
136
137/**
138 * What is wrong with a query, or null. The same rules, and the same
139 * words, as `DatasetQuery::validate` in Rust. It checks the query against
140 * the catalog only; who may run it is the owning service's to decide.
141 */
142export function datasetQueryError(query: DatasetQuery): string | null {
143 const spec = DATASETS[query.dataset];
144 if (!spec) return `There is no dataset called ${query.dataset}.`;
145 const name = query.dataset;
146 const { op } = query.measure;
147 const field = query.measure.field ?? null;
148 if (op === "count") {
149 if (field !== null) return "count takes no field.";
150 } else if (op === "rate") {
151 if (field === null) return "rate needs a field.";
152 if (!spec.rates.includes(field)) return `${name} has no yes-or-no field ${field} to take the rate of.`;
153 } else {
154 if (field === null) return `${op} needs a field.`;
155 if (!spec.measures.includes(field)) return `${name} has no number field ${field}.`;
156 }
157 const groupBy = query.group_by ?? null;
158 if (groupBy !== null && !spec.dimensions.includes(groupBy)) return `${name} can't be grouped by ${groupBy}.`;
159 const time = query.time ?? null;
160 if (time !== null && !spec.times.includes(time)) return `${name} has no time field ${time}.`;
161 const filters = query.filters ?? [];
162 if (filters.length > DATASET_MAX_FILTERS) return `A query takes at most ${DATASET_MAX_FILTERS} filters.`;
163 for (const filter of filters) {
164 if (spec.dimensions.includes(filter.field)) {
165 if (filter.op === "in") {
166 if (!Array.isArray(filter.value) || filter.value.length === 0 || filter.value.length > DATASET_MAX_IN_VALUES) {
167 return `in on ${filter.field} takes a list of 1 to ${DATASET_MAX_IN_VALUES} values.`;
168 }
169 } else if (filter.op === "eq" || filter.op === "neq") {
170 if (typeof filter.value !== "string") return `${filter.op} on ${filter.field} takes text.`;
171 } else {
172 return `${filter.field} is text: filter it with eq, neq or in.`;
173 }
174 } else if (spec.measures.includes(filter.field)) {
175 if (filter.op === "in") return `${filter.field} is a number: filter it with eq, neq, gte or lte.`;
176 if (typeof filter.value !== "number" || !Number.isFinite(filter.value)) return `${filter.op} on ${filter.field} takes a number.`;
177 } else {
178 return `${name} can't be filtered by ${filter.field}.`;
179 }
180 }
181 const range = query.range ?? null;
182 if (range !== null && typeof range === "object") {
183 const from = datasetTime(range.from);
184 const to = datasetTime(range.to);
185 if (from === null || to === null) return "A range's from and to are dates or RFC 3339 UTC times.";
186 if (from >= to) return "A range's from comes before its to.";
187 if (to - from > DATASET_MAX_RANGE_DAYS * DAY_MS) return `A range is at most ${DATASET_MAX_RANGE_DAYS} days.`;
188 }
189 const limit = query.limit ?? null;
190 if (limit !== null && (!Number.isInteger(limit) || limit < 1 || limit > DATASET_MAX_ROWS)) return `limit is between 1 and ${DATASET_MAX_ROWS}.`;
191 return null;
192}
193
194/** A query from untrusted JSON: well-formed and valid, or why not. Unknown keys are dropped. */
195export function parseDatasetQuery(input: unknown): { query: DatasetQuery } | { error: string } {
196 if (!isObject(input)) return { error: "A query is an object." };
197 if (typeof input.dataset !== "string" || !(DATASET_IDS as readonly string[]).includes(input.dataset)) {
198 return { error: `There is no dataset called ${String(input.dataset)}.` };
199 }
200 const measure = input.measure;
201 if (!isObject(measure) || typeof measure.op !== "string" || !(DATASET_MEASURE_OPS as readonly string[]).includes(measure.op)) {
202 return { error: "measure.op is count, sum, avg, p50, p95 or rate." };
203 }
204 if (!optional(measure.field, "string")) return { error: "measure.field is text." };
205 if (!optional(input.group_by, "string")) return { error: "group_by is text." };
206 if (!optional(input.time, "string")) return { error: "time is text." };
207 if (input.interval != null && !(DATASET_INTERVALS as readonly unknown[]).includes(input.interval)) return { error: "interval is day, week or month." };
208 if (input.limit != null && typeof input.limit !== "number") return { error: "limit is a number." };
209 let filters: DatasetFilter[] | null = null;
210 if (input.filters != null) {
211 if (!Array.isArray(input.filters)) return { error: "filters is a list." };
212 filters = [];
213 for (const filter of input.filters) {
214 if (!isObject(filter) || typeof filter.field !== "string" || !(DATASET_FILTER_OPS as readonly unknown[]).includes(filter.op)) {
215 return { error: "A filter has a field and an op: eq, neq, in, gte or lte." };
216 }
217 const value = filter.value;
218 const ok =
219 typeof value === "string" || (typeof value === "number" && Number.isFinite(value)) || (Array.isArray(value) && value.every((item) => typeof item === "string"));
220 if (!ok) return { error: "A filter's value is text, a number or a list of text." };
221 filters.push({ field: filter.field, op: filter.op as DatasetFilterOp, value: value as DatasetFilter["value"] });
222 }
223 }
224 let range: DatasetRange | null = null;
225 if (input.range != null) {
226 if (typeof input.range === "string" && (DATASET_RANGE_PRESETS as readonly string[]).includes(input.range)) range = input.range as DatasetRangePreset;
227 else if (isObject(input.range) && typeof input.range.from === "string" && typeof input.range.to === "string") range = { from: input.range.from, to: input.range.to };
228 else return { error: "range is 7d, 30d, 90d or { from, to }." };
229 }
230 const query: DatasetQuery = {
231 dataset: input.dataset as DatasetId,
232 measure: { op: measure.op as DatasetMeasureOp, field: (measure.field as string | null | undefined) ?? null },
233 group_by: (input.group_by as string | null | undefined) ?? null,
234 interval: (input.interval as DatasetInterval | null | undefined) ?? null,
235 time: (input.time as string | null | undefined) ?? null,
236 filters,
237 range,
238 limit: (input.limit as number | null | undefined) ?? null,
239 };
240 const error = datasetQueryError(query);
241 return error ? { error } : { query };
242}
243
244function isObject(value: unknown): value is Record<string, unknown> {
245 return typeof value === "object" && value !== null && !Array.isArray(value);
246}
247
248function optional(value: unknown, type: "string"): boolean {
249 return value === undefined || value === null || typeof value === type;
250}