Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp
301 linesCodeBlameRaw
1/**
2 * Datasets: the safe query layer behind dashboards in Artifacts. Mirrors
3 * `crates/contracts/src/datasets.rs`; a Rust test keeps the catalog the
4 * same and runs both validators over `datasets.fixtures.json`.
5 *
6 * Not SQL. A query names a dataset from a declared catalog, one measure,
7 * at most one dimension to group by, a time interval, filters on declared
8 * fields and a range. The service that owns the data (`service` in the
9 * catalog) runs it for the viewer, over only what the viewer can read, with
10 * a fixed query per measure and dimension, and caps the rows.
11 *
12 * Values never live in a folio: not in its Yjs document, versions, text,
13 * preview, index or templates. Only the query does.
14 *
15 * Wire shapes are snake_case.
16 */
17
18export type DatasetId = "work_items" | "pull_requests" | "workflow_runs" | "deployments" | "spend" | "agent_sessions";
19
20export const DATASET_IDS: readonly DatasetId[] = ["work_items", "pull_requests", "workflow_runs", "deployments", "spend", "agent_sessions"];
21
22/** How rows are summed up. `count` takes no field; `rate` takes a rate field; the rest a measure field. */
23export type DatasetMeasureOp = "count" | "sum" | "avg" | "p50" | "p95" | "rate";
24export const DATASET_MEASURE_OPS: readonly DatasetMeasureOp[] = ["count", "sum", "avg", "p50", "p95", "rate"];
25
26export type DatasetInterval = "day" | "week" | "month";
27export const DATASET_INTERVALS: readonly DatasetInterval[] = ["day", "week", "month"];
28
29export type DatasetFilterOp = "eq" | "neq" | "in" | "gte" | "lte";
30export const DATASET_FILTER_OPS: readonly DatasetFilterOp[] = ["eq", "neq", "in", "gte", "lte"];
31
32export type DatasetRangePreset = "7d" | "30d" | "90d";
33export const DATASET_RANGE_PRESETS: readonly DatasetRangePreset[] = ["7d", "30d", "90d"];
34
35/**
36 * A time range: a preset counted back from now, or between two times.
37 * `from` and `to` are dates (`2026-10-01`) or RFC 3339 UTC times
38 * (`2026-10-01T09:00:00Z`); `to` is exclusive.
39 */
40export type DatasetRange = DatasetRangePreset | { from: string; to: string };
41
42export type DatasetFilter = {
43 field: string;
44 op: DatasetFilterOp;
45 /** Text for `eq`/`neq` on a dimension, a list for `in`, a number on a measure. */
46 value: string | number | string[];
47};
48
49export type DatasetQuery = {
50 dataset: DatasetId;
51 measure: { op: DatasetMeasureOp; field?: string | null };
52 /** A declared dimension only. */
53 group_by?: string | null;
54 /** A time series, over the dataset's time field. */
55 interval?: DatasetInterval | null;
56 /** Which of the dataset's time fields the range and interval use; its first when left out. */
57 time?: string | null;
58 filters?: DatasetFilter[] | null;
59 /** The tile's own range; else the dashboard's. */
60 range?: DatasetRange | null;
61 /** At most `DATASET_MAX_ROWS`. */
62 limit?: number | null;
63};
64
65export type DatasetColumnType = "string" | "number" | "time" | "money";
66
67export type DatasetResult = {
68 columns: { name: string; type: DatasetColumnType }[];
69 rows: (string | number | null)[][];
70 /** More rows matched than were returned. */
71 truncated: boolean;
72 /** The viewer cannot read everything the query covers: shown as "Based on what you can see". */
73 partial: boolean;
74 /** When the numbers were computed, RFC 3339. */
75 as_of: string;
76};
77
78/** The services that own datasets, each answering `query_dataset` for its own. */
79export type DatasetService = "work" | "actions" | "deployments" | "billing" | "agents";
80
81/** What a viewer needs to query a dataset: membership, or the workspace's billing role. */
82export type DatasetNeeds = "member" | "billing";
83
84export type DatasetSpec = {
85 label: string;
86 service: DatasetService;
87 needs: DatasetNeeds;
88 /** Time fields, the default first. */
89 times: string[];
90 /** Fields to group and filter by (text). */
91 dimensions: string[];
92 /** Number fields for sum, avg, p50, p95, and number filters. */
93 measures: string[];
94 /** Yes-or-no fields `rate` gives the share of. */
95 rates: string[];
96};
97
98/** The catalog. One dataset per line: the Rust mirror test reads it that way. */
99export const DATASETS: Record<DatasetId, DatasetSpec> = {
100 work_items: {
101 label: "Work items",
102 service: "work",
103 needs: "member",
104 times: ["created_at", "closed_at"],
105 dimensions: ["repo", "label", "state", "author_kind", "assignee_kind", "milestone"],
106 measures: ["time_to_close_hours", "comments"],
107 rates: ["closed"],
108 },
109 pull_requests: {
110 label: "Pull requests",
111 service: "work",
112 needs: "member",
113 times: ["created_at", "merged_at", "closed_at"],
114 dimensions: ["repo", "label", "state", "author_kind", "base_branch"],
115 measures: ["cycle_time_hours", "time_to_first_review_hours", "review_count", "additions", "deletions", "changed_files"],
116 rates: ["merged"],
117 },
118 workflow_runs: {
119 label: "Workflow runs",
120 service: "actions",
121 needs: "member",
122 times: ["started_at", "completed_at"],
123 dimensions: ["repo", "workflow", "branch", "event", "conclusion", "runner_kind"],
124 measures: ["duration_seconds", "queue_seconds"],
125 rates: ["succeeded"],
126 },
127 deployments: {
128 label: "Deployments",
129 service: "deployments",
130 needs: "member",
131 times: ["created_at"],
132 dimensions: ["repo", "project", "environment", "state"],
133 measures: ["duration_seconds", "time_to_restore_hours"],
134 rates: ["failed"],
135 },
136 spend: {
137 label: "Spend",
138 service: "billing",
139 needs: "billing",
140 times: ["day"],
141 dimensions: ["product", "project", "person", "model"],
142 measures: ["amount_micros"],
143 rates: [],
144 },
145 agent_sessions: {
146 label: "Agent sessions",
147 service: "agents",
148 needs: "member",
149 times: ["started_at"],
150 dimensions: ["agent", "repo", "outcome", "model", "trigger"],
151 measures: ["duration_seconds", "cost_micros", "tokens"],
152 rates: ["succeeded"],
153 },
154};
155
156/** The most rows a query returns. */
157export const DATASET_MAX_ROWS = 100;
158/** The most filters on one query. */
159export const DATASET_MAX_FILTERS = 10;
160/** The most values in an `in` filter. */
161export const DATASET_MAX_IN_VALUES = 50;
162/** The longest `{ from, to }` range, in days. */
163export const DATASET_MAX_RANGE_DAYS = 366;
164
165const DAY_MS = 86_400_000;
166const TIME = /^(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,3}))?Z)?$/;
167
168/** A range end as milliseconds since the epoch, or null when it is not a real date or RFC 3339 UTC time. */
169export function datasetTime(text: string): number | null {
170 const match = TIME.exec(text);
171 if (!match) return null;
172 const [year, month, day] = [Number(match[1]), Number(match[2]), Number(match[3])];
173 const [hour, minute, second] = [Number(match[4] ?? 0), Number(match[5] ?? 0), Number(match[6] ?? 0)];
174 const millis = Number((match[7] ?? "").padEnd(3, "0") || 0);
175 if (month < 1 || month > 12 || day < 1 || day > daysInMonth(year, month)) return null;
176 if (hour > 23 || minute > 59 || second > 59) return null;
177 return Date.UTC(year, month - 1, day, hour, minute, second, millis);
178}
179
180function daysInMonth(year: number, month: number): number {
181 if (month === 2) return (year % 4 === 0 && year % 100 !== 0) || year % 400 === 0 ? 29 : 28;
182 return [4, 6, 9, 11].includes(month) ? 30 : 31;
183}
184
185/**
186 * What is wrong with a query, or null. The same rules, and the same
187 * words, as `DatasetQuery::validate` in Rust. It checks the query against
188 * the catalog only; who may run it is the owning service's to decide.
189 */
190export function datasetQueryError(query: DatasetQuery): string | null {
191 const spec = DATASETS[query.dataset];
192 if (!spec) return `There is no dataset called ${query.dataset}.`;
193 const name = query.dataset;
194 const { op } = query.measure;
195 const field = query.measure.field ?? null;
196 if (op === "count") {
197 if (field !== null) return "count takes no field.";
198 } else if (op === "rate") {
199 if (field === null) return "rate needs a field.";
200 if (!spec.rates.includes(field)) return `${name} has no yes-or-no field ${field} to take the rate of.`;
201 } else {
202 if (field === null) return `${op} needs a field.`;
203 if (!spec.measures.includes(field)) return `${name} has no number field ${field}.`;
204 }
205 const groupBy = query.group_by ?? null;
206 if (groupBy !== null && !spec.dimensions.includes(groupBy)) return `${name} can't be grouped by ${groupBy}.`;
207 const time = query.time ?? null;
208 if (time !== null && !spec.times.includes(time)) return `${name} has no time field ${time}.`;
209 const filters = query.filters ?? [];
210 if (filters.length > DATASET_MAX_FILTERS) return `A query takes at most ${DATASET_MAX_FILTERS} filters.`;
211 for (const filter of filters) {
212 if (spec.dimensions.includes(filter.field)) {
213 if (filter.op === "in") {
214 if (!Array.isArray(filter.value) || filter.value.length === 0 || filter.value.length > DATASET_MAX_IN_VALUES) {
215 return `in on ${filter.field} takes a list of 1 to ${DATASET_MAX_IN_VALUES} values.`;
216 }
217 } else if (filter.op === "eq" || filter.op === "neq") {
218 if (typeof filter.value !== "string") return `${filter.op} on ${filter.field} takes text.`;
219 } else {
220 return `${filter.field} is text: filter it with eq, neq or in.`;
221 }
222 } else if (spec.measures.includes(filter.field)) {
223 if (filter.op === "in") return `${filter.field} is a number: filter it with eq, neq, gte or lte.`;
224 if (typeof filter.value !== "number" || !Number.isFinite(filter.value)) return `${filter.op} on ${filter.field} takes a number.`;
225 } else {
226 return `${name} can't be filtered by ${filter.field}.`;
227 }
228 }
229 const range = query.range ?? null;
230 if (range !== null && typeof range === "object") {
231 const from = datasetTime(range.from);
232 const to = datasetTime(range.to);
233 if (from === null || to === null) return "A range's from and to are dates or RFC 3339 UTC times.";
234 if (from >= to) return "A range's from comes before its to.";
235 if (to - from > DATASET_MAX_RANGE_DAYS * DAY_MS) return `A range is at most ${DATASET_MAX_RANGE_DAYS} days.`;
236 }
237 const limit = query.limit ?? null;
238 if (limit !== null && (!Number.isInteger(limit) || limit < 1 || limit > DATASET_MAX_ROWS)) return `limit is between 1 and ${DATASET_MAX_ROWS}.`;
239 return null;
240}
241
242/** A query from untrusted JSON: well-formed and valid, or why not. Unknown keys are dropped. */
243export function parseDatasetQuery(input: unknown): { query: DatasetQuery } | { error: string } {
244 if (!isObject(input)) return { error: "A query is an object." };
245 if (typeof input.dataset !== "string" || !(DATASET_IDS as readonly string[]).includes(input.dataset)) {
246 return { error: `There is no dataset called ${String(input.dataset)}.` };
247 }
248 const measure = input.measure;
249 if (!isObject(measure) || typeof measure.op !== "string" || !(DATASET_MEASURE_OPS as readonly string[]).includes(measure.op)) {
250 return { error: "measure.op is count, sum, avg, p50, p95 or rate." };
251 }
252 if (!optional(measure.field, "string")) return { error: "measure.field is text." };
253 if (!optional(input.group_by, "string")) return { error: "group_by is text." };
254 if (!optional(input.time, "string")) return { error: "time is text." };
255 if (input.interval != null && !(DATASET_INTERVALS as readonly unknown[]).includes(input.interval)) return { error: "interval is day, week or month." };
256 if (input.limit != null && typeof input.limit !== "number") return { error: "limit is a number." };
257 let filters: DatasetFilter[] | null = null;
258 if (input.filters != null) {
259 if (!Array.isArray(input.filters)) return { error: "filters is a list." };
260 filters = [];
261 for (const filter of input.filters) {
262 if (!isObject(filter) || typeof filter.field !== "string" || !(DATASET_FILTER_OPS as readonly unknown[]).includes(filter.op)) {
263 return { error: "A filter has a field and an op: eq, neq, in, gte or lte." };
264 }
265 const value = filter.value;
266 const ok =
267 typeof value === "string" ||
268 (typeof value === "number" && Number.isFinite(value)) ||
269 (Array.isArray(value) && value.every((item) => typeof item === "string"));
270 if (!ok) return { error: "A filter's value is text, a number or a list of text." };
271 filters.push({ field: filter.field, op: filter.op as DatasetFilterOp, value: value as DatasetFilter["value"] });
272 }
273 }
274 let range: DatasetRange | null = null;
275 if (input.range != null) {
276 if (typeof input.range === "string" && (DATASET_RANGE_PRESETS as readonly string[]).includes(input.range)) range = input.range as DatasetRangePreset;
277 else if (isObject(input.range) && typeof input.range.from === "string" && typeof input.range.to === "string")
278 range = { from: input.range.from, to: input.range.to };
279 else return { error: "range is 7d, 30d, 90d or { from, to }." };
280 }
281 const query: DatasetQuery = {
282 dataset: input.dataset as DatasetId,
283 measure: { op: measure.op as DatasetMeasureOp, field: (measure.field as string | null | undefined) ?? null },
284 group_by: (input.group_by as string | null | undefined) ?? null,
285 interval: (input.interval as DatasetInterval | null | undefined) ?? null,
286 time: (input.time as string | null | undefined) ?? null,
287 filters,
288 range,
289 limit: (input.limit as number | null | undefined) ?? null,
290 };
291 const error = datasetQueryError(query);
292 return error ? { error } : { query };
293}
294
295function isObject(value: unknown): value is Record<string, unknown> {
296 return typeof value === "object" && value !== null && !Array.isArray(value);
297}
298
299function optional(value: unknown, type: "string"): boolean {
300 return value === undefined || value === null || typeof value === type;
301}