Sign in

flagon-io/g1t

public

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp

g1t/services/work/src/memory.rs

544 lines21,990 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Agents and memory, checks and conflicts, profiles, slug renames, custom domains1//! Memory: what agents and people have learned that the next agent should
2//! know, at two levels. A project's memory is about its codebase; the
3//! workspace's is true across its projects ("we use pnpm everywhere",
4//! "staging lives at …").
5//!
6//! It is members-only, since it can hold internal knowledge, and it never
7//! holds a secret: text that looks like a key or a token is refused. Every
8//! g1t agent run is given it (`memory_context`): pinned first, then what
9//! was used most recently, within a size budget.
10
11use g1t_contracts::agents::*;
12use g1t_contracts::repos::{Repo, RepoPath};
13use g1t_contracts::time::rfc3339;
14use g1t_contracts::{FailureCode, Outcome, PrincipalKind, User, Viewer, new_id};
15use g1t_kit::now_ms;
16use serde::Deserialize;
17use worker::Result;
18use worker::wasm_bindgen::JsValue;
19
20use crate::Work;
21use crate::runs::member_of;
22
23/// A workspace renamed: its runs and memories move to the slug it has now.
24/// `?1` is the current slug, `?2` a stale one (see `g1t_kit::rename`).
25pub(crate) const RENAMED: &[&str] = &[
26 "UPDATE agent_runs SET workspace = ?1 WHERE workspace = ?2",
27 "UPDATE agent_runs SET repo = ?1 || substr(repo, length(?2) + 1) WHERE substr(repo, 1, length(?2) + 1) = ?2 || '/'",
28 "UPDATE memories SET scope_key = ?1 WHERE scope = 'workspace' AND scope_key = ?2",
29 "UPDATE memories SET workspace = ?1 WHERE workspace = ?2",
30 "UPDATE memories SET repo = ?1 || substr(repo, length(?2) + 1) WHERE substr(repo, 1, length(?2) + 1) = ?2 || '/'",
31 "UPDATE memories SET source_repo = ?1 || substr(source_repo, length(?2) + 1) WHERE substr(source_repo, 1, length(?2) + 1) = ?2 || '/'",
32];
33
34/// The most memories one project, or one workspace, keeps.
35const MAX_PER_SCOPE: u32 = 500;
36/// What an agent is given by default, in characters.
37const DEFAULT_BUDGET: u32 = 6000;
38const MAX_BUDGET: u32 = 20_000;
39const DEFAULT_RECALL: u32 = 20;
40
41#[derive(Deserialize)]
42struct MemoryRow {
43 id: String,
44 scope: String,
45 workspace: String,
46 repo: Option<String>,
47 text: String,
48 kind: String,
49 source_kind: String,
50 source_run: Option<String>,
51 source_repo: Option<String>,
52 source_number: Option<u32>,
53 created_by: String,
54 pinned: u32,
55 created_at: String,
56 updated_at: String,
57 last_used_at: Option<String>,
58}
59
60fn path(text: &str) -> Option<RepoPath> {
61 let (namespace, name) = text.split_once('/')?;
62 Some(RepoPath {
63 namespace: namespace.to_owned(),
64 name: name.to_owned(),
65 })
66}
67
68impl From<MemoryRow> for Memory {
69 fn from(row: MemoryRow) -> Self {
70 Memory {
71 id: row.id,
72 scope: if row.scope == "workspace" {
73 MemoryScope::Workspace
74 } else {
75 MemoryScope::Project
76 },
77 workspace: row.workspace,
78 repo: row.repo.as_deref().and_then(path),
79 text: row.text,
80 kind: MemoryKind::parse(&row.kind).unwrap_or_default(),
81 source: MemorySource {
82 kind: row.source_kind,
83 run_id: row.source_run,
84 repo: row.source_repo.as_deref().and_then(path),
85 number: row.source_number,
86 },
87 created_by: row.created_by,
88 pinned: row.pinned != 0,
89 created_at: row.created_at,
90 updated_at: row.updated_at,
91 last_used_at: row.last_used_at,
92 }
93 }
94}
95
96/// The text tidied, or why it cannot be kept.
97fn valid_text(text: &str) -> std::result::Result<String, String> {
98 let text = text.trim();
99 if text.is_empty() {
100 return Err("Write what should be remembered.".into());
101 }
102 if text.chars().count() > MAX_MEMORY_CHARS {
103 return Err(format!(
104 "Keep a memory to {MAX_MEMORY_CHARS} characters: one fact, convention, decision or gotcha each."
105 ));
106 }
107 if let Some(what) = secret_in(text) {
108 return Err(format!(
109 "That looks like {what}. Memory is read by every agent in the workspace, so it never holds secrets. Say where the secret lives instead, such as \"the deploy key is the DEPLOY_KEY secret\"."
110 ));
111 }
112 Ok(text.to_owned())
113}
114
115fn denied<T>() -> Outcome<T> {
116 Outcome::fail(
117 FailureCode::Forbidden,
118 "Memory is for members of the workspace and its agents.",
119 )
120}
121
122/// Whether `actor` may change `workspace`'s memory.
123fn may_write(actor: &User, workspace: &str) -> bool {
124 actor.is_member(workspace) && (actor.verified || actor.kind != PrincipalKind::User)
125}
126
127/// The order memories are given and listed in: pinned, then most recently
128/// used or changed.
129const ORDER: &str = "pinned DESC, COALESCE(last_used_at, updated_at) DESC, created_at DESC";
130
131/// What an agent is told about memory, ahead of the memories themselves.
132const PREAMBLE: &str = "What people and agents have learned here before you, kept as memory. Treat it as notes from colleagues: usually right, sometimes out of date. Where it disagrees with the code, the code wins; say so in your summary.";
133
134/// How an agent is told to add to memory.
135const HOW_TO_REMEMBER: &str = "When you learn something the next agent here would need (how to build or test, a convention, a decision and why, a trap), save it with the remember tool: scope project for this codebase, workspace for what holds across the workspace's projects. One short fact each. Never a secret, a key or a token. recall searches what is kept.";
136
137/// The context an agent gets: as many memories as fit in `budget`, each
138/// level labelled, and the ids of those given.
139fn compose(workspace: &[Memory], project: &[Memory], repo: &RepoPath, budget: usize) -> (Option<String>, Vec<String>) {
140 let line = |memory: &Memory| {
141 format!(
142 "- [{}{}] {}",
143 memory.kind.as_str(),
144 if memory.pinned { ", pinned" } else { "" },
145 memory.text.replace('\n', " ")
146 )
147 };
148 let mut used = PREAMBLE.len() + HOW_TO_REMEMBER.len() + 200;
149 let mut given = Vec::new();
150 let mut sections = Vec::new();
151 // Pinned memories of either level go in before anything else does.
152 let mut take = |memories: &[Memory], pinned: bool, out: &mut Vec<String>| {
153 for memory in memories.iter().filter(|memory| memory.pinned == pinned) {
154 let text = line(memory);
155 if used + text.len() + 1 > budget {
156 continue;
157 }
158 used += text.len() + 1;
159 given.push(memory.id.clone());
160 out.push(text);
161 }
162 };
163 let (mut ws, mut own) = (Vec::new(), Vec::new());
164 take(workspace, true, &mut ws);
165 take(project, true, &mut own);
166 take(project, false, &mut own);
167 take(workspace, false, &mut ws);
168 if ws.is_empty() && own.is_empty() {
169 return (None, given);
170 }
171 sections.push(PREAMBLE.to_owned());
172 if !ws.is_empty() {
173 sections.push(format!(
174 "Workspace memory (true across the {} workspace's projects):\n{}",
175 repo.namespace,
176 ws.join("\n")
177 ));
178 }
179 if !own.is_empty() {
180 sections.push(format!(
181 "Project memory ({}/{}):\n{}",
182 repo.namespace,
183 repo.name,
184 own.join("\n")
185 ));
186 }
187 sections.push(HOW_TO_REMEMBER.to_owned());
188 (Some(sections.join("\n\n")), given)
189}
190
191impl Work {
192 async fn memories_of(&self, scope: MemoryScope, key: &str, limit: u32) -> Result<Vec<Memory>> {
193 Ok(self
194 .db
195 .prepare(format!(
196 "SELECT * FROM memories WHERE scope = ? AND scope_key = ? ORDER BY {ORDER} LIMIT ?"
197 ))
198 .bind(&[scope.as_str().into(), key.into(), limit.into()])?
199 .all()
200 .await?
201 .results::<MemoryRow>()?
202 .into_iter()
203 .map(Memory::from)
204 .collect())
205 }
206
207 async fn memory_row(&self, workspace: &str, id: &str) -> Result<Option<Memory>> {
208 Ok(self
209 .db
210 .prepare("SELECT * FROM memories WHERE id = ? AND workspace = ?")
211 .bind(&[id.into(), workspace.into()])?
212 .first::<MemoryRow>(None)
213 .await?
214 .map(Memory::from))
215 }
216
217 async fn mark_used(&self, ids: &[String]) -> Result<()> {
218 if ids.is_empty() {
219 return Ok(());
220 }
221 self.db
222 .prepare("UPDATE memories SET last_used_at = ? WHERE id IN (SELECT value FROM json_each(?))")
223 .bind(&[rfc3339(now_ms()).into(), serde_json::to_string(ids)?.into()])?
224 .run()
225 .await?;
226 Ok(())
227 }
228
229 /// The project's repository, which must be in `workspace`.
230 async fn project_repo(&self, path: &RepoPath, viewer: &Viewer, workspace: &str) -> Result<Outcome<Repo>> {
231 Ok(match self.repo(path, viewer).await? {
232 Outcome::Ok(repo) if repo.namespace.to_lowercase() == workspace => Outcome::Ok(repo),
233 Outcome::Ok(_) => Outcome::fail(FailureCode::Invalid, "That project is in another workspace."),
234 Outcome::Fail(failure) => Outcome::Fail(failure),
235 })
236 }
237
238 pub(crate) async fn list_memories(&self, a: ListMemoriesArgs) -> Result<Outcome<Memories>> {
239 let workspace = a.workspace.to_lowercase();
240 if !a.viewer.as_ref().is_some_and(|viewer| viewer.is_member(&workspace)) {
241 return Ok(denied());
242 }
243 let project = match &a.repo {
244 Some(path) => match self.project_repo(path, &a.viewer, &workspace).await? {
245 Outcome::Ok(repo) => self.memories_of(MemoryScope::Project, &repo.id, MAX_PER_SCOPE).await?,
246 Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)),
247 },
248 None => Vec::new(),
249 };
250 Ok(Outcome::Ok(Memories {
251 project,
252 workspace: self.memories_of(MemoryScope::Workspace, &workspace, MAX_PER_SCOPE).await?,
253 }))
254 }
255
256 pub(crate) async fn add_memory(&self, a: AddMemoryArgs) -> Result<Outcome<Memory>> {
257 let workspace = a.workspace.to_lowercase();
258 if !may_write(&a.actor, &workspace) {
259 return Ok(denied());
260 }
261 let text = match valid_text(&a.text) {
262 Ok(text) => text,
263 Err(message) => return Ok(Outcome::fail(FailureCode::Invalid, message)),
264 };
265 let viewer = Some(a.actor.clone());
266 let repo = match &a.repo {
267 Some(path) => match self.project_repo(path, &viewer, &workspace).await? {
268 Outcome::Ok(repo) => Some(repo),
269 Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)),
270 },
271 None => None,
272 };
273 let key = match (a.scope, &repo) {
274 (MemoryScope::Workspace, _) => workspace.clone(),
275 (MemoryScope::Project, Some(repo)) => repo.id.clone(),
276 (MemoryScope::Project, None) => {
277 return Ok(Outcome::fail(FailureCode::Invalid, "Name the project this memory is about."));
278 }
279 };
280 // The same thing remembered twice is one memory, freshened.
281 let now = rfc3339(now_ms());
282 let same = self
283 .db
284 .prepare("UPDATE memories SET updated_at = ? WHERE scope = ? AND scope_key = ? AND text = ? RETURNING id AS value")
285 .bind(&[now.as_str().into(), a.scope.as_str().into(), key.as_str().into(), text.as_str().into()])?
286 .first::<String>(Some("value"))
287 .await?;
288 if let Some(id) = same {
289 return Ok(match self.memory_row(&workspace, &id).await? {
290 Some(memory) => Outcome::Ok(memory),
291 None => Outcome::fail(FailureCode::NotFound, "Memory not found."),
292 });
293 }
294 let count = self
295 .db
296 .prepare("SELECT count(*) AS value FROM memories WHERE scope = ? AND scope_key = ?")
297 .bind(&[a.scope.as_str().into(), key.as_str().into()])?
298 .first::<u32>(Some("value"))
299 .await?
300 .unwrap_or_default();
301 if count >= MAX_PER_SCOPE {
302 return Ok(Outcome::fail(
303 FailureCode::Conflict,
304 format!("This {} already keeps {MAX_PER_SCOPE} memories. Remove some that no longer hold first.", a.scope.as_str()),
305 ));
306 }
307 // Where it came from: a person, or an agent, and the run it was in.
308 let from = match (&repo, a.from_number) {
309 (Some(repo), Some(number)) => Some((repo, number)),
310 _ => None,
311 };
312 let run = match (a.actor.kind, from) {
313 (PrincipalKind::Agent, Some((repo, number))) => self.active_run_on(&repo.id, number).await?,
314 _ => None,
315 };
316 let source_kind = match (a.actor.kind, &run) {
317 (PrincipalKind::User, _) => "person",
318 (_, Some(_)) => "run",
319 _ => "agent",
320 };
321 let id = new_id("mem", now_ms());
322 let repo_text = repo.as_ref().map(|repo| format!("{}/{}", repo.namespace, repo.name));
323 self.db
324 .prepare(
325 "INSERT INTO memories
326 (id, scope, scope_key, workspace, repo, text, kind, source_kind, source_run,
327 source_repo, source_number, created_by, pinned, created_at, updated_at)
328 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
329 )
330 .bind(&[
331 id.as_str().into(),
332 a.scope.as_str().into(),
333 key.into(),
334 workspace.as_str().into(),
335 // A workspace's memory belongs to no one project, though it
336 // remembers which it was learned in.
337 if a.scope == MemoryScope::Project { repo_text.clone().map_or(JsValue::NULL, JsValue::from) } else { JsValue::NULL },
338 text.into(),
339 a.kind.as_str().into(),
340 source_kind.into(),
341 run.map_or(JsValue::NULL, JsValue::from),
342 repo_text.map_or(JsValue::NULL, JsValue::from),
343 from.map_or(JsValue::NULL, |(_, number)| number.into()),
344 a.actor.username.as_str().into(),
345 u32::from(a.pinned).into(),
346 now.as_str().into(),
347 now.as_str().into(),
348 ])?
349 .run()
350 .await?;
351 Ok(match self.memory_row(&workspace, &id).await? {
352 Some(memory) => Outcome::Ok(memory),
353 None => Outcome::fail(FailureCode::NotFound, "Memory not found."),
354 })
355 }
356
357 pub(crate) async fn update_memory(&self, a: UpdateMemoryArgs) -> Result<Outcome<Memory>> {
358 let workspace = a.workspace.to_lowercase();
359 if !may_write(&a.actor, &workspace) || a.actor.kind == PrincipalKind::Agent {
360 return Ok(Outcome::fail(FailureCode::Forbidden, "Only members of the workspace can change its memory."));
361 }
362 if self.memory_row(&workspace, &a.id).await?.is_none() {
363 return Ok(Outcome::fail(FailureCode::NotFound, "Memory not found."));
364 }
365 let text = match a.text.as_deref().map(valid_text) {
366 Some(Err(message)) => return Ok(Outcome::fail(FailureCode::Invalid, message)),
367 Some(Ok(text)) => Some(text),
368 None => None,
369 };
370 self.db
371 .prepare(
372 "UPDATE memories SET text = COALESCE(?, text), kind = COALESCE(?, kind),
373 pinned = COALESCE(?, pinned), updated_at = ?
374 WHERE id = ? AND workspace = ?",
375 )
376 .bind(&[
377 text.map_or(JsValue::NULL, JsValue::from),
378 a.kind.map_or(JsValue::NULL, |kind| kind.as_str().into()),
379 a.pinned.map_or(JsValue::NULL, |pinned| u32::from(pinned).into()),
380 rfc3339(now_ms()).into(),
381 a.id.as_str().into(),
382 workspace.as_str().into(),
383 ])?
384 .run()
385 .await?;
386 Ok(match self.memory_row(&workspace, &a.id).await? {
387 Some(memory) => Outcome::Ok(memory),
388 None => Outcome::fail(FailureCode::NotFound, "Memory not found."),
389 })
390 }
391
392 pub(crate) async fn delete_memory(&self, a: DeleteMemoryArgs) -> Result<Outcome<bool>> {
393 let workspace = a.workspace.to_lowercase();
394 if !may_write(&a.actor, &workspace) || a.actor.kind == PrincipalKind::Agent {
395 return Ok(Outcome::fail(FailureCode::Forbidden, "Only members of the workspace can change its memory."));
396 }
397 let gone = self
398 .db
399 .prepare("DELETE FROM memories WHERE id = ? AND workspace = ? RETURNING id AS value")
400 .bind(&[a.id.as_str().into(), workspace.as_str().into()])?
401 .first::<String>(Some("value"))
402 .await?;
403 Ok(match gone {
404 Some(_) => Outcome::Ok(true),
405 None => Outcome::fail(FailureCode::NotFound, "Memory not found."),
406 })
407 }
408
409 pub(crate) async fn recall(&self, a: RecallArgs) -> Result<Outcome<Memories>> {
410 let repo = match self.repo(&a.repo, &a.viewer).await? {
411 Outcome::Ok(repo) => repo,
412 Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)),
413 };
414 let workspace = repo.namespace.to_lowercase();
415 if !a.viewer.as_ref().is_some_and(|viewer| viewer.is_member(&workspace)) {
416 return Ok(denied());
417 }
418 let words: Vec<String> = a
419 .query
420 .as_deref()
421 .unwrap_or_default()
422 .split_whitespace()
423 .map(str::to_lowercase)
424 .collect();
425 let limit = a.limit.unwrap_or(DEFAULT_RECALL).clamp(1, 100) as usize;
426 let matching = |memories: Vec<Memory>| -> Vec<Memory> {
427 memories
428 .into_iter()
429 .filter(|memory| {
430 let text = memory.text.to_lowercase();
431 words.iter().all(|word| text.contains(word.as_str()))
432 })
433 .take(limit)
434 .collect()
435 };
436 let found = Memories {
437 project: matching(self.memories_of(MemoryScope::Project, &repo.id, MAX_PER_SCOPE).await?),
438 workspace: matching(self.memories_of(MemoryScope::Workspace, &workspace, MAX_PER_SCOPE).await?),
439 };
440 let ids: Vec<String> = found
441 .project
442 .iter()
443 .chain(&found.workspace)
444 .map(|memory| memory.id.clone())
445 .collect();
446 self.mark_used(&ids).await?;
447 Ok(Outcome::Ok(found))
448 }
449
450 pub(crate) async fn memory_context(&self, a: MemoryContextArgs) -> Result<MemoryContext> {
451 let service = User {
452 id: "g1t_runner".into(),
453 username: "g1t".into(),
454 ..User::default()
455 };
456 let Outcome::Ok(repo) = self.repo(&a.repo, &member_of(&service, &a.repo.namespace)).await? else {
457 return Ok(MemoryContext::default());
458 };
459 let workspace = self
460 .memories_of(MemoryScope::Workspace, &repo.namespace.to_lowercase(), MAX_PER_SCOPE)
461 .await?;
462 let project = self.memories_of(MemoryScope::Project, &repo.id, MAX_PER_SCOPE).await?;
463 let budget = a.budget.unwrap_or(DEFAULT_BUDGET).clamp(500, MAX_BUDGET) as usize;
464 let path = RepoPath {
465 namespace: repo.namespace.clone(),
466 name: repo.name.clone(),
467 };
468 let (text, given) = compose(&workspace, &project, &path, budget);
469 self.mark_used(&given).await?;
470 Ok(MemoryContext {
471 text,
472 count: given.len() as u32,
473 })
474 }
475}
476
477#[cfg(test)]
478mod tests {
479 use super::*;
480
481 fn memory(id: &str, text: &str, pinned: bool) -> Memory {
482 Memory {
483 id: id.into(),
484 scope: MemoryScope::Project,
485 workspace: "acme".into(),
486 repo: None,
487 text: text.into(),
488 kind: MemoryKind::Fact,
489 source: MemorySource::default(),
490 created_by: "ana".into(),
491 pinned,
492 created_at: String::new(),
493 updated_at: String::new(),
494 last_used_at: None,
495 }
496 }
497
498 fn repo() -> RepoPath {
499 RepoPath {
500 namespace: "acme".into(),
501 name: "web".into(),
502 }
503 }
504
505 #[test]
506 fn rename_statements_take_the_two_slugs() {
507 for sql in RENAMED {
508 assert_eq!(g1t_kit::rename::parameters(sql), 2, "{sql}");
509 }
510 }
511
512 #[test]
513 fn nothing_kept_is_nothing_said() {
514 assert_eq!(compose(&[], &[], &repo(), 6000), (None, Vec::new()));
515 }
516
517 #[test]
518 fn levels_are_labelled_and_pinned_come_first() {
519 let workspace = [memory("w1", "We use pnpm everywhere.", false)];
520 let project = [memory("p1", "Tests need TZ=UTC.", false), memory("p2", "Never edit generated.rs.", true)];
521 let (text, given) = compose(&workspace, &project, &repo(), 6000);
522 let text = text.unwrap();
523 assert!(text.contains("Workspace memory (true across the acme workspace's projects)"));
524 assert!(text.contains("Project memory (acme/web)"));
525 assert!(text.find("Never edit").unwrap() < text.find("Tests need").unwrap());
526 assert_eq!(given, ["p2", "p1", "w1"]);
527 }
528
529 #[test]
530 fn the_budget_is_kept() {
531 let project: Vec<Memory> = (0..100).map(|n| memory(&format!("p{n}"), &"x".repeat(200), false)).collect();
532 let (text, given) = compose(&[], &project, &repo(), 3000);
533 assert!(text.unwrap().len() <= 3000);
534 assert!(given.len() < 100 && !given.is_empty());
535 }
536
537 #[test]
538 fn secrets_are_refused_with_a_way_out() {
539 let refused = valid_text("deploy with ghp_abcdefghijklmnopqrstuvwxyz0123456789").unwrap_err();
540 assert!(refused.contains("never holds secrets"));
541 assert_eq!(valid_text(" We use pnpm. ").unwrap(), "We use pnpm.");
542 assert!(valid_text(" ").is_err());
543 }
544}