Sign in

pulls/pr_01m47d15m3e54sn21z27rpy5n9

public

pr_01m47d15m3e54sn21z27rpy5n9/services/integrations/src/github.rs

1,156 lines49,653 bytesCodeBlame
1//! g1t's GitHub App: the installations a workspace records, the
2//! repositories brought across through them, and the app's webhook.
3//!
4//! A person installs the app on a GitHub account, and GitHub sends them
5//! back to `g1t.sh/integrations/github/setup`. The site asks this service
6//! to record the installation against a workspace, which it does only after
7//! checking with the person's own GitHub user token (from identity) that
8//! the installation is one they can see. Repositories are listed with that
9//! same user token, so a person only ever sees what both they and the app
10//! can reach.
11//!
12//! Git goes over HTTPS with an installation access token, which this
13//! service gets by signing a JWT as the app (see `github_jwt.rs`) and keeps,
14//! sealed, until five minutes before it expires. Tokens are opaque: no
15//! length or format is assumed.
16//!
17//! A repository comes across one of three ways (`GithubMode`): imported
18//! once; mirrored, so each push on GitHub is fetched into g1t; or pushed,
19//! so each push on g1t is sent to GitHub. Either way g1t holds a full copy,
20//! and the project built from it is hosted on g1t like any other.
21//!
22//! The webhook, `https://api.g1t.sh/hooks/github`, is checked against
23//! GITHUB_APP_WEBHOOK_SECRET in constant time, de-duplicated by
24//! `X-GitHub-Delivery`, answered with 202 at once and acted on afterwards,
25//! as every inbound hook in this service is.
26
27use std::collections::{HashMap, HashSet};
28
29use g1t_contracts::access::{self, Capability};
30use g1t_contracts::events::Event;
31use g1t_contracts::github::*;
32use g1t_contracts::integrations::Received;
33use g1t_contracts::repos::{CreateArgs, MirrorArgs, MirrorDirection, Mirrored, Repo, RepoPath};
34use g1t_contracts::time::rfc3339;
35use g1t_contracts::work::{Issue, IssueActionArgs, IssueReason, OpenIssueArgs};
36use g1t_contracts::{FailureCode, Outcome, PrincipalKind, Role, User, is_valid_repo_name};
37use g1t_kit::{args, now_ms, reply};
38use g1t_secrets::{self as crypto, Sealer};
39use serde::Deserialize;
40use serde_json::{Value, json};
41use worker::wasm_bindgen::JsValue;
42use worker::{Context, D1Database, Env, Fetcher, Method, Response, Result};
43
44use crate::github_jwt;
45use crate::http::{self, Answer};
46
47const API: &str = "https://api.github.com";
48/// An installation token is used until this close to its expiry.
49const TOKEN_MARGIN_MS: u64 = 5 * 60 * 1000;
50/// The most issues copied in one import, and per page asked of GitHub.
51const MAX_ISSUES: usize = 200;
52const PER_PAGE: u32 = 50;
53/// How long a delivery id is remembered, to drop GitHub's retries.
54const DELIVERY_DAYS: u64 = 7;
55
56/// The app, as this g1t is configured. Only `configured()` ones are used.
57pub struct AppConfig {
58 pub app_id: String,
59 pub slug: String,
60 pub client_id: String,
61 pub private_key: Option<String>,
62 pub webhook_secret: Option<String>,
63}
64
65impl AppConfig {
66 pub fn from_env(env: &Env) -> Self {
67 let var = |name: &str| env.var(name).map(|v| v.to_string().trim().to_owned()).unwrap_or_default();
68 let secret = |name: &str| {
69 env.secret(name)
70 .ok()
71 .map(|value| value.to_string())
72 .filter(|value| !value.trim().is_empty())
73 };
74 AppConfig {
75 app_id: var("GITHUB_APP_ID"),
76 slug: var("GITHUB_APP_SLUG"),
77 client_id: var("GITHUB_APP_CLIENT_ID"),
78 private_key: secret("GITHUB_APP_PRIVATE_KEY"),
79 webhook_secret: secret("GITHUB_APP_WEBHOOK_SECRET"),
80 }
81 }
82
83 pub fn configured(&self) -> bool {
84 !self.slug.is_empty() && !self.issuer().is_empty() && self.private_key.is_some()
85 }
86
87 /// Who the app's JWTs say they are from: its client ID, as GitHub now
88 /// recommends, or its app ID.
89 pub fn issuer(&self) -> &str {
90 if self.client_id.is_empty() { &self.app_id } else { &self.client_id }
91 }
92
93 pub fn install_url(&self) -> String {
94 format!("https://github.com/apps/{}/installations/new", self.slug)
95 }
96}
97
98// --- Pure parts, tested below ----------------------------------------------
99
100/// Milliseconds since the epoch of an RFC 3339 UTC time such as GitHub's
101/// `2026-10-05T12:00:00Z`.
102pub fn parse_time(text: &str) -> Option<u64> {
103 let text = text.trim().trim_end_matches('Z');
104 let (date, time) = text.split_once('T')?;
105 let mut date = date.splitn(3, '-').map(|part| part.parse::<i64>().ok());
106 let (year, month, day) = (date.next()??, date.next()??, date.next()??);
107 let mut time = time.splitn(3, ':');
108 let hour: i64 = time.next()?.parse().ok()?;
109 let minute: i64 = time.next()?.parse().ok()?;
110 let second: f64 = time.next()?.split('+').next()?.parse().ok()?;
111 // Days from the civil date (Howard Hinnant's algorithm).
112 let year = if month <= 2 { year - 1 } else { year };
113 let era = year.div_euclid(400);
114 let year_of_era = year - era * 400;
115 let day_of_year = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1;
116 let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year;
117 let days = era * 146_097 + day_of_era - 719_468;
118 let ms = ((days * 86_400 + hour * 3_600 + minute * 60) as f64 + second) * 1000.0;
119 (ms >= 0.0).then_some(ms as u64)
120}
121
122/// A g1t repository name for a GitHub one.
123pub fn repo_name(github_name: &str) -> String {
124 let name: String = github_name
125 .trim()
126 .to_lowercase()
127 .chars()
128 .map(|c| if c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-') { c } else { '-' })
129 .collect();
130 let name = name.trim_start_matches('.');
131 name.strip_suffix(".git").unwrap_or(name).chars().take(100).collect()
132}
133
134/// Whether a webhook delivery was signed with the app's secret.
135pub fn authentic(secret: &str, body: &str, headers: &HashMap<String, String>) -> bool {
136 headers
137 .get("x-hub-signature-256")
138 .is_some_and(|signature| signature.trim().starts_with("sha256=") && crypto::signed(secret, body, signature))
139}
140
141/// Labels as g1t keeps them: lowercase, at most 40 characters, ten each.
142pub fn labels_of(issue: &Value) -> Vec<String> {
143 let mut labels: Vec<String> = Vec::new();
144 for label in issue["labels"].as_array().into_iter().flatten() {
145 let name = label["name"].as_str().or(label.as_str()).unwrap_or_default().trim().to_lowercase();
146 if !name.is_empty() && name.chars().count() <= 40 && !labels.contains(&name) {
147 labels.push(name);
148 }
149 }
150 labels.truncate(10);
151 labels
152}
153
154/// The opening of an imported issue's body: where it came from and who
155/// wrote it, by their g1t name when their GitHub account is linked.
156pub fn imported_header(issue: &Value, full_name: &str, g1t_name: Option<&str>) -> String {
157 let login = issue["user"]["login"].as_str().unwrap_or("ghost");
158 let author = match g1t_name {
159 Some(name) => format!("@{name} (@{login} on GitHub)"),
160 None => format!("[@{login}](https://github.com/{login}) on GitHub"),
161 };
162 let date = issue["created_at"].as_str().unwrap_or_default().get(..10).unwrap_or_default();
163 let mut header = format!(
164 "> Imported from GitHub: [{full_name}#{}]({}), opened by {author}{}.",
165 issue["number"],
166 issue["html_url"].as_str().unwrap_or_default(),
167 if date.is_empty() { String::new() } else { format!(" on {date}") },
168 );
169 if let Some(milestone) = issue["milestone"]["title"].as_str() {
170 header.push_str(&format!("\n> Milestone: {milestone}"));
171 }
172 header
173}
174
175// --- The service --------------------------------------------------------------
176
177#[derive(Deserialize)]
178struct InstallationRow {
179 id: u64,
180 workspace: String,
181 account: String,
182 account_type: String,
183 repository_selection: String,
184 settings_url: String,
185 suspended_at: Option<String>,
186 created_at: String,
187}
188
189impl From<InstallationRow> for GithubInstallation {
190 fn from(row: InstallationRow) -> Self {
191 GithubInstallation {
192 id: row.id,
193 workspace: row.workspace,
194 account: row.account,
195 account_type: row.account_type,
196 repository_selection: row.repository_selection,
197 suspended: row.suspended_at.is_some(),
198 settings_url: row.settings_url,
199 created_at: row.created_at,
200 }
201 }
202}
203
204#[derive(Deserialize)]
205struct LinkRow {
206 repo_id: String,
207 repo: String,
208 installation_id: u64,
209 github_repo_id: u64,
210 full_name: String,
211 mode: String,
212 synced_at: Option<String>,
213 last_error: Option<String>,
214 issues_imported: u32,
215}
216
217impl From<LinkRow> for GithubRepoLink {
218 fn from(row: LinkRow) -> Self {
219 GithubRepoLink {
220 repo_id: row.repo_id,
221 repo: row.repo,
222 installation_id: row.installation_id,
223 github_repo_id: row.github_repo_id,
224 full_name: row.full_name,
225 mode: GithubMode::parse(&row.mode),
226 synced_at: row.synced_at,
227 last_error: row.last_error,
228 issues_imported: row.issues_imported,
229 }
230 }
231}
232
233/// Issues to copy after an import has answered.
234pub struct IssueJob {
235 actor: User,
236 repo: RepoPath,
237 repo_id: String,
238 full_name: String,
239 installation_id: u64,
240}
241
242pub struct GithubApp {
243 db: D1Database,
244 sealer: Option<Sealer>,
245 identity: Fetcher,
246 work: Fetcher,
247 repos: Fetcher,
248 config: AppConfig,
249}
250
251fn fail<T>(code: FailureCode, message: impl Into<String>) -> Outcome<T> {
252 Outcome::fail(code, message)
253}
254
255/// Why `actor` may not do `capability` to a linked repository, if they may
256/// not. Without its visibility at hand, it is taken as private: whoever has
257/// no role on it is told it is not found, as for a private one, and the
258/// roles that act on it are never had through being public anyway.
259fn refused<T>(actor: &User, row: &LinkRow, capability: Capability) -> Option<Outcome<T>> {
260 let namespace = row.repo.split('/').next().unwrap_or_default();
261 let target = access::RepoRef { id: &row.repo_id, namespace, private: true };
262 match access::check(Some(actor), target, capability) {
263 Ok(()) => None,
264 Err(access::Denied::NotFound) => Some(fail(FailureCode::NotFound, "Repository not found.")),
265 Err(access::Denied::Forbidden) => Some(fail(FailureCode::Forbidden, access::needs(capability, &row.repo))),
266 }
267}
268
269fn null_or(value: Option<&str>) -> JsValue {
270 value.map_or(JsValue::NULL, Into::into)
271}
272
273fn number(value: u64) -> JsValue {
274 (value as f64).into()
275}
276
277const NOT_SET_UP: &str = "GitHub is not set up on this g1t.";
278
279impl GithubApp {
280 pub fn new(env: &Env) -> Result<Self> {
281 Ok(GithubApp {
282 db: env.d1("DB")?,
283 sealer: env.secret("INTEGRATIONS_KEY").ok().and_then(|key| Sealer::new(&key.to_string())),
284 identity: env.service("IDENTITY")?,
285 work: env.service("WORK")?,
286 repos: env.service("REPOS")?,
287 config: AppConfig::from_env(env),
288 })
289 }
290
291 async fn github(&self, method: Method, path: &str, token: &str, body: Option<Value>) -> Result<Answer> {
292 let authorization = format!("Bearer {token}");
293 let url = if path.starts_with("https://") { path.to_owned() } else { format!("{API}{path}") };
294 http::send(
295 method,
296 &url,
297 &[
298 ("authorization", &authorization),
299 ("accept", "application/vnd.github+json"),
300 ("x-github-api-version", "2022-11-28"),
301 ],
302 body.map(|body| body.to_string()),
303 )
304 .await
305 }
306
307 /// The person's GitHub user token, from identity.
308 async fn user_token(&self, user: &User) -> Result<Outcome<String>> {
309 g1t_kit::call(&self.identity, "github_user_token", &GithubUserTokenArgs { user_id: user.id.clone() }).await
310 }
311
312 /// An installation access token, from the cache or fresh from GitHub.
313 async fn installation_token(&self, installation_id: u64) -> Result<std::result::Result<String, String>> {
314 #[derive(Deserialize)]
315 struct Cached {
316 token: String,
317 expires_ms: f64,
318 }
319 let bound = format!("ghi:{installation_id}");
320 let now = now_ms();
321 let cached = self
322 .db
323 .prepare("SELECT token, expires_ms FROM github_tokens WHERE installation_id = ?")
324 .bind(&[number(installation_id)])?
325 .first::<Cached>(None)
326 .await?;
327 if let (Some(cached), Some(sealer)) = (cached, &self.sealer)
328 && cached.expires_ms as u64 > now + TOKEN_MARGIN_MS
329 && let Some(token) = sealer.open(&cached.token, &bound)
330 {
331 return Ok(Ok(token));
332 }
333 let Some(key) = self.config.private_key.as_deref().filter(|_| self.config.configured()) else {
334 return Ok(Err(NOT_SET_UP.to_owned()));
335 };
336 let jwt = github_jwt::app_jwt(self.config.issuer(), key, now / 1000).await?;
337 let answer = self
338 .github(Method::Post, &format!("/app/installations/{installation_id}/access_tokens"), &jwt, None)
339 .await?;
340 if !answer.ok() {
341 return Ok(Err(answer.problem("GitHub")));
342 }
343 let body = answer.json();
344 let Some(token) = body["token"].as_str().filter(|token| !token.is_empty()).map(str::to_owned) else {
345 return Ok(Err("GitHub sent no installation token.".to_owned()));
346 };
347 // GitHub's tokens last an hour; trust its own expiry when it says.
348 let expires = body["expires_at"].as_str().and_then(parse_time).unwrap_or(now + 60 * 60 * 1000);
349 if let Some(sealer) = &self.sealer {
350 self.db
351 .prepare(
352 "INSERT INTO github_tokens (installation_id, token, expires_ms) VALUES (?1, ?2, ?3)
353 ON CONFLICT (installation_id) DO UPDATE SET token = excluded.token, expires_ms = excluded.expires_ms",
354 )
355 .bind(&[number(installation_id), sealer.seal(&token, &bound).into(), (expires as f64).into()])?
356 .run()
357 .await?;
358 }
359 Ok(Ok(token))
360 }
361
362 async fn installation(&self, workspace: &str, id: u64) -> Result<Option<InstallationRow>> {
363 self.db
364 .prepare("SELECT * FROM github_installations WHERE workspace = ? AND id = ?")
365 .bind(&[workspace.into(), number(id)])?
366 .first::<InstallationRow>(None)
367 .await
368 }
369
370 async fn link(&self, repo_id: &str) -> Result<Option<LinkRow>> {
371 self.db
372 .prepare("SELECT * FROM github_repos WHERE repo_id = ?")
373 .bind(&[repo_id.into()])?
374 .first::<LinkRow>(None)
375 .await
376 }
377
378 async fn note(&self, repo_id: &str, problem: Option<&str>) -> Result<()> {
379 let sql = if problem.is_some() {
380 "UPDATE github_repos SET last_error = ?2 WHERE repo_id = ?1"
381 } else {
382 "UPDATE github_repos SET last_error = ?2, synced_at = ?3 WHERE repo_id = ?1"
383 };
384 let statement = self.db.prepare(sql);
385 let statement = if problem.is_some() {
386 statement.bind(&[repo_id.into(), null_or(problem)])?
387 } else {
388 statement.bind(&[repo_id.into(), JsValue::NULL, rfc3339(now_ms()).into()])?
389 };
390 statement.run().await?;
391 Ok(())
392 }
393
394 pub async fn status(&self, a: GithubStatusArgs) -> Result<Outcome<GithubAppStatus>> {
395 let workspace = a.workspace.to_lowercase();
396 if !a.viewer.is_member(&workspace) {
397 return Ok(fail(FailureCode::Forbidden, "Only members of the workspace can see its GitHub installations."));
398 }
399 if !self.config.configured() {
400 return Ok(Outcome::Ok(GithubAppStatus::default()));
401 }
402 let account: GithubAccountView =
403 g1t_kit::call(&self.identity, "github_account", &json!({ "user": a.viewer })).await?;
404 let installations = self
405 .db
406 .prepare("SELECT * FROM github_installations WHERE workspace = ? ORDER BY account")
407 .bind(&[workspace.as_str().into()])?
408 .all()
409 .await?
410 .results::<InstallationRow>()?;
411 Ok(Outcome::Ok(GithubAppStatus {
412 configured: true,
413 install_url: Some(self.config.install_url()),
414 linked: account.account.is_some_and(|account| account.authorized),
415 installations: installations.into_iter().map(Into::into).collect(),
416 }))
417 }
418
419 fn owner_only<T>(actor: &User, workspace: &str) -> Option<Outcome<T>> {
420 (actor.role_in(workspace) != Some(Role::Owner) || actor.kind != PrincipalKind::User).then(|| {
421 fail(FailureCode::Forbidden, "Only an owner of the workspace can add or remove GitHub accounts.")
422 })
423 }
424
425 /// Records an installation against a workspace, once the person's own
426 /// GitHub token shows it is one they can see.
427 pub async fn add_installation(&self, a: GithubInstallationArgs) -> Result<Outcome<GithubInstallation>> {
428 let workspace = a.workspace.to_lowercase();
429 if let Some(refused) = Self::owner_only(&a.actor, &workspace) {
430 return Ok(refused);
431 }
432 if !self.config.configured() {
433 return Ok(fail(FailureCode::NotFound, NOT_SET_UP));
434 }
435 let token = match self.user_token(&a.actor).await? {
436 Outcome::Ok(token) => token,
437 Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)),
438 };
439 let mut found = None;
440 for page in 1..=5 {
441 let answer = self
442 .github(Method::Get, &format!("/user/installations?per_page=100&page={page}"), &token, None)
443 .await?;
444 if !answer.ok() {
445 return Ok(fail(FailureCode::Conflict, answer.problem("GitHub")));
446 }
447 let body = answer.json();
448 let listed = body["installations"].as_array().cloned().unwrap_or_default();
449 found = listed.iter().find(|item| item["id"].as_u64() == Some(a.installation_id)).cloned();
450 if found.is_some() || listed.len() < 100 {
451 break;
452 }
453 }
454 let Some(item) = found else {
455 return Ok(fail(
456 FailureCode::Forbidden,
457 "Your GitHub account cannot see that installation. Install the app from your own GitHub account or an organization you manage.",
458 ));
459 };
460 let now = rfc3339(now_ms());
461 let row = InstallationRow {
462 id: a.installation_id,
463 workspace: workspace.clone(),
464 account: item["account"]["login"].as_str().unwrap_or_default().to_owned(),
465 account_type: item["account"]["type"].as_str().or(item["target_type"].as_str()).unwrap_or("User").to_owned(),
466 repository_selection: item["repository_selection"].as_str().unwrap_or("selected").to_owned(),
467 settings_url: item["html_url"].as_str().unwrap_or("https://github.com/settings/installations").to_owned(),
468 suspended_at: item["suspended_at"].as_str().map(str::to_owned),
469 created_at: now,
470 };
471 self.db
472 .prepare(
473 "INSERT INTO github_installations
474 (id, workspace, account, account_type, repository_selection, settings_url, suspended_at, added_by, created_at)
475 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
476 ON CONFLICT (id, workspace) DO UPDATE SET account = excluded.account,
477 repository_selection = excluded.repository_selection, settings_url = excluded.settings_url,
478 suspended_at = excluded.suspended_at",
479 )
480 .bind(&[
481 number(row.id),
482 row.workspace.as_str().into(),
483 row.account.as_str().into(),
484 row.account_type.as_str().into(),
485 row.repository_selection.as_str().into(),
486 row.settings_url.as_str().into(),
487 null_or(row.suspended_at.as_deref()),
488 a.actor.id.as_str().into(),
489 row.created_at.as_str().into(),
490 ])?
491 .run()
492 .await?;
493 Ok(Outcome::Ok(row.into()))
494 }
495
496 /// Forgets an installation in a workspace; its mirrors stop. The app
497 /// stays installed on GitHub until it is uninstalled there.
498 pub async fn remove_installation(&self, a: GithubInstallationArgs) -> Result<Outcome<bool>> {
499 let workspace = a.workspace.to_lowercase();
500 if let Some(refused) = Self::owner_only(&a.actor, &workspace) {
501 return Ok(refused);
502 }
503 self.db
504 .prepare("DELETE FROM github_installations WHERE workspace = ? AND id = ?")
505 .bind(&[workspace.as_str().into(), number(a.installation_id)])?
506 .run()
507 .await?;
508 self.db
509 .prepare("UPDATE github_repos SET mode = 'import' WHERE workspace = ? AND installation_id = ?")
510 .bind(&[workspace.as_str().into(), number(a.installation_id)])?
511 .run()
512 .await?;
513 Ok(Outcome::Ok(true))
514 }
515
516 pub async fn repositories(&self, a: GithubRepositoriesArgs) -> Result<Outcome<GithubRepositories>> {
517 let workspace = a.workspace.to_lowercase();
518 if !a.actor.is_member(&workspace) {
519 return Ok(fail(FailureCode::Forbidden, "Only members of the workspace can bring repositories into it."));
520 }
521 if self.installation(&workspace, a.installation_id).await?.is_none() {
522 return Ok(fail(FailureCode::NotFound, "That GitHub account is not connected to this workspace."));
523 }
524 let token = match self.user_token(&a.actor).await? {
525 Outcome::Ok(token) => token,
526 Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)),
527 };
528 let page = a.page.unwrap_or(1).max(1);
529 let answer = self
530 .github(
531 Method::Get,
532 &format!("/user/installations/{}/repositories?per_page={PER_PAGE}&page={page}", a.installation_id),
533 &token,
534 None,
535 )
536 .await?;
537 if !answer.ok() {
538 return Ok(fail(FailureCode::Conflict, answer.problem("GitHub")));
539 }
540 let body = answer.json();
541 let listed = body["repositories"].as_array().cloned().unwrap_or_default();
542 let ids: Vec<u64> = listed.iter().filter_map(|repo| repo["id"].as_u64()).collect();
543 let mut linked = HashMap::new();
544 if !ids.is_empty() {
545 #[derive(Deserialize)]
546 struct Linked {
547 github_repo_id: u64,
548 repo: String,
549 }
550 let marks = vec!["?"; ids.len()].join(", ");
551 let mut bind = vec![JsValue::from(workspace.as_str())];
552 bind.extend(ids.iter().map(|id| number(*id)));
553 let rows = self
554 .db
555 .prepare(format!(
556 "SELECT github_repo_id, repo FROM github_repos WHERE workspace = ? AND github_repo_id IN ({marks})"
557 ))
558 .bind(&bind)?
559 .all()
560 .await?
561 .results::<Linked>()?;
562 linked = rows.into_iter().map(|row| (row.github_repo_id, row.repo)).collect();
563 }
564 Ok(Outcome::Ok(GithubRepositories {
565 repositories: listed
566 .iter()
567 .filter_map(|repo| {
568 let id = repo["id"].as_u64()?;
569 Some(GithubRepository {
570 id,
571 full_name: repo["full_name"].as_str()?.to_owned(),
572 name: repo["name"].as_str()?.to_owned(),
573 private: repo["private"].as_bool().unwrap_or(true),
574 description: repo["description"].as_str().map(str::to_owned),
575 default_branch: repo["default_branch"].as_str().unwrap_or("main").to_owned(),
576 linked_to: linked.get(&id).cloned(),
577 })
578 })
579 .collect(),
580 total: body["total_count"].as_u64().unwrap_or(listed.len() as u64) as u32,
581 page,
582 per_page: PER_PAGE,
583 }))
584 }
585
586 /// Brings one GitHub repository across. Returns the issues still to
587 /// copy, which the caller does after answering.
588 pub async fn import(&self, a: GithubImportArgs) -> Result<(Outcome<GithubRepoLink>, Option<IssueJob>)> {
589 let workspace = a.workspace.to_lowercase();
590 let refuse = |code, message: &str| Ok((fail(code, message), None));
591 if !a.actor.is_member(&workspace) {
592 return refuse(FailureCode::Forbidden, "Only members of the workspace can bring repositories into it.");
593 }
594 if self.installation(&workspace, a.installation_id).await?.is_none() {
595 return refuse(FailureCode::NotFound, "That GitHub account is not connected to this workspace.");
596 }
597 let user_token = match self.user_token(&a.actor).await? {
598 Outcome::Ok(token) => token,
599 Outcome::Fail(failure) => return Ok((Outcome::Fail(failure), None)),
600 };
601 // Read with the person's token: only a repository both they and the
602 // installation can reach answers.
603 let answer = self
604 .github(Method::Get, &format!("/repositories/{}", a.github_repo_id), &user_token, None)
605 .await?;
606 if !answer.ok() {
607 return refuse(FailureCode::NotFound, "That GitHub repository is not one you and the app can both reach.");
608 }
609 let github = answer.json();
610 let (Some(full_name), Some(clone_url)) = (github["full_name"].as_str(), github["clone_url"].as_str()) else {
611 return refuse(FailureCode::Conflict, "GitHub did not describe the repository.");
612 };
613 let name = a
614 .name
615 .as_deref()
616 .map(str::trim)
617 .filter(|name| !name.is_empty())
618 .map(str::to_lowercase)
619 .unwrap_or_else(|| repo_name(github["name"].as_str().unwrap_or_default()));
620 if !is_valid_repo_name(&name) {
621 return refuse(FailureCode::Invalid, "Use letters, digits, dots, hyphens and underscores only.");
622 }
623 let token = match self.installation_token(a.installation_id).await? {
624 Ok(token) => token,
625 Err(reason) => return refuse(FailureCode::Conflict, &reason),
626 };
627 let created: Outcome<Repo> = g1t_kit::call(
628 &self.repos,
629 "create",
630 &CreateArgs {
631 owner: a.actor.clone(),
632 namespace: workspace.clone(),
633 name,
634 description: github["description"].as_str().map(|text| text.chars().take(200).collect()),
635 is_private: a.private.unwrap_or_else(|| github["private"].as_bool().unwrap_or(true)),
636 import_url: Some(clone_url.to_owned()),
637 import_token: Some(token),
638 },
639 )
640 .await?;
641 let repo = match created {
642 Outcome::Ok(repo) => repo,
643 Outcome::Fail(failure) => return Ok((Outcome::Fail(failure), None)),
644 };
645 let path = format!("{}/{}", repo.namespace, repo.name);
646 let now = rfc3339(now_ms());
647 self.db
648 .prepare(
649 "INSERT INTO github_repos
650 (repo_id, workspace, repo, installation_id, github_repo_id, full_name, mode, synced_at, created_by, created_at)
651 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?8)",
652 )
653 .bind(&[
654 repo.id.as_str().into(),
655 workspace.as_str().into(),
656 path.as_str().into(),
657 number(a.installation_id),
658 number(a.github_repo_id),
659 full_name.into(),
660 a.mode.as_str().into(),
661 now.as_str().into(),
662 a.actor.id.as_str().into(),
663 ])?
664 .run()
665 .await?;
666 let job = a.issues.then(|| IssueJob {
667 actor: a.actor.clone(),
668 repo: RepoPath {
669 namespace: repo.namespace.clone(),
670 name: repo.name.clone(),
671 },
672 repo_id: repo.id.clone(),
673 full_name: full_name.to_owned(),
674 installation_id: a.installation_id,
675 });
676 let link = self.link(&repo.id).await?.map(GithubRepoLink::from);
677 Ok((link.map_or_else(|| fail(FailureCode::NotFound, "The link was not kept."), Outcome::Ok), job))
678 }
679
680 /// Copies a repository's issues, oldest first, with their labels,
681 /// milestone and state. Pull requests are left: their branches came
682 /// with the git data.
683 pub async fn copy_issues(&self, job: IssueJob) -> Result<()> {
684 let token = match self.installation_token(job.installation_id).await? {
685 Ok(token) => token,
686 Err(reason) => return self.note(&job.repo_id, Some(&format!("Issues were not copied: {reason}"))).await,
687 };
688 let mut issues: Vec<Value> = Vec::new();
689 let mut more = false;
690 for page in 1..=4 {
691 let answer = self
692 .github(
693 Method::Get,
694 &format!("/repos/{}/issues?state=all&sort=created&direction=asc&per_page=100&page={page}", job.full_name),
695 &token,
696 None,
697 )
698 .await?;
699 if !answer.ok() {
700 return self.note(&job.repo_id, Some(&format!("Issues were not copied: {}", answer.problem("GitHub")))).await;
701 }
702 let listed = answer.json().as_array().cloned().unwrap_or_default();
703 let full = listed.len() == 100;
704 issues.extend(listed.into_iter().filter(|issue| issue.get("pull_request").is_none()));
705 if issues.len() >= MAX_ISSUES {
706 more = issues.len() > MAX_ISSUES || full;
707 issues.truncate(MAX_ISSUES);
708 break;
709 }
710 if !full {
711 break;
712 }
713 }
714 let authors: HashSet<u64> = issues.iter().filter_map(|issue| issue["user"]["id"].as_u64()).collect();
715 let names: HashMap<String, String> = g1t_kit::call(
716 &self.identity,
717 "github_usernames",
718 &GithubUsernamesArgs {
719 github_ids: authors.into_iter().collect(),
720 },
721 )
722 .await
723 .unwrap_or_default();
724 let mut copied = 0u32;
725 for issue in &issues {
726 let g1t_name = issue["user"]["id"].as_u64().and_then(|id| names.get(&id.to_string())).map(String::as_str);
727 let mut body = imported_header(issue, &job.full_name, g1t_name);
728 if let Some(text) = issue["body"].as_str().filter(|text| !text.trim().is_empty()) {
729 body.push_str("\n\n");
730 body.push_str(&http::shorten(text.trim(), 60_000));
731 }
732 let opened: Outcome<Issue> = g1t_kit::call(
733 &self.work,
734 "open_issue",
735 &OpenIssueArgs {
736 actor: job.actor.clone(),
737 repo: job.repo.clone(),
738 title: issue["title"].as_str().unwrap_or("Untitled").chars().take(200).collect(),
739 body,
740 labels: labels_of(issue),
741 checks: Vec::new(),
742 },
743 )
744 .await?;
745 let Outcome::Ok(opened) = opened else { continue };
746 if issue["state"].as_str() == Some("closed") {
747 let reason = if issue["state_reason"].as_str() == Some("not_planned") {
748 IssueReason::NotPlanned
749 } else {
750 IssueReason::Completed
751 };
752 let _: Outcome<Value> = g1t_kit::call(
753 &self.work,
754 "close_issue",
755 &IssueActionArgs {
756 actor: job.actor.clone(),
757 repo: job.repo.clone(),
758 number: opened.number,
759 reason: Some(reason),
760 },
761 )
762 .await?;
763 }
764 copied += 1;
765 }
766 self.db
767 .prepare("UPDATE github_repos SET issues_imported = ? WHERE repo_id = ?")
768 .bind(&[copied.into(), job.repo_id.as_str().into()])?
769 .run()
770 .await?;
771 if more {
772 self.note(&job.repo_id, Some(&format!("Copied the first {MAX_ISSUES} issues; the rest stay on GitHub.")))
773 .await?;
774 }
775 Ok(())
776 }
777
778 pub async fn link_for(&self, a: GithubLinkArgs) -> Result<Option<GithubRepoLink>> {
779 Ok(self.link(&a.repo_id).await?.map(Into::into))
780 }
781
782 /// Stops a mirror: the g1t repository keeps what it has and is its own.
783 pub async fn unlink_repo(&self, a: GithubUnlinkRepoArgs) -> Result<Outcome<bool>> {
784 let Some(row) = self.link(&a.repo_id).await? else {
785 return Ok(Outcome::Ok(false));
786 };
787 if let Some(refused) = refused(&a.actor, &row, Capability::ManageIntegrations) {
788 return Ok(refused);
789 }
790 self.db
791 .prepare("UPDATE github_repos SET mode = 'import' WHERE repo_id = ?")
792 .bind(&[a.repo_id.as_str().into()])?
793 .run()
794 .await?;
795 Ok(Outcome::Ok(true))
796 }
797
798 /// Copies refs now, in the link's direction.
799 async fn sync(&self, row: &LinkRow) -> Result<std::result::Result<Mirrored, String>> {
800 let direction = match GithubMode::parse(&row.mode) {
801 GithubMode::Mirror => MirrorDirection::Pull,
802 GithubMode::Push => MirrorDirection::Push,
803 GithubMode::Import => return Ok(Err("This repository was imported once; it is not mirrored.".to_owned())),
804 };
805 let token = match self.installation_token(row.installation_id).await? {
806 Ok(token) => token,
807 Err(reason) => {
808 self.note(&row.repo_id, Some(&reason)).await?;
809 return Ok(Err(reason));
810 }
811 };
812 let done: Outcome<Mirrored> = g1t_kit::call(
813 &self.repos,
814 "mirror",
815 &MirrorArgs {
816 repo_id: row.repo_id.clone(),
817 url: format!("https://github.com/{}.git", row.full_name),
818 token,
819 direction,
820 },
821 )
822 .await?;
823 Ok(match done {
824 Outcome::Ok(mirrored) => {
825 self.note(&row.repo_id, None).await?;
826 Ok(mirrored)
827 }
828 Outcome::Fail(failure) => {
829 self.note(&row.repo_id, Some(&failure.message)).await?;
830 Err(failure.message)
831 }
832 })
833 }
834
835 pub async fn sync_now(&self, a: GithubUnlinkRepoArgs) -> Result<Outcome<GithubRepoLink>> {
836 let Some(row) = self.link(&a.repo_id).await? else {
837 return Ok(fail(FailureCode::NotFound, "This repository is not linked to GitHub."));
838 };
839 // Syncing brings commits in, as pushing does.
840 if let Some(refused) = refused(&a.actor, &row, Capability::Push) {
841 return Ok(refused);
842 }
843 if let Err(reason) = self.sync(&row).await? {
844 return Ok(fail(FailureCode::Conflict, reason));
845 }
846 Ok(self.link(&a.repo_id).await?.map_or_else(|| fail(FailureCode::NotFound, "Gone."), |row| Outcome::Ok(row.into())))
847 }
848
849 // --- The webhook -----------------------------------------------------------
850
851 /// Checks and records a delivery. What it asks for is done by
852 /// `process`, after the answer has gone.
853 pub async fn receive(&self, a: &GithubReceiveArgs) -> Result<(Received, Option<(String, Value)>)> {
854 let answer = |status: u16, message: &str| Received {
855 status,
856 message: message.to_owned(),
857 };
858 let Some(secret) = self.config.webhook_secret.as_deref() else {
859 return Ok((answer(404, "GitHub is not set up on this g1t."), None));
860 };
861 if !authentic(secret, &a.body, &a.headers) {
862 return Ok((answer(401, "The request was not signed with the app's webhook secret."), None));
863 }
864 let event = a.headers.get("x-github-event").cloned().unwrap_or_default();
865 let Some(delivery) = a.headers.get("x-github-delivery").filter(|id| !id.is_empty() && id.len() <= 100) else {
866 return Ok((answer(400, "No X-GitHub-Delivery."), None));
867 };
868 let now = now_ms();
869 let fresh = self
870 .db
871 .prepare("INSERT OR IGNORE INTO github_deliveries (id, event, received_ms) VALUES (?, ?, ?) RETURNING id")
872 .bind(&[delivery.as_str().into(), event.as_str().into(), (now as f64).into()])?
873 .first::<Value>(None)
874 .await?;
875 if fresh.is_none() {
876 return Ok((answer(200, "Already received."), None));
877 }
878 let Ok(payload) = serde_json::from_str::<Value>(&a.body) else {
879 return Ok((answer(400, "The body is not JSON."), None));
880 };
881 Ok((answer(202, "Received."), Some((event, payload))))
882 }
883
884 pub async fn process(&self, event: &str, payload: &Value) -> Result<()> {
885 let action = payload["action"].as_str().unwrap_or_default();
886 let installation = payload["installation"]["id"].as_u64();
887 match (event, action) {
888 ("installation", "deleted") | ("installation", "suspend") | ("installation", "unsuspend") => {
889 let Some(id) = installation else { return Ok(()) };
890 match action {
891 "deleted" => self.installation_gone(id, "The GitHub App was uninstalled from this account.").await?,
892 "suspend" => {
893 self.db
894 .prepare("UPDATE github_installations SET suspended_at = ? WHERE id = ?")
895 .bind(&[rfc3339(now_ms()).into(), number(id)])?
896 .run()
897 .await?;
898 }
899 _ => {
900 self.db
901 .prepare("UPDATE github_installations SET suspended_at = NULL WHERE id = ?")
902 .bind(&[number(id)])?
903 .run()
904 .await?;
905 }
906 }
907 }
908 ("installation_repositories", _) => {
909 let Some(id) = installation else { return Ok(()) };
910 if let Some(selection) = payload["repository_selection"].as_str() {
911 self.db
912 .prepare("UPDATE github_installations SET repository_selection = ? WHERE id = ?")
913 .bind(&[selection.into(), number(id)])?
914 .run()
915 .await?;
916 }
917 for removed in payload["repositories_removed"].as_array().into_iter().flatten() {
918 if let Some(repo) = removed["id"].as_u64() {
919 self.mark_github_repo(repo, "GitHub no longer lets the app see this repository: add it back to the installation to keep it in step.")
920 .await?;
921 }
922 }
923 }
924 ("push", _) => {
925 let Some(repo) = payload["repository"]["id"].as_u64() else { return Ok(()) };
926 let rows = self
927 .db
928 .prepare("SELECT * FROM github_repos WHERE github_repo_id = ? AND mode = 'mirror'")
929 .bind(&[number(repo)])?
930 .all()
931 .await?
932 .results::<LinkRow>()?;
933 for row in rows {
934 if let Err(reason) = self.sync(&row).await? {
935 worker::console_log!("github mirror of {} not synced: {reason}", row.repo);
936 }
937 }
938 }
939 ("repository", "renamed") | ("repository", "transferred") => {
940 let (Some(repo), Some(full_name)) = (payload["repository"]["id"].as_u64(), payload["repository"]["full_name"].as_str()) else {
941 return Ok(());
942 };
943 self.db
944 .prepare("UPDATE github_repos SET full_name = ? WHERE github_repo_id = ?")
945 .bind(&[full_name.into(), number(repo)])?
946 .run()
947 .await?;
948 }
949 ("repository", "deleted") => {
950 if let Some(repo) = payload["repository"]["id"].as_u64() {
951 self.mark_github_repo(repo, "The repository was deleted on GitHub. The copy on g1t is kept.").await?;
952 self.db
953 .prepare("UPDATE github_repos SET mode = 'import' WHERE github_repo_id = ?")
954 .bind(&[number(repo)])?
955 .run()
956 .await?;
957 }
958 }
959 ("github_app_authorization", "revoked") => {
960 if let Some(github_id) = payload["sender"]["id"].as_u64() {
961 let _: u32 = g1t_kit::call(&self.identity, "github_revoked", &GithubRevokedArgs { github_id }).await?;
962 }
963 }
964 ("meta", "deleted") => {
965 let ids = self
966 .db
967 .prepare("SELECT DISTINCT id FROM github_installations")
968 .all()
969 .await?
970 .results::<Value>()?;
971 for row in ids {
972 if let Some(id) = row["id"].as_u64() {
973 self.installation_gone(id, "g1t's GitHub App was deleted.").await?;
974 }
975 }
976 }
977 _ => {}
978 }
979 // Delivery ids are kept a week, long enough for GitHub's retries.
980 self.db
981 .prepare("DELETE FROM github_deliveries WHERE received_ms < ?")
982 .bind(&[((now_ms().saturating_sub(DELIVERY_DAYS * 86_400_000)) as f64).into()])?
983 .run()
984 .await?;
985 Ok(())
986 }
987
988 async fn installation_gone(&self, id: u64, why: &str) -> Result<()> {
989 self.db.prepare("DELETE FROM github_installations WHERE id = ?").bind(&[number(id)])?.run().await?;
990 self.db.prepare("DELETE FROM github_tokens WHERE installation_id = ?").bind(&[number(id)])?.run().await?;
991 self.db
992 .prepare("UPDATE github_repos SET mode = 'import', last_error = ? WHERE installation_id = ? AND mode != 'import'")
993 .bind(&[why.into(), number(id)])?
994 .run()
995 .await?;
996 Ok(())
997 }
998
999 async fn mark_github_repo(&self, github_repo_id: u64, why: &str) -> Result<()> {
1000 self.db
1001 .prepare("UPDATE github_repos SET last_error = ? WHERE github_repo_id = ? AND mode != 'import'")
1002 .bind(&[why.into(), number(github_repo_id)])?
1003 .run()
1004 .await?;
1005 Ok(())
1006 }
1007
1008 /// A push on g1t: a repository GitHub follows is pushed out.
1009 pub async fn on_event(&self, event: &Event) -> Result<()> {
1010 if event.kind != "git.push" {
1011 return Ok(());
1012 }
1013 let Some(repo_id) = event.repo_id.as_deref() else {
1014 return Ok(());
1015 };
1016 if let Some(row) = self.link(repo_id).await?.filter(|row| row.mode == "push")
1017 && let Err(reason) = self.sync(&row).await?
1018 {
1019 worker::console_log!("github push mirror of {} failed: {reason}", row.repo);
1020 }
1021 Ok(())
1022 }
1023}
1024
1025/// Answers the GitHub methods; `None` for any other.
1026pub async fn route(method: &str, body: &Value, env: &Env, ctx: &Context) -> Option<Result<Response>> {
1027 if !method.starts_with("github_") {
1028 return None;
1029 }
1030 Some(handle(method, body.clone(), env, ctx).await)
1031}
1032
1033async fn handle(method: &str, body: Value, env: &Env, ctx: &Context) -> Result<Response> {
1034 let app = GithubApp::new(env)?;
1035 match method {
1036 "github_status" => reply(&app.status(args(body)?).await?),
1037 "github_add_installation" => reply(&app.add_installation(args(body)?).await?),
1038 "github_remove_installation" => reply(&app.remove_installation(args(body)?).await?),
1039 "github_repositories" => reply(&app.repositories(args(body)?).await?),
1040 "github_import" => {
1041 let (outcome, job) = app.import(args(body)?).await?;
1042 if let Some(job) = job {
1043 let env = env.clone();
1044 ctx.wait_until(async move {
1045 let Ok(app) = GithubApp::new(&env) else { return };
1046 if let Err(error) = app.copy_issues(job).await {
1047 worker::console_error!("github: copying issues failed: {error}");
1048 }
1049 });
1050 }
1051 reply(&outcome)
1052 }
1053 "github_link" => reply(&app.link_for(args(body)?).await?),
1054 "github_unlink_repo" => reply(&app.unlink_repo(args(body)?).await?),
1055 "github_sync" => reply(&app.sync_now(args(body)?).await?),
1056 "github_receive" => {
1057 let received: GithubReceiveArgs = args(body)?;
1058 let (answer, work) = app.receive(&received).await?;
1059 if let Some((event, payload)) = work {
1060 let env = env.clone();
1061 ctx.wait_until(async move {
1062 let Ok(app) = GithubApp::new(&env) else { return };
1063 if let Err(error) = app.process(&event, &payload).await {
1064 worker::console_error!("github: acting on a {event} delivery failed: {error}");
1065 }
1066 });
1067 }
1068 reply(&answer)
1069 }
1070 _ => Response::error("Unknown method", 404),
1071 }
1072}
1073
1074/// For the queue: pushes on g1t that GitHub follows.
1075pub async fn on_event(env: &Env, event: &Event) -> Result<()> {
1076 if event.kind != "git.push" || !AppConfig::from_env(env).configured() {
1077 return Ok(());
1078 }
1079 GithubApp::new(env)?.on_event(event).await
1080}
1081
1082#[cfg(test)]
1083mod tests {
1084 use super::*;
1085
1086 #[test]
1087 fn times_are_read_as_github_writes_them() {
1088 assert_eq!(parse_time("1970-01-01T00:00:00Z"), Some(0));
1089 assert_eq!(parse_time("2016-07-11T22:14:10Z"), Some(1_468_275_250_000));
1090 assert_eq!(parse_time("2024-02-29T12:00:00.5Z"), Some(1_709_208_000_500));
1091 assert_eq!(parse_time("not a time"), None);
1092 }
1093
1094 #[test]
1095 fn names_follow_g1ts_rules() {
1096 assert_eq!(repo_name("Hello-World"), "hello-world");
1097 assert_eq!(repo_name(".github"), "github");
1098 assert_eq!(repo_name("site.git"), "site");
1099 assert!(is_valid_repo_name(&repo_name("My Repo_1.x")));
1100 }
1101
1102 fn headers(signature: &str) -> HashMap<String, String> {
1103 HashMap::from([("x-hub-signature-256".to_owned(), signature.to_owned())])
1104 }
1105
1106 #[test]
1107 fn webhooks_must_carry_the_apps_signature() {
1108 // GitHub's documented example: secret "It's a Secret to Everybody",
1109 // payload "Hello, World!".
1110 let secret = "It's a Secret to Everybody";
1111 let signature = "sha256=757107ea0eb2509fc211221cce984b8a37570b6d7586c22c46f4379c8b043e17";
1112 assert!(authentic(secret, "Hello, World!", &headers(signature)));
1113 assert!(!authentic(secret, "Hello, World?", &headers(signature)));
1114 assert!(!authentic("another secret", "Hello, World!", &headers(signature)));
1115 // The bare hex form is not what GitHub sends; it is refused.
1116 assert!(!authentic(secret, "Hello, World!", &headers(signature.trim_start_matches("sha256="))));
1117 assert!(!authentic(secret, "Hello, World!", &HashMap::new()));
1118 }
1119
1120 #[test]
1121 fn imported_issues_say_where_they_came_from() {
1122 let issue = json!({
1123 "number": 12,
1124 "html_url": "https://github.com/acme/site/issues/12",
1125 "user": { "login": "octocat", "id": 1 },
1126 "created_at": "2024-01-02T03:04:05Z",
1127 "milestone": { "title": "v1" },
1128 "labels": [{ "name": "Bug" }, { "name": "bug" }, { "name": "x".repeat(41) }, "Help Wanted"],
1129 });
1130 let linked = imported_header(&issue, "acme/site", Some("octo"));
1131 assert!(linked.contains("[acme/site#12](https://github.com/acme/site/issues/12)"));
1132 assert!(linked.contains("@octo (@octocat on GitHub)"));
1133 assert!(linked.contains("on 2024-01-02"));
1134 assert!(linked.contains("Milestone: v1"));
1135 assert!(imported_header(&issue, "acme/site", None).contains("[@octocat](https://github.com/octocat) on GitHub"));
1136 assert_eq!(labels_of(&issue), vec!["bug", "help wanted"]);
1137 }
1138
1139 #[test]
1140 fn the_jwt_issuer_prefers_the_client_id() {
1141 let mut config = AppConfig {
1142 app_id: "5203641".to_owned(),
1143 slug: "g1t-sh".to_owned(),
1144 client_id: String::new(),
1145 private_key: Some("key".to_owned()),
1146 webhook_secret: None,
1147 };
1148 assert_eq!(config.issuer(), "5203641");
1149 config.client_id = "Iv23liZS94alfjIUn1eW".to_owned();
1150 assert_eq!(config.issuer(), "Iv23liZS94alfjIUn1eW");
1151 assert!(config.configured());
1152 assert_eq!(config.install_url(), "https://github.com/apps/g1t-sh/installations/new");
1153 config.private_key = None;
1154 assert!(!config.configured());
1155 }
1156}