diff --git a/.claude/INDEX.md b/.claude/INDEX.md new file mode 100644 index 0000000000000000000000000000000000000000..730f32950bbcb84d1d873e5d9c156b943ddf953d --- /dev/null +++ b/.claude/INDEX.md @@ -0,0 +1,455 @@ +# SourceHut documentation mirror — index + +Auto-generated by `.claude/scripts/build-index.sh`. Regenerated at the end of every `sourcehut-refresh` run. Do not edit by hand — changes will be overwritten. + +Read this **before** grepping. Each section gives you the canonical place to look for symbols in that repo. + +_Generated: 2026-05-19 05:22:31 UTC_ + +## Repos + +- `api.sr.ht` — master +- `builds.sr.ht` — 0.103.12 +- `core-go` — master +- `core.sr.ht` — 0.83.3 +- `forgeperf` — master +- `gensokyo` — master +- `git-am.io` — master +- `git-rebase.io` — master +- `git-send-email.io` — 0.1.17 +- `git.sr.ht` — 0.97.6 +- `go-away-config` — main +- `gql.sr.ht` — master +- `hub.sr.ht` — 0.27.6 +- `lists.sr.ht` — 0.70.6 +- `man.sr.ht` — 0.19.27 +- `meta.sr.ht` — 0.88.4 +- `pages.sr.ht` — 0.17.12 +- `paste.sr.ht` — 0.19.1 +- `sourcehut-go` — master +- `sourcehut-migrate` — v0.3.1 +- `sourcehut-ssh` — 0.5.0 +- `sourcehut.org` — master +- `sr.ht-apkbuilds` — master +- `sr.ht-docs` — master +- `sr.ht-nginx` — master +- `sr.ht-pkgbuilds` — master +- `srht.site` — master +- `status.sr.ht` — main +- `todo.sr.ht` — 0.84.4 + +## api.sr.ht + +_Tag:_ `(branch: master)` + +**Go files (root)**: auth.go, main.go + +--- + +## builds.sr.ht + +_Tag:_ `0.103.12` + +**GraphQL types** (`buildsrht/schema.graphqls`) + +- `type`: Artifact, EmailTrigger, Job, JobCursor, JobEvent, JobGroup, Log, Mutation, OAuthClient, PGPKey, Query, SSHKey, SecretCursor, SecretFile, Settings, Task, User, UserWebhookSubscription, Version, WebhookDelivery, WebhookDeliveryCursor, WebhookSubscriptionCursor, WebhookTrigger +- `input`: EmailTriggerInput, TriggerInput, UserWebhookInput, WebhookTriggerInput +- `enum`: AccessKind, AccessScope, JobStatus, TaskStatus, TriggerCondition, TriggerType, Visibility, WebhookEvent +- `interface`: Entity, Secret, Trigger, WebhookPayload, WebhookSubscription +- `scalar`: Binary, Cursor, File, Time + +**SQL tables** (`schema.sql`) + +artifact, gql_user_wh_delivery, gql_user_wh_sub, job, job_group, secret, task, trigger, user + +_Migrations: 2 files, latest `migrations/0002_drop_legacy_tables.sql`_ + +**Python blueprints** (`buildsrht/blueprints/`): admin, jobs, secrets, settings + +**Python GraphQL queries** (`buildsrht/graphql/`): builds, secrets + +**Go binaries** (`cmd/`): api, worker + +**Go packages**: `api, worker` + +--- + +## core-go + +_Tag:_ `(branch: master)` + +**Go binaries** (`cmd/`): token + +**Go files (root)**: placeholder.go + +**Go packages**: `auth, client, config, crypto, database, email, errors, feature, model, objects, redis, server, valid, webhooks` + +--- + +## core.sr.ht + +_Tag:_ `0.83.3` + +**Python modules** (`srht/`): api, assets, cache, config, crypto, database, debug, email, flagtype, gql_lexer, markdown, metrics, redis, rid, search, validation + +**Python subpackages** (`srht/`): app, ariadne, graphql, icons, oauth, templates, webhook + +--- + +## forgeperf + +_Tag:_ `(branch: master)` + +**Go files (root)**: main.go + +**Go packages**: `schema` + +--- + +## gensokyo + +_Tag:_ `(branch: master)` + +--- + +## git-am.io + +_Tag:_ `(branch: master)` + +--- + +## git-rebase.io + +_Tag:_ `(branch: master)` + +--- + +## git-send-email.io + +_Tag:_ `0.1.17` + +--- + +## git.sr.ht + +_Tag:_ `0.97.6` + +**GraphQL types** (`gitsrht/schema.graphqls`) + +- `type`: ACL, ACLCursor, Artifact, ArtifactCursor, BinaryBlob, Commit, CommitCursor, Features, GitEvent, GitWebhookSubscription, Mutation, OAuthClient, Query, Redirect, Reference, ReferenceCursor, Repository, RepositoryCursor, RepositoryEvent, Settings, Signature, Tag, TextBlob, Trailer, Tree, TreeEntry, TreeEntryCursor, UpdatedRef, User, UserWebhookSubscription, Version, WebhookDelivery, WebhookDeliveryCursor, WebhookSubscriptionCursor +- `input`: Filter, GitEventInput, GitWebhookInput, RepoInput, UpdatedRefInput, UserWebhookInput +- `enum`: AccessKind, AccessMode, AccessScope, ObjectType, Visibility, WebhookEvent +- `interface`: Blob, Entity, Object, WebhookPayload, WebhookSubscription +- `scalar`: Cursor, Time, URL, Upload + +**SQL tables** (`schema.sql`) + +access, artifacts, gql_git_wh_delivery, gql_git_wh_sub, gql_user_wh_delivery, gql_user_wh_sub, redirect, repository, user + +_Migrations: 6 files, latest `migrations/0006_add_resource_ids.sql`_ + +**Python blueprints** (`gitsrht/blueprints/`): artifacts, email, manage, public, repo + +**Python GraphQL queries** (`gitsrht/graphql/`): artifacts, manage, settings + +**Go binaries** (`cmd/`): api, http-clone, shell, update-hook + +**Go packages**: `update-hook` + +--- + +## go-away-config + +_Tag:_ `(branch: main)` + +--- + +## gql.sr.ht + +_Tag:_ `(branch: master)` + +**Go files (root)**: directives.go, email.go, server.go + +**Go packages**: `auth, client, config, crypto, database, model, redis` + +--- + +## hub.sr.ht + +_Tag:_ `0.27.6` + +**GraphQL types** (`hubsrht/schema.graphqls`) + +- `type`: Features, MailingList, MailingListCursor, Mutation, Project, ProjectCursor, Query, SourceRepo, SourceRepoCursor, Tracker, TrackerCursor, User, Version +- `enum`: AccessKind, AccessScope, RepoType, Visibility +- `interface`: Entity, ProjectResource +- `scalar`: Cursor, Time, Upload + +**SQL tables** (`schema.sql`) + +event, event_project_association, features, mailing_list, project, redirect, source_repo, tracker, user, user_webhooks + +_Migrations: 8 files, latest `migrations/0008_repo-type_enum.sql`_ + +**Python blueprints** (`hubsrht/blueprints/`): mailing_lists, projects, public, sources, trackers, users, webhooks + +**Go packages**: `api` + +--- + +## lists.sr.ht + +_Tag:_ `0.70.6` + +**GraphQL types** (`api/graph/schema.graphqls`) + +- `type`: ActivitySubscriptionCursor, ByteRange, Email, EmailCursor, EmailEvent, GeneralACL, Mailbox, MailingList, MailingListACL, MailingListACLCursor, MailingListCursor, MailingListEvent, MailingListSubscription, MailingListWebhookSubscription, Mutation, OAuthClient, Patch, Patchset, PatchsetCursor, PatchsetEvent, PatchsetTool, Preferences, Query, Thread, ThreadBlock, ThreadCursor, Trailer, User, UserWebhookSubscription, Version, WebhookDelivery, WebhookDeliveryCursor, WebhookSubscriptionCursor +- `input`: ACLInput, MailingListInput, MailingListWebhookInput, PreferencesInput, UserWebhookInput +- `enum`: AccessKind, AccessScope, PatchsetStatus, ToolIcon, Visibility, WebhookEvent +- `interface`: ACL, ActivitySubscription, Entity, WebhookPayload, WebhookSubscription +- `scalar`: ConfirmationToken, Cursor, Time, URL, Upload + +**SQL tables** (`schema.sql`) + +access, email, gql_list_wh_delivery, gql_list_wh_sub, gql_user_wh_delivery, gql_user_wh_sub, list, mirror, patchset, patchset_tool, subscription, subscription_request, user + +_Migrations: 10 files, latest `migrations/0010_add_supersedes_id.sql`_ + +**Python blueprints** (`listssrht/blueprints/`): archives, patches, settings, user + +**Python GraphQL queries** (`listssrht/graphql/`): queries + +**Go packages**: `api, ingress` + +--- + +## man.sr.ht + +_Tag:_ `0.19.27` + +**GraphQL types** (`api/graph/schema.graphqls`) + +- `type`: Mutation, Query, Version +- `scalar`: Cursor, Time, Upload + +**SQL tables** (`schema.sql`) + +backing_repo, root_wiki, user, wiki + +_Migrations: 3 files, latest `migrations/0003_remove_backing_repo.sql`_ + +**Python blueprints** (`mansrht/blueprints/`): create, html, manage, public + +**Go binaries** (`cmd/`): api + +--- + +## meta.sr.ht + +_Tag:_ `0.88.4` + +**GraphQL types** (`metasrht/schema.graphqls`) + +- `type`: AuditLogCursor, AuditLogEntry, BillingAddress, BillingSubscription, Features, Invoice, InvoiceCursor, LoginSecurity, Mutation, OAuthClient, OAuthClientRegistration, OAuthGrant, OAuthGrantRegistration, OAuthPersonalToken, OAuthPersonalTokenRegistration, PGPKey, PGPKeyCursor, PGPKeyEvent, PaymentMethod, PaymentOutcome, Product, ProductPrice, ProfileUpdateEvent, ProfileWebhookSubscription, Query, SSHKey, SSHKeyCursor, SSHKeyEvent, StripePaymentIntent, StripeSetupIntent, TOTPConfig, User, Version, WebhookDelivery, WebhookDeliveryCursor, WebhookSubscriptionCursor +- `input`: BillingAddressInput, ProductPriceInput, ProfileWebhookInput, UpdateBillingSubscriptionInput, UserInput +- `enum`: AccessKind, AccessScope, Currency, PaymentIntentStatus, PaymentInterval, PaymentStatus, SetupIntentStatus, SubscriptionStatus, UserType, WebhookEvent +- `interface`: Entity, PaymentIntent, SetupIntent, WebhookPayload, WebhookSubscription +- `scalar`: Country, Cursor, Time, URL, Upload + +**SQL tables** (`schema.sql`) + +audit_log_entry, billing_address, gql_profile_wh_delivery, gql_profile_wh_sub, invoice, oauth2_client, oauth2_grant, oauthclient, oauthtoken, payment_method, pgpkey, product, product_price, reserved_usernames, sshkey, subscription, user, user_auth_factor, user_email_change, user_notes, user_password_change, user_payment_processor, user_registration, user_webhook_delivery, user_webhook_subscription, webhook_subscription + +_Migrations: 12 files, latest `migrations/0012_make_fingerprint_nullable.sql`_ + +**Python blueprints** (`metasrht/blueprints/`): auth, billing, keys, oauth2, privacy, profile, security, users + +**Python GraphQL queries** (`metasrht/graphql/`): admin, auth, billing, email, keys, oauth2, profile, security + +**Go binaries** (`cmd/`): api + +--- + +## pages.sr.ht + +_Tag:_ `0.17.12` + +**GraphQL types** (`graph/schema.graphqls`) + +- `type`: Mutation, OAuthClient, Query, Site, SiteACL, SiteACLCursor, SiteCursor, SiteEvent, User, UserWebhookSubscription, Version, WebhookDelivery, WebhookDeliveryCursor, WebhookSubscriptionCursor +- `input`: ACLInput, FileConfig, FileOptions, SiteConfig, UserWebhookInput +- `enum`: AccessKind, AccessScope, Protocol, WebhookEvent +- `interface`: ACL, Entity, WebhookPayload, WebhookSubscription +- `scalar`: Cursor, Time, Upload + +**SQL tables** (`schema.sql`) + +gql_user_wh_delivery, gql_user_wh_sub, site_access, sites, user + +_Migrations: 1 files, latest `migrations/0001_switch_to_brant.sql`_ + +**Go binaries** (`cmd/`): daily, pages.sr.ht + +**Go packages**: `account, graph, webhooks` + +--- + +## paste.sr.ht + +_Tag:_ `0.19.1` + +**GraphQL types** (`pastesrht/schema.graphqls`) + +- `type`: File, Mutation, OAuthClient, Paste, PasteCursor, PasteEvent, Query, User, UserWebhookSubscription, Version, WebhookDelivery, WebhookDeliveryCursor, WebhookSubscriptionCursor +- `input`: UserWebhookInput +- `enum`: AccessKind, AccessScope, Visibility, WebhookEvent +- `interface`: Entity, WebhookPayload, WebhookSubscription +- `scalar`: Cursor, Time, URL, Upload + +**SQL tables** (`schema.sql`) + +blob, gql_user_wh_delivery, gql_user_wh_sub, paste, paste_file, user + +_Migrations: 3 files, latest `migrations/0003_drop_legacy_tables.sql`_ + +**Python blueprints** (`pastesrht/blueprints/`): public + +**Python GraphQL queries** (`pastesrht/graphql/`): mutations + +**Go binaries** (`cmd/`): api + +--- + +## sourcehut-go + +_Tag:_ `(branch: master)` + +**Go binaries** (`cmd/`): git.sr.ht, meta.sr.ht + +**Go files (root)**: client.go, config.go, doc.go, entry.go, resources.go + +**Go packages**: `git.sr.ht, meta.sr.ht` + +--- + +## sourcehut-migrate + +_Tag:_ `v0.3.1` + +**Go binaries** (`cmd/`): sourcehut-migrate + +--- + +## sourcehut-ssh + +_Tag:_ `0.5.0` + +**Go binaries** (`cmd/`): sourcehut-ssh + +**Go packages**: `dispatch, handler, meta, shell, ssh` + +--- + +## sourcehut.org + +_Tag:_ `(branch: master)` + +--- + +## sr.ht-apkbuilds + +_Tag:_ `(branch: master)` + +--- + +## sr.ht-docs + +_Tag:_ `(branch: master)` + +--- + +## sr.ht-nginx + +_Tag:_ `(branch: master)` + +--- + +## sr.ht-pkgbuilds + +_Tag:_ `(branch: master)` + +--- + +## srht.site + +_Tag:_ `(branch: master)` + +--- + +## status.sr.ht + +_Tag:_ `(branch: main)` + +--- + +## todo.sr.ht + +_Tag:_ `0.84.4` + +**GraphQL types** (`todosrht/schema.graphqls`) + +- `type`: ACLCursor, ActivitySubscriptionCursor, Assignment, Comment, Created, DefaultACL, EmailAddress, Event, EventCreated, EventCursor, ExternalUser, Label, LabelCursor, LabelEvent, LabelUpdate, Mutation, OAuthClient, Preferences, Query, StatusChange, Ticket, TicketCursor, TicketDeletedEvent, TicketEvent, TicketMention, TicketSubscription, TicketWebhookSubscription, Tracker, TrackerACL, TrackerCursor, TrackerEvent, TrackerSubscription, TrackerWebhookSubscription, User, UserMention, UserWebhookSubscription, Version, WebhookDelivery, WebhookDeliveryCursor, WebhookSubscriptionCursor +- `input`: ACLInput, ImportInput, PreferencesInput, SubmitCommentEmailInput, SubmitCommentInput, SubmitTicketEmailInput, SubmitTicketInput, TicketWebhookInput, TrackerInput, TrackerWebhookInput, UpdateLabelInput, UpdateStatusInput, UpdateTicketInput, UserWebhookInput +- `enum`: AccessKind, AccessScope, Authenticity, EmailCmd, EventType, TicketResolution, TicketStatus, Visibility, WebhookEvent +- `interface`: ACL, ActivitySubscription, Entity, EventDetail, WebhookPayload, WebhookSubscription +- `scalar`: Cursor, Time, URL, Upload + +**SQL tables** (`schema.sql`) + +event, event_notification, gql_ticket_wh_delivery, gql_ticket_wh_sub, gql_tracker_wh_delivery, gql_tracker_wh_sub, gql_user_wh_delivery, gql_user_wh_sub, label, participant, redirect, ticket, ticket_assignee, ticket_comment, ticket_label, ticket_subscription, tracker, user, user_access + +_Migrations: 5 files, latest `migrations/0005_add_resource_ids.sql`_ + +**Python blueprints** (`todosrht/blueprints/`): html, settings, ticket, tracker + +**Python GraphQL queries** (`todosrht/graphql/`): lmtp, settings, tickets, trackers + +**Go binaries** (`cmd/`): api + +**Go packages**: `api` + +--- + +## Cross-repo GraphQL type map + +When the same type name appears in multiple repos, it is a *different* type in each — they are not federated as one. Use this map to pick the right service before reading. + +- `ACLCursor` → git.sr.ht, todo.sr.ht +- `ACLInput` → lists.sr.ht, pages.sr.ht, todo.sr.ht +- `AccessKind` → builds.sr.ht, git.sr.ht, hub.sr.ht, lists.sr.ht, meta.sr.ht, pages.sr.ht, paste.sr.ht, todo.sr.ht +- `AccessScope` → builds.sr.ht, git.sr.ht, hub.sr.ht, lists.sr.ht, meta.sr.ht, pages.sr.ht, paste.sr.ht, todo.sr.ht +- `ActivitySubscriptionCursor` → lists.sr.ht, todo.sr.ht +- `Artifact` → builds.sr.ht, git.sr.ht +- `Features` → git.sr.ht, hub.sr.ht, meta.sr.ht +- `MailingList` → hub.sr.ht, lists.sr.ht +- `MailingListCursor` → hub.sr.ht, lists.sr.ht +- `Mutation` → builds.sr.ht, git.sr.ht, hub.sr.ht, lists.sr.ht, man.sr.ht, meta.sr.ht, pages.sr.ht, paste.sr.ht, todo.sr.ht +- `OAuthClient` → builds.sr.ht, git.sr.ht, lists.sr.ht, meta.sr.ht, pages.sr.ht, paste.sr.ht, todo.sr.ht +- `PGPKey` → builds.sr.ht, meta.sr.ht +- `Preferences` → lists.sr.ht, todo.sr.ht +- `PreferencesInput` → lists.sr.ht, todo.sr.ht +- `Query` → builds.sr.ht, git.sr.ht, hub.sr.ht, lists.sr.ht, man.sr.ht, meta.sr.ht, pages.sr.ht, paste.sr.ht, todo.sr.ht +- `SSHKey` → builds.sr.ht, meta.sr.ht +- `Settings` → builds.sr.ht, git.sr.ht +- `Tracker` → hub.sr.ht, todo.sr.ht +- `TrackerCursor` → hub.sr.ht, todo.sr.ht +- `Trailer` → git.sr.ht, lists.sr.ht +- `User` → builds.sr.ht, git.sr.ht, hub.sr.ht, lists.sr.ht, meta.sr.ht, pages.sr.ht, paste.sr.ht, todo.sr.ht +- `UserWebhookInput` → builds.sr.ht, git.sr.ht, lists.sr.ht, pages.sr.ht, paste.sr.ht, todo.sr.ht +- `UserWebhookSubscription` → builds.sr.ht, git.sr.ht, lists.sr.ht, pages.sr.ht, paste.sr.ht, todo.sr.ht +- `Version` → builds.sr.ht, git.sr.ht, hub.sr.ht, lists.sr.ht, man.sr.ht, meta.sr.ht, pages.sr.ht, paste.sr.ht, todo.sr.ht +- `Visibility` → builds.sr.ht, git.sr.ht, hub.sr.ht, lists.sr.ht, paste.sr.ht, todo.sr.ht +- `WebhookDelivery` → builds.sr.ht, git.sr.ht, lists.sr.ht, meta.sr.ht, pages.sr.ht, paste.sr.ht, todo.sr.ht +- `WebhookDeliveryCursor` → builds.sr.ht, git.sr.ht, lists.sr.ht, meta.sr.ht, pages.sr.ht, paste.sr.ht, todo.sr.ht +- `WebhookEvent` → builds.sr.ht, git.sr.ht, lists.sr.ht, meta.sr.ht, pages.sr.ht, paste.sr.ht, todo.sr.ht +- `WebhookSubscriptionCursor` → builds.sr.ht, git.sr.ht, lists.sr.ht, meta.sr.ht, pages.sr.ht, paste.sr.ht, todo.sr.ht diff --git a/.claude/scripts/build-index.sh b/.claude/scripts/build-index.sh new file mode 100755 index 0000000000000000000000000000000000000000..010b59b76f98ebf898d2c4c304955ad2747063e3 --- /dev/null +++ b/.claude/scripts/build-index.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# Regenerate .claude/INDEX.md — a per-service inventory of GraphQL types, SQL +# tables, Python blueprints, and notable Go packages. Read by the +# sourcehut-lookup skill as the first step of any documentation lookup. +# +# Runs in seconds. Safe to re-run. Called as the final step of sourcehut-refresh. + +set -u +cd "$(dirname "$0")/../.." || exit 1 +ROOT="$PWD" +OUT="$ROOT/.claude/INDEX.md" + +# Comma-join stdin lines, sorted unique. +join_csv() { sort -u | paste -sd',' - | sed 's/,/, /g'; } + +# Identify each direct child that is a git repo. +mapfile -t REPOS < <( + find . -mindepth 2 -maxdepth 2 -name .git -type d 2>/dev/null \ + | sed -E 's:^\./([^/]+)/\.git$:\1:' \ + | sort +) + +tmp=$(mktemp) +trap 'rm -f "$tmp"' EXIT + +{ + echo "# SourceHut documentation mirror — index" + echo + echo "Auto-generated by \`.claude/scripts/build-index.sh\`. Regenerated at the end of every \`sourcehut-refresh\` run. Do not edit by hand — changes will be overwritten." + echo + echo "Read this **before** grepping. Each section gives you the canonical place to look for symbols in that repo." + echo + echo "_Generated: $(date -u +'%Y-%m-%d %H:%M:%S UTC')_" + echo + echo "## Repos" + echo + for repo in "${REPOS[@]}"; do + tag=$(git -C "$repo" describe --tags --exact-match 2>/dev/null \ + || git -C "$repo" rev-parse --abbrev-ref HEAD 2>/dev/null) + echo "- \`$repo\` — $tag" + done + echo +} > "$tmp" + +extract_one() { + local repo="$1" + local tag + tag=$(git -C "$repo" describe --tags --exact-match 2>/dev/null \ + || echo "(branch: $(git -C "$repo" rev-parse --abbrev-ref HEAD 2>/dev/null))") + + echo "## $repo" + echo + echo "_Tag:_ \`$tag\`" + echo + + # GraphQL types — typically /api/graph/schema.graphqls + local graphqls + graphqls=$(find "$repo" -name '*.graphqls' -not -path '*/node_modules/*' 2>/dev/null) + if [ -n "$graphqls" ]; then + local gqlpath + gqlpath=$(printf '%s\n' "$graphqls" | head -n1 | sed "s|^$repo/||") + echo "**GraphQL types** (\`$gqlpath\`)" + echo + for kind in type input enum interface union scalar; do + local names + names=$(grep -rhE "^${kind} [A-Z][A-Za-z0-9_]*" $graphqls 2>/dev/null \ + | awk -v k="$kind" '{print $2}' \ + | sed 's/[({].*//' \ + | join_csv) + [ -n "$names" ] && echo "- \`${kind}\`: $names" + done + echo + fi + + # SQL tables — /schema.sql + if [ -f "$repo/schema.sql" ]; then + local tables + tables=$(grep -hE '^CREATE TABLE' "$repo/schema.sql" 2>/dev/null \ + | sed -E 's/CREATE TABLE (IF NOT EXISTS )?"?([A-Za-z0-9_]+)"?.*/\2/' \ + | join_csv) + if [ -n "$tables" ]; then + echo "**SQL tables** (\`schema.sql\`)" + echo + echo "$tables" + echo + fi + # Migrations count + if [ -d "$repo/migrations" ]; then + local mcount latest + mcount=$(find "$repo/migrations" -name '*.sql' 2>/dev/null | wc -l | tr -d ' ') + latest=$(find "$repo/migrations" -name '*.sql' 2>/dev/null | sort | tail -n1 | sed "s|$repo/||") + [ "$mcount" -gt 0 ] && echo "_Migrations: $mcount files, latest \`$latest\`_" && echo + fi + fi + + # Python blueprints — /srht/blueprints/*.py (require a prefix so we + # don't match core.sr.ht's bare `srht/` — that's handled separately below) + local py_pkg + py_pkg=$(find "$repo" -maxdepth 2 -type d -name '?*srht' -not -path '*/contrib/*' 2>/dev/null | head -n1) + if [ -n "$py_pkg" ] && [ -d "$py_pkg/blueprints" ]; then + local bps + bps=$(find "$py_pkg/blueprints" -maxdepth 1 -name '*.py' -not -name '__init__.py' 2>/dev/null \ + | xargs -n1 basename 2>/dev/null \ + | sed 's/\.py$//' \ + | join_csv) + [ -n "$bps" ] && echo "**Python blueprints** (\`$(basename "$py_pkg")/blueprints/\`): $bps" && echo + fi + # Python GraphQL client queries + if [ -n "$py_pkg" ] && [ -d "$py_pkg/graphql" ]; then + local pyql + pyql=$(find "$py_pkg/graphql" -maxdepth 1 -name '*.graphql' 2>/dev/null \ + | xargs -n1 basename 2>/dev/null | sed 's/\.graphql$//' | join_csv) + [ -n "$pyql" ] && echo "**Python GraphQL queries** (\`$(basename "$py_pkg")/graphql/\`): $pyql" && echo + fi + + # Notable Go subdirs — cmd/*, top-level dirs with *.go, plus root-level .go files + if [ -f "$repo/go.mod" ]; then + local cmds gopkgs rootgo + cmds=$(find "$repo/cmd" -mindepth 1 -maxdepth 1 -type d 2>/dev/null \ + | xargs -n1 basename 2>/dev/null | join_csv) + [ -n "$cmds" ] && echo "**Go binaries** (\`cmd/\`): $cmds" && echo + # Repo-root .go files (e.g. api.sr.ht has main.go + auth.go at top) + rootgo=$(find "$repo" -maxdepth 1 -name '*.go' 2>/dev/null \ + | xargs -n1 basename 2>/dev/null | sort | join_csv) + [ -n "$rootgo" ] && echo "**Go files (root)**: $rootgo" && echo + # Top-level Go packages + gopkgs=$(find "$repo" -mindepth 2 -maxdepth 2 -name '*.go' -not -path '*/generated*' 2>/dev/null \ + | sed -E "s:^$repo/::; s:/[^/]+\.go$::" \ + | sort -u \ + | grep -vE '^(cmd|migrations|node_modules)' \ + | head -20 \ + | join_csv) + [ -n "$gopkgs" ] && echo "**Go packages**: \`$gopkgs\`" && echo + fi + + # Plain Python package (no `srht` directory) — e.g. core.sr.ht ships `srht/` + if [ -z "$py_pkg" ] && [ -d "$repo/srht" ]; then + local srht_mods + srht_mods=$(find "$repo/srht" -maxdepth 1 -name '*.py' -not -name '__init__.py' 2>/dev/null \ + | xargs -n1 basename 2>/dev/null | sed 's/\.py$//' | join_csv) + local srht_subs + srht_subs=$(find "$repo/srht" -mindepth 1 -maxdepth 1 -type d -not -name '__pycache__' 2>/dev/null \ + | xargs -n1 basename 2>/dev/null | join_csv) + [ -n "$srht_mods" ] && echo "**Python modules** (\`srht/\`): $srht_mods" && echo + [ -n "$srht_subs" ] && echo "**Python subpackages** (\`srht/\`): $srht_subs" && echo + fi + + echo "---" + echo +} + +for repo in "${REPOS[@]}"; do + extract_one "$repo" >> "$tmp" +done + +# Cross-repo symbol map: which repos define each common GraphQL type name? +{ + echo "## Cross-repo GraphQL type map" + echo + echo "When the same type name appears in multiple repos, it is a *different* type in each — they are not federated as one. Use this map to pick the right service before reading." + echo + declare -A type_to_repos + for repo in "${REPOS[@]}"; do + while IFS= read -r name; do + [ -z "$name" ] && continue + type_to_repos[$name]+="$repo " + done < <( + find "$repo" -name '*.graphqls' -exec grep -hE '^(type|input|enum) [A-Z][A-Za-z0-9_]*' {} + 2>/dev/null \ + | awk '{print $2}' | sed 's/[({].*//' | sort -u + ) + done + # Print only types defined in >=2 repos (the interesting cases). + for name in $(printf '%s\n' "${!type_to_repos[@]}" | sort); do + repos="${type_to_repos[$name]}" + count=$(printf '%s\n' $repos | wc -l | tr -d ' ') + if [ "$count" -ge 2 ]; then + echo "- \`$name\` → $(printf '%s, ' $repos | sed 's/, $//')" + fi + done +} >> "$tmp" + +mv "$tmp" "$OUT" +trap - EXIT +echo "Wrote $OUT ($(wc -l < "$OUT" | tr -d ' ') lines, $(wc -c < "$OUT" | tr -d ' ') bytes)" diff --git a/.claude/skills/sourcehut-lookup/SKILL.md b/.claude/skills/sourcehut-lookup/SKILL.md new file mode 100644 index 0000000000000000000000000000000000000000..e0b877159c4f930ae42dfc390fb202b549daf7a9 --- /dev/null +++ b/.claude/skills/sourcehut-lookup/SKILL.md @@ -0,0 +1,83 @@ +--- +name: sourcehut-lookup +description: Fast traversal of the SourceHut documentation mirror at ~/data/home/sourcehut. Use when the user asks how any SourceHut feature works (auth, webhooks, GraphQL, builds, OAuth, federation, jobs, ACLs, repos, lists, tickets, migrations, etc.) or asks to find/cite the canonical implementation of a SourceHut concept. +--- + +# sourcehut-lookup + +The workspace at `~/data/home/sourcehut` is a read-only mirror of every SourceHut subproject pinned to its latest tag. Treat it as documentation: when a question is about *how SourceHut works*, the answer lives here, not in your training data. + +## Mental model + +Every user-facing service (`builds.sr.ht`, `git.sr.ht`, `hub.sr.ht`, `lists.sr.ht`, `meta.sr.ht`, `paste.sr.ht`, `todo.sr.ht`, `man.sr.ht`, `pages.sr.ht`) has the same shape. To understand any feature, read these in this order: + +1. **The GraphQL contract** — `/api/graph/schema.graphqls`. This is the authoritative description of what the service exposes. Read it first. +2. **The SQL schema** — `/schema.sql` and the newest `/migrations/*.sql`. This is what the service actually stores. +3. **Go resolvers** — `/api/graph/*.go` (request handlers) and `/api/loaders/*.go` (batched DB lookups). This is *the* API implementation. +4. **Python web tier** — `/srht/blueprints/*.py` (Flask routes, HTML UI) and `/srht/templates/`. The Python side calls its own GraphQL API; it is rarely the source of truth. + +Cross-cutting concerns (auth, webhooks, OAuth, config, GraphQL helpers, templates) live in **`core.sr.ht/srht/`** (Python) and **`core-go/`** (Go). Always check there before assuming a service implements something locally. + +The aggregator **`api.sr.ht`** is a pure Go gateway that federates each `-api` into one GraphQL endpoint via the `thistle` library. Look here only for federation, internal auth (`auth.go`), and request routing. + +## Concept → where to look (search this map BEFORE grepping blind) + +| Concept | Primary paths | +| -------------------------------- | --------------------------------------------------------------------------------------------------- | +| OAuth 2.0 (server side) | `meta.sr.ht/metasrht/blueprints/oauth*.py`, `meta.sr.ht/api/graph/` (PersonalAccessToken, OAuthClient) | +| OAuth (client / token validation)| `core.sr.ht/srht/oauth/`, `core-go/auth/` | +| Internal service-to-service auth | `core-go/auth/internal.go`, `api.sr.ht/auth.go` (`InternalAuthTransport`) | +| GraphQL federation gateway | `api.sr.ht/main.go`, `api.sr.ht/auth.go`; library `git.sr.ht/~adnano/thistle` | +| GraphQL server scaffolding | `core-go/server/`, per-service `api/graph/generate.go`, `gqlgen.yml`, generated `api/graph/api/generated.go` | +| DataLoader batching | `/api/loaders/*_gen.go` (generated), `/api/loaders/generate.go` (`//go:generate` directives) | +| Webhooks (new GraphQL-native) | `core-go/webhooks/`, `/api/webhooks/` | +| Webhooks (legacy HTTP) | `core.sr.ht/srht/webhook/` | +| Build manifests / job lifecycle | `builds.sr.ht/api/manifest.go`, `manifest_test.go`, `api/task.go`; worker in `builds.sr.ht/worker/` | +| Build images and shell | `builds.sr.ht/images/`, `builds.sr.ht/builds.sr.ht-shell/` | +| Build job queue | `builds.sr.ht/cmd/worker/`, `gocelery` + Redis (see go.mod), `worker/tasks.go` | +| Git push/pull access control | `git.sr.ht/gitsrht/`, `git.sr.ht/update-hook/`, `git.sr.ht/api/`, `sourcehut-ssh/dispatch/` | +| SSH dispatch (git/hg shells) | `sourcehut-ssh/` (`cmd/`, `dispatch/`, `shell/`) | +| Mailing-list ingestion | `lists.sr.ht/ingress/` (LMTP), `lists.sr.ht/listssrht/` | +| Ticket-tracker LMTP | `todo.sr.ht/todo.sr.ht-lmtp/` | +| Page hosting | `pages.sr.ht/graph/`, `pages.sr.ht/cmd/`, `pages.sr.ht/account/` | +| Hub / project aggregation | `hub.sr.ht/hubsrht/`, `hub.sr.ht/api/` | +| Config (INI loader) | `core-go/config/config.go`, `core.sr.ht/srht/config.py`; per-service `config.example.ini` | +| DB / migration tooling | `core-go/database/`, `core.sr.ht/srht/database.py`, `/schema.sql` + `migrations/` | +| Redis usage | `core-go/redis/`, `core.sr.ht/srht/redis.py` | +| Email | `core-go/email/`, `core.sr.ht/srht/email.py` | +| Shared SCSS / theme | `core.sr.ht/scss/` (everything else's `scss/main.scss` imports from here) | +| nginx routing in production | `sr.ht-nginx/*.conf` | +| Kubernetes deployment | `gensokyo/` (RBAC, ingress, monitoring, mirror) | +| Package builds | `sr.ht-apkbuilds/` (Alpine), `sr.ht-pkgbuilds/` (Arch) | +| Docs site (man.sr.ht content) | `sr.ht-docs/` | +| Go client library | `sourcehut-go/` | +| Marketing site | `sourcehut.org/`, `srht.site/` | +| Performance benchmarks | `forgeperf/` | + +## Workflow + +0. **Read `~/data/home/sourcehut/.claude/INDEX.md` first** if you have not already this session. It is auto-generated by `sourcehut-refresh` and lists, per service: current tag, every GraphQL type name, every SQL table name, every Python blueprint, every Go package, plus a cross-repo type map (which services define `User`, `Mutation`, `OAuthClient`, etc.). It is ~16 KB and answers "which service defines X?" without any grep. If `INDEX.md` is missing, run the `sourcehut-refresh` skill — it regenerates it as its final step. +1. **Identify which service(s)** the question belongs to — usually obvious from the feature name (`builds.sr.ht` = CI, `meta.sr.ht` = accounts/OAuth, etc.). If it's a cross-cutting concern, start with `core.sr.ht` / `core-go`. For ambiguous symbol names, the cross-repo type map at the bottom of `INDEX.md` resolves it. +2. **Read the table above** for likely file paths. Read those files directly — do not grep first. +3. **For specifics inside a service**, prefer `rg` scoped to the service directory rather than the whole workspace: + ```bash + rg --type go 'symbol' ~/data/home/sourcehut// + rg --type py 'symbol' ~/data/home/sourcehut//srht/ + ``` + Searching the entire workspace at once is almost always wrong — every service has its own `User`, `Job`, `Token`, etc. and you will drown in matches. +4. **For "where is type X defined?"** — start in `api/graph/schema.graphqls`. The GraphQL schema names line up with Go struct names in `api/graph/model/` (auto-generated by gqlgen). The SQL table names usually line up too. +5. **For "what calls X?"** — Python and Go sides do *not* call each other directly. Python web tier calls its own service's GraphQL API (queries are in `srht/graphql/*.graphql`, generated client in `srht/graphql/__init__.py` via `ariadne-codegen`). Inter-service Python calls go through `srht/meta/*.graphql` (the meta.sr.ht client). +6. **Cite file:line** in answers. Prefer absolute paths under `~/data/home/sourcehut/` so the user can click them. + +## Anti-patterns to avoid + +- Do **not** use Bash to read files — use the Read tool. The workspace is large and `cat`/`head` clutter context. +- Do **not** answer from training data when the file is right there. Tag versions move; your priors about SourceHut may be stale. Read the actual file. +- Do **not** edit, commit, or branch in these clones. `sourcehut-refresh` will destroy any local changes. +- Do **not** assume a feature exists in every service — `paste.sr.ht`, `man.sr.ht`, and `pages.sr.ht` are intentionally smaller and may lack things like webhooks or jobs. +- Do **not** grep the workspace root for common identifiers. Scope to a service. +- If unsure whether the canonical implementation is Python or Go: it is **Go** if a `cmd/api` exists, **Python** for the web UI, and **Go** for everything new (`pages.sr.ht`, `sourcehut-ssh`, `api.sr.ht`, `forgeperf`). + +## When to refresh first + +If the user's question is about a recently-released feature (last few months), or your answer would depend on the exact tag in use, run the `sourcehut-refresh` skill before answering. Otherwise the current pinned tags are sufficient. diff --git a/.claude/skills/sourcehut-refresh/SKILL.md b/.claude/skills/sourcehut-refresh/SKILL.md new file mode 100644 index 0000000000000000000000000000000000000000..37e897cd419e10d385826928cf50af4e64c0b90f --- /dev/null +++ b/.claude/skills/sourcehut-refresh/SKILL.md @@ -0,0 +1,136 @@ +--- +name: sourcehut-refresh +description: Refresh the SourceHut documentation mirror — scrape the upstream source listing pages, clone any new repos, and update every existing repo to the latest tag. Use when the user says "refresh", "update", "sync", or "fetch new" sourcehut repos, or when answers seem stale. +--- + +# sourcehut-refresh + +This workspace at `~/data/home/sourcehut` is a documentation mirror of the SourceHut project. Running this skill re-discovers the canonical list of subprojects from upstream and brings every clone up to its latest tag. + +## What this skill does + +The workspace is a **git superproject** — each SourceHut repo is a submodule pinned to a specific commit, listed in `.gitmodules`. Refresh therefore means: re-discover upstream, add submodules for new repos, fast-forward existing submodules to the latest tag, then stage the bumped gitlinks so the user can review and commit. + +1. Scrape `https://sr.ht/~sircmpwn/sourcehut/sources` and any `?page=N` continuations until no `?page=N+1` link appears. +2. Extract every repository name linked under `~sircmpwn/`. +3. For each name: + - **Not yet a submodule** → `git submodule add https://git.sr.ht/~sircmpwn/`, then check out latest tag inside. + - **Already a submodule** → enter it, `git fetch --tags --prune`, check out latest tag. +4. Tag selection: highest-versioned tag via `git tag --sort=-v:refname | head -n1`; if no tags, stay on default branch and fast-forward. +5. `git add ` in the superproject for every submodule whose HEAD moved. +6. Regenerate `.claude/INDEX.md`. +7. Report `OK / NEW / UPDATED / NOTAG / FAIL` and **leave the staged changes uncommitted** — the user reviews and commits. + +## How to run + +Always run from `~/data/home/sourcehut`. Do all the work via Bash — do not invent intermediate scripts unless the inline pipeline below is genuinely insufficient. + +### 1. Discover the repo list from upstream + +```bash +cd ~/data/home/sourcehut +discover_repos() { + local page=1 url repos="" got + while :; do + if [ "$page" -eq 1 ]; then + url="https://sr.ht/~sircmpwn/sourcehut/sources" + else + url="https://sr.ht/~sircmpwn/sourcehut/sources?page=$page" + fi + got=$(curl -fsSL "$url" 2>/dev/null) || break + # Anchor: href="/~sircmpwn/" where contains no slash. + # Exclude the hub itself ("sourcehut") and any '?page=' query strings. + page_repos=$(printf '%s\n' "$got" \ + | grep -oE 'href="/~sircmpwn/[A-Za-z0-9._+-]+"' \ + | sed -E 's:.*/~sircmpwn/([^"/]+)".*:\1:' \ + | grep -vE '^(sourcehut)$' \ + | sort -u) + [ -z "$page_repos" ] && break + repos="$repos"$'\n'"$page_repos" + # Stop when there is no link to the next page. + if ! printf '%s\n' "$got" | grep -q "sources?page=$((page+1))"; then + break + fi + page=$((page+1)) + done + printf '%s\n' "$repos" | sed '/^$/d' | sort -u +} +discover_repos > /tmp/srht-repos.txt +wc -l /tmp/srht-repos.txt +``` + +If the discovered list is empty or surprisingly short (< 20 entries), **stop and report to the user** — the upstream HTML may have changed. Do not proceed to mutate clones. + +### 2. Update each submodule (adds new, fetches existing, checks out latest tag) + +Submodule operations are not safely parallel (they touch the shared superproject index), so this loop is serial. Still finishes in well under a minute for ~30 repos. + +```bash +cd ~/data/home/sourcehut +while IFS= read -r name; do + url="https://git.sr.ht/~sircmpwn/$name" + status="" + if ! git config -f .gitmodules --get submodule."$name".path >/dev/null 2>&1; then + # New repo — register as submodule. submodule add clones into place. + if git submodule add --quiet "$url" "$name" 2>/dev/null; then + status="NEW" + else + printf 'FAIL %s (submodule add)\n' "$name"; continue + fi + else + # Existing — make sure it is initialized, then fetch. + git submodule update --init --quiet "$name" 2>/dev/null + git -C "$name" fetch --quiet --tags --prune 2>/dev/null \ + || { printf 'FAIL %s (fetch)\n' "$name"; continue; } + status="OK" + fi + tag=$(git -C "$name" tag --sort=-v:refname 2>/dev/null | head -n1) + if [ -z "$tag" ]; then + git -C "$name" pull --ff-only --quiet 2>/dev/null || true + printf 'NOTAG %s (on %s)\n' "$name" "$(git -C "$name" rev-parse --abbrev-ref HEAD)" + git add "$name" 2>/dev/null + continue + fi + current=$(git -C "$name" describe --tags --exact-match 2>/dev/null || echo "") + if [ "$current" = "$tag" ] && [ "$status" != "NEW" ]; then + printf '%-7s %s @ %s\n' "$status" "$name" "$tag" + else + if git -C "$name" -c advice.detachedHead=false checkout --quiet "$tag" 2>/dev/null; then + printf 'UPDATED %s @ %s\n' "$name" "$tag" + git add "$name" + else + printf 'FAIL %s (checkout %s)\n' "$name" "$tag" + fi + fi +done < /tmp/srht-repos.txt | tee /tmp/srht-refresh.log +``` + +After this loop, `git status` in the superproject will show new gitlinks for every submodule whose HEAD moved (and any `NEW` submodules will also have a `.gitmodules` change). **Do not commit automatically** — show the user `git submodule status` and `git status --short`, summarize, and let them commit. + +### 3. Rebuild the index + +After all clones are up-to-date, regenerate `.claude/INDEX.md`: + +```bash +bash ~/data/home/sourcehut/.claude/scripts/build-index.sh +``` + +This walks every repo and writes a per-service inventory of GraphQL types, SQL tables, Python blueprints, Go packages, and a cross-repo type map. The `sourcehut-lookup` skill reads it before doing anything else, so a stale index degrades every later lookup. Always run this — even if no clones changed — because the script also captures the tag of each repo. It takes ~5 seconds. + +### 4. Stage `.claude/INDEX.md` + +```bash +git add .claude/INDEX.md +``` + +### 5. Report + +Summarize counts of NEW / UPDATED / OK / NOTAG / FAIL. Highlight any FAIL lines. If `NEW` repos appeared, mention them by name — they likely warrant a one-line addition to the layout section of `CLAUDE.md`. Show `git status --short` and suggest a commit message like `refresh: bump submodules to latest tags` — let the user commit. + +## Notes + +- The workspace is a git superproject. Submodules live in `.gitmodules` and their checkout state is the gitlink stored in the superproject's tree. **A refresh produces uncommitted submodule pointer bumps** that the user reviews and commits. +- This skill performs **destructive checkout** on each submodule (detached-HEAD at a tag). If a submodule has local uncommitted changes (it shouldn't — this is a read-only mirror), `git checkout ` will fail and the repo will be reported as FAIL. Do not force. +- Submodule updates are serial because they share the superproject's index lockfile. Do not parallelize. +- Never remove submodules that disappear from upstream listings — flag them in the report and let the user decide whether to `git submodule deinit && git rm `. +- `.clone-repos.sh` is now legacy/historical. Keep it as documentation of the original bootstrap; do not edit it as part of refresh. diff --git a/.clone-repos.sh b/.clone-repos.sh new file mode 100644 index 0000000000000000000000000000000000000000..13f9b5b64c5ea95af8b0ff36352a69524a5a54dd --- /dev/null +++ b/.clone-repos.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Clone all sourcehut-related repos and check out the latest tag. +set -u + +REPOS=( + sr.ht-apkbuilds + builds.sr.ht + git.sr.ht + sr.ht-docs + sr.ht-nginx + core-go + meta.sr.ht + todo.sr.ht + core.sr.ht + go-away-config + gensokyo + status.sr.ht + hub.sr.ht + lists.sr.ht + man.sr.ht + paste.sr.ht + sourcehut-ssh + pages.sr.ht + git-send-email.io + sourcehut.org + sourcehut-migrate + srht.site + api.sr.ht + sr.ht-pkgbuilds + sourcehut-go + forgeperf + gql.sr.ht + git-am.io + git-rebase.io +) + +clone_one() { + local name="$1" + local url="https://git.sr.ht/~sircmpwn/$name" + local log="/tmp/srht-clone-$name.log" + + if [ -d "$name/.git" ]; then + echo "SKIP $name (already cloned)" + return 0 + fi + + if ! git clone --quiet "$url" "$name" >"$log" 2>&1; then + echo "FAIL $name (clone)" + return 1 + fi + + local tag + tag=$(git -C "$name" tag --sort=-v:refname 2>/dev/null | head -n1) + if [ -z "$tag" ]; then + local branch + branch=$(git -C "$name" rev-parse --abbrev-ref HEAD) + echo "NOTAG $name (on $branch)" + return 0 + fi + + if git -C "$name" -c advice.detachedHead=false checkout --quiet "$tag" 2>>"$log"; then + echo "OK $name @ $tag" + else + echo "FAIL $name (checkout $tag)" + fi +} + +export -f clone_one + +printf '%s\n' "${REPOS[@]}" | xargs -n1 -P6 -I{} bash -c 'clone_one "{}"' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..1d2e60333c0c04add80015b3779a3e5fa6fce753 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.DS_Store +*.swp + +# Override global ignores — this superproject deliberately tracks its +# Claude tooling alongside the submoduled SourceHut clones. +!CLAUDE.md +!.claude +!.claude/** diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000000000000000000000000000000000..4d0a6f0b01896afe0941e2e4039e82404a847dfa --- /dev/null +++ b/.gitmodules @@ -0,0 +1,87 @@ +[submodule "api.sr.ht"] + path = api.sr.ht + url = https://git.sr.ht/~sircmpwn/api.sr.ht +[submodule "builds.sr.ht"] + path = builds.sr.ht + url = https://git.sr.ht/~sircmpwn/builds.sr.ht +[submodule "core-go"] + path = core-go + url = https://git.sr.ht/~sircmpwn/core-go +[submodule "core.sr.ht"] + path = core.sr.ht + url = https://git.sr.ht/~sircmpwn/core.sr.ht +[submodule "forgeperf"] + path = forgeperf + url = https://git.sr.ht/~sircmpwn/forgeperf +[submodule "gensokyo"] + path = gensokyo + url = https://git.sr.ht/~sircmpwn/gensokyo +[submodule "git-am.io"] + path = git-am.io + url = https://git.sr.ht/~sircmpwn/git-am.io +[submodule "git-rebase.io"] + path = git-rebase.io + url = https://git.sr.ht/~sircmpwn/git-rebase.io +[submodule "git-send-email.io"] + path = git-send-email.io + url = https://git.sr.ht/~sircmpwn/git-send-email.io +[submodule "git.sr.ht"] + path = git.sr.ht + url = https://git.sr.ht/~sircmpwn/git.sr.ht +[submodule "go-away-config"] + path = go-away-config + url = https://git.sr.ht/~sircmpwn/go-away-config +[submodule "gql.sr.ht"] + path = gql.sr.ht + url = https://git.sr.ht/~sircmpwn/gql.sr.ht +[submodule "hub.sr.ht"] + path = hub.sr.ht + url = https://git.sr.ht/~sircmpwn/hub.sr.ht +[submodule "lists.sr.ht"] + path = lists.sr.ht + url = https://git.sr.ht/~sircmpwn/lists.sr.ht +[submodule "man.sr.ht"] + path = man.sr.ht + url = https://git.sr.ht/~sircmpwn/man.sr.ht +[submodule "meta.sr.ht"] + path = meta.sr.ht + url = https://git.sr.ht/~sircmpwn/meta.sr.ht +[submodule "pages.sr.ht"] + path = pages.sr.ht + url = https://git.sr.ht/~sircmpwn/pages.sr.ht +[submodule "paste.sr.ht"] + path = paste.sr.ht + url = https://git.sr.ht/~sircmpwn/paste.sr.ht +[submodule "sourcehut-go"] + path = sourcehut-go + url = https://git.sr.ht/~sircmpwn/sourcehut-go +[submodule "sourcehut-migrate"] + path = sourcehut-migrate + url = https://git.sr.ht/~sircmpwn/sourcehut-migrate +[submodule "sourcehut-ssh"] + path = sourcehut-ssh + url = https://git.sr.ht/~sircmpwn/sourcehut-ssh +[submodule "sourcehut.org"] + path = sourcehut.org + url = https://git.sr.ht/~sircmpwn/sourcehut.org +[submodule "sr.ht-apkbuilds"] + path = sr.ht-apkbuilds + url = https://git.sr.ht/~sircmpwn/sr.ht-apkbuilds +[submodule "sr.ht-docs"] + path = sr.ht-docs + url = https://git.sr.ht/~sircmpwn/sr.ht-docs +[submodule "sr.ht-nginx"] + path = sr.ht-nginx + url = https://git.sr.ht/~sircmpwn/sr.ht-nginx +[submodule "sr.ht-pkgbuilds"] + path = sr.ht-pkgbuilds + url = https://git.sr.ht/~sircmpwn/sr.ht-pkgbuilds +[submodule "srht.site"] + path = srht.site + url = https://git.sr.ht/~sircmpwn/srht.site +[submodule "status.sr.ht"] + path = status.sr.ht + url = https://git.sr.ht/~sircmpwn/status.sr.ht +[submodule "todo.sr.ht"] + path = todo.sr.ht + url = https://git.sr.ht/~sircmpwn/todo.sr.ht diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000000000000000000000000000000000..30e48b00bf97afe27656ef4be297f0d88d15a26c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,96 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Purpose + +**This directory is a read-only documentation mirror, not a development workspace.** It exists to let Claude inspect the *current state* of every SourceHut service in one place when answering questions about how SourceHut works. + +The set of repos is discovered by scraping the upstream source listing pages: +- `https://sr.ht/~sircmpwn/sourcehut/sources` +- `https://sr.ht/~sircmpwn/sourcehut/sources?page=2` (and additional pages if they exist) + +This workspace is itself a **git superproject**: every cloned subproject is a submodule pinned to a specific commit (typically the latest upstream tag), tracked in `.gitmodules`. Use the `sourcehut-refresh` skill to re-discover upstream and bump submodules; use `sourcehut-lookup` for fast cross-repo documentation traversal. A refresh leaves submodule pointer bumps **uncommitted** so the user can review and commit one snapshot at a time. + +Submodules' `.git` directories are absorbed into `.git/modules//` of the superproject. Each clone's worktree contains a `.git` *file* (gitlink), not a directory — do not be confused by this. + +A pre-built inventory lives at `.claude/INDEX.md` — per-service GraphQL types, SQL tables, Python blueprints, Go packages, plus a cross-repo type map. Regenerated automatically by `sourcehut-refresh`; the `sourcehut-lookup` skill reads it first to skip a lot of grepping. + +## Production deviations (phoebe-lab) + +The user runs a patched SourceHut in production under `~/data/home/phoebe-lab/srht/`. Fixes not yet upstreamed are mirrored here in `patches/` for reference — they are **not applied** to the clones in this workspace. When answering questions about behavior that might diverge from upstream, check `patches/README.md` first. Currently tracked: + +- `patches/core-go-checksum.patch` — disables AWS SDK v2 default request/response checksum calculation in `core-go/objects/middleware.go` `NewClient`. Upstream defaults break streaming `PutObject` (pages publish, builds artifact upload) on non-AWS S3 backends. + +Do not commit, push, or develop in these clones — they will be wiped/replaced by the next refresh. If you need to make changes, clone the upstream repo separately. + +## Repository layout + +This is **not a single repo** — it is a workspace of ~28 sibling clones of upstream SourceHut subprojects (`https://git.sr.ht/~sircmpwn/`). Each subdirectory is an independent git repo with its own LICENSE, build, and history. Always `cd` into the specific subproject before running per-repo commands. + +## Service topology + +SourceHut is a federation of services that share a common architecture. Categorize a repo by inspecting its top level: + +- **Hybrid Python+Go service** (`api/`, `*srht/`, `Makefile`, `run.py`, `schema.sql`, `migrations/`, `pyproject.toml`, `go.mod`): + `builds.sr.ht`, `git.sr.ht`, `hub.sr.ht`, `lists.sr.ht`, `meta.sr.ht`, `paste.sr.ht`, `todo.sr.ht`, `man.sr.ht`, `pages.sr.ht`. + Python Flask app (`srht/app.py`) serves the user-facing web UI; Go binary (`cmd/api` → `-api`) serves the GraphQL API. `builds.sr.ht` additionally has `cmd/worker` → `builds.sr.ht-worker` (job runner using Celery via `gocelery`, Redis, and `builds.sr.ht-shell`). +- **Pure Go service**: `api.sr.ht` (the federated GraphQL gateway built on `git.sr.ht/~adnano/thistle`), `sourcehut-ssh` (SSH dispatch for git/hg shells), `pages.sr.ht` (newer version, Go-only). +- **Shared libraries**: `core.sr.ht` (Python — `srht` package: ORM, OAuth, GraphQL helpers, templates, SCSS/Bootstrap assets) and `core-go` (Go equivalent: `auth`, `client`, `config`, `database`, `redis`, `server`, `webhooks`, `crypto`). +- **Static sites / docs**: `sourcehut.org` and `srht.site` (Hugo), `sr.ht-docs` (man.sr.ht content), `git-am.io`, `git-rebase.io`, `git-send-email.io`. +- **Ops / infra**: `gensokyo` (Kubernetes YAML for the production cluster), `sr.ht-nginx` (nginx configs), `sr.ht-apkbuilds` / `sr.ht-pkgbuilds` (Alpine/Arch packaging), `sourcehut-migrate` (data migration tooling), `status.sr.ht`, `go-away-config`. +- **Misc Go clients/tools**: `sourcehut-go` (Go client library and CLI examples), `forgeperf` (forge performance benchmark harness using Chromium+Lighthouse). + +## Build commands + +Each service that builds has a top-level `Makefile`. The hybrid Python+Go services follow a uniform pattern (driven by `core.sr.ht`'s `Makefile` conventions): + +``` +make # default: all-bin all-share all-python +make all-bin # Go binaries: -api, -worker (if present) +make all-share # compile SCSS via sassc → static/main.min.css +make all-python # generate buildsrht/graphql/__init__.py via ariadne-codegen +make install # install to $(PREFIX)=/usr/local +make clean +``` + +Toolchain prerequisites (vary by service): +- Go: project go.mod versions range from `1.16` (older repos like `api.sr.ht`) to `1.22+` with `toolchain go1.24.0` (newer repos like `builds.sr.ht`). +- Python: needs the `srht` package from `core.sr.ht` installed (editable or via system package). Production uses `pip install -e .` after `pip install -e ../core.sr.ht`. +- `sassc` (or set `SASSC`), `minify`, `ariadne-codegen` (Python package, used to generate GraphQL client stubs from `*.graphql` queries). +- GraphQL server code is generated by `gqlgen` (Go) — `cd api && go generate ./graph` (uses `gqlgen.yml`, reads `api/graph/schema.graphqls`, writes `api/graph/api/generated.go`). +- DataLoader code is generated by `dataloaden` — `cd api && go generate ./loaders` (the `api/loaders/gen` shell helper + directives in `generate.go`). +- Build-time version strings come from `sourcehut-buildver` / `sourcehut-builddate` (installed from `core.sr.ht`) and are injected via `-ldflags` into `core-go/server.BuildVersion`/`BuildDate`. + +### Running a service locally + +For Python web tier: `python3 run.py` (calls `srht.debug.run_service(app)`). For the Go API: `./-api` after `make`. Both read `config.ini` (copy from `config.example.ini`). + +### Database / migrations + +Each service ships its full `schema.sql` plus incremental `migrations/*.sql`. Tests/dev use Postgres; Redis is required for job queues, GraphQL pubsub, and webhooks. `lists.sr.ht` additionally has an `ingress/` (LMTP) component; `todo.sr.ht` has `todo.sr.ht-lmtp`. + +### Tests + +`builds.sr.ht/api/manifest_test.go` shows the Go test convention — standard `go test ./...` from inside each service's repo. There is no top-level "run all tests" target across the workspace; each repo is tested independently. + +## Architecture notes that span files + +- **GraphQL is the API contract.** Each service publishes `api/graph/schema.graphqls`; clients (the Python web tier of the *same* service, the federated gateway `api.sr.ht`, and the Python web tiers of *other* services) consume it. Python clients are generated under `srht/graphql/` and `srht/meta/` via `ariadne-codegen`. Go resolvers live under `api/graph/`, with batched lookups in `api/loaders/`. +- **`api.sr.ht` is the federation layer.** It uses `thistle` to merge per-service schemas into one endpoint and forwards requests to each `-api` based on the type. `InternalAuthTransport` in `api.sr.ht/auth.go` is how it elevates calls when needed. +- **Authentication paths:** OAuth 2.0 with `meta.sr.ht` as the IdP. `core-go/auth` and `core.sr.ht/srht/oauth` are the canonical client implementations — every other service depends on one of them. Internal service-to-service calls use the "internal user" with HMAC-signed headers (`core-go/auth/internal.go`). +- **Webhooks** (legacy HTTP and new GraphQL-native): `core-go/webhooks` defines the worker; each service registers concrete event subscriptions in `api/webhooks/`. +- **SCSS / static assets:** `core.sr.ht/scss/` provides the shared bootstrap-derived theme; each service has its own `scss/main.scss` that imports it. The Makefile compiles to a hashed `main.min..css` for cache busting. +- **CI manifests for the project itself** live in each repo's `.builds/` directory (if present) and run on `builds.sr.ht` — see the `sourcehut-ci` skill for authoring rules. + +## Upstream and contributions + +Upstream is hosted at `git.sr.ht/~sircmpwn/`. Patches go to the `sr.ht-dev` mailing list via `git send-email`. **This workspace is a read-only mirror — do not commit, push, or develop here.** To contribute, clone the upstream repo independently outside this workspace. + +## Gotchas + +- `make` from the workspace root does nothing useful — there is no top-level Makefile. Always work inside a single subproject. +- The Go modules use `git.sr.ht/~sircmpwn/core-go` from upstream tags; do **not** add a `replace` directive to point at the local `./core-go` clone unless explicitly working on a coordinated change, since the local clone is pinned to a tag that may not match what the service expects. +- Several repos still target Go 1.16 (`api.sr.ht`, possibly older clones). Use the toolchain declared in each `go.mod`. +- `gensokyo` uses the Go-based `yq` (`github.com/mikefarah/yq`), not the Python one — they are not interchangeable. +- `forgeperf` is an offline benchmark harness, not a service — it needs Chromium, an X server, and `lighthouse` via npm to run. diff --git a/patches/README.md b/patches/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a87a10b6db629455b585e2a2ce34453da8b19186 --- /dev/null +++ b/patches/README.md @@ -0,0 +1,33 @@ +# Local patches + +Fixes carried in the **phoebe-lab production deployment** of SourceHut that are *not* upstream yet. These are stored here for reference and provenance — they are **not applied** to the clones in this workspace, which mirrors upstream verbatim. + +If a question is about behavior that diverges from upstream, check here first to see whether phoebe-lab is running a patched version. + +Patches are kept in `git format-patch`/`git diff` format against the target upstream repo and are sourced from `~/data/home/phoebe-lab/srht/patches/`. + +## Patches + +### `core-go-checksum.patch` + +**Repo:** `core-go`  ·  **File:** `objects/middleware.go` (`NewClient`) + +**Problem.** AWS SDK for Go v2 defaults `RequestChecksumCalculation` to `WhenSupported`, which causes the S3 client to **seek the request body** before `PutObject` to compute a checksum. Non-seekable bodies (streamed uploads) fail or get fully buffered into memory. + +This hits two upload paths in production: + +- `pages.sr.ht` Publish — streaming the site tarball into S3. +- `builds.sr.ht` artifact upload — workers POST job artifacts as a stream. + +**Fix.** Set both `RequestChecksumCalculation` and `ResponseChecksumValidation` to `WhenRequired`, so the SDK only checksums when the protocol mandates it. This restores pre-SDK-v1.30 behavior and is the workaround Amazon documents for non-AWS S3 backends (Garage/Ceph/Minio) where strict checksum negotiation is also unreliable. + +**Status.** Applied in phoebe-lab production; **not upstreamed yet**. + +**Verify it still applies cleanly:** + +```bash +cd ~/data/home/sourcehut/core-go +git apply --check ../patches/core-go-checksum.patch +``` + +If `git apply --check` fails after a `sourcehut-refresh`, upstream has changed that file — re-derive the patch against the new context and update the copy in `~/data/home/phoebe-lab/srht/patches/` first, then re-import here. diff --git a/patches/core-go-checksum.patch b/patches/core-go-checksum.patch new file mode 100644 index 0000000000000000000000000000000000000000..aca2ed7e359b8a2ed2b7813eaaf6dd88aecb410a --- /dev/null +++ b/patches/core-go-checksum.patch @@ -0,0 +1,16 @@ +diff --git a/objects/middleware.go b/objects/middleware.go +index efdb4b5..520781b 100644 +--- a/objects/middleware.go ++++ b/objects/middleware.go +@@ -88,6 +88,11 @@ func NewClient(conf ini.File) (*s3.Client, error) { + Region: region, + Credentials: creds, + }, func(opts *s3.Options) { ++ // Patched (phoebe-lab): AWS SDK v2 defaults to WhenSupported, which seeks ++ // non-seekable PutObject bodies (pages Publish, builds artifact upload) ++ // for checksum computation. Use WhenRequired to skip pre-flight hashing. ++ opts.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired ++ opts.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired + opts.BaseEndpoint = aws.String(scheme + upstream) + opts.EndpointResolverV2 = &S3Resolver{ + conf,