diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml new file mode 100644 index 0000000..5cc8347 --- /dev/null +++ b/.gitea/workflows/ci.yaml @@ -0,0 +1,40 @@ +name: CI + +# Go CI for sherlock. Runs `go vet`, `go test -race`, `go build` on every +# push and PR. No deploy pipeline — sherlock is operator-installed via +# `go install`, not host-deployed. +# +# Runner notes: +# - The Charlie runner image (gitea/act_runner + git/sops/docker-cli/bash) +# does NOT ship a Go toolchain. setup-go installs one at job time. +# - runs-on: self-hosted picks any Charlie runner. No SSH-into-host +# gymnastics required because there's no docker build / no host state +# to mutate; everything runs inside the runner container. + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + build: + runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.23" + cache: true + + - name: go vet + run: go vet ./... + + - name: go test -race + run: go test ./... -race + + - name: go build + run: go build ./... diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3c568ae --- /dev/null +++ b/.gitignore @@ -0,0 +1,25 @@ +# Editor / IDE +.idea/ +.vscode/ +*.swp +*.swo +.DS_Store + +# Go build output +/sherlock +/sherlock-broker +/dist/ +/bin/ + +# Go test / tooling +coverage.out +coverage.html +*.test +*.prof + +# Vendored deps (we use modules) +/vendor/ + +# Local config (operator-specific, never committed) +/.envrc +/.env diff --git a/README.md b/README.md new file mode 100644 index 0000000..7fb3e9e --- /dev/null +++ b/README.md @@ -0,0 +1,14 @@ +# sherlock + +Per-operator credential broker + agent-CLI wrapper for the Charlie homelab. Holds the Authentik session once, injects per-service tokens into MCP servers, and execs the agent CLI of your choice (Copilot today, Claude Code or any other MCP-aware agent tomorrow). + +Design + phasing: [plan.md](plan.md). + +## Docs + +- [Architecture](docs/architecture.md) +- [Auth model](docs/auth-model.md) +- [Service registry](docs/service-registry.md) +- [Agent profiles](docs/agent-profiles.md) +- [gssh integration](docs/gssh-integration.md) +- [Conventions](docs/conventions.md) diff --git a/cmd/sherlock-broker/main.go b/cmd/sherlock-broker/main.go new file mode 100644 index 0000000..e114b18 --- /dev/null +++ b/cmd/sherlock-broker/main.go @@ -0,0 +1,46 @@ +// Command sherlock-broker is the long-running per-user daemon that owns +// the Authentik OAuth state, caches per-service tokens, performs RFC 8693 +// token exchange, and hosts the loopback HTTP listener for browser +// callbacks. +// +// Phase 0 ships only `version` so CI has something to build. The actual +// daemon lands in Phase 1. +package main + +import ( + "flag" + "fmt" + "os" +) + +// Version is overwritten at build time via -ldflags "-X main.Version=...". +var Version = "0.0.0-dev" + +func main() { + flag.Usage = func() { + fmt.Fprintf(os.Stderr, `sherlock-broker - Charlie credential broker daemon (Phase 1+) + +Usage: + sherlock-broker [args...] + +Subcommands: + version Print the broker version and exit. +`) + } + flag.Parse() + + args := flag.Args() + if len(args) == 0 { + flag.Usage() + os.Exit(2) + } + + switch args[0] { + case "version", "--version", "-v": + fmt.Println(Version) + default: + fmt.Fprintf(os.Stderr, "sherlock-broker: unknown subcommand %q\n", args[0]) + flag.Usage() + os.Exit(2) + } +} diff --git a/cmd/sherlock/main.go b/cmd/sherlock/main.go new file mode 100644 index 0000000..c67ee14 --- /dev/null +++ b/cmd/sherlock/main.go @@ -0,0 +1,45 @@ +// Command sherlock is the operator-facing CLI: login, status, run-agent wrapper. +// +// Phase 0 ships only `version`. Login / status / run land in Phase 1. +package main + +import ( + "flag" + "fmt" + "os" +) + +// Version is overwritten at build time via -ldflags "-X main.Version=...". +var Version = "0.0.0-dev" + +func main() { + flag.Usage = func() { + fmt.Fprintf(os.Stderr, `sherlock - Charlie homelab credential broker + agent wrapper + +Usage: + sherlock [args...] + +Subcommands: + version Print the sherlock version and exit. + +Phase 1 will add: login, logout, status, run, copilot. +See plan.md. +`) + } + flag.Parse() + + args := flag.Args() + if len(args) == 0 { + flag.Usage() + os.Exit(2) + } + + switch args[0] { + case "version", "--version", "-v": + fmt.Println(Version) + default: + fmt.Fprintf(os.Stderr, "sherlock: unknown subcommand %q\n", args[0]) + flag.Usage() + os.Exit(2) + } +} diff --git a/docs/agent-profiles.md b/docs/agent-profiles.md new file mode 100644 index 0000000..ebaae58 --- /dev/null +++ b/docs/agent-profiles.md @@ -0,0 +1,133 @@ +# Agent profiles + +How sherlock supports multiple agent CLIs without code changes. + +## Why this exists + +`sherlock` should not be Copilot-shaped. We want to swap the underlying agent CLI (Copilot, Claude Code, Aider, future tools) without changing sherlock's code, because: +- We may switch agents. +- Different operators may prefer different agents. +- Per-task we may want a specific agent (a smaller model for batch ops, a bigger one for review). + +An **agent profile** is a TOML file describing one agent CLI: how to launch it, how to pass it the per-session MCP config sherlock renders, and what env it needs. Adding a new agent = adding one TOML file. No Go changes. + +## Location + +- **Built-in:** `internal/agent/profiles/*.toml`, embedded via `//go:embed`. Sherlock ships with at least `copilot.toml`. +- **User overrides:** `~/.config/sherlock/agents.d/*.toml`. Same filename as a built-in profile shadows the built-in entirely. + +## Invocation + +``` +sherlock run [args...] # canonical +sherlock copilot [args...] # sugar for `run copilot` +sherlock claude [args...] # sugar for `run claude` once claude.toml exists +``` + +The aliases are auto-derived from the set of loaded profiles, so they appear/disappear with `agents.d/`. + +## Schema + +```toml +[agent] +name = "copilot" # unique; matches the file's basename +executable = "copilot" # looked up on PATH unless absolute +description = "GitHub Copilot CLI" + +[mcp] +# How the rendered MCP config is delivered to this agent. +mode = "cli-flag" # cli-flag | env-var | config-file +# When mode = "cli-flag": the flag name. +flag = "--mcp-config" +# When mode = "env-var": the env var name (value is the path to the rendered file). +# var = "MCP_CONFIG" +# When mode = "config-file": the absolute path the rendered file is written to. +# path = "~/.config/claude-code/mcp.json" + +[env] +# Static env injected into the agent process. Values may reference +# ${HOME}, ${XDG_RUNTIME_DIR}, etc. +# Example: +# GH_TOKEN_KIND = "oauth" + +[argv] +# argv built around the agent's positional args: +# +# When mcp.mode = "cli-flag", the flag + value are appended after prefix. +prefix = [] +suffix = [] + +[forbid] +# Env vars sherlock will strip before exec'ing the agent, even if they exist +# in the operator's shell. Defense against accidental credential leaks. +env = [] +``` + +## Built-in: `copilot.toml` + +```toml +[agent] +name = "copilot" +executable = "copilot" +description = "GitHub Copilot CLI" + +[mcp] +mode = "cli-flag" +flag = "--mcp-config" + +[argv] +prefix = [] +suffix = [] + +[forbid] +env = [] +``` + +## Worked example: Claude Code (operator-supplied, not built-in yet) + +```toml +[agent] +name = "claude" +executable = "claude" +description = "Anthropic Claude Code" + +[mcp] +# Claude Code reads a JSON config file from a stable path. +mode = "config-file" +path = "${XDG_CONFIG_HOME}/claude-code/sherlock.mcp.json" + +[argv] +# Claude Code is invoked with the project dir as a positional; users pass +# extra flags after. +prefix = [] +suffix = [] + +[forbid] +# Make sure no stray ANTHROPIC_API_KEY from the shell leaks in if we want +# the broker to be the only source of credentials in future revisions. +env = ["ANTHROPIC_API_KEY"] +``` + +Drop the file at `~/.config/sherlock/agents.d/claude.toml` and `sherlock claude` starts working — no rebuild. + +## Resolution order + +1. `internal/agent/profiles/` (embedded built-ins) — provides defaults. +2. `~/.config/sherlock/agents.d/` — operator overrides; same `agent.name` shadows the built-in. + +A profile is malformed → it's logged and skipped. Sherlock never silently runs an agent with a half-loaded profile. + +## What the wrapper does at runtime (Phase 1) + +1. Parse `sherlock ...`. Resolve the agent profile by name. +2. Ensure the broker is up; ensure the operator is logged in (`whoami` succeeds). +3. For each service registered under `services.d/`: ask the broker for a per-service token and collect `{ env_var: token }` pairs. +4. Render an MCP config file in `$XDG_RUNTIME_DIR/sherlock/.mcp.json` listing each MCP's `command`, `args`, and the `env` block populated with the tokens from step 3. +5. Build argv from the profile (`prefix + [mcp-flag pair if applicable] + user args + suffix`). +6. Build env from the operator's env + `[env]` + strip `[forbid].env`. +7. `exec` the agent. No fork: sherlock itself exits, the agent becomes the foreground process. + +## Non-goals + +- We do **not** translate MCP protocol versions per agent. All supported agents must speak the MCP version sherlock ships with. +- We do **not** rewrite the agent's own state. Each agent keeps its config / history / login wherever it normally does. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..59fc6c2 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,65 @@ +# Architecture + +Condensed from [`../plan.md`](../plan.md). Read this first; jump to the plan for the longer rationale. + +## One-paragraph summary + +Sherlock is a per-user credential broker + agent-CLI wrapper that runs on the operator's workstation. It owns a single Authentik session, exchanges it for per-service tokens on demand, injects those tokens as environment variables into thin stdio MCP servers, and then `exec`s the agent CLI of your choice (Copilot, Claude Code, …) with the right MCP config. No long-lived service tokens live on disk in the clear, and the agent never sees a credential it isn't supposed to. + +## Diagram + +``` +┌────────────────────────────────────────────────────────────┐ +│ user │ +│ │ │ +│ │ `sherlock run copilot` (or `sherlock copilot`) │ +│ ▼ │ +│ ┌─────────────┐ spawns, injects env ┌──────────────┐ │ +│ │ copilot CLI │ ────────────────────▶ │ gitea-mcp │ │ +│ │ │ │ grafana-mcp │ │ +│ │ │ │ gssh-mcp │ │ +│ └─────┬───────┘ └──────┬───────┘ │ +│ │ unix socket │ │ +│ ▼ ▼ │ +│ ┌──────────────────────────────────────────────────┐ │ +│ │ sherlock-broker (systemd --user daemon) │ │ +│ │ - owns the Authentik OAuth state │ │ +│ │ - per-service token cache + refresh │ │ +│ │ - RFC 8693 token exchange │ │ +│ │ - hosts loopback HTTP for browser callbacks │ │ +│ │ - exposes sherlock-mcp (stdio) for consent flow │ │ +│ └────────────────────┬─────────────────────────────┘ │ +│ │ OIDC / token exchange │ +│ ▼ │ +│ Authentik (id.alexandru.macocian.me) │ +└────────────────────────────────────────────────────────────┘ +``` + +## Components + +| Component | Lives at | Owns | +|---|---|---| +| `sherlock` (CLI) | `cmd/sherlock/` | `login`, `logout`, `status`, `run`, agent-name aliases (`copilot`, `claude`, …). Renders the per-session MCP config, then `exec`s the agent per its profile. | +| `sherlock-broker` (daemon) | `cmd/sherlock-broker/` | Long-running per-user daemon. Authentik PKCE; loopback redirect; token cache (encrypted at rest); single-flight refresh; RFC 8693 token exchange; Unix-socket RPC. | +| `internal/agent/` | — | Agent-profile loader + spawner. **Single integration point** with any agent CLI; adding one = a new TOML under `agents.d/`, never code here. See [agent-profiles.md](agent-profiles.md). | +| `internal/authn/` | — | OIDC / PKCE / token-exchange primitives. See [auth-model.md](auth-model.md). | +| `internal/config/` | — | TOML loaders for `~/.config/sherlock/{agents,services}.d/`. | +| `internal/socket/` | — | UDS RPC server (broker side) + client (CLI + MCPs). | +| `internal/mcp/` | — | Shared stdio-MCP helpers (used by every per-service MCP binary). | +| `cmd/gitea-mcp/` (Phase 2) | — | First per-service MCP. Reads `GITEA_TOKEN` from env, refreshes via broker socket on 401. | +| `cmd/gssh-mcp/` (Phase 3) | — | Thin HTTP+WS client to the existing gssh server. No local certs. See [gssh-integration.md](gssh-integration.md). | +| `cmd/grafana-mcp/` (Phase 4) | — | Grafana tools, OIDC-federated via Authentik. | +| `cmd/sherlock-mcp/` (Phase 4) | — | The only **interactive** MCP. Exposes `oauth.consent(service)`, `auth.whoami()`, `auth.list_services()`, `auth.revoke(service)`. | + +## Why the broker is separate from the wrapper + +- One Authentik browser flow per workstation, shared across every terminal. +- A stable loopback listener for non-Authentik OAuth callbacks. +- Token refresh races are solved once, in one place. +- The wrapper can stay agent-agnostic: it only knows "ask the broker for a token, then exec the agent". Swap Copilot for Claude Code without touching auth code. + +## Boundaries + +- Sherlock does **not** federate identities — that's Authentik's job. +- Sherlock does **not** issue SSH certificates — gssh already does that internally when it authenticates a WebSocket session. See [gssh-integration.md](gssh-integration.md). +- Sherlock does **not** know what tools an MCP exposes — that's the MCP's job. Sherlock only ensures the MCP has the credential it needs. diff --git a/docs/auth-model.md b/docs/auth-model.md new file mode 100644 index 0000000..c253af5 --- /dev/null +++ b/docs/auth-model.md @@ -0,0 +1,56 @@ +# Auth model + +How sherlock obtains and distributes credentials. + +## The two layers + +1. **Operator → Authentik.** A single OAuth 2.1 authorization-code + PKCE flow against Authentik (`id.alexandru.macocian.me`). The broker owns the resulting ID/access/refresh tokens. This is the only browser flow per workstation per session. +2. **Broker → downstream service.** For each per-service request from an MCP, the broker mints a service-scoped token using one of the three grant kinds below. + +## Grant kinds + +Set per service in `~/.config/sherlock/services.d/.toml` (see [service-registry.md](service-registry.md)). + +### `oidc-federated` + +The downstream service already federates against Authentik (Gitea, Grafana, Caddy-protected apps). The broker either: +- Hands the existing Authentik-issued ID token (when the service's expected audience matches), or +- Performs **RFC 8693 token exchange** against Authentik's token endpoint, swapping the operator's ID token for a service-scoped access token. + +The MCP receives the token in an env var (e.g. `GITEA_TOKEN`). + +### `own-oauth` + +The service runs its own OAuth/OIDC stack, not federated through Authentik (think GitHub.com, a SaaS API, …). The broker: +- Holds a per-(operator, service) OAuth client registration. +- Runs the auth-code + PKCE flow in a browser the first time, with the callback received on the broker's loopback listener. +- Caches the refresh token alongside the Authentik creds. +- Refreshes silently after that. + +Triggered explicitly via `oauth.consent(service)` from `sherlock-mcp` (Phase 4) or via `sherlock login --service `. + +### `static-pat` + +For services that have no OAuth at all, or where the only sane option is a long-lived PAT. The broker reads the PAT from an `age`-encrypted blob and injects it. Avoid where possible; document why every time it's used. + +## Token storage + +- Authentik creds + per-service tokens persisted under `~/.config/sherlock/` (encrypted at rest — exact mechanism decided at the start of Phase 1; `age`-encrypted blob is the leading candidate, `libsecret`/OS keyring under evaluation). +- In-memory cache mirrors the on-disk store; the on-disk write is debounced and atomic. +- Refresh is single-flight: a 401 storm across N MCPs collapses to one refresh. + +## Audience binding + +Per the MCP 2025-06-18 spec, downstream tokens MUST be audience-bound. The broker never hands an MCP a token whose `aud` claim doesn't name that service. This is the "confused deputy" mitigation the spec calls out. + +## Scope minimisation + +Default service registrations request read-only scopes. Write scopes require either: +- The operator passing `--write` to `sherlock run` (Phase 5), or +- An explicit `scopes` override in the service's TOML. + +## Out of scope for sherlock + +- Issuing SSH certs (gssh handles this — [gssh-integration.md](gssh-integration.md)). +- Multi-user / shared tenancy. Sherlock is per-operator, full stop. +- Acting as a Resource Server itself (Phase 4's `sherlock-mcp` is a stdio MCP, not HTTP — the agent CLI talks to it over a pipe, no inbound auth). diff --git a/docs/conventions.md b/docs/conventions.md new file mode 100644 index 0000000..4f68d5a --- /dev/null +++ b/docs/conventions.md @@ -0,0 +1,59 @@ +# Conventions + +Rules that govern this repo. Anything that becomes load-bearing belongs here, in `conventions.md`, never in a section appended to another doc. + +## Repository layout + +``` +cmd/ one subdir per shippable binary; package main only +internal/ every other package; never imported outside this module +internal/agent/profiles built-in TOML profiles, embedded with //go:embed +docs/ one topic per file (see "Docs" below) +plan.md the long-form design doc; source-of-truth for phasing +README.md TOC only +``` + +No top-level `pkg/` until we have an external consumer. + +## Go + +- Module path: `gitea.alexandru.macocian.me/Charlie/sherlock`. +- Target Go toolchain: `go 1.23` (pin via `go.mod`'s `go` directive). +- One `package main` per `cmd//`. No multiple-`main`-files trickery. +- Every other package has a top-of-file `Package ...` doc comment in `doc.go`. +- No third-party deps in Phase 0. Phase 1+ deps must be added with a one-line justification in the PR description. + +## Docs + +- `README.md` is **TOC only**. One-line description + a bulleted index of `docs/*.md`. +- **One topic per file under `docs/`.** Never append a new section to an existing doc to cover a new concern; create `docs/.md` and link it from `README.md`. +- `plan.md` is the long-form design + phasing doc. It is allowed to be long. +- Cross-link aggressively: every doc should link to the other docs whose concerns it touches. +- ASCII diagrams over images. Diagrams live next to the prose that explains them. + +## Naming + +- Binaries are kebab-case (`sherlock-broker`, `gitea-mcp`). +- Built-in agent profiles match the agent's canonical CLI name (`copilot`, `claude`, `aider`). +- Service registry files match the service's canonical name (`gitea.toml`, `grafana.toml`). +- Env vars are `_TOKEN` (e.g. `GITEA_TOKEN`, `GSSH_TOKEN`). + +## Extensibility invariants + +- **Agent extensibility:** adding a new agent CLI MUST be a TOML drop-in under `~/.config/sherlock/agents.d/`. If a code change is required, the design has drifted — fix the abstraction in `internal/agent/`, don't accept the code change. +- **Service extensibility:** adding a new downstream service MUST be a TOML drop-in under `~/.config/sherlock/services.d/` + (optionally) a new `cmd/-mcp/` binary. The broker itself does not learn about individual services in Go code. + +## Commits + +- Conventional-ish: `area: short imperative` (e.g. `broker: persist tokens with age`, `docs: add gssh-integration`). +- One logical change per commit. CI must pass on every commit on `main`. + +## CI + +- `.gitea/workflows/ci.yaml` runs on every push + PR. It runs `go vet`, `go test -race`, `go build`. No other gates. +- No deploy pipeline — sherlock is operator-installed via `go install`. Release artifacts (binaries to the gitea registry / releases) land in Phase 5 if at all. + +## Security + +- No secrets in this repo, ever. Not in tests, not in fixtures, not in comments. +- The operator's per-service credentials live in `~/.config/sherlock/`, encrypted at rest. Mechanism is decided at the start of Phase 1; this doc will gain a link to the new `docs/storage.md` then. diff --git a/docs/gssh-integration.md b/docs/gssh-integration.md new file mode 100644 index 0000000..1d3b447 --- /dev/null +++ b/docs/gssh-integration.md @@ -0,0 +1,61 @@ +# gssh integration + +How `gssh-mcp` will reuse the existing gssh server. Decided in Phase-0 planning; implementation in Phase 3. + +## Why reuse + +The gssh server at `gssh.alexandru.macocian.me` already: +- Authenticates inbound requests via Authentik JWTs (`HttpContext.User`). +- Resolves the per-user host allow-list (`SessionController.GetHosts` → `SshConnectorService.Hosts`). +- Establishes SSH sessions internally using the host's CA-signed credentials (the operator never sees an SSH key). +- Streams stdin/stdout as a binary WebSocket and accepts text control messages for terminal resize (`SessionSocketRoute`). + +That's the entire job of "let an authorized human run a command on a Charlie host". Building a parallel SSH broker inside sherlock would duplicate the CA integration, the host-allow-list logic, and the audit trail — and put another piece of cert-handling code in our blast radius. We do not do that. + +## What `gssh-mcp` actually is + +A thin Go HTTP+WebSocket client to the existing gssh server, wrapped in a stdio MCP. It: + +1. Reads `GSSH_TOKEN` from env at startup (an Authentik JWT minted by the broker, `aud=gssh`). +2. Calls `POST /api/v1/session/initialize` with `Authorization: Bearer ` to materialise the user's session on the gssh server. +3. Caches the list of permitted hosts from `GET /api/v1/session/hosts`. +4. Exposes MCP tools: + - `ssh.list_hosts()` — returns the cached host list. + - `ssh.run(host, command, timeout?)` — opens a WebSocket to the session route for `host`, writes `command + "; echo __SHERLOCK_DONE__$?\n"`, reads until it sees the sentinel, returns `{stdout, exit_code}`. + - `ssh.put_file(host, path, content)` (later) — same shell channel, base64-decode + `tee` on the far side, sentinel as above. +5. On 401, asks the broker for a refreshed token via the UDS RPC and retries once. + +No SSH key handling. No cert minting. No `~/Dev/gssh` shell-out. + +## Endpoints sherlock relies on (gssh contract) + +| Method | Path | Used for | +|---|---|---| +| `POST` | `/api/v1/session/initialize` | Materialise the user's session before opening the WebSocket. | +| `GET` | `/api/v1/session/hosts` | Per-user host allow-list. | +| `GET` | `/api/v1/users/me` | Sanity check / debug. | +| `WS` | `/?hostName=` (exact path defined in gssh's routing) | Bidirectional binary stream = stdin/stdout of the SSH session. Text frames are JSON control messages (e.g. `ResizeMessage`). | + +If a gssh release ever changes one of these, `gssh-mcp` is the only thing in sherlock that needs to follow. + +## Token shape + +- Issuer: `https://id.alexandru.macocian.me/application/o/gssh/` +- Audience: `gssh.alexandru.macocian.me` +- Service kind in the registry: `oidc-federated` +- `exchange.mode = "passthrough"` is safe because gssh already accepts the operator's Authentik ID token (same audience model gssh's web UI uses today). Switch to `rfc8693` only if we ever introduce per-tool scope splitting (e.g. read-only vs write). + +## Completion sentinel — open detail for Phase 3 + +The gssh WebSocket is interactive (PTY-style). `gssh-mcp` needs a deterministic way to know a command has finished. Two options: + +1. **Client-side wrap:** `gssh-mcp` sends `printf '%s\n' "; echo __SHERLOCK_DONE__$?" | ` and parses for the marker. +2. **Server-side one-shot mode:** add a small endpoint / route flag on the gssh server that runs a single command and closes the socket with the exit code in the close frame. Cleaner, but it's a gssh change. + +Phase 0 decision: try (1) first because it requires no gssh change; revisit (2) if we hit edge cases (TTY echo, prompt clutter, multi-line stdout truncation). + +## Out of scope + +- `gssh-mcp` does NOT speak gssh's WebSocket protocol "directly" by hand-rolling SSH packet framing. It just speaks the documented `/api/v1/session/*` HTTP + WS surface. +- `gssh-mcp` does NOT shell out to `/mnt/seagate/Dev/gssh` (which is the C# server source, not a client binary). +- `gssh-mcp` does NOT enforce per-host policy locally. The gssh server already does that against JWT claims; duplicating it client-side is a footgun. diff --git a/docs/service-registry.md b/docs/service-registry.md new file mode 100644 index 0000000..cfe9247 --- /dev/null +++ b/docs/service-registry.md @@ -0,0 +1,115 @@ +# Service registry + +How operators register a downstream service so the broker can mint per-service tokens. + +## Location + +`~/.config/sherlock/services.d/.toml` — one file per service. The broker hot-reloads on change. + +A built-in registry under `internal/agent/profiles/`… no, services are NOT built-in (unlike agent profiles). Every service is operator-registered. + +## Schema + +```toml +[service] +name = "gitea" # unique; matches the file's basename +kind = "oidc-federated" # oidc-federated | own-oauth | static-pat +issuer = "https://id.alexandru.macocian.me/application/o/gitea/" +audience = "gitea.alexandru.macocian.me" +scopes = ["openid", "profile", "email", "read:repository"] + +[exchange] # required when kind = "oidc-federated" +mode = "rfc8693" # rfc8693 | passthrough +subject_token_source = "authentik-id-token" +# Optional: override the token endpoint if it's not the issuer's default. +# token_endpoint = "https://id.alexandru.macocian.me/application/o/token/" + +[mcp] # how the rendered MCP config refers to this service +env_var = "GITEA_TOKEN" # env var the MCP binary reads on startup +command = "gitea-mcp" # executable name (looked up on PATH) +args = [] +``` + +### `kind = "own-oauth"` extras + +```toml +[oauth] +authorization_endpoint = "https://github.com/login/oauth/authorize" +token_endpoint = "https://github.com/login/oauth/access_token" +client_id = "Iv1.abc123" +# client_secret is read from a sibling `.secret.age` file, never inline. +redirect_path = "/cb/github" # appended to the broker's loopback base URL +scopes = ["read:org", "read:user"] +``` + +### `kind = "static-pat"` extras + +```toml +[pat] +# Path to an age-encrypted file containing the PAT. Decrypted by the broker +# on demand using the operator's age key. +file = "~/.config/sherlock/secrets/dockerhub.pat.age" +``` + +## Field reference + +| Field | Required | Notes | +|---|---|---| +| `service.name` | yes | Must equal the filename stem. The broker rejects mismatches. | +| `service.kind` | yes | See [auth-model.md](auth-model.md) for the three grant kinds. | +| `service.issuer` | `oidc-federated` only | OIDC issuer URL; used for metadata discovery. | +| `service.audience` | `oidc-federated` only | Goes into the `audience` parameter of the token-exchange request. | +| `service.scopes` | yes | Default scopes. Override per-invocation with `sherlock run --scope`. | +| `exchange.mode` | `oidc-federated` only | `rfc8693` does a real exchange; `passthrough` hands the existing ID token unchanged (only safe when audiences match). | +| `mcp.env_var` | yes | The MCP binary reads this env var on startup. Convention: `_TOKEN`. | +| `mcp.command` / `mcp.args` | yes | Used by the wrapper when it renders the per-session MCP config for the agent. | + +## Loading + +1. The broker scans `~/.config/sherlock/services.d/*.toml` on startup. +2. On filesystem change (fsnotify), it re-parses and atomically swaps its registry. +3. Parse errors are logged but do not crash the broker — the offending service is just marked unavailable. + +## Worked examples + +### Gitea (Phase 2) + +```toml +[service] +name = "gitea" +kind = "oidc-federated" +issuer = "https://id.alexandru.macocian.me/application/o/gitea/" +audience = "gitea.alexandru.macocian.me" +scopes = ["openid", "profile", "read:repository", "write:issue"] + +[exchange] +mode = "rfc8693" +subject_token_source = "authentik-id-token" + +[mcp] +env_var = "GITEA_TOKEN" +command = "gitea-mcp" +args = [] +``` + +### gssh (Phase 3) + +```toml +[service] +name = "gssh" +kind = "oidc-federated" +issuer = "https://id.alexandru.macocian.me/application/o/gssh/" +audience = "gssh.alexandru.macocian.me" +scopes = ["openid", "profile", "email"] + +[exchange] +mode = "passthrough" +subject_token_source = "authentik-id-token" + +[mcp] +env_var = "GSSH_TOKEN" +command = "gssh-mcp" +args = [] +``` + +See [gssh-integration.md](gssh-integration.md) for why `passthrough` is OK here. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..53f8b58 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module gitea.alexandru.macocian.me/Charlie/sherlock + +go 1.23 diff --git a/internal/agent/doc.go b/internal/agent/doc.go new file mode 100644 index 0000000..ed32480 --- /dev/null +++ b/internal/agent/doc.go @@ -0,0 +1,12 @@ +// Package agent loads agent profiles (Copilot, Claude Code, ...) from +// ~/.config/sherlock/agents.d/ plus built-in defaults, renders the +// per-session MCP config, and exec's the underlying agent CLI with the +// correct argv and environment. +// +// The package is the single integration point between sherlock and any +// supported agent CLI; adding a new agent must be a config-only change +// (a new TOML file under agents.d/), never a code change here. +// +// Implementation lands in Phase 1. See docs/agent-profiles.md for the +// TOML schema. +package agent diff --git a/internal/agent/profiles/copilot.toml b/internal/agent/profiles/copilot.toml new file mode 100644 index 0000000..0f10196 --- /dev/null +++ b/internal/agent/profiles/copilot.toml @@ -0,0 +1,19 @@ +# Built-in agent profile for GitHub Copilot CLI. +# Schema documented at: docs/agent-profiles.md +# This file is embedded into the sherlock binary via //go:embed. + +[agent] +name = "copilot" +executable = "copilot" +description = "GitHub Copilot CLI" + +[mcp] +mode = "cli-flag" +flag = "--mcp-config" + +[argv] +prefix = [] +suffix = [] + +[forbid] +env = [] diff --git a/internal/authn/doc.go b/internal/authn/doc.go new file mode 100644 index 0000000..0a8d069 --- /dev/null +++ b/internal/authn/doc.go @@ -0,0 +1,4 @@ +// Package authn implements the Authentik OIDC/OAuth client: PKCE, +// loopback redirect, refresh, and RFC 8693 token exchange for downstream +// service tokens. Implementation lands in Phase 1. +package authn diff --git a/internal/broker/doc.go b/internal/broker/doc.go new file mode 100644 index 0000000..96effb8 --- /dev/null +++ b/internal/broker/doc.go @@ -0,0 +1,3 @@ +// Package broker owns the long-running daemon loop: lifecycle, RPC server +// wiring, token-cache supervision. Implementation lands in Phase 1. +package broker diff --git a/internal/config/doc.go b/internal/config/doc.go new file mode 100644 index 0000000..e2f904e --- /dev/null +++ b/internal/config/doc.go @@ -0,0 +1,4 @@ +// Package config loads the operator-edited TOML files under +// ~/.config/sherlock/{agents,services}.d/. Implementation lands in +// Phase 1 (agents) and Phase 2 (services). +package config diff --git a/internal/mcp/doc.go b/internal/mcp/doc.go new file mode 100644 index 0000000..671525d --- /dev/null +++ b/internal/mcp/doc.go @@ -0,0 +1,5 @@ +// Package mcp provides shared helpers used by every per-service MCP +// binary: stdio loop, JSON-RPC framing, broker-socket client wrapper, +// 401-triggered single-flight refresh. Implementation lands in Phase 2 +// (first consumer is gitea-mcp). +package mcp diff --git a/internal/socket/doc.go b/internal/socket/doc.go new file mode 100644 index 0000000..84e284f --- /dev/null +++ b/internal/socket/doc.go @@ -0,0 +1,5 @@ +// Package socket implements the Unix-socket RPC transport that the +// broker exposes at $XDG_RUNTIME_DIR/sherlock.sock and that the +// service MCPs (and the sherlock CLI) dial into. Implementation lands +// in Phase 1. +package socket diff --git a/plan.md b/plan.md index bf091fd..9926f5e 100644 --- a/plan.md +++ b/plan.md @@ -158,37 +158,63 @@ Go with **Option C**, but build it in slices so we have something usable after e ## Phased plan +Sliced finer than the original draft so the first end-to-end demo (login + wrapper) lands before any service MCP work. + ### Phase 0 — bootstrap - Create `Charlie/sherlock` repo on Gitea; clone to `~/Dev/charlie/sherlock/`. -- Standard project-charlie repo scaffold: `README.md`, `.sops.yaml`, `.gitignore`, `LICENSE`, CI workflow stub. -- Decide service-registry format (TOML / YAML / JSON, under `~/.config/sherlock/services.d/`). -- Add a row to `project-charlie/docs/inventory.md` and link from `project-charlie/README.md`. +- Sherlock-flavoured scaffold (NOT the per-app stack scaffold — sherlock is operator-side tooling, no `docker-compose.yml`, no `deploy-stack.yml` caller): + - `README.md` (TOC only), `.gitignore` (Go), `go.mod` (`gitea.alexandru.macocian.me/Charlie/sherlock`, Go 1.23). + - Directory skeleton `cmd/{sherlock,sherlock-broker}/`, `internal/{broker,authn,agent,agent/profiles,config,socket,mcp}/`. + - Minimal `cmd/sherlock/main.go` with `version` subcommand so CI has something to build. + - `.gitea/workflows/ci.yaml` — `go vet` + `go test -race` + `go build`. +- `docs/` with one file per topic: `architecture.md`, `auth-model.md`, `service-registry.md`, `agent-profiles.md`, `gssh-integration.md`, `conventions.md`. Service-registry and agent-profile TOML schemas are documented only — loaders land in Phase 1. +- Built-in Copilot agent profile under `internal/agent/profiles/copilot.toml` (embed-only; not yet read by code). +- Add a row to `project-charlie/docs/inventory.md` (new "Operator tooling" section) and link from `project-charlie/README.md`. -### Phase 1 — broker MVP + Gitea MCP (proves the pattern end-to-end) +### Phase 1 — login broker + agent-agnostic wrapper (no MCPs yet) -- `sherlock-broker` with Unix-socket RPC, Authentik OAuth client (PKCE, loopback callback on a stable port), in-memory token cache with disk persistence (encrypted via `age` using the host's existing `/etc/charlie/age.key` or a per-user key). -- `sherlock login` / `sherlock status`. -- `gitea-mcp` (stdio) exposing a minimal toolset: `list_repos`, `get_file`, `list_issues`, `create_issue`, `search_code`. Authenticated to Gitea via OIDC token from Authentik (Gitea already federates). -- `sherlock copilot` wrapper that renders the MCP config and execs. +Smallest slice that proves the auth path and the wrapper indirection end-to-end. No per-service MCP work in this phase. -**Exit criterion:** `sherlock copilot` then `> list my charlie repos` works, no PAT pasted anywhere. +- `sherlock-broker`: + - Authentik PKCE flow with loopback redirect on a stable port. + - Persisted token cache (mechanism — `age`-encrypted blob vs OS keyring — decided at the top of this phase). + - Unix-socket RPC at `$XDG_RUNTIME_DIR/sherlock.sock`: `get_id_token()`, `whoami()`, `status()`. + - Single-flight refresh. +- `sherlock` CLI: + - `sherlock login` / `sherlock logout` / `sherlock status`. + - `sherlock run [args…]` — loads the agent profile, renders the per-session MCP config (empty in Phase 1), `exec`s the agent's binary with the right argv + env. + - `sherlock copilot [args…]` — sugar for `sherlock run copilot`. +- Agent-profile loader in `internal/agent/`. Built-in `copilot.toml` is the only profile shipped; user overrides under `~/.config/sherlock/agents.d/` are picked up. A worked Claude Code profile in `docs/agent-profiles.md` proves the design is extensible. -### Phase 2 — gssh MCP (the interesting one) +**Exit criterion:** `sherlock login` then `sherlock copilot` execs copilot with a (possibly empty) MCP config. `sherlock status` shows a valid JWT. Dropping a `claude.toml` under `agents.d/` makes `sherlock claude` start working with no rebuild. -- `gssh-mcp` exposes `ssh.run(host, command, timeout)` and `ssh.put_file(host, path, content)`. -- Each call: broker hands it a fresh OIDC token → it shells out to `~/Dev/gssh` (or speaks the same WebSocket protocol directly) → returns stdout/stderr/exit code. -- Per-host allow/deny policy in the broker config (don't let Copilot ssh into `radagon` without explicit opt-in). +### Phase 2 — Gitea MCP (first real service) -**Exit criterion:** `> on melina, show me docker ps` works and the SSH session is logged centrally. +- `gitea-mcp` (stdio): `list_repos`, `get_file`, `list_issues`, `create_issue`, `search_code`. +- Broker performs RFC 8693 token exchange against Authentik (or passthrough of the ID token if audiences align). +- The wrapper renders an MCP config listing `gitea-mcp` with the right `env`. -### Phase 3 — Grafana MCP + `sherlock-mcp` consent server +**Exit criterion:** `sherlock copilot` then `> list my charlie repos` works, no PAT pasted anywhere. Per-user audit trail in Gitea logs. + +### Phase 3 — gssh MCP + +Reuses the existing gssh server (`gssh.alexandru.macocian.me`) — sherlock does NOT shell out to a local client and does NOT mint SSH certs. Full design in [docs/gssh-integration.md](docs/gssh-integration.md). + +- `gssh-mcp`: thin Go HTTP+WebSocket client to gssh's `/api/v1/session/*` endpoints, authenticated with an Authentik JWT from the broker. +- Tools: `ssh.run(host, command, timeout)`, `ssh.list_hosts()`, later `ssh.put_file(host, path, content)`. +- Per-host policy is already enforced server-side by gssh's `SessionService` against JWT claims — sherlock does not duplicate it. +- Completion sentinel strategy: `command; echo __SHERLOCK_DONE__$?` in the WebSocket shell stream, parsed client-side. + +**Exit criterion:** `> on melina, show me docker ps` works and the SSH session is logged centrally by gssh. + +### Phase 4 — Grafana MCP + `sherlock-mcp` consent server - `grafana-mcp` with `query`, `list_dashboards`, `get_dashboard`. Auth via Authentik OIDC (Grafana already federates). - `sherlock-mcp` (stdio) with `oauth.consent`, `auth.whoami`, `auth.list_services`, `auth.revoke`. - First "non-Authentik" service (something with its own OAuth — e.g. GitHub.com itself, or a SaaS) registered to exercise the consent path end-to-end. -### Phase 4 — polish +### Phase 5 — polish - `systemd --user` units for the broker. - `sherlock add-service` scaffolder. @@ -201,9 +227,20 @@ Go with **Option C**, but build it in slices so we have something usable after e ## Decided - **Name:** `sherlock`. (Replaces "sherlock" everywhere — `sherlock-broker`, `sherlock-mcp`, `sherlock copilot …`, etc.) -- **Repo location:** new repo `Charlie/sherlock`, cloned to `~/Dev/charlie/sherlock/` (sibling of the existing per-app stack repos like `gitea/`, `runners/`, etc.). Will be added to `docs/inventory.md` and linked from `project-charlie/README.md`. +- **Repo location:** new repo `Charlie/sherlock`, cloned to `~/Dev/charlie/sherlock/` (sibling of the existing per-app stack repos like `gitea/`, `runners/`, etc.). Added to `docs/inventory.md` under a new "Operator tooling" section and linked from `project-charlie/README.md`. +- **Language:** Go. Single static binaries per `cmd//`; best-in-class OAuth/OIDC libs (`golang.org/x/oauth2`, `coreos/go-oidc`); official MCP Go SDK; native UDS + systemd integration. +- **Phase 2 Gitea auth model:** broker mints an OIDC token *as the operator* via Authentik. Every MCP call shows up under the operator in Gitea's audit log. No dedicated `sherlock` service account. +- **Service-registry format:** TOML, under `~/.config/sherlock/services.d/*.toml`. Hot-reloaded by the broker. +- **SSH backend:** reuse the existing gssh server. `gssh-mcp` is a thin JWT-authenticated HTTP+WS client; no local SSH-cert work in sherlock. See [docs/gssh-integration.md](docs/gssh-integration.md). +- **Wrapper extensibility:** agent-agnostic. Each supported agent CLI (Copilot, Claude Code, Aider, …) is described by a TOML profile under `~/.config/sherlock/agents.d/`, with built-in defaults embedded in `internal/agent/profiles/`. `sherlock copilot` and `sherlock claude` are sugar for `sherlock run copilot` / `sherlock run claude`. Adding a new agent MUST be a TOML drop-in, never a code change. +- **Docs convention:** `README.md` is TOC only; every topic lives as its own file under `docs/`. New concerns get new files, never appended sections. ## Still open -1. **Language** for the broker + MCPs (TypeScript / Go / Python). -2. **Phase 1 Gitea auth model** — broker mints an OIDC token *as you* (proper audit trail, but every MCP call shows up under your user in Gitea's log) vs. a dedicated `sherlock` service account (cleaner separation, but loses user attribution). +Tracked here and revisited at the top of the relevant phase. + +1. **Phase 1 — token persistence:** `age`-encrypted blob in `~/.config/sherlock/` vs `libsecret`/`pass` vs OS keyring. +2. **Phase 1 — RPC framing** on the Unix socket: JSON-over-newline vs gRPC-over-UDS. +3. **Phase 1 — broker lifecycle:** does `sherlock run` auto-start a `systemd --user` broker unit (which then needs to exist as part of Phase 1), or does it fork the broker as a child process and let Phase 5 introduce the systemd unit? +4. **Phase 1 — loopback redirect port** for Authentik PKCE; pre-register an OAuth2 provider in Authentik for sherlock (separate from the existing per-app providers). +5. **Phase 3 — gssh completion semantics:** keep the client-side `__SHERLOCK_DONE__` sentinel, or land a server-side one-shot run-and-close mode in gssh.