From 47af17df37542e54ffdeb8362da8e6b403a5b8f5 Mon Sep 17 00:00:00 2001 From: Alexandru Macocian Date: Tue, 5 May 2026 16:18:41 +0200 Subject: [PATCH] Switch deploy-stack to SSH-into-host model; add charlie-ssh composite actions --- .gitea/actions/charlie-ssh-close/action.yml | 10 + .gitea/actions/charlie-ssh-open/action.yml | 35 +++ .gitea/actions/charlie-ssh-target/action.yml | 31 +++ .gitea/actions/sops-decrypt/action.yml | 102 -------- .gitea/workflows/deploy-stack.yml | 224 ++++++++-------- docs/onboarding.md | 257 ++++++++++++++++++- 6 files changed, 453 insertions(+), 206 deletions(-) create mode 100644 .gitea/actions/charlie-ssh-close/action.yml create mode 100644 .gitea/actions/charlie-ssh-open/action.yml create mode 100644 .gitea/actions/charlie-ssh-target/action.yml delete mode 100644 .gitea/actions/sops-decrypt/action.yml diff --git a/.gitea/actions/charlie-ssh-close/action.yml b/.gitea/actions/charlie-ssh-close/action.yml new file mode 100644 index 0000000..b0b0b62 --- /dev/null +++ b/.gitea/actions/charlie-ssh-close/action.yml @@ -0,0 +1,10 @@ +name: charlie-ssh-close +description: | + Tear down the SSH multiplex session opened by `charlie-ssh-open` and wipe + the ephemeral key + cert. Idempotent; intended for `if: always()`. + +runs: + using: composite + steps: + - shell: bash + run: charlie-ssh-close diff --git a/.gitea/actions/charlie-ssh-open/action.yml b/.gitea/actions/charlie-ssh-open/action.yml new file mode 100644 index 0000000..dac4874 --- /dev/null +++ b/.gitea/actions/charlie-ssh-open/action.yml @@ -0,0 +1,35 @@ +name: charlie-ssh-open +description: | + Open a multiplex SSH session to a target host using a short-lived + step-ca-issued cert. Subsequent steps in the same job use + `charlie-ssh-target` to run commands through the multiplex socket; a final + step calls `charlie-ssh-close` (with `if: always()`) to tear it down. + +inputs: + host: + description: Hostname (LAN-resolvable) to SSH into. Maps to runner labels too. + required: true + user: + description: Remote user to SSH in as. + required: false + default: sys-gitea-runner + duration: + description: Cert validity. Max bound by the `runners` provisioner config in step-ca (currently 10m). + required: false + default: 5m + +runs: + using: composite + steps: + - name: Open SSH session + shell: bash + env: + HOST: ${{ inputs.host }} + CHARLIE_SSH_USER: ${{ inputs.user }} + CHARLIE_SSH_DURATION: ${{ inputs.duration }} + run: | + set -euo pipefail + # charlie-ssh-open prints KEY=VAL lines for the session metadata. + # Promote them to job env so charlie-ssh-target / charlie-ssh-close + # in subsequent steps see them. + charlie-ssh-open "$HOST" >> "$GITHUB_ENV" diff --git a/.gitea/actions/charlie-ssh-target/action.yml b/.gitea/actions/charlie-ssh-target/action.yml new file mode 100644 index 0000000..d290352 --- /dev/null +++ b/.gitea/actions/charlie-ssh-target/action.yml @@ -0,0 +1,31 @@ +name: charlie-ssh-target +description: | + Run a command on the SSH target opened by `charlie-ssh-open`. Pass the + command via `run:` (multi-line OK; runs as `bash -lc ` on the + target). The session env (CHARLIE_SSH_*) is read from $GITHUB_ENV + automatically. + +inputs: + run: + description: Shell command to execute on the target. Multi-line allowed. + required: true + workdir: + description: Optional working directory to cd into before running. + required: false + default: "" + +runs: + using: composite + steps: + - shell: bash + env: + CMD: ${{ inputs.run }} + WORKDIR: ${{ inputs.workdir }} + run: | + set -euo pipefail + if [ -n "$WORKDIR" ]; then + remote="cd $(printf %q "$WORKDIR") && $CMD" + else + remote="$CMD" + fi + charlie-ssh-target "$remote" diff --git a/.gitea/actions/sops-decrypt/action.yml b/.gitea/actions/sops-decrypt/action.yml deleted file mode 100644 index 738a406..0000000 --- a/.gitea/actions/sops-decrypt/action.yml +++ /dev/null @@ -1,102 +0,0 @@ -name: SOPS decrypt -description: | - Decrypt one or more SOPS-encrypted files using the host's age key. - - Each line of `decrypt` is a `src -> dst` pair. The destination's extension - determines the output format (`.env`/dotenv, `.json`, `.yaml|.yml`, `.ini`, - anything else = raw/binary). Plaintext files are written with mode 0600. - - Cleanup is the caller's responsibility. The list of decrypted destinations - is exported via `SOPS_DECRYPT_CLEANUP` (path to a newline-separated file) - and as the step output `cleanup-list`. The reusable workflow - `deploy-stack.yml` already wires an `if: always()` cleanup step around it; - if you call this action directly, do the same. - -inputs: - decrypt: - description: | - Multi-line list of `src -> dst` pairs. Example: - secrets/env.sops.yaml -> .env - secrets/config.sops.yaml -> config/app.yaml - secrets/credentials.sops.json -> credentials.json - Blank lines and `#` comments are ignored. - required: true - age-key-file: - description: Path to the host's age private key. - required: false - default: /etc/charlie/age.key - workdir: - description: Working directory the `src` and `dst` paths are resolved against. - required: false - default: ${{ github.workspace }} - -outputs: - cleanup-list: - description: Path to a newline-separated file listing decrypted destinations. - value: ${{ steps.decrypt.outputs.cleanup-list }} - -runs: - using: composite - steps: - - id: decrypt - shell: bash - env: - SOPS_AGE_KEY_FILE: ${{ inputs.age-key-file }} - DECRYPT_INPUT: ${{ inputs.decrypt }} - WORKDIR: ${{ inputs.workdir }} - run: | - set -euo pipefail - - command -v sops >/dev/null 2>&1 || { echo "::error::sops binary not on PATH"; exit 1; } - if [ ! -r "$SOPS_AGE_KEY_FILE" ]; then - echo "::error::age key not readable at $SOPS_AGE_KEY_FILE (need root or appropriate perms)" - exit 1 - fi - - cd "$WORKDIR" - - cleanup_list="${RUNNER_TEMP:-/tmp}/sops-decrypt-cleanup.$$.list" - : > "$cleanup_list" - - while IFS= read -r raw; do - line="${raw%%#*}" - line="$(printf '%s' "$line" | awk '{$1=$1;print}')" - [ -z "$line" ] && continue - - case "$line" in - *"->"*) ;; - *) echo "::error::invalid decrypt entry (missing '->'): $raw"; exit 1 ;; - esac - - src="$(printf '%s' "${line%%->*}" | awk '{$1=$1;print}')" - dst="$(printf '%s' "${line#*->}" | awk '{$1=$1;print}')" - - if [ -z "$src" ] || [ -z "$dst" ]; then - echo "::error::invalid decrypt entry: $raw" - exit 1 - fi - if [ ! -f "$src" ]; then - echo "::error::source not found: $src" - exit 1 - fi - - case "$dst" in - *.env|.env) out_format="dotenv" ;; - *.json) out_format="json" ;; - *.yaml|*.yml) out_format="yaml" ;; - *.ini) out_format="ini" ;; - *) out_format="binary" ;; - esac - - dst_dir="$(dirname -- "$dst")" - [ "$dst_dir" = "." ] || mkdir -p "$dst_dir" - - umask 077 - sops --decrypt --output-type "$out_format" --output "$dst" "$src" - chmod 600 "$dst" - printf '%s\n' "$dst" >> "$cleanup_list" - echo "decrypted: $src -> $dst" - done <<< "$DECRYPT_INPUT" - - echo "cleanup-list=$cleanup_list" >> "$GITHUB_OUTPUT" - echo "SOPS_DECRYPT_CLEANUP=$cleanup_list" >> "$GITHUB_ENV" diff --git a/.gitea/workflows/deploy-stack.yml b/.gitea/workflows/deploy-stack.yml index cd661bd..abce418 100644 --- a/.gitea/workflows/deploy-stack.yml +++ b/.gitea/workflows/deploy-stack.yml @@ -2,6 +2,12 @@ name: Deploy stack # Reusable workflow. Replaces the old per-repo `deploy.sh`. # +# This version uses the SSH-into-host model: the runner is stateless and +# every operation happens on the target host as `sys-gitea-runner` (LDAP, +# member of charlie-deploy) over a short-lived step-ca-issued SSH user +# cert. Sidesteps the Gitea reusable-workflow label-routing bug +# (https://github.com/go-gitea/gitea/issues/32348). +# # Caller pattern (from a stack repo): # # on: @@ -9,22 +15,12 @@ name: Deploy stack # branches: [main] # jobs: # deploy: -# uses: Charlie/project-charlie/.gitea/workflows/deploy-stack.yml@main +# uses: https://sys-gitea-runner:${{ secrets.RUNNER_REPO_PAT }}@gitea.alexandru.macocian.me/Charlie/project-charlie/.gitea/workflows/deploy-stack.yml@main # with: # host: morgott # decrypt: | # secrets/env.sops.yaml -> .env -# -# For a multi-host stack, give it a matrix: -# -# jobs: -# deploy: -# strategy: -# matrix: -# host: [morgott, melina, mohg, miquella] -# uses: Charlie/project-charlie/.gitea/workflows/deploy-stack.yml@main -# with: -# host: ${{ matrix.host }} +# secrets: inherit # # Per-host compose / env resolution mirrors docs/strategy.md ("Shared stacks"): # 1. If `docker-compose..yml` exists -> use only that. @@ -35,7 +31,7 @@ on: workflow_call: inputs: host: - description: Hostname this deploy targets. Must match a self-hosted runner label. + description: Hostname this deploy targets. The runner SSHes in here. required: true type: string stack-path: @@ -47,8 +43,9 @@ on: default: "" decrypt: description: | - Multi-line `src -> dst` SOPS decrypt list. Empty for stacks without secrets. - See `.gitea/actions/sops-decrypt`. + Multi-line `src -> dst` SOPS decrypt list. Empty for stacks without + secrets. Decryption happens on the target host using its + `/etc/charlie/age.key`. required: false type: string default: "" @@ -77,17 +74,10 @@ on: jobs: deploy: - # Both labels required: - # `self-hosted` — magic label that puts act_runner in host-execution - # mode (no docker image pulled for the job container). - # `` — the host short name. Each runner advertises its host - # label uniquely so this disambiguates which runner picks up the job. - # - # Gitea's label matching IS supposed to be AND for arrays, but in - # practice the host label alone is more reliable. We keep both because - # dropping `self-hosted` makes act default to docker mode and pull an - # ubuntu image, which we don't want. - runs-on: [self-hosted, "${{ inputs.host }}"] + # `self-hosted` is the only label requirement now. Routing to the right + # host is done by SSH inside the job, not by Gitea's runner picker. + # Sidesteps Gitea's reusable-workflow `runs-on` matching bug. + runs-on: self-hosted steps: - name: Resolve stack path and ref id: resolve @@ -105,96 +95,130 @@ jobs: if [ -z "$ref" ]; then ref="${GITHUB_REF_NAME:-main}" fi - if [ ! -d "$path/.git" ]; then - echo "::error::stack path '$path' is not a git working tree" - exit 1 - fi echo "path=$path" >> "$GITHUB_OUTPUT" echo "ref=$ref" >> "$GITHUB_OUTPUT" + - name: Open SSH session + uses: https://sys-gitea-runner:${{ secrets.RUNNER_REPO_PAT }}@gitea.alexandru.macocian.me/Charlie/project-charlie/.gitea/actions/charlie-ssh-open@main + with: + host: ${{ inputs.host }} + + - name: Verify stack working tree exists + uses: https://sys-gitea-runner:${{ secrets.RUNNER_REPO_PAT }}@gitea.alexandru.macocian.me/Charlie/project-charlie/.gitea/actions/charlie-ssh-target@main + with: + run: | + test -d "${{ steps.resolve.outputs.path }}/.git" || { + echo "::error::stack path '${{ steps.resolve.outputs.path }}' is not a git working tree" + exit 1 + } + - name: Sync working tree - shell: bash - working-directory: ${{ steps.resolve.outputs.path }} - env: - REF: ${{ steps.resolve.outputs.ref }} - run: | - set -euo pipefail - sudo -n git fetch --prune origin - sudo -n git reset --hard "origin/$REF" - # Keep decrypted .env-style files out of the way; never touch /mnt/nas/data. - sudo -n git clean -fd --exclude='.env' --exclude='secrets/*.plain*' + uses: https://sys-gitea-runner:${{ secrets.RUNNER_REPO_PAT }}@gitea.alexandru.macocian.me/Charlie/project-charlie/.gitea/actions/charlie-ssh-target@main + with: + workdir: ${{ steps.resolve.outputs.path }} + run: | + set -euo pipefail + git fetch --prune origin + git reset --hard "origin/${{ steps.resolve.outputs.ref }}" + # Keep decrypted .env-style files out of the way; never touch /mnt/nas/data. + git clean -fd --exclude='.env' --exclude='secrets/*.plain*' - name: Decrypt SOPS secrets if: ${{ inputs.decrypt != '' }} - # Absolute URL form (with auth) is required so act doesn't default to - # github.com. Same pattern as the caller's `uses:` for this workflow. - # See https://github.com/go-gitea/gitea/issues/25929. - uses: https://sys-gitea-runner:${{ secrets.RUNNER_REPO_PAT }}@gitea.alexandru.macocian.me/Charlie/project-charlie/.gitea/actions/sops-decrypt@main + uses: https://sys-gitea-runner:${{ secrets.RUNNER_REPO_PAT }}@gitea.alexandru.macocian.me/Charlie/project-charlie/.gitea/actions/charlie-ssh-target@main with: - decrypt: ${{ inputs.decrypt }} workdir: ${{ steps.resolve.outputs.path }} + run: | + set -euo pipefail + export SOPS_AGE_KEY_FILE=/etc/charlie/age.key + cleanup_list=$(mktemp) + while IFS= read -r raw; do + line="${raw%%#*}" + line="$(printf '%s' "$line" | awk '{$1=$1;print}')" + [ -z "$line" ] && continue + case "$line" in + *"->"*) ;; + *) echo "::error::invalid decrypt entry (missing '->'): $raw"; exit 1 ;; + esac + src="$(printf '%s' "${line%%->*}" | awk '{$1=$1;print}')" + dst="$(printf '%s' "${line#*->}" | awk '{$1=$1;print}')" + [ -z "$src" ] || [ -z "$dst" ] && { + echo "::error::invalid decrypt entry: $raw"; exit 1; } + [ -f "$src" ] || { echo "::error::source not found: $src"; exit 1; } + case "$dst" in + *.env|.env) out_format="dotenv" ;; + *.json) out_format="json" ;; + *.yaml|*.yml) out_format="yaml" ;; + *.ini) out_format="ini" ;; + *) out_format="binary" ;; + esac + dst_dir="$(dirname -- "$dst")" + [ "$dst_dir" = "." ] || mkdir -p "$dst_dir" + umask 077 + sops --decrypt --output-type "$out_format" --output "$dst" "$src" + chmod 600 "$dst" + printf '%s\n' "$dst" >> "$cleanup_list" + echo "decrypted: $src -> $dst" + done <<< "${{ inputs.decrypt }}" + mv "$cleanup_list" /tmp/charlie-deploy-cleanup.list - name: Resolve compose invocation - id: compose - shell: bash - working-directory: ${{ steps.resolve.outputs.path }} - env: - HOST: ${{ inputs.host }} - run: | - set -euo pipefail - host="$(printf '%s' "$HOST" | tr '[:upper:]' '[:lower:]')" - - files=() - if [ -f "docker-compose.$host.yml" ]; then - files+=(-f "docker-compose.$host.yml") - else - files+=(-f "docker-compose.yml") - [ -f "overrides/$host.yml" ] && files+=(-f "overrides/$host.yml") - fi - - envfile=() - [ -f "envs/$host.env" ] && envfile+=(--env-file "envs/$host.env") - - # Persist as a shell-quoted string the next steps can re-eval. - printf 'compose_args=' > "$GITHUB_OUTPUT.tmp" - printf '%q ' "${envfile[@]}" "${files[@]}" >> "$GITHUB_OUTPUT.tmp" - printf '\n' >> "$GITHUB_OUTPUT.tmp" - cat "$GITHUB_OUTPUT.tmp" >> "$GITHUB_OUTPUT" - rm -f "$GITHUB_OUTPUT.tmp" + uses: https://sys-gitea-runner:${{ secrets.RUNNER_REPO_PAT }}@gitea.alexandru.macocian.me/Charlie/project-charlie/.gitea/actions/charlie-ssh-target@main + with: + workdir: ${{ steps.resolve.outputs.path }} + run: | + set -euo pipefail + host="$(printf '%s' "${{ inputs.host }}" | tr '[:upper:]' '[:lower:]')" + files=() + if [ -f "docker-compose.$host.yml" ]; then + files+=(-f "docker-compose.$host.yml") + else + files+=(-f "docker-compose.yml") + [ -f "overrides/$host.yml" ] && files+=(-f "overrides/$host.yml") + fi + envfile=() + [ -f "envs/$host.env" ] && envfile+=(--env-file "envs/$host.env") + args=$(printf '%q ' "${envfile[@]}" "${files[@]}") + printf '%s\n' "$args" > /tmp/charlie-deploy-compose-args + echo "compose args: $args" - name: Pull images if: ${{ inputs.pull }} - shell: bash - working-directory: ${{ steps.resolve.outputs.path }} - env: - COMPOSE_ARGS: ${{ steps.compose.outputs.compose_args }} - run: | - set -euo pipefail - eval "set -- $COMPOSE_ARGS" - sudo -n docker compose "$@" pull + uses: https://sys-gitea-runner:${{ secrets.RUNNER_REPO_PAT }}@gitea.alexandru.macocian.me/Charlie/project-charlie/.gitea/actions/charlie-ssh-target@main + with: + workdir: ${{ steps.resolve.outputs.path }} + run: | + set -euo pipefail + args=$(cat /tmp/charlie-deploy-compose-args) + eval "set -- $args" + docker compose "$@" pull - name: Up - shell: bash - working-directory: ${{ steps.resolve.outputs.path }} - env: - COMPOSE_ARGS: ${{ steps.compose.outputs.compose_args }} - EXTRA_ARGS: ${{ inputs.compose-args }} - run: | - set -euo pipefail - eval "set -- $COMPOSE_ARGS" - # shellcheck disable=SC2086 - sudo -n docker compose "$@" up -d $EXTRA_ARGS + uses: https://sys-gitea-runner:${{ secrets.RUNNER_REPO_PAT }}@gitea.alexandru.macocian.me/Charlie/project-charlie/.gitea/actions/charlie-ssh-target@main + with: + workdir: ${{ steps.resolve.outputs.path }} + run: | + set -euo pipefail + args=$(cat /tmp/charlie-deploy-compose-args) + eval "set -- $args" + # shellcheck disable=SC2086 + docker compose "$@" up -d ${{ inputs.compose-args }} - name: Cleanup decrypted plaintext + if: ${{ always() && inputs.decrypt != '' }} + uses: https://sys-gitea-runner:${{ secrets.RUNNER_REPO_PAT }}@gitea.alexandru.macocian.me/Charlie/project-charlie/.gitea/actions/charlie-ssh-target@main + with: + workdir: ${{ steps.resolve.outputs.path }} + run: | + set -u + list=/tmp/charlie-deploy-cleanup.list + [ -f "$list" ] || exit 0 + while IFS= read -r f; do + [ -z "$f" ] && continue + rm -f -- "$f" && echo "removed: $f" + done < "$list" + rm -f "$list" /tmp/charlie-deploy-compose-args + + - name: Close SSH session if: always() - shell: bash - working-directory: ${{ steps.resolve.outputs.path }} - run: | - set -u - list="${SOPS_DECRYPT_CLEANUP:-}" - [ -n "$list" ] && [ -f "$list" ] || exit 0 - while IFS= read -r f; do - [ -z "$f" ] && continue - sudo -n rm -f -- "$f" && echo "removed: $f" - done < "$list" - rm -f "$list" + uses: https://sys-gitea-runner:${{ secrets.RUNNER_REPO_PAT }}@gitea.alexandru.macocian.me/Charlie/project-charlie/.gitea/actions/charlie-ssh-close@main diff --git a/docs/onboarding.md b/docs/onboarding.md index b391a0f..04880ea 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -50,9 +50,10 @@ Recover by reversing: paste the note contents back into `~/.config/sops/age/keys 7. [Trusted CA](#trusted-ca) 8. [Host age key (SOPS)](#host-age-key-sops) 9. [LDAP / SSSD](#ldap--sssd) -10. [SSH defaults](#ssh-defaults) +10. [Charlie deploy group](#charlie-deploy-group) +11. [SSH defaults](#ssh-defaults) -Every step except (4) and (9) is required. (4) is required if the host runs containers. (9) is required if humans will SSH into it. +Every step except (4) and (9) is required. (4) is required if the host runs containers. (9) is required if humans will SSH into it. (10) is required if Gitea Actions deploys land here. One-time tasks not tied to a specific host: @@ -181,6 +182,35 @@ sudo launchctl load -w /Library/LaunchDaemons/dev.colima.start.plist > `home/plist/` is Tier 5 backlog — see [`plist`](repos.md#pending--tier-5-long-tail). Path will move once that repo is migrated. +### Charlie registry login + +Stacks that pull images from `gitea.alexandru.macocian.me` (e.g. +`Charlie/runners` -> `amacocian/gitea-runner`, `Charlie/ldap` -> sidecars, +etc.) need authenticated docker pulls. Log in once **as root**, the +credentials persist in `/root/.docker/config.json`: + +```bash +sudo docker login gitea.alexandru.macocian.me +# Username: sys-host- +# Password: +# +# PAT belongs to the host's LDAP system account (sys-host-morgott, +# sys-host-melina, etc.) and lives in vaultwarden as +# "Charlie sys-host- Gitea Docker PAT". +``` + +⚠️ Make sure you ran with `sudo` — `docker login` writes to the running +user's `~/.docker/config.json`. If you log in as your normal user, root +(which is what runs `sudo docker compose pull`) won't find the +credentials. + +Verify: + +```bash +sudo grep -A2 gitea /root/.docker/config.json +sudo docker pull gitea.alexandru.macocian.me/amacocian/gitea-runner:0.6.1-5 +``` + --- ## Portainer agent @@ -593,10 +623,14 @@ Add a row to the [Age public keys (SOPS)](hosts.md#age-public-keys-sops) table i ### Make the host actually able to decrypt -SOPS looks for the age key at `$SOPS_AGE_KEY_FILE`. The [`sops-decrypt`](../.gitea/actions/sops-decrypt/action.yml) composite action sets this to `/etc/charlie/age.key` by default. No system-wide config needed. +SOPS looks for the age key at `$SOPS_AGE_KEY_FILE`. The deploy automation +sets this to `/etc/charlie/age.key` and runs as `sys-gitea-runner` over +SSH — which has read access via the `charlie-deploy` group (see +[Charlie deploy group](#charlie-deploy-group)). Until that section runs, +the key is root-only. ```bash -# Sanity-check that root can read the key (sops always runs as root via the runner's sudo). +# Sanity-check that root can read the key (initially the only reader). sudo test -r /etc/charlie/age.key && echo OK ``` @@ -728,6 +762,221 @@ sudo visudo -cf /etc/sudoers.d/90-ldap-admins --- +## Charlie deploy group + +The `charlie-deploy` LDAP group is the single identity that owns deployment +on every host. Members of this group can: + +- Read/write `/mnt/nas/stacks//` (working trees the deploy workflow operates on) +- Talk to the host's docker daemon (via `/var/run/docker.sock` group ownership) +- Decrypt SOPS-encrypted secrets (read `/etc/charlie/age.key`) + +This setup means the deploy workflow runs **without `sudo`** as +`sys-gitea-runner` (or any future deploy identity). One LDAP group gates +everything; no scoped sudoers to maintain. + +Pre-req: `charlie-deploy` exists in LDAP, `sys-gitea-runner` is a member. +GID is whatever LDAP picked; SSSD resolves it consistently across hosts. + +### Ubuntu + +```bash +# 1. Tell dockerd to use charlie-deploy as the socket-owning group, instead +# of the default local `docker` group. Means LDAP membership in +# charlie-deploy alone grants docker access. No local group juggling. +sudo install -d -m 0755 /etc/docker +sudo tee /etc/docker/daemon.json >/dev/null <<'JSON' +{ + "group": "charlie-deploy" +} +JSON + +# 2. Override docker.socket to use charlie-deploy as the socket group. +# On Ubuntu, dockerd is socket-activated: docker.socket creates +# /var/run/docker.sock with SocketGroup=docker hardcoded BEFORE dockerd +# starts, then dockerd inherits the FD via `fd://`. The `group` setting +# in daemon.json only applies when dockerd creates the socket itself, +# not when systemd hands it one. So override the socket unit too. +sudo install -d -m 0755 /etc/systemd/system/docker.socket.d +sudo tee /etc/systemd/system/docker.socket.d/group.conf >/dev/null <<'INI' +[Socket] +SocketGroup=charlie-deploy +INI + +# 3. Make sure dockerd starts after SSSD is up, so the charlie-deploy group +# is resolvable when docker.service starts. Without this, dockerd may +# fall back to the default group on first boot. +sudo install -d -m 0755 /etc/systemd/system/docker.service.d +sudo tee /etc/systemd/system/docker.service.d/wait-for-sssd.conf >/dev/null <<'INI' +[Unit] +After=sssd.service +Wants=sssd.service +INI + +sudo systemctl daemon-reload +sudo systemctl restart docker.socket docker.service + +# 4. Verify the docker socket is owned by charlie-deploy. +ls -l /var/run/docker.sock +# Expect: srw-rw---- 1 root charlie-deploy ... + +# 5. Verify a charlie-deploy member can talk to docker without sudo. +sudo -u sys-gitea-runner -H docker ps +# Should list containers (or "no containers"), no permission error. +``` + +```bash +# 5. Stack working trees: sys-gitea-runner owns them, charlie-deploy is the +# group, group-writable, with setgid on directories so new files (e.g. +# created by `git fetch`) inherit the charlie-deploy group automatically. +# +# User ownership matters because git refuses to operate on a working tree +# owned by another user ("dubious ownership"). Setting the user to +# sys-gitea-runner avoids needing per-host safe.directory config files. +sudo chown -R sys-gitea-runner:charlie-deploy /mnt/nas/stacks +sudo chmod -R g+w /mnt/nas/stacks +sudo find /mnt/nas/stacks -type d -exec chmod g+s {} \; + +# Verify a charlie-deploy member can do a write against the working tree. +# Pick any stack you have on this host (e.g. homarr). +sudo -u sys-gitea-runner -H -- bash -c ' + cd /mnt/nas/stacks/homarr && + git status +' +``` + +```bash +# 6. SOPS host key: charlie-deploy readable. The directory must also be +# traversable by the group, otherwise sys-gitea-runner can read the file +# permission but not actually open it. +sudo chgrp charlie-deploy /etc/charlie +sudo chmod 0750 /etc/charlie +sudo chgrp charlie-deploy /etc/charlie/age.key +sudo chmod 0640 /etc/charlie/age.key + +ls -ld /etc/charlie /etc/charlie/age.key +sudo -u sys-gitea-runner -H test -r /etc/charlie/age.key && echo OK +``` + +```bash +# 7. Pre-create sys-gitea-runner's home directory and seed it with a Gitea +# PAT so `git fetch/clone/push` against Charlie/* repos works without +# interactive prompts. +# +# LDAP defines the home as /Users/sys-gitea-runner (mac-style; ready for +# a future shared-home rollout). pam_mkhomedir only creates the home on +# interactive login; `sudo -u ... -H` doesn't trigger it, so we make it +# ourselves on each new host. +# +# The PAT is the same one used by the runner image's `RUNNER_REPO_PAT` +# org-level secret. It belongs to sys-gitea-runner and is read-only on +# the Charlie org. Source of truth: vaultwarden secure note +# `Charlie sys-gitea-runner Gitea Repos PAT`. +SR_HOME="/Users/sys-gitea-runner" + +sudo install -d -m 0755 /Users +sudo install -d -m 0700 -o sys-gitea-runner -g charlie-deploy "$SR_HOME" + +PAT="" + +sudo tee "$SR_HOME/.git-credentials" >/dev/null </dev/null <<'EOF' +[credential] + helper = store +EOF +sudo chown sys-gitea-runner:charlie-deploy "$SR_HOME/.gitconfig" +sudo chmod 644 "$SR_HOME/.gitconfig" + +# Verify: should list refs without prompting for a password. +sudo -u sys-gitea-runner -H git ls-remote https://gitea.alexandru.macocian.me/Charlie/runners | head -2 +``` + +> When NFS-shared homes land (planned), `/Users/` becomes the bind mount. +> The per-host `.gitconfig`/`.git-credentials` above become superfluous +> (the shared home brings them along). Migration: delete the local copies +> after the bind takes over. + +```bash +# 8. Set up root's git credentials, using THIS host's system account +# (sys-host-). Used for any sudo git operations done by humans +# or by other root-level automation. The runner uses sys-gitea-runner +# via the credential file in step 7; `root` here is for everything else. +# +# Source of truth: vaultwarden secure note +# `Charlie sys-host- Gitea Repos PAT`. +HOST_PAT="" + +sudo tee /root/.git-credentials >/dev/null </dev/null <<'EOF' +[credential] + helper = store +EOF +sudo chmod 644 /root/.gitconfig + +# Verify +sudo git ls-remote https://gitea.alexandru.macocian.me/Charlie/runners | head -2 +``` + +## PAT identities reference + +Three Gitea identities are used per host. Keep them straight: + +| Identity | Where the PAT lives | Used for | Scopes | +|---|---|---|---| +| `sys-gitea-runner` | `~sys-gitea-runner/.git-credentials` (host) + `RUNNER_REPO_PAT` org secret + runner image env | Runner-side git operations: `act` cross-repo `uses:` clones, runner deploys via SSH-to-host. | `read:repository` | +| `sys-host-` | `/root/.git-credentials` (host) | Root-level git operations on the host: humans doing `sudo git ...`, manual `git fetch && git reset --hard` for fallback deploys. | `read:repository` | +| `sys-host-` (separate PAT) | `/root/.docker/config.json` (host) | Root-level docker registry pulls: `sudo docker compose pull`. | `read:package` | + +Having three separate PATs is deliberate: independent rotation, scope-limited compromise. All three live in vaultwarden under predictable names (`Charlie sys- Gitea PAT`). + +### Verify end-to-end + +`sys-gitea-runner` should now be able to do the entire deploy by hand, +without `sudo`: + +```bash +# Become the deploy identity. -H sets HOME so credential helpers work. +sudo -u sys-gitea-runner -H bash + +# Inside that shell — no sudo for any of these: +cd /mnt/nas/stacks/ +git fetch --prune origin && git reset --hard origin/main +SOPS_AGE_KEY_FILE=/etc/charlie/age.key sops --decrypt --output .env secrets/env.sops.yaml +docker compose pull +docker compose up -d +rm -f .env +``` + +If any step fails with permission errors, fix the corresponding chgrp/chmod +above. The deploy automation calls these exact commands over SSH as +`sys-gitea-runner`; if it works manually, it works in CI. + +### NFS GID mapping caveat + +`/mnt/nas/stacks/` lives on the NAS (DSM/miquella). DSM doesn't know about +LDAP; when you `chgrp charlie-deploy ` on a host, NFS sends the +numeric GID to DSM, which stores it as-is. Other hosts then read it back +and SSSD resolves the GID to `charlie-deploy`. Works because the kernel +cares about GIDs, not names. + +Cosmetic side effect: from a host where SSSD is down, `ls -l` shows the +numeric GID instead of the group name. Functionally fine. + +If DSM's NFS export is configured with `all_squash` or any UID/GID +mapping, this approach breaks. Verify by `chgrp` from one host, `ls` from +another — both should resolve to `charlie-deploy`. + +--- + ## SSH defaults Global readline config (advanced command-line editing):