actions/runner: scrub claimed secrets on log ingest

Status	File	+	-
M	`cmd/shithubd-runner/run.go`	22	17
M	`internal/actions/queries/workflow_step_log_chunks.sql`	17	0
M	`internal/actions/sqlc/querier.go`	3	0
M	`internal/actions/sqlc/workflow_step_log_chunks.sql.go`	66	0
M	`internal/infra/metrics/metrics.go`	8	0
M	`internal/runner/api/client.go`	2	0
M	`internal/runner/config/config.go`	96	19
M	`internal/runner/config/config_test.go`	38	5
M	`internal/runner/engine/docker.go`	8	0
M	`internal/runner/engine/docker_test.go`	35	0
M	`internal/runner/engine/types.go`	1	0
M	`internal/runner/runner.go`	13	0
M	`internal/runner/scrub/scrub.go`	38	3
M	`internal/runner/scrub/scrub_test.go`	6	0
M	`internal/web/auth_wiring.go`	2	0
M	`internal/web/handlers/api/api.go`	2	0
M	`internal/web/handlers/api/runners.go`	225	25
M	`internal/web/handlers/api/runners_test.go`	195	0
M	`internal/web/server.go`	10	2

cmd/shithubd-runner/run.gomodified

  			SeccompProfile: cfg.Engine.SeccompProfile,
  			User:           cfg.Engine.User,
  			PidsLimit:      cfg.Engine.PidsLimit,
++			DNSServers:     cfg.Engine.DNSServers,
  			Stdout:         os.Stdout,
  			Stderr:         os.Stderr,
  			Logger:         logger,
  	runCmd.Flags().String("seccomp-profile", "", "Container seccomp profile path")
  	runCmd.Flags().String("container-user", "", "Default container user")
  	runCmd.Flags().Int("pids-limit", 0, "Container PID limit")
++	runCmd.Flags().String("network-allowlist", "", "Comma-separated host patterns allowed by the runner DNS policy")
++	runCmd.Flags().String("dns-servers", "", "Comma-separated DNS servers passed to step containers")
  	runCmd.Flags().String("log-level", "", "Log level: debug, info, warn, error")
  	runCmd.Flags().String("log-format", "", "Log format: text or json")
  }
  		"seccomp-profile":   "engine.seccomp_profile",
  		"container-user":    "engine.user",
  		"pids-limit":        "engine.pids_limit",
++		"network-allowlist": "runner.network_allowlist",
++		"dns-servers":       "engine.dns_servers",
  		"log-level":         "log.level",
  		"log-format":        "log.format",
  	}

internal/actions/queries/workflow_step_log_chunks.sqlmodified

  ORDER BY seq ASC
  LIMIT $3;
++-- name: GetStepLogChunkBefore :one
++SELECT id, step_id, seq, chunk, created_at
++FROM workflow_step_log_chunks
++WHERE step_id = $1 AND seq < $2
++ORDER BY seq DESC
++LIMIT 1;
++
++-- name: GetStepLogChunkByStepSeq :one
++SELECT id, step_id, seq, chunk, created_at
++FROM workflow_step_log_chunks
++WHERE step_id = $1 AND seq = $2;
++
++-- name: UpdateStepLogChunk :exec
++UPDATE workflow_step_log_chunks
++SET chunk = $2
++WHERE id = $1;
++
  -- name: DeleteStepLogChunks :exec
  DELETE FROM workflow_step_log_chunks WHERE step_id = $1;

internal/actions/sqlc/querier.gomodified

  	GetRunnerByID(ctx context.Context, db DBTX, id int64) (WorkflowRunner, error)
  	GetRunnerByName(ctx context.Context, db DBTX, name string) (WorkflowRunner, error)
  	GetRunnerByTokenHash(ctx context.Context, db DBTX, tokenHash []byte) (GetRunnerByTokenHashRow, error)
++	GetStepLogChunkBefore(ctx context.Context, db DBTX, arg GetStepLogChunkBeforeParams) (WorkflowStepLogChunk, error)
++	GetStepLogChunkByStepSeq(ctx context.Context, db DBTX, arg GetStepLogChunkByStepSeqParams) (WorkflowStepLogChunk, error)
  	GetWorkflowJobByID(ctx context.Context, db DBTX, id int64) (WorkflowJob, error)
  	GetWorkflowRunByID(ctx context.Context, db DBTX, id int64) (WorkflowRun, error)
  	GetWorkflowStepByID(ctx context.Context, db DBTX, id int64) (WorkflowStep, error)
  	NextRunIndexForRepo(ctx context.Context, db DBTX, repoID int64) (int64, error)
  	RevokeAllTokensForRunner(ctx context.Context, db DBTX, runnerID int64) error
  	TouchRunnerHeartbeat(ctx context.Context, db DBTX, arg TouchRunnerHeartbeatParams) error
++	UpdateStepLogChunk(ctx context.Context, db DBTX, arg UpdateStepLogChunkParams) error
  	UpdateWorkflowJobStatus(ctx context.Context, db DBTX, arg UpdateWorkflowJobStatusParams) (WorkflowJob, error)
  	UpdateWorkflowStepLogObject(ctx context.Context, db DBTX, arg UpdateWorkflowStepLogObjectParams) (WorkflowStep, error)
  	UpdateWorkflowStepStatus(ctx context.Context, db DBTX, arg UpdateWorkflowStepStatusParams) (WorkflowStep, error)

internal/actions/sqlc/workflow_step_log_chunks.sql.gomodified

  	return err
+ }
++const getStepLogChunkBefore = `-- name: GetStepLogChunkBefore :one
++SELECT id, step_id, seq, chunk, created_at
++FROM workflow_step_log_chunks
++WHERE step_id = $1 AND seq < $2
++ORDER BY seq DESC
++LIMIT 1
++`
++
++type GetStepLogChunkBeforeParams struct {
++	StepID int64
++	Seq    int32
++}
++
++func (q *Queries) GetStepLogChunkBefore(ctx context.Context, db DBTX, arg GetStepLogChunkBeforeParams) (WorkflowStepLogChunk, error) {
++	row := db.QueryRow(ctx, getStepLogChunkBefore, arg.StepID, arg.Seq)
++	var i WorkflowStepLogChunk
++	err := row.Scan(
++		&i.ID,
++		&i.StepID,
++		&i.Seq,
++		&i.Chunk,
++		&i.CreatedAt,
++	)
++	return i, err
++}
++
++const getStepLogChunkByStepSeq = `-- name: GetStepLogChunkByStepSeq :one
++SELECT id, step_id, seq, chunk, created_at
++FROM workflow_step_log_chunks
++WHERE step_id = $1 AND seq = $2
++`
++
++type GetStepLogChunkByStepSeqParams struct {
++	StepID int64
++	Seq    int32
++}
++
++func (q *Queries) GetStepLogChunkByStepSeq(ctx context.Context, db DBTX, arg GetStepLogChunkByStepSeqParams) (WorkflowStepLogChunk, error) {
++	row := db.QueryRow(ctx, getStepLogChunkByStepSeq, arg.StepID, arg.Seq)
++	var i WorkflowStepLogChunk
++	err := row.Scan(
++		&i.ID,
++		&i.StepID,
++		&i.Seq,
++		&i.Chunk,
++		&i.CreatedAt,
++	)
++	return i, err
++}
++
  const listStepLogChunks = `-- name: ListStepLogChunks :many
  SELECT id, step_id, seq, chunk, created_at
  FROM workflow_step_log_chunks
+ 	}
  	return items, nil
+ }
++
++const updateStepLogChunk = `-- name: UpdateStepLogChunk :exec
++UPDATE workflow_step_log_chunks
++SET chunk = $2
++WHERE id = $1
++`
++
++type UpdateStepLogChunkParams struct {
++	ID    int64
++	Chunk []byte
++}
++
++func (q *Queries) UpdateStepLogChunk(ctx context.Context, db DBTX, arg UpdateStepLogChunkParams) error {
++	_, err := db.Exec(ctx, updateStepLogChunk, arg.ID, arg.Chunk)
++	return err
++}

internal/infra/metrics/metrics.gomodified

  		},
  		[]string{"result"},
  	)
++	ActionsLogScrubReplacementsTotal = prometheus.NewCounterVec(
++		prometheus.CounterOpts{
++			Name: "shithub_actions_log_scrub_replacements_total",
++			Help: "Total exact secret-value replacements performed on Actions log chunks.",
++		},
++		[]string{"location"},
++	)
  )
  func init() {
  		ActionsRunnerRegistrationsTotal,
  		ActionsRunnerHeartbeatsTotal,
  		ActionsRunnerJWTTotal,
++		ActionsLogScrubReplacementsTotal,
  	)
  }

internal/runner/api/client.gomodified

  	If             string            `json:"if"`
  	TimeoutMinutes int32             `json:"timeout_minutes"`
  	Permissions    json.RawMessage   `json:"permissions"`
++	Secrets        map[string]string `json:"secrets"`
++	MaskValues     []string          `json:"mask_values"`
  	Env            map[string]string `json:"env"`
  	Steps          []Step            `json:"steps"`
  }

internal/runner/config/config.gomodified

  import (
  	"errors"
  	"fmt"
++	"net/netip"
  	"net/url"
  	"os"
  	"reflect"
  	defaultContainerPIDMax = 512
+ )
++var defaultNetworkAllowlist = []string{
++	"api.github.com",
++	"auth.docker.io",
++	"codeload.github.com",
++	"github.com",
++	"objects.githubusercontent.com",
++	"production.cloudflare.docker.com",
++	"registry-1.docker.io",
++	"*.githubusercontent.com",
++}
++
  // LoadOptions controls config resolution. Zero value uses the default path,
  // process environment, and no CLI overrides.
  type LoadOptions struct {
  	PollInterval     time.Duration `toml:"poll_interval"`
  	WorkspaceRoot    string        `toml:"workspace_root"`
  	WorkspaceTTL     time.Duration `toml:"workspace_ttl"`
++	NetworkAllowlist []string      `toml:"network_allowlist"`
+ }
  type EngineConfig struct {
  	SeccompProfile string   `toml:"seccomp_profile"`
  	User           string   `toml:"user"`
  	PidsLimit      int      `toml:"pids_limit"`
++	DNSServers     []string `toml:"dns_servers"`
+ }
  type LogConfig struct {
  			PollInterval:     5 * time.Second,
  			WorkspaceRoot:    "/var/lib/shithubd-runner/workspaces",
  			WorkspaceTTL:     24 * time.Hour,
++			NetworkAllowlist: append([]string{}, defaultNetworkAllowlist...),
  		},
  		Engine: EngineConfig{
  			Kind:           "docker",
  	if c.Runner.WorkspaceTTL <= 0 {
  		return errors.New("runner config: runner.workspace_ttl must be positive")
+ 	}
++	allowlist, err := normalizeHostPatterns(c.Runner.NetworkAllowlist)
++	if err != nil {
++		return fmt.Errorf("runner config: runner.network_allowlist: %w", err)
++	}
++	c.Runner.NetworkAllowlist = allowlist
  	switch strings.ToLower(strings.TrimSpace(c.Engine.Kind)) {
  	case "docker", "podman":
  	if c.Engine.PidsLimit <= 0 {
  		return fmt.Errorf("runner config: engine.pids_limit must be positive, got %d", c.Engine.PidsLimit)
+ 	}
++	dnsServers, err := normalizeDNSServers(c.Engine.DNSServers)
++	if err != nil {
++		return fmt.Errorf("runner config: engine.dns_servers: %w", err)
++	}
++	c.Engine.DNSServers = dnsServers
  	switch strings.ToLower(c.Log.Level) {
  	case "debug", "info", "warn", "error":
  	return nil
+ }
++func normalizeHostPatterns(patterns []string) ([]string, error) {
++	seen := map[string]struct{}{}
++	out := make([]string, 0, len(patterns))
++	for _, p := range patterns {
++		p = strings.ToLower(strings.TrimSpace(p))
++		if p == "" {
++			continue
++		}
++		if strings.ContainsAny(p, "/:") || strings.Trim(p, "*.abcdefghijklmnopqrstuvwxyz0123456789-") != "" {
++			return nil, fmt.Errorf("invalid host pattern %q", p)
++		}
++		if strings.Contains(p, "**") || strings.Contains(p, "..") || strings.HasPrefix(p, ".") || strings.HasSuffix(p, ".") {
++			return nil, fmt.Errorf("invalid host pattern %q", p)
++		}
++		if strings.Contains(p, "*") && !strings.HasPrefix(p, "*.") {
++			return nil, fmt.Errorf("invalid wildcard host pattern %q", p)
++		}
++		if _, ok := seen[p]; ok {
++			continue
++		}
++		seen[p] = struct{}{}
++		out = append(out, p)
++	}
++	if len(out) == 0 {
++		return nil, errors.New("must contain at least one host pattern")
++	}
++	return out, nil
++}
++
++func normalizeDNSServers(servers []string) ([]string, error) {
++	seen := map[string]struct{}{}
++	out := make([]string, 0, len(servers))
++	for _, s := range servers {
++		s = strings.TrimSpace(s)
++		if s == "" {
++			continue
++		}
++		if strings.ContainsAny(s, " \t\r\n") {
++			return nil, fmt.Errorf("invalid DNS server %q", s)
++		}
++		if _, err := netip.ParseAddr(s); err != nil {
++			return nil, fmt.Errorf("invalid DNS server %q", s)
++		}
++		if _, ok := seen[s]; ok {
++			continue
++		}
++		seen[s] = struct{}{}
++		out = append(out, s)
++	}
++	return out, nil
++}
++
  func normalizeLabels(labels []string) ([]string, error) {
  	if len(labels) == 1 && strings.Contains(labels[0], ",") {
  		return runnerlabels.ParseCSV(labels[0])

internal/runner/config/config_test.gomodified

  	if cfg.Engine.PidsLimit != 512 {
  		t.Fatalf("Engine.PidsLimit: %d", cfg.Engine.PidsLimit)
+ 	}
++	if want := []string{"api.github.com", "auth.docker.io", "codeload.github.com", "github.com", "objects.githubusercontent.com", "production.cloudflare.docker.com", "registry-1.docker.io", "*.githubusercontent.com"}; !reflect.DeepEqual(cfg.Runner.NetworkAllowlist, want) {
++		t.Fatalf("NetworkAllowlist: got %#v want %#v", cfg.Runner.NetworkAllowlist, want)
++	}
  	if cfg.Runner.PollInterval != 5*time.Second {
  		t.Fatalf("PollInterval: %v", cfg.Runner.PollInterval)
+ 	}
  poll_interval = "10s"
  workspace_root = "/tmp/file"
  workspace_ttl = "12h"
++network_allowlist = ["github.com", "*.githubusercontent.com"]
  [engine]
  kind = "docker"
  seccomp_profile = "/file/seccomp.json"
  user = "1000:1000"
  pids_limit = 64
++dns_servers = ["172.30.0.10"]
+ `
  	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
  		t.Fatalf("WriteFile: %v", err)
  		Environ: []string{
  			"SHITHUB_RUNNER_TOKEN=alias-token",
  			"SHITHUB_RUNNER_ENGINE__PIDS_LIMIT=256",
++			"SHITHUB_RUNNER_ENGINE__DNS_SERVERS=172.30.0.11,172.30.0.12",
  			"SHITHUB_RUNNER_RUNNER__CAPACITY=3",
  			"SHITHUB_RUNNER_RUNNER__LABELS=self-hosted,linux,x64",
  		},
  			"server.base_url":          "https://flag.example/path/",
  			"runner.capacity":          "4",
  			"runner.poll_interval":     "2s",
++			"runner.network_allowlist": "api.github.com,github.com",
  			"engine.seccomp_profile":   "/flag/seccomp.json",
  			"engine.user":              "123:456",
  		},
  	if cfg.Engine.PidsLimit != 256 {
  		t.Fatalf("PidsLimit: %d", cfg.Engine.PidsLimit)
+ 	}
++	if want := []string{"api.github.com", "github.com"}; !reflect.DeepEqual(cfg.Runner.NetworkAllowlist, want) {
++		t.Fatalf("NetworkAllowlist: got %#v want %#v", cfg.Runner.NetworkAllowlist, want)
++	}
++	if want := []string{"172.30.0.11", "172.30.0.12"}; !reflect.DeepEqual(cfg.Engine.DNSServers, want) {
++		t.Fatalf("DNSServers: got %#v want %#v", cfg.Engine.DNSServers, want)
++	}
+ }
  func TestLoad_RequiresToken(t *testing.T) {
  		t.Fatal("Validate returned nil error")
+ 	}
+ }
++
++func TestValidate_RejectsBadNetworkAllowlist(t *testing.T) {
++	t.Parallel()
++	cfg := Defaults()
++	cfg.Runner.Token = "tok"
++	cfg.Runner.NetworkAllowlist = []string{"https://github.com"}
++	if err := Validate(&cfg); err == nil {
++		t.Fatal("Validate returned nil error")
++	}
++}
++
++func TestValidate_RejectsBadDNSServer(t *testing.T) {
++	t.Parallel()
++	cfg := Defaults()
++	cfg.Runner.Token = "tok"
++	cfg.Engine.DNSServers = []string{"dns.internal"}
++	if err := Validate(&cfg); err == nil {
++		t.Fatal("Validate returned nil error")
++	}
++}

internal/runner/engine/docker.gomodified

  	SeccompProfile   string
  	User             string
  	PidsLimit        int
++	DNSServers       []string
  	LogChunkBytes    int
  	LogFlushInterval time.Duration
  	StepLogLimit     int64
  		"--workdir=" + workdir,
  		"--mount", "type=bind,src=" + job.WorkspaceDir + ",dst=/workspace,rw",
  	}
++	for _, dns := range d.cfg.DNSServers {
++		dns = strings.TrimSpace(dns)
++		if dns != "" {
++			args = append(args, "--dns", dns)
++		}
++	}
  	env, err := validateEnv(rendered.Env)
  	if err != nil {
  		return dockerInvocation{}, err
  		_ = json.Unmarshal([]byte(job.Event), &event)
  	}
  	return expr.Context{
++		Secrets: job.Secrets,
  		Shithub: expr.ShithubContext{
  			Event: event,
  			RunID: fmt.Sprintf("%d", job.RunID),

internal/runner/engine/docker_test.gomodified

+ 	}
+ }
++func TestDockerExecute_AddsConfiguredDNSServers(t *testing.T) {
++	t.Parallel()
++	rec := &recordingRunner{}
++	d := NewDocker(DockerConfig{
++		DefaultImage: "runner-image",
++		Network:      "actions-net",
++		Memory:       "2g",
++		CPUs:         "2",
++		DNSServers:   []string{"172.30.0.10", "172.30.0.11"},
++		Runner:       rec,
++	})
++	if _, err := d.Execute(t.Context(), Job{
++		ID:           1,
++		WorkspaceDir: t.TempDir(),
++		Steps:        []Step{{Run: "curl https://github.com"}},
++	}); err != nil {
++		t.Fatalf("Execute: %v", err)
++	}
++	if argAfterN(rec.args, "--dns", 0) != "172.30.0.10" || argAfterN(rec.args, "--dns", 1) != "172.30.0.11" {
++		t.Fatalf("dns args missing: %#v", rec.args)
++	}
++}
++
  func TestDockerExecute_StreamsStepLogs(t *testing.T) {
  	t.Parallel()
  	d := NewDocker(DockerConfig{
+ 	}
  	return ""
+ }
++
++func argAfterN(args []string, flag string, n int) string {
++	for i, arg := range args {
++		if arg == flag {
++			if n == 0 && i+1 < len(args) {
++				return args[i+1]
++			}
++			n--
++		}
++	}
++	return ""
++}

internal/runner/engine/types.gomodified


 	If             string
 	TimeoutMinutes int32
 	Permissions    json.RawMessage
+	Secrets        map[string]string
 	Env            map[string]string
 	Steps          []Step
 	WorkspaceDir   string

internal/runner/runner.gomodified

  		If:             job.If,
  		TimeoutMinutes: job.TimeoutMinutes,
  		Permissions:    job.Permissions,
++		Secrets:        cloneStringMap(job.Secrets),
  		Env:            job.Env,
  		Steps:          steps,
  		WorkspaceDir:   workspaceDir,
  		Image:          defaultImage,
++		MaskValues:     append([]string{}, job.MaskValues...),
+ 	}
+ }
++func cloneStringMap(in map[string]string) map[string]string {
++	if len(in) == 0 {
++		return nil
++	}
++	out := make(map[string]string, len(in))
++	for k, v := range in {
++		out[k] = v
++	}
++	return out
++}
++
  func defaultSleep(ctx context.Context, d time.Duration) error {
  	timer := time.NewTimer(d)
  	defer timer.Stop()

internal/runner/scrub/scrub.gomodified

  	values       []string
  	replacer     *strings.Replacer
  	tail         string
++	replacements uint64
+ }
  func New(values []string) *Scrubber {
+ 	}
  	emit := combined[:len(combined)-keep]
  	s.tail = combined[len(combined)-keep:]
++	s.replacements += countReplacements(emit, s.values)
  	return []byte(s.replacer.Replace(emit))
+ }
  	if s.replacer == nil {
  		return []byte(tail)
+ 	}
++	s.replacements += countReplacements(tail, s.values)
  	return []byte(s.replacer.Replace(tail))
+ }
++func (s *Scrubber) Replacements() uint64 {
++	if s == nil {
++		return 0
++	}
++	return s.replacements
++}
++
  func normalize(values []string) []string {
  	seen := map[string]struct{}{}
  	out := make([]string, 0, len(values))
  	return out
+ }
++func countReplacements(input string, values []string) uint64 {
++	var count uint64
++	rest := input
++	for rest != "" {
++		bestAt := -1
++		best := ""
++		for _, value := range values {
++			at := strings.Index(rest, value)
++			if at < 0 {
++				continue
++			}
++			if bestAt == -1 || at < bestAt || (at == bestAt && len(value) > len(best)) {
++				bestAt = at
++				best = value
++			}
++		}
++		if bestAt == -1 {
++			return count
++		}
++		count++
++		rest = rest[bestAt+len(best):]
++	}
++	return count
++}
++
  func (s *Scrubber) pendingSuffixLen(combined string) int {
  	keep := 0
  	for _, secret := range s.values {

internal/runner/scrub/scrub_test.gomodified

  	if got != want {
  		t.Fatalf("scrubbed:\ngot  %q\nwant %q", got, want)
  	}
++	if s.Replacements() != 2 {
++		t.Fatalf("replacements: got %d, want 2", s.Replacements())
++	}
  }
  func TestScrubber_MasksAcrossChunkBoundary(t *testing.T) {
  	if got != want {
  		t.Fatalf("scrubbed:\ngot  %q\nwant %q", got, want)
  	}
++	if s.Replacements() != 1 {
++		t.Fatalf("replacements: got %d, want 1", s.Replacements())
++	}
  }
  func TestScrubber_NoSecretsIsCopyingNoop(t *testing.T) {

internal/web/auth_wiring.gomodified

  	pool *pgxpool.Pool,
  	objectStore storage.ObjectStore,
  	runnerJWT *runnerjwt.Signer,
++	secretBox *secretbox.Box,
  	rateLimiter *ratelimit.Limiter,
  	logger *slog.Logger,
  ) (*apih.Handlers, error) {
  		Logger:      logger,
  		ObjectStore: objectStore,
  		RunnerJWT:   runnerJWT,
++		SecretBox:   secretBox,
  		RateLimiter: rateLimiter,
  	})
  }

internal/web/handlers/api/api.gomodified

  	"github.com/tenseleyFlow/shithub/internal/auth/pat"
  	"github.com/tenseleyFlow/shithub/internal/auth/runnerjwt"
++	"github.com/tenseleyFlow/shithub/internal/auth/secretbox"
  	"github.com/tenseleyFlow/shithub/internal/infra/storage"
  	"github.com/tenseleyFlow/shithub/internal/ratelimit"
  	usersdb "github.com/tenseleyFlow/shithub/internal/users/sqlc"
  	Logger      *slog.Logger
  	ObjectStore storage.ObjectStore
  	RunnerJWT   *runnerjwt.Signer
++	SecretBox   *secretbox.Box
  	RateLimiter *ratelimit.Limiter
  }

internal/web/handlers/api/runners.gomodified

  	"io"
  	"net/http"
  	"regexp"
++	"sort"
  	"strconv"
  	"strings"
  	"time"
  	"github.com/tenseleyFlow/shithub/internal/actions/finalize"
  	"github.com/tenseleyFlow/shithub/internal/actions/runnerlabels"
  	"github.com/tenseleyFlow/shithub/internal/actions/runnertoken"
++	"github.com/tenseleyFlow/shithub/internal/actions/secrets"
  	actionsdb "github.com/tenseleyFlow/shithub/internal/actions/sqlc"
  	"github.com/tenseleyFlow/shithub/internal/auth/runnerjwt"
  	"github.com/tenseleyFlow/shithub/internal/checks"
  	checksdb "github.com/tenseleyFlow/shithub/internal/checks/sqlc"
  	"github.com/tenseleyFlow/shithub/internal/infra/metrics"
  	"github.com/tenseleyFlow/shithub/internal/ratelimit"
++	reposdb "github.com/tenseleyFlow/shithub/internal/repos/sqlc"
++	"github.com/tenseleyFlow/shithub/internal/runner/scrub"
  	"github.com/tenseleyFlow/shithub/internal/worker"
+ )
+ 	}
  	metrics.ActionsRunnerHeartbeatsTotal.WithLabelValues("claimed").Inc()
  	metrics.ActionsRunnerJWTTotal.WithLabelValues("issued").Inc()
--	writeJSON(w, http.StatusOK, presentRunnerClaim(job, steps, token, time.Unix(claims.Exp, 0)))
++	resolvedSecrets, err := h.resolveVisibleSecrets(r.Context(), job.RepoID)
++	if err != nil {
++		h.d.Logger.ErrorContext(r.Context(), "runner secret resolution failed", "repo_id", job.RepoID, "job_id", job.ID, "error", err)
++		writeAPIError(w, http.StatusInternalServerError, "runner secret resolution failed")
++		return
++	}
++	writeJSON(w, http.StatusOK, presentRunnerClaim(job, steps, resolvedSecrets, token, time.Unix(claims.Exp, 0)))
+ }
  func (h *Handlers) authenticateRunner(w http.ResponseWriter, r *http.Request) (actionsdb.GetRunnerByTokenHashRow, bool) {
  		writeAPIError(w, http.StatusBadRequest, "chunk must be between 1 and 524288 bytes")
  		return
+ 	}
++	values, err := h.logMaskValues(r.Context(), auth.Claims.RepoID)
++	if err != nil {
++		h.d.Logger.ErrorContext(r.Context(), "runner log mask resolution failed", "repo_id", auth.Claims.RepoID, "job_id", auth.Claims.JobID, "error", err)
++		writeAPIError(w, http.StatusInternalServerError, "log mask resolution failed")
++		return
++	}
  	stepID, ok := h.resolveLogStep(w, r, auth.Job.ID, body.StepID)
  	if !ok {
  		return
+ 	}
--	if _, err := actionsdb.New().AppendStepLogChunk(r.Context(), h.d.Pool, actionsdb.AppendStepLogChunkParams{
++	if err := h.appendScrubbedLogChunk(r.Context(), stepID, body.Seq, chunk, values); err != nil {
--		StepID: stepID,
--		Seq:    body.Seq,
--		Chunk:  chunk,
--	}); err != nil && !errors.Is(err, pgx.ErrNoRows) {
  		writeAPIError(w, http.StatusInternalServerError, "append log failed")
  		return
+ 	}
  	})
+ }
++func (h *Handlers) resolveVisibleSecrets(ctx context.Context, repoID int64) (map[string]string, error) {
++	if h.d.SecretBox == nil {
++		return nil, nil
++	}
++	repo, err := reposdb.New().GetRepoByID(ctx, h.d.Pool, repoID)
++	if err != nil {
++		return nil, err
++	}
++	store := secrets.Deps{Pool: h.d.Pool, Box: h.d.SecretBox, Logger: h.d.Logger}
++	out := map[string]string{}
++	if repo.OwnerOrgID.Valid {
++		if err := h.mergeSecrets(ctx, store, secrets.OrgScope(repo.OwnerOrgID.Int64), out); err != nil {
++			return nil, err
++		}
++	}
++	if err := h.mergeSecrets(ctx, store, secrets.RepoScope(repo.ID), out); err != nil {
++		return nil, err
++	}
++	if len(out) == 0 {
++		return nil, nil
++	}
++	return out, nil
++}
++
++func (h *Handlers) mergeSecrets(ctx context.Context, store secrets.Deps, scope secrets.Scope, out map[string]string) error {
++	items, err := store.List(ctx, scope)
++	if err != nil {
++		return err
++	}
++	for _, item := range items {
++		plaintext, err := store.Get(ctx, scope, item.Name)
++		if err != nil {
++			return err
++		}
++		out[item.Name] = string(plaintext)
++	}
++	return nil
++}
++
++func (h *Handlers) logMaskValues(ctx context.Context, repoID int64) ([]string, error) {
++	resolved, err := h.resolveVisibleSecrets(ctx, repoID)
++	if err != nil {
++		return nil, err
++	}
++	return secretMaskValues(resolved), nil
++}
++
++func secretMaskValues(resolved map[string]string) []string {
++	if len(resolved) == 0 {
++		return nil
++	}
++	values := make([]string, 0, len(resolved))
++	for _, value := range resolved {
++		values = append(values, value)
++	}
++	sort.Strings(values)
++	return values
++}
++
++func cloneStringMap(in map[string]string) map[string]string {
++	if len(in) == 0 {
++		return nil
++	}
++	out := make(map[string]string, len(in))
++	for k, v := range in {
++		out[k] = v
++	}
++	return out
++}
++
++func (h *Handlers) appendScrubbedLogChunk(ctx context.Context, stepID int64, seq int32, chunk []byte, values []string) error {
++	q := actionsdb.New()
++	if len(values) == 0 {
++		_, err := q.AppendStepLogChunk(ctx, h.d.Pool, actionsdb.AppendStepLogChunkParams{
++			StepID: stepID,
++			Seq:    seq,
++			Chunk:  chunk,
++		})
++		if errors.Is(err, pgx.ErrNoRows) {
++			return nil
++		}
++		return err
++	}
++
++	tx, err := h.d.Pool.Begin(ctx)
++	if err != nil {
++		return err
++	}
++	committed := false
++	defer func() {
++		if !committed {
++			_ = tx.Rollback(ctx)
++		}
++	}()
++
++	if _, err := q.GetStepLogChunkByStepSeq(ctx, tx, actionsdb.GetStepLogChunkByStepSeqParams{
++		StepID: stepID,
++		Seq:    seq,
++	}); err == nil {
++		if err := tx.Commit(ctx); err != nil {
++			return err
++		}
++		committed = true
++		return nil
++	} else if !errors.Is(err, pgx.ErrNoRows) {
++		return err
++	}
++
++	var replacements uint64
++	prev, err := q.GetStepLogChunkBefore(ctx, tx, actionsdb.GetStepLogChunkBeforeParams{
++		StepID: stepID,
++		Seq:    seq,
++	})
++	if err == nil {
++		if carry := scrubCarryLen(prev.Chunk, values); carry > 0 {
++			prefix := append([]byte(nil), prev.Chunk[:len(prev.Chunk)-carry]...)
++			combined := append(append([]byte(nil), prev.Chunk[len(prev.Chunk)-carry:]...), chunk...)
++			chunk, replacements = scrubChunk(combined, values)
++			if err := q.UpdateStepLogChunk(ctx, tx, actionsdb.UpdateStepLogChunkParams{
++				ID:    prev.ID,
++				Chunk: prefix,
++			}); err != nil {
++				return err
++			}
++		} else {
++			chunk, replacements = scrubChunk(chunk, values)
++		}
++	} else if errors.Is(err, pgx.ErrNoRows) {
++		chunk, replacements = scrubChunk(chunk, values)
++	} else {
++		return err
++	}
++
++	if _, err := q.AppendStepLogChunk(ctx, tx, actionsdb.AppendStepLogChunkParams{
++		StepID: stepID,
++		Seq:    seq,
++		Chunk:  chunk,
++	}); err != nil && !errors.Is(err, pgx.ErrNoRows) {
++		return err
++	}
++	if err := tx.Commit(ctx); err != nil {
++		return err
++	}
++	committed = true
++	if replacements > 0 {
++		metrics.ActionsLogScrubReplacementsTotal.WithLabelValues("server").Add(float64(replacements))
++	}
++	return nil
++}
++
++func scrubChunk(chunk []byte, values []string) ([]byte, uint64) {
++	if len(values) == 0 {
++		return chunk, 0
++	}
++	s := scrub.New(values)
++	out := s.Scrub(chunk)
++	return append(out, s.Flush()...), s.Replacements()
++}
++
++func scrubCarryLen(chunk []byte, values []string) int {
++	if len(chunk) == 0 || len(values) == 0 {
++		return 0
++	}
++	text := string(chunk)
++	keep := 0
++	for _, value := range values {
++		if value == "" {
++			continue
++		}
++		max := len(value) - 1
++		if max > len(text) {
++			max = len(text)
++		}
++		for n := max; n > keep; n-- {
++			if strings.HasSuffix(text, value[:n]) {
++				keep = n
++				break
++			}
++		}
++	}
++	return keep
++}
++
  func (h *Handlers) writeNextTokenResponse(
  	w http.ResponseWriter,
  	r *http.Request,
  	If             string            `json:"if"`
  	TimeoutMinutes int32             `json:"timeout_minutes"`
  	Permissions    json.RawMessage   `json:"permissions"`
++	Secrets        map[string]string `json:"secrets"`
++	MaskValues     []string          `json:"mask_values"`
  	Env            json.RawMessage   `json:"env"`
  	Steps          []runnerStep      `json:"steps"`
+ }
  func presentRunnerClaim(
  	job actionsdb.ClaimQueuedWorkflowJobRow,
  	steps []actionsdb.ListRunnerStepsForJobRow,
++	resolvedSecrets map[string]string,
  	token string,
  	expiresAt time.Time,
  ) runnerClaimResponse {
  			If:             job.IfExpr,
  			TimeoutMinutes: job.TimeoutMinutes,
  			Permissions:    rawJSONOrObject(job.Permissions),
++			Secrets:        cloneStringMap(resolvedSecrets),
++			MaskValues:     secretMaskValues(resolvedSecrets),
  			Env:            rawJSONOrObject(job.JobEnv),
  			Steps:          outSteps,
  		},

internal/web/handlers/api/runners_test.gomodified

  	"github.com/tenseleyFlow/shithub/internal/actions/finalize"
  	"github.com/tenseleyFlow/shithub/internal/actions/runnertoken"
++	actionsecrets "github.com/tenseleyFlow/shithub/internal/actions/secrets"
  	actionsdb "github.com/tenseleyFlow/shithub/internal/actions/sqlc"
  	"github.com/tenseleyFlow/shithub/internal/actions/trigger"
  	"github.com/tenseleyFlow/shithub/internal/actions/workflow"
  	"github.com/tenseleyFlow/shithub/internal/auth/runnerjwt"
++	"github.com/tenseleyFlow/shithub/internal/auth/secretbox"
  	"github.com/tenseleyFlow/shithub/internal/infra/storage"
  	reposdb "github.com/tenseleyFlow/shithub/internal/repos/sqlc"
  	"github.com/tenseleyFlow/shithub/internal/testing/dbtest"
+ 	}
+ }
++func TestRunnerSecretsAreClaimedAndServerScrubsLogs(t *testing.T) {
++	ctx := context.Background()
++	pool := dbtest.NewTestDB(t)
++	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
++	repoID, userID := setupRunnerAPIRepo(t, pool)
++	runID := enqueueRunnerAPIRun(t, pool, logger, repoID, userID)
++	box := testRunnerAPISecretBox(t)
++	if err := (actionsecrets.Deps{Pool: pool, Box: box}).Set(ctx, actionsecrets.RepoScope(repoID), "TOKEN", []byte("hunter2"), userID); err != nil {
++		t.Fatalf("Set secret: %v", err)
++	}
++
++	token, _ := registerRunnerForTest(t, pool, []string{"ubuntu-latest", "linux"}, 1)
++	signer := runnerAPISigner(t, time.Date(2026, 5, 10, 12, 0, 0, 0, time.UTC))
++	router := newRunnerAPIRouterWithSecretBox(t, pool, logger, signer, box)
++
++	req := httptest.NewRequest(http.MethodPost, "/api/v1/runners/heartbeat",
++		strings.NewReader(`{"labels":["ubuntu-latest","linux"],"capacity":1}`))
++	req.Header.Set("Authorization", "Bearer "+token)
++	rr := httptest.NewRecorder()
++	router.ServeHTTP(rr, req)
++	if rr.Code != http.StatusOK {
++		t.Fatalf("heartbeat status: got %d, want 200; body=%s", rr.Code, rr.Body.String())
++	}
++	var claim struct {
++		Token string `json:"token"`
++		Job   struct {
++			ID         int64             `json:"id"`
++			RunID      int64             `json:"run_id"`
++			Secrets    map[string]string `json:"secrets"`
++			MaskValues []string          `json:"mask_values"`
++		} `json:"job"`
++	}
++	if err := json.Unmarshal(rr.Body.Bytes(), &claim); err != nil {
++		t.Fatalf("decode claim: %v", err)
++	}
++	if claim.Job.RunID != runID || claim.Job.Secrets["TOKEN"] != "hunter2" || !containsString(claim.Job.MaskValues, "hunter2") {
++		t.Fatalf("claim did not include masked secret context: %+v", claim.Job)
++	}
++
++	rawLog := []byte("before hunter2 after\n")
++	logBody := fmt.Sprintf(`{"seq":0,"chunk":%q}`, base64.StdEncoding.EncodeToString(rawLog))
++	req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/v1/jobs/%d/logs", claim.Job.ID), strings.NewReader(logBody))
++	req.Header.Set("Authorization", "Bearer "+claim.Token)
++	rr = httptest.NewRecorder()
++	router.ServeHTTP(rr, req)
++	if rr.Code != http.StatusAccepted {
++		t.Fatalf("logs status: got %d, want 202; body=%s", rr.Code, rr.Body.String())
++	}
++	step, err := actionsdb.New().GetFirstStepForJob(ctx, pool, claim.Job.ID)
++	if err != nil {
++		t.Fatalf("GetFirstStepForJob: %v", err)
++	}
++	chunks, err := actionsdb.New().ListStepLogChunks(ctx, pool, actionsdb.ListStepLogChunksParams{
++		StepID: step.ID,
++		Seq:    -1,
++		Limit:  10,
++	})
++	if err != nil {
++		t.Fatalf("ListStepLogChunks: %v", err)
++	}
++	if len(chunks) != 1 {
++		t.Fatalf("chunks: %#v", chunks)
++	}
++	got := string(chunks[0].Chunk)
++	if strings.Contains(got, "hunter2") || got != "before *** after\n" {
++		t.Fatalf("stored log chunk was not scrubbed: %q", got)
++	}
++}
++
++func TestRunnerServerScrubsSecretSplitAcrossLogPosts(t *testing.T) {
++	ctx := context.Background()
++	pool := dbtest.NewTestDB(t)
++	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
++	repoID, userID := setupRunnerAPIRepo(t, pool)
++	enqueueRunnerAPIRun(t, pool, logger, repoID, userID)
++	box := testRunnerAPISecretBox(t)
++	if err := (actionsecrets.Deps{Pool: pool, Box: box}).Set(ctx, actionsecrets.RepoScope(repoID), "TOKEN", []byte("hunter2"), userID); err != nil {
++		t.Fatalf("Set secret: %v", err)
++	}
++
++	token, _ := registerRunnerForTest(t, pool, []string{"ubuntu-latest", "linux"}, 1)
++	signer := runnerAPISigner(t, time.Date(2026, 5, 10, 12, 0, 0, 0, time.UTC))
++	router := newRunnerAPIRouterWithSecretBox(t, pool, logger, signer, box)
++
++	req := httptest.NewRequest(http.MethodPost, "/api/v1/runners/heartbeat",
++		strings.NewReader(`{"labels":["ubuntu-latest","linux"],"capacity":1}`))
++	req.Header.Set("Authorization", "Bearer "+token)
++	rr := httptest.NewRecorder()
++	router.ServeHTTP(rr, req)
++	if rr.Code != http.StatusOK {
++		t.Fatalf("heartbeat status: got %d, want 200; body=%s", rr.Code, rr.Body.String())
++	}
++	var claim struct {
++		Token string `json:"token"`
++		Job   struct {
++			ID int64 `json:"id"`
++		} `json:"job"`
++	}
++	if err := json.Unmarshal(rr.Body.Bytes(), &claim); err != nil {
++		t.Fatalf("decode claim: %v", err)
++	}
++
++	next := postRunnerLogChunk(t, router, claim.Job.ID, claim.Token, 0, []byte("before hun"))
++	next = postRunnerLogChunk(t, router, claim.Job.ID, next, 1, []byte("ter2 after\n"))
++
++	step, err := actionsdb.New().GetFirstStepForJob(ctx, pool, claim.Job.ID)
++	if err != nil {
++		t.Fatalf("GetFirstStepForJob: %v", err)
++	}
++	chunks, err := actionsdb.New().ListStepLogChunks(ctx, pool, actionsdb.ListStepLogChunksParams{
++		StepID: step.ID,
++		Seq:    -1,
++		Limit:  10,
++	})
++	if err != nil {
++		t.Fatalf("ListStepLogChunks: %v", err)
++	}
++	var combined strings.Builder
++	for _, chunk := range chunks {
++		combined.Write(chunk.Chunk)
++	}
++	got := combined.String()
++	if strings.Contains(got, "hunter2") || got != "before *** after\n" {
++		t.Fatalf("stored log chunks were not scrubbed across boundary: chunks=%#v combined=%q next=%q", chunks, got, next)
++	}
++}
++
  func TestRunnerHeartbeatRejectsBadToken(t *testing.T) {
  	pool := dbtest.NewTestDB(t)
  	router := newRunnerAPIRouter(t, pool, slog.New(slog.NewTextHandler(io.Discard, nil)), runnerAPISigner(t, time.Now()))
+ 	}
+ }
++func postRunnerLogChunk(t *testing.T, router http.Handler, jobID int64, token string, seq int32, chunk []byte) string {
++	t.Helper()
++	body := fmt.Sprintf(`{"seq":%d,"chunk":%q}`, seq, base64.StdEncoding.EncodeToString(chunk))
++	req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/v1/jobs/%d/logs", jobID), strings.NewReader(body))
++	req.Header.Set("Authorization", "Bearer "+token)
++	rr := httptest.NewRecorder()
++	router.ServeHTTP(rr, req)
++	if rr.Code != http.StatusAccepted {
++		t.Fatalf("logs status: got %d, want 202; body=%s", rr.Code, rr.Body.String())
++	}
++	var resp struct {
++		NextToken string `json:"next_token"`
++	}
++	if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
++		t.Fatalf("decode log response: %v", err)
++	}
++	if resp.NextToken == "" {
++		t.Fatalf("empty next token in log response: %s", rr.Body.String())
++	}
++	return resp.NextToken
++}
++
  func newRunnerAPIRouter(
  	t *testing.T,
  	pool *pgxpool.Pool,
  	return r
+ }
++func newRunnerAPIRouterWithSecretBox(
++	t *testing.T,
++	pool *pgxpool.Pool,
++	logger *slog.Logger,
++	signer *runnerjwt.Signer,
++	box *secretbox.Box,
++) http.Handler {
++	t.Helper()
++	h, err := apih.New(apih.Deps{
++		Pool:      pool,
++		Logger:    logger,
++		RunnerJWT: signer,
++		SecretBox: box,
++	})
++	if err != nil {
++		t.Fatalf("api.New: %v", err)
++	}
++	r := chi.NewRouter()
++	h.Mount(r)
++	return r
++}
++
++func testRunnerAPISecretBox(t *testing.T) *secretbox.Box {
++	t.Helper()
++	key, err := secretbox.GenerateKey()
++	if err != nil {
++		t.Fatalf("GenerateKey: %v", err)
++	}
++	box, err := secretbox.FromBytes(key)
++	if err != nil {
++		t.Fatalf("secretbox.FromBytes: %v", err)
++	}
++	return box
++}
++
  func setupRunnerAPIRepo(t *testing.T, pool *pgxpool.Pool) (repoID, userID int64) {
  	t.Helper()
  	ctx := context.Background()
  	return token, runner.ID
+ }
++func containsString(items []string, want string) bool {
++	for _, item := range items {
++		if item == want {
++			return true
++		}
++	}
++	return false
++}
++
  func runnerAPISigner(t *testing.T, now time.Time) *runnerjwt.Signer {
  	t.Helper()
  	signer, err := runnerjwt.NewFromKey(

internal/web/server.gomodified

  	"golang.org/x/crypto/chacha20poly1305"
  	"github.com/tenseleyFlow/shithub/internal/auth/runnerjwt"
++	"github.com/tenseleyFlow/shithub/internal/auth/secretbox"
  	"github.com/tenseleyFlow/shithub/internal/auth/session"
  	"github.com/tenseleyFlow/shithub/internal/infra/config"
  	"github.com/tenseleyFlow/shithub/internal/infra/db"
  		}
  		deps.AuthMounter = auth.Mount
--		var runnerJWT *runnerjwt.Signer
++		var (
++			runnerJWT  *runnerjwt.Signer
++			actionsBox *secretbox.Box
++		)
  		if cfg.Auth.TOTPKeyB64 != "" {
  			runnerJWT, err = runnerjwt.NewFromTOTPKeyB64(cfg.Auth.TOTPKeyB64)
  			if err != nil {
  				return fmt.Errorf("runner jwt: %w", err)
  			}
++			actionsBox, err = secretbox.FromBase64(cfg.Auth.TOTPKeyB64)
++			if err != nil {
++				return fmt.Errorf("actions secretbox: %w", err)
++			}
  		} else {
  			logger.Warn("actions runner API disabled: auth.totp_key_b64 is not configured",
  				"hint", "set SHITHUB_TOTP_KEY=$(openssl rand -base64 32) to enable runner job JWTs")
  		}
--		api, err := buildAPIHandlers(pool, objectStore, runnerJWT, ratelimit.New(pool), logger)
++		api, err := buildAPIHandlers(pool, objectStore, runnerJWT, actionsBox, ratelimit.New(pool), logger)
  		if err != nil {
  			return fmt.Errorf("api handlers: %w", err)
  		}

tenseleyflow/shithub / `07bdfb8`

19 changed files

`@@ -46,6 +46,7 @@` type Job struct {
46	If string	46	If string
47	TimeoutMinutes int32	47	TimeoutMinutes int32
48	Permissions json.RawMessage	48	Permissions json.RawMessage
		49	+ Secrets map[string]string
49	Env map[string]string	50	Env map[string]string
50	Steps []Step	51	Steps []Step
51	WorkspaceDir string	52	WorkspaceDir string