`9eff67e`

rclone: move shared config from /root/.config to /etc/rclone-shithub.conf

The previous path was unreachable to the postgres user (Postgres
invokes archive_command as itself, /root is mode 0700). Single
file at the new path serves both root-run scripts (backup, sync,
restore-drill, provisioner) and the postgres-run archive_command.

Authored by

espadonne 3 days ago

SHA: 9eff67eb77e9380c69ff81179ff3ddf16bea972f
Parents: 4aa3385
Tree: fecf0a1

10 changed files

Status	File	+	-
M	`deploy/cutover/provision-wal-buckets.sh`	5	5
M	`deploy/docs-site/sync-to-spaces.sh`	1	1
M	`deploy/postgres/archive_command.sh`	1	1
M	`deploy/postgres/backup-daily.sh`	1	1
M	`deploy/restore-drill/run.sh`	2	2
M	`deploy/spaces/sync-cross-region.sh`	2	2
M	`docs/internal/runbooks/backups.md`	3	3
M	`docs/internal/runbooks/day-one.md`	1	1
M	`docs/public/self-host/backup-restore.md`	1	1
M	`docs/public/self-host/troubleshooting.md`	1	1

deploy/cutover/provision-wal-buckets.shmodified


 # both buckets using its on-disk config (which references the just-
 # updated scoped key). A failure here means either the key cache
 # hasn't propagated (wait 30s, re-run) or the scoped key isn't the
+# one in /etc/rclone-shithub.conf (check by hand).
 echo "verifying droplet → both WAL buckets..." >&2
 ssh -o BatchMode=yes "root@$DEPLOY_HOST" "
         set -e
         echo wal-write-probe-\$(date -u +%Y%m%dT%H%M%SZ) \
+                | rclone --config /etc/rclone-shithub.conf \
                         --s3-no-check-bucket \
                         rcat spaces-prod:$WAL_BUCKET/.write-probe
         echo wal-write-probe-\$(date -u +%Y%m%dT%H%M%SZ) \
+                | rclone --config /etc/rclone-shithub.conf \
                         --s3-no-check-bucket \
                         rcat spaces-dr:$WAL_DR_BUCKET/.write-probe
+        rclone --config /etc/rclone-shithub.conf --s3-no-check-bucket \
                 delete spaces-prod:$WAL_BUCKET/.write-probe spaces-dr:$WAL_DR_BUCKET/.write-probe
         echo OK
 "

 
 Verify within ~60s:
   ssh root@$DEPLOY_HOST 'sudo -u postgres psql -xc "SELECT * FROM pg_stat_archiver"'
+  ssh root@$DEPLOY_HOST 'rclone --config /etc/rclone-shithub.conf --s3-no-check-bucket lsf spaces-prod:$WAL_BUCKET/ --recursive | head'
 ==============================================================
 DONE

deploy/docs-site/sync-to-spaces.shmodified


 fi
 
 echo "syncing to $BUCKET..."
+rclone --config /etc/rclone-shithub.conf \
        sync --transfers 8 --checkers 16 \
        build/docs "$BUCKET"
 

deploy/postgres/archive_command.shmodified


 # --s3-no-check-bucket: scoped Spaces keys lack GetBucketLocation; the
 # actual PUT works fine on a key with bucket-level readwrite. Matches
 # the same flag in backup-daily.sh + sync-cross-region.sh.
+rclone --config /etc/rclone-shithub.conf \
        --s3-no-check-bucket \
        --quiet \
        copyto "$SRC" "$BUCKET/$(date +%Y/%m/%d)/$NAME"

deploy/postgres/backup-daily.shmodified


 # --s3-no-check-bucket: skip the GetBucketLocation pre-check that
 # requires a permission our scoped-RW Spaces key doesn't grant.
 # The actual PUT works fine on a key with bucket-level readwrite.
+rclone --config /etc/rclone-shithub.conf --s3-no-check-bucket \
        copyto "$LOCAL_DIR/$NAME" "$BUCKET/daily/$(date -u +%Y/%m/%d)/$NAME"
 
 # Local retention: keep the last 7 dumps; bucket lifecycle handles

deploy/restore-drill/run.shmodified


 # 1. Resolve dump path. --s3-no-check-bucket: scoped Spaces keys lack
 # GetBucketLocation; the actual GET works fine.
 if [[ -z "$DUMP" ]]; then
+  LATEST="$(rclone --config /etc/rclone-shithub.conf --s3-no-check-bucket \
                    lsf "$BUCKET/daily/" --recursive --files-only \
                 | sort | tail -n 1)"
   if [[ -z "$LATEST" ]]; then

   fi
   DUMP="$WORK/$(basename "$LATEST")"
   say "fetching $LATEST"
+  rclone --config /etc/rclone-shithub.conf --s3-no-check-bucket \
          copyto "$BUCKET/daily/$LATEST" "$DUMP"
 fi
 chown postgres:postgres "$DUMP"

deploy/spaces/sync-cross-region.shmodified


 {
   echo "[$(ts)] sync start"
 
+  rclone --config /etc/rclone-shithub.conf --s3-no-check-bucket \
          copy --transfers 8 --checkers 16 --fast-list \
          "$PRIMARY" "$DR"
 
+  rclone --config /etc/rclone-shithub.conf --s3-no-check-bucket \
          copy --transfers 8 --checkers 16 --fast-list \
          "$WAL_PRIMARY" "$WAL_DR"
 

docs/internal/runbooks/backups.mdmodified


    Edit) to grant `readwrite` on `shithub-wal`. The `dr` key needs
    `readwrite` on `shithub-wal-dr` so `sync-cross-region.sh` can push.
 3. **Confirm the rclone config on the app droplet** has both keys
+   (`/etc/rclone-shithub.conf` — `spaces-prod` and
    `spaces-dr` remotes).
 4. **Re-run ansible** (or drop the conf.d file by hand at
    `/etc/postgresql/16/main/conf.d/99_shithub_archive.conf`), then

    # last_archived_wal: 000000010000000000000003 (or similar)
    # last_archived_time: <recent timestamp>
    # failed_count: 0
+   rclone --config /etc/rclone-shithub.conf --s3-no-check-bucket \
           lsf spaces-prod:shithub-wal/ --recursive | head
    ```
 6. **If `failed_count > 0`** before any successful archive:

 
 ```sh
 ssh db
+sudo -u postgres rclone --config /etc/rclone-shithub.conf \
      lsf spaces-prod:shithub-backups/daily/$(date -u +%Y/%m/%d)/
 ```
 

docs/internal/runbooks/day-one.mdmodified


 1. **Backups.** The first daily logical backup should have run.
    ```sh
    ssh db
+   sudo -u postgres rclone --config /etc/rclone-shithub.conf \
         lsf spaces-prod:shithub-backups/daily/$(date -u +%Y/%m/%d)/
    ```
    Should list one `.dump` file. If empty, see

docs/public/self-host/backup-restore.mdmodified


 
 ```sh
 ssh db
+sudo -u postgres rclone --config /etc/rclone-shithub.conf \
      lsf spaces-prod:shithub-backups/daily/$(date -u +%Y/%m/%d)/
 ```
 

docs/public/self-host/troubleshooting.mdmodified


 Confirm by hand:
 
 ```sh
+sudo -u postgres rclone --config /etc/rclone-shithub.conf \
      lsd spaces-prod:
 ```
 

`@@ -127,19 +127,19 @@` doctl spaces keys update "$PROD_KEY_ID" \
127	# both buckets using its on-disk config (which references the just-	127	# both buckets using its on-disk config (which references the just-
128	# updated scoped key). A failure here means either the key cache	128	# updated scoped key). A failure here means either the key cache
129	# hasn't propagated (wait 30s, re-run) or the scoped key isn't the	129	# hasn't propagated (wait 30s, re-run) or the scoped key isn't the
130	-# one in /root/.config/rclone/rclone.conf (check by hand).	130	+# one in /etc/rclone-shithub.conf (check by hand).
131	echo "verifying droplet → both WAL buckets..." >&2	131	echo "verifying droplet → both WAL buckets..." >&2
132	ssh -o BatchMode=yes "root@$DEPLOY_HOST" "	132	ssh -o BatchMode=yes "root@$DEPLOY_HOST" "
133	set -e	133	set -e
134	echo wal-write-probe-\$(date -u +%Y%m%dT%H%M%SZ) \	134	echo wal-write-probe-\$(date -u +%Y%m%dT%H%M%SZ) \
135	- \| rclone --config /root/.config/rclone/rclone.conf \	135	+ \| rclone --config /etc/rclone-shithub.conf \
136	--s3-no-check-bucket \	136	--s3-no-check-bucket \
137	rcat spaces-prod:$WAL_BUCKET/.write-probe	137	rcat spaces-prod:$WAL_BUCKET/.write-probe
138	echo wal-write-probe-\$(date -u +%Y%m%dT%H%M%SZ) \	138	echo wal-write-probe-\$(date -u +%Y%m%dT%H%M%SZ) \
139	- \| rclone --config /root/.config/rclone/rclone.conf \	139	+ \| rclone --config /etc/rclone-shithub.conf \
140	--s3-no-check-bucket \	140	--s3-no-check-bucket \
141	rcat spaces-dr:$WAL_DR_BUCKET/.write-probe	141	rcat spaces-dr:$WAL_DR_BUCKET/.write-probe
142	- rclone --config /root/.config/rclone/rclone.conf --s3-no-check-bucket \	142	+ rclone --config /etc/rclone-shithub.conf --s3-no-check-bucket \
143	delete spaces-prod:$WAL_BUCKET/.write-probe spaces-dr:$WAL_DR_BUCKET/.write-probe	143	delete spaces-prod:$WAL_BUCKET/.write-probe spaces-dr:$WAL_DR_BUCKET/.write-probe
144	echo OK	144	echo OK
145	"	145	"
`@@ -162,6 +162,6 @@` already in place — see the WAL archive PR for that change).
162		162
163	Verify within ~60s:	163	Verify within ~60s:
164	ssh root@$DEPLOY_HOST 'sudo -u postgres psql -xc "SELECT * FROM pg_stat_archiver"'	164	ssh root@$DEPLOY_HOST 'sudo -u postgres psql -xc "SELECT * FROM pg_stat_archiver"'
165	- ssh root@$DEPLOY_HOST 'rclone --config /root/.config/rclone/rclone.conf --s3-no-check-bucket lsf spaces-prod:$WAL_BUCKET/ --recursive \| head'	165	+ ssh root@$DEPLOY_HOST 'rclone --config /etc/rclone-shithub.conf --s3-no-check-bucket lsf spaces-prod:$WAL_BUCKET/ --recursive \| head'
166	==============================================================	166	==============================================================
167	DONE	167	DONE

`@@ -22,7 +22,7 @@` BUCKET="${SHITHUB_WAL_BUCKET:-spaces-prod:shithub-wal}"
22	# --s3-no-check-bucket: scoped Spaces keys lack GetBucketLocation; the	22	# --s3-no-check-bucket: scoped Spaces keys lack GetBucketLocation; the
23	# actual PUT works fine on a key with bucket-level readwrite. Matches	23	# actual PUT works fine on a key with bucket-level readwrite. Matches
24	# the same flag in backup-daily.sh + sync-cross-region.sh.	24	# the same flag in backup-daily.sh + sync-cross-region.sh.
25	-rclone --config /root/.config/rclone/rclone.conf \	25	+rclone --config /etc/rclone-shithub.conf \
26	--s3-no-check-bucket \	26	--s3-no-check-bucket \
27	--quiet \	27	--quiet \
28	copyto "$SRC" "$BUCKET/$(date +%Y/%m/%d)/$NAME"	28	copyto "$SRC" "$BUCKET/$(date +%Y/%m/%d)/$NAME"

`@@ -28,11 +28,11 @@` ts() { date -u +%Y-%m-%dT%H:%M:%SZ; }
28	{	28	{
29	echo "[$(ts)] sync start"	29	echo "[$(ts)] sync start"
30		30
31	- rclone --config /root/.config/rclone/rclone.conf --s3-no-check-bucket \	31	+ rclone --config /etc/rclone-shithub.conf --s3-no-check-bucket \
32	copy --transfers 8 --checkers 16 --fast-list \	32	copy --transfers 8 --checkers 16 --fast-list \
33	"$PRIMARY" "$DR"	33	"$PRIMARY" "$DR"
34		34
35	- rclone --config /root/.config/rclone/rclone.conf --s3-no-check-bucket \	35	+ rclone --config /etc/rclone-shithub.conf --s3-no-check-bucket \
36	copy --transfers 8 --checkers 16 --fast-list \	36	copy --transfers 8 --checkers 16 --fast-list \
37	"$WAL_PRIMARY" "$WAL_DR"	37	"$WAL_PRIMARY" "$WAL_DR"
38		38

`@@ -27,7 +27,7 @@` ships zero WAL segments until the operator runs through this once:
27	Edit) to grant `readwrite` on `shithub-wal`. The `dr` key needs	27	Edit) to grant `readwrite` on `shithub-wal`. The `dr` key needs
28	`readwrite` on `shithub-wal-dr` so `sync-cross-region.sh` can push.	28	`readwrite` on `shithub-wal-dr` so `sync-cross-region.sh` can push.
29	3. Confirm the rclone config on the app droplet has both keys	29	3. Confirm the rclone config on the app droplet has both keys
30	- (`/root/.config/rclone/rclone.conf` — `spaces-prod` and	30	+ (`/etc/rclone-shithub.conf` — `spaces-prod` and
31	`spaces-dr` remotes).	31	`spaces-dr` remotes).
32	4. Re-run ansible (or drop the conf.d file by hand at	32	4. Re-run ansible (or drop the conf.d file by hand at
33	`/etc/postgresql/16/main/conf.d/99_shithub_archive.conf`), then	33	`/etc/postgresql/16/main/conf.d/99_shithub_archive.conf`), then
`@@ -39,7 +39,7 @@` ships zero WAL segments until the operator runs through this once:
39	# last_archived_wal: 000000010000000000000003 (or similar)	39	# last_archived_wal: 000000010000000000000003 (or similar)
40	# last_archived_time: <recent timestamp>	40	# last_archived_time: <recent timestamp>
41	# failed_count: 0	41	# failed_count: 0
42	- rclone --config /root/.config/rclone/rclone.conf --s3-no-check-bucket \	42	+ rclone --config /etc/rclone-shithub.conf --s3-no-check-bucket \
43	lsf spaces-prod:shithub-wal/ --recursive \| head	43	lsf spaces-prod:shithub-wal/ --recursive \| head
44	```	44	```
45	6. If `failed_count > 0` before any successful archive:	45	6. If `failed_count > 0` before any successful archive:
`@@ -62,7 +62,7 @@` If you want to confirm by hand:
62		62
63	```sh	63	```sh
64	ssh db	64	ssh db
65	-sudo -u postgres rclone --config /root/.config/rclone/rclone.conf \	65	+sudo -u postgres rclone --config /etc/rclone-shithub.conf \
66	lsf spaces-prod:shithub-backups/daily/$(date -u +%Y/%m/%d)/	66	lsf spaces-prod:shithub-backups/daily/$(date -u +%Y/%m/%d)/
67	```	67	```
68		68

`@@ -42,7 +42,7 @@` if you haven't.
42	1. Backups. The first daily logical backup should have run.	42	1. Backups. The first daily logical backup should have run.
43	```sh	43	```sh
44	ssh db	44	ssh db
45	- sudo -u postgres rclone --config /root/.config/rclone/rclone.conf \	45	+ sudo -u postgres rclone --config /etc/rclone-shithub.conf \
46	lsf spaces-prod:shithub-backups/daily/$(date -u +%Y/%m/%d)/	46	lsf spaces-prod:shithub-backups/daily/$(date -u +%Y/%m/%d)/
47	```	47	```
48	Should list one `.dump` file. If empty, see	48	Should list one `.dump` file. If empty, see

`@@ -30,7 +30,7 @@` if [[ ! -d build/docs ]]; then
30	fi	30	fi
31		31
32	echo "syncing to $BUCKET..."	32	echo "syncing to $BUCKET..."
33	-rclone --config /root/.config/rclone/rclone.conf \	33	+rclone --config /etc/rclone-shithub.conf \
34	sync --transfers 8 --checkers 16 \	34	sync --transfers 8 --checkers 16 \
35	build/docs "$BUCKET"	35	build/docs "$BUCKET"
36		36

`@@ -32,7 +32,7 @@` pg_restore --list "$LOCAL_DIR/$NAME" >/dev/null
32	# --s3-no-check-bucket: skip the GetBucketLocation pre-check that	32	# --s3-no-check-bucket: skip the GetBucketLocation pre-check that
33	# requires a permission our scoped-RW Spaces key doesn't grant.	33	# requires a permission our scoped-RW Spaces key doesn't grant.
34	# The actual PUT works fine on a key with bucket-level readwrite.	34	# The actual PUT works fine on a key with bucket-level readwrite.
35	-rclone --config /root/.config/rclone/rclone.conf --s3-no-check-bucket \	35	+rclone --config /etc/rclone-shithub.conf --s3-no-check-bucket \
36	copyto "$LOCAL_DIR/$NAME" "$BUCKET/daily/$(date -u +%Y/%m/%d)/$NAME"	36	copyto "$LOCAL_DIR/$NAME" "$BUCKET/daily/$(date -u +%Y/%m/%d)/$NAME"
37		37
38	# Local retention: keep the last 7 dumps; bucket lifecycle handles	38	# Local retention: keep the last 7 dumps; bucket lifecycle handles

`@@ -70,7 +70,7 @@` say "restore drill start (work=$WORK port=$PGPORT pg=$PG_BIN)"
70	# 1. Resolve dump path. --s3-no-check-bucket: scoped Spaces keys lack	70	# 1. Resolve dump path. --s3-no-check-bucket: scoped Spaces keys lack
71	# GetBucketLocation; the actual GET works fine.	71	# GetBucketLocation; the actual GET works fine.
72	if [[ -z "$DUMP" ]]; then	72	if [[ -z "$DUMP" ]]; then
73	- LATEST="$(rclone --config /root/.config/rclone/rclone.conf --s3-no-check-bucket \	73	+ LATEST="$(rclone --config /etc/rclone-shithub.conf --s3-no-check-bucket \
74	lsf "$BUCKET/daily/" --recursive --files-only \	74	lsf "$BUCKET/daily/" --recursive --files-only \
75	\| sort \| tail -n 1)"	75	\| sort \| tail -n 1)"
76	if [[ -z "$LATEST" ]]; then	76	if [[ -z "$LATEST" ]]; then
`@@ -79,7 +79,7 @@` if [[ -z "$DUMP" ]]; then
79	fi	79	fi
80	DUMP="$WORK/$(basename "$LATEST")"	80	DUMP="$WORK/$(basename "$LATEST")"
81	say "fetching $LATEST"	81	say "fetching $LATEST"
82	- rclone --config /root/.config/rclone/rclone.conf --s3-no-check-bucket \	82	+ rclone --config /etc/rclone-shithub.conf --s3-no-check-bucket \
83	copyto "$BUCKET/daily/$LATEST" "$DUMP"	83	copyto "$BUCKET/daily/$LATEST" "$DUMP"
84	fi	84	fi
85	chown postgres:postgres "$DUMP"	85	chown postgres:postgres "$DUMP"

`@@ -28,7 +28,7 @@` By hand:
28		28
29	```sh	29	```sh
30	ssh db	30	ssh db
31	-sudo -u postgres rclone --config /root/.config/rclone/rclone.conf \	31	+sudo -u postgres rclone --config /etc/rclone-shithub.conf \
32	lsf spaces-prod:shithub-backups/daily/$(date -u +%Y/%m/%d)/	32	lsf spaces-prod:shithub-backups/daily/$(date -u +%Y/%m/%d)/
33	```	33	```
34		34

`@@ -107,7 +107,7 @@` will print the rclone error. Common causes:
107	Confirm by hand:	107	Confirm by hand:
108		108
109	```sh	109	```sh
110	-sudo -u postgres rclone --config /root/.config/rclone/rclone.conf \	110	+sudo -u postgres rclone --config /etc/rclone-shithub.conf \
111	lsd spaces-prod:	111	lsd spaces-prod:
112	```	112	```
113		113