Fixing CloudFlare mount

2023-08-29 22:55:40 -04:00 · 2023-08-29 22:55:40 -04:00 · ca95d1ee76
commit ca95d1ee76
parent 8a56619bf0
8 changed files with 130 additions and 22 deletions
--- a/home/.chezmoiscripts/disabled/run_onchange_after_14-gcloud.sh.tmpl
+++ b/home/.chezmoiscripts/disabled/run_onchange_after_14-gcloud.sh.tmpl
@ -0,0 +1,40 @@
+{{- if false -}}
+#!/usr/bin/env bash
+# @file Google Cloud Platform
+# @brief Ensures S3 buckets are created
+# @description
+#     This script ensures that the S3 buckets are created on Google Cloud Platform. It creates
+#     the following buckets:
+#
+#     - gs://docker.s3.{{ .host.domain }}
+#     - gs://private.s3.{{ .host.domain }}
+#     - gs://public.s3.{{ .host.domain }}
+#     - gs://system.s3.{{ .host.domain }}
+#     - gs://user.s3.{{ .host.domain }}
+
+exit 0
+
+if command -v gcloud > /dev/null; then
+  if [ -f "${$XDG_CONFIG_HOME:-$HOME/.config}/gcloud/s3-client-secret.json" ]; then
+    logg info 'Authenticating with gcloud S3 service account'
+    echo Y | gcloud auth login --cred-file=s3-gcp.json
+    if gcloud config get-value account > /dev/null && logg info 'gcloud is authenticated'; then
+      logg info 'Disabling usage reporting' && gcloud config set disable_usage_reporting true
+      BUCKETS="$(gcloud storage buckets list | grep 'storage_url:')"
+      for BUCKET in "docker" "private" "public" "system" "user"; do
+        if logg info "Checking for $BUCKET S3 presence on GCP" && ! echo "$BUCKETS" | grep "gs://${BUCKET}.s3.{{ .host.domain }}" > /dev/null; then
+          logg info "Creating gs://${BUCKET}.s3.{{ .host.domain }}" && gcloud storage buckets create "gs://${BUCKET}.s3.{{ .host.domain }}"
+        fi
+      done
+      logg info 'Finished ensuring S3 buckets on gcloud are created'
+    else
+      logg error 'gcloud is not authenticated'
+    fi
+  else
+    logg info 'No client secret available for gcloud S3 service account'
+  fi
+else
+  logg error 'The gcloud executable is not available'
+fi
+
+{{ end -}}
--- a/home/.chezmoitemplates/files/gcp-s3-client-secret
+++ b/home/.chezmoitemplates/files/gcp-s3-client-secret
@ -0,0 +1,56 @@
+-----BEGIN AGE ENCRYPTED FILE-----
+YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzaGV4ZFVZdGF1K2ZBa0JV
+WW9FVHF2V29KY3FVOFd4eURSVkxBTFI1ZkMwClRwaW1aSWQyYjM4NTdXSmx4OWZz
+bVhwaWtURU1iWWl2Y0l5b3c3bUFBUVEKLS0tIHVpQzVkVDJDMUhCUTVFbXUya0ZZ
+ZUVuYTFSRXd6VnUzTW5wLzNVN1RhY2cK6lva90NCZuyEaavciX3n8gHA0jFbq/h7
+BVD1gSeVDB833p1SdfmmFgqjJk8fGuxRhWKzkzO/3Ip6iXvXXygcOBcp3RZytCGd
+mJHWZ7jCSOr+Z1NypzdvkHY8iLkvZwQC1q7BK8t8WDSd8/SnIQWbLm1UbT6oJlbG
+z/51YCs2bsV6fW3OxXUjqqFe57wTOvO4kaQ7Bjhr0k5LQ6vFKZIyBcgUv5mlN+6w
+sxG5fimcqIYF2FKSL+D+D0HA9lZth+84DT4qR1D9n/27bpky3o5BRyjbTlf+KUnR
+hP/woWKwu9z5aGfjlQXNWbithCE88sz3m0RAHXpcT4T8HleVr6vKhPd+4aRy7pwV
+wAVh8tfx+BEM1/UkVTxswChrkavzOcBa7QEM8udnFfPzHwtVAN1si+4Wp509Zd+9
+FiyOz80pxsQB10HefSG9oPp8zzY8gqTRcpxqFToi2D0CmvRWJMsOzN4i0t3EMwK2
+F2nHAF8DPmurIY5HOOCt2RccplzanRJITrw1yNDRinp129cyIdXGDZjmwCUA2xgN
+dvJpGpeiffmkkEx4U36HTlFSCBZeCCF8WJbg8P2wQ/efJG+vwH/vWl48mKRkpMVN
+cebATEPWaFK3R84ebxJarro9E/QklYn7UBOMbMY/FCiWaglPr+gZlxizL0FmpzYx
+uenurhobbApQc/03uXXsHILhBkkg3Q5VvLzgn51BoKj4Sa69GVcnamsoZ6sda19i
+MVwgF0enTg0vqkqDZZbO/DD6YzXXIBi64AQs5f9I2eCjSRPtUBCgKhhbq8LRuoHk
+n8XFK/+Dfd2zF9iW67kKq2qhnr21dJ5Vn/HF+eCR1lWT4hg3OaqXPN0jxGvhbBzy
+Fy2S9awEyUDSS010f5IPZlqL6H501+aUrzWmydgS1PONIt3pGqKatPApp5hHrtmV
+Zmlv+k3MU1OGImFqfP+OuMeyiC7YkuxUcGDuOGqnMSzqSbIrTO3GaiBkSfjUZEbq
+p72f5dEWANlftdx3GJgQtDZLgV5vQFqp5fkNfUjtzLEk+tFIoBi7NefcjMnBdy15
+EArGLN5dYRMooVlBcKYqOoLCet8z4DMx33IomYeVLrurVGIAV4DuA043ZMG5cUn0
+v2q7vmowOjvqGTIIWaufWZqEkgJrGmygrR32iq/Rbi3eycEGACDyUN7arzCIsQuw
+c19IePXm4lD5VmOft3TYYYKoQ9avzg6+mY89uZuO8qK1hwj/JJMV2BK00hKKpGMx
+11+k7Q2RCSQq1c3+yAsRSVxnEyh/EIbd8RgLIyFBx1RLGs09RSxPzOXLyvW34zPj
+YN9yhcZNDFZYJEO9RxnRSkEn9pEcL8AY/0lhPxLc8DKsrU8o+ty2Vq7Yc1zOkiWh
+KZMS6QPJO+FtvPm1vAoLduXM1ImVCImBxftf3izjeQ0/UvawCeW6iogWw3Ok6xDY
+HF/wGUZc+A8fIg8iQIcxZffPHefIU8nFHIgXXazqt3Y7tOyPgc7sZQyJnBVmFpJ
+i3h6J2LUskcVjbeHXx64iskbgznInqW+a2UfnOwIRewKeiZtpDHD4uXigVOkrYJf
+GxeRI3JAxnbooYJRPnTdfQJPbZltEPjkcT+A6nUbXK6oU8oZAfWf5oJ7Hqbp+NvX
+/XlpYMoeLid+m/RZE6BHtOUZSTMICcduIHKq8VRdNjdv0MtZXLWLynK1Xv+yxQom
+WdLX3RHKoUq6ygD3j7/xYCpNJcHm47eJI4fxlBXZ6S1NMSlSsorOU1XWVpz/Jwbv
+sKrvYO1UthaBo8wMYhuCPhoAYhtZD8pcYunpZnjiiEy8YV0nVPy74/KeEWKkdKIY
+VCNVsUG3/Mm3sMhjt3e2pnsraGuTuaYW6oRPfX9gYoctSaFAqvPSl20kE9IOghxp
+ojw9Rl32Mlc1ThJEaXN0dKhnv2/OpFgUe1UGRlM4H+aLvhQxDjR+B2grPGJYr5tF
+S8oXF52N/AAHi4OB2Hib6qETEqk+oivBzZhmsXGGXl12mXk6GZbcAxBXbSIXiUuc
+SEia4SnEipchiew28w7P/Xh56eOH7e0g5+3fBmGIumQ+9MomEsF3bONmtDs+asgu
+zyPEI1dAbEWhgfCMvP5nj0NK3SRJ6c7jPnR1GYLEEoMoN7SXG32DX/OoUWKEKFp5
+JkAd/TnVw+nYtXsShQfpllryzelFUuOWP3xS40jpUAatnkntcDsh2TBnhwdYKH0i
+5EWpSHZDQFdVAIxVF2bbPWx0vjEBBGmCkfDSZlBvMBV13LBaHgIXNtfL73wC++Wk
+lhJQT6DCncsFZJAZ3TkS/Cms4rOHXd0N0Rwq3NHYNc58jhRcgwXJLrnX8uRgmDS+
+ByBtXiFZWiH9WVx3bz48Mks8blOgGcGj4Mi22lxnwtXRJ3BiuQtxpmwf4SK9Nr0W
+wWPWFcH6V9ZBlFmbxSWri+vC+ZO6+2/1dkqYWLGCpoLReZAu6Uy6WLWoUTjjIAZ/
+hLjAVgsSWQWfkJj+f0WnhMbKovPyKjM8NPCzlfNfHFi5njmtmHgm2Qba8NdSho4T
+vkBfi6kYhsFwU0zH7Q/kcgV9ifsL033SecVs9LjsmJXvpa+BrctvrrfE3X6iIszi
+FoPxjODzH3I5VU3sbDxILUlAtT/I1wb3Ad9h5Qq/XfYs/xsEIf4DiVDiT7kjjzyM
+IswwLVctpPLKLSII6YONgQ28sQCYtXYxk3qwswH5cTpEdPZpzs3n4aTus3GMWHj5
+oH+TdiYphNHawhElcQ/0IGCyi5ppiaRdTp4IedXyY+brpSUn8gy0NJvLwnEzRHuT
+7lw0tpL9TV2QYIg9E0KbRu8dyM08vQ6RyS+08Lk0DQe3Lg1fPFg3e5mzzNI2A7wh
+kQAslX3VbIRpJF6RDMEgcxO75DkmRdQ702i4t6A7aqbN/PsFgVLo1N+0F80Req6X
+SsUg155ET/d46QYxQ8i2o1e/wmfHGvS4oarD4xhF3Ujv6ZsodHJ9vfkDibiO7zgS
+wRUMrE7MPG7GBytlpOX4lWOWGmhfQrl1MjHyXMq7OJZPuCLZgyYddUztGX2Vs18V
+gE+nYO03JbLrTMLwIFNoh/VRrbWaXMmvbW1H1NaIhnBaxQNGZlru6PjgoLcAzn9/
+lKlLz28EahOy5eAY2n/6r/UvnwNrnhHlPDC5C7t3mxM08MA9C0mlP7paiB3twCkF
+YFGgDphmginqXdYFpy2c8OU=
+-----END AGE ENCRYPTED FILE-----
--- a/home/.chezmoitemplates/secrets/CLOUDFLARE_R2_ID
+++ b/home/.chezmoitemplates/secrets/CLOUDFLARE_R2_ID
@ -1,7 +1,7 @@
 -----BEGIN AGE ENCRYPTED FILE-----
-YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2bFM5VUFoTDlxb2NjV2RV
-d0UvM2pPYWZMeDRZeDZKYmZ3YlhwNlBsYlRvCmVTcEVzUndwSG1lQ3pKTFpxZ1Bs
-NGtXcksrNnRmR1UxOXR2UGpiNHplOHcKLS0tIFBEZHBibnEzSnBxTUlxcHdQQmhT
-MlUyZnRHWHY5UE43OXV1cFJjUnJGRHcK9s3V7BN+uHHJt8ekqFpP0XYaa+WwanmW
-qQ7rr6AB5ZT7z8y9vpQNK+mzuB49zL87AiNspAacKP/RtKNUPmdEzpY=
+YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5NDdQM2xxaFlPeVpkNVN5
+em1OWXVKdG5yN2V0SDVmRnF3c2o5MXpnZ2lzCkdWQVZkaGhVZUUvQWt5YkZkNzBk
+eUtsS1UxRGRDUFNjZ3ZSOXorQ21mVWsKLS0tIGllSGpiUFdpL3VGT01qZDlsTTAz
+MFYwWmJVbWpBQ2lQby9Zc0hxd1lIa28KxEVddgAMlDC/ySyUoGiRfi4Iya41iByf
+cw7E/FVT6+9iiaW1tfXYpeRUFbXinR8HiiJavbUO7vfT8/0rdmd/7g==
 -----END AGE ENCRYPTED FILE-----
--- a/home/.chezmoitemplates/secrets/CLOUDFLARE_R2_SECRET
+++ b/home/.chezmoitemplates/secrets/CLOUDFLARE_R2_SECRET
@ -1,8 +1,8 @@
 -----BEGIN AGE ENCRYPTED FILE-----
-YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpdjhHNWJEMlJjNTlCUmJv
-RzRvUzRhUmR6OWpxWTVudDJ0NnVqbklqQmlrClhyNWpSZEZ1SHpEU0FROWZFYzlL
-RmhEbmJ1ZWJtS2xjNmRsaVhZb3ExK0UKLS0tIE90dzZ5T0liQitNV0hQTHNmcFlj
-eEdKZWdvK0NOdU1PK3I1NGxmTEVtQWsKJWhE2Q5wCLtvy7ZrrPwNvceLWEp7rV9I
-YEVpLY6lWuHWIbg6h8GkwlrbP/e3evFpZ7T9eLmhsMIfYm7hPtYV3BkASNqpWRh/
-o94FfrDqtg7Nu1/pZO8o/dt7QnVh0lMPYw==
+YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMOE5nRHY3bE1rNE5KVDZW
+TkVHVHdqa0MvNW91UFAyYU5ZMWIyb3Bkb2pJCmV0N1VCdFdjZ0xiRzV6bXl4WHNO
+M0xpUXRINHkxUi96Ym96S0dQSnVwOG8KLS0tIGpEN2ozWUZ2aU91cWc4dHYzamtr
+MUhCanIzYWdpWVFJUnBDdXpHQytYYzgKtAbLV9I58aZWYeViHFKQCZA6mf0+dK4y
+Ac6W5tQ/Fjk+ge+E5uNgLi22G3PW9qyshwJk/axVa/0wpMZHM2HtEvyIrBEj4YsQ
+EIFNsM0Llui6GOJHPbq423j1V0ZvhlIe
 -----END AGE ENCRYPTED FILE-----
--- a/home/.chezmoitemplates/secrets/CLOUDFLARE_R2_TOKEN
+++ b/home/.chezmoitemplates/secrets/CLOUDFLARE_R2_TOKEN
@ -0,0 +1,7 @@
+-----BEGIN AGE ENCRYPTED FILE-----
+YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBWEFveDRNY0laM3BPSDVK
+VktjeVU1dGd1QU9zWWV0b3VBT2FKN1MxY1NVCk5OcTBnNlg4TU44OWV1b3BiOUhD
+SWFmeHA0NWdFTCtFMUhlWFVoK1RSZmMKLS0tIFBxREdEMEV1Y2FRTHdCSzRCZXps
+TFI4Q2t6Rm9HY1Q5cll6d1IwV3dOeUkKc8+PHTsEBFLpd37vxIYVSj6dbEv27HQG
+9UwZMEKsY2FmEQ+24i4L+q0CYi84BPgnx4LfUWQiZs3ZJ+8b0/Jh/9/IdHD3/VoG
+-----END AGE ENCRYPTED FILE-----
--- a/home/dot_config/gcp/gcp.json.TODO
+++ b/home/dot_config/gcp/gcp.json.TODO
--- a/home/dot_config/rclone/private_rclone.conf.tmpl
+++ b/home/dot_config/rclone/private_rclone.conf.tmpl
@ -1,5 +1,5 @@
 {{- if and (or (and (stat (joinPath .host.home ".config" "age" "chezmoi.txt")) (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_ID"))) (env "CLOUDFLARE_R2_ID")) (or (and (stat (joinPath .host.home ".config" "age" "chezmoi.txt")) (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_SECRET"))) (env "CLOUDFLARE_R2_SECRET")) (ne .user.cloudflare.r2 "") -}}
-# MEGABYTE LABS MANAGED S3
+# INSTALL DOCTOR MANAGED S3
 [{{ .user.username}}-s3]
 access_key_id = {{ if (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_ID")) }}{{- includeTemplate "secrets/CLOUDFLARE_R2_ID" | decrypt | trim -}}{{ else }}{{- env "CLOUDFLARE_R2_ID" -}}{{ end }}
 acl = private
@ -8,6 +8,5 @@ provider = Cloudflare
 region = auto
 secret_access_key = {{ if (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_SECRET")) }}{{- includeTemplate "secrets/CLOUDFLARE_R2_SECRET" | decrypt | trim -}}{{ else }}{{- env "CLOUDFLARE_R2_SECRET" -}}{{ end }}
 type = s3
-# MEGABYTE LABS MANAGED S3
-EOT
+# INSTALL DOCTOR MANAGED S3
 {{- end -}}
--- a/home/dot_local/bin/executable_rclone-mount
+++ b/home/dot_local/bin/executable_rclone-mount
@ -14,7 +14,7 @@ if [ "$TYPE" = 'user' ]; then
  fi
  CACHE_FOLDER="/$USER_FOLDER/$USER/.cache/rclone"
  CONFIG_FOLDER="/$USER_FOLDER/$USER/.config/rclone"
-  LOG_FOLDER="/$USER_FOLDER/$USER/.local/log"
+  LOG_FOLDER="/$USER_FOLDER/$USER/.local/share/rclone"
  LOG_FILE="$LOG_FOLDER/$MOUNT.log"
  MOUNT_PATH="/$USER_FOLDER/$USER/.local/mnt/$MOUNT"
 else
@ -46,7 +46,13 @@ if [ ! -f "$RCLONE_IGNORE" ] && [ -f "/etc/rcloneignore" ]; then
 fi

 ### Mount
-MOUNT_LOWERCASE="$(echo "$MOUNT" | tr "[:upper:]" "[:lower:]")"
+if [ "$TYPE" == 'user' ]; then
+  MOUNT_REF="$USER-$MOUNT"
+  MOUNT_LOWERCASE="user/$USER-$MOUNT"
+else
+  MOUNT_REF="$MOUNT"
+  MOUNT_LOWERCASE="$(echo "$MOUNT" | tr "[:upper:]" "[:lower:]")"
+fi
 unset AWS_CA_BUNDLE
 export PATH="$PATH:/usr/local/bin:/usr/bin"
 # TODO: Only launch with --rc-web-gui if the servers hostname is the {{ .kubernetesHost }}
@ -56,11 +62,11 @@ sudo rclone --config "$CONFIG_FOLDER/rclone.conf" \
  --allow-other \
  --buffer-size 4G \
  --bwlimit 40M \
-  --cache-chunk-path "$CACHE_FOLDER/$MOUNT-chunks" \
-  --cache-db-path "$CACHE_FOLDER/$MOUNT-db" \
-  --cache-dir "$CACHE_FOLDER/$MOUNT-vfs" \
+  --cache-chunk-path "$CACHE_FOLDER/$MOUNT_REF-chunks" \
+  --cache-db-path "$CACHE_FOLDER/$MOUNT_REF-db" \
+  --cache-dir "$CACHE_FOLDER/$MOUNT_REF-vfs" \
  --cache-info-age 60m \
-  --cache-tmp-upload-path "$CACHE_FOLDER/$MOUNT-upload" \
+  --cache-tmp-upload-path "$CACHE_FOLDER/$MOUNT_REF-upload" \
  --cache-workers 8 \
  --cache-writes \
  --checkers 16 \
@ -85,5 +91,5 @@ sudo rclone --config "$CONFIG_FOLDER/rclone.conf" \
  --vfs-read-ahead 128M \
  --vfs-read-chunk-size 16M \
  --vfs-read-chunk-size-limit 128M \
-  --volname "$MOUNT" \
-  "$MOUNT":"$MOUNT_LOWERCASE" "$MOUNT_PATH"
+  --volname "$MOUNT_REF" \
+  "$MOUNT_REF":"$MOUNT_LOWERCASE" "$MOUNT_PATH"