Fixing CloudFlare mount

This commit is contained in:
Brian Zalewski 2023-08-29 22:55:40 -04:00 committed by GitHub
parent 8a56619bf0
commit ca95d1ee76
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 130 additions and 22 deletions

View file

@ -0,0 +1,40 @@
{{- if false -}}
#!/usr/bin/env bash
# @file Google Cloud Platform
# @brief Ensures S3 buckets are created
# @description
# This script ensures that the S3 buckets are created on Google Cloud Platform. It creates
# the following buckets:
#
# - gs://docker.s3.{{ .host.domain }}
# - gs://private.s3.{{ .host.domain }}
# - gs://public.s3.{{ .host.domain }}
# - gs://system.s3.{{ .host.domain }}
# - gs://user.s3.{{ .host.domain }}
exit 0
if command -v gcloud > /dev/null; then
if [ -f "${$XDG_CONFIG_HOME:-$HOME/.config}/gcloud/s3-client-secret.json" ]; then
logg info 'Authenticating with gcloud S3 service account'
echo Y | gcloud auth login --cred-file=s3-gcp.json
if gcloud config get-value account > /dev/null && logg info 'gcloud is authenticated'; then
logg info 'Disabling usage reporting' && gcloud config set disable_usage_reporting true
BUCKETS="$(gcloud storage buckets list | grep 'storage_url:')"
for BUCKET in "docker" "private" "public" "system" "user"; do
if logg info "Checking for $BUCKET S3 presence on GCP" && ! echo "$BUCKETS" | grep "gs://${BUCKET}.s3.{{ .host.domain }}" > /dev/null; then
logg info "Creating gs://${BUCKET}.s3.{{ .host.domain }}" && gcloud storage buckets create "gs://${BUCKET}.s3.{{ .host.domain }}"
fi
done
logg info 'Finished ensuring S3 buckets on gcloud are created'
else
logg error 'gcloud is not authenticated'
fi
else
logg info 'No client secret available for gcloud S3 service account'
fi
else
logg error 'The gcloud executable is not available'
fi
{{ end -}}

View file

@ -0,0 +1,56 @@
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzaGV4ZFVZdGF1K2ZBa0JV
WW9FVHF2V29KY3FVOFd4eURSVkxBTFI1ZkMwClRwaW1aSWQyYjM4NTdXSmx4OWZz
bVhwaWtURU1iWWl2Y0l5b3c3bUFBUVEKLS0tIHVpQzVkVDJDMUhCUTVFbXUya0ZZ
ZUVuYTFSRXd6VnUzTW5wLzNVN1RhY2cK6lva90NCZuyEaavciX3n8gHA0jFbq/h7
BVD1gSeVDB833p1SdfmmFgqjJk8fGuxRhWKzkzO/3Ip6iXvXXygcOBcp3RZytCGd
mJHWZ7jCSOr+Z1NypzdvkHY8iLkvZwQC1q7BK8t8WDSd8/SnIQWbLm1UbT6oJlbG
z/51YCs2bsV6fW3OxXUjqqFe57wTOvO4kaQ7Bjhr0k5LQ6vFKZIyBcgUv5mlN+6w
sxG5fimcqIYF2FKSL+D+D0HA9lZth+84DT4qR1D9n/27bpky3o5BRyjbTlf+KUnR
hP/woWKwu9z5aGfjlQXNWbithCE88sz3m0RAHXpcT4T8HleVr6vKhPd+4aRy7pwV
wAVh8tfx+BEM1/UkVTxswChrkavzOcBa7QEM8udnFfPzHwtVAN1si+4Wp509Zd+9
FiyOz80pxsQB10HefSG9oPp8zzY8gqTRcpxqFToi2D0CmvRWJMsOzN4i0t3EMwK2
F2nHAF8DPmurIY5HOOCt2RccplzanRJITrw1yNDRinp129cyIdXGDZjmwCUA2xgN
dvJpGpeiffmkkEx4U36HTlFSCBZeCCF8WJbg8P2wQ/efJG+vwH/vWl48mKRkpMVN
cebATEPWaFK3R84ebxJarro9E/QklYn7UBOMbMY/FCiWaglPr+gZlxizL0FmpzYx
uenurhobbApQc/03uXXsHILhBkkg3Q5VvLzgn51BoKj4Sa69GVcnamsoZ6sda19i
MVwgF0enTg0vqkqDZZbO/DD6YzXXIBi64AQs5f9I2eCjSRPtUBCgKhhbq8LRuoHk
n8XFK/+Dfd2zF9iW67kKq2qhnr21dJ5Vn/HF+eCR1lWT4hg3OaqXPN0jxGvhbBzy
Fy2S9awEyUDSS010f5IPZlqL6H501+aUrzWmydgS1PONIt3pGqKatPApp5hHrtmV
Zmlv+k3MU1OGImFqfP+OuMeyiC7YkuxUcGDuOGqnMSzqSbIrTO3GaiBkSfjUZEbq
p72f5dEWANlftdx3GJgQtDZLgV5vQFqp5fkNfUjtzLEk+tFIoBi7NefcjMnBdy15
EArGLN5dYRMooVlBcKYqOoLCet8z4DMx33IomYeVLrurVGIAV4DuA043ZMG5cUn0
v2q7vmowOjvqGTIIWaufWZqEkgJrGmygrR32iq/Rbi3eycEGACDyUN7arzCIsQuw
c19IePXm4lD5VmOft3TYYYKoQ9avzg6+mY89uZuO8qK1hwj/JJMV2BK00hKKpGMx
11+k7Q2RCSQq1c3+yAsRSVxnEyh/EIbd8RgLIyFBx1RLGs09RSxPzOXLyvW34zPj
YN9yhcZNDFZYJEO9RxnRSkEn9pEcL8AY/0lhPxLc8DKsrU8o+ty2Vq7Yc1zOkiWh
KZMS6QPJO+FtvPm1vAoLduXM1ImVCImBxftf3izjeQ0/UvawCeW6iogWw3Ok6xDY
+HF/wGUZc+A8fIg8iQIcxZffPHefIU8nFHIgXXazqt3Y7tOyPgc7sZQyJnBVmFpJ
i3h6J2LUskcVjbeHXx64iskbgznInqW+a2UfnOwIRewKeiZtpDHD4uXigVOkrYJf
GxeRI3JAxnbooYJRPnTdfQJPbZltEPjkcT+A6nUbXK6oU8oZAfWf5oJ7Hqbp+NvX
/XlpYMoeLid+m/RZE6BHtOUZSTMICcduIHKq8VRdNjdv0MtZXLWLynK1Xv+yxQom
WdLX3RHKoUq6ygD3j7/xYCpNJcHm47eJI4fxlBXZ6S1NMSlSsorOU1XWVpz/Jwbv
sKrvYO1UthaBo8wMYhuCPhoAYhtZD8pcYunpZnjiiEy8YV0nVPy74/KeEWKkdKIY
VCNVsUG3/Mm3sMhjt3e2pnsraGuTuaYW6oRPfX9gYoctSaFAqvPSl20kE9IOghxp
ojw9Rl32Mlc1ThJEaXN0dKhnv2/OpFgUe1UGRlM4H+aLvhQxDjR+B2grPGJYr5tF
S8oXF52N/AAHi4OB2Hib6qETEqk+oivBzZhmsXGGXl12mXk6GZbcAxBXbSIXiUuc
SEia4SnEipchiew28w7P/Xh56eOH7e0g5+3fBmGIumQ+9MomEsF3bONmtDs+asgu
zyPEI1dAbEWhgfCMvP5nj0NK3SRJ6c7jPnR1GYLEEoMoN7SXG32DX/OoUWKEKFp5
JkAd/TnVw+nYtXsShQfpllryzelFUuOWP3xS40jpUAatnkntcDsh2TBnhwdYKH0i
5EWpSHZDQFdVAIxVF2bbPWx0vjEBBGmCkfDSZlBvMBV13LBaHgIXNtfL73wC++Wk
lhJQT6DCncsFZJAZ3TkS/Cms4rOHXd0N0Rwq3NHYNc58jhRcgwXJLrnX8uRgmDS+
ByBtXiFZWiH9WVx3bz48Mks8blOgGcGj4Mi22lxnwtXRJ3BiuQtxpmwf4SK9Nr0W
wWPWFcH6V9ZBlFmbxSWri+vC+ZO6+2/1dkqYWLGCpoLReZAu6Uy6WLWoUTjjIAZ/
hLjAVgsSWQWfkJj+f0WnhMbKovPyKjM8NPCzlfNfHFi5njmtmHgm2Qba8NdSho4T
vkBfi6kYhsFwU0zH7Q/kcgV9ifsL033SecVs9LjsmJXvpa+BrctvrrfE3X6iIszi
FoPxjODzH3I5VU3sbDxILUlAtT/I1wb3Ad9h5Qq/XfYs/xsEIf4DiVDiT7kjjzyM
IswwLVctpPLKLSII6YONgQ28sQCYtXYxk3qwswH5cTpEdPZpzs3n4aTus3GMWHj5
oH+TdiYphNHawhElcQ/0IGCyi5ppiaRdTp4IedXyY+brpSUn8gy0NJvLwnEzRHuT
7lw0tpL9TV2QYIg9E0KbRu8dyM08vQ6RyS+08Lk0DQe3Lg1fPFg3e5mzzNI2A7wh
kQAslX3VbIRpJF6RDMEgcxO75DkmRdQ702i4t6A7aqbN/PsFgVLo1N+0F80Req6X
SsUg155ET/d46QYxQ8i2o1e/wmfHGvS4oarD4xhF3Ujv6ZsodHJ9vfkDibiO7zgS
wRUMrE7MPG7GBytlpOX4lWOWGmhfQrl1MjHyXMq7OJZPuCLZgyYddUztGX2Vs18V
gE+nYO03JbLrTMLwIFNoh/VRrbWaXMmvbW1H1NaIhnBaxQNGZlru6PjgoLcAzn9/
lKlLz28EahOy5eAY2n/6r/UvnwNrnhHlPDC5C7t3mxM08MA9C0mlP7paiB3twCkF
YFGgDphmginqXdYFpy2c8OU=
-----END AGE ENCRYPTED FILE-----

View file

@ -1,7 +1,7 @@
-----BEGIN AGE ENCRYPTED FILE----- -----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2bFM5VUFoTDlxb2NjV2RV YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5NDdQM2xxaFlPeVpkNVN5
d0UvM2pPYWZMeDRZeDZKYmZ3YlhwNlBsYlRvCmVTcEVzUndwSG1lQ3pKTFpxZ1Bs em1OWXVKdG5yN2V0SDVmRnF3c2o5MXpnZ2lzCkdWQVZkaGhVZUUvQWt5YkZkNzBk
NGtXcksrNnRmR1UxOXR2UGpiNHplOHcKLS0tIFBEZHBibnEzSnBxTUlxcHdQQmhT eUtsS1UxRGRDUFNjZ3ZSOXorQ21mVWsKLS0tIGllSGpiUFdpL3VGT01qZDlsTTAz
MlUyZnRHWHY5UE43OXV1cFJjUnJGRHcK9s3V7BN+uHHJt8ekqFpP0XYaa+WwanmW MFYwWmJVbWpBQ2lQby9Zc0hxd1lIa28KxEVddgAMlDC/ySyUoGiRfi4Iya41iByf
qQ7rr6AB5ZT7z8y9vpQNK+mzuB49zL87AiNspAacKP/RtKNUPmdEzpY= cw7E/FVT6+9iiaW1tfXYpeRUFbXinR8HiiJavbUO7vfT8/0rdmd/7g==
-----END AGE ENCRYPTED FILE----- -----END AGE ENCRYPTED FILE-----

View file

@ -1,8 +1,8 @@
-----BEGIN AGE ENCRYPTED FILE----- -----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpdjhHNWJEMlJjNTlCUmJv YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMOE5nRHY3bE1rNE5KVDZW
RzRvUzRhUmR6OWpxWTVudDJ0NnVqbklqQmlrClhyNWpSZEZ1SHpEU0FROWZFYzlL TkVHVHdqa0MvNW91UFAyYU5ZMWIyb3Bkb2pJCmV0N1VCdFdjZ0xiRzV6bXl4WHNO
RmhEbmJ1ZWJtS2xjNmRsaVhZb3ExK0UKLS0tIE90dzZ5T0liQitNV0hQTHNmcFlj M0xpUXRINHkxUi96Ym96S0dQSnVwOG8KLS0tIGpEN2ozWUZ2aU91cWc4dHYzamtr
eEdKZWdvK0NOdU1PK3I1NGxmTEVtQWsKJWhE2Q5wCLtvy7ZrrPwNvceLWEp7rV9I MUhCanIzYWdpWVFJUnBDdXpHQytYYzgKtAbLV9I58aZWYeViHFKQCZA6mf0+dK4y
YEVpLY6lWuHWIbg6h8GkwlrbP/e3evFpZ7T9eLmhsMIfYm7hPtYV3BkASNqpWRh/ Ac6W5tQ/Fjk+ge+E5uNgLi22G3PW9qyshwJk/axVa/0wpMZHM2HtEvyIrBEj4YsQ
o94FfrDqtg7Nu1/pZO8o/dt7QnVh0lMPYw== EIFNsM0Llui6GOJHPbq423j1V0ZvhlIe
-----END AGE ENCRYPTED FILE----- -----END AGE ENCRYPTED FILE-----

View file

@ -0,0 +1,7 @@
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBWEFveDRNY0laM3BPSDVK
VktjeVU1dGd1QU9zWWV0b3VBT2FKN1MxY1NVCk5OcTBnNlg4TU44OWV1b3BiOUhD
SWFmeHA0NWdFTCtFMUhlWFVoK1RSZmMKLS0tIFBxREdEMEV1Y2FRTHdCSzRCZXps
TFI4Q2t6Rm9HY1Q5cll6d1IwV3dOeUkKc8+PHTsEBFLpd37vxIYVSj6dbEv27HQG
9UwZMEKsY2FmEQ+24i4L+q0CYi84BPgnx4LfUWQiZs3ZJ+8b0/Jh/9/IdHD3/VoG
-----END AGE ENCRYPTED FILE-----

View file

@ -1,5 +1,5 @@
{{- if and (or (and (stat (joinPath .host.home ".config" "age" "chezmoi.txt")) (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_ID"))) (env "CLOUDFLARE_R2_ID")) (or (and (stat (joinPath .host.home ".config" "age" "chezmoi.txt")) (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_SECRET"))) (env "CLOUDFLARE_R2_SECRET")) (ne .user.cloudflare.r2 "") -}} {{- if and (or (and (stat (joinPath .host.home ".config" "age" "chezmoi.txt")) (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_ID"))) (env "CLOUDFLARE_R2_ID")) (or (and (stat (joinPath .host.home ".config" "age" "chezmoi.txt")) (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_SECRET"))) (env "CLOUDFLARE_R2_SECRET")) (ne .user.cloudflare.r2 "") -}}
# MEGABYTE LABS MANAGED S3 # INSTALL DOCTOR MANAGED S3
[{{ .user.username}}-s3] [{{ .user.username}}-s3]
access_key_id = {{ if (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_ID")) }}{{- includeTemplate "secrets/CLOUDFLARE_R2_ID" | decrypt | trim -}}{{ else }}{{- env "CLOUDFLARE_R2_ID" -}}{{ end }} access_key_id = {{ if (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_ID")) }}{{- includeTemplate "secrets/CLOUDFLARE_R2_ID" | decrypt | trim -}}{{ else }}{{- env "CLOUDFLARE_R2_ID" -}}{{ end }}
acl = private acl = private
@ -8,6 +8,5 @@ provider = Cloudflare
region = auto region = auto
secret_access_key = {{ if (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_SECRET")) }}{{- includeTemplate "secrets/CLOUDFLARE_R2_SECRET" | decrypt | trim -}}{{ else }}{{- env "CLOUDFLARE_R2_SECRET" -}}{{ end }} secret_access_key = {{ if (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_SECRET")) }}{{- includeTemplate "secrets/CLOUDFLARE_R2_SECRET" | decrypt | trim -}}{{ else }}{{- env "CLOUDFLARE_R2_SECRET" -}}{{ end }}
type = s3 type = s3
# MEGABYTE LABS MANAGED S3 # INSTALL DOCTOR MANAGED S3
EOT
{{- end -}} {{- end -}}

View file

@ -14,7 +14,7 @@ if [ "$TYPE" = 'user' ]; then
fi fi
CACHE_FOLDER="/$USER_FOLDER/$USER/.cache/rclone" CACHE_FOLDER="/$USER_FOLDER/$USER/.cache/rclone"
CONFIG_FOLDER="/$USER_FOLDER/$USER/.config/rclone" CONFIG_FOLDER="/$USER_FOLDER/$USER/.config/rclone"
LOG_FOLDER="/$USER_FOLDER/$USER/.local/log" LOG_FOLDER="/$USER_FOLDER/$USER/.local/share/rclone"
LOG_FILE="$LOG_FOLDER/$MOUNT.log" LOG_FILE="$LOG_FOLDER/$MOUNT.log"
MOUNT_PATH="/$USER_FOLDER/$USER/.local/mnt/$MOUNT" MOUNT_PATH="/$USER_FOLDER/$USER/.local/mnt/$MOUNT"
else else
@ -46,7 +46,13 @@ if [ ! -f "$RCLONE_IGNORE" ] && [ -f "/etc/rcloneignore" ]; then
fi fi
### Mount ### Mount
MOUNT_LOWERCASE="$(echo "$MOUNT" | tr "[:upper:]" "[:lower:]")" if [ "$TYPE" == 'user' ]; then
MOUNT_REF="$USER-$MOUNT"
MOUNT_LOWERCASE="user/$USER-$MOUNT"
else
MOUNT_REF="$MOUNT"
MOUNT_LOWERCASE="$(echo "$MOUNT" | tr "[:upper:]" "[:lower:]")"
fi
unset AWS_CA_BUNDLE unset AWS_CA_BUNDLE
export PATH="$PATH:/usr/local/bin:/usr/bin" export PATH="$PATH:/usr/local/bin:/usr/bin"
# TODO: Only launch with --rc-web-gui if the servers hostname is the {{ .kubernetesHost }} # TODO: Only launch with --rc-web-gui if the servers hostname is the {{ .kubernetesHost }}
@ -56,11 +62,11 @@ sudo rclone --config "$CONFIG_FOLDER/rclone.conf" \
--allow-other \ --allow-other \
--buffer-size 4G \ --buffer-size 4G \
--bwlimit 40M \ --bwlimit 40M \
--cache-chunk-path "$CACHE_FOLDER/$MOUNT-chunks" \ --cache-chunk-path "$CACHE_FOLDER/$MOUNT_REF-chunks" \
--cache-db-path "$CACHE_FOLDER/$MOUNT-db" \ --cache-db-path "$CACHE_FOLDER/$MOUNT_REF-db" \
--cache-dir "$CACHE_FOLDER/$MOUNT-vfs" \ --cache-dir "$CACHE_FOLDER/$MOUNT_REF-vfs" \
--cache-info-age 60m \ --cache-info-age 60m \
--cache-tmp-upload-path "$CACHE_FOLDER/$MOUNT-upload" \ --cache-tmp-upload-path "$CACHE_FOLDER/$MOUNT_REF-upload" \
--cache-workers 8 \ --cache-workers 8 \
--cache-writes \ --cache-writes \
--checkers 16 \ --checkers 16 \
@ -85,5 +91,5 @@ sudo rclone --config "$CONFIG_FOLDER/rclone.conf" \
--vfs-read-ahead 128M \ --vfs-read-ahead 128M \
--vfs-read-chunk-size 16M \ --vfs-read-chunk-size 16M \
--vfs-read-chunk-size-limit 128M \ --vfs-read-chunk-size-limit 128M \
--volname "$MOUNT" \ --volname "$MOUNT_REF" \
"$MOUNT":"$MOUNT_LOWERCASE" "$MOUNT_PATH" "$MOUNT_REF":"$MOUNT_LOWERCASE" "$MOUNT_PATH"