From df3ad69a5a491f712f3344259ed78c929ecf3316 Mon Sep 17 00:00:00 2001 From: meliurwen Date: Mon, 24 Aug 2020 19:37:44 +0200 Subject: [PATCH] First chunk of refactoring done --- .env.example | 31 +++++++ .gitignore | 1 + all.env.example | 186 +++++++++++++++++++++++++++++++++++++ docker-compose.yml | 97 +++++++++++++++++++ nginx.env.example | 8 ++ nginx/funkwhale.template | 112 ++++++++++++++++++++++ nginx/funkwhale_proxy.conf | 19 ++++ postgres.env.example | 17 ++++ 8 files changed, 471 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 all.env.example create mode 100644 docker-compose.yml create mode 100644 nginx.env.example create mode 100644 nginx/funkwhale.template create mode 100644 nginx/funkwhale_proxy.conf create mode 100644 postgres.env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b59794d --- /dev/null +++ b/.env.example @@ -0,0 +1,31 @@ +# Global Settings +TZ=Etc/UTC +LOCAL_STACK_DIR=/srv/docker/volumes/funkwhale + +# Global Settings (optional) +FUNKWHALE_VERSION= + +# Networks +NETWORK=webservices + +# postgres (optional) +DB_IMG= +DB_TAG= +DB_CONTAINER_NAME= +DB_RESTART= + +# redis (optional) +RD_IMG= +RD_TAG= +RD_CONTAINER_NAME= +RD_RESTART= + + +CELERYD_CONCURRENCY= + + +MUSIC_DIRECTORY_PATH=/music +MUSIC_DIRECTORY_SERVE_PATH=/srv/docker/volumes/funkwhale/media/import_music +MEDIA_ROOT=/srv/docker/volumes/funkwhale/media/media +STATIC_ROOT=/srv/docker/volumes/funkwhale/api/static +FUNKWHALE_FRONTEND_PATH=/srv/docker/volumes/funkwhale/api/front/dist diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..03bd412 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.env diff --git a/all.env.example b/all.env.example new file mode 100644 index 0000000..0c0d58a --- /dev/null +++ b/all.env.example @@ -0,0 +1,186 @@ +# If you have any doubts about what a setting does, +# check https://docs.funkwhale.audio/configuration.html#configuration-reference + +# If you're tweaking this file from the template, ensure you edit at least the +# following variables: +# - DJANGO_SECRET_KEY +# - FUNKWHALE_HOSTNAME +# - EMAIL_CONFIG and DEFAULT_FROM_EMAIL if you plan to send emails) +# On non-docker setup **only**, you'll also have to tweak/uncomment those variables: +# - DATABASE_URL +# - CACHE_URL +# +# You **don't** need to update those variables on pure docker setups. +# +# Additional options you may want to check: +# - MUSIC_DIRECTORY_PATH and MUSIC_DIRECTORY_SERVE_PATH if you plan to use +# in-place import +# +# Docker only +# ----------- + +# The tag of the image we should use +# (it will be interpolated in docker-compose file) +# You can comment or ignore this if you're not using docker +FUNKWHALE_VERSION=latest + +# End of Docker-only configuration + +# General configuration +# --------------------- + +# Set this variables to bind the API server to another interface/port +# example: FUNKWHALE_API_IP=0.0.0.0 +# example: FUNKWHALE_API_PORT=5678 +FUNKWHALE_API_IP=127.0.0.1 +FUNKWHALE_API_PORT=5000 +# The number of web workers to start in parallel. Higher means you can handle +# more concurrent requests, but also leads to higher CPU/Memory usage +FUNKWHALE_WEB_WORKERS=1 +# Replace this by the definitive, public domain you will use for +# your instance. It cannot be changed after initial deployment +# without breaking your instance. +FUNKWHALE_HOSTNAME=msubdomain.domain.tld +FUNKWHALE_PROTOCOL=https + +# Configure email sending using this variale +# By default, funkwhale will output emails sent to stdout +# here are a few examples for this setting +# EMAIL_CONFIG=consolemail:// # output emails to console (the default) +# EMAIL_CONFIG=dummymail:// # disable email sending completely +# On a production instance, you'll usually want to use an external SMTP server: +# EMAIL_CONFIG=smtp://user@:password@youremail.host:25 +# EMAIL_CONFIG=smtp+ssl://user@:password@youremail.host:465 +# EMAIL_CONFIG=smtp+tls://user@:password@youremail.host:587 + +# Make email verification mandatory before using the service +# Doesn't apply to admins. +# ACCOUNT_EMAIL_VERIFICATION_ENFORCE=false + +# The email address to use to send system emails. +# DEFAULT_FROM_EMAIL=noreply@yourdomain + +# Depending on the reverse proxy used in front of your funkwhale instance, +# the API will use different kind of headers to serve audio files +# Allowed values: nginx, apache2 +REVERSE_PROXY_TYPE=nginx + +# API/Django configuration + +# Database configuration +# Examples: +# DATABASE_URL=postgresql://:@:/ +# DATABASE_URL=postgresql://funkwhale:passw0rd@localhost:5432/funkwhale_database +# Use the next one if you followed Debian installation guide +# DATABASE_URL=postgresql://funkwhale@:5432/funkwhale + +# Cache configuration +# Examples: +# CACHE_URL=redis://:/ +# CACHE_URL=redis://localhost:6379/0c +# With a password: +# CACHE_URL=redis://:password@localhost:6379/0 +# (the extra semicolon is important) +# Use the next one if you followed Debian installation guide +# +# CACHE_URL=redis://127.0.0.1:6379/0 +# +# If you want to use Redis over unix sockets, you'll actually need two variables: +# For the cache part: +# CACHE_URL=redis:///run/redis/redis.sock?db=0 +# For the Celery/asynchronous tasks part: +# CELERY_BROKER_URL=redis+socket:///run/redis/redis.sock?virtual_host=0 + +# Number of worker processes to execute. Defaults to 0, in which case it uses your number of CPUs +# Celery workers handle background tasks (such file imports or federation +# messaging). The more processes a worker gets, the more tasks +# can be processed in parallel. However, more processes also means +# a bigger memory footprint. +# CELERYD_CONCURRENCY=0 + +# Where media files (such as album covers or audio tracks) should be stored +# on your system? +# (Ensure this directory actually exists) +MEDIA_ROOT=/srv/docker/volumes/funkwhale/media/media + +# Where static files (such as API css or icons) should be compiled +# on your system? +# (Ensure this directory actually exists) +STATIC_ROOT=/srv/docker/volumes/funkwhale/api/static + +# which settings module should django use? +# You don't have to touch this unless you really know what you're doing +DJANGO_SETTINGS_MODULE=config.settings.production + +# Generate one using `openssl rand -base64 45`, for example +DJANGO_SECRET_KEY= + +# You don't have to edit this, but you can put the admin on another URL if you +# want to +# DJANGO_ADMIN_URL=^api/admin/ + +# Sentry/Raven error reporting (server side) +# Enable Raven if you want to help improve funkwhale by +# automatically sending error reports our Sentry instance. +# This will help us detect and correct bugs +RAVEN_ENABLED=false +RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5 + +# In-place import settings +# You can safely leave those settings uncommented if you don't plan to use +# in place imports. +# Typical docker setup: +# MUSIC_DIRECTORY_PATH=/music # docker-only +# MUSIC_DIRECTORY_SERVE_PATH=/srv/docker/volumes/funkwhale/media/import_music +# Typical non-docker setup: +# MUSIC_DIRECTORY_PATH=/srv/docker/volumes/funkwhale/media/import_music +# # MUSIC_DIRECTORY_SERVE_PATH= # stays commented, not needed + +MUSIC_DIRECTORY_PATH=/music +MUSIC_DIRECTORY_SERVE_PATH=/srv/docker/volumes/funkwhale/media/import_music + +# LDAP settings +# Use the following options to allow authentication on your Funkwhale instance +# using a LDAP directory. +# Have a look at https://docs.funkwhale.audio/installation/ldap.html for +# detailed instructions. + +# LDAP_ENABLED=False +# LDAP_SERVER_URI=ldap://your.server:389 +# LDAP_BIND_DN=cn=admin,dc=domain,dc=com +# LDAP_BIND_PASSWORD=bindpassword +# LDAP_SEARCH_FILTER=(|(cn={0})(mail={0})) +# LDAP_START_TLS=False +# LDAP_ROOT_DN=dc=domain,dc=com + +FUNKWHALE_FRONTEND_PATH=/srv/docker/volumes/funkwhale/api/front/dist + +## External storages configuration +# Funkwhale can store uploaded files on Amazon S3 and S3-compatible storages (such as Minio) +# Uncomment and fill the variables below + +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_STORAGE_BUCKET_NAME= +# An optional bucket subdirectory were you want to store the files. This is especially useful +# if you plan to use share the bucket with other services +# AWS_LOCATION= + +# If you use a S3-compatible storage such as minio, set the following variable +# the full URL to the storage server. Example: +# AWS_S3_ENDPOINT_URL=https://minio.mydomain.com +# AWS_S3_ENDPOINT_URL= + +# If you want to serve media directly from your S3 bucket rather than through a proxy, +# set this to true +# PROXY_MEDIA=false + +# If you are using Amazon S3 to serve media directly, you will need to specify your region +# name in order to access files. Example: +# AWS_S3_REGION_NAME=eu-west-2 +# AWS_S3_REGION_NAME= + +# If you are using Amazon S3, use this setting to configure how long generated URLs should stay +# valid. The default value is 3600 (60 minutes). The maximum accepted value is 604800 (7 days) + +# AWS_QUERYSTRING_EXPIRE= diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4366b68 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,97 @@ +version: "3" + +services: + postgres: + image: ${DB_IMG:-postgres}:${DB_TAG:-12} + container_name: ${DB_CONTAINER_NAME:-funkwhale-db} + restart: ${DB_RESTART:-unless-stopped} + volumes: + - ${LOCAL_STACK_DIR}/postgres/data:/var/lib/postgresql/data + expose: + - 5432 + env_file: + - postgres.env + + redis: + image: ${RD_IMG:-redis}:${RD_TAG:-5} + container_name: ${RD_CONTAINER_NAME:-funkwhale-cache} + restart: ${RD_RESTART:-unless-stopped} + volumes: + - ${LOCAL_STACK_DIR}/redis/data:/data + expose: + - 6379 + + celeryworker: + restart: unless-stopped + image: funkwhale/funkwhale:${FUNKWHALE_VERSION:-latest} + depends_on: + - postgres + - redis + volumes: + - "${MUSIC_DIRECTORY_SERVE_PATH-/srv/docker/volumes/funkwhale/media/import_music}:${MUSIC_DIRECTORY_PATH-/music}:ro" + - "${MEDIA_ROOT}:${MEDIA_ROOT}" + env_file: all.env + environment: + - C_FORCE_ROOT=true + # Celery workers handle background tasks (such file imports or federation + # messaging). The more processes a worker gets, the more tasks + # can be processed in parallel. However, more processes also means + # a bigger memory footprint. + # By default, a worker will span a number of process equal to your number + # of CPUs. You can adjust this, by explicitly setting the --concurrency + # flag: + # celery -A funkwhale_api.taskapp worker -l INFO --concurrency=4 + command: celery -A funkwhale_api.taskapp worker -l INFO --concurrency=${CELERYD_CONCURRENCY-0} + + celerybeat: + image: funkwhale/funkwhale:${FUNKWHALE_VERSION:-latest} + restart: unless-stopped + depends_on: + - postgres + - redis + env_file: all.env + command: celery -A funkwhale_api.taskapp beat --pidfile= -l INFO + + api: + image: funkwhale/funkwhale:${FUNKWHALE_VERSION:-latest} + restart: unless-stopped + depends_on: + - postgres + - redis + volumes: + - "${MUSIC_DIRECTORY_SERVE_PATH-/srv/docker/volumes/funkwhale/media/import_music}:${MUSIC_DIRECTORY_PATH-/music}:ro" + - "${MEDIA_ROOT}:${MEDIA_ROOT}" + - "${STATIC_ROOT}:${STATIC_ROOT}" + - "${FUNKWHALE_FRONTEND_PATH}:/frontend" + expose: + - 5000 + env_file: all.env + + nginx: + image: nginx + restart: unless-stopped + depends_on: + - api + volumes: + - "./nginx/funkwhale.template:/etc/nginx/conf.d/funkwhale.template:ro" + - "./nginx/funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf:ro" + - "${MUSIC_DIRECTORY_SERVE_PATH-/srv/docker/volumes/funkwhale/media/import_music}:${MUSIC_DIRECTORY_SERVE_PATH-/srv/docker/volumes/funkwhale/media/import_music}:ro" + - "${MEDIA_ROOT}:${MEDIA_ROOT}:ro" + - "${STATIC_ROOT}:${STATIC_ROOT}:ro" + - "${FUNKWHALE_FRONTEND_PATH}:/frontend:ro" + expose: + - 80 + env_file: + - all.env + - nginx.env + command: > + sh -c "envsubst \"`env | awk -F = '{printf \" $$%s\", $$1}'`\" + < /etc/nginx/conf.d/funkwhale.template + > /etc/nginx/conf.d/default.conf + && cat /etc/nginx/conf.d/default.conf + && nginx -g 'daemon off;'" + +networks: + default: + external: + name: ${NETWORK} diff --git a/nginx.env.example b/nginx.env.example new file mode 100644 index 0000000..633435a --- /dev/null +++ b/nginx.env.example @@ -0,0 +1,8 @@ +# nginx +NGINX_MAX_BODY_SIZE=100M + +# Reverse-proxy and certbot +VIRTUAL_HOST=subdomain.domain.tld +VIRTUAL_PORT=80 +LETSENCRYPT_HOST=subdomain.domain.tld +LETSENCRYPT_EMAIL=username@domain.tld diff --git a/nginx/funkwhale.template b/nginx/funkwhale.template new file mode 100644 index 0000000..b5f01ea --- /dev/null +++ b/nginx/funkwhale.template @@ -0,0 +1,112 @@ +upstream funkwhale-api { + # depending on your setup, you may want to update this + server api:5000; +} + + +# required for websocket support +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +server { + listen 80; + server_name ${FUNKWHALE_HOSTNAME}; + + # TLS + # Feel free to use your own configuration for SSL here or simply remove the + # lines and move the configuration to the previous server block if you + # don't want to run funkwhale behind https (this is not recommended) + # have a look here for let's encrypt configuration: + # https://certbot.eff.org/all-instructions/#debian-9-stretch-nginx + + root /frontend; + + # If you are using S3 to host your files, remember to add your S3 URL to the + # media-src and img-src headers (e.g. img-src 'self' https:// data:) + + add_header Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; object-src 'none'; media-src 'self' data:"; + add_header Referrer-Policy "strict-origin-when-cross-origin"; + + + location / { + include /etc/nginx/funkwhale_proxy.conf; + # this is needed if you have file import via upload enabled + client_max_body_size ${NGINX_MAX_BODY_SIZE}; + proxy_pass http://funkwhale-api/; + } + + location /front/ { + add_header Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; object-src 'none'; media-src 'self' data:"; + add_header Referrer-Policy "strict-origin-when-cross-origin"; + add_header Service-Worker-Allowed "/"; + add_header X-Frame-Options "ALLOW"; + alias /frontend/; + expires 30d; + add_header Pragma public; + add_header Cache-Control "public, must-revalidate, proxy-revalidate"; + } + + location /front/embed.html { + add_header Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; object-src 'none'; media-src 'self' data:"; + add_header Referrer-Policy "strict-origin-when-cross-origin"; + + add_header X-Frame-Options "ALLOW"; + alias /frontend/embed.html; + expires 30d; + add_header Pragma public; + add_header Cache-Control "public, must-revalidate, proxy-revalidate"; + } + + location /federation/ { + include /etc/nginx/funkwhale_proxy.conf; + proxy_pass http://funkwhale-api/federation/; + } + + # You can comment this if you do not plan to use the Subsonic API + location /rest/ { + include /etc/nginx/funkwhale_proxy.conf; + proxy_pass http://funkwhale-api/api/subsonic/rest/; + } + + location /.well-known/ { + include /etc/nginx/funkwhale_proxy.conf; + proxy_pass http://funkwhale-api/.well-known/; + } + + location /media/ { + alias ${MEDIA_ROOT}/; + } + + # this is an internal location that is used to serve + # audio files once correct permission / authentication + # has been checked on API side + location /_protected/media { + internal; + alias ${MEDIA_ROOT}; + + } + # Comment the previous location and uncomment this one if you're storing + # media files in a S3 bucket + # location ~ /_protected/media/(.+) { + # internal; + # # Needed to ensure DSub auth isn't forwarded to S3/Minio, see #932 + # proxy_set_header Authorization ""; + # proxy_pass $1; + # } + + location /_protected/music { + # this is an internal location that is used to serve + # audio files once correct permission / authentication + # has been checked on API side + # Set this to the same value as your MUSIC_DIRECTORY_PATH setting + internal; + alias ${MUSIC_DIRECTORY_PATH}; + } + + location /staticfiles/ { + # django static files + alias ${STATIC_ROOT}/; + } +} diff --git a/nginx/funkwhale_proxy.conf b/nginx/funkwhale_proxy.conf new file mode 100644 index 0000000..dccc782 --- /dev/null +++ b/nginx/funkwhale_proxy.conf @@ -0,0 +1,19 @@ +# use this one if you put the nginx container behind another proxy +# you will have to set some headers on this proxy as well to ensure +# everything works correctly, you can use the ones from the funkwhale_proxy.conf file +# at https://dev.funkwhale.audio/funkwhale/funkwhale/blob/develop/deploy/funkwhale_proxy.conf +# your proxy will also need to support websockets + +proxy_set_header Host $host; +proxy_set_header X-Real-IP $remote_addr; + +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; +proxy_set_header X-Forwarded-Host $http_x_forwarded_host; +proxy_set_header X-Forwarded-Port $http_x_forwarded_port; +proxy_redirect off; + +# websocket support +proxy_http_version 1.1; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection $connection_upgrade; diff --git a/postgres.env.example b/postgres.env.example new file mode 100644 index 0000000..2de1696 --- /dev/null +++ b/postgres.env.example @@ -0,0 +1,17 @@ +# postgres +POSTGRES_HOST_AUTH_METHOD=trust + +# From: https://hub.docker.com/_/postgres +# This optional variable can be used to control the auth-method for host +# connections for all databases, all users, and all addresses. If unspecified +# then md5 password authentication is used. On an uninitialized database, +# this will populate pg_hba.conf via this approximate line: +# echo "host all all all $POSTGRES_HOST_AUTH_METHOD" >> pg_hba.conf +# See the PostgreSQL documentation on pg_hba.conf for more information about +# possible values and their meanings. +# Note 1: It is not recommended to use trust since it allows anyone to connect +# without a password, even if one is set (like via POSTGRES_PASSWORD). +# For more information see the PostgreSQL documentation on Trust +# Authentication. +# Note 2: If you set POSTGRES_HOST_AUTH_METHOD to trust, then POSTGRES_PASSWORD +# is not required.