Merge pull request #7 from akochari/main

Additional restructuring to fit SciLifeLab Serve

README.md

@@ -46,25 +46,27 @@ To use this project, follow these steps:
     docker-compose up --build
     ```
 
-### Scilifelab Deployment
+### Scilifelab Serve Deployment
 
 1. Build container
     ```sh
     docker build -t ptp .
     ```
 2. Push container to registry
 
-
 3. Deploy
+    - By default the folder /app/ext_storage/ should be mounted to outside storage to avoid ram bloat. It will contain database, media, and models.
+    - Place the downloaded models (from https://huggingface.co/pharmbio/ptp) to /app/ext_storage/models/
 
 4. Configuration
-    - Set environment variables
-    Use `DOWNLOAD=true` to download models prior to startup.
- - preferebly mount `/app/inference/models` directory to outside storage to avoid ram bloat.
-
- - Use `MAX_MODELS` to limit the number of active models (for debug purposes) .otherwise it iterates all models present (currently 800+).
-
-- If desired use `MODEL_DIR` to change path where model are found (default /app/inference/models/models)
+    - Currently static files are served by Django itself even with DEBUG=False
+    - All defaults of environmental variables make sense for SciLifeLab Serve. Here are some environmental variables that can be changed if needed:
+    - `EMAIL_HOST`, `EMAIL_PORT`, `EMAIL_HOST_USER` etc for email settings.
+    - `MAX_MODELS` to limit the number of active models (for debug purposes) .otherwise it iterates all models present (currently 800+).
+    - `MEDIA_DIR` for media directory
+    - `MODEL_DIR` for models directory
+    - `DATABASE_DIR` for database directory
+    - `SITE_URL` for generating download links in emails
 
 
 ## References

ptp/Dockerfile

@@ -32,7 +32,8 @@ FROM djangobase AS djangoapp
 COPY . /app/
 
 WORKDIR /app
-RUN mkdir /app/inference/models/
+RUN mkdir -p /app/ext_storage/
+# Models also need to be added. They be downloaded and placed in /app/ext_storage/models/ through the SciLifeLab Serve interface
 
 # Expose the port the Django app runs on
 EXPOSE 8000
@@ -50,10 +51,5 @@ RUN chmod +x /app/start-django.sh
 # Make sure the container is running as non-root
 USER $USER
 
-ENV CELERY_BROKER_URL="redis://localhost:6379/0"
-ENV REDIS_URL="redis://localhost:6379/0"
-ENV MODEL_DIR="/app/inference/models/models"
-# Models will be downloaded and placed in /app/inference/models/models through SciLifeLab Serve interface
-
 # Start supervisord
 CMD ["sh", "-c", "/app/start-script.sh"]

ptp/inference/tasks.py

@@ -6,8 +6,10 @@
 from django.conf import settings
 import pandas as pd
 from datetime import timezone
+from pathlib import Path
 
-model_dir = os.environ.get("MODEL_DIR", "/app/inference/models/models/")
+model_dir = os.environ.get("MODEL_DIR", "/app/ext_storage/models")
+Path(model_dir).mkdir(parents=True, exist_ok=True)
 max_models = os.environ.get("MAX_MODELS", False)
 
 @shared_task

ptp/inference/urls.py

@@ -1,7 +1,7 @@
+from django.conf import settings
 from django.urls import path
 from . import views
 
-
 app_name = 'inference'
 
 urlpatterns = [

ptp/ptp/settings.py

@@ -10,20 +10,19 @@
 
 # SECURITY WARNING: don't run with debug turned on in production!
 
-#DEBUG = True
+DEBUG = False
 if os.environ.get('DEBUG', False) == 'True':
     DEBUG = True
-else:
-    DEBUG = False
 
 ALLOWED_HOSTS = ['*', 'localhost']
 
-CSRF_TRUSTED_ORIGINS=['http://localhost:8000']
-if os.environ.get('CSRF_TRUSTED_ORIGINS', False):
-    CSRF_TRUSTED_ORIGINS = os.environ.get('CSRF_TRUSTED_ORIGINS').split(',')
 
 
 
+CSRF_TRUSTED_ORIGINS=['https://ptp2-inference.serve.scilifelab.se']
+if os.environ.get('CSRF_TRUSTED_ORIGINS', False):
+    CSRF_TRUSTED_ORIGINS = os.environ.get('CSRF_TRUSTED_ORIGINS').split(',')
+
 # Application definition
 INSTALLED_APPS = [
     'django.contrib.admin',
@@ -68,7 +67,8 @@
 
 WSGI_APPLICATION = 'ptp.wsgi.application'
 
-DATABASE_DIR = os.environ.get('DATABASE_DIR', BASE_DIR)
+DATABASE_DIR = Path(os.environ.get('DATABASE_DIR', '/app/ext_storage/database'))
+DATABASE_DIR.mkdir(parents=True, exist_ok=True)
 # Database
 DATABASES = {
     'default': {
@@ -108,31 +108,26 @@
     #'/var/www/static',
 ]
 
-
 # Media files (Uploaded files, results)
 MEDIA_URL = '/media/'
 
-MEDIA_DIR = os.environ.get('MEDIA_DIR', BASE_DIR)
+MEDIA_DIR = os.environ.get('MEDIA_DIR', '/app/ext_storage/media')
 MEDIA_ROOT = os.path.join(MEDIA_DIR, 'media')
 
 # Email settings (for sending job completion notifications)
-if os.environ.get('EMAIL_HOST', False):
-    EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
-    EMAIL_HOST = os.environ.get('EMAIL_HOST', 'smtp.example.com')
-    EMAIL_PORT = os.environ.get('EMAIL_PORT',587)
-    EMAIL_USE_TLS = os.environ.get('EMAIL_USE_TLS',True)
-    EMAIL_HOST_USER = os.environ.get('EMAIL_HOST_USER','[email protected]')
-
-    # Not superfond of this..
-    EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD','your-email-password')
-    # Perhaps?
-    if os.environ.get("EMAIL_PASSWORD_FILE", False):
-        filename = os.environ.get("EMAIL_PASSWORD_FILE", False)
-        with open(filename) as f:
-            EMAIL_HOST_PASSWORD = f.read().strip()
-
-else:
-    EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'  # Use console backend for testing
+EMAIL_BACKEND = (
+    "django.core.mail.backends.smtp.EmailBackend" if not DEBUG else "django.core.mail.backends.console.EmailBackend"
+)
+EMAIL_HOST = os.environ.get('EMAIL_HOST', "smtp.gmail.com")
+EMAIL_PORT = os.environ.get('EMAIL_PORT', 465)
+EMAIL_USE_SSL = os.environ.get('EMAIL_USE_SSL', True)
+EMAIL_USE_TLS = os.environ.get('EMAIL_USE_TLS', False)
+EMAIL_HOST_USER = os.environ.get('EMAIL_HOST_USER','[email protected]')
+EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD','your-email-password')
+if os.environ.get("EMAIL_PASSWORD_FILE", False):
+    filename = os.environ.get("EMAIL_PASSWORD_FILE", False)
+    with open(filename) as f:
+        EMAIL_HOST_PASSWORD = f.read().strip()
 
 # Celery configuration
 CELERY_BROKER_URL = "redis://localhost:6379/0"
@@ -152,14 +147,16 @@
 
 # Set the SITE_URL for generating download links in emails
 
-SITE_URL = os.environ.get('EMAIL_DOMAIN','https://yourdomain.com')
+SITE_URL = os.environ.get('SITE_URL','https://ptp2-inference.serve.scilifelab.se')
+
 
 STAGE_ENV = os.environ.get('STAGE', False)
 
 if not STAGE_ENV:
     # Security settings
     SECURE_BROWSER_XSS_FILTER = True
     X_FRAME_OPTIONS = 'DENY'
-    SECURE_SSL_REDIRECT = not DEBUG
+    #SECURE_SSL_REDIRECT = not DEBUG
     CSRF_COOKIE_SECURE = not DEBUG
     SESSION_COOKIE_SECURE = not DEBUG
+

ptp/ptp/urls.py

@@ -15,13 +15,18 @@
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
 from django.contrib import admin
-from django.urls import path, include
+from django.urls import path, include, re_path
 from django.conf import settings
 
 from django.conf.urls.static import static
+from django.views.static import serve
 urlpatterns = [
     path('admin/', admin.site.urls),
     path('', include('inference.urls')),
 
-] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+] 
 
+urlpatterns += [
+    re_path(r'^static/(?P<path>.*)$', serve, {'document_root': settings.STATIC_ROOT}),
+    re_path(r'^media/(?P<path>.*)$', serve, {'document_root': settings.MEDIA_ROOT}),
+]