[spcgeonode] several modifications

- cleanup
- remove custom setting files and use env vars instead
- move envvars to docker-compose when possible
- run tests without async
- fix async
- ci : cache per branch

.circleci/config.yml

@@ -11,21 +11,9 @@ jobs:
         type: boolean
       save_docker_cache:
         type: boolean
-      run_tests_smoke:
-        type: boolean
-        default: false
-      run_tests_core:
-        type: boolean
-        default: false
-      run_tests_internal:
-        type: boolean
-        default: false
-      run_tests_contrib:
-        type: boolean
-        default: false
-      run_tests_integration:
-        type: boolean
-        default: false
+      test_suite:
+        type: string
+        default: ""
 
     steps:
 
@@ -38,7 +26,7 @@ jobs:
           steps:
             - restore_cache:
                 keys:
-                  - v1-docker-images
+                  - v1-docker-images-{{ .Branch }}
             - run:
                 name: Load Docker layers cache
                 command: |
@@ -59,7 +47,7 @@ jobs:
                   docker save -o ~/docker-layers-cache.tar $(docker images -a --format "{{.ID}}")
                 when: always
             - save_cache:
-                key: v1-docker-images-{{ epoch }}
+                key: v1-docker-images-{{ .Branch }}-{{ epoch }}
                 paths:
                   - ~/docker-layers-cache.tar
                 when: always
@@ -106,48 +94,15 @@ jobs:
           when: on_fail
 
       - when:
-          condition: <<parameters.run_tests_smoke>>
-          steps:
-            - run:
-                name: Run smoke tests
-                command: |
-                  docker-compose exec django bash -c 'python manage.py test geonode.tests.smoke'
-                working_directory: scripts/spcgeonode/
-
-      - when:
-          condition: <<parameters.run_tests_core>>
-          steps:
-            - run:
-                name: Run core applications tests
-                command: |
-                  docker-compose exec django bash -c 'python manage.py test $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a for a in settings.GEONODE_CORE_APPS]))")'
-                working_directory: scripts/spcgeonode/
-
-      - when:
-          condition: <<parameters.run_tests_internal>>
-          steps:
-            - run:
-                name: Run internal applications tests
-                command: |
-                  docker-compose exec django bash -c 'python manage.py test $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a for a in settings.GEONODE_INTERNAL_APPS]))")'
-                working_directory: scripts/spcgeonode/
-
-      - when:
-          condition: <<parameters.run_tests_contrib>>
-          steps:
-            - run:
-                name: Run contrib applications tests
-                command: |
-                  docker-compose exec django bash -c 'python manage.py test $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a for a in settings.GEONODE_CONTRIB_APPS]))")'
-                working_directory: scripts/spcgeonode/
-
-      - when:
-          condition: <<parameters.run_tests_integration>>
+          condition: <<parameters.test_suite>>
           steps:
             - run:
-                name: Run integrations tests
+                name: Run test suite
                 command: |
-                  docker-compose exec django bash -c 'python manage.py test geonode.tests.integration'
+                  docker-compose -f docker-compose.yml exec django bash -c 'echo "Running <<parameters.test_suite>>"'
+                  docker-compose -f docker-compose.yml exec postgres psql -U postgres -c 'SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();'
+                  docker-compose -f docker-compose.yml exec postgres createdb -U postgres -T postgres test_postgres
+                  docker-compose -f docker-compose.yml run --rm django bash -c 'ASYNC_SIGNALS=False python manage.py test -v 3 --keepdb <<parameters.test_suite>>'
                 working_directory: scripts/spcgeonode/
 
 workflows:
@@ -158,33 +113,33 @@ workflows:
           name: setup
           load_docker_cache: true
           save_docker_cache: true
-          run_tests_smoke: true
+          test_suite: 'geonode.tests.smoke'
       - build:
           name: tests_integration
           load_docker_cache: true
           save_docker_cache: false
-          run_tests_integration: true
+          test_suite: 'geonode.tests.integration'
           requires:
             - setup
       - build:
           name: tests_core
           load_docker_cache: true
           save_docker_cache: false
-          run_tests_core: true
+          test_suite: $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a+'\''.tests'\'' for a in settings.GEONODE_CORE_APPS]))")
           requires:
             - setup
       - build:
           name: tests_internal
           load_docker_cache: true
           save_docker_cache: false
-          run_tests_internal: true
+          test_suite: $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a+'\''.tests'\'' for a in settings.GEONODE_INTERNAL_APPS]))")
           requires:
             - setup
       - build:
           name: tests_contrib
           load_docker_cache: true
           save_docker_cache: false
-          run_tests_contrib: true
+          test_suite: $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a+'\''.tests'\'' for a in settings.GEONODE_CONTRIB_APPS]))")
           requires:
             - setup
 
@@ -196,12 +151,9 @@ workflows:
             branches:
               only:
                 - spcgeonode-release
+                - spcgeonode
     jobs:
       - build:
           load_docker_cache: false
           save_docker_cache: true
-          run_tests_smoke: true
-          run_tests_core: true
-          run_tests_internal: true
-          run_tests_contrib: true
-          run_tests_integration: true
+          test_suite: geonode.tests.smoke geonode.tests.integration $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a+'\''.tests'\'' for a in settings.GEONODE_APPS]))")

scripts/spcgeonode/.env

@@ -26,6 +26,9 @@ ADMIN_USERNAME=super
 ADMIN_PASSWORD=duper
 ADMIN_EMAIL=[email protected]
 
+# Django secret key (replace this by any complex and random string)
+SECRET_KEY=1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ
+
 # Let's Encrypt certificates for https encryption. You must have a domain name as HTTPS_HOST (doesn't work
 # with an ip) and it must be reachable from the outside. This can be one of the following :
 # disabled : we do not get a certificate at all (a placeholder certificate will be used)

scripts/spcgeonode/django/Dockerfile

@@ -48,19 +48,4 @@ RUN chmod +x scripts/spcgeonode/django/docker-entrypoint.sh
 # Export ports
 EXPOSE 8000
 
-# Set environnment variables
-ENV PYTHONPATH=/spcgeonode/scripts/spcgeonode/django:$PYTHONPATH
-ENV DJANGO_SETTINGS_MODULE=settings_spcgeonode
-ENV DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres
-ENV BROKER_URL=amqp://guest:guest@rabbitmq:5672/
-ENV STATIC_ROOT=/spcgeonode-static/
-ENV MEDIA_ROOT=/spcgeonode-media/
-ENV STATIC_URL=/static/
-ENV MEDIA_URL=/uploaded/
-# TODO : we should probably remove this and set Celery to use JSON serialization instead of pickle
-ENV C_FORCE_ROOT=True
-
-# We get an exception after migrations on startup (it seems the monitoring app tries to resolve the geoserver domain name after it's migration, which can happen before oauth migrations on which geoserver startup depends...)
-ENV MONITORING_ENABLED=False
-
 # We provide no command or entrypoint as this image can be used to serve the django project or run celery tasks

scripts/spcgeonode/django/docker-entrypoint.sh

@@ -5,8 +5,25 @@ set -e
 
 echo $"\n\n\n"
 echo "-----------------------------------------------------"
-echo "STARTING DJANGO ENTRYPOINT --------------------------"
-date
+echo "STARTING DJANGO ENTRYPOINT $(date)"
+echo "-----------------------------------------------------"
+
+# Setting dynamic env vars (some of this could probably be put in docker-compose once
+# https://github.com/docker/compose/pull/5268 is merged, or even better hardcoded if
+# geonode supported relative site urls)
+if [ ! -z "$HTTPS_HOST" ]; then
+    export SITEURL="https://$HTTPS_HOST"
+    if [ "$HTTPS_PORT" != "443" ]; then
+        SITEURL="$SITEURL:$HTTPS_PORT"
+    fi
+else
+    export SITEURL="http://$HTTP_HOST"
+    if [ "$HTTP_PORT" != "80" ]; then
+        SITEURL="$SITEURL:$HTTP_PORT"
+    fi
+fi
+
+export GEOSERVER_PUBLIC_LOCATION="${SITEURL}/geoserver/"
 
 # Run migrations
 echo 'Running initialize.py...'

scripts/spcgeonode/docker-compose.yml

@@ -8,16 +8,35 @@ x-common-django:
     context: ../../
     dockerfile: scripts/spcgeonode/django/Dockerfile
   environment:
+    # editable in .env
     - HTTPS_HOST=${HTTPS_HOST}
     - HTTPS_PORT=${HTTPS_PORT}
     - HTTP_HOST=${HTTP_HOST}
     - HTTP_PORT=${HTTP_PORT}
     - ADMIN_USERNAME=${ADMIN_USERNAME}
     - ADMIN_PASSWORD=${ADMIN_PASSWORD}
     - ADMIN_EMAIL=${ADMIN_EMAIL}
-    - DEBUG=False
+    - GEOSERVER_ADMIN_USER=${ADMIN_USERNAME}
+    - GEOSERVER_ADMIN_PASSWORD=${ADMIN_PASSWORD}
     - REGISTRATION_OPEN=${REGISTRATION_OPEN}
     - TIME_ZONE=${TIME_ZONE}
+    - ALLOWED_HOSTS=['nginx','127.0.0.1','localhost','$HTTPS_HOST','$HTTP_HOST']
+    - SECRET_KEY=${SECRET_KEY}
+    # hardcoded
+    - DEBUG=False
+    - DJANGO_SETTINGS_MODULE=geonode.settings
+    - DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres
+    - BROKER_URL=amqp://rabbitmq:5672
+    - STATIC_ROOT=/spcgeonode-static/
+    - MEDIA_ROOT=/spcgeonode-media/
+    - STATIC_URL=/static/
+    - MEDIA_URL=/uploaded/
+    - GEOSERVER_LOCATION=http://nginx/geoserver/
+    - ASYNC_SIGNALS=True
+    # TODO : we should probably remove this and set Celery to use JSON serialization instead of pickle
+    - C_FORCE_ROOT=True
+    # We get an exception after migrations on startup (it seems the monitoring app tries to resolve the geoserver domain name after it's migration, which can happen before oauth migrations on which geoserver startup depends...)
+    - MONITORING_ENABLED=False
   volumes:
     - static:/spcgeonode-static/
     - media:/spcgeonode-media/
@@ -41,7 +60,7 @@ services:
   celery:
     << : *default-common-django
     entrypoint: []
-    command: 'celery worker --app=geonode.celery_app:app -l info'
+    command: 'celery worker --app=geonode.celery_app:app -l info -E'
 
   # Celery beat that triggers scheduled tasks
   celerybeat: