From 4de6f5a6fd5c57d237f251cc73e1df022be48f21 Mon Sep 17 00:00:00 2001 From: Baohua Yang Date: Thu, 19 Mar 2020 17:15:47 -0700 Subject: [PATCH] [Issue-119] Clean up docker build for api-engine The current building process involves several useless steps. And it will trigger security alerts with the old version of django packages. This PR cleans unused code and remove unused scripts and packages. Close#119. Signed-off-by: Baohua Yang --- Makefile | 9 +- .../docker/common/api-engine/Dockerfile.in | 27 ++-- .../docker/common/api-engine/entrypoint.sh | 20 ++- .../docker/common/api-engine}/server.ini | 0 src/api-engine/api_engine/settings.py.initial | 123 ------------------ .../docker/scripts/change_local_settings.sh | 5 - src/api-engine/docker/scripts/initial.sh | 6 - src/api-engine/requirements.txt | 4 +- 8 files changed, 40 insertions(+), 154 deletions(-) rename {src/api-engine/docker/uwsgi => build_image/docker/common/api-engine}/server.ini (100%) delete mode 100644 src/api-engine/api_engine/settings.py.initial delete mode 100755 src/api-engine/docker/scripts/change_local_settings.sh delete mode 100755 src/api-engine/docker/scripts/initial.sh diff --git a/Makefile b/Makefile index 1f1ac650b..3f1404861 100755 --- a/Makefile +++ b/Makefile @@ -120,13 +120,13 @@ build/docker/%/.push: build/docker/%/$(DUMMY) --password=$(DOCKER_HUB_PASSWORD) @docker push $(BASENAME)-$(patsubst build/docker/%/.push,%,$@):$(IMG_TAG) -common-docker: $(patsubst %,build/docker/common/%/$(DUMMY),$(COMMON_DOCKER_IMAGES)) ##@Generate docker images locally +docker-common: $(patsubst %,build/docker/common/%/$(DUMMY),$(COMMON_DOCKER_IMAGES)) ##@Generate docker images locally agent-docker: $(patsubst %,build/docker/agent/%/$(DUMMY),$(AGENT_DOCKER_IMAGES)) ##@Generate docker images locally -docker: common-docker agent-docker +docker: docker-common agent-docker -common-docker-%: +docker-common-%: @$(MAKE) build/docker/common/$*/$(DUMMY) agent-docker-%: @@ -272,6 +272,9 @@ HELP_FUN = \ }; \ print "\n"; } +api-engine: # for debug only now + docker build -t hyperledger/cello-api-engine:latest -f build_image/docker/common/api-engine/Dockerfile.in ./ + .PHONY: \ all \ check \ diff --git a/build_image/docker/common/api-engine/Dockerfile.in b/build_image/docker/common/api-engine/Dockerfile.in index 0d62b409f..ca2f0c3ec 100644 --- a/build_image/docker/common/api-engine/Dockerfile.in +++ b/build_image/docker/common/api-engine/Dockerfile.in @@ -1,20 +1,25 @@ FROM python:3.6 -RUN apt-get update && apt-get install -y gettext-base graphviz libgraphviz-dev && \ - apt-get autoclean && apt-get clean && apt-get autoremove && rm -rf /var/cache/apt/ -COPY src/api-engine/requirements.txt / -RUN cd / && \ - pip install -r requirements.txt +# Install software +RUN apt-get update \ + && apt-get install -y gettext-base graphviz libgraphviz-dev \ + && apt-get autoclean \ + && apt-get clean \ + && apt-get autoremove && rm -rf /var/cache/apt/ -COPY src/api-engine /var/www/server -COPY src/api-engine/docker/uwsgi/server.ini /etc/uwsgi/apps-enabled/ -COPY src/api-engine/docker/scripts /scripts -COPY build_image/docker/common/api-engine/entrypoint.sh / +# Set the working dir +WORKDIR /var/www/server -RUN cd /var/www/server/api_engine && cp settings.py.initial settings.py && cd .. && python manage.py collectstatic --noinput +# Copy source code to the working dir +COPY src/api-engine ./ -WORKDIR /var/www/server +# Install python dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Add uwsgi configuration file +COPY build_image/docker/common/api-engine/server.ini /etc/uwsgi/apps-enabled/ ENV RUN_MODE server +COPY build_image/docker/common/api-engine/entrypoint.sh / CMD bash /entrypoint.sh diff --git a/build_image/docker/common/api-engine/entrypoint.sh b/build_image/docker/common/api-engine/entrypoint.sh index 41c83c13f..684d972d1 100755 --- a/build_image/docker/common/api-engine/entrypoint.sh +++ b/build_image/docker/common/api-engine/entrypoint.sh @@ -1,13 +1,25 @@ #!/usr/bin/env bash -bash /scripts/initial.sh; +#bash /scripts/initial.sh; + +echo "Generating the settings.py for api_engine" +LOCAL_SETTINGS="/var/www/server/api_engine/settings.py" +RAW_LOCAL_SETTINGS="/var/www/server/api_engine/settings.py.example" + +envsubst < ${RAW_LOCAL_SETTINGS} > ${LOCAL_SETTINGS} + holdup -t 120 tcp://${DB_HOST}:${DB_PORT}; if [[ "$RUN_MODE" == "server" ]]; then python manage.py makemigrations && python manage.py migrate; - python manage.py create_user --username ${ADMIN_USERNAME} --password ${ADMIN_PASSWORD} --is_superuser --email ${ADMIN_EMAIL} --role operator - if [[ "$DEBUG" == "True" ]]; then + python manage.py create_user \ + --username ${ADMIN_USERNAME:-admin} \ + --password ${ADMIN_PASSWORD:-pass} \ + --email ${ADMIN_EMAIL:-admin@cello} \ + --is_superuser \ + --role operator + if [[ "$DEBUG" == "True" ]]; then # For dev, use pure Django directly python manage.py runserver 0.0.0.0:8080; - else + else # For production, use uwsgi in front uwsgi --ini /etc/uwsgi/apps-enabled/server.ini; fi else diff --git a/src/api-engine/docker/uwsgi/server.ini b/build_image/docker/common/api-engine/server.ini similarity index 100% rename from src/api-engine/docker/uwsgi/server.ini rename to build_image/docker/common/api-engine/server.ini diff --git a/src/api-engine/api_engine/settings.py.initial b/src/api-engine/api_engine/settings.py.initial deleted file mode 100644 index 07f644bd8..000000000 --- a/src/api-engine/api_engine/settings.py.initial +++ /dev/null @@ -1,123 +0,0 @@ -""" -Django settings for api_engine project. - -Generated by 'django-admin startproject' using Django 2.1.4. - -For more information on this file, see -https://docs.djangoproject.com/en/2.1/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/2.1/ref/settings/ -""" - -import os - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = '=5-oa588z5-5ow4wd8+=xoj%uy_rd6a65edkfvn3&zw+1=qhwd' - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = [] - - -# Application definition - -INSTALLED_APPS = [ - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'rest_framework', - 'api', - 'drf_yasg' -] - -MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', -] - -ROOT_URLCONF = 'api_engine.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - -WSGI_APPLICATION = 'api_engine.wsgi.application' - - -# Database -# https://docs.djangoproject.com/en/2.1/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), - } -} - - -# Password validation -# https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, -] - - -# Internationalization -# https://docs.djangoproject.com/en/2.1/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/2.1/howto/static-files/ - -STATIC_URL = '/static/' -STATIC_ROOT = '/var/www/server/static' diff --git a/src/api-engine/docker/scripts/change_local_settings.sh b/src/api-engine/docker/scripts/change_local_settings.sh deleted file mode 100755 index dec18eee8..000000000 --- a/src/api-engine/docker/scripts/change_local_settings.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -LOCAL_SETTINGS="/var/www/server/api_engine/settings.py" -RAW_LOCAL_SETTINGS="/var/www/server/api_engine/settings.py.example" - -envsubst < ${RAW_LOCAL_SETTINGS} > ${LOCAL_SETTINGS} \ No newline at end of file diff --git a/src/api-engine/docker/scripts/initial.sh b/src/api-engine/docker/scripts/initial.sh deleted file mode 100755 index abbe3aa97..000000000 --- a/src/api-engine/docker/scripts/initial.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -SCRIPTS_PATH=/scripts - -bash ${SCRIPTS_PATH}/change_local_settings.sh - -echo "All initial Done!!" \ No newline at end of file diff --git a/src/api-engine/requirements.txt b/src/api-engine/requirements.txt index c8e97b2bc..2bf045cf5 100644 --- a/src/api-engine/requirements.txt +++ b/src/api-engine/requirements.txt @@ -1,4 +1,4 @@ -Django<=2.1.4 +Django>=2.2.10,<=3.0.4 uwsgi<=2.0.17.1 djangorestframework<=3.9.0 django-filter<=1.1.0 @@ -11,7 +11,7 @@ django-extensions<=2.1.6 pygraphviz<=1.5 celery<=4.2.1 redis<=3.2.0 -docker-compose>=1.18.0,<=1.23.2 +# docker-compose>=1.18.0,<=1.25.4 watchdog<=0.9.0 django-allauth<=0.39.1 django-rest-auth<=0.9.3