Add Native Docker config - Ansible free

Dockerfile

@@ -1,77 +1,119 @@
 FROM ubuntu:focal as app
 
-# Packages installed:
-# git; Used to pull in particular requirements from github rather than pypi, 
-# and to check the sha of the code checkout.
-
-# ppa:deadsnakes/ppa; since Ubuntu doesn't ship with python 3.8 till 20, we need deadsnakes to install
-# python 3.8 on older ubuntu versions
-
-# language-pack-en locales; ubuntu locale support so that system utilities have a consistent
-# language and time zone.
-
-# python3.8-dev; to install python 3.8
-# python3-venv; installs venv module required to create virtual environments
-
-# libssl-dev; # mysqlclient wont install without this.
-
-# libmysqlclient-dev; to install header files needed to use native C implementation for 
-# MySQL-python for performance gains.
-# software-properties-common; to get apt-add-repository
-# deadsnakes PPA to install Python 3.8
-# If you add a package here please include a comment above describing what it is used for
-
-RUN apt-get update && \
-    apt-get install -y software-properties-common && \
-    apt-add-repository -y ppa:deadsnakes/ppa && \
-    apt-get update && apt-get upgrade -qy && \
-    apt-get install \
-    language-pack-en \
-    locales \
-    git \
-    libmysqlclient-dev \
-    libssl-dev \
-    build-essential \
-    python3.8-dev \
-    python3.8-distutils \
-    python3.8-venv -qy && \
-    rm -rf /var/lib/apt/lists/*
-
-ENV VIRTUAL_ENV=/edx/app/edx-notes-api/venvs/edx-notes-api
-RUN python3.8 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-
-
+# System requirements.
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt update && \
+  apt-get install -qy \ 
+  curl \
+  vim \
+  git-core \
+  language-pack-en \
+  build-essential \
+  python3.8-dev \
+  python3-pip \
+  python3-virtualenv \
+  python3.8-distutils \
+  libmysqlclient-dev \
+  libssl-dev \
+  libcairo2-dev && \
+  rm -rf /var/lib/apt/lists/*
+
+
+# Use UTF-8.
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
-ENV EDXNOTES_CONFIG_ROOT /edx/etc
-ENV DJANGO_SETTINGS_MODULE notesserver.settings.yaml_config
 
-EXPOSE 8120
-RUN useradd -m --shell /bin/false app
 
-WORKDIR /edx/app/notes
 
-# Copy the requirements explicitly even though we copy everything below
-# this prevents the image cache from busting unless the dependencies have changed.
-COPY requirements/base.txt /edx/app/notes/requirements/base.txt
-COPY requirements/pip.txt /edx/app/notes/requirements/pip.txt
+ARG COMMON_APP_DIR="/edx/app"
+ARG EDX_NOTES_API_SERVICE_NAME="edx_notes_api"
+ENV EDX_NOTES_API_HOME "${COMMON_APP_DIR}/${EDX_NOTES_API_SERVICE_NAME}"
+ARG EDX_NOTES_API_APP_DIR="${COMMON_APP_DIR}/${EDX_NOTES_API_SERVICE_NAME}"
+ARG SUPERVISOR_APP_DIR="${COMMON_APP_DIR}/supervisor"
+ARG EDX_NOTES_API_VENV_DIR="${COMMON_APP_DIR}/${EDX_NOTES_API_SERVICE_NAME}/venvs/${EDX_NOTES_API_SERVICE_NAME}"
+ARG SUPERVISOR_VENVS_DIR="${SUPERVISOR_APP_DIR}/venvs"
+ARG SUPERVISOR_VENV_DIR="${SUPERVISOR_VENVS_DIR}/supervisor"
+ARG EDX_NOTES_API_CODE_DIR="${EDX_NOTES_API_APP_DIR}/${EDX_NOTES_API_SERVICE_NAME}"
+ARG SUPERVISOR_AVAILABLE_DIR="${COMMON_APP_DIR}/supervisor/conf.available.d"
+ARG SUPERVISOR_VENV_BIN="${SUPERVISOR_VENV_DIR}/bin"
+ARG SUPEVISOR_CTL="${SUPERVISOR_VENV_BIN}/supervisorctl"
+ARG SUPERVISOR_VERSION="4.2.1"
+ARG SUPERVISOR_CFG_DIR="${SUPERVISOR_APP_DIR}/conf.d"
+
+
+ENV HOME /root
+ENV PATH "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
+ENV PATH "${EDX_NOTES_API_VENV_DIR}/bin:$PATH"
+ENV COMMON_CFG_DIR "/edx/etc"
+ENV EDX_NOTES_API_CFG_DIR "${COMMON_CFG_DIR}/edx_notes_api"
+ENV EDX_NOTES_API_CFG "/edx/etc/edx_notes_api.yml"
+
+RUN addgroup edx_notes_api
+RUN adduser --disabled-login --disabled-password edx_notes_api --ingroup edx_notes_api
+
+
+# Make necessary directories and environment variables.
+RUN mkdir -p /edx/var/edx_notes_api/staticfiles
+RUN mkdir -p /edx/var/edx_notes_api/media
+# Log dir
+RUN mkdir /edx/var/log/
+
+RUN virtualenv -p python3.8 --always-copy ${EDX_NOTES_API_VENV_DIR}
+RUN virtualenv -p python3.8 --always-copy ${SUPERVISOR_VENV_DIR}
+
+#install supervisor and deps in its virtualenv
+RUN . ${SUPERVISOR_VENV_BIN}/activate && \
+  pip install supervisor==${SUPERVISOR_VERSION} backoff==1.4.3 boto==2.48.0 && \
+  deactivate
+
+COPY requirements/base.txt ${EDX_NOTES_API_CODE_DIR}/requirements/base.txt
+
+RUN pip install -r ${EDX_NOTES_API_CODE_DIR}/requirements/base.txt
+
+# Working directory will be root of repo.
+WORKDIR ${EDX_NOTES_API_CODE_DIR}
+
+# Copy over rest of code.
+# We do this AFTER requirements so that the requirements cache isn't busted
+# every time any bit of code is changed.
+COPY . .
+COPY /configuration_files/edx_notes_api_gunicorn.py ${EDX_NOTES_API_HOME}/edx_notes_api_gunicorn.py
+# COPY /configuration_files/discovery-workers.sh ${DISCOVERY_HOME}/discovery-workers.sh
+# COPY /configuration_files/discovery.yml ${DISCOVERY_CFG}
+COPY /scripts/edx_notes_api.sh ${EDX_NOTES_API_HOME}/edx_notes_api.sh
+# COPY /configuration_files/supervisor.conf ${SUPERVISOR_APP_DIR}/supervisord.conf
+# create supervisor job
+COPY /configuration_files/supervisor.service /etc/systemd/system/supervisor.service
+COPY /configuration_files/supervisor.conf ${SUPERVISOR_CFG_DIR}/supervisor.conf
+COPY /configuration_files/supervisorctl ${SUPERVISOR_VENV_BIN}/supervisorctl
+# Manage.py symlink
+COPY /manage.py /edx/bin/manage.edx_notes_api
+
+# Expose canonical Discovery port
+EXPOSE 18281
+
+FROM app as prod
+
+ENV DJANGO_SETTINGS_MODULE "notesserver.settings.dev"
+
+RUN make static
+
+ENTRYPOINT ["/edx/app/edx_notes_api/edx_notes_api.sh"]
+
+FROM app as dev
+
+ENV DJANGO_SETTINGS_MODULE "notesserver.settings.devstack"
 
-# Dependencies are installed as root so they cannot be modified by the application user.
-RUN pip install -r requirements/pip.txt
-RUN pip install -r requirements/base.txt
+RUN pip install -r ${EDX_NOTES_API_CODE_DIR}/requirements/base.txt
 
-RUN mkdir -p /edx/var/log
+COPY /scripts/devstack.sh ${EDX_NOTES_API_HOME}/devstack.sh
 
-# Code is owned by root so it cannot be modified by the application user.
-# So we copy it before changing users.
-USER app
+RUN chown edx_notes_api:edx_notes_api /edx/app/edx_notes_api/devstack.sh && chmod a+x /edx/app/edx_notes_api/devstack.sh
 
-# Gunicorn 19 does not log to stdout or stderr by default. Once we are past gunicorn 19, the logging to STDOUT need not be specified.
-CMD gunicorn --workers=2 --name notes -c /edx/app/notes/notesserver/docker_gunicorn_configuration.py --log-file - --max-requests=1000 notesserver.wsgi:application
+# Devstack related step for backwards compatibility
+RUN touch /edx/app/${EDX_NOTES_API_SERVICE_NAME}/${EDX_NOTES_API_SERVICE_NAME}_env
 
-# This line is after the requirements so that changes to the code will not
-# bust the image cache
-COPY . /edx/app/notes
+ENTRYPOINT ["/edx/app/edx_notes_api/devstack.sh"]
+CMD ["start"]

backup

@@ -0,0 +1,78 @@
+# To build this Dockerfile:
+#
+# From the root of configuration:
+#
+# docker build -f docker/build/notes/Dockerfile .
+#
+# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible
+# with the currently checked-out configuration repo.
+
+ARG BASE_IMAGE_TAG=latest
+FROM edxops/focal-common:${BASE_IMAGE_TAG}
+LABEL maintainer="edxops"
+
+# ARG OPENEDX_RELEASE=master
+# ENV OPENEDX_RELEASE=${OPENEDX_RELEASE}
+# ENV NOTES_VERSION=${OPENEDX_RELEASE}
+# ENV REPO_OWNER=edx
+
+
+ENV EDX_NOTES_API_VENV="/edx/edx_notes_api/venvs/edx_notes_api"
+
+# ADD . /edx/app/edx_ansible/edx_ansible
+
+# WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
+WORKDIR /edx/app
+
+# COPY docker/build/notes/ansible_overrides.yml /
+# COPY docker/build/notes/edx_notes_api.yml /edx/etc/edx_notes_api.yml
+
+
+# Ansible Free work start
+
+# Creating Path
+RUN mkdir -p /edx/app/supervisor/conf.available.d
+RUN mkdir -p /edx/app/supervisor/conf.d
+
+## tag: install
+RUN sudo apt-get update && sudo apt-get -y install python3-dev libmysqlclient-dev python3-virtualenv python3-pip
+RUN apt-get install -y sudo
+ENV PATH="$EDX_NOTES_API_VENV/bin:$PATH"
+
+
+
+
+# Copying files
+COPY requirements/base.txt /edx/app/edx_notes_api/requirements/base.txt
+COPY conf_files/edx_notes_api_gunicorn.py /edx/app/edx_notes_api/edx_notes_api_gunicorn.py
+COPY conf_files/edx_notes_api.sh /edx/app/edx_notes_api/edx_notes_api.sh
+COPY conf_files/edx_notes_api.conf /edx/app/supervisor/conf.available.d/edx_notes_api.conf
+COPY conf_files/edx_notes_api_env /edx/app/edx_notes_api/edx_notes_api_env
+COPY conf_files/edx_notes_api.conf /edx/app/supervisor/conf.d/edx_notes_api.conf
+COPY conf_files/manage.edx_notes_api /edx/bin/manage.edx_notes_api
+
+RUN pip install -r /edx/app/edx_notes_api/requirements/base.txt
+
+
+## tag:devstack:install
+COPY conf_files/devstack.sh /edx/app/edx_notes_api/devstack.sh
+
+## tag:assets pending/in progress
+# RUN make assets
+
+
+# Ansible Free work end
+
+
+
+
+
+# RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook notes.yml \
+#     -c local -i '127.0.0.1,' \
+#     -t 'install,assets,devstack:install' \
+#     --extra-vars="@/ansible_overrides.yml" \
+#     --extra-vars="EDX_NOTES_API_VERSION=$NOTES_VERSION" \
+#     --extra-vars="COMMON_GIT_PATH=$REPO_OWNER"
+
+# USER root
+ENTRYPOINT ["/edx/app/edx_notes_api/devstack.sh"]

configuration_files/edx_notes_api_env

@@ -0,0 +1,7 @@
+# Ansible managed
+
+export EDXNOTES_CONFIG_ROOT="/edx/etc"
+export LANG="en-us"
+export DJANGO_SETTINGS_MODULE="notesserver.settings.devstack"
+export SERVICE_VARIANT="edx_notes_api"
+export PATH="/edx/app/edx_notes_api/venvs/edx_notes_api/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"

configuration_files/edx_notes_api_gunicorn.py

@@ -0,0 +1,50 @@
+"""
+gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html
+
+Ansible managed
+"""
+import multiprocessing
+
+preload_app = True
+timeout = 300
+bind = "0.0.0.0:8120"
+pythonpath = "/edx/app/edx_notes_api/edx_notes_api"
+limit_request_field_size = 16384
+
+workers = (multiprocessing.cpu_count()-1) * 2 + 2
+
+def pre_request(worker, req):
+    worker.log.info("%s %s" % (req.method, req.path))
+
+
+def close_all_caches():
+    # Close the cache so that newly forked workers cannot accidentally share
+    # the socket with the processes they were forked from. This prevents a race
+    # condition in which one worker could get a cache response intended for
+    # another worker.
+    # We do this in a way that is safe for 1.4 and 1.8 while we still have some
+    # 1.4 installations.
+    from django.conf import settings
+    from django.core import cache as django_cache
+    if hasattr(django_cache, 'caches'):
+        get_cache = django_cache.caches.__getitem__
+    else:
+        get_cache = django_cache.get_cache
+    for cache_name in settings.CACHES:
+        cache = get_cache(cache_name)
+        if hasattr(cache, 'close'):
+            cache.close()
+
+    # The 1.4 global default cache object needs to be closed also: 1.4
+    # doesn't ensure you get the same object when requesting the same
+    # cache. The global default is a separate Python object from the cache
+    # you get with get_cache("default"), so it will have its own connection
+    # that needs to be closed.
+    cache = django_cache.cache
+    if hasattr(cache, 'close'):
+        cache.close()
+
+
+def post_fork(server, worker):
+    close_all_caches()
+

configuration_files/supervisor.conf

@@ -0,0 +1,17 @@
+[program:nginx]
+command=nginx -g 'daemon off;'
+killasgroup=true
+stopasgroup=true
+
+[program:edx_notes_api]
+command=/edx/app/edx_notes_api/edx_notes_api.sh
+user=www-data
+directory=/edx/app/edx_notes_api/edx_notes_api
+stdout_logfile=/edx/var/log/supervisor/%(program_name)-stdout.log
+stderr_logfile=/edx/var/log/supervisor/%(program_name)-stderr.log
+killasgroup=true
+stopasgroup=true
+
+[supervisord]
+
+[supervisorctl]

configuration_files/supervisor.service

@@ -0,0 +1,29 @@
+[Unit]
+Description=supervisord - Supervisor process control system
+Documentation=http://supervisord.org
+After=network.target
+
+
+[Service]
+
+# User will be applied only to ExecStart, not other commands (i.e. ExecStartPre)
+# This is needed because pre_supervisor needs to write to supervisor/conf.d, which
+# supervisor_service_user does not have permission to do.
+PermissionsStartOnly=true
+User=www-data
+
+Type=forking
+TimeoutSec=432000
+
+ExecStart=/edx/app/supervisor/venvs/supervisor/bin/supervisord --configuration /edx/app/supervisor/supervisord.conf
+ExecReload=/edx/app/supervisor/venvs/supervisor/bin/supervisorctl reload
+ExecStop=/edx/app/supervisor/venvs/supervisor/bin/supervisorctl shutdown
+
+# Trust supervisor to kill all its children
+# Otherwise systemd will see that ExecStop ^ comes back synchronously and say "Oh, I can kill everyone in this cgroup"
+# https://www.freedesktop.org/software/systemd/man/systemd.service.html#ExecStop=
+# https://www.freedesktop.org/software/systemd/man/systemd.kill.html
+KillMode=none
+
+[Install]
+WantedBy=multi-user.target

configuration_files/supervisorctl

@@ -0,0 +1,10 @@
+#!/edx/app/supervisor/venvs/supervisor/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+
+from supervisor.supervisorctl import main
+
+if __name__ == '__main__':
+    sys.exit(main())
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])

manage.py

@@ -3,6 +3,7 @@
 import sys
 
 if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "notesserver.settings.dev")
     from django.core.management import execute_from_command_line
 
     execute_from_command_line(sys.argv)