From c84b4e6229b248be14c7f3d42c60de94595f541e Mon Sep 17 00:00:00 2001 From: zubairshakoorarbisoft Date: Fri, 16 Sep 2022 20:36:17 +0500 Subject: [PATCH 1/9] fix: install and devstack:install tags converted into docker(not tested) --- Dockerfile | 107 ++++++++++++--------------- conf_files/devstack.sh | 29 ++++++++ conf_files/edx_notes_api.conf | 1 + conf_files/edx_notes_api.sh | 13 ++++ conf_files/edx_notes_api_env | 7 ++ conf_files/edx_notes_api_gunicorn.py | 50 +++++++++++++ conf_files/manage.edx_notes_api | 1 + 7 files changed, 148 insertions(+), 60 deletions(-) create mode 100755 conf_files/devstack.sh create mode 120000 conf_files/edx_notes_api.conf create mode 100644 conf_files/edx_notes_api.sh create mode 100644 conf_files/edx_notes_api_env create mode 100644 conf_files/edx_notes_api_gunicorn.py create mode 120000 conf_files/manage.edx_notes_api diff --git a/Dockerfile b/Dockerfile index 0aa53ac0..720bff82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,79 +1,66 @@ -FROM ubuntu:focal as app -MAINTAINER sre@edx.org +# To build this Dockerfile: +# +# From the root of configuration: +# +# docker build -f docker/build/notes/Dockerfile . +# +# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible +# with the currently checked-out configuration repo. +ARG BASE_IMAGE_TAG=latest +FROM edxops/focal-common:${BASE_IMAGE_TAG} +LABEL maintainer="edxops" -# Packages installed: -# git; Used to pull in particular requirements from github rather than pypi, -# and to check the sha of the code checkout. +ARG OPENEDX_RELEASE=master +ENV OPENEDX_RELEASE=${OPENEDX_RELEASE} +ENV NOTES_VERSION=${OPENEDX_RELEASE} +ENV REPO_OWNER=edx -# ppa:deadsnakes/ppa; since Ubuntu doesn't ship with python 3.8 till 20, we need deadsnakes to install -# python 3.8 on older ubuntu versions -# language-pack-en locales; ubuntu locale support so that system utilities have a consistent -# language and time zone. +ENV EDX_NOTES_API_VENV="/edx/edx_notes_api/venvs/edx_notes_api" -# python3.8-dev; to install python 3.8 -# python3-venv; installs venv module required to create virtual environments +ADD . /edx/app/edx_ansible/edx_ansible -# libssl-dev; # mysqlclient wont install without this. +WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays -# libmysqlclient-dev; to install header files needed to use native C implementation for -# MySQL-python for performance gains. -# software-properties-common; to get apt-add-repository -# deadsnakes PPA to install Python 3.8 -# If you add a package here please include a comment above describing what it is used for +COPY docker/build/notes/ansible_overrides.yml / +COPY docker/build/notes/edx_notes_api.yml /edx/etc/edx_notes_api.yml -RUN apt-get update && \ - apt-get install -y software-properties-common && \ - apt-add-repository -y ppa:deadsnakes/ppa && \ - apt-get update && apt-get upgrade -qy && \ - apt-get install \ - language-pack-en \ - locales \ - git \ - libmysqlclient-dev \ - libssl-dev \ - build-essential \ - python3.8-dev \ - python3.8-distutils \ - python3.8-venv -qy && \ - rm -rf /var/lib/apt/lists/* -ENV VIRTUAL_ENV=/edx/app/edx-notes-api/venvs/edx-notes-api -RUN python3.8 -m venv $VIRTUAL_ENV -ENV PATH="$VIRTUAL_ENV/bin:$PATH" +# Ansible Free work start +## tag: install +RUN sudo apt-get update && sudo apt-get -y install python3-dev libmysqlclient-dev python3-virtualenv python3-pip +RUN apt-get install -y sudo +ENV PATH="$EDX_NOTES_API_VENV/bin:$PATH" +RUN pip install -r /edx/edx_notes_api/edx_notes_api/requirements/base.txt +# Copying files +COPY /edx/app/notes/conf_files/edx_notes_api_gunicorn.py /edx/app/edx_notes_api/edx_notes_api_gunicorn.py +COPY edx/app/notes/conf_files/edx_notes_api.sh /edx/app/edx_notes_api/edx_notes_api.sh +COPY /edx/app/notes/conf_files/edx_notes_api.conf /edx/app/supervisor/conf.available.d/edx_notes_api.conf +COPY /edx/app/notes/conf_files/edx_notes_api_env /edx/app/edx_notes_api/edx_notes_api_env +COPY /edx/app/notes/conf_files/edx_notes_api.conf /edx/app/supervisor/conf.d/edx_notes_api.conf +COPY /edx/app/notes/conf_files/manage.edx_notes_api /edx/bin/manage.edx_notes_api -RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 -ENV EDXNOTES_CONFIG_ROOT /edx/etc -ENV DJANGO_SETTINGS_MODULE notesserver.settings.yaml_config +## tag:devstack:install +COPY /edx/app/notes/conf_files/devstack.sh /edx/app/edx_notes_api/devstack.sh -EXPOSE 8120 -RUN useradd -m --shell /bin/false app +## tag:assets pending/in progress -WORKDIR /edx/app/notes -# Copy the requirements explicitly even though we copy everything below -# this prevents the image cache from busting unless the dependencies have changed. -COPY requirements/base.txt /edx/app/notes/requirements/base.txt -COPY requirements/pip.txt /edx/app/notes/requirements/pip.txt +# Ansible Free work end -# Dependencies are installed as root so they cannot be modified by the application user. -RUN pip install -r requirements/pip.txt -RUN pip install -r requirements/base.txt -RUN mkdir -p /edx/var/log -# Code is owned by root so it cannot be modified by the application user. -# So we copy it before changing users. -USER app -# Gunicorn 19 does not log to stdout or stderr by default. Once we are past gunicorn 19, the logging to STDOUT need not be specified. -CMD gunicorn --workers=2 --name notes -c /edx/app/notes/notesserver/docker_gunicorn_configuration.py --log-file - --max-requests=1000 notesserver.wsgi:application -# This line is after the requirements so that changes to the code will not -# bust the image cache -COPY . /edx/app/notes +# RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook notes.yml \ +# -c local -i '127.0.0.1,' \ +# -t 'install,assets,devstack:install' \ +# --extra-vars="@/ansible_overrides.yml" \ +# --extra-vars="EDX_NOTES_API_VERSION=$NOTES_VERSION" \ +# --extra-vars="COMMON_GIT_PATH=$REPO_OWNER" + +USER root +ENTRYPOINT ["/edx/app/edx_notes_api/devstack.sh"] +CMD ["start"] diff --git a/conf_files/devstack.sh b/conf_files/devstack.sh new file mode 100755 index 00000000..957a9e1b --- /dev/null +++ b/conf_files/devstack.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Ansible managed + +source /edx/app/edx_notes_api/edx_notes_api_env +COMMAND=$1 + +case $COMMAND in + start) + /edx/app/supervisor/venvs/supervisor/bin/supervisord -n --configuration /edx/app/supervisor/supervisord.conf + ;; + open) + . /edx/app/edx_notes_api/venvs/edx_notes_api/bin/activate + cd /edx/app/edx_notes_api/edx_notes_api + + /bin/bash + ;; + exec) + shift + + . /edx/app/edx_notes_api/venvs/edx_notes_api/bin/activate + cd /edx/app/edx_notes_api/edx_notes_api + + "$@" + ;; + *) + "$@" + ;; +esac diff --git a/conf_files/edx_notes_api.conf b/conf_files/edx_notes_api.conf new file mode 120000 index 00000000..ffb6777b --- /dev/null +++ b/conf_files/edx_notes_api.conf @@ -0,0 +1 @@ +/edx/app/supervisor/conf.available.d/edx_notes_api.conf \ No newline at end of file diff --git a/conf_files/edx_notes_api.sh b/conf_files/edx_notes_api.sh new file mode 100644 index 00000000..5251f250 --- /dev/null +++ b/conf_files/edx_notes_api.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Ansible managed + + +source /edx/app/edx_notes_api/edx_notes_api_env + +export PID=/var/tmp/edx_notes_api.pid +export PORT=8120 +export ADDRESS=0.0.0.0 + +# We exec so that gunicorn is the child of supervisor and can be managed properly +exec /edx/app/edx_notes_api/venvs/edx_notes_api/bin/gunicorn -c /edx/app/edx_notes_api/edx_notes_api_gunicorn.py notesserver.wsgi:application diff --git a/conf_files/edx_notes_api_env b/conf_files/edx_notes_api_env new file mode 100644 index 00000000..a455b039 --- /dev/null +++ b/conf_files/edx_notes_api_env @@ -0,0 +1,7 @@ +# Ansible managed + +export EDXNOTES_CONFIG_ROOT="/edx/etc" +export LANG="en-us" +export DJANGO_SETTINGS_MODULE="notesserver.settings.devstack" +export SERVICE_VARIANT="edx_notes_api" +export PATH="/edx/app/edx_notes_api/venvs/edx_notes_api/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin" diff --git a/conf_files/edx_notes_api_gunicorn.py b/conf_files/edx_notes_api_gunicorn.py new file mode 100644 index 00000000..9f06c4cc --- /dev/null +++ b/conf_files/edx_notes_api_gunicorn.py @@ -0,0 +1,50 @@ +""" +gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html + +Ansible managed +""" +import multiprocessing + +preload_app = True +timeout = 300 +bind = "0.0.0.0:8120" +pythonpath = "/edx/app/edx_notes_api/edx_notes_api" +limit_request_field_size = 16384 + +workers = (multiprocessing.cpu_count()-1) * 2 + 2 + +def pre_request(worker, req): + worker.log.info("%s %s" % (req.method, req.path)) + + +def close_all_caches(): + # Close the cache so that newly forked workers cannot accidentally share + # the socket with the processes they were forked from. This prevents a race + # condition in which one worker could get a cache response intended for + # another worker. + # We do this in a way that is safe for 1.4 and 1.8 while we still have some + # 1.4 installations. + from django.conf import settings + from django.core import cache as django_cache + if hasattr(django_cache, 'caches'): + get_cache = django_cache.caches.__getitem__ + else: + get_cache = django_cache.get_cache + for cache_name in settings.CACHES: + cache = get_cache(cache_name) + if hasattr(cache, 'close'): + cache.close() + + # The 1.4 global default cache object needs to be closed also: 1.4 + # doesn't ensure you get the same object when requesting the same + # cache. The global default is a separate Python object from the cache + # you get with get_cache("default"), so it will have its own connection + # that needs to be closed. + cache = django_cache.cache + if hasattr(cache, 'close'): + cache.close() + + +def post_fork(server, worker): + close_all_caches() + diff --git a/conf_files/manage.edx_notes_api b/conf_files/manage.edx_notes_api new file mode 120000 index 00000000..482d9491 --- /dev/null +++ b/conf_files/manage.edx_notes_api @@ -0,0 +1 @@ +/edx/app/edx_notes_api/edx_notes_api/manage.py \ No newline at end of file From b80cc9ffdbbba47bb2e2e86d6ad21896f5e3d975 Mon Sep 17 00:00:00 2001 From: Sarina Canelake Date: Sat, 10 Sep 2022 15:59:59 -0400 Subject: [PATCH 2/9] fix: fix github url strings (org edx -> openedx) --- CONTRIBUTING.rst | 2 +- README.rst | 2 +- db_keyword_overrides.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 10b3156f..e06ffb5a 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -3,7 +3,7 @@ How To Contribute Contributions are very welcome. -Please read `How To Contribute `_ for details. +Please read `How To Contribute `_ for details. Even though it was written with ``edx-platform`` in mind, the guidelines should be followed for Open edX code in general. diff --git a/README.rst b/README.rst index cfd09aff..070c58f2 100644 --- a/README.rst +++ b/README.rst @@ -74,7 +74,7 @@ How To Contribute Contributions are very welcome. -Please read `How To Contribute `_ for details. +Please read `How To Contribute `_ for details. Even though it was written with ``edx-platform`` in mind, the guidelines should be followed for Open edX code in general. diff --git a/db_keyword_overrides.yml b/db_keyword_overrides.yml index a9806769..4313f80b 100644 --- a/db_keyword_overrides.yml +++ b/db_keyword_overrides.yml @@ -1,7 +1,7 @@ # This file is used by the 'check_reserved_keywords' management command to allow specific field names to be overridden # when checking for conflicts with lists of restricted keywords used in various database/data warehouse tools. -# For more information, see: https://github.com/edx/edx-django-release-util/release_util/management/commands/check_reserved_keywords.py +# For more information, see: https://github.com/openedx/edx-django-release-util/release_util/management/commands/check_reserved_keywords.py # # overrides should be added in the following format: # - ModelName.field_name From 55d27ffb1a6ceb64192c2f879746b89f0329754c Mon Sep 17 00:00:00 2001 From: Sarina Canelake Date: Sat, 10 Sep 2022 17:47:28 -0400 Subject: [PATCH 3/9] fix: update path to .github workflows to read from openedx org --- .github/workflows/commitlint.yml | 2 +- .github/workflows/upgrade-python-requirements.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml index e2b06615..fec11d6c 100644 --- a/.github/workflows/commitlint.yml +++ b/.github/workflows/commitlint.yml @@ -7,4 +7,4 @@ on: jobs: commitlint: - uses: edx/.github/.github/workflows/commitlint.yml@master + uses: openedx/.github/.github/workflows/commitlint.yml@master diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml index 4ec4fd83..0cadb2cc 100644 --- a/.github/workflows/upgrade-python-requirements.yml +++ b/.github/workflows/upgrade-python-requirements.yml @@ -12,7 +12,7 @@ on: jobs: call-upgrade-python-requirements-workflow: - uses: edx/.github/.github/workflows/upgrade-python-requirements.yml@master + uses: openedx/.github/.github/workflows/upgrade-python-requirements.yml@master with: branch: ${{ github.event.inputs.branch || 'master' }} # optional parameters below; fill in if you'd like github or email notifications From 332bce7ae8a33e0d83320ace592bd433edf843e9 Mon Sep 17 00:00:00 2001 From: Sarina Canelake Date: Fri, 16 Sep 2022 18:11:50 -0400 Subject: [PATCH 4/9] chore: run edx_lint update with the current version of the repo. --- pylintrc | 660 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 363 insertions(+), 297 deletions(-) diff --git a/pylintrc b/pylintrc index 9a4879a3..a12ac82f 100644 --- a/pylintrc +++ b/pylintrc @@ -1,319 +1,385 @@ +# *************************** +# ** DO NOT EDIT THIS FILE ** +# *************************** +# +# This file was generated by edx-lint: https://github.com/openedx/edx-lint +# +# If you want to change this file, you have two choices, depending on whether +# you want to make a local change that applies only to this repo, or whether +# you want to make a central change that applies to all repos using edx-lint. +# +# Note: If your pylintrc file is simply out-of-date relative to the latest +# pylintrc in edx-lint, ensure you have the latest edx-lint installed +# and then follow the steps for a "LOCAL CHANGE". +# +# LOCAL CHANGE: +# +# 1. Edit the local pylintrc_tweaks file to add changes just to this +# repo's file. +# +# 2. Run: +# +# $ edx_lint write pylintrc +# +# 3. This will modify the local file. Submit a pull request to get it +# checked in so that others will benefit. +# +# +# CENTRAL CHANGE: +# +# 1. Edit the pylintrc file in the edx-lint repo at +# https://github.com/openedx/edx-lint/blob/master/edx_lint/files/pylintrc +# +# 2. install the updated version of edx-lint (in edx-lint): +# +# $ pip install . +# +# 3. Run (in edx-lint): +# +# $ edx_lint write pylintrc +# +# 4. Make a new version of edx_lint, submit and review a pull request with the +# pylintrc update, and after merging, update the edx-lint version and +# publish the new version. +# +# 5. In your local repo, install the newer version of edx-lint. +# +# 6. Run: +# +# $ edx_lint write pylintrc +# +# 7. This will modify the local file. Submit a pull request to get it +# checked in so that others will benefit. +# +# +# +# +# +# STAY AWAY FROM THIS FILE! +# +# +# +# +# +# SERIOUSLY. +# +# ------------------------------ +# Generated by edx-lint version: 5.3.0 +# ------------------------------ [MASTER] - -# Specify a configuration file. -#rcfile= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS, settings - -# Pickle collected data for later comparisons. -persistent=yes - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - +ignore = +persistent = yes +load-plugins = edx_lint.pylint,pylint_django,pylint_celery [MESSAGES CONTROL] - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. See also the "--disable" option for examples. -#enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable= - - locally-disabled, - no-self-use, - invalid-name, - no-member, - too-few-public-methods, - +enable = + blacklisted-name, + line-too-long, + + abstract-class-instantiated, + abstract-method, + access-member-before-definition, + anomalous-backslash-in-string, + anomalous-unicode-escape-in-string, + arguments-differ, + assert-on-tuple, + assigning-non-slot, + assignment-from-no-return, + assignment-from-none, + attribute-defined-outside-init, + bad-except-order, + bad-format-character, + bad-format-string-key, + bad-format-string, + bad-open-mode, + bad-reversed-sequence, + bad-staticmethod-argument, + bad-str-strip-call, + bad-super-call, + binary-op-exception, + boolean-datetime, + catching-non-exception, + cell-var-from-loop, + confusing-with-statement, + continue-in-finally, + dangerous-default-value, + duplicate-argument-name, + duplicate-bases, + duplicate-except, + duplicate-key, + expression-not-assigned, + format-combined-specification, + format-needs-mapping, + function-redefined, + global-variable-undefined, + import-error, + import-self, + inconsistent-mro, + inherit-non-class, + init-is-generator, + invalid-all-object, + invalid-format-index, + invalid-length-returned, + invalid-sequence-index, + invalid-slice-index, + invalid-slots-object, + invalid-slots, + invalid-unary-operand-type, + logging-too-few-args, + logging-too-many-args, + logging-unsupported-format, + lost-exception, + method-hidden, + misplaced-bare-raise, + misplaced-future, + missing-format-argument-key, + missing-format-attribute, + missing-format-string-key, + no-member, + no-method-argument, + no-name-in-module, + no-self-argument, + no-value-for-parameter, + non-iterator-returned, + non-parent-method-called, + nonexistent-operator, + not-a-mapping, + not-an-iterable, + not-callable, + not-context-manager, + not-in-loop, + pointless-statement, + pointless-string-statement, + raising-bad-type, + raising-non-exception, + redefined-builtin, + redefined-outer-name, + redundant-keyword-arg, + repeated-keyword, + return-arg-in-generator, + return-in-init, + return-outside-function, + signature-differs, + super-init-not-called, + super-method-not-called, + syntax-error, + test-inherits-tests, + too-few-format-args, + too-many-format-args, + too-many-function-args, + translation-of-non-string, + truncated-format-string, + undefined-all-variable, + undefined-loop-variable, + undefined-variable, + unexpected-keyword-arg, + unexpected-special-method-signature, + unpacking-non-sequence, + unreachable, + unsubscriptable-object, + unsupported-binary-operation, + unsupported-membership-test, + unused-format-string-argument, + unused-format-string-key, + used-before-assignment, + using-constant-test, + yield-outside-function, + + astroid-error, + fatal, + method-check-failed, + parse-error, + raw-checker-failed, + + empty-docstring, + invalid-characters-in-docstring, + missing-docstring, + wrong-spelling-in-comment, + wrong-spelling-in-docstring, + + unused-argument, + unused-import, + unused-variable, + + eval-used, + exec-used, + + bad-classmethod-argument, + bad-mcs-classmethod-argument, + bad-mcs-method-argument, + bare-except, + broad-except, + consider-iterating-dictionary, + consider-using-enumerate, + global-at-module-level, + global-variable-not-assigned, + literal-used-as-attribute, + logging-format-interpolation, + logging-not-lazy, + multiple-imports, + multiple-statements, + no-classmethod-decorator, + no-staticmethod-decorator, + protected-access, + redundant-unittest-assert, + reimported, + simplifiable-if-statement, + simplifiable-range, + singleton-comparison, + superfluous-parens, + unidiomatic-typecheck, + unnecessary-lambda, + unnecessary-pass, + unnecessary-semicolon, + unneeded-not, + useless-else-on-loop, + wrong-assert-type, + + deprecated-method, + deprecated-module, + + too-many-boolean-expressions, + too-many-nested-blocks, + too-many-statements, + + wildcard-import, + wrong-import-order, + wrong-import-position, + + missing-final-newline, + mixed-line-endings, + trailing-newlines, + trailing-whitespace, + unexpected-line-ending-format, + + bad-inline-option, + bad-option-value, + deprecated-pragma, + unrecognized-inline-option, + useless-suppression, +disable = + bad-indentation, + consider-using-f-string, + duplicate-code, + file-ignored, + fixme, + global-statement, + invalid-name, + locally-disabled, + no-else-return, + suppressed-message, + too-few-public-methods, + too-many-ancestors, + too-many-arguments, + too-many-branches, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-public-methods, + too-many-return-statements, + ungrouped-imports, + unspecified-encoding, + unused-wildcard-import, + use-maxsplit-arg, + + feature-toggle-needs-doc, + illegal-waffle-usage, + + logging-fstring-interpolation, [REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no - -# Tells whether to display a full report or only the messages -reports=yes - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - +output-format = text +reports = no +score = no [BASIC] - -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,apply,input,file - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Regular expression matching correct function names -function-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for function names -function-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for variable names -variable-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct constant names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for attribute names -attr-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for argument names -argument-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for method names -method-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=__.*__ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_$|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - - -[TYPECHECK] - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis -ignored-modules= - -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). -ignored-classes=SQLObject - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E0201 when accessed. Python regular -# expressions are accepted. -generated-members=REQUEST,acl_users,aq_parent - +module-rgx = (([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +const-rgx = (([A-Z_][A-Z0-9_]*)|(__.*__)|log|urlpatterns)$ +class-rgx = [A-Z_][a-zA-Z0-9]+$ +function-rgx = ([a-z_][a-z0-9_]{2,40}|test_[a-z0-9_]+)$ +method-rgx = ([a-z_][a-z0-9_]{2,40}|setUp|set[Uu]pClass|tearDown|tear[Dd]ownClass|assert[A-Z]\w*|maxDiff|test_[a-z0-9_]+)$ +attr-rgx = [a-z_][a-z0-9_]{2,30}$ +argument-rgx = [a-z_][a-z0-9_]{2,30}$ +variable-rgx = [a-z_][a-z0-9_]{2,30}$ +class-attribute-rgx = ([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ +inlinevar-rgx = [A-Za-z_][A-Za-z0-9_]*$ +good-names = f,i,j,k,db,ex,Run,_,__ +bad-names = foo,bar,baz,toto,tutu,tata +no-docstring-rgx = __.*__$|test_.+|setUp$|setUpClass$|tearDown$|tearDownClass$|Meta$ +docstring-min-length = 5 [FORMAT] - -# Maximum number of characters on a single line. -max-line-length=119 - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - -# List of optional constructs for which whitespace checking is disabled -no-space-check=trailing-comma,dict-separator - -# Maximum number of lines in a module -max-module-lines=1000 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - +max-line-length = 120 +ignore-long-lines = ^\s*(# )?((?)|(\.\. \w+: .*))$ +single-line-if-stmt = no +max-module-lines = 1000 +indent-string = ' ' [MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO - +notes = FIXME,XXX,TODO [SIMILARITIES] +min-similarity-lines = 4 +ignore-comments = yes +ignore-docstrings = yes +ignore-imports = no -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,TERMIOS,Bastion,rexec - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of statements in function / method body -max-statements=50 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 +[TYPECHECK] +ignore-mixin-members = yes +ignored-classes = SQLObject +unsafe-load-any-extension = yes +generated-members = + REQUEST, + acl_users, + aq_parent, + objects, + DoesNotExist, + can_read, + can_write, + get_url, + size, + content, + status_code, + create, + build, + fields, + tag, + org, + course, + category, + name, + revision, + _meta, +[VARIABLES] +init-import = no +dummy-variables-rgx = _|dummy|unused|.*_unused +additional-builtins = [CLASSES] +defining-attr-methods = __init__,__new__,setUp +valid-classmethod-first-arg = cls +valid-metaclass-classmethod-first-arg = mcs -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs +[DESIGN] +max-args = 5 +ignored-argument-names = _.* +max-locals = 15 +max-returns = 6 +max-branches = 12 +max-statements = 50 +max-parents = 7 +max-attributes = 7 +min-public-methods = 2 +max-public-methods = 20 +[IMPORTS] +deprecated-modules = regsub,TERMIOS,Bastion,rexec +import-graph = +ext-import-graph = +int-import-graph = [EXCEPTIONS] +overgeneral-exceptions = Exception -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception +# cbdfd50557cf2747f24a700b1caba610ff09bc6b From 5124c259f15e72a06a9995a7f1688f3572c0a2fd Mon Sep 17 00:00:00 2001 From: zubairshakoorarbisoft Date: Tue, 20 Sep 2022 17:25:41 +0500 Subject: [PATCH 5/9] fix: assets tag Dockerized --- Dockerfile | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 720bff82..bf0a7f92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,41 +11,53 @@ ARG BASE_IMAGE_TAG=latest FROM edxops/focal-common:${BASE_IMAGE_TAG} LABEL maintainer="edxops" -ARG OPENEDX_RELEASE=master -ENV OPENEDX_RELEASE=${OPENEDX_RELEASE} -ENV NOTES_VERSION=${OPENEDX_RELEASE} -ENV REPO_OWNER=edx +# ARG OPENEDX_RELEASE=master +# ENV OPENEDX_RELEASE=${OPENEDX_RELEASE} +# ENV NOTES_VERSION=${OPENEDX_RELEASE} +# ENV REPO_OWNER=edx ENV EDX_NOTES_API_VENV="/edx/edx_notes_api/venvs/edx_notes_api" -ADD . /edx/app/edx_ansible/edx_ansible +# ADD . /edx/app/edx_ansible/edx_ansible -WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays +# WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays -COPY docker/build/notes/ansible_overrides.yml / -COPY docker/build/notes/edx_notes_api.yml /edx/etc/edx_notes_api.yml +# COPY docker/build/notes/ansible_overrides.yml / +# COPY docker/build/notes/edx_notes_api.yml /edx/etc/edx_notes_api.yml # Ansible Free work start +# Creating Path +RUN mkdir -p /edx/app/supervisor/conf.available.d +RUN mkdir -p /edx/app/supervisor/conf.d + ## tag: install RUN sudo apt-get update && sudo apt-get -y install python3-dev libmysqlclient-dev python3-virtualenv python3-pip RUN apt-get install -y sudo ENV PATH="$EDX_NOTES_API_VENV/bin:$PATH" -RUN pip install -r /edx/edx_notes_api/edx_notes_api/requirements/base.txt + + + + # Copying files -COPY /edx/app/notes/conf_files/edx_notes_api_gunicorn.py /edx/app/edx_notes_api/edx_notes_api_gunicorn.py -COPY edx/app/notes/conf_files/edx_notes_api.sh /edx/app/edx_notes_api/edx_notes_api.sh -COPY /edx/app/notes/conf_files/edx_notes_api.conf /edx/app/supervisor/conf.available.d/edx_notes_api.conf -COPY /edx/app/notes/conf_files/edx_notes_api_env /edx/app/edx_notes_api/edx_notes_api_env -COPY /edx/app/notes/conf_files/edx_notes_api.conf /edx/app/supervisor/conf.d/edx_notes_api.conf -COPY /edx/app/notes/conf_files/manage.edx_notes_api /edx/bin/manage.edx_notes_api +COPY requirements/base.txt /edx/app/edx_notes_api/requirements/base.txt +COPY conf_files/edx_notes_api_gunicorn.py /edx/app/edx_notes_api/edx_notes_api_gunicorn.py +COPY conf_files/edx_notes_api.sh /edx/app/edx_notes_api/edx_notes_api.sh +COPY conf_files/edx_notes_api.conf /edx/app/supervisor/conf.available.d/edx_notes_api.conf +COPY conf_files/edx_notes_api_env /edx/app/edx_notes_api/edx_notes_api_env +COPY conf_files/edx_notes_api.conf /edx/app/supervisor/conf.d/edx_notes_api.conf +# COPY conf_files/manage.edx_notes_api /edx/bin/manage.edx_notes_api + +RUN pip install -r /edx/app/edx_notes_api/requirements/base.txt + ## tag:devstack:install -COPY /edx/app/notes/conf_files/devstack.sh /edx/app/edx_notes_api/devstack.sh +COPY conf_files/devstack.sh /edx/app/edx_notes_api/devstack.sh ## tag:assets pending/in progress +# RUN make assets # Ansible Free work end From 94d33c5a1c7e78755c5a7cdf66bc1b954baa7007 Mon Sep 17 00:00:00 2001 From: zubairshakoorarbisoft Date: Wed, 21 Sep 2022 19:31:12 +0500 Subject: [PATCH 6/9] ARGS and ENV names updated --- Dockerfile | 135 +++++++++++------- backup | 78 ++++++++++ conf_files/edx_notes_api.conf | 1 - conf_files/edx_notes_api.sh | 13 -- .../edx_notes_api_env | 0 .../edx_notes_api_gunicorn.py | 0 .../manage.edx_notes_api | 0 configuration_files/supervisor.conf | 13 ++ configuration_files/supervisor.service | 29 ++++ configuration_files/supervisorctl | 10 ++ {conf_files => scripts}/devstack.sh | 0 scripts/edx_notes_api.sh | 6 + 12 files changed, 222 insertions(+), 63 deletions(-) create mode 100644 backup delete mode 120000 conf_files/edx_notes_api.conf delete mode 100644 conf_files/edx_notes_api.sh rename {conf_files => configuration_files}/edx_notes_api_env (100%) rename {conf_files => configuration_files}/edx_notes_api_gunicorn.py (100%) rename {conf_files => configuration_files}/manage.edx_notes_api (100%) create mode 100644 configuration_files/supervisor.conf create mode 100644 configuration_files/supervisor.service create mode 100644 configuration_files/supervisorctl rename {conf_files => scripts}/devstack.sh (100%) create mode 100644 scripts/edx_notes_api.sh diff --git a/Dockerfile b/Dockerfile index bf0a7f92..56722e6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,78 +1,115 @@ -# To build this Dockerfile: -# -# From the root of configuration: -# -# docker build -f docker/build/notes/Dockerfile . -# -# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible -# with the currently checked-out configuration repo. - -ARG BASE_IMAGE_TAG=latest -FROM edxops/focal-common:${BASE_IMAGE_TAG} +FROM ubuntu:focal as app LABEL maintainer="edxops" -# ARG OPENEDX_RELEASE=master -# ENV OPENEDX_RELEASE=${OPENEDX_RELEASE} -# ENV NOTES_VERSION=${OPENEDX_RELEASE} -# ENV REPO_OWNER=edx +# System requirements. +ENV DEBIAN_FRONTEND=noninteractive +RUN apt update && \ + apt-get install -qy \ + curl \ + vim \ + git-core \ + language-pack-en \ + build-essential \ + python3.8-dev \ + python3-virtualenv \ + python3.8-distutils \ + libmysqlclient-dev \ + libssl-dev \ + libcairo2-dev && \ + rm -rf /var/lib/apt/lists/* + +ARG OPENEDX_RELEASE=master +ARG COMMON_APP_DIR="/edx/app" +ARG EDX_NOTES_API_SERVICE_NAME="edx_notes_api" +ARG EDX_NOTES_API_APP_DIR="${COMMON_APP_DIR}/${EDX_NOTES_API_SERVICE_NAME}" +ARG EDX_NOTES_API_CODE_DIR="${EDX_NOTES_API_APP_DIR}/${EDX_NOTES_API_SERVICE_NAME}" +ARG EDX_NOTES_API_VENV_DIR="${COMMON_APP_DIR}/${EDX_NOTES_API_SERVICE_NAME}/venvs/${EDX_NOTES_API_SERVICE_NAME}" +ARG SUPERVISOR_APP_DIR="${COMMON_APP_DIR}/supervisor" +ARG SUPERVISOR_AVAILABLE_DIR="${COMMON_APP_DIR}/supervisor/conf.available.d" +ARG SUPERVISOR_CFG_DIR="${SUPERVISOR_APP_DIR}/conf.d" +ARG SUPERVISOR_VENVS_DIR="${SUPERVISOR_APP_DIR}/venvs" +ARG SUPERVISOR_VENV_DIR="${SUPERVISOR_VENVS_DIR}/supervisor" +ARG SUPERVISOR_VENV_BIN="${SUPERVISOR_VENV_DIR}/bin" +ARG SUPERVISOR_VERSION="4.2.1" + + +ENV OPENEDX_RELEASE=${OPENEDX_RELEASE} +ENV NOTES_VERSION=${OPENEDX_RELEASE} +ENV REPO_OWNER=edx +ENV HOME /root +RUN addgroup edx_notes_api +RUN adduser --disabled-login --disabled-password edx_notes_api --ingroup edx_notes_api +ENV EDX_NOTES_API_HOME "${COMMON_APP_DIR}/${EDX_NOTES_API_SERVICE_NAME}" +ENV PATH "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin" +ENV PATH="$EDX_NOTES_API_VENV_DIR/bin:$PATH" + + +RUN virtualenv -p python3.8 --always-copy ${EDX_NOTES_API_VENV_DIR} +RUN virtualenv -p python3.8 --always-copy ${SUPERVISOR_VENV_DIR} + + +#install supervisor and deps in its virtualenv +RUN . ${SUPERVISOR_VENV_BIN}/activate && \ + pip install supervisor==${SUPERVISOR_VERSION} backoff==1.4.3 boto==2.48.0 && \ + deactivate + +COPY requirements/base.txt ${EDX_NOTES_API_CODE_DIR}/requirements/base.txt +RUN pip install -r ${EDX_NOTES_API_CODE_DIR}/requirements/base.txt + +# RUN sudo apt-get update && sudo apt-get -y install python3-dev libmysqlclient-dev python3-virtualenv python3-pip +ENV PATH="$EDX_NOTES_API_VENV/bin:$PATH" +# Working directory will be root of repo. +WORKDIR ${EDX_NOTES_API_CODE_DIR} -ENV EDX_NOTES_API_VENV="/edx/edx_notes_api/venvs/edx_notes_api" +# Copy over rest of code. +# We do this AFTER requirements so that the requirements cache isn't busted +# every time any bit of code is changed. +COPY . . -# ADD . /edx/app/edx_ansible/edx_ansible +# Copying files +COPY /configuration_files/edx_notes_api_gunicorn.py ${EDX_NOTES_API_HOME}/edx_notes_api_gunicorn.py +COPY /scripts/edx_notes_api.sh ${EDX_NOTES_API_HOME}/edx_notes_api.sh +COPY /configuration_files/supervisor.conf ${SUPERVISOR_APP_DIR}/supervisord.conf +COPY /manage.py /edx/bin/manage.edx_notes_api -# WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays -# COPY docker/build/notes/ansible_overrides.yml / -# COPY docker/build/notes/edx_notes_api.yml /edx/etc/edx_notes_api.yml +# create supervisor job +COPY /configuration_files/supervisor.service /etc/systemd/system/supervisor.service +COPY /configuration_files/supervisor.conf ${SUPERVISOR_CFG_DIR}/supervisor.conf +COPY /configuration_files/supervisorctl ${SUPERVISOR_VENV_BIN}/supervisorctl -# Ansible Free work start -# Creating Path -RUN mkdir -p /edx/app/supervisor/conf.available.d -RUN mkdir -p /edx/app/supervisor/conf.d -## tag: install -RUN sudo apt-get update && sudo apt-get -y install python3-dev libmysqlclient-dev python3-virtualenv python3-pip -RUN apt-get install -y sudo -ENV PATH="$EDX_NOTES_API_VENV/bin:$PATH" +EXPOSE 18281 -# Copying files -COPY requirements/base.txt /edx/app/edx_notes_api/requirements/base.txt -COPY conf_files/edx_notes_api_gunicorn.py /edx/app/edx_notes_api/edx_notes_api_gunicorn.py -COPY conf_files/edx_notes_api.sh /edx/app/edx_notes_api/edx_notes_api.sh -COPY conf_files/edx_notes_api.conf /edx/app/supervisor/conf.available.d/edx_notes_api.conf -COPY conf_files/edx_notes_api_env /edx/app/edx_notes_api/edx_notes_api_env -COPY conf_files/edx_notes_api.conf /edx/app/supervisor/conf.d/edx_notes_api.conf -# COPY conf_files/manage.edx_notes_api /edx/bin/manage.edx_notes_api +FROM app as prod + +ENV DJANGO_SETTINGS_MODULE "notesserver.settings.dev" -RUN pip install -r /edx/app/edx_notes_api/requirements/base.txt +RUN make static +ENTRYPOINT ["/edx/app/edx_notes_api/edx_notes_api.sh"] -## tag:devstack:install -COPY conf_files/devstack.sh /edx/app/edx_notes_api/devstack.sh +FROM app as dev -## tag:assets pending/in progress -# RUN make assets +ENV DJANGO_SETTINGS_MODULE "notesserver.settings.dev" +RUN pip install -r ${EDX_NOTES_API_CODE_DIR}/requirements/base.txt -# Ansible Free work end +COPY /scripts/devstack.sh ${EDX_NOTES_API_HOME}/devstack.sh +RUN chown edx_notes_api:edx_notes_api /edx/app/edx_notes_api/devstack.sh && chmod a+x /edx/app/edx_notes_api/devstack.sh +# Devstack related step for backwards compatibility +RUN touch /edx/app/${EDX_NOTES_API_SERVICE_NAME}/${EDX_NOTES_API_SERVICE_NAME}_env -# RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook notes.yml \ -# -c local -i '127.0.0.1,' \ -# -t 'install,assets,devstack:install' \ -# --extra-vars="@/ansible_overrides.yml" \ -# --extra-vars="EDX_NOTES_API_VERSION=$NOTES_VERSION" \ -# --extra-vars="COMMON_GIT_PATH=$REPO_OWNER" -USER root ENTRYPOINT ["/edx/app/edx_notes_api/devstack.sh"] CMD ["start"] diff --git a/backup b/backup new file mode 100644 index 00000000..abe2ec03 --- /dev/null +++ b/backup @@ -0,0 +1,78 @@ +# To build this Dockerfile: +# +# From the root of configuration: +# +# docker build -f docker/build/notes/Dockerfile . +# +# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible +# with the currently checked-out configuration repo. + +ARG BASE_IMAGE_TAG=latest +FROM edxops/focal-common:${BASE_IMAGE_TAG} +LABEL maintainer="edxops" + +# ARG OPENEDX_RELEASE=master +# ENV OPENEDX_RELEASE=${OPENEDX_RELEASE} +# ENV NOTES_VERSION=${OPENEDX_RELEASE} +# ENV REPO_OWNER=edx + + +ENV EDX_NOTES_API_VENV="/edx/edx_notes_api/venvs/edx_notes_api" + +# ADD . /edx/app/edx_ansible/edx_ansible + +# WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays +WORKDIR /edx/app + +# COPY docker/build/notes/ansible_overrides.yml / +# COPY docker/build/notes/edx_notes_api.yml /edx/etc/edx_notes_api.yml + + +# Ansible Free work start + +# Creating Path +RUN mkdir -p /edx/app/supervisor/conf.available.d +RUN mkdir -p /edx/app/supervisor/conf.d + +## tag: install +RUN sudo apt-get update && sudo apt-get -y install python3-dev libmysqlclient-dev python3-virtualenv python3-pip +RUN apt-get install -y sudo +ENV PATH="$EDX_NOTES_API_VENV/bin:$PATH" + + + + +# Copying files +COPY requirements/base.txt /edx/app/edx_notes_api/requirements/base.txt +COPY conf_files/edx_notes_api_gunicorn.py /edx/app/edx_notes_api/edx_notes_api_gunicorn.py +COPY conf_files/edx_notes_api.sh /edx/app/edx_notes_api/edx_notes_api.sh +COPY conf_files/edx_notes_api.conf /edx/app/supervisor/conf.available.d/edx_notes_api.conf +COPY conf_files/edx_notes_api_env /edx/app/edx_notes_api/edx_notes_api_env +COPY conf_files/edx_notes_api.conf /edx/app/supervisor/conf.d/edx_notes_api.conf +COPY conf_files/manage.edx_notes_api /edx/bin/manage.edx_notes_api + +RUN pip install -r /edx/app/edx_notes_api/requirements/base.txt + + +## tag:devstack:install +COPY conf_files/devstack.sh /edx/app/edx_notes_api/devstack.sh + +## tag:assets pending/in progress +# RUN make assets + + +# Ansible Free work end + + + + + +# RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook notes.yml \ +# -c local -i '127.0.0.1,' \ +# -t 'install,assets,devstack:install' \ +# --extra-vars="@/ansible_overrides.yml" \ +# --extra-vars="EDX_NOTES_API_VERSION=$NOTES_VERSION" \ +# --extra-vars="COMMON_GIT_PATH=$REPO_OWNER" + +# USER root +ENTRYPOINT ["/edx/app/edx_notes_api/devstack.sh"] diff --git a/conf_files/edx_notes_api.conf b/conf_files/edx_notes_api.conf deleted file mode 120000 index ffb6777b..00000000 --- a/conf_files/edx_notes_api.conf +++ /dev/null @@ -1 +0,0 @@ -/edx/app/supervisor/conf.available.d/edx_notes_api.conf \ No newline at end of file diff --git a/conf_files/edx_notes_api.sh b/conf_files/edx_notes_api.sh deleted file mode 100644 index 5251f250..00000000 --- a/conf_files/edx_notes_api.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -# Ansible managed - - -source /edx/app/edx_notes_api/edx_notes_api_env - -export PID=/var/tmp/edx_notes_api.pid -export PORT=8120 -export ADDRESS=0.0.0.0 - -# We exec so that gunicorn is the child of supervisor and can be managed properly -exec /edx/app/edx_notes_api/venvs/edx_notes_api/bin/gunicorn -c /edx/app/edx_notes_api/edx_notes_api_gunicorn.py notesserver.wsgi:application diff --git a/conf_files/edx_notes_api_env b/configuration_files/edx_notes_api_env similarity index 100% rename from conf_files/edx_notes_api_env rename to configuration_files/edx_notes_api_env diff --git a/conf_files/edx_notes_api_gunicorn.py b/configuration_files/edx_notes_api_gunicorn.py similarity index 100% rename from conf_files/edx_notes_api_gunicorn.py rename to configuration_files/edx_notes_api_gunicorn.py diff --git a/conf_files/manage.edx_notes_api b/configuration_files/manage.edx_notes_api similarity index 100% rename from conf_files/manage.edx_notes_api rename to configuration_files/manage.edx_notes_api diff --git a/configuration_files/supervisor.conf b/configuration_files/supervisor.conf new file mode 100644 index 00000000..6682a8f8 --- /dev/null +++ b/configuration_files/supervisor.conf @@ -0,0 +1,13 @@ +[program:nginx] +command=nginx -g 'daemon off;' +killasgroup=true +stopasgroup=true + +[program:edx_notes_api] +command=/edx/app/edx_notes_api/edx_notes_api.sh +user=www-data +directory=/edx/app/edx_notes_api/edx_notes_api +stdout_logfile=/edx/var/log/supervisor/%(program_name)-stdout.log +stderr_logfile=/edx/var/log/supervisor/%(program_name)-stderr.log +killasgroup=true +stopasgroup=true diff --git a/configuration_files/supervisor.service b/configuration_files/supervisor.service new file mode 100644 index 00000000..22e4edec --- /dev/null +++ b/configuration_files/supervisor.service @@ -0,0 +1,29 @@ +[Unit] +Description=supervisord - Supervisor process control system +Documentation=http://supervisord.org +After=network.target + + +[Service] + +# User will be applied only to ExecStart, not other commands (i.e. ExecStartPre) +# This is needed because pre_supervisor needs to write to supervisor/conf.d, which +# supervisor_service_user does not have permission to do. +PermissionsStartOnly=true +User=www-data + +Type=forking +TimeoutSec=432000 + +ExecStart=/edx/app/supervisor/venvs/supervisor/bin/supervisord --configuration /edx/app/supervisor/supervisord.conf +ExecReload=/edx/app/supervisor/venvs/supervisor/bin/supervisorctl reload +ExecStop=/edx/app/supervisor/venvs/supervisor/bin/supervisorctl shutdown + +# Trust supervisor to kill all its children +# Otherwise systemd will see that ExecStop ^ comes back synchronously and say "Oh, I can kill everyone in this cgroup" +# https://www.freedesktop.org/software/systemd/man/systemd.service.html#ExecStop= +# https://www.freedesktop.org/software/systemd/man/systemd.kill.html +KillMode=none + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/configuration_files/supervisorctl b/configuration_files/supervisorctl new file mode 100644 index 00000000..afb04433 --- /dev/null +++ b/configuration_files/supervisorctl @@ -0,0 +1,10 @@ +#!/edx/app/supervisor/venvs/supervisor/bin/python +# -*- coding: utf-8 -*- +import re +import sys + +from supervisor.supervisorctl import main + +if __name__ == '__main__': + sys.exit(main()) + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) \ No newline at end of file diff --git a/conf_files/devstack.sh b/scripts/devstack.sh similarity index 100% rename from conf_files/devstack.sh rename to scripts/devstack.sh diff --git a/scripts/edx_notes_api.sh b/scripts/edx_notes_api.sh new file mode 100644 index 00000000..0f074893 --- /dev/null +++ b/scripts/edx_notes_api.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + + +export EDX_REST_API_CLIENT_NAME="default_env-default_deployment-edx_notes_api" + +exec /edx/app/edx_notes_api/venvs/edx_notes_api/bin/gunicorn -c /edx/app/edx_notes_api/edx_notes_api_gunicorn.py --reload notesserver.wsgi:application \ No newline at end of file From 2203eb6a9471843bc78e8958d1257a40fa296937 Mon Sep 17 00:00:00 2001 From: zubairshakoorarbisoft Date: Wed, 21 Sep 2022 21:21:53 +0500 Subject: [PATCH 7/9] latest changes for configs added --- Dockerfile | 68 +++++++++++++----------- configuration_files/manage.edx_notes_api | 1 - configuration_files/supervisor.conf | 4 ++ manage.py | 1 + scripts/devstack.sh | 2 +- 5 files changed, 42 insertions(+), 34 deletions(-) delete mode 120000 configuration_files/manage.edx_notes_api diff --git a/Dockerfile b/Dockerfile index 56722e6c..aaa8a8c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,4 @@ FROM ubuntu:focal as app -LABEL maintainer="edxops" # System requirements. ENV DEBIAN_FRONTEND=noninteractive @@ -11,6 +10,7 @@ RUN apt update && \ language-pack-en \ build-essential \ python3.8-dev \ + python3-pip \ python3-virtualenv \ python3.8-distutils \ libmysqlclient-dev \ @@ -18,46 +18,59 @@ RUN apt update && \ libcairo2-dev && \ rm -rf /var/lib/apt/lists/* -ARG OPENEDX_RELEASE=master + +# Use UTF-8. +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + + + ARG COMMON_APP_DIR="/edx/app" ARG EDX_NOTES_API_SERVICE_NAME="edx_notes_api" +ENV EDX_NOTES_API_HOME "${COMMON_APP_DIR}/${EDX_NOTES_API_SERVICE_NAME}" ARG EDX_NOTES_API_APP_DIR="${COMMON_APP_DIR}/${EDX_NOTES_API_SERVICE_NAME}" -ARG EDX_NOTES_API_CODE_DIR="${EDX_NOTES_API_APP_DIR}/${EDX_NOTES_API_SERVICE_NAME}" -ARG EDX_NOTES_API_VENV_DIR="${COMMON_APP_DIR}/${EDX_NOTES_API_SERVICE_NAME}/venvs/${EDX_NOTES_API_SERVICE_NAME}" ARG SUPERVISOR_APP_DIR="${COMMON_APP_DIR}/supervisor" -ARG SUPERVISOR_AVAILABLE_DIR="${COMMON_APP_DIR}/supervisor/conf.available.d" -ARG SUPERVISOR_CFG_DIR="${SUPERVISOR_APP_DIR}/conf.d" +ARG EDX_NOTES_API_VENV_DIR="${COMMON_APP_DIR}/${EDX_NOTES_API_SERVICE_NAME}/venvs/${EDX_NOTES_API_SERVICE_NAME}" ARG SUPERVISOR_VENVS_DIR="${SUPERVISOR_APP_DIR}/venvs" ARG SUPERVISOR_VENV_DIR="${SUPERVISOR_VENVS_DIR}/supervisor" +ARG EDX_NOTES_API_CODE_DIR="${EDX_NOTES_API_APP_DIR}/${EDX_NOTES_API_SERVICE_NAME}" +ARG SUPERVISOR_AVAILABLE_DIR="${COMMON_APP_DIR}/supervisor/conf.available.d" ARG SUPERVISOR_VENV_BIN="${SUPERVISOR_VENV_DIR}/bin" +ARG SUPEVISOR_CTL="${SUPERVISOR_VENV_BIN}/supervisorctl" ARG SUPERVISOR_VERSION="4.2.1" +ARG SUPERVISOR_CFG_DIR="${SUPERVISOR_APP_DIR}/conf.d" -ENV OPENEDX_RELEASE=${OPENEDX_RELEASE} -ENV NOTES_VERSION=${OPENEDX_RELEASE} -ENV REPO_OWNER=edx ENV HOME /root +ENV PATH "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin" +ENV PATH "${EDX_NOTES_API_VENV_DIR}}/bin:$PATH" +ENV COMMON_CFG_DIR "/edx/etc" +ENV EDX_NOTES_API_CFG_DIR "${COMMON_CFG_DIR}/edx_notes_api" +ENV EDX_NOTES_API_CFG "/edx/etc/edx_notes_api.yml" + RUN addgroup edx_notes_api RUN adduser --disabled-login --disabled-password edx_notes_api --ingroup edx_notes_api -ENV EDX_NOTES_API_HOME "${COMMON_APP_DIR}/${EDX_NOTES_API_SERVICE_NAME}" -ENV PATH "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin" -ENV PATH="$EDX_NOTES_API_VENV_DIR/bin:$PATH" +# Make necessary directories and environment variables. +RUN mkdir -p /edx/var/edx_notes_api/staticfiles +RUN mkdir -p /edx/var/edx_notes_api/media +# Log dir +RUN mkdir /edx/var/log/ + RUN virtualenv -p python3.8 --always-copy ${EDX_NOTES_API_VENV_DIR} RUN virtualenv -p python3.8 --always-copy ${SUPERVISOR_VENV_DIR} - #install supervisor and deps in its virtualenv RUN . ${SUPERVISOR_VENV_BIN}/activate && \ pip install supervisor==${SUPERVISOR_VERSION} backoff==1.4.3 boto==2.48.0 && \ deactivate COPY requirements/base.txt ${EDX_NOTES_API_CODE_DIR}/requirements/base.txt -RUN pip install -r ${EDX_NOTES_API_CODE_DIR}/requirements/base.txt -# RUN sudo apt-get update && sudo apt-get -y install python3-dev libmysqlclient-dev python3-virtualenv python3-pip -ENV PATH="$EDX_NOTES_API_VENV/bin:$PATH" +RUN pip install -r ${EDX_NOTES_API_CODE_DIR}/requirements/base.txt # Working directory will be root of repo. WORKDIR ${EDX_NOTES_API_CODE_DIR} @@ -66,25 +79,19 @@ WORKDIR ${EDX_NOTES_API_CODE_DIR} # We do this AFTER requirements so that the requirements cache isn't busted # every time any bit of code is changed. COPY . . - -# Copying files COPY /configuration_files/edx_notes_api_gunicorn.py ${EDX_NOTES_API_HOME}/edx_notes_api_gunicorn.py +# COPY /configuration_files/discovery-workers.sh ${DISCOVERY_HOME}/discovery-workers.sh +# COPY /configuration_files/discovery.yml ${DISCOVERY_CFG} COPY /scripts/edx_notes_api.sh ${EDX_NOTES_API_HOME}/edx_notes_api.sh -COPY /configuration_files/supervisor.conf ${SUPERVISOR_APP_DIR}/supervisord.conf -COPY /manage.py /edx/bin/manage.edx_notes_api - - +# COPY /configuration_files/supervisor.conf ${SUPERVISOR_APP_DIR}/supervisord.conf # create supervisor job COPY /configuration_files/supervisor.service /etc/systemd/system/supervisor.service COPY /configuration_files/supervisor.conf ${SUPERVISOR_CFG_DIR}/supervisor.conf COPY /configuration_files/supervisorctl ${SUPERVISOR_VENV_BIN}/supervisorctl +# Manage.py symlink +COPY /manage.py /edx/bin/manage.edx_notes_api - - - - - - +# Expose canonical Discovery port EXPOSE 18281 FROM app as prod @@ -97,7 +104,7 @@ ENTRYPOINT ["/edx/app/edx_notes_api/edx_notes_api.sh"] FROM app as dev -ENV DJANGO_SETTINGS_MODULE "notesserver.settings.dev" +ENV DJANGO_SETTINGS_MODULE "notesserver.settings.devstack" RUN pip install -r ${EDX_NOTES_API_CODE_DIR}/requirements/base.txt @@ -108,8 +115,5 @@ RUN chown edx_notes_api:edx_notes_api /edx/app/edx_notes_api/devstack.sh && chmo # Devstack related step for backwards compatibility RUN touch /edx/app/${EDX_NOTES_API_SERVICE_NAME}/${EDX_NOTES_API_SERVICE_NAME}_env - - - ENTRYPOINT ["/edx/app/edx_notes_api/devstack.sh"] CMD ["start"] diff --git a/configuration_files/manage.edx_notes_api b/configuration_files/manage.edx_notes_api deleted file mode 120000 index 482d9491..00000000 --- a/configuration_files/manage.edx_notes_api +++ /dev/null @@ -1 +0,0 @@ -/edx/app/edx_notes_api/edx_notes_api/manage.py \ No newline at end of file diff --git a/configuration_files/supervisor.conf b/configuration_files/supervisor.conf index 6682a8f8..4b08b1c4 100644 --- a/configuration_files/supervisor.conf +++ b/configuration_files/supervisor.conf @@ -11,3 +11,7 @@ stdout_logfile=/edx/var/log/supervisor/%(program_name)-stdout.log stderr_logfile=/edx/var/log/supervisor/%(program_name)-stderr.log killasgroup=true stopasgroup=true + +[supervisord] + +[supervisorctl] diff --git a/manage.py b/manage.py index 746c5d00..e0a14a2d 100755 --- a/manage.py +++ b/manage.py @@ -3,6 +3,7 @@ import sys if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "notesserver.settings.dev") from django.core.management import execute_from_command_line execute_from_command_line(sys.argv) diff --git a/scripts/devstack.sh b/scripts/devstack.sh index 957a9e1b..a44f0c05 100755 --- a/scripts/devstack.sh +++ b/scripts/devstack.sh @@ -7,7 +7,7 @@ COMMAND=$1 case $COMMAND in start) - /edx/app/supervisor/venvs/supervisor/bin/supervisord -n --configuration /edx/app/supervisor/supervisord.conf + /edx/app/supervisor/venvs/supervisor/bin/supervisord -n --configuration /edx/app/supervisor/conf.d/supervisor.conf ;; open) . /edx/app/edx_notes_api/venvs/edx_notes_api/bin/activate From bd15fb54ee3f5a86d7d9ee514f85ecc5e6fc2a38 Mon Sep 17 00:00:00 2001 From: zubairshakoorarbisoft Date: Thu, 22 Sep 2022 12:55:52 +0500 Subject: [PATCH 8/9] fix: syntax error fixed --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aaa8a8c6..ff0fb31f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,7 @@ ARG SUPERVISOR_CFG_DIR="${SUPERVISOR_APP_DIR}/conf.d" ENV HOME /root ENV PATH "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin" -ENV PATH "${EDX_NOTES_API_VENV_DIR}}/bin:$PATH" +ENV PATH "${EDX_NOTES_API_VENV_DIR}/bin:$PATH" ENV COMMON_CFG_DIR "/edx/etc" ENV EDX_NOTES_API_CFG_DIR "${COMMON_CFG_DIR}/edx_notes_api" ENV EDX_NOTES_API_CFG "/edx/etc/edx_notes_api.yml" From 68a51cfd2bb88dcf59b31f4bdffbb42f3c535ae2 Mon Sep 17 00:00:00 2001 From: Adam Blackwell Date: Wed, 2 Nov 2022 16:31:00 -0400 Subject: [PATCH 9/9] fix: Remove MAINTAINER label --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0aa53ac0..2c2ad6a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,4 @@ FROM ubuntu:focal as app -MAINTAINER sre@edx.org - # Packages installed: # git; Used to pull in particular requirements from github rather than pypi,