From 7f96c40c4edd0aeb2ad08155dbf9e5154ae00e87 Mon Sep 17 00:00:00 2001 From: wan Date: Mon, 4 Mar 2019 09:46:46 +0900 Subject: [PATCH] refs #87 python3.6 support, docker correspond --- ChangeLog.rst | 4 ++++ Dockerfile | 13 ----------- Makefile | 44 +++++++++++++++++++++++++++++++++++ README.md | 48 ++++++++++++--------------------------- conf/mysql/Dockerfile | 4 ++++ conf/mysql/my.cnf | 11 +++++++++ conf/python/Dockerfile | 21 +++++++++++++++++ docker-compose.yml | 32 +++++++++++++++++--------- env.sample | 7 ++++-- nohup.out | 0 scripts/logfilter.py | 38 +++++++++++++++++++++++++++++++ scripts/runserver.sh | 3 +++ scripts/start_logging.sh | 13 +++++++++++ scripts/wait_for_mysql.sh | 14 ++++++++++++ src/constraints.txt | 4 +++- src/requirements.txt | 3 +++ tox.ini | 6 ++--- 17 files changed, 201 insertions(+), 64 deletions(-) delete mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 conf/mysql/Dockerfile create mode 100644 conf/mysql/my.cnf create mode 100644 conf/python/Dockerfile create mode 100644 nohup.out create mode 100644 scripts/logfilter.py create mode 100644 scripts/runserver.sh create mode 100644 scripts/start_logging.sh create mode 100644 scripts/wait_for_mysql.sh diff --git a/ChangeLog.rst b/ChangeLog.rst index 78bd4aa..a35a447 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,10 @@ Unreleased ---------- +- [#177] 開発をDockerのみで行う様にする +- [#87] Python3.6対応 +- [#56] slackbotのバージョンを上げる + Release Notes - 2019-03-04 -------------------------- - [#174] $cleaning list の表示名はディスプレイ名を優先する diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 20b103a..0000000 --- a/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM ubuntu:16.04 - -# 基本的な設定 -RUN apt update -y && apt install -y wget tzdata python3 sqlite3 -RUN wget -O- https://bootstrap.pypa.io/get-pip.py | python3 -RUN cp /usr/share/zoneinfo/Asia/Tokyo /etc/localtime - -# アプリの実行環境設定 -COPY ./src/constraints.txt /tmp -COPY ./src/requirements.txt /tmp -RUN pip install -r /tmp/requirements.txt - -RUN ln -s /usr/bin/python3 /usr/bin/python diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..21c693b --- /dev/null +++ b/Makefile @@ -0,0 +1,44 @@ +DC := docker-compose + +MYSQL_EXPOSE_PORT := 33061 + +help: + @echo "Please use \`make ' where is one of" + @echo " build to build docker images." + @echo " deploy to deploy bot" + @echo " clean to remove all docker containers, images" + @echo " destroy to remove all docker containers, images, and all volumes" + @echo " down to down all docker containers, images" + @echo " test to excute all tests." + @echo " migrations to generate alembic migration files." + @echo " migrate to apply alembic migration." + @echo " help to show this help messages" + +clean: + docker system prune + ${DC} down --remove-orphans --rmi all + +destroy: + docker system prune + ${DC} down --volume --remove-orphans --rmi all + +down: + ${DC} down + +build: + ${DC} build + +deploy: build + ${DC} up -d + sh scripts/start_logging.sh + ${DC} exec bot sh scripts/wait_for_mysql.sh honcho start migrate + ${DC} restart bot + +test: + ${DC} exec bot tox -e py36 + +migrations: + ${DC} exec bot honcho start makemigrations + +migrate: + ${DC} exec bot honcho start migrate diff --git a/README.md b/README.md index 3906b29..0077853 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Haro is [slackbot](https://github.com/lins05/slackbot "lins05/slackbot: A chat b ## 事前準備 -- Macのローカルで開発する場合、Docker、ansibleを構築する必要なし +- 開発環境には、Dockerを利用します - 起動方法まで参照することで、ローカル環境が構築できる ### APIトークンの取得 @@ -16,42 +16,23 @@ Haro is [slackbot](https://github.com/lins05/slackbot "lins05/slackbot: A chat b - 「Save Integration」ボタンをクリックして保存する - API Token(``xoxb-XXXXXXX-XXXXXXX``)をこのあと使用するので、コピーしておく -### Requirements -- Python 3.5.2 or later. +## 開発方法 -```bash -$ python3 -m venv env -$ git clone git@github.com:beproud/beproudbot.git -$ cd beproudbot -$ source /path/env/bin/activate -(env)$ cp env.sample .env -(env)$ vi .env # API Token 等を記入する -(env)$ export SLACK_API_TOKEN=xoxb-XXXX　# API Tokenがターミナルの環境変数で.envに反映されない場合 -(env)$ pip install -r src/requirements.txt -``` +- 開発に必要な操作はMakefileを参照 -## 起動方法 - -```bash -$ source /path/env/bin/activate -# configには環境変数を指定します -(env)$ export $(cat .env |grep -v '#') -(env)$ cd src && python run.py ``` - -### Docker - -```bash -# MySQL を使用する場合先に立ち上げておく -# docker-compose up -d db -# bot の起動 -$ docker-compose build bot -$ docker-compose run -d bot -# コンテナにはいる -$ docker-compose run --rm bot bash -# 終了 -# docker-compose down +Makefile +$ make +Please use `make ' where is one of + build to build docker images. + deploy to deploy apps (local or dev) + test to excute all tests. + flake8 to excute flake8 to python codes. + clean to remove all docker containers, images + destroy to remove all docker containers, images, and all volumes + down to down all docker containers, images + help to show this help messages ``` ## DB操作 @@ -81,7 +62,6 @@ alembic を使用します honchoは .env を自動的に読み込み、スクリプトを開始することができます。 ```bash -(env)$ pip install honcho # honcho start bot # honcho start migrate # honcho start makemigrations diff --git a/conf/mysql/Dockerfile b/conf/mysql/Dockerfile new file mode 100644 index 0000000..96f2190 --- /dev/null +++ b/conf/mysql/Dockerfile @@ -0,0 +1,4 @@ +FROM mysql:5.7 + + +COPY ./conf/mysql/my.cnf /etc/mysql/conf.d/ diff --git a/conf/mysql/my.cnf b/conf/mysql/my.cnf new file mode 100644 index 0000000..73691eb --- /dev/null +++ b/conf/mysql/my.cnf @@ -0,0 +1,11 @@ +[mysqld] +character-set-server = utf8 +collation-server = utf8_general_ci +init-connect = SET NAMES utf8 +innodb-file-format = barracuda +innodb_file_format_max = barracuda +innodb_file_per_table = 1 +innodb_large_prefix = 1 + +[client] +loose-default-character-set = utf8 diff --git a/conf/python/Dockerfile b/conf/python/Dockerfile new file mode 100644 index 0000000..bba72da --- /dev/null +++ b/conf/python/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.6-slim + +ENV PYTHONUNBUFFERED 1 +RUN mkdir /proj +WORKDIR /proj +COPY /src/constraints.txt /proj/src/ +COPY /src/requirements.txt /proj/src/ + +RUN pip install --upgrade pip\ + && apt-get update\ + && apt-get install -y mysql-client \ + default-libmysqlclient-dev \ + build-essential \ + python3-dev \ + && pip install -r src/requirements.txt\ + && rm -rf ~/.cache/pip \ + && rm -rf /var/lib/apt/lists/* + +COPY . /proj/ + +CMD python3 diff --git a/docker-compose.yml b/docker-compose.yml index 426d9f5..b7375bb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,20 +1,30 @@ -version: '2' +version: '3' services: bot: - build: . - #environment: - # LANG: ja_JP.UTF-8 + build: + context: ./ + dockerfile: ./conf/python/Dockerfile env_file: .env volumes: - .:/proj - working_dir: /proj restart: always - command: "bash -c 'cd src && alembic --config alembic/conf.ini upgrade head && python3 run.py'" + command: /bin/bash /proj/scripts/runserver.sh + depends_on: + - db db: - image: library/mysql:5.6 - environment: - MYSQL_ROOT_PASSWORD: example - MYSQL_DATABASE: haro - command: mysqld --character-set-server=utf8 --collation-server=utf8_unicode_ci + restart: always + env_file: .env + build: + context: ./ + dockerfile: ./conf/mysql/Dockerfile + volumes: + - mysql-data:/var/lib/mysql + ports: + - '${MYSQL_EXPOSE_PORT:-33061}:3306' + + +volumes: + mysql-data: + driver: local diff --git a/env.sample b/env.sample index 2457078..9589e1d 100644 --- a/env.sample +++ b/env.sample @@ -9,11 +9,14 @@ SLACK_ERRORS_TO=#my-channel or username REDMINE_URL=http://localhost/redmine_url # REDMINE_URL=https://project.beproud.jp/redmine HARO_DEBUG=True or False -SQLALCHEMY_URL=sqlite:///haro.sqlite # SQLALCHEMY_URL=mysql+pymysql://root:example@db/haro?charset=utf8 +SQLALCHEMY_URL=mysql+pymysql://haro-user:haro-pass@db/haro?charset=utf8 SQLALCHEMY_ECHO=true or false #SQLALCHEMY_POOL_SIZE=20 - +MYSQL_USER=haro-user +MYSQL_DATABASE=haro +MYSQL_PASSWORD=haro-pass +MYSQL_ROOT_PASSWORD=pass # ansible # ENVIRONMENT_FILE_PATH=/home/haro/beproudbot/.env # use_local_mysql_server=true diff --git a/nohup.out b/nohup.out new file mode 100644 index 0000000..e69de29 diff --git a/scripts/logfilter.py b/scripts/logfilter.py new file mode 100644 index 0000000..2fc25af --- /dev/null +++ b/scripts/logfilter.py @@ -0,0 +1,38 @@ +from datetime import datetime +import fileinput +import os + +here = os.path.dirname(__file__) +LOG_DIR = os.path.abspath(os.path.join(here, '..', 'logs')) +LOG_PREFIX = ( + 'db', + 'bot' +) + + +def match_prefix(name): + for prefix in LOG_PREFIX: + if name.startswith(prefix): + return True + return False + + +def main(): + for line in fileinput.input(): + today = datetime.now().strftime('%Y%m%d') + data = line.split(' ') + if match_prefix(data[0]): + log_name = '{}_{}.log'.format(data[0].strip(), today) + log_file = os.path.join(LOG_DIR, log_name) + row = ' '.join(data[5:]) + else: + log_name = 'other_{}.log'.format(today) + log_file = os.path.join(LOG_DIR, log_name) + row = line + + with open(log_file, 'a') as fp: + fp.write(row) + + +if __name__ == '__main__': + main() diff --git a/scripts/runserver.sh b/scripts/runserver.sh new file mode 100644 index 0000000..06efd01 --- /dev/null +++ b/scripts/runserver.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +cd src && python3 run.py diff --git a/scripts/start_logging.sh b/scripts/start_logging.sh new file mode 100644 index 0000000..4893b61 --- /dev/null +++ b/scripts/start_logging.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +cd "$(dirname "$0")/.." + +LOG_PID=`ps -ef | grep "docker-compose logs" | grep -v "grep" | awk '{print $2}'` + +if [ -z "$LOG_PID" ];then + echo "Not found logging pid." +else + kill -9 $LOG_PID +fi + +nohup sh -c "docker-compose logs --no-color -f | python3 scripts/logfilter.py" & diff --git a/scripts/wait_for_mysql.sh b/scripts/wait_for_mysql.sh new file mode 100644 index 0000000..11ab7c7 --- /dev/null +++ b/scripts/wait_for_mysql.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +# https://qiita.com/ry0f/items/6e29fa9f689b97058085 +set -e + +CMD="$@" + +until mysqladmin ping -h db --silent; do + echo 'Waiting for mysqld to be connectable...' + sleep 5 +done + +echo "Command start" +exec $CMD diff --git a/src/constraints.txt b/src/constraints.txt index c06e19b..9478273 100644 --- a/src/constraints.txt +++ b/src/constraints.txt @@ -14,9 +14,11 @@ python-dateutil==2.6.1 python-editor==1.0.3 requests==2.18.4 six==1.11.0 -slackbot==0.5.1 +slackbot==0.5.3 slacker==0.9.60 SQLAlchemy==1.2.2 urllib3==1.22 websocket-client==0.44.0 python-redmine==2.0.2 +honcho==1.0.1 +tox==3.7.0 diff --git a/src/requirements.txt b/src/requirements.txt index 9cf1031..77459f4 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -15,3 +15,6 @@ geopy kml2geojson pillow + +honcho +tox diff --git a/tox.ini b/tox.ini index 07ce176..460af75 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py35, flake8 +envlist = py36, flake8 skipsdist = True [testenv] @@ -21,13 +21,13 @@ deps = commands = pytest {posargs} -[testenv:py35] +[testenv:py36] commands = pytest {posargs} \ --junitxml={toxinidir}/test-results/pytest.xml [testenv:flake8] deps = - flake8==3.5 + flake8==3.6 flake8-blind-except flake8-import-order==0.14 mccabe