refs #87 python3.6 support, docker correspond

ChangeLog.rst

@@ -1,6 +1,10 @@
 Unreleased
 ----------
 
+- [#177] 開発をDockerのみで行う様にする
+- [#87] Python3.6対応
+- [#56] slackbotのバージョンを上げる
+
 Release Notes - 2019-03-04
 --------------------------
 - [#174] $cleaning list の表示名はディスプレイ名を優先する

Makefile

@@ -0,0 +1,44 @@
+DC := docker-compose
+
+MYSQL_EXPOSE_PORT := 33061
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  build           to build docker images."
+	@echo "  deploy          to deploy bot"
+	@echo "  clean           to remove all docker containers, images"
+	@echo "  destroy         to remove all docker containers, images, and all volumes"
+	@echo "  down            to down all docker containers, images"
+	@echo "  test            to excute all tests."
+	@echo "  migrations      to generate alembic migration files."
+	@echo "  migrate         to apply alembic migration."
+	@echo "  help            to show this help messages"
+
+clean:
+	docker system prune
+	${DC} down --remove-orphans --rmi all
+
+destroy:
+	docker system prune
+	${DC} down --volume --remove-orphans --rmi all
+
+down:
+	${DC} down
+
+build:
+	${DC} build
+
+deploy: build
+	${DC} up -d
+	sh scripts/start_logging.sh
+	${DC} exec bot sh scripts/wait_for_mysql.sh honcho start migrate
+	${DC} restart bot
+
+test:
+	${DC} exec bot tox -e py36
+
+migrations:
+	${DC} exec bot honcho start makemigrations
+
+migrate:
+	${DC} exec bot honcho start migrate

README.md

@@ -6,7 +6,7 @@ Haro is [slackbot](https://github.com/lins05/slackbot "lins05/slackbot: A chat b
 
 ## 事前準備
 
-- Macのローカルで開発する場合、Docker、ansibleを構築する必要なし
+- 開発環境には、Dockerを利用します
 - 起動方法まで参照することで、ローカル環境が構築できる
 
 ### APIトークンの取得
@@ -16,42 +16,23 @@ Haro is [slackbot](https://github.com/lins05/slackbot "lins05/slackbot: A chat b
 - 「Save Integration」ボタンをクリックして保存する
   - API Token(``xoxb-XXXXXXX-XXXXXXX``)をこのあと使用するので、コピーしておく
 
-### Requirements
 
-- Python 3.5.2 or later.
+## 開発方法
 
-```bash
-$ python3 -m venv env
-$ git clone [email protected]:beproud/beproudbot.git
-$ cd beproudbot
-$ source /path/env/bin/activate
-(env)$ cp env.sample .env
-(env)$ vi .env # API Token 等を記入する
-(env)$ export SLACK_API_TOKEN=xoxb-XXXX　# API Tokenがターミナルの環境変数で.envに反映されない場合
-(env)$ pip install -r src/requirements.txt
-```
+- 開発に必要な操作はMakefileを参照
 
-## 起動方法
-
-```bash
-$ source /path/env/bin/activate
-# configには環境変数を指定します
-(env)$ export $(cat .env |grep -v '#')
-(env)$ cd src && python run.py
 ```
-
-### Docker
-
-```bash
-# MySQL を使用する場合先に立ち上げておく
-# docker-compose up -d db
-# bot の起動
-$ docker-compose build bot
-$ docker-compose run -d bot
-# コンテナにはいる
-$ docker-compose run --rm bot bash
-# 終了
-# docker-compose down
+Makefile
+$ make
+Please use `make <target>' where <target> is one of
+  build           to build docker images.
+  deploy          to deploy apps (local or dev)
+  test            to excute all tests.
+  flake8          to excute flake8 to python codes.
+  clean           to remove all docker containers, images
+  destroy         to remove all docker containers, images, and all volumes
+  down            to down all docker containers, images
+  help            to show this help messages
 ```
 
 ## DB操作
@@ -81,7 +62,6 @@ alembic を使用します
 honchoは .env を自動的に読み込み、スクリプトを開始することができます。
 
 ```bash
-(env)$ pip install honcho
 # honcho start bot
 # honcho start migrate
 # honcho start makemigrations

conf/mysql/Dockerfile

@@ -0,0 +1,4 @@
+FROM mysql:5.7
+
+
+COPY ./conf/mysql/my.cnf /etc/mysql/conf.d/

conf/mysql/my.cnf

@@ -0,0 +1,11 @@
+[mysqld]
+character-set-server = utf8
+collation-server = utf8_general_ci
+init-connect = SET NAMES utf8
+innodb-file-format = barracuda
+innodb_file_format_max = barracuda
+innodb_file_per_table = 1
+innodb_large_prefix = 1
+
+[client]
+loose-default-character-set = utf8

conf/python/Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.6-slim
+
+ENV PYTHONUNBUFFERED 1
+RUN mkdir /proj
+WORKDIR /proj
+COPY /src/constraints.txt /proj/src/
+COPY /src/requirements.txt /proj/src/
+
+RUN pip install --upgrade pip\
+    && apt-get update\
+    && apt-get install -y mysql-client \
+        default-libmysqlclient-dev \
+        build-essential \
+        python3-dev \
+    && pip install -r src/requirements.txt\
+    && rm -rf ~/.cache/pip \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY . /proj/
+
+CMD python3

docker-compose.yml

@@ -1,20 +1,30 @@
-version: '2'
+version: '3'
 services:
 
   bot:
-    build: .
-    #environment:
-    #  LANG: ja_JP.UTF-8
+    build:
+      context: ./
+      dockerfile: ./conf/python/Dockerfile
     env_file: .env
     volumes:
       - .:/proj
-    working_dir: /proj
     restart: always
-    command: "bash -c 'cd src && alembic --config alembic/conf.ini upgrade head && python3 run.py'"
+    command: /bin/bash /proj/scripts/runserver.sh
+    depends_on:
+      - db
 
   db:
-    image: library/mysql:5.6
-    environment:
-      MYSQL_ROOT_PASSWORD: example
-      MYSQL_DATABASE: haro
-    command: mysqld --character-set-server=utf8 --collation-server=utf8_unicode_ci
+    restart: always
+    env_file: .env
+    build:
+      context: ./
+      dockerfile: ./conf/mysql/Dockerfile
+    volumes:
+      - mysql-data:/var/lib/mysql
+    ports:
+      - '${MYSQL_EXPOSE_PORT:-33061}:3306'
+
+
+volumes:
+  mysql-data:
+    driver: local

env.sample

@@ -9,11 +9,14 @@ SLACK_ERRORS_TO=#my-channel or username
 REDMINE_URL=http://localhost/redmine_url
 # REDMINE_URL=https://project.beproud.jp/redmine
 HARO_DEBUG=True or False
-SQLALCHEMY_URL=sqlite:///haro.sqlite
 # SQLALCHEMY_URL=mysql+pymysql://root:example@db/haro?charset=utf8
+SQLALCHEMY_URL=mysql+pymysql://haro-user:haro-pass@db/haro?charset=utf8
 SQLALCHEMY_ECHO=true or false
 #SQLALCHEMY_POOL_SIZE=20
-
+MYSQL_USER=haro-user
+MYSQL_DATABASE=haro
+MYSQL_PASSWORD=haro-pass
+MYSQL_ROOT_PASSWORD=pass
 # ansible
 # ENVIRONMENT_FILE_PATH=/home/haro/beproudbot/.env
 # use_local_mysql_server=true

scripts/logfilter.py

@@ -0,0 +1,38 @@
+from datetime import datetime
+import fileinput
+import os
+
+here = os.path.dirname(__file__)
+LOG_DIR = os.path.abspath(os.path.join(here, '..', 'logs'))
+LOG_PREFIX = (
+    'db',
+    'bot'
+)
+
+
+def match_prefix(name):
+    for prefix in LOG_PREFIX:
+        if name.startswith(prefix):
+            return True
+    return False
+
+
+def main():
+    for line in fileinput.input():
+        today = datetime.now().strftime('%Y%m%d')
+        data = line.split(' ')
+        if match_prefix(data[0]):
+            log_name = '{}_{}.log'.format(data[0].strip(), today)
+            log_file = os.path.join(LOG_DIR, log_name)
+            row = ' '.join(data[5:])
+        else:
+            log_name = 'other_{}.log'.format(today)
+            log_file = os.path.join(LOG_DIR, log_name)
+            row = line
+
+        with open(log_file, 'a') as fp:
+            fp.write(row)
+
+
+if __name__ == '__main__':
+    main()

scripts/runserver.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+cd src && python3 run.py

scripts/start_logging.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+cd "$(dirname "$0")/.."
+
+LOG_PID=`ps -ef | grep "docker-compose logs" | grep -v "grep" | awk '{print $2}'`
+
+if [ -z "$LOG_PID" ];then
+  echo "Not found logging pid."
+else
+  kill -9 $LOG_PID
+fi
+
+nohup sh -c "docker-compose logs --no-color -f | python3 scripts/logfilter.py" &

scripts/wait_for_mysql.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# https://qiita.com/ry0f/items/6e29fa9f689b97058085
+set -e
+
+CMD="$@"
+
+until mysqladmin ping -h db --silent; do
+  echo 'Waiting for mysqld to be connectable...'
+  sleep 5
+done
+
+echo "Command start"
+exec $CMD

src/constraints.txt

@@ -14,9 +14,11 @@ python-dateutil==2.6.1
 python-editor==1.0.3
 requests==2.18.4
 six==1.11.0
-slackbot==0.5.1
+slackbot==0.5.3
 slacker==0.9.60
 SQLAlchemy==1.2.2
 urllib3==1.22
 websocket-client==0.44.0
 python-redmine==2.0.2
+honcho==1.0.1
+tox==3.7.0

src/requirements.txt

@@ -15,3 +15,6 @@ geopy
 kml2geojson
 
 pillow
+
+honcho
+tox

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py35, flake8
+envlist = py36, flake8
 skipsdist = True
 
 [testenv]
@@ -21,13 +21,13 @@ deps =
 
 commands = pytest {posargs}
 
-[testenv:py35]
+[testenv:py36]
 commands = pytest {posargs} \
            --junitxml={toxinidir}/test-results/pytest.xml
 
 [testenv:flake8]
 deps =
-    flake8==3.5
+    flake8==3.6
     flake8-blind-except
     flake8-import-order==0.14
     mccabe