Merge pull request #484 from lappis-unb/update_pipeline

Update pipeline
lappis-unb · May 8, 2019 · 9e6d01a · 9e6d01a
2 parents 213cbf4 + 31636ea
commit 9e6d01a
Show file tree

Hide file tree

Showing 12 changed files with 90 additions and 55 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -4,13 +4,11 @@ stages:
   - test style
   - validate format
   - test stories
+  - build requirements
+  - build coach
   - build
   - deploy
 
-variables:
-  BOT_LATEST_IMAGE: $CI_REGISTRY_IMAGE/bot:latest
-  WEB_LATEST_IMAGE: $CI_REGISTRY_IMAGE/web:latest
-
 .set_ssh_config: &set_ssh_config
   before_script:
     - apt-get update -y
@@ -26,45 +24,83 @@ test style:
 
 run dataset validator:
   stage: validate format
-  image: registry.gitlab.com/lappis-unb/services/tais/bot
+  image: lappis/coach:latest
   script:
     - cd coach/
     - make run-validator
 
 test stories:
   stage: test stories
-  image: registry.gitlab.com/lappis-unb/services/tais/bot
+  image: lappis/bot:latest
   script:
     - cd bot/
-    - make train > /dev/null 2>&1
     - make test-stories
+
+build requirements:
+  stage: build requirements
+  image: docker
+  tags:
+    - docker
+  services:
+    - docker:dind
+  script:
+    - docker login -u $DOCKERHUB_USER -p $DOCKERHUB_PASSWORD
+    - cd ./docker/bot
+    - ./build-base.sh publish
   only:
-    - merge_requests
+    refs:
+      - master
+    changes:
+      - ./docker/bot/requirements.txt
+  environment: homolog
+
+build coach:
+  stage: build coach
+  image: docker
+  tags:
+    - docker
+  services:
+    - docker:dind
+  script:
+    - docker login -u $DOCKERHUB_USER -p $DOCKERHUB_PASSWORD
+    - docker build . -f docker/bot/coach.Dockerfile -t lappis/coach:latest
+    - docker push lappis/coach:latest
+  only:
+    refs:
+      - master
+    changes:
+      - ./coach/data
+  environment: homolog
 
 build bot:
   stage: build
   image: docker
+  tags:
+    - docker
   services:
     - docker:dind
   script:
-    - docker login -u "gitlab-ci-token" -p "$CI_JOB_TOKEN" $CI_REGISTRY
-    - docker build -f docker/bot/bot.Dockerfile -t $BOT_LATEST_IMAGE .
-    - docker push $BOT_LATEST_IMAGE
+    - docker login -u $DOCKERHUB_USER -p $DOCKERHUB_PASSWORD
+    - docker build -f docker/bot/bot.Dockerfile -t lappis/bot:latest .
+    - docker push lappis/bot:latest
   only:
     - master
+  environment: homolog
 
 build web:
   stage: build
   image: docker
+  tags:
+    - docker
   services:
     - docker:dind
   script:
-    - docker login -u "gitlab-ci-token" -p "$CI_JOB_TOKEN" $CI_REGISTRY
-
-    - docker build -f docker/web/web.Dockerfile -t $WEB_LATEST_IMAGE .
-    - docker push $WEB_LATEST_IMAGE
+    - docker login -u $DOCKERHUB_USER -p $DOCKERHUB_PASSWORD
+    - docker build -f docker/web/web.Dockerfile -t lappis/bot-web:latest .
+    - docker push lappis/bot-web:latest
   only:
     - master
+  environment: homolog
 
 deploy bot to homolog:
   stage: deploy

diff --git a/Makefile b/Makefile
@@ -1,2 +1,6 @@
 train: 
 	docker build . -f docker/bot/coach.Dockerfile -t lappis/coach:latest
+	docker-compose build bot
+
+test-dialogue:
+	docker-compose run --rm bot make e2e
diff --git a/bot/Makefile b/bot/Makefile
@@ -1,27 +1,31 @@
 run-rocketchat:
+	make config-bot
 	python3 run-rocketchat.py
 
 run-api:
-	python3 -m rasa_core.run -d models/dialogue -u models/nlu/current --debug \
+	python3 -m rasa_core.run -d /models/dialogue -u /models/nlu/current --debug \
   --port 5005 --enable_api
 
 run-console:
-	python3 -m rasa_core.run -d models/dialogue -u models/nlu/current --debug
+	python3 -m rasa_core.run -d /models/dialogue -u /models/nlu/current --debug
 
 evaluate:
-	python3 -m rasa_core.evaluate -d models/dialogue -s data/core/
+	python3 -m rasa_core.evaluate -d /models/dialogue -s data/core/
 
 run-validator:
 	python3 validator.py --intents data/intents/ --stories data/stories --domain domain.yml
 
 # Force Makefile to execute e2e command every time
 .PHONY: all e2e clean
 e2e:
-	python3 -m rasa_core.evaluate default --core models/dialogue --nlu models/nlu/current --stories e2e/e2e_stories_1.md --e2e
-	python3 -m rasa_core.evaluate default --core models/dialogue --nlu models/nlu/current --stories e2e/e2e_stories_2.md --e2e
+	python3 -m rasa_core.evaluate default --core /models/dialogue --nlu /models/nlu/current --stories e2e/e2e_stories_1.md --e2e
+	python3 -m rasa_core.evaluate default --core /models/dialogue --nlu /models/nlu/current --stories e2e/e2e_stories_2.md --e2e
 
 e2e-test:	
-	python3 -m rasa_core.evaluate default --core models/dialogue --nlu models/nlu/current --stories e2e/e2e_stories_erro.md --e2e
+	python3 -m rasa_core.evaluate default --core /models/dialogue --nlu /models/nlu/current --stories e2e/e2e_stories_erro.md --e2e
 
 test-stories:
 	python3 test_stories.py --stories e2e/ --e2e
+
+config-bot:
+	python /scripts/bot_config.py -r ${ROCKETCHAT_URL} -an ${ROCKETCHAT_ADMIN_USERNAME} -ap ${ROCKETCHAT_ADMIN_PASSWORD} -bu ${ROCKETCHAT_BOT_USERNAME} -bp ${ROCKETCHAT_BOT_PASSWORD}
diff --git a/bot/run-rocketchat.py b/bot/run-rocketchat.py
@@ -56,4 +56,4 @@ def run(core_dir, nlu_dir):
 
 
 if __name__ == '__main__':
-    run('models/dialogue', 'models/nlu/current')
+    run('/models/dialogue', '/models/nlu/current')
diff --git a/bot/test_stories.py b/bot/test_stories.py
@@ -21,8 +21,8 @@
 FAILED_COLOR = utils.bcolors.FAIL
 BLUE_COLOR = utils.bcolors.OKBLUE
 BOLD_COLOR = utils.bcolors.BOLD
-CORE_DIR = 'models/dialogue'
-NLU_DIR = 'models/nlu/current'
+CORE_DIR = '/models/dialogue'
+NLU_DIR = '/models/nlu/current'
 
 
 parser = argparse.ArgumentParser()

diff --git a/coach/Makefile b/coach/Makefile
@@ -9,7 +9,7 @@ train: train-nlu train-core
 
 train-online:
 	python3 -m rasa_core.train -d domain.yml -s data/stories -o \
-	models/dialogue --epochs 15 --online --nlu models/nlu/current/
+	/src_models/dialogue --epochs 15 --online --nlu /src_models/nlu/current/
 
 run-validator:
 	python3 validator.py --intents data/intents/ --stories data/stories --domain domain.yml
diff --git a/coach/domain.yml b/coach/domain.yml
@@ -111,7 +111,7 @@ templates:
     - text:
           A minha missão é te guiar para a inscrição da sua proposta,
           Se você já sabe isso é porque você já tem um conhecimento avançado e talvez eu não consiga sanar as suas dúvidas.
-          Mas podemos continuar conversando sobre alguns conceitos básicos da lei Rouanet =)
+          Mas podemos continuar conversando sobre alguns conceitos básicos da Lei de Incentivo à Cultura =)
 
   utter_expressoes_indesejadas:
     - text: |

diff --git a/coach/policy_config.yml b/coach/policy_config.yml
@@ -1,6 +1,6 @@
 policies:
   - name: KerasPolicy
-    epochs: 10
+    epochs: 7
     batch_size: 10
     featurizer:
       - name: FullDialogueTrackerFeaturizer

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -32,28 +32,23 @@ services:
       context: .
       dockerfile: ./docker/bot/bot.Dockerfile
     restart: unless-stopped
+    command: make run-rocketchat
+    # OR
+    # command: make run-console
+    depends_on:
+      - rocketchat
     env_file:
       - ./docker/bot/bot.env
     ports:
       - 5005:5005
     volumes:
-      - models:/bot/models/:ro
       - ./bot:/bot
       - ./scripts:/scripts
-    depends_on:
-      - coach
-
-  coach:
-    image: lappis/coach:latest
-    command: cp -a /src_models/. /models/
-    volumes:
-      - models:/models/
-      - ./coach:/coach
-      - ./scripts:/scripts
+
 
   # =================================== Web ===================================
   web:
-    image: registry.gitlab.com/lappis-unb/services/tais/web
+    image: lappis/web:latest
     build:
       context: .
       dockerfile: ./docker/web/web.Dockerfile
@@ -192,5 +187,3 @@ volumes:
   esdata:
     driver: local
   esbackup:
-  models:
-
diff --git a/docker/bot/bot.Dockerfile b/docker/bot/bot.Dockerfile
@@ -1,6 +1,9 @@
+FROM lappis/coach:latest as coach
+
 FROM lappis/botrequirements:latest
 
 COPY ./bot /bot
+COPY --from=coach /src_models/ /models/
 COPY ./scripts /scripts
 
 WORKDIR /bot
@@ -19,8 +22,3 @@ ENV ROCKETCHAT_URL=rocketchat:3000         \
     ELASTICSEARCH_URL=elasticsearch:9200   
 
 RUN find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf
-
-CMD python /scripts/bot_config.py -r $ROCKETCHAT_URL                        \
-    -an $ROCKETCHAT_ADMIN_USERNAME -ap $ROCKETCHAT_ADMIN_PASSWORD    \
-    -bu $ROCKETCHAT_BOT_USERNAME -bp $ROCKETCHAT_BOT_PASSWORD \
-    && make run-rocketchat
diff --git a/scripts/deploy_bot.sh b/scripts/deploy_bot.sh
@@ -1,9 +1,9 @@
 #!/bin/bash
 
 sshpass  -p $1 ssh -o StrictHostKeyChecking=no root@$2 <<-'ENDSSH'
-		cd rouana/
-    docker-compose -f ./beta.docker-compose.yml stop bot
-		docker-compose -f ./beta.docker-compose.yml rm -f bot
-    docker rmi -f registry.gitlab.com/lappis-unb/services/tais/bot
-    docker-compose -f ./beta.docker-compose.yml up -d bot
+    cd rouana/
+    docker-compose stop bot
+		docker-compose rm -f bot
+		docker-compose pull bot
+    docker-compose up -d bot
 ENDSSH
diff --git a/scripts/deploy_web.sh b/scripts/deploy_web.sh
@@ -2,8 +2,8 @@
 
 sshpass  -p $1 ssh -o StrictHostKeyChecking=no root@$2 <<-'ENDSSH'
 		cd rouana/
-    docker-compose -f ./beta.docker-compose.yml stop web
-		docker-compose -f ./beta.docker-compose.yml rm -f web
-    docker rmi -f registry.gitlab.com/lappis-unb/services/tais/web
-    docker-compose -f ./beta.docker-compose.yml up -d web
+    docker-compose stop web
+		docker-compose rm -f web
+		docker-compose pull web
+    docker-compose up -d web
 ENDSSH
Original file line number	Diff line number	Diff line change
Expand Up		@@ -56,4 +56,4 @@ def run(core_dir, nlu_dir):


		if __name__ == '__main__':
		run('models/dialogue', 'models/nlu/current')
		run('/models/dialogue', '/models/nlu/current')