Adding Eureka Service Discovery to Keyserver

deployments/docker/Makefile

@@ -6,8 +6,9 @@ help:
 	@echo "Usage: make <target>\n"
 	@echo "where <target> is: 'bootstrap', 'up', 'all-up', 'ps', 'down', 'network' or 'clean'\n"
 
-private bootstrap:
+private/cega.yml private/ega_swe1.yml private/ega_fin1.yml private bootstrap:
 	@docker run --rm -it \
+		    -v /dev/urandom:/dev/random \
 		    -v ${PWD}:/ega \
 		    -v ${PWD}/../../extras/db.sql:/tmp/db.sql \
 		    -v ${PWD}/../../extras/generate_pgp_key.py:/tmp/generate_pgp_key.py \
@@ -17,8 +18,10 @@ private bootstrap:
 network:
 	@docker network inspect cega &>/dev/null || docker network create cega &>/dev/null
 
-up:network
-	@docker-compose up -d cega-mq cega-users mq-swe1 db-swe1 inbox-swe1 vault-swe1 ingest-swe1 keys-swe1 mq-fin1 inbox-fin1 db-fin1 vault-fin1 ingest-fin1 keys-fin1
+up:network private/cega.yml private/ega_swe1.yml private/ega_fin1.yml
+	@docker-compose -f private/cega.yml -f private/ega_swe1.yml -f private/ega_fin1.yml \
+	 		up -d \
+			cega-mq cega-users cega-eureka mq-swe1 db-swe1 inbox-swe1 vault-swe1 ingest-swe1 keys-swe1 mq-fin1 inbox-fin1 db-fin1 vault-fin1 ingest-fin1 keys-fin1
 
 all-up:
 	@docker-compose -f private/cega.yml -f private/ega_swe1.yml -f private/ega_fin1.yml up -d
@@ -27,9 +30,8 @@ ps:
 	@docker-compose ps
 
 down: #.env
-	@docker-compose down -v
+	@[[ -f private/cega.yml ]] && [[ -f private/ega_swe1.yml ]] && [[ -f private/ega_fin1.yml ]] && docker-compose down -v || echo "No recipe to bring containers down\nHave you bootstrapped? (ie make bootstrap)"
 
 clean:
 	rm -rf .env private
 	-docker network rm cega &>/dev/null
-

deployments/docker/bootstrap/boot.sh

@@ -59,8 +59,8 @@ cat >> ${PRIVATE}/cega/env <<EOF
 LEGA_INSTANCES=${INSTANCES// /,}
 EOF
 
-# Central EGA Users
-source ${HERE}/cega_users.sh
+# Central EGA Users and Eureka server
+source ${HERE}/cega.sh
 
 # Generate the configuration for each instance
 for INSTANCE in ${INSTANCES}

deployments/docker/bootstrap/cega_users.sh → deployments/docker/bootstrap/cega.sh

@@ -5,7 +5,7 @@ echomsg "Generating fake Central EGA users"
 
 [[ -x $(readlink ${OPENSSL}) ]] && echo "${OPENSSL} is not executable. Adjust the setting with --openssl" && exit 3
 
-mkdir -p ${PRIVATE}/cega/users
+mkdir -p ${PRIVATE}/cega/users/pgp
 
 EGA_USER_PASSWORD_JOHN=$(generate_password 16)
 EGA_USER_PASSWORD_JANE=$(generate_password 16)
@@ -56,6 +56,26 @@ chmod 777 ${PRIVATE}/cega/users/{swe1,fin1}
     ln -s ../john.yml .
 )
 
+echomsg "Generating PGP keys for EGA users"
+
+if [[ -f /tmp/generate_pgp_key.py ]]; then
+    # Running in a container
+    GEN_KEY="python3.6 /tmp/generate_pgp_key.py"
+else
+    # Running on host, outside a container
+    GEN_KEY="python ${EXTRAS}/generate_pgp_key.py"
+fi
+
+${GEN_KEY} "John Travolta" "[email protected]" "John" --passphrase "hi-john" --pub ${PRIVATE}/cega/users/pgp/john.pub --priv ${PRIVATE}/cega/users/pgp/john.sec --armor
+chmod 644 ${PRIVATE}/cega/users/pgp/john.pub
+
+${GEN_KEY} "Jane Fonda" "[email protected]" "Jane" --passphrase "hi-jane" --pub ${PRIVATE}/cega/users/pgp/jane.pub --priv ${PRIVATE}/cega/users/pgp/jane.sec --armor
+chmod 644 ${PRIVATE}/cega/users/pgp/jane.pub
+
+${GEN_KEY} "Taylor Swift" "[email protected]" "Taylor" --passphrase "hi-taylor" --pub ${PRIVATE}/cega/users/pgp/taylor.pub --priv ${PRIVATE}/cega/users/pgp/taylor.sec --armor
+chmod 644 ${PRIVATE}/cega/users/pgp/taylor.pub
+
+
 cat >> ${PRIVATE}/cega/.trace <<EOF
 #####################################################################
 #
@@ -67,6 +87,10 @@ EGA_USER_PASSWORD_JOHN    = ${EGA_USER_PASSWORD_JOHN}
 EGA_USER_PUBKEY_JOHN      = ./private/cega/users/john.pub
 EGA_USER_PUBKEY_JANE      = ./private/cega/users/jane.pub
 EGA_USER_PASSWORD_TAYLOR  = ${EGA_USER_PASSWORD_TAYLOR}
+#
+EGA_USER_PGP_JOHN         = ./private/cega/users/pgp/john.pub
+EGA_USER_PGP_JANE         = ./private/cega/users/pgp/jane.pub
+EGA_USER_PGP_TAYLOR       = ./private/cega/users/pgp/taylor.pub
 # =============================
 EOF
 
@@ -103,21 +127,37 @@ services:
     image: nbisweden/ega-base
     hostname: cega-users
     container_name: cega-users
-    ports:
-      - "9100:80"
+    #ports:
+    #  - "9100:80"
     expose:
       - "80"
     volumes:
       - ./cega/users:/cega/users:rw
-      - ../images/cega-users/users.html:/cega/users.html
-      - ../images/cega-users/server.py:/cega/server.py
+      - ../images/cega/users.html:/cega/users.html
+      - ../images/cega/server.py:/cega/server.py
       # - ../..:/root/.local/lib/python3.6/site-packages:ro
     restart: on-failure:3
     networks:
       - cega
     command: ["python3.6", "/cega/server.py"]
 
-
+  ############################################
+  # Fake Eureka server
+  ############################################
+  cega-eureka:
+    hostname: cega-eureka
+    #ports:
+    #  - "8761:8761"
+    expose:
+      - 8761
+    image: nbisweden/ega-base
+    container_name: cega-eureka
+    volumes:
+      - ../images/cega/eureka.py:/cega/eureka.py
+    restart: on-failure:3
+    networks:
+      - cega
+    command: ["python3.6", "/cega/eureka.py"]
 EOF
 
 # For the compose file

deployments/docker/bootstrap/instance.sh

@@ -45,14 +45,9 @@ chmod 644 ${PRIVATE}/${INSTANCE}/pgp/ega2.pub
 
 #########################################################################
 
-echomsg "\t* the RSA public and private key"
-#${OPENSSL} genpkey -algorithm RSA -pass pass:"${RSA_PASSPHRASE}" -out ${PRIVATE}/${INSTANCE}/rsa/ega.sec -pkeyopt rsa_keygen_bits:2048
-${OPENSSL} genpkey -algorithm RSA -out ${PRIVATE}/${INSTANCE}/rsa/ega.sec -pkeyopt rsa_keygen_bits:2048
-${OPENSSL} rsa -pubout -in ${PRIVATE}/${INSTANCE}/rsa/ega.sec -out ${PRIVATE}/${INSTANCE}/rsa/ega.pub
-
-#${OPENSSL} genpkey -algorithm RSA -pass pass:"${RSA_PASSPHRASE}" -out ${PRIVATE}/${INSTANCE}/rsa/ega2.sec -pkeyopt rsa_keygen_bits:2048
-${OPENSSL} genpkey -algorithm RSA -out ${PRIVATE}/${INSTANCE}/rsa/ega2.sec -pkeyopt rsa_keygen_bits:2048
-${OPENSSL} rsa -pubout -in ${PRIVATE}/${INSTANCE}/rsa/ega2.sec -out ${PRIVATE}/${INSTANCE}/rsa/ega2.pub
+echomsg "\t* the RSA private key"
+${OPENSSL} genpkey -algorithm RSA -pass pass:"${RSA_PASSPHRASE}" -out ${PRIVATE}/${INSTANCE}/rsa/ega.sec -pkeyopt rsa_keygen_bits:2048 -aes-256-cbc
+${OPENSSL} genpkey -algorithm RSA -pass pass:"${RSA_PASSPHRASE}" -out ${PRIVATE}/${INSTANCE}/rsa/ega2.sec -pkeyopt rsa_keygen_bits:2048 -aes-256-cbc
 
 #########################################################################
 
@@ -68,24 +63,22 @@ rsa : rsa.key.1
 pgp : pgp.key.1
 
 [rsa.key.1]
-public : /etc/ega/rsa/ega.pub
-private : /etc/ega/rsa/ega.sec
-#passphrase : ${RSA_PASSPHRASE}
+path : /etc/ega/rsa/ega.sec
+passphrase : ${RSA_PASSPHRASE}
+expire: 30/MAR/19 08:00:00
 
 [rsa.key.2]
-public : /etc/ega/rsa/ega2.pub
-private : /etc/ega/rsa/ega2.sec
-#passphrase : ${RSA_PASSPHRASE}
+path : /etc/ega/rsa/ega2.sec
+passphrase : ${RSA_PASSPHRASE}
+expire: 30/MAR/19 08:00:00
 
 [pgp.key.1]
-public : /etc/ega/pgp/ega.pub
-private : /etc/ega/pgp/ega.sec
+path : /etc/ega/pgp/ega.sec
 passphrase : ${PGP_PASSPHRASE}
 expire: 30/MAR/19 08:00:00
 
 [pgp.key.2]
-public : /etc/ega/pgp/ega2.pub
-private : /etc/ega/pgp/ega2.sec
+path : /etc/ega/pgp/ega2.sec
 passphrase : ${PGP_PASSPHRASE}
 expire: 30/MAR/18 08:00:00
 EOF
@@ -102,6 +95,10 @@ keyserver_endpoint_rsa = http://ega-keys-${INSTANCE}:443/active/rsa
 
 decrypt_cmd = python3.6 -u -m lega.openpgp %(file)s
 
+[outgestion]
+# Just for test
+keyserver_endpoint = https://ega-keys-${INSTANCE}:443/temp/file/%s
+
 ## Connecting to Local EGA
 [broker]
 host = ega-mq-${INSTANCE}
@@ -111,6 +108,9 @@ host = ega-db-${INSTANCE}
 username = ${DB_USER}
 password = ${DB_PASSWORD}
 try = ${DB_TRY}
+
+[eureka]
+endpoint = http://cega-eureka:8761
 EOF
 
 # echomsg "\t* SFTP Inbox port"
@@ -424,9 +424,6 @@ services:
       - mq-${INSTANCE}
       - keys-${INSTANCE}
     image: nbisweden/ega-base
-    # Required external link
-    external_links:
-      - cega-mq:cega-mq
     environment:
       - MQ_INSTANCE=ega-mq-${INSTANCE}
       - KEYSERVER_INSTANCE=ega-keys-${INSTANCE}
@@ -440,7 +437,6 @@ services:
     restart: on-failure:3
     networks:
       - lega_${INSTANCE}
-      - cega
     entrypoint: ["/bin/bash", "/usr/local/bin/entrypoint.sh"]
 
   # Key server
@@ -449,29 +445,30 @@ services:
     hostname: ega-keys-${INSTANCE}
     container_name: ega-keys-${INSTANCE}
     image: nbisweden/ega-base
-    tty: true
+    # For the /temp/file/{stable_id} queries
+    depends_on:
+      - db-${INSTANCE}
     expose:
       - "443"
-    ports:
-      - "${DOCKER_PORT_keyserver}:443"
+    #ports:
+    #  - "${DOCKER_PORT_keyserver}:443"
     volumes:
        - ./${INSTANCE}/ega.conf:/etc/ega/conf.ini:ro
        - ./${INSTANCE}/logger.yml:/etc/ega/logger.yml:ro
        - ./${INSTANCE}/keys.conf:/etc/ega/keys.ini:ro
        - ./${INSTANCE}/certs/ssl.cert:/etc/ega/ssl.cert:ro
        - ./${INSTANCE}/certs/ssl.key:/etc/ega/ssl.key:ro
-       - ./${INSTANCE}/pgp/ega.pub:/etc/ega/pgp/ega.pub:ro
        - ./${INSTANCE}/pgp/ega.sec:/etc/ega/pgp/ega.sec:ro
-       - ./${INSTANCE}/pgp/ega2.pub:/etc/ega/pgp/ega2.pub:ro
        - ./${INSTANCE}/pgp/ega2.sec:/etc/ega/pgp/ega2.sec:ro
-       - ./${INSTANCE}/rsa/ega.pub:/etc/ega/rsa/ega.pub:ro
        - ./${INSTANCE}/rsa/ega.sec:/etc/ega/rsa/ega.sec:ro
-       - ./${INSTANCE}/rsa/ega2.pub:/etc/ega/rsa/ega2.pub:ro
        - ./${INSTANCE}/rsa/ega2.sec:/etc/ega/rsa/ega2.sec:ro
        - ../../../lega:/root/.local/lib/python3.6/site-packages/lega
     restart: on-failure:3
+    external_links:
+      - cega-eureka:cega-eureka
     networks:
       - lega_${INSTANCE}
+      - cega
     entrypoint: ["ega-keyserver","--keys","/etc/ega/keys.ini"]
 
   # Vault
@@ -483,12 +480,8 @@ services:
     hostname: ega-vault
     container_name: ega-vault-${INSTANCE}
     image: nbisweden/ega-base
-    # Required external link
-    external_links:
-      - cega-mq:cega-mq
     environment:
       - MQ_INSTANCE=ega-mq-${INSTANCE}
-      - CEGA_INSTANCE=cega-mq
     volumes:
        - staging_${INSTANCE}:/ega/staging
        - vault_${INSTANCE}:/ega/vault
@@ -499,7 +492,6 @@ services:
     restart: on-failure:3
     networks:
       - lega_${INSTANCE}
-      - cega
     entrypoint: ["/bin/bash", "/usr/local/bin/entrypoint.sh"]
 
   # Logging & Monitoring (ELK: Elasticsearch, Logstash, Kibana).

deployments/docker/bootstrap/settings/fin1

@@ -4,7 +4,7 @@ set -e
 DOCKER_PORT_inbox=2223
 DOCKER_PORT_mq=15673
 DOCKER_PORT_kibana=5602
-DOCKER_PORT_keyserver=8444
+#DOCKER_PORT_keyserver=8444
 
 LEGA_GREETINGS="Welcome to Local EGA Finland @ CSC"
 CEGA_MQ_PASSWORD=$(generate_password 16)

deployments/docker/bootstrap/settings/swe1

@@ -4,7 +4,7 @@ set -e
 DOCKER_PORT_inbox=2222
 DOCKER_PORT_mq=15672
 DOCKER_PORT_kibana=5601
-DOCKER_PORT_keyserver=8443
+#DOCKER_PORT_keyserver=8443
 
 LEGA_GREETINGS="Welcome to Local EGA Sweden @ NBIS"
 CEGA_MQ_PASSWORD=$(generate_password 16)

deployments/docker/images/Makefile

@@ -20,7 +20,7 @@ TARGET=nbisweden/ega
 
 all: base inbox
 
-base: PIP_EGA_PACKAGES=pika==0.11.0 pycryptodomex==3.4.7 psycopg2==2.7.4 cryptography==2.1.3 aiohttp==2.3.8 aiohttp-jinja2==0.13.0 pgpy fusepy aiopg==0.13.0
+base: PIP_EGA_PACKAGES=pika==0.11.0 psycopg2==2.7.4 cryptography==2.1.4 aiohttp==2.3.8 aiohttp-jinja2==0.13.0 pgpy fusepy aiopg==0.13.0
 base inbox:
 	docker build --build-arg checkout=$(CHECKOUT) \
                      --build-arg PIP_EGA_PACKAGES="$(PIP_EGA_PACKAGES)" \

deployments/docker/images/cega/eureka.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+'''\
+A fake Eureka server.
+
+Spinning the whole Spring Framework Netflix Eureka would take too long,
+thus we are going to fake the responses.
+'''
+
+import sys
+import asyncio
+from aiohttp import web
+
+import logging as LOG
+
+
+routes = web.RouteTableDef()
+
+# Followjng the responses from https://github.com/Netflix/eureka/wiki/Eureka-REST-operations
+
+
+@routes.post('/eureka/apps/{app_name}')
+async def register(request):
+    """No matter the app it should register with success response 204."""
+    return web.HTTPNoContent()
+
+@routes.delete('/eureka/apps/{app_name}/{instance_id}')
+async def deregister(request):
+    """No matter the app it should deregister with success response 200."""
+    return web.HTTPOk()
+
+@routes.put('/eureka/apps/{app_name}/{instance_id}')
+async def heartbeat(request):
+    """No matter the app it should renew lease with success response 200."""
+    return web.HTTPOk()
+
+async def init(app):
+    '''Initialization running before the loop.run_forever'''
+    LOG.info('Initializing')
+
+async def shutdown(app):
+    '''Function run after a KeyboardInterrupt. After that: cleanup'''
+    LOG.info('Shutting down the database engine')
+
+async def cleanup(app):
+    '''Function run after a KeyboardInterrupt. Right after, the loop is closed'''
+    LOG.info('Cancelling all pending tasks')
+
+
+def main(args=None):
+    """Where the magic happens."""
+
+    host = sys.argv[1] if len(sys.argv) > 1 else "0.0.0.0"
+    port = 8761
+    sslcontext = None
+
+    loop = asyncio.get_event_loop()
+    eureka = web.Application(loop=loop)
+    eureka.router.add_routes(routes)
+
+    # Registering some initialization and cleanup routines
+    LOG.info('Setting up callbacks')
+    eureka.on_startup.append(init)
+    eureka.on_shutdown.append(shutdown)
+    eureka.on_cleanup.append(cleanup)
+
+    LOG.info(f"Start fake eureka on {host}:{port}")
+    web.run_app(eureka, host=host, port=port, shutdown_timeout=0, ssl_context=sslcontext)
+
+
+if __name__ == '__main__':
+    main()