Merge pull request #263 from NBISweden/feature/keyserver

Handling stable IDs from CentralEGA.

deployments/docker/bootstrap/instance.sh

@@ -46,9 +46,11 @@ chmod 644 ${PRIVATE}/${INSTANCE}/pgp/ega2.pub
 #########################################################################
 
 echomsg "\t* the RSA public and private key"
+#${OPENSSL} genpkey -algorithm RSA -pass pass:"${RSA_PASSPHRASE}" -out ${PRIVATE}/${INSTANCE}/rsa/ega.sec -pkeyopt rsa_keygen_bits:2048
 ${OPENSSL} genpkey -algorithm RSA -out ${PRIVATE}/${INSTANCE}/rsa/ega.sec -pkeyopt rsa_keygen_bits:2048
 ${OPENSSL} rsa -pubout -in ${PRIVATE}/${INSTANCE}/rsa/ega.sec -out ${PRIVATE}/${INSTANCE}/rsa/ega.pub
 
+#${OPENSSL} genpkey -algorithm RSA -pass pass:"${RSA_PASSPHRASE}" -out ${PRIVATE}/${INSTANCE}/rsa/ega2.sec -pkeyopt rsa_keygen_bits:2048
 ${OPENSSL} genpkey -algorithm RSA -out ${PRIVATE}/${INSTANCE}/rsa/ega2.sec -pkeyopt rsa_keygen_bits:2048
 ${OPENSSL} rsa -pubout -in ${PRIVATE}/${INSTANCE}/rsa/ega2.sec -out ${PRIVATE}/${INSTANCE}/rsa/ega2.pub
 
@@ -68,10 +70,12 @@ pgp : pgp.key.1
 [rsa.key.1]
 public : /etc/ega/rsa/ega.pub
 private : /etc/ega/rsa/ega.sec
+#passphrase : ${RSA_PASSPHRASE}
 
 [rsa.key.2]
 public : /etc/ega/rsa/ega2.pub
 private : /etc/ega/rsa/ega2.sec
+#passphrase : ${RSA_PASSPHRASE}
 
 [pgp.key.1]
 public : /etc/ega/pgp/ega.pub
@@ -93,8 +97,8 @@ log = /etc/ega/logger.yml
 
 [ingestion]
 # Keyserver communication
-keyserver_endpoint_pgp = https://ega-keys-${INSTANCE}/retrieve/pgp/%s
-keyserver_endpoint_rsa = https://ega-keys-${INSTANCE}/active/rsa
+keyserver_endpoint_pgp = http://ega-keys-${INSTANCE}:443/retrieve/pgp/%s
+keyserver_endpoint_rsa = http://ega-keys-${INSTANCE}:443/active/rsa
 
 decrypt_cmd = python3.6 -u -m lega.openpgp %(file)s
 
@@ -448,6 +452,8 @@ services:
     tty: true
     expose:
       - "443"
+    ports:
+      - "${DOCKER_PORT_keyserver}:443"
     volumes:
        - ./${INSTANCE}/ega.conf:/etc/ega/conf.ini:ro
        - ./${INSTANCE}/logger.yml:/etc/ega/logger.yml:ro
@@ -489,7 +495,7 @@ services:
        - ./${INSTANCE}/ega.conf:/etc/ega/conf.ini:ro
        - ./${INSTANCE}/logger.yml:/etc/ega/logger.yml:ro
        - ../images/vault/entrypoint.sh:/usr/local/bin/entrypoint.sh
-       # - ../../../lega:/root/.local/lib/python3.6/site-packages/lega
+       - ../../../lega:/root/.local/lib/python3.6/site-packages/lega
     restart: on-failure:3
     networks:
       - lega_${INSTANCE}

deployments/docker/bootstrap/settings/fin1

@@ -4,6 +4,7 @@ set -e
 DOCKER_PORT_inbox=2223
 DOCKER_PORT_mq=15673
 DOCKER_PORT_kibana=5602
+DOCKER_PORT_keyserver=8444
 
 LEGA_GREETINGS="Welcome to Local EGA Finland @ CSC"
 CEGA_MQ_PASSWORD=$(generate_password 16)
@@ -20,4 +21,6 @@ PGP_COMMENT="@CSC"
 PGP_EMAIL="[email protected]"
 PGP_PASSPHRASE=$(generate_password 16)
 
+RSA_PASSPHRASE=$(generate_password 16)
+
 LOG_LEVEL=INFO

deployments/docker/bootstrap/settings/swe1

@@ -4,6 +4,7 @@ set -e
 DOCKER_PORT_inbox=2222
 DOCKER_PORT_mq=15672
 DOCKER_PORT_kibana=5601
+DOCKER_PORT_keyserver=8443
 
 LEGA_GREETINGS="Welcome to Local EGA Sweden @ NBIS"
 CEGA_MQ_PASSWORD=$(generate_password 16)
@@ -20,4 +21,6 @@ PGP_COMMENT="@NBIS"
 PGP_EMAIL="[email protected]"
 PGP_PASSPHRASE=$(generate_password 16)
 
+RSA_PASSPHRASE=$(generate_password 16)
+
 LOG_LEVEL=DEBUG

deployments/docker/images/Makefile

@@ -20,7 +20,7 @@ TARGET=nbisweden/ega
 
 all: base inbox
 
-base: PIP_EGA_PACKAGES=pika==0.11.0 pycryptodomex==3.4.7 psycopg2==2.7.4 cryptography==2.1.3 aiohttp==2.3.8 aiohttp-jinja2==0.13.0 pgpy fusepy
+base: PIP_EGA_PACKAGES=pika==0.11.0 pycryptodomex==3.4.7 psycopg2==2.7.4 cryptography==2.1.3 aiohttp==2.3.8 aiohttp-jinja2==0.13.0 pgpy fusepy aiopg==0.13.0
 base inbox:
 	docker build --build-arg checkout=$(CHECKOUT) \
                      --build-arg PIP_EGA_PACKAGES="$(PIP_EGA_PACKAGES)" \

extras/db.sql

@@ -21,6 +21,7 @@ CREATE TABLE files (
 	status         status,
 	staging_name   TEXT,
 	stable_id      TEXT,
+	filepath       TEXT,
 	reenc_info     TEXT,
 	reenc_size     INTEGER,
 	reenc_checksum TEXT, -- sha256
@@ -30,19 +31,31 @@ CREATE TABLE files (
 
 CREATE FUNCTION insert_file(filename    files.filename%TYPE,
 			    eid         files.elixir_id%TYPE,
+			    stable_id   files.stable_id%TYPE,
 			    status      files.status%TYPE)
     RETURNS files.id%TYPE AS $insert_file$
     #variable_conflict use_column
     DECLARE
         file_id files.id%TYPE;
     BEGIN
-	INSERT INTO files (filename,elixir_id,status)
-	VALUES(filename,eid,status) RETURNING files.id
+	INSERT INTO files (filename,elixir_id,stable_id,status)
+	VALUES(filename,eid,stable_id,status) RETURNING files.id
 	INTO file_id;
 	RETURN file_id;
     END;
 $insert_file$ LANGUAGE plpgsql;
 
+CREATE FUNCTION translate_fileid_to_filepath(sid files.stable_id%TYPE)
+    RETURNS files.filepath%TYPE AS $translate_fileid_to_filepath$
+    #variable_conflict use_column
+    DECLARE
+        filepath files.filepath%TYPE;
+    BEGIN
+	SELECT filepath FROM files WHERE stable_id = sid LIMIT 1 INTO filepath;
+	RETURN filepath;
+    END;
+$translate_fileid_to_filepath$ LANGUAGE plpgsql;
+
 -- ##################################################
 --                      ERRORS
 -- ##################################################

extras/publish.py

@@ -28,13 +28,17 @@
 
 args = parser.parse_args()
 
-message = { 'user': args.user, 'filepath': args.filepath }
+stable_id = 'EGAF_'+str(uuid.uuid4())
+
+print('Ingesting file',stable_id)
+
+message = { 'user': args.user, 'filepath': args.filepath, 'stable_id': stable_id }
 if args.enc:
     message['encrypted_integrity'] = { 'checksum': args.enc, 'algorithm': args.enc_algo, }
 if args.unenc:
     message['unencrypted_integrity'] = { 'checksum': args.unenc, 'algorithm': args.unenc_algo, }
 
-print('Publishing:',message)
+#print('Publishing:',message)
 
 parameters = pika.URLParameters(args.connection)
 connection = pika.BlockingConnection(parameters)
@@ -44,4 +48,4 @@
                       properties=pika.BasicProperties(correlation_id=str(uuid.uuid4()), content_type='application/json',delivery_mode=2))
 
 connection.close()
-print('Message published')
+print('Message published to CentralEGA')

lega/conf/defaults.ini

@@ -44,3 +44,4 @@ ssl_certfile = /etc/ega/ssl.cert
 ssl_keyfile = /etc/ega/ssl.key
 host = 0.0.0.0
 port = 443
+eureka_endpoint = https://eureka.eu/register/service

lega/ingest.py

@@ -52,13 +52,14 @@ def work(active_master_key, master_pubkey, data):
     '''
 
     filepath = data['filepath']
-    LOG.info(f"Processing {filepath}")
+    stable_id = data['stable_id']
+    LOG.info(f"Processing {filepath} (with stable_id: {stable_id})")
 
     # Use user_id, and not elixir_id
     user_id = sanitize_user_id(data['user'])
 
     # Insert in database
-    file_id = db.insert_file(filepath, user_id)
+    file_id = db.insert_file(filepath, user_id, stable_id)
 
     # early record
     internal_data = {