From eec0e8dcb95449087baa262cce3c3b6c5a3b779a Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Mon, 12 Mar 2018 11:10:12 +0200 Subject: [PATCH 1/6] Retrieving public key instead of private for RSA and adding generate/pgp endpoint.: --- lega/keyserver.py | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/lega/keyserver.py b/lega/keyserver.py index e588f4b0..d4a38844 100644 --- a/lega/keyserver.py +++ b/lega/keyserver.py @@ -40,7 +40,7 @@ from .openpgp.packet import iter_packets from .conf import CONF, KeysConfiguration from .utils import get_file_content -# from .openpgp.generate import generate_pgp_key +from .openpgp.generate import generate_pgp_key LOG = logging.getLogger('keyserver') routes = web.RouteTableDef() @@ -149,19 +149,20 @@ def load_key(self): class ReEncryptionKey: """ReEncryption currently done with a RSA key.""" - def __init__(self, key_id, secret_path, passphrase=''): + def __init__(self, key_id, public_path, secret_path=None, passphrase=''): """Intialise PrivateKey.""" self.secret_path = secret_path + self.public_path = public_path self.key_id = key_id assert( isinstance(passphrase,str) ) self.passphrase = passphrase.encode() def load_key(self): """Load key and return tuple for reconstruction.""" - data = get_file_content(self.secret_path) + public_data = get_file_content(self.public_path) # unlock it with the passphrase # TODO - return (self.key_id, data) + return (self.key_id, public_data) async def activate_key(key_name, data): @@ -171,7 +172,7 @@ async def activate_key(key_name, data): obj_key = PGPPrivateKey(data.get('private'), data.get('passphrase')) _cache = _pgp_cache elif key_name.startswith("rsa"): - obj_key = ReEncryptionKey(key_name, data.get('private'), passphrase='') + obj_key = ReEncryptionKey(key_name, data.get('public'), None, passphrase='') _cache = _rsa_cache else: LOG.error(f"Unrecognised key type: {key_name}") @@ -336,21 +337,21 @@ async def unlock_key(request): return web.HTTPBadRequest() -# @routes.post('generate/pgp') -# async def generate_pgp_key_pair(request): -# """Generate PGP key pair""" -# key_options = await request.json() -# LOG.debug(f'Admin generate PGP key pair: {key_options}') -# if all(k in key_options for k in("name", "comment", "email")): -# # By default we can return armored -# pub_data, sec_data = generate_pgp_key(key_options['name'], -# key_options['email'], -# key_options['comment'], -# key_options.get('passphrase', None)) -# # TO DO return the key pair or the path where it is stored. -# return web.HTTPAccepted() -# else: -# return web.HTTPBadRequest() +@routes.post('/generate/pgp') +async def generate_pgp_key_pair(request): + """Generate PGP key pair""" + key_options = await request.json() + LOG.debug(f'Admin generate PGP key pair: {key_options}') + if all(k in key_options for k in("name", "comment", "email")): + # By default we can return armored + pub_data, sec_data = generate_pgp_key(key_options['name'], + key_options['email'], + key_options['comment'], + key_options.get('passphrase', None)) + # TO DO return the key pair or the path where it is stored. + return web.HTTPAccepted() + else: + return web.HTTPBadRequest() @routes.get('/admin/ttl') From 5bce8727eeb7677fb4f4355e26414bd42ccfd9c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Haziza?= Date: Mon, 12 Mar 2018 15:18:25 +0100 Subject: [PATCH 2/6] Turning off SSL and adjusting the keyserver answers with hex-based dicts. --- deployments/docker/bootstrap/instance.sh | 4 +- lega/keyserver.py | 73 +++++++++++------------- lega/openpgp/__main__.py | 4 +- lega/openpgp/packet.py | 6 +- lega/openpgp/utils.py | 71 ++++++++++++++++++----- 5 files changed, 97 insertions(+), 61 deletions(-) diff --git a/deployments/docker/bootstrap/instance.sh b/deployments/docker/bootstrap/instance.sh index 37a1230f..c4d2df16 100755 --- a/deployments/docker/bootstrap/instance.sh +++ b/deployments/docker/bootstrap/instance.sh @@ -93,8 +93,8 @@ log = /etc/ega/logger.yml [ingestion] # Keyserver communication -keyserver_endpoint_pgp = https://ega-keys-${INSTANCE}/retrieve/pgp/%s -keyserver_endpoint_rsa = https://ega-keys-${INSTANCE}/active/rsa +keyserver_endpoint_pgp = http://ega-keys-${INSTANCE}:443/retrieve/pgp/%s +keyserver_endpoint_rsa = http://ega-keys-${INSTANCE}:443/active/rsa decrypt_cmd = python3.6 -u -m lega.openpgp %(file)s diff --git a/lega/keyserver.py b/lega/keyserver.py index d4a38844..f72ef2c3 100644 --- a/lega/keyserver.py +++ b/lega/keyserver.py @@ -130,20 +130,17 @@ def __init__(self, secret_path, passphrase): def load_key(self): """Load key and return tuple for reconstruction.""" - _public_key_material = None - _private_key_material = None + data = None with open(self.secret_path, 'rb') as infile: for packet in iter_packets(unarmor(infile)): LOG.info(str(packet)) if packet.tag == 5: - _public_key_material, _private_key_material = packet.unlock(self.passphrase) - _public_length = struct.pack('>I', len(_public_key_material)) - _private_length = struct.pack('>I', len(_private_key_material)) + data = packet.unlock(self.passphrase) self.key_id = packet.key_id else: packet.skip() - return (self.key_id.upper(), (_public_length, _public_key_material, _private_length, _private_key_material)) + return (self.key_id.upper(), data) class ReEncryptionKey: @@ -189,24 +186,19 @@ async def activate_key(key_name, data): @routes.get('/active/pgp') async def active_pgp_key(request): - """Retrieve tuple to reconstruced active unlocked key. + """Returns a JSON-formated list of numbers to reconstruct the active key, unlocked. - In case the output is not JSON, we use the following encoding: - First, 4 bytes for the length of the public part, followed by the public part. - Then, 4 bytes for the length of the private part, followed by the private part. + The JOSN response contains a "type" attribute to specify which key it is. + If type is "rsa", the public and private attributes contain ('n','e') and ('d','p','q','u') respectively. + If type is "dsa", the public and private attributes contain ('p','q','g','y') and ('x') respectively. + If type is "elg", the public and private attributes contain ('p','g','y') and ('x') respectively. + Other key types are not supported """ key_id = _pgp_cache.get("active_pgp_key") - request_type = request.content_type - LOG.debug(f'Requested active PGP key | {request_type}') + LOG.debug(f'Requesting active PGP key') value = _pgp_cache.get(key_id) if value: - if request_type == 'application/json': - return web.json_response({'public': value[1].hex(), 'private': value[3].hex()}) - response_body = b''.join(value) - if request_type == 'text/hex': - return web.Response(body=response_body.hex(), content_type='text/hex') - else: - return web.Response(body=response_body, content_type='application/octed-stream') + return web.json_response(value) else: LOG.warn(f"Active key not found.") return web.HTTPNotFound() @@ -219,7 +211,8 @@ async def active_pgp_key_private(request): LOG.debug(f'Requested active PGP (private) key.') value = _pgp_cache.get(key_id) if value: - return web.Response(body=value[3].hex()) + del value['public'] + return web.json_response(value) else: LOG.warn(f"Requested active PGP key not found.") return web.HTTPNotFound() @@ -232,7 +225,8 @@ async def active_pgp_key_public(request): LOG.debug(f'Requested active PGP (public) key.') value = _pgp_cache.get(key_id) if value: - return web.Response(body=value[1].hex()) + del value['private'] + return web.json_response(value) else: LOG.warn(f"Requested PGP key not found.") return web.HTTPNotFound() @@ -256,25 +250,20 @@ async def retrieve_active_rsa(request): @routes.get('/retrieve/pgp/{requested_id}') async def retrieve_pgp_key(request): - """Retrieve tuple to reconstruced unlocked key. + """Returns a JSON-formated list of numbers to reconstruct an unlocked key. - In case the output is not JSON, we use the following encoding: - First, 4 bytes for the length of the public part, followed by the public part. - Then, 4 bytes for the length of the private part, followed by the private part. + The JOSN response contains a "type" attribute to specify which key it is. + If type is "rsa", the public and private attributes contain ('n','e') and ('d','p','q','u') respectively. + If type is "dsa", the public and private attributes contain ('p','q','g','y') and ('x') respectively. + If type is "elg", the public and private attributes contain ('p','g','y') and ('x') respectively. + Other key types are not supported """ requested_id = request.match_info['requested_id'] - request_type = request.content_type - LOG.debug(f'Requested PGP key with ID {requested_id} | {request_type}') + LOG.debug(f'Requested PGP key with ID {requested_id}') key_id = requested_id[-16:].upper() value = _pgp_cache.get(key_id) if value: - if request_type == 'application/json': - return web.json_response({'public': value[1].hex(), 'private': value[3].hex()}) - response_body = b''.join(value) - if request_type == 'text/hex': - return web.Response(body=response_body.hex(), content_type='text/hex') - else: - return web.Response(body=response_body, content_type='application/octed-stream') + return web.json_response(value) else: LOG.warn(f"Requested PGP key {requested_id} not found.") return web.HTTPNotFound() @@ -391,14 +380,16 @@ def main(args=None): host = CONF.get('keyserver', 'host') # fallbacks are in defaults.ini port = CONF.getint('keyserver', 'port') - ssl_certfile = Path(CONF.get('keyserver', 'ssl_certfile')).expanduser() - ssl_keyfile = Path(CONF.get('keyserver', 'ssl_keyfile')).expanduser() - LOG.debug(f'Certfile: {ssl_certfile}') - LOG.debug(f'Keyfile: {ssl_keyfile}') + # ssl_certfile = Path(CONF.get('keyserver', 'ssl_certfile')).expanduser() + # ssl_keyfile = Path(CONF.get('keyserver', 'ssl_keyfile')).expanduser() + # LOG.debug(f'Certfile: {ssl_certfile}') + # LOG.debug(f'Keyfile: {ssl_keyfile}') + + # sslcontext = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + # sslcontext.check_hostname = False + # sslcontext.load_cert_chain(ssl_certfile, ssl_keyfile) - sslcontext = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - sslcontext.check_hostname = False - sslcontext.load_cert_chain(ssl_certfile, ssl_keyfile) + sslcontext = None # Turning off SSL for the moment loop = asyncio.get_event_loop() keyserver = web.Application(loop=loop) diff --git a/lega/openpgp/__main__.py b/lega/openpgp/__main__.py index 960abb80..9488150e 100644 --- a/lega/openpgp/__main__.py +++ b/lega/openpgp/__main__.py @@ -26,10 +26,8 @@ def fetch_private_key(key_id): LOG.info('Opening connection to %s', keyurl) with urlopen(req, context=ssl_ctx) as response: data = json.loads(response.read().decode()) - public_key_material = bytes.fromhex(data['public']) - private_key_material = bytes.fromhex(data['private']) LOG.info('Connection to the server closed for %s', key_id) - return make_key(public_key_material, private_key_material) + return make_key(data) except HTTPError as e: LOG.critical('Unknown PGP key %s', key_id) sys.exit(1) diff --git a/lega/openpgp/packet.py b/lega/openpgp/packet.py index 70992593..f4b4a79d 100644 --- a/lega/openpgp/packet.py +++ b/lega/openpgp/packet.py @@ -10,7 +10,7 @@ from .utils import (PGPError, read_1_byte, read_2_bytes, read_4_bytes, new_tag_length, old_tag_length, - get_mpi, parse_public_key_material, parse_private_key_material, + get_mpi, parse_public_key_material, parse_private_key_material, pack_key_material, derive_key, decryptor, make_decryptor, decompressor, @@ -289,7 +289,9 @@ def unlock(self, passphrase): validate_private_data(clear_private_data, self.s2k_usage) LOG.info('Passphrase correct') - return (self.public_part.getvalue(), clear_private_data) + + # Packing the return data + return pack_key_material(self.public_part, io.BytesIO(clear_private_data)) def __repr__(self): s = super().__repr__() diff --git a/lega/openpgp/utils.py b/lega/openpgp/utils.py index 5b7a78bc..be88efd5 100644 --- a/lega/openpgp/utils.py +++ b/lega/openpgp/utils.py @@ -268,21 +268,42 @@ def derive_key(passphrase, keylen, s2k_type, hash_algo, salt, count): return b''.join(_h.digest() for _h in h)[:keylen] -def make_rsa_key(n, e, d, p, q, u): +def make_rsa_key(material): + '''Convert a hex-based dict of values to an RSA key''' backend = default_backend() - pub = rsa.RSAPublicNumbers(e, n) + public_material = material['public'] + private_material = material['private'] + e = int(public_material['e'], 16) + n = int(public_material['n'], 16) + d = int(private_material['d'], 16) + p = int(private_material['p'], 16) + q = int(private_material['q'], 16) + pub = rsa.RSAPublicNumbers(e,n) dmp1 = rsa.rsa_crt_dmp1(d, p) dmq1 = rsa.rsa_crt_dmq1(d, q) iqmp = rsa.rsa_crt_iqmp(p, q) return rsa.RSAPrivateNumbers(p, q, d, dmp1, dmq1, iqmp, pub).private_key(backend), padding.PKCS1v15() -def make_dsa_key(y, g, p, q, x): +def make_dsa_key(material): + '''Convert a hex-based dict of values to a DSA key''' backend = default_backend() + public_material = material['public'] + private_material = material['private'] + p = int(public_material['p'], 16) + q = int(public_material['q'], 16) + g = int(public_material['g'], 16) + y = int(public_material['y'], 16) + x = int(private_material['x'], 16) params = dsa.DSAParameterNumbers(p,q,g) pn = dsa.DSAPublicNumbers(y, params) return dsa.DSAPrivateNumbers(x, pn).private_key(backend), None -def make_elg_key(y, g, p, q, x): +def make_elg_key(material): + # backend = default_backend() + # p = int(material['p'], 16) + # q = int(material['q'], 16) + # y = int(material['y'], 16) + # x = int(material['x'], 16) raise NotImplementedError() def parse_public_key_material(data, buf=None): @@ -337,23 +358,47 @@ def parse_private_key_material(raw_pub_algorithm, data, buf=None): raise PGPError(f"Experimental private key part: {raw_pub_algorithm}") raise PGPError(f"Unsupported public key algorithm {raw_pub_algorithm}") -def make_key(pub_stream, priv_stream): - '''Given the public and private part, as byte sequences, this function - parses them and return a key object''' - raw_alg, key_type, *public_key_material = parse_public_key_material(io.BytesIO(pub_stream)) - private_key_material = parse_private_key_material(raw_alg, io.BytesIO(priv_stream)) +def make_key(key_material): + '''Given the key_material, this function returns a key object''' - args = (int.from_bytes(n, "big") for n in chain(public_key_material, private_key_material)) + LOG.debug(f'-------------------- MAKE KEY from: {key_material}') + key_type = key_material["type"] if key_type == "rsa": - return make_rsa_key(*args) + return make_rsa_key(key_material) if key_type == "dsa": - return make_dsa_key(*args) + return make_dsa_key(key_material) if key_type == "elg": - return make_elg_key(*args) + return make_elg_key(key_material) assert False, "should not come here" return None +def pack_key_material(pub_stream, priv_stream): + pub_stream.seek(0,io.SEEK_SET) # rewind to beginning + priv_stream.seek(0,io.SEEK_SET) + raw_alg, key_type, *public_key_material = parse_public_key_material(pub_stream) + private_key_material = parse_private_key_material(raw_alg, priv_stream) + + if key_type == "rsa": + material_keys_pub = ('n','e') + material_keys_priv = ('d','p','q','u') + elif key_type == "dsa": + material_keys_pub = ('p','q','g','y') + material_keys_priv = ('x') + elif key_type == "elg": + material_keys_pub = ('p','g','y') + material_keys_priv = ('x') + else: + raise PGPError(f'Cannot pack a "{key_material}" key material') + + return { + "type": key_type, + "public": dict(zip(material_keys_pub, (v.hex() for v in public_key_material))), + #"private": dict(zip(chain(material_keys_pub, material_keys_priv), chain(public_key_material, private_key_material))), + "private": dict(zip(material_keys_priv, (v.hex() for v in private_key_material))), + } + + def validate_private_data(data, s2k_usage): if s2k_usage == 254: From b337b85460a8a71f7a9a04757aa43fa7e0255dd1 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 13 Mar 2018 09:06:03 +0200 Subject: [PATCH 3/6] keyserver new version and fix for DSA/ELG keys --- lega/keyserver.py | 181 +++++++++++++++++++++++++----------------- lega/openpgp/utils.py | 4 +- 2 files changed, 110 insertions(+), 75 deletions(-) diff --git a/lega/keyserver.py b/lega/keyserver.py index f72ef2c3..e2781f67 100644 --- a/lega/keyserver.py +++ b/lega/keyserver.py @@ -146,7 +146,7 @@ def load_key(self): class ReEncryptionKey: """ReEncryption currently done with a RSA key.""" - def __init__(self, key_id, public_path, secret_path=None, passphrase=''): + def __init__(self, key_id, public_path, secret_path, passphrase=''): """Intialise PrivateKey.""" self.secret_path = secret_path self.public_path = public_path @@ -156,10 +156,15 @@ def __init__(self, key_id, public_path, secret_path=None, passphrase=''): def load_key(self): """Load key and return tuple for reconstruction.""" - public_data = get_file_content(self.public_path) + public_data = get_file_content(self.public_path).hex() # unlock it with the passphrase + private_data = None + if self.secret_path: + private_data = get_file_content(self.secret_path).hex() # TODO - return (self.key_id, public_data) + return (self.key_id, {'id': self.key_id, + 'public': public_data, + 'private': private_data}) async def activate_key(key_name, data): @@ -169,7 +174,7 @@ async def activate_key(key_name, data): obj_key = PGPPrivateKey(data.get('private'), data.get('passphrase')) _cache = _pgp_cache elif key_name.startswith("rsa"): - obj_key = ReEncryptionKey(key_name, data.get('public'), None, passphrase='') + obj_key = ReEncryptionKey(key_name, data.get('public'), data.get('private', None), passphrase='') _cache = _rsa_cache else: LOG.error(f"Unrecognised key type: {key_name}") @@ -184,72 +189,90 @@ async def activate_key(key_name, data): # Retrieve the active keys # -@routes.get('/active/pgp') -async def active_pgp_key(request): +@routes.get('/active/{key_type}') +async def active_key(request): """Returns a JSON-formated list of numbers to reconstruct the active key, unlocked. - The JOSN response contains a "type" attribute to specify which key it is. - If type is "rsa", the public and private attributes contain ('n','e') and ('d','p','q','u') respectively. - If type is "dsa", the public and private attributes contain ('p','q','g','y') and ('x') respectively. - If type is "elg", the public and private attributes contain ('p','g','y') and ('x') respectively. + For PGP: + + * The JOSN response contains a "type" attribute to specify which key it is. + * If type is "rsa", the public and private attributes contain ('n','e') and ('d','p','q','u') respectively. + * If type is "dsa", the public and private attributes contain ('p','q','g','y') and ('x') respectively. + * If type is "elg", the public and private attributes contain ('p','g','y') and ('x') respectively. + + For RSA the public and private parts are retrieved in hex format. + Other key types are not supported """ - key_id = _pgp_cache.get("active_pgp_key") - LOG.debug(f'Requesting active PGP key') - value = _pgp_cache.get(key_id) + key_type = request.match_info['key_type'].lower() + if key_type == 'pgp': + active_key = "active_pgp_key" + _cache = _pgp_cache + elif key_type == 'rsa': + active_key = "active_rsa_key" + _cache = _rsa_cache + else: + return web.HTTPBadRequest() + key_id = _cache.get(active_key) + LOG.debug(f'Requesting active %s key', key_type.upper()) + value = _cache.get(key_id) if value: - return web.json_response(value) + return web.json_response(value) else: LOG.warn(f"Active key not found.") return web.HTTPNotFound() -@routes.get('/active/pgp/private') -async def active_pgp_key_private(request): +@routes.get('/active/{key_type}/private') +async def active_key_private(request): """Retrieve private part to reconstruced unlocked active key.""" - key_id = _pgp_cache.get("active_pgp_key") - LOG.debug(f'Requested active PGP (private) key.') - value = _pgp_cache.get(key_id) + key_type = request.match_info['key_type'].lower() + if key_type == 'pgp': + active_key = "active_pgp_key" + _cache = _pgp_cache + elif key_type == 'rsa': + active_key = "active_rsa_key" + _cache = _rsa_cache + else: + return web.HTTPBadRequest() + key_id = _cache.get(active_key) + LOG.debug(f'Requesting active %s (private) key', key_type.upper()) + value = dict(_cache.get(key_id)) if value: del value['public'] return web.json_response(value) else: - LOG.warn(f"Requested active PGP key not found.") + LOG.warn(f"Requested active %s (private) key not found.", key_type.upper()) return web.HTTPNotFound() -@routes.get('/active/pgp/public') -async def active_pgp_key_public(request): +@routes.get('/active/{key_type}/public') +async def active_key_public(request): """Retrieve public to reconstruced unlocked active key.""" - key_id = _pgp_cache.get("active_pgp_key") - LOG.debug(f'Requested active PGP (public) key.') - value = _pgp_cache.get(key_id) + key_type = request.match_info['key_type'].lower() + if key_type == 'pgp': + active_key = "active_pgp_key" + _cache = _pgp_cache + elif key_type == 'rsa': + active_key = "active_rsa_key" + _cache = _rsa_cache + else: + return web.HTTPBadRequest() + key_id = _cache.get(active_key) + LOG.debug(f'Requesting active %s (public) key', key_type.upper()) + value = dict(_cache.get(key_id)) if value: del value['private'] return web.json_response(value) else: - LOG.warn(f"Requested PGP key not found.") - return web.HTTPNotFound() - - -@routes.get('/active/rsa') -async def retrieve_active_rsa(request): - """Retrieve RSA reencryption key.""" - key_id = _rsa_cache.get("active_rsa_key") - LOG.debug(f'Requested active RSA key') - value = _rsa_cache.get(key_id) - if value: - return web.json_response({ 'id': key_id, - 'public': value.hex()}) - else: - LOG.warn(f"Requested ReEncryption Key not found.") + LOG.warn(f"Requested %s key (public) not found.", key_type.upper()) return web.HTTPNotFound() # Just want to get a key by its key_id PGP or RSA -@routes.get('/retrieve/pgp/{requested_id}') -async def retrieve_pgp_key(request): +@routes.get('/retrieve/{key_type}/{requested_id}') +async def retrieve_key(request): """Returns a JSON-formated list of numbers to reconstruct an unlocked key. The JOSN response contains a "type" attribute to specify which key it is. @@ -259,55 +282,67 @@ async def retrieve_pgp_key(request): Other key types are not supported """ requested_id = request.match_info['requested_id'] - LOG.debug(f'Requested PGP key with ID {requested_id}') - key_id = requested_id[-16:].upper() - value = _pgp_cache.get(key_id) + key_type = request.match_info['key_type'].lower() + if key_type == 'pgp': + _cache = _pgp_cache + key_id = requested_id[-16:].upper() + elif key_type == 'rsa': + _cache = _rsa_cache + key_id = requested_id + else: + return web.HTTPBadRequest() + LOG.debug(f'Requested {key_type.upper()} key with ID {requested_id}') + value = _cache.get(key_id) if value: return web.json_response(value) else: - LOG.warn(f"Requested PGP key {requested_id} not found.") + LOG.warn(f"Requested {key_type.upper()} key {requested_id} not found.") return web.HTTPNotFound() -@routes.get('/retrieve/pgp/{requested_id}/private') -async def retrieve_pgp_key_private(request): +@routes.get('/retrieve/{key_type}/{requested_id}/private') +async def retrieve_key_private(request): """Retrieve private part to reconstruced unlocked key.""" requested_id = request.match_info['requested_id'] - LOG.debug(f'Requested PGP (private) key with ID {requested_id}') - key_id = requested_id[-16:].upper() - value = _pgp_cache.get(key_id) + key_type = request.match_info['key_type'].lower() + if key_type == 'pgp': + _cache = _pgp_cache + key_id = requested_id[-16:].upper() + elif key_type == 'rsa': + _cache = _rsa_cache + key_id = requested_id + else: + return web.HTTPBadRequest() + LOG.debug(f'Requested {key_type.upper()} (private) key with ID {requested_id}') + value = dict(_cache.get(key_id)) if value: - return web.Response(body=value[3].hex()) + del value['public'] + return web.json_response(value) else: - LOG.warn(f"Requested PGP key {requested_id} not found.") + LOG.warn(f"Requested {key_type.upper()} (private) key {requested_id} not found.") return web.HTTPNotFound() -@routes.get('/retrieve/pgp/{requested_id}/public') -async def retrieve_pgp_key_public(request): +@routes.get('/retrieve/{key_type}/{requested_id}/public') +async def retrieve_key_public(request): """Retrieve public to reconstruced unlocked key.""" requested_id = request.match_info['requested_id'] - LOG.debug(f'Requested PGP (public) key with ID {requested_id}') - key_id = requested_id[-16:].upper() - value = _pgp_cache.get(key_id) - if value: - return web.Response(body=value[1].hex()) + key_type = request.match_info['key_type'].lower() + if key_type == 'pgp': + _cache = _pgp_cache + key_id = requested_id[-16:].upper() + elif key_type == 'rsa': + _cache = _rsa_cache + key_id = requested_id else: - LOG.warn(f"Requested PGP key {requested_id} not found.") - return web.HTTPNotFound() - - -@routes.get('/retrieve/rsa/{requested_id}') -async def retrieve_reencryt_key(request): - """Retrieve RSA reencryption key.""" - requested_id = request.match_info['requested_id'] - LOG.debug(f'Requested RSA key with ID {requested_id}') - value = _rsa_cache.get(requested_id) + return web.HTTPBadRequest() + LOG.debug(f'Requested {key_type.upper()} (public) key with ID {requested_id}') + value = dict(_cache.get(key_id)) if value: - return web.json_response({ 'id': requested_id, - 'public': value.hex()}) + del value['private'] + return web.json_response(value) else: - LOG.warn(f"Requested ReEncryption Key not found.") + LOG.warn(f"Requested {key_type.upper()} (public) key {requested_id} not found.") return web.HTTPNotFound() diff --git a/lega/openpgp/utils.py b/lega/openpgp/utils.py index be88efd5..2c2bde0a 100644 --- a/lega/openpgp/utils.py +++ b/lega/openpgp/utils.py @@ -348,11 +348,11 @@ def parse_private_key_material(raw_pub_algorithm, data, buf=None): elif raw_pub_algorithm == 17: # x x = get_mpi(data, buf=buf) - return x + return (x, ) elif raw_pub_algorithm in (16, 20): # x x = get_mpi(data, buf=buf) - return x + return (x, ) elif 100 <= raw_pub_algorithm <= 110: # Private/Experimental algorithms, just move on raise PGPError(f"Experimental private key part: {raw_pub_algorithm}") From 70fe600e03acb8fa03699d6d58297aeb8b9deda4 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 13 Mar 2018 13:24:45 +0200 Subject: [PATCH 4/6] documentation, healthcheck endpoint for keyserver and small fixes --- lega/keyserver.py | 88 ++++++++++++++++++++++++++++++++----------- lega/openpgp/utils.py | 4 +- 2 files changed, 69 insertions(+), 23 deletions(-) diff --git a/lega/keyserver.py b/lega/keyserver.py index e2781f67..f90ff0b6 100644 --- a/lega/keyserver.py +++ b/lega/keyserver.py @@ -5,19 +5,21 @@ --------- The Keyserver provides a REST endpoint for retrieving PGP and Re-encryption keys. -Active keys endpoint: +Active keys endpoint (current key types supported are PGP and RSA): -* ``/active/pgp`` - GET request for the active PGP key -* ``/active/rsa`` - GET request for the active RSA key for re-encryption -* ``/active/pgp/private`` - GET request for the private part of the active PGP key -* ``/active/pgp/public`` - GET request for the public part of the active PGP key +* ``/active/\{key_type\}`` - GET request for the active key +* ``/active/\{key_type\}/private`` - GET request for the private part of the active key +* ``/active/\{key_type\}/public`` - GET request for the public part of the active key Retrieve keys endpoint: -* ``/retrieve/pgp/\{key_id\}`` - GET request for the active PGP key with a known keyID of fingerprint -* ``/retrieve/rsa/\{key_id\}`` - GET request for the active RSA key for re-encryption with a known keyID -* ``/retrieve/pgp/\{key_id\}/private`` - GET request for the private part of the active PGP key with a known keyID of fingerprint -* ``/retrieve/pgp/\{key_id\}/public`` - GET request for the public part of the active PGP key with a known keyID of fingerprint +* ``/retrieve/\{key_type\}/\{key_id\}`` - GET request for the active PGP key with a known keyID of fingerprint +* ``/retrieve/\{key_type\}/\{key_id\}/private`` - GET request for the private part of the active PGP key with a known keyID of fingerprint +* ``/retrieve/\{key_type\}/\{key_id\}/public`` - GET request for the public part of the active PGP key with a known keyID of fingerprint + +Generate endpoint: + +* ``/generate/pgp`` - POST request to generate a PGP key pair Admin endpoint: @@ -193,7 +195,7 @@ async def activate_key(key_name, data): async def active_key(request): """Returns a JSON-formated list of numbers to reconstruct the active key, unlocked. - For PGP: + For PGP key types: * The JOSN response contains a "type" attribute to specify which key it is. * If type is "rsa", the public and private attributes contain ('n','e') and ('d','p','q','u') respectively. @@ -202,7 +204,7 @@ async def active_key(request): For RSA the public and private parts are retrieved in hex format. - Other key types are not supported + Other key types are not supported. """ key_type = request.match_info['key_type'].lower() if key_type == 'pgp': @@ -225,7 +227,19 @@ async def active_key(request): @routes.get('/active/{key_type}/private') async def active_key_private(request): - """Retrieve private part to reconstruced unlocked active key.""" + """Retrieve private part to reconstruced unlocked active key. + + For PGP key types: + + * The JOSN response contains a "type" attribute to specify which key it is. + * If type is "rsa", the private attribute contains ('d','p','q','u'). + * If type is "dsa", the private attribute contains ('x'). + * If type is "elg", the private attribute contains ('x'). + + For RSA the public and private parts are retrieved in hex format. + + Other key types are not supported. + """ key_type = request.match_info['key_type'].lower() if key_type == 'pgp': active_key = "active_pgp_key" @@ -248,7 +262,19 @@ async def active_key_private(request): @routes.get('/active/{key_type}/public') async def active_key_public(request): - """Retrieve public to reconstruced unlocked active key.""" + """Retrieve public to reconstruced unlocked active key. + + For PGP key types: + + * The JOSN response contains a "type" attribute to specify which key it is. + * If type is "rsa", the public attribute contains ('n','e'). + * If type is "dsa", the public attribute contains ('p','q','g','y'). + * If type is "elg", the public attribute contains ('p','g','y'). + + For RSA the public part is retrieved in hex format. + + Other key types are not supported. + """ key_type = request.match_info['key_type'].lower() if key_type == 'pgp': active_key = "active_pgp_key" @@ -275,11 +301,16 @@ async def active_key_public(request): async def retrieve_key(request): """Returns a JSON-formated list of numbers to reconstruct an unlocked key. - The JOSN response contains a "type" attribute to specify which key it is. - If type is "rsa", the public and private attributes contain ('n','e') and ('d','p','q','u') respectively. - If type is "dsa", the public and private attributes contain ('p','q','g','y') and ('x') respectively. - If type is "elg", the public and private attributes contain ('p','g','y') and ('x') respectively. - Other key types are not supported + For PGP key types: + + * The JOSN response contains a "type" attribute to specify which key it is. + * If type is "rsa", the public and private attributes contain ('n','e') and ('d','p','q','u') respectively. + * If type is "dsa", the public and private attributes contain ('p','q','g','y') and ('x') respectively. + * If type is "elg", the public and private attributes contain ('p','g','y') and ('x') respectively. + + For RSA the public and private parts are retrieved in hex format. + + Other key types are not supported. """ requested_id = request.match_info['requested_id'] key_type = request.match_info['key_type'].lower() @@ -302,7 +333,10 @@ async def retrieve_key(request): @routes.get('/retrieve/{key_type}/{requested_id}/private') async def retrieve_key_private(request): - """Retrieve private part to reconstruced unlocked key.""" + """Retrieve private part to reconstruct unlocked key. + + :py:func:`lega.keyserver.active_key_private` + """ requested_id = request.match_info['requested_id'] key_type = request.match_info['key_type'].lower() if key_type == 'pgp': @@ -325,7 +359,10 @@ async def retrieve_key_private(request): @routes.get('/retrieve/{key_type}/{requested_id}/public') async def retrieve_key_public(request): - """Retrieve public to reconstruced unlocked key.""" + """Retrieve public part to reconstruct unlocked key. + + :py:func:`lega.keyserver.active_key_private` + """ requested_id = request.match_info['requested_id'] key_type = request.match_info['key_type'].lower() if key_type == 'pgp': @@ -378,11 +415,20 @@ async def generate_pgp_key_pair(request): return web.HTTPBadRequest() +@routes.get('/health') +async def healthcheck(request): + """A health endpoint for service discovery. + It will always return ok. + """ + LOG.debug('Healthcheck called') + return web.HTTPOk() + + @routes.get('/admin/ttl') async def check_ttl(request): """Evict from the cache if TTL expired and return the keys that survived""" # ehh...why? /Fred - LOG.debug(f'Admin TTL') + LOG.debug('Admin TTL') pgp_expire = _pgp_cache.check_ttl() rsa_expire = _rsa_cache.check_ttl() if pgp_expire or rsa_expire: diff --git a/lega/openpgp/utils.py b/lega/openpgp/utils.py index 2c2bde0a..85608126 100644 --- a/lega/openpgp/utils.py +++ b/lega/openpgp/utils.py @@ -384,10 +384,10 @@ def pack_key_material(pub_stream, priv_stream): material_keys_priv = ('d','p','q','u') elif key_type == "dsa": material_keys_pub = ('p','q','g','y') - material_keys_priv = ('x') + material_keys_priv = ('x', ) elif key_type == "elg": material_keys_pub = ('p','g','y') - material_keys_priv = ('x') + material_keys_priv = ('x', ) else: raise PGPError(f'Cannot pack a "{key_material}" key material') From 1979b96207ec248137d5d3934755bb66927eac22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Haziza?= Date: Tue, 13 Mar 2018 16:06:24 +0100 Subject: [PATCH 5/6] Handling EGA file IDs --- deployments/docker/bootstrap/instance.sh | 8 +++- deployments/docker/bootstrap/settings/fin1 | 3 ++ deployments/docker/bootstrap/settings/swe1 | 3 ++ deployments/docker/images/Makefile | 2 +- extras/db.sql | 17 ++++++- extras/publish.py | 10 +++-- lega/conf/defaults.ini | 1 + lega/ingest.py | 5 ++- lega/keyserver.py | 52 ++++++++++++++++++---- lega/utils/db.py | 46 +++++++++++++++---- lega/verify.py | 4 +- requirements.txt | 3 +- setup.py | 1 + 13 files changed, 125 insertions(+), 30 deletions(-) diff --git a/deployments/docker/bootstrap/instance.sh b/deployments/docker/bootstrap/instance.sh index c4d2df16..9b52db51 100755 --- a/deployments/docker/bootstrap/instance.sh +++ b/deployments/docker/bootstrap/instance.sh @@ -46,9 +46,11 @@ chmod 644 ${PRIVATE}/${INSTANCE}/pgp/ega2.pub ######################################################################### echomsg "\t* the RSA public and private key" +#${OPENSSL} genpkey -algorithm RSA -pass pass:"${RSA_PASSPHRASE}" -out ${PRIVATE}/${INSTANCE}/rsa/ega.sec -pkeyopt rsa_keygen_bits:2048 ${OPENSSL} genpkey -algorithm RSA -out ${PRIVATE}/${INSTANCE}/rsa/ega.sec -pkeyopt rsa_keygen_bits:2048 ${OPENSSL} rsa -pubout -in ${PRIVATE}/${INSTANCE}/rsa/ega.sec -out ${PRIVATE}/${INSTANCE}/rsa/ega.pub +#${OPENSSL} genpkey -algorithm RSA -pass pass:"${RSA_PASSPHRASE}" -out ${PRIVATE}/${INSTANCE}/rsa/ega2.sec -pkeyopt rsa_keygen_bits:2048 ${OPENSSL} genpkey -algorithm RSA -out ${PRIVATE}/${INSTANCE}/rsa/ega2.sec -pkeyopt rsa_keygen_bits:2048 ${OPENSSL} rsa -pubout -in ${PRIVATE}/${INSTANCE}/rsa/ega2.sec -out ${PRIVATE}/${INSTANCE}/rsa/ega2.pub @@ -68,10 +70,12 @@ pgp : pgp.key.1 [rsa.key.1] public : /etc/ega/rsa/ega.pub private : /etc/ega/rsa/ega.sec +#passphrase : ${RSA_PASSPHRASE} [rsa.key.2] public : /etc/ega/rsa/ega2.pub private : /etc/ega/rsa/ega2.sec +#passphrase : ${RSA_PASSPHRASE} [pgp.key.1] public : /etc/ega/pgp/ega.pub @@ -448,6 +452,8 @@ services: tty: true expose: - "443" + ports: + - "${DOCKER_PORT_keyserver}:443" volumes: - ./${INSTANCE}/ega.conf:/etc/ega/conf.ini:ro - ./${INSTANCE}/logger.yml:/etc/ega/logger.yml:ro @@ -489,7 +495,7 @@ services: - ./${INSTANCE}/ega.conf:/etc/ega/conf.ini:ro - ./${INSTANCE}/logger.yml:/etc/ega/logger.yml:ro - ../images/vault/entrypoint.sh:/usr/local/bin/entrypoint.sh - # - ../../../lega:/root/.local/lib/python3.6/site-packages/lega + - ../../../lega:/root/.local/lib/python3.6/site-packages/lega restart: on-failure:3 networks: - lega_${INSTANCE} diff --git a/deployments/docker/bootstrap/settings/fin1 b/deployments/docker/bootstrap/settings/fin1 index 56e8ed7e..bce32956 100644 --- a/deployments/docker/bootstrap/settings/fin1 +++ b/deployments/docker/bootstrap/settings/fin1 @@ -4,6 +4,7 @@ set -e DOCKER_PORT_inbox=2223 DOCKER_PORT_mq=15673 DOCKER_PORT_kibana=5602 +DOCKER_PORT_keyserver=8444 LEGA_GREETINGS="Welcome to Local EGA Finland @ CSC" CEGA_MQ_PASSWORD=$(generate_password 16) @@ -20,4 +21,6 @@ PGP_COMMENT="@CSC" PGP_EMAIL="ega@csc.fi" PGP_PASSPHRASE=$(generate_password 16) +RSA_PASSPHRASE=$(generate_password 16) + LOG_LEVEL=INFO diff --git a/deployments/docker/bootstrap/settings/swe1 b/deployments/docker/bootstrap/settings/swe1 index 8c9d96ce..17d91b27 100644 --- a/deployments/docker/bootstrap/settings/swe1 +++ b/deployments/docker/bootstrap/settings/swe1 @@ -4,6 +4,7 @@ set -e DOCKER_PORT_inbox=2222 DOCKER_PORT_mq=15672 DOCKER_PORT_kibana=5601 +DOCKER_PORT_keyserver=8443 LEGA_GREETINGS="Welcome to Local EGA Sweden @ NBIS" CEGA_MQ_PASSWORD=$(generate_password 16) @@ -20,4 +21,6 @@ PGP_COMMENT="@NBIS" PGP_EMAIL="ega@nbis.se" PGP_PASSPHRASE=$(generate_password 16) +RSA_PASSPHRASE=$(generate_password 16) + LOG_LEVEL=DEBUG diff --git a/deployments/docker/images/Makefile b/deployments/docker/images/Makefile index e364099d..8df4b4f3 100644 --- a/deployments/docker/images/Makefile +++ b/deployments/docker/images/Makefile @@ -20,7 +20,7 @@ TARGET=nbisweden/ega all: base inbox -base: PIP_EGA_PACKAGES=pika==0.11.0 pycryptodomex==3.4.7 psycopg2==2.7.4 cryptography==2.1.3 aiohttp==2.3.8 aiohttp-jinja2==0.13.0 pgpy fusepy +base: PIP_EGA_PACKAGES=pika==0.11.0 pycryptodomex==3.4.7 psycopg2==2.7.4 cryptography==2.1.3 aiohttp==2.3.8 aiohttp-jinja2==0.13.0 pgpy fusepy aiopg==0.13.0 base inbox: docker build --build-arg checkout=$(CHECKOUT) \ --build-arg PIP_EGA_PACKAGES="$(PIP_EGA_PACKAGES)" \ diff --git a/extras/db.sql b/extras/db.sql index 0f18ae8d..eda659b6 100644 --- a/extras/db.sql +++ b/extras/db.sql @@ -21,6 +21,7 @@ CREATE TABLE files ( status status, staging_name TEXT, stable_id TEXT, + filepath TEXT, reenc_info TEXT, reenc_size INTEGER, reenc_checksum TEXT, -- sha256 @@ -30,19 +31,31 @@ CREATE TABLE files ( CREATE FUNCTION insert_file(filename files.filename%TYPE, eid files.elixir_id%TYPE, + stable_id files.stable_id%TYPE, status files.status%TYPE) RETURNS files.id%TYPE AS $insert_file$ #variable_conflict use_column DECLARE file_id files.id%TYPE; BEGIN - INSERT INTO files (filename,elixir_id,status) - VALUES(filename,eid,status) RETURNING files.id + INSERT INTO files (filename,elixir_id,stable_id,status) + VALUES(filename,eid,stable_id,status) RETURNING files.id INTO file_id; RETURN file_id; END; $insert_file$ LANGUAGE plpgsql; +CREATE FUNCTION translate_fileid_to_filepath(sid files.stable_id%TYPE) + RETURNS files.filepath%TYPE AS $translate_fileid_to_filepath$ + #variable_conflict use_column + DECLARE + filepath files.filepath%TYPE; + BEGIN + SELECT filepath FROM files WHERE stable_id = sid LIMIT 1 INTO filepath; + RETURN filepath; + END; +$translate_fileid_to_filepath$ LANGUAGE plpgsql; + -- ################################################## -- ERRORS -- ################################################## diff --git a/extras/publish.py b/extras/publish.py index d6521020..09806097 100644 --- a/extras/publish.py +++ b/extras/publish.py @@ -28,13 +28,17 @@ args = parser.parse_args() -message = { 'user': args.user, 'filepath': args.filepath } +stable_id = 'EGAF_'+str(uuid.uuid4()) + +print('Ingesting file',stable_id) + +message = { 'user': args.user, 'filepath': args.filepath, 'stable_id': stable_id } if args.enc: message['encrypted_integrity'] = { 'checksum': args.enc, 'algorithm': args.enc_algo, } if args.unenc: message['unencrypted_integrity'] = { 'checksum': args.unenc, 'algorithm': args.unenc_algo, } -print('Publishing:',message) +#print('Publishing:',message) parameters = pika.URLParameters(args.connection) connection = pika.BlockingConnection(parameters) @@ -44,4 +48,4 @@ properties=pika.BasicProperties(correlation_id=str(uuid.uuid4()), content_type='application/json',delivery_mode=2)) connection.close() -print('Message published') +print('Message published to CentralEGA') diff --git a/lega/conf/defaults.ini b/lega/conf/defaults.ini index 5ebfc3ab..363d0a40 100644 --- a/lega/conf/defaults.ini +++ b/lega/conf/defaults.ini @@ -44,3 +44,4 @@ ssl_certfile = /etc/ega/ssl.cert ssl_keyfile = /etc/ega/ssl.key host = 0.0.0.0 port = 443 +eureka_endpoint = https://eureka.eu/register/service diff --git a/lega/ingest.py b/lega/ingest.py index 0c17fe84..6faef612 100644 --- a/lega/ingest.py +++ b/lega/ingest.py @@ -52,13 +52,14 @@ def work(active_master_key, master_pubkey, data): ''' filepath = data['filepath'] - LOG.info(f"Processing {filepath}") + stable_id = data['stable_id'] + LOG.info(f"Processing {filepath} (with stable_id: {stable_id})") # Use user_id, and not elixir_id user_id = sanitize_user_id(data['user']) # Insert in database - file_id = db.insert_file(filepath, user_id) + file_id = db.insert_file(filepath, user_id, stable_id) # early record internal_data = { diff --git a/lega/keyserver.py b/lega/keyserver.py index f90ff0b6..14329d79 100644 --- a/lega/keyserver.py +++ b/lega/keyserver.py @@ -41,7 +41,7 @@ from .openpgp.utils import unarmor from .openpgp.packet import iter_packets from .conf import CONF, KeysConfiguration -from .utils import get_file_content +from .utils import get_file_content, db from .openpgp.generate import generate_pgp_key LOG = logging.getLogger('keyserver') @@ -436,19 +436,47 @@ async def check_ttl(request): else: return web.HTTPBadRequest() - -async def load_keys_conf(KEYS): +@routes.get('/temp/file/{file_id}') +async def translate_file_id_to_filepath(request): + """Translate a file_id to a file_path""" + file_id = request.match_info['file_id'] + LOG.debug(f'Translation {file_id} to filepath') + filepath = await db.get_filepath(request.app['db'], file_id) + LOG.debug(f'Filepath {filepath}') + if filepath: + return web.Response(text=filepath) + raise web.HTTPNotFound(text=f'Dunno anything about a file with id "{file_id}"\n') + +async def load_keys_conf(store): """Parse and load keys configuration.""" # Cache the active key names - for name, value in KEYS.defaults().items(): + for name, value in store.defaults().items(): if name == 'pgp': _pgp_cache.set('active_pgp_key', value) if name == 'rsa': _rsa_cache.set('active_rsa_key', value) # Load all the keys in the store - for section in KEYS.sections(): - await activate_key(section, dict(KEYS.items(section))) - + for section in store.sections(): + await activate_key(section, dict(store.items(section))) + +async def init(app): + '''Initialization running before the loop.run_forever''' + app['db'] = await db.create_pool(loop=app.loop) + LOG.info('DB Connection pool created') + # Note: will exit on failure + await load_keys_conf(app['store']) + +async def shutdown(app): + '''Function run after a KeyboardInterrupt. After that: cleanup''' + LOG.info('Shutting down the database engine') + app['db'].close() + await app['db'].wait_closed() + +async def cleanup(app): + '''Function run after a KeyboardInterrupt. Right after, the loop is closed''' + LOG.info('Cancelling all pending tasks') + for task in asyncio.Task.all_tasks(): + task.cancel() def main(args=None): """Where the magic happens.""" @@ -456,7 +484,6 @@ def main(args=None): args = sys.argv[1:] CONF.setup(args) - KEYS = KeysConfiguration(args) host = CONF.get('keyserver', 'host') # fallbacks are in defaults.ini port = CONF.getint('keyserver', 'port') @@ -476,7 +503,14 @@ def main(args=None): keyserver = web.Application(loop=loop) keyserver.router.add_routes(routes) - loop.run_until_complete(load_keys_conf(KEYS)) + # Adding the keystore to the server + keyserver['store'] = KeysConfiguration(args) + + # Registering some initialization and cleanup routines + LOG.info('Setting up callbacks') + keyserver.on_startup.append(init) + keyserver.on_shutdown.append(shutdown) + keyserver.on_cleanup.append(cleanup) LOG.info(f"Start keyserver on {host}:{port}") web.run_app(keyserver, host=host, port=port, shutdown_timeout=0, ssl_context=sslcontext) diff --git a/lega/utils/db.py b/lega/utils/db.py index 2be1952f..6aef8b3f 100644 --- a/lega/utils/db.py +++ b/lega/utils/db.py @@ -17,6 +17,7 @@ from socket import gethostname from time import sleep import asyncio +import aiopg from ..conf import CONF from .exceptions import FromUser @@ -126,13 +127,15 @@ def connect(): return psycopg2.connect(**db_args) -def insert_file(filename, user_id): +def insert_file(filename, user_id, stable_id): with connect() as conn: with conn.cursor() as cur: - cur.execute('SELECT insert_file(%(filename)s,%(user_id)s,%(status)s);',{ + cur.execute('SELECT insert_file(%(filename)s,%(user_id)s,%(stable_id)s, %(status)s);',{ 'filename': filename, 'user_id': user_id, - 'status' : Status.Received.value }) + 'status' : Status.Received.value, + 'stable_id': stable_id, + }) file_id = (cur.fetchone())[0] if file_id: LOG.debug(f'Created id {file_id} for {filename}') @@ -161,7 +164,7 @@ def set_error(file_id, error, from_user=False): def get_details(file_id): with connect() as conn: with conn.cursor() as cur: - query = 'SELECT filename, org_checksum, org_checksum_algo, stable_id, reenc_checksum from files WHERE id = %(file_id)s;' + query = 'SELECT filename, org_checksum, org_checksum_algo, filepath, stable_id, reenc_checksum from files WHERE id = %(file_id)s;' cur.execute(query, { 'file_id': file_id}) return cur.fetchone() @@ -191,16 +194,41 @@ def set_encryption(file_id, info, digest): cur.execute('UPDATE files SET reenc_info = %(reenc_info)s, reenc_checksum = %(digest)s, status = %(status)s WHERE id = %(file_id)s;', {'reenc_info': info, 'file_id': file_id, 'digest': digest, 'status': Status.Completed.value}) -def finalize_file(file_id, stable_id, filesize): +def finalize_file(file_id, filepath, filesize): assert file_id, 'Eh? No file_id?' - assert stable_id, 'Eh? No stable_id?' - LOG.debug(f'Setting final name for file_id {file_id}: {stable_id}') + assert filepath, 'Eh? No filepath?' + LOG.debug(f'Setting final name for file_id {file_id}: {filepath}') with connect() as conn: with conn.cursor() as cur: cur.execute('UPDATE files ' - 'SET status = %(status)s, stable_id = %(stable_id)s, reenc_size = %(filesize)s ' + 'SET status = %(status)s, filepath = %(filepath)s, reenc_size = %(filesize)s ' 'WHERE id = %(file_id)s;', - {'stable_id': stable_id, 'file_id': file_id, 'status': Status.Archived.value, 'filesize': filesize}) + {'filepath': filepath, 'file_id': file_id, 'status': Status.Archived.value, 'filesize': filesize}) + +###################################### +## Async code ## +###################################### + +@retry_loop(on_failure=_do_exit) +async def create_pool(loop): + '''\ + Async function to create a pool of connection to the database. + Used by the frontend. + ''' + db_args = fetch_args(CONF) + return await aiopg.create_pool(**db_args, loop=loop, echo=True) + +async def get_filepath(conn, file_id): + assert file_id, 'Eh? No file ID?' + try: + with (await conn.cursor()) as cur: + query = 'SELECT translate_fileid_to_filepath(%(file_id)s)' + #query = "SELECT filepath from files where stable_id = '%(file_id)s';" + await cur.execute(query, {'file_id': file_id}) + return (await cur.fetchone())[0] + except psycopg2.InternalError as pgerr: + LOG.debug(f'File Info for {file_id}: {pgerr!r}') + return None ###################################### ## Decorator ## diff --git a/lega/verify.py b/lega/verify.py index 2fb349ac..c8530101 100644 --- a/lega/verify.py +++ b/lega/verify.py @@ -30,12 +30,12 @@ def work(data): LOG.debug(f'Verifying message: {data}') file_id = data.pop('internal_data') # can raise KeyError - filename, _, org_hash_algo, vault_filename, vault_checksum = db.get_details(file_id) + filename, _, org_hash_algo, vault_filename, stable_id, vault_checksum = db.get_details(file_id) if not checksum.is_valid(vault_filename, vault_checksum, hashAlgo='sha256'): raise exceptions.VaultDecryption(vault_filename) - data['status'] = { 'state': 'COMPLETED', 'details': file_id } + data['status'] = { 'state': 'COMPLETED', 'details': stable_id } return data def main(args=None): diff --git a/requirements.txt b/requirements.txt index 308e66bb..ffe1755a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ pika==0.11.0 colorama==0.3.7 -aiohttp==2.3.8 +aiohttp==3.0.7 aiohttp-jinja2==0.13.0 fusepy sphinx_rtd_theme @@ -8,3 +8,4 @@ pycryptodomex==3.4.7 cryptography==2.1.3 pgpy psycopg2==2.7.4 +aiopg==0.13.0 diff --git a/setup.py b/setup.py index 6587e054..82eb5328 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ # 'pika==0.11.0', # 'colorama==0.3.7', # 'psycopg2==2.7.4', + # 'aiopg'==0.13.0, # 'aiohttp==2.3.8', # 'aiohttp-jinja2==0.13.0', # 'fusepy', From ce807ef7eba2a1becaddd6aa4bfc539ae35d66b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Haziza?= Date: Tue, 13 Mar 2018 16:20:39 +0100 Subject: [PATCH 6/6] Updating the testsuite with the stable_id --- tests/src/test/java/se/nbis/lega/cucumber/Utils.java | 1 + .../test/java/se/nbis/lega/cucumber/publisher/Message.java | 3 +++ .../src/test/java/se/nbis/lega/cucumber/steps/Ingestion.java | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/src/test/java/se/nbis/lega/cucumber/Utils.java b/tests/src/test/java/se/nbis/lega/cucumber/Utils.java index a9f5eec8..9f95a3f9 100644 --- a/tests/src/test/java/se/nbis/lega/cucumber/Utils.java +++ b/tests/src/test/java/se/nbis/lega/cucumber/Utils.java @@ -222,6 +222,7 @@ public void publishCEGA(String connection, String user, String filepath, String Message message = new Message(); message.setUser(user); message.setFilepath(filepath); + message.setStableID("EGAF_" + UUID.randomUUID().toString()); Checksum unencrypted = new Checksum(); unencrypted.setAlgorithm("md5"); diff --git a/tests/src/test/java/se/nbis/lega/cucumber/publisher/Message.java b/tests/src/test/java/se/nbis/lega/cucumber/publisher/Message.java index a2e9a1fb..64e1a328 100755 --- a/tests/src/test/java/se/nbis/lega/cucumber/publisher/Message.java +++ b/tests/src/test/java/se/nbis/lega/cucumber/publisher/Message.java @@ -14,6 +14,9 @@ public class Message { @JsonProperty("filepath") private String filepath; + @JsonProperty("stable_id") + private String stableID; + @JsonProperty("encrypted_integrity") private Checksum encryptedIntegrity; diff --git a/tests/src/test/java/se/nbis/lega/cucumber/steps/Ingestion.java b/tests/src/test/java/se/nbis/lega/cucumber/steps/Ingestion.java index 5a99967c..067e6adf 100644 --- a/tests/src/test/java/se/nbis/lega/cucumber/steps/Ingestion.java +++ b/tests/src/test/java/se/nbis/lega/cucumber/steps/Ingestion.java @@ -55,7 +55,7 @@ public Ingestion(Context context) { Then("^the file is ingested successfully$", () -> { try { String output = utils.executeDBQuery(context.getTargetInstance(), - String.format("select stable_id from files where filename = '%s'", context.getEncryptedFile().getName())); + String.format("select filepath from files where filename = '%s'", context.getEncryptedFile().getName())); String vaultFileName = output.split(System.getProperty("line.separator"))[2].trim(); String cat = utils.executeWithinContainer(utils.findContainer(utils.getProperty("images.name.vault"), utils.getProperty("container.prefix.vault") + context.getTargetInstance()), "cat", vaultFileName); @@ -69,7 +69,7 @@ public Ingestion(Context context) { Then("^ingestion failed$", () -> { try { String output = utils.executeDBQuery(context.getTargetInstance(), - String.format("select stable_id from files where filename = '%s'", context.getEncryptedFile().getName())); + String.format("select filepath from files where filename = '%s'", context.getEncryptedFile().getName())); String vaultFileName = output.split(System.getProperty("line.separator"))[2].trim(); Assertions.assertThat(vaultFileName).isEmpty(); } catch (IOException | InterruptedException e) {