Update docker-compose to make a full environment, with kms and auth

CHANGELOG.md

@@ -1,29 +1,8 @@
 # Changelog
 
-## 6.4.1
-
-* When the environment variable `ENABLE_SAVE_LAST_DECRYPTION_TIME` is set to true, this change will
-  save the archived credential in addition to the credential itself, which was saved in v6.4.0.
-  The archived credential will not be saved if the ID specified in the endpoint is already an
-  archived credential.
-
-## 6.4.0
-
-* This release adds support for 2 small features:
-
-  * Add a `metadata_only` param to `GET /v1/credentials/<ID>`. For instance, if the request is
-    `GET /v1/credentials/123?metadata_only=true`, the response will not contain the credential pairs.
-    `metadata_only` defaults to `false` so that it is backwards compatible. The purpose of this
-    is to give users finer controls when deciding whether to send back `credential_pairs`.
-  * Automatically update the `last_decrypted_date` on a credential when the `credential_pairs` are
-    sent back to the client. Sending `credential_pairs` to the client implies that a credential has been
-    decrypted and is likely to have been read by a human. This is also an OPT IN change.
-    An environment variable `ENABLE_SAVE_LAST_DECRYPTION_TIME` must be set to true in order to
-    update `last_decrypted_date`.
-
 ## 6.3.0
 
-* This release adds support for keeping track of when credentials should be rotated.
+* Added support for keeping track of when credentials should be rotated.
   Three new fields have been added to the Credential model:
 
   * tags: `tags` are a set of strings that can be used to categorize a credential. For instance
@@ -47,6 +26,24 @@
      that keys with this tag should be rotated. For instance, we could have a `ROTATION_DAYS_CONFIG` that
      looks something like '{"ADMIN_PRIV": 30, "FINANCIAL_DATA": 10}'
 
+* Add a `metadata_only` param to `GET /v1/credentials/<ID>`. For instance, if the request is
+  `GET /v1/credentials/123?metadata_only=true`, the response will not contain the credential pairs.
+  `metadata_only` defaults to `false` so that it is backwards compatible. The purpose of this
+  is to give users finer controls when deciding whether to send back `credential_pairs`.
+
+* Automatically update the `last_decrypted_date` on a credential when the `credential_pairs` are
+  sent back to the client. Sending `credential_pairs` to the client implies that a credential has been
+  decrypted and is likely to have been read by a human. This is also an OPT IN change.
+  An environment variable `ENABLE_SAVE_LAST_DECRYPTION_TIME` must be set to true in order to
+  update `last_decrypted_date`.
+
+* Added `config/gunicorn.conf` and `config/logging.conf` files, which can be used to enable structured
+  json logs for logging output.
+
+* Updated the docker-compose setup to have a fully functional production-like environment, with
+  local dynamodb, local kms, and a local simplesamlphp IDP. The developer environment also has a
+  configuration for the PKI, which will generate self-signed certificates.
+
 ## 6.2.0
 
 * This release fixes a python3 stacktrace in SAML auth, when using the `SAML_SP_KEY_FILE` setting.

VERSION

@@ -1 +1 @@
-6.4.1
+6.3.0

confidant/authnz/__init__.py

@@ -35,6 +35,7 @@ def _get_validator():
             scoped_auth_keys=settings.SCOPED_AUTH_KEYS,
             token_cache_size=settings.KMS_AUTH_TOKEN_CACHE_SIZE,
             stats=stats,
+            endpoint_url=settings.KMS_URL,
         )
     return _VALIDATOR
 

confidant/clients/__init__.py

@@ -15,7 +15,8 @@ def get_boto_client(
         aws_access_key_id=None,
         aws_secret_access_key=None,
         aws_session_token=None,
-        config=None
+        config=None,
+        endpoint_url=None,
         ):
     """Get a boto3 client connection."""
     if config is None:
@@ -41,7 +42,8 @@ def get_boto_client(
 
     CLIENT_CACHE[cache_key] = session.client(
         client,
-        config=config.get('config')
+        config=config.get('config'),
+        endpoint_url=endpoint_url,
     )
     return CLIENT_CACHE[cache_key]
 
@@ -52,7 +54,8 @@ def get_boto_resource(
         aws_access_key_id=None,
         aws_secret_access_key=None,
         aws_session_token=None,
-        config=None
+        config=None,
+        endpoint_url=None,
         ):
     """Get a boto resource connection."""
     if config is None:
@@ -78,7 +81,8 @@ def get_boto_resource(
 
     RESOURCE_CACHE[cache_key] = session.resource(
         resource,
-        config=config.get('config')
+        config=config.get('config'),
+        endpoint_url=endpoint_url,
     )
     return RESOURCE_CACHE[cache_key]
 

confidant/services/keymanager.py

@@ -10,35 +10,47 @@
 from confidant.lib import cryptolib
 
 logger = logging.getLogger(__name__)
-config = botocore.config.Config(
-    connect_timeout=settings.KMS_CONNECTION_TIMEOUT,
-    read_timeout=settings.KMS_READ_TIMEOUT,
-    max_pool_connections=settings.KMS_MAX_POOL_CONNECTIONS
-)
-auth_kms_client = confidant.clients.get_boto_client(
-    'kms',
-    config={'name': 'keymanager', 'config': config}
-)
-at_rest_kms_client = confidant.clients.get_boto_client(
-    'kms',
-    config={'name': 'keymanager', 'config': config}
-)
-iam_resource = confidant.clients.get_boto_resource('iam')
-
-DATAKEYS = {}
-KEY_METADATA = {}
+
+_DATAKEYS = {}
+_KEY_METADATA = {}
+
+
+def _get_boto_config():
+    return botocore.config.Config(
+        connect_timeout=settings.KMS_CONNECTION_TIMEOUT,
+        read_timeout=settings.KMS_READ_TIMEOUT,
+        max_pool_connections=settings.KMS_MAX_POOL_CONNECTIONS,
+    )
+
+
+def _get_auth_kms_client():
+    return confidant.clients.get_boto_client(
+        'kms',
+        config={'name': 'keymanager_auth', 'config': _get_boto_config()},
+        endpoint_url=settings.KMS_URL,
+    )
+
+
+def _get_at_rest_kms_client():
+    return confidant.clients.get_boto_client(
+        'kms',
+        config={'name': 'keymanager_at_rest', 'config': _get_boto_config()},
+        endpoint_url=settings.KMS_URL,
+    )
 
 
 def get_key_id(key_alias):
-    if key_alias not in KEY_METADATA:
-        KEY_METADATA[key_alias] = auth_kms_client.describe_key(KeyId=key_alias)
-    return KEY_METADATA[key_alias]['KeyMetadata']['KeyId']
+    auth_kms_client = _get_auth_kms_client()
+    if key_alias not in _KEY_METADATA:
+        _KEY_METADATA[key_alias] = auth_kms_client.describe_key(KeyId=key_alias)
+    return _KEY_METADATA[key_alias]['KeyMetadata']['KeyId']
 
 
 def create_datakey(encryption_context):
     '''
     Create a datakey from KMS.
     '''
+    at_rest_kms_client = _get_at_rest_kms_client()
     # Disabled encryption is dangerous, so we don't use falsiness here.
     if settings.USE_ENCRYPTION is False:
         logger.warning(
@@ -60,6 +72,7 @@ def decrypt_datakey(data_key, encryption_context=None):
     '''
     Decrypt a datakey.
     '''
+    at_rest_kms_client = _get_at_rest_kms_client()
     # Disabled encryption is dangerous, so we don't use falsiness here.
     if settings.USE_ENCRYPTION is False:
         logger.warning(
@@ -69,18 +82,19 @@ def decrypt_datakey(data_key, encryption_context=None):
         )
         return cryptolib.decrypt_mock_datakey(data_key)
     sha = hashlib.sha256(data_key).hexdigest()
-    if sha not in DATAKEYS:
+    if sha not in _DATAKEYS:
         stats.incr('at_rest_action')
         plaintext = cryptolib.decrypt_datakey(
             data_key,
             encryption_context,
             client=at_rest_kms_client
         )
-        DATAKEYS[sha] = plaintext
-    return DATAKEYS[sha]
+        _DATAKEYS[sha] = plaintext
+    return _DATAKEYS[sha]
 
 
 def get_grants():
+    auth_kms_client = _get_auth_kms_client()
     _grants = []
     next_marker = None
     while True:
@@ -110,6 +124,7 @@ def ensure_grants(service_name):
     TODO: We should probably orchestrate this, rather than doing it in
           confidant.
     '''
+    iam_resource = confidant.clients.get_boto_resource('iam')
     if not settings.KMS_AUTH_MANAGE_GRANTS:
         return
     try:
@@ -125,6 +140,7 @@ def ensure_grants(service_name):
 
 
 def grants_exist(service_name):
+    iam_resource = confidant.clients.get_boto_resource('iam')
     try:
         role = iam_resource.Role(name=service_name)
         role.load()
@@ -174,6 +190,7 @@ def _grants_exist(role, grants):
 
 
 def _ensure_grants(role, grants):
+    auth_kms_client = _get_auth_kms_client()
     encrypt_constraint = {
         'EncryptionContextSubset': {
             'from': role.role_name

confidant/settings.py

@@ -258,6 +258,8 @@ def str_env(var_name, default=''):
 # equivalent to the number of tokens you expect to generate within the lifetime
 # of your tokens.
 KMS_AUTH_TOKEN_CACHE_SIZE = int_env('KMS_AUTH_TOKEN_CACHE_SIZE', 4096)
+# A custom endpoint url for KMS, for use in development
+KMS_URL = str_env('KMS_URL', None)
 
 # SSL redirection and HSTS
 

config/development/confidant.env

@@ -0,0 +1,49 @@
+AWS_ACCESS_KEY_ID=1
+AWS_SECRET_ACCESS_KEY=1
+AWS_DEFAULT_REGION=us-east-1
+GEVENT_RESOLVER=ares
+DEBUG=true
+HOST=0.0.0.0
+PORT=80
+USER_AUTH_MODULE=saml
+SAML_CONFIDANT_URL_ROOT=http://localhost
+SAML_DEBUG=true
+SAML_IDP_ENTITY_ID=http://localhost:8080/simplesaml/saml2/idp/metadata.php
+SAML_IDP_SIGNON_URL=http://localhost:8080/simplesaml/saml2/idp/SSOService.php
+SAML_IDP_LOGOUT_URL=http://localhost:8080/simplesaml/saml2/idp/SingleLogoutService.php
+SAML_SECURITY_SLO_RESP_SIGNED=false
+SAML_SECURITY_MESSAGES_SIGNED=false
+SAML_SECURITY_ASSERTIONS_SIGNED=false
+SAML_WANT_ATTRIBUTE_STATEMENT=false
+SAML_IDP_CERT_FILE=/etc/confidant/idp.crt
+AUTH_CONTEXT=confidant-development
+AUTH_KEY=alias/auth
+USER_AUTH_KEY=alias/auth
+KMS_AUTH_USER_TYPES=service
+KMS_AUTH_MANAGE_GRANTS=false
+KMS_URL=http://kms:8080
+SSLIFY=false
+# this is a pre-generated string, meant to be used in dev only.
+# normally this would be set through the SECRETS_BOOTSTRAP
+SESSION_SECRET=6eeJKEyFYx863mELaYBULYxW0XuBZqtLThdJRA8st81e9xju9R
+DYNAMODB_CREATE_TABLE=true
+DYNAMODB_URL=http://dynamodb:8080
+DYNAMODB_TABLE=confidant-development
+DYNAMODB_TABLE_ARCHIVE=confidant-development-archive
+DYNAMODB_CREATE_TABLE=true
+HISTORY_PAGE_LIMIT=20
+KMS_MASTER_KEY=alias/atrest
+MAINTENANCE_MODE_TOUCH_FILE=/config/maintenance
+BACKGROUND_CACHE_IAM_ROLES=false
+ACM_PRIVATE_CAS=development
+ACM_PRIVATE_CA_SELF_SIGN_DEVELOPMENT=true
+ACM_PRIVATE_CA_CSR_COUNTRY_NAME_DEVELOPMENT=US
+ACM_PRIVATE_CA_CSR_STATE_OR_PROVINCE_NAME_DEVELOPMENT=California
+ACM_PRIVATE_CA_CSR_LOCALITY_NAME_DEVELOPMENT=San Francisco
+ACM_PRIVATE_CA_CSR_ORGANIZATION_NAME_DEVELOPMENT=Example Inc.
+ACM_PRIVATE_CA_CERTIFICATE_USE_CACHE_DEVELOPMENT=true
+# example match: example-development
+ACM_PRIVATE_CA_DOMAIN_REGEX_DEVELOPMENT=(?P<service_name>[\w]+)\-development
+MAXIMUM_ROTATION_DAYS=10
+TAGS_EXCLUDING_ROTATION=["ROTATION_EXCLUDED"]
+ROTATION_DAYS_CONFIG={"FINANCIALLY_SENSITIVE": 1}

config/development/dynamodb.env

@@ -0,0 +1,3 @@
+AWS_ACCESS_KEY_ID=1
+AWS_SECRET_ACCESS_KEY=1
+AWS_REGION=us-east-1

config/development/idp.crt

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDXTCCAkWgAwIBAgIJALmVVuDWu4NYMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV
+BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
+aWRnaXRzIFB0eSBMdGQwHhcNMTYxMjMxMTQzNDQ3WhcNNDgwNjI1MTQzNDQ3WjBF
+MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50
+ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAzUCFozgNb1h1M0jzNRSCjhOBnR+uVbVpaWfXYIR+AhWDdEe5ryY+Cgav
+Og8bfLybyzFdehlYdDRgkedEB/GjG8aJw06l0qF4jDOAw0kEygWCu2mcH7XOxRt+
+YAH3TVHa/Hu1W3WjzkobqqqLQ8gkKWWM27fOgAZ6GieaJBN6VBSMMcPey3HWLBmc
++TYJmv1dbaO2jHhKh8pfKw0W12VM8P1PIO8gv4Phu/uuJYieBWKixBEyy0lHjyix
+YFCR12xdh4CA47q958ZRGnnDUGFVE1QhgRacJCOZ9bd5t9mr8KLaVBYTCJo5ERE8
+jymab5dPqe5qKfJsCZiqWglbjUo9twIDAQABo1AwTjAdBgNVHQ4EFgQUxpuwcs/C
+YQOyui+r1G+3KxBNhxkwHwYDVR0jBBgwFoAUxpuwcs/CYQOyui+r1G+3KxBNhxkw
+DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAAiWUKs/2x/viNCKi3Y6b
+lEuCtAGhzOOZ9EjrvJ8+COH3Rag3tVBWrcBZ3/uhhPq5gy9lqw4OkvEws99/5jFs
+X1FJ6MKBgqfuy7yh5s1YfM0ANHYczMmYpZeAcQf2CGAaVfwTTfSlzNLsF2lW/ly7
+yapFzlYSJLGoVE+OHEu8g5SlNACUEfkXw+5Eghh+KzlIN7R6Q7r2ixWNFBC/jWf7
+NKUfJyX8qIG5md1YUeT6GBW9Bm2/1/RiO24JTaYlfLdKK9TYb8sG5B+OLab2DImG
+99CJ25RkAcSobWNF5zD0O6lgOo3cEdB/ksCq3hmtlC/DlLZ/D8CJ+7VuZnS1rR2n
+aQ==
+-----END CERTIFICATE-----

config/development/kms-seed.yaml

@@ -0,0 +1,18 @@
+Keys:
+  - Metadata:
+      KeyId: bc436485-5092-42b8-92a3-0aa8b93536dc
+    BackingKeys:
+      - 34743777217A25432A46294A404E635266556A586E3272357538782F413F4428
+
+  - Metadata:
+      KeyId: 49c5492b-b1bc-42a8-9a5c-b2015e810c1c
+    BackingKeys:
+      - 5cdaead27fe7da2de47945d73cd6d79e36494e73802f3cd3869f1d2cb0b5d7a9
+
+
+Aliases:
+  - AliasName: alias/atrest
+    TargetKeyId: bc436485-5092-42b8-92a3-0aa8b93536dc
+
+  - AliasName: alias/auth
+    TargetKeyId: 49c5492b-b1bc-42a8-9a5c-b2015e810c1c

config/development/kms.env

@@ -0,0 +1 @@
+KMS_REGION=us-east-1