explorer rdev (#144)

.dockerignore

@@ -1,5 +1,4 @@
 bin
-client
 dist
 docs
-server
+/server/eb/customize

docker-compose.yml

@@ -0,0 +1,67 @@
+version: "3.8"
+
+services:
+
+  localstack:
+    image: localstack/localstack@sha256:7c6635493185d25165979995fb073fd789c72b6d8b17ef3a70b798d55576732f
+    ports:
+      - "4566:4566"
+    environment:
+      - HOSTNAME_EXTERNAL=localstack
+      - SERVICES=s3,secretsmanager,stepfunctions,iam,lambda
+      - DEBUG=1
+      - DATA_DIR=/tmp/localstack/data
+      - PORT_WEB_UI=${PORT_WEB_UI- }
+      - HOST_TMP_FOLDER=${TMPDIR}
+      - DEFAULT_REGION=us-west-2
+    volumes:
+      - localstack:/tmp/localstack
+    networks:
+       corporanet:
+          aliases:
+            - localstack.corporanet.local
+
+  explorer:
+    image: "${DOCKER_REPO}corpora-explorer"
+    build:
+      context: .
+      dockerfile: hosted/Dockerfile
+      cache_from:
+        - "${DOCKER_REPO}corpora-explorer:branch-main"
+      args:
+        - BUILDKIT_INLINE_CACHE=1
+        - HAPPY_COMMIT
+        - HAPPY_BRANCH
+        - HAPPY_TAG
+        - HAPPY_ENV
+    restart: always
+    depends_on:
+      - localstack
+    ports:
+      - "6000:6000"
+    environment:
+      - PYTHONUNBUFFERED=1
+      - CORPORA_LOCAL_DEV=true
+      - AWS_REGION=us-west-2
+      - AWS_DEFAULT_REGION=us-west-2
+      - AWS_ACCESS_KEY_ID=test
+      - AWS_SECRET_ACCESS_KEY=test
+      - BOTO_ENDPOINT_URL=http://localstack:4566
+      - DEPLOYMENT_STAGE=test
+      - RESTART_ON_FAILURE=yes
+      - DEV_MODE_COOKIES=1
+      - IS_DOCKER_DEV=yes # This skips some unit tests.
+    volumes:
+      # Mount dirs in a layout suitable for running tests.
+      - .:/corpora-data-portal
+      # SecretsManager population relies on oauth json
+      - ./oauth/users.json:/oauth/users.json
+    networks:
+       corporanet:
+          aliases:
+            - backend.corporanet.local
+networks:
+  corporanet:
+volumes:
+  database:
+  localstack:

hosted/Dockerfile

@@ -0,0 +1,37 @@
+FROM ubuntu:focal
+
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+ARG HAPPY_ENV
+ENV HAPPY_ENV=$HAPPY_ENV
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+ADD hosted/requirements.txt .
+ADD hosted/create_rdev_config.py .
+
+ADD server/ server/
+ADD client/ client/
+ADD Makefile Makefile
+ADD common.mk common.mk
+ADD environment.default.json environment.default.json
+
+RUN apt-get update && \
+    apt-get install -y gettext moreutils build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp llvm jq npm git zip && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1
+
+RUN python3 -m pip install psycopg2-binary
+RUN python3 -m pip install -r requirements.txt
+
+RUN python3 create_rdev_config.py
+
+RUN make build-client
+
+WORKDIR server/eb
+RUN make build-ecs
+
+ENV CXG_CONFIG_FILE=/config.yaml
+ENV FLASK_ENV=development
+CMD ["flask", "run", "--host", "0.0.0.0", "--port", "5000"]

hosted/create_rdev_config.py

@@ -0,0 +1,146 @@
+import os
+env = os.getenv("HAPPY_ENV")
+
+assert(env is not None, "No HAPPY_ENV specified")
+
+config = f"""
+server:
+  app:
+    verbose: true
+    debug: true
+    host: 0.0.0.0
+    port: null
+    open_browser: false
+    force_https: false
+    flask_secret_key: "asd"
+    generate_cache_control_headers: true
+    server_timing_headers: true
+    csp_directives:
+      img-src:
+        - https://www.google-analytics.com
+      script-src:
+        - https://www.google-analytics.com
+        - https://ssl.google-analytics.com
+        - browser.sentry-cdn.com
+      connect-src:
+        - https://www.google-analytics.com
+        - sentry.prod.si.czi.technology
+
+    api_base_url: https://{env}-explorer.rdev.single-cell.czi.technology/cellxgene/
+    web_base_url: https://{env}-explorer.rdev.single-cell.czi.technology/
+
+  multi_dataset:
+    dataroot:
+      covid19:
+        base_url: d
+        dataroot: s3://env-rdev-cellxgene/{env}
+      corpora_data_portal:
+        base_url: e
+        dataroot: s3://env-rdev-cellxgene/{env}
+
+    # The index page when in multi-dataset mode:
+    #   false or null:  this returns a 404 code
+    #   true:  loads a test index page, which links to the datasets that are available in the dataroot
+    #   string/URL:  redirect to this URL:  flask.redirect(config.multi_dataset__index)
+    index: https://{env}-explorer.rdev.single-cell.czi.technology/
+
+    # A list of allowed matrix types.  If an empty list, then all matrix types are allowed
+    allowed_matrix_types: []
+
+    matrix_cache:
+      # The maximum number of datasets that may be opened at one time.  The least recently used dataset
+      # is evicted from the cache first.
+      max_datasets: 100
+
+      # A matrix is automatically removed from the cache after timelimit_s number of seconds.
+      # If timelimit_s is set to None, then there is no time limit.
+      timelimit_s: 30
+
+    metadata_cache:
+      # The maximum number of datasets that may be opened at one time.  The least recently used dataset
+      # is evicted from the cache first.
+      max_datasets: 10000
+      # A matrix is automatically removed from the cache after timelimit_s number of seconds.
+      # If timelimit_s is set to None, then there is no time limit.
+      timelimit_s: 300
+
+  diffexp:
+    alg_cxg:
+      # The number of threads to use is computed from: min(max_workers, cpu_multipler * cpu_count).
+      # Where cpu_count is determined at runtime.
+      max_workers: 64
+      cpu_multiplier: 4
+
+      # The target number of matrix elements that are evaluated
+      # together in one thread.
+      target_workunit: 16_000_000
+
+  data_locator:
+    api_base: http://{env}-backend.internal.rdev.single-cell.czi.technology/dp/v1
+    s3:
+      # s3 region name.
+      #   if true, then the s3 location is automatically determined from the datapath or dataroot.
+      #   if false/null, then do not set.
+      #   if a string, then use that value (e.g. us-east-1).
+      region_name: true
+
+  adaptor:
+    cxg_adaptor:
+      # The key/values under tiledb_ctx will be used to initialize the tiledb Context.
+      # If 'vfs.s3.region' is not set, then it will automatically use the setting from
+      # data_locator / s3 / region_name.
+      tiledb_ctx:
+        sm.tile_cache_size: 60129542144 # 56 GB
+        sm.num_reader_threads: 32
+
+    anndata_adaptor:
+      backed: false
+
+  limits:
+    column_request_max: 32
+    diffexp_cellcount_max: 50000
+
+dataset:
+  app:
+    # Scripts can be a list of either file names (string) or dicts containing keys src, integrity and crossorigin.
+    # these will be injected into the index template as script tags with these attributes set.
+    scripts:
+      - src: "https://browser.sentry-cdn.com/5.15.5/bundle.min.js"
+        integrity: "sha384-wF7Jc4ZlWVxe/L8Ji3hOIBeTgo/HwFuaeEfjGmS3EXAG7Y+7Kjjr91gJpJtr+PAT"
+        crossorigin: "anonymous"
+
+    # Inline scripts are a list of file names, where the contents of the file will be injected into the index.
+    # inline_scripts:
+    #  - rdev-ga.js
+    #  - staging-sentry.js
+
+    about_legal_tos: /tos/
+    about_legal_privacy: /privacy/
+
+
+  presentation:
+    max_categories: 1000
+    custom_colors: true
+
+  user_annotations:
+    enable: false
+    type: hosted_tiledb_array
+    hosted_tiledb_array:
+      db_uri: null
+      hosted_file_directory: s3://hosted-cellxgene-dev/user_annotations/
+    local_file_csv:
+      directory: null
+      file: null
+
+  embeddings:
+    names: []
+
+  diffexp:
+    enable: true
+    lfc_cutoff: 0.01
+    top_n: 10
+"""
+
+if __name__ == "__main__":
+    with open("config.yaml", "w") as f:
+        f.write(config)

hosted/requirements.txt

@@ -0,0 +1,52 @@
+Authlib==0.14.3
+aniso8601==8.0.0
+anndata==0.7.6 # we use to_memory(), added in 0.7.6
+boto3==1.13.2
+botocore==1.16.2
+Brotli==1.0.7
+certifi==2020.4.5.1
+chardet==3.0.4
+click==7.1.2
+docutils==0.15.2
+Flask==1.1.4
+Flask-Compress==1.10.1
+Flask-Cors==3.0.10
+Flask-RESTful==0.3.9
+flask-server-timing==0.1.2
+flask-talisman==0.7.0
+flatbuffers==1.12
+flatten-dict==0.3.0
+fsspec==0.7.3
+gunicorn==20.0.4
+h5py==3.3.0
+idna==2.9
+importlib-metadata==1.6.0
+itsdangerous==1.1.0
+Jinja2==2.11.2
+jmespath==0.9.5
+MarkupSafe==1.1.1
+natsort==7.0.1
+numba==0.53.1
+numpy==1.21.1
+packaging==20.3
+pandas==1.3.0
+pathlib2==2.3.5
+# psycopg2==2.9.1
+pyparsing==2.4.7
+python-dateutil==2.8.1
+python-jose==3.1.0
+pytz==2020.1
+PyYAML==5.4.1
+requests==2.23.0
+s3fs==0.4.2
+s3transfer==0.3.3
+scipy==1.4.1
+scanpy==1.8.1
+umap-learn==0.4.6
+sentry-sdk[flask]==0.14.3
+six==1.14.0
+sqlalchemy==1.3.18
+tiledb==0.10.1
+urllib3==1.25.9
+Werkzeug==1.0.1
+zipp==3.1.0