diff --git a/.github/workflows/check_ci.yaml b/.github/workflows/check_ci.yaml index d74bea25..a42bd5f5 100644 --- a/.github/workflows/check_ci.yaml +++ b/.github/workflows/check_ci.yaml @@ -18,8 +18,11 @@ jobs: uses: ./.github/actions/prepare_poetry_env - name: Run pytest - run: poetry run pytest test/ci/test_install_dependencies.py + run: > + poetry run pytest + test/unit + test/integration/test_create_dss_docker_image.py env: # Set the secret as an env variable - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} diff --git a/doc/changes/changes_0.1.0.md b/doc/changes/changes_0.1.0.md index 1eb59701..c1a707b0 100644 --- a/doc/changes/changes_0.1.0.md +++ b/doc/changes/changes_0.1.0.md @@ -14,6 +14,7 @@ Version: 0.1.0 - #11: Created a notebook to show training with scikit-learn in the notebook - #15: Installed exasol-notebook-connector via ansible + - #30: Added script to build the Data Science Sandbox as Docker Image ## Bug Fixes diff --git a/doc/developer_guide/developer_guide.md b/doc/developer_guide/developer_guide.md index 012c5e3b..61c5032d 100644 --- a/doc/developer_guide/developer_guide.md +++ b/doc/developer_guide/developer_guide.md @@ -39,11 +39,11 @@ A CLI command has normally a respective function in the `lib` submodule. Hence, There are generally three types of commands: -| Type | Explanation | -| ----- | --------- | -| Release Commands | used during the release | -| Deployment Commands | used to deploy infrastructure onto AWS cloud | -| Development Commands | used to identify problems or for testing | +| Type | Explanation | +|----------------------|----------------------------------------------| +| Release Commands | used during the release | +| Deployment Commands | used to deploy infrastructure onto AWS cloud | +| Development Commands | used to identify problems or for testing | ### Release commands @@ -71,6 +71,7 @@ The following commands can be used to deploy the infrastructure onto a given AWS - `setup-vm-bucket` - deploys the AWS Bucket cloudformation stack which will be used to deploy the VM images - `setup-release-codebuild` - deploys the AWS Codebuild cloudformation stack which will be used for the release-build - `setup-vm-bucket-waf` - deploys the AWS Codebuild cloudformation stack which contains the WAF Acl configuration for the Cloudfront distribution of the VM Bucket +- `create-docker-image` - creates a Docker image for data-science-sandbox and deploys it to hub.docker.com/exasol/data-science-sandbox ## Flow diff --git a/exasol/ds/sandbox/cli/commands/__init__.py b/exasol/ds/sandbox/cli/commands/__init__.py index 02efc9f2..7785f62f 100644 --- a/exasol/ds/sandbox/cli/commands/__init__.py +++ b/exasol/ds/sandbox/cli/commands/__init__.py @@ -13,3 +13,4 @@ from .update_release import update_release from .make_ami_public import make_ami_public from .setup_vm_bucket_waf import setup_vm_bucket_waf +from .create_docker_image import create_docker_image diff --git a/exasol/ds/sandbox/cli/commands/create_docker_image.py b/exasol/ds/sandbox/cli/commands/create_docker_image.py new file mode 100644 index 00000000..16ffb587 --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/create_docker_image.py @@ -0,0 +1,44 @@ +import click + +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.lib.dss_docker import DssDockerImage +from exasol.ds.sandbox.lib.logging import SUPPORTED_LOG_LEVELS +from exasol.ds.sandbox.lib.logging import set_log_level + + +@cli.command() +@add_options([ + click.option( + '--repository', type=str, metavar="ORG/REPO", show_default=True, + default="exasol/data-science-sandbox", + help="Organization and repository on hub.docker.com to publish the docker image to"), + click.option('--version', type=str, help="Docker image version tag"), + click.option( + '--publish', type=bool, is_flag=True, + help="Whether to publish the created Docker image"), + click.option( + '--keep-container', type=bool, is_flag=True, + help="""Keep the Docker Container running after creating the image. + Otherwise stop and remove the container."""), +]) +@add_options(logging_options) +def create_docker_image( + repository: str, + version: str, + publish: bool, + keep_container: bool, + log_level: str, +): + """ + Create a Docker image for data-science-sandbox and deploy + it to a Docker repository. + """ + set_log_level(log_level) + DssDockerImage( + repository=repository, + version=version, + publish=publish, + keep_container=keep_container, + ).create() diff --git a/exasol/ds/sandbox/lib/ansible/ansible_access.py b/exasol/ds/sandbox/lib/ansible/ansible_access.py index 9a3770e0..f653c7db 100644 --- a/exasol/ds/sandbox/lib/ansible/ansible_access.py +++ b/exasol/ds/sandbox/lib/ansible/ansible_access.py @@ -1,8 +1,10 @@ -from typing import Callable - import ansible_runner +import logging + +from typing import Callable from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType class AnsibleException(RuntimeError): @@ -10,15 +12,16 @@ class AnsibleException(RuntimeError): class AnsibleAccess: - """ Provides access to ansible runner. @raises: AnsibleException if ansible execution fails """ @staticmethod def run(private_data_dir: str, run_ctx: AnsibleRunContext, printer: Callable[[str], None]): + quiet = not get_status_logger(LogType.ANSIBLE).isEnabledFor(logging.INFO) r = ansible_runner.run(private_data_dir=private_data_dir, playbook=run_ctx.playbook, + quiet=quiet, extravars=run_ctx.extra_vars) for e in r.events: printer(e) diff --git a/exasol/ds/sandbox/lib/ansible/ansible_runner.py b/exasol/ds/sandbox/lib/ansible/ansible_runner.py index dd9c53dc..a1e492ee 100644 --- a/exasol/ds/sandbox/lib/ansible/ansible_runner.py +++ b/exasol/ds/sandbox/lib/ansible/ansible_runner.py @@ -20,7 +20,7 @@ def __init__(self, ansible_access: AnsibleAccess, work_dir: Path): @staticmethod def printer(msg: str): - LOG.info(msg) + LOG.debug(msg) def run(self, ansible_run_context: AnsibleRunContext, host_infos: Tuple[HostInfo]): inventory_content = render_template("inventory.jinja", host_infos=host_infos) diff --git a/exasol/ds/sandbox/lib/dss_docker/Dockerfile b/exasol/ds/sandbox/lib/dss_docker/Dockerfile new file mode 100644 index 00000000..0d6d7245 --- /dev/null +++ b/exasol/ds/sandbox/lib/dss_docker/Dockerfile @@ -0,0 +1,4 @@ +FROM ubuntu:20.04 +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install --no-install-recommends --assume-yes python3 python3-pexpect +EXPOSE 8888/tcp diff --git a/exasol/ds/sandbox/lib/dss_docker/__init__.py b/exasol/ds/sandbox/lib/dss_docker/__init__.py new file mode 100644 index 00000000..f62d46be --- /dev/null +++ b/exasol/ds/sandbox/lib/dss_docker/__init__.py @@ -0,0 +1 @@ +from .create_image import DssDockerImage diff --git a/exasol/ds/sandbox/lib/dss_docker/create_image.py b/exasol/ds/sandbox/lib/dss_docker/create_image.py new file mode 100644 index 00000000..aba083fb --- /dev/null +++ b/exasol/ds/sandbox/lib/dss_docker/create_image.py @@ -0,0 +1,103 @@ +import docker +import humanfriendly +import importlib_resources + +from datetime import datetime +from docker.types import Mount +from exasol.ds.sandbox.lib import pretty_print +from importlib_metadata import version +from pathlib import Path + +from exasol.ds.sandbox.lib.config import ConfigObject, SLC_VERSION +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.ansible import ansible_repository +from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies + + +DSS_VERSION = version("exasol-data-science-sandbox") +_logger = get_status_logger(LogType.DOCKER_IMAGE) + + +class DssDockerImage: + @classmethod + def timestamp(cls) -> str: + return f'{datetime.now().timestamp():.0f}' + + def __init__( + self, + repository: str, + version: str = None, + publish: bool = False, + keep_container: bool = False, + ): + version = version if version else DSS_VERSION + self.container_name = f"ds-sandbox-{DssDockerImage.timestamp()}" + self.image_name = f"{repository}:{version}" + self.publish = publish + self.keep_container = keep_container + + def _ansible_run_context(self) -> AnsibleRunContext: + extra_vars = { + "docker_container": self.container_name, + } + return AnsibleRunContext( + playbook="dss_docker_playbook.yml", + extra_vars=extra_vars, + ) + + def _ansible_config(self) -> ConfigObject: + return ConfigObject( + time_to_wait_for_polling=0.1, + slc_version=SLC_VERSION, + ) + + def _docker_file(self) -> importlib_resources.abc.Traversable: + return ( + importlib_resources + .files("exasol.ds.sandbox.lib.dss_docker") + .joinpath("Dockerfile") + ) + + def create(self): + docker_file = self._docker_file() + try: + start = datetime.now() + docker_client = docker.from_env() + _logger.info(f"Creating docker image {self.image_name} from {docker_file}") + with docker_file.open("rb") as fileobj: + docker_client.images.build(fileobj=fileobj, tag=self.image_name) + container = docker_client.containers.create( + image=self.image_name, + name=self.container_name, + command="sleep infinity", + detach=True, + ) + _logger.info("Starting container") + container.start() + _logger.info("Installing dependencies") + run_install_dependencies( + AnsibleAccess(), + configuration=self._ansible_config(), + host_infos=tuple(), + ansible_run_context=self._ansible_run_context(), + ansible_repositories=ansible_repository.default_repositories, + ) + _logger.info("Committing changes to docker container") + image = container.commit( + repository=self.image_name, + ) + except Exception as ex: + raise ex + finally: + if self.keep_container: + _logger.info("Keeping container running") + else: + _logger.info("Stopping container") + container.stop() + _logger.info("Removing container") + container.remove() + size = humanfriendly.format_size(image.attrs["Size"]) + elapsed = pretty_print.elapsed(start) + _logger.info(f"Built Docker image {self.image_name} size {size} in {elapsed}.") diff --git a/exasol/ds/sandbox/lib/logging.py b/exasol/ds/sandbox/lib/logging.py index 5d54d2a5..e6301f85 100644 --- a/exasol/ds/sandbox/lib/logging.py +++ b/exasol/ds/sandbox/lib/logging.py @@ -13,6 +13,7 @@ class LogType(Enum): SETUP_CI_CODEBUILD = "setup_ci_codebuild" AWS_ACCESS = "aws_access" ANSIBLE = "ansible" + DOCKER_IMAGE = "docker_image" CREATE_VM = "create_vm" SETUP_RELEASE_BUILD = "setup_release_build" RELEASE_BUILD = "release_build" diff --git a/exasol/ds/sandbox/lib/pretty_print.py b/exasol/ds/sandbox/lib/pretty_print.py new file mode 100644 index 00000000..3b4e648f --- /dev/null +++ b/exasol/ds/sandbox/lib/pretty_print.py @@ -0,0 +1,8 @@ +from datetime import datetime, timedelta + + +def elapsed(start: datetime, round_to_seconds=True) -> str: + d = datetime.now() - start + if round_to_seconds: + d = d - timedelta(microseconds=d.microseconds) + return str(d) diff --git a/exasol/ds/sandbox/runtime/ansible/dss_docker_playbook.yml b/exasol/ds/sandbox/runtime/ansible/dss_docker_playbook.yml new file mode 100644 index 00000000..f4436bc5 --- /dev/null +++ b/exasol/ds/sandbox/runtime/ansible/dss_docker_playbook.yml @@ -0,0 +1,23 @@ +- name: Prepare environment + hosts: localhost + gather_facts: false + vars: + ansible_python_interpreter: python3 + tasks: + - name: Add docker container to inventory + add_host: + name: "{{docker_container}}" + groups: docker_container_group + ansible_connection: docker + +- name: Setup DSS Docker Container + hosts: docker_container_group + gather_facts: false + vars: + ansible_python_interpreter: python3 + user_name: root + user_home: /root + need_sudo: false + docker_integration_test: true + tasks: + - import_tasks: general_setup_tasks.yml diff --git a/exasol/ds/sandbox/runtime/ansible/ec2_setup_tasks.yml b/exasol/ds/sandbox/runtime/ansible/ec2_setup_tasks.yml new file mode 100644 index 00000000..8a4b6450 --- /dev/null +++ b/exasol/ds/sandbox/runtime/ansible/ec2_setup_tasks.yml @@ -0,0 +1,6 @@ +- name: Install Script_languages + include_role: + name: script_languages +- name: Update netplan + include_role: + name: netplan diff --git a/exasol/ds/sandbox/runtime/ansible/slc_setup_tasks.yml b/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml similarity index 53% rename from exasol/ds/sandbox/runtime/ansible/slc_setup_tasks.yml rename to exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml index a87823fd..132405d5 100644 --- a/exasol/ds/sandbox/runtime/ansible/slc_setup_tasks.yml +++ b/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml @@ -5,17 +5,14 @@ become: "{{need_sudo}}" - name: Install Poetry include_role: - name: poetry + name: poetry - name: Install Jupyter include_role: - name: jupyter + name: jupyter +- name: Clear pip cache + ansible.builtin.file: + path: /root/.cache/pip + state: absent - name: Install Docker include_role: - name: docker -- name: Install Script_languages - include_role: - name: script_languages -- name: Update netplan - include_role: - name: netplan - + name: docker diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt index f6722fd5..a87f6c6b 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt @@ -1,4 +1,4 @@ uncertainties==3.1.7 numpy==1.23.1 pandas==1.4.3 -exasol-notebook-connector==0.1.0 +exasol-notebook-connector==0.2.0 diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml index f73ebaa1..81680907 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml @@ -3,7 +3,7 @@ - name: Setup Jupyter block: - - name: Install dependant apt packages + - name: Install dependent apt packages apt: name: "{{apt_dependencies}}" state: present diff --git a/exasol/ds/sandbox/runtime/ansible/roles/poetry/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/poetry/tasks/main.yml index f79606fb..12b8f67f 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/poetry/tasks/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/poetry/tasks/main.yml @@ -1,6 +1,6 @@ --- -- name: Install dependant apt packages +- name: Install dependent apt packages apt: name: "{{apt_dependencies}}" state: present diff --git a/exasol/ds/sandbox/runtime/ansible/roles/script_languages/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/script_languages/tasks/main.yml index ab550ebe..b91da0f0 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/script_languages/tasks/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/script_languages/tasks/main.yml @@ -1,6 +1,6 @@ --- -- name: Install dependant apt packages +- name: Install dependent apt packages apt: name: "{{apt_dependencies}}" state: present diff --git a/exasol/ds/sandbox/runtime/ansible/slc_setup.yml b/exasol/ds/sandbox/runtime/ansible/slc_setup.yml index 410a07c8..29991cb1 100644 --- a/exasol/ds/sandbox/runtime/ansible/slc_setup.yml +++ b/exasol/ds/sandbox/runtime/ansible/slc_setup.yml @@ -7,4 +7,5 @@ need_sudo: yes remote_user: ubuntu tasks: - - import_tasks: slc_setup_tasks.yml + - import_tasks: general_setup_tasks.yml + - import_tasks: ec2_setup_tasks.yml diff --git a/poetry.lock b/poetry.lock index c915c523..9738680c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "ansible" version = "6.7.0" description = "Radically simple IT automation" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -19,7 +18,6 @@ ansible-core = ">=2.13.7,<2.14.0" name = "ansible-core" version = "2.13.10" description = "Radically simple IT automation" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -38,7 +36,6 @@ resolvelib = ">=0.5.3,<0.9.0" name = "ansible-runner" version = "2.3.3" description = "\"Consistent Ansible Python API and CLI with container and process isolation runtime capabilities\"" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -58,7 +55,6 @@ six = "*" name = "apispec" version = "6.3.0" description = "A pluggable API specification generator. Currently supports the OpenAPI Specification (f.k.a. the Swagger specification)." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -82,7 +78,6 @@ yaml = ["PyYAML (>=3.10)"] name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -101,7 +96,6 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "aws-sam-translator" version = "1.70.0" description = "AWS SAM Translator is a library that transform SAM templates into AWS CloudFormation templates" -category = "dev" optional = false python-versions = ">=3.7, <=4.0, !=4.0" files = [ @@ -110,19 +104,18 @@ files = [ ] [package.dependencies] -boto3 = ">=1.19.5,<2.0.0" +boto3 = ">=1.19.5,<2.dev0" jsonschema = ">=3.2,<5" pydantic = ">=1.8,<2.0" typing-extensions = ">=4.4,<5" [package.extras] -dev = ["black (==23.1.0)", "boto3 (>=1.23,<2)", "boto3-stubs[appconfig,serverlessrepo] (>=1.19.5,<2.0.0)", "coverage (>=5.3,<8)", "dateparser (>=1.1,<2.0)", "importlib-metadata", "mypy (>=1.1.0,<1.2.0)", "parameterized (>=0.7,<1.0)", "pytest (>=6.2,<8)", "pytest-cov (>=2.10,<5)", "pytest-env (>=0.6,<1)", "pytest-rerunfailures (>=9.1,<12)", "pytest-xdist (>=2.5,<4)", "pyyaml (>=6.0,<7.0)", "requests (>=2.28,<3.0)", "ruamel.yaml (==0.17.21)", "ruff (==0.0.263)", "tenacity (>=8.0,<9.0)", "types-PyYAML (>=6.0,<7.0)", "types-jsonschema (>=3.2,<4.0)"] +dev = ["black (==23.1.0)", "boto3 (>=1.23,<2)", "boto3-stubs[appconfig,serverlessrepo] (>=1.19.5,<2.dev0)", "coverage (>=5.3,<8)", "dateparser (>=1.1,<2.0)", "importlib-metadata", "mypy (>=1.1.0,<1.2.0)", "parameterized (>=0.7,<1.0)", "pytest (>=6.2,<8)", "pytest-cov (>=2.10,<5)", "pytest-env (>=0.6,<1)", "pytest-rerunfailures (>=9.1,<12)", "pytest-xdist (>=2.5,<4)", "pyyaml (>=6.0,<7.0)", "requests (>=2.28,<3.0)", "ruamel.yaml (==0.17.21)", "ruff (==0.0.263)", "tenacity (>=8.0,<9.0)", "types-PyYAML (>=6.0,<7.0)", "types-jsonschema (>=3.2,<4.0)"] [[package]] name = "bcrypt" version = "4.0.1" description = "Modern password hashing for your software and your servers" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -157,7 +150,6 @@ typecheck = ["mypy"] name = "boto3" version = "1.27.0" description = "The AWS SDK for Python" -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -177,7 +169,6 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] name = "botocore" version = "1.30.0" description = "Low-level, data-driven core of boto 3." -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -197,7 +188,6 @@ crt = ["awscrt (==0.16.9)"] name = "cachetools" version = "3.1.1" description = "Extensible memoizing collections and decorators" -category = "dev" optional = false python-versions = "*" files = [ @@ -209,7 +199,6 @@ files = [ name = "certifi" version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -221,7 +210,6 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." -category = "main" optional = false python-versions = "*" files = [ @@ -298,7 +286,6 @@ pycparser = "*" name = "cfn-lint" version = "0.65.1" description = "Checks CloudFormation templates for practices and behaviour that could potentially be improved" -category = "dev" optional = false python-versions = ">=3.7, <=4.0, !=4.0" files = [ @@ -320,7 +307,6 @@ sarif-om = ">=1.0.4,<1.1.0" name = "chardet" version = "4.0.0" description = "Universal encoding detector for Python 2 and 3" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -332,7 +318,6 @@ files = [ name = "click" version = "8.1.3" description = "Composable command line interface toolkit" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -347,7 +332,6 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -359,7 +343,6 @@ files = [ name = "commonmark" version = "0.9.1" description = "Python parser for the CommonMark Markdown spec" -category = "main" optional = false python-versions = "*" files = [ @@ -374,7 +357,6 @@ test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] name = "cryptography" version = "41.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -416,7 +398,6 @@ test-randomorder = ["pytest-randomly"] name = "deepdiff" version = "6.3.0" description = "Deep Difference and Search of any Python object/data. Recreate objects by adding adding deltas to each other." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -435,7 +416,6 @@ optimize = ["orjson"] name = "deprecated" version = "1.2.14" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -453,7 +433,6 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] name = "dill" version = "0.3.2" description = "serialize all of python" -category = "dev" optional = false python-versions = ">=2.6, !=3.0.*" files = [ @@ -467,7 +446,6 @@ graph = ["objgraph (>=1.7.2)"] name = "dnslib" version = "0.9.23" description = "Simple library to encode/decode DNS wire-format packets" -category = "dev" optional = false python-versions = "*" files = [ @@ -480,7 +458,6 @@ files = [ name = "dnspython" version = "2.3.0" description = "DNS toolkit" -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -501,7 +478,6 @@ wmi = ["wmi (>=1.5.1,<2.0.0)"] name = "docker" version = "5.0.3" description = "A Python library for the Docker Engine API." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -521,7 +497,6 @@ tls = ["cryptography (>=3.4.7)", "idna (>=2.0.0)", "pyOpenSSL (>=17.5.0)"] name = "docutils" version = "0.20.1" description = "Docutils -- Python Documentation Utilities" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -533,7 +508,6 @@ files = [ name = "ecdsa" version = "0.18.0" description = "ECDSA cryptographic signature library (pure python)" -category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -552,7 +526,6 @@ gmpy2 = ["gmpy2"] name = "exasol-error-reporting-python" version = "0.2.0" description = "Exasol Python Error Reporting" -category = "main" optional = false python-versions = ">=3.6.1,<4.0" files = [] @@ -568,7 +541,6 @@ resolved_reference = "e21f181502b32f18d09660dde84cc20159c27b29" name = "exasol-integration-test-docker-environment" version = "0.11.0" description = "Integration Test Docker Environment for Exasol" -category = "main" optional = false python-versions = ">=3.8,<4" files = [ @@ -598,7 +570,6 @@ url = "https://github.com/exasol/integration-test-docker-environment/releases/do name = "exasol-script-languages-container-ci" version = "0.4.0" description = "Implements CI builds for script-language-container." -category = "main" optional = false python-versions = ">=3.8.0,<4.0" files = [ @@ -615,11 +586,11 @@ PyGithub = ">=1.55.0,<2.0.0" [package.source] type = "url" url = "https://github.com/exasol/script-languages-container-ci/releases/download/0.4.0/exasol_script_languages_container_ci-0.4.0-py3-none-any.whl" + [[package]] name = "exasol-script-languages-container-ci-setup" version = "0.5.0" description = "Manages AWS cloud CI build infrastructure." -category = "main" optional = false python-versions = ">=3.8.0,<4.0" files = [ @@ -639,11 +610,11 @@ PyGithub = ">=1.55.0,<2.0.0" [package.source] type = "url" url = "https://github.com/exasol/script-languages-container-ci-setup/releases/download/0.5.0/exasol_script_languages_container_ci_setup-0.5.0-py3-none-any.whl" + [[package]] name = "exasol-script-languages-container-tool" version = "0.14.0" description = "Script Languages Container Tool" -category = "main" optional = false python-versions = ">=3.8,<4" files = [ @@ -658,11 +629,11 @@ importlib-resources = ">=5.4.0" [package.source] type = "url" url = "https://github.com/exasol/script-languages-container-tool/releases/download/0.14.0/exasol_script_languages_container_tool-0.14.0-py3-none-any.whl" + [[package]] name = "exasol-script-languages-release" version = "5.0.0" description = "Script Languages Release" -category = "main" optional = false python-versions = ">=3.8,<4" files = [] @@ -683,7 +654,6 @@ resolved_reference = "f56c64c52a91b3d5fcf45d6f11330b5825198b79" name = "exceptiongroup" version = "1.1.2" description = "Backport of PEP 654 (exception groups)" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -698,7 +668,6 @@ test = ["pytest (>=6)"] name = "fabric" version = "2.7.1" description = "High level SSH command execution" -category = "dev" optional = false python-versions = "*" files = [ @@ -719,7 +688,6 @@ testing = ["mock (>=2.0.0,<3.0)"] name = "gitdb" version = "4.0.10" description = "Git Object Database" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -734,7 +702,6 @@ smmap = ">=3.0.1,<6" name = "gitpython" version = "3.1.31" description = "GitPython is a Python library used to interact with Git repositories" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -749,7 +716,6 @@ gitdb = ">=4.0.1,<5" name = "humanfriendly" version = "10.0" description = "Human friendly output for text interfaces using Python" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -764,7 +730,6 @@ pyreadline3 = {version = "*", markers = "sys_platform == \"win32\" and python_ve name = "idna" version = "2.10" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -776,7 +741,6 @@ files = [ name = "importlib-metadata" version = "6.2.1" description = "Read metadata from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -796,7 +760,6 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "importlib-resources" version = "5.12.0" description = "Read resources from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -815,7 +778,6 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -827,7 +789,6 @@ files = [ name = "invoke" version = "1.7.3" description = "Pythonic task execution" -category = "dev" optional = false python-versions = "*" files = [ @@ -839,7 +800,6 @@ files = [ name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -857,7 +817,6 @@ i18n = ["Babel (>=2.7)"] name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -869,7 +828,6 @@ files = [ name = "jschema-to-python" version = "1.2.3" description = "Generate source code for Python classes from a JSON schema." -category = "dev" optional = false python-versions = ">= 2.7" files = [ @@ -886,7 +844,6 @@ pbr = "*" name = "jsonpatch" version = "1.33" description = "Apply JSON-Patches (RFC 6902)" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" files = [ @@ -901,7 +858,6 @@ jsonpointer = ">=1.9" name = "jsonpickle" version = "3.0.1" description = "Python library for serializing any arbitrary object graph into JSON" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -918,7 +874,6 @@ testing-libs = ["simplejson", "ujson"] name = "jsonpointer" version = "2.4" description = "Identify specific nodes in a JSON document (RFC 6901)" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" files = [ @@ -930,7 +885,6 @@ files = [ name = "jsonschema" version = "3.2.0" description = "An implementation of JSON Schema validation for Python" -category = "main" optional = false python-versions = "*" files = [ @@ -952,7 +906,6 @@ format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-va name = "junit-xml" version = "1.9" description = "Creates JUnit XML test result documents that can be read by tools such as Jenkins" -category = "dev" optional = false python-versions = "*" files = [ @@ -967,7 +920,6 @@ six = "*" name = "localstack" version = "0.14.5" description = "LocalStack - A fully functional local Cloud stack" -category = "dev" optional = false python-versions = "*" files = [ @@ -1002,7 +954,6 @@ test = ["coverage[toml] (>=5.5)", "deepdiff (>=5.5.0)", "jsonpath-ng (>=1.5.3)", name = "localstack-client" version = "2.2" description = "A lightweight Python client for LocalStack." -category = "dev" optional = false python-versions = "*" files = [ @@ -1019,7 +970,6 @@ test = ["black", "coverage", "flake8", "isort", "localstack", "pytest"] name = "localstack-ext" version = "0.14.5" description = "Extensions for LocalStack" -category = "dev" optional = false python-versions = "*" files = [ @@ -1049,7 +999,6 @@ test-overrides = ["moto-ext[all] (>=3.1.10)"] name = "lockfile" version = "0.12.2" description = "Platform-independent file locking module" -category = "main" optional = false python-versions = "*" files = [ @@ -1061,7 +1010,6 @@ files = [ name = "luigi" version = "3.3.0" description = "Workflow mgmgt + task scheduling + dependency resolution." -category = "main" optional = false python-versions = "*" files = [ @@ -1083,7 +1031,6 @@ toml = ["toml (<2.0.0)"] name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1107,6 +1054,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1143,7 +1100,6 @@ files = [ name = "netaddr" version = "0.8.0" description = "A network address manipulation library for Python" -category = "main" optional = false python-versions = "*" files = [ @@ -1155,7 +1111,6 @@ files = [ name = "networkx" version = "2.8.8" description = "Python package for creating and manipulating graphs and networks" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1174,7 +1129,6 @@ test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] name = "numpy" version = "1.24.4" description = "Fundamental package for array computing in Python" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1212,7 +1166,6 @@ files = [ name = "numpy" version = "1.25.0" description = "Fundamental package for array computing in Python" -category = "main" optional = false python-versions = ">=3.9" files = [ @@ -1247,7 +1200,6 @@ files = [ name = "ordered-set" version = "4.1.0" description = "An OrderedSet is a custom MutableSet that remembers its order, so that every" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1262,7 +1214,6 @@ dev = ["black", "mypy", "pytest"] name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1274,7 +1225,6 @@ files = [ name = "pandas" version = "1.5.3" description = "Powerful data structures for data analysis, time series, and statistics" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1310,8 +1260,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, ] python-dateutil = ">=2.8.1" pytz = ">=2020.1" @@ -1323,7 +1273,6 @@ test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] name = "paramiko" version = "3.2.0" description = "SSH2 protocol library" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1345,7 +1294,6 @@ invoke = ["invoke (>=2.0)"] name = "pathlib2" version = "2.3.7.post1" description = "Object-oriented filesystem paths" -category = "dev" optional = false python-versions = "*" files = [ @@ -1360,7 +1308,6 @@ six = "*" name = "pbr" version = "5.11.1" description = "Python Build Reasonableness" -category = "dev" optional = false python-versions = ">=2.6" files = [ @@ -1372,7 +1319,6 @@ files = [ name = "pexpect" version = "4.8.0" description = "Pexpect allows easy control of interactive console applications." -category = "main" optional = false python-versions = "*" files = [ @@ -1387,7 +1333,6 @@ ptyprocess = ">=0.5" name = "pluggy" version = "1.2.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1403,7 +1348,6 @@ testing = ["pytest", "pytest-benchmark"] name = "plux" version = "1.3.2" description = "A dynamic code loading framework for building pluggable Python distributions" -category = "dev" optional = false python-versions = "*" files = [ @@ -1421,7 +1365,6 @@ dev = ["black (==21.6b0)", "isort (==5.9.1)", "pytest (==6.2.4)"] name = "psutil" version = "5.9.5" description = "Cross-platform lib for process and system monitoring in Python." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1448,7 +1391,6 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] name = "ptyprocess" version = "0.7.0" description = "Run a subprocess in a pseudo terminal" -category = "main" optional = false python-versions = "*" files = [ @@ -1460,7 +1402,6 @@ files = [ name = "pyaes" version = "1.6.1" description = "Pure-Python Implementation of the AES block-cipher and common modes of operation" -category = "dev" optional = false python-versions = "*" files = [ @@ -1471,7 +1412,6 @@ files = [ name = "pyasn1" version = "0.5.0" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -1483,7 +1423,6 @@ files = [ name = "pycparser" version = "2.21" description = "C parser in Python" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1495,7 +1434,6 @@ files = [ name = "pydantic" version = "1.10.10" description = "Data validation and settings management using python type hints" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1548,7 +1486,6 @@ email = ["email-validator (>=1.0.3)"] name = "pydot" version = "1.4.2" description = "Python interface to Graphviz's Dot" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1563,7 +1500,6 @@ pyparsing = ">=2.1.4" name = "pygithub" version = "1.59.0" description = "Use the full Github API v3" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1581,7 +1517,6 @@ requests = ">=2.14.0" name = "pygments" version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1596,7 +1531,6 @@ plugins = ["importlib-metadata"] name = "pyjwt" version = "2.7.0" description = "JSON Web Token implementation in Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1617,7 +1551,6 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] name = "pynacl" version = "1.5.0" description = "Python binding to the Networking and Cryptography (NaCl) library" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1644,7 +1577,6 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] name = "pyparsing" version = "3.1.0" description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "main" optional = false python-versions = ">=3.6.8" files = [ @@ -1659,7 +1591,6 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "pyreadline3" version = "3.4.1" description = "A python implementation of GNU readline." -category = "main" optional = false python-versions = "*" files = [ @@ -1671,7 +1602,6 @@ files = [ name = "pyrsistent" version = "0.19.3" description = "Persistent/Functional/Immutable data structures" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1708,7 +1638,6 @@ files = [ name = "pytest" version = "7.4.0" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1731,7 +1660,6 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-mock" version = "3.11.1" description = "Thin-wrapper around the mock package for easier use with pytest" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1749,7 +1677,6 @@ dev = ["pre-commit", "pytest-asyncio", "tox"] name = "python-daemon" version = "3.0.1" description = "Library to implement a well-behaved Unix daemon process." -category = "main" optional = false python-versions = ">=3" files = [ @@ -1770,7 +1697,6 @@ test = ["coverage", "docutils", "testscenarios (>=0.4)", "testtools"] name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -1785,7 +1711,6 @@ six = ">=1.5" name = "python-dotenv" version = "1.0.0" description = "Read key-value pairs from a .env file and set them as environment variables" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1800,7 +1725,6 @@ cli = ["click (>=5.0)"] name = "python-jose" version = "3.3.0" description = "JOSE implementation in Python" -category = "dev" optional = false python-versions = "*" files = [ @@ -1823,7 +1747,6 @@ pycryptodome = ["pyasn1", "pycryptodome (>=3.3.1,<4.0.0)"] name = "pytz" version = "2023.3" description = "World timezone definitions, modern and historical" -category = "main" optional = false python-versions = "*" files = [ @@ -1835,7 +1758,6 @@ files = [ name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1885,7 +1807,6 @@ files = [ name = "requests" version = "2.25.1" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1907,7 +1828,6 @@ socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] name = "resolvelib" version = "0.8.1" description = "Resolve abstract dependencies into concrete ones" -category = "main" optional = false python-versions = "*" files = [ @@ -1925,7 +1845,6 @@ test = ["commentjson", "packaging", "pytest"] name = "rich" version = "12.6.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -category = "main" optional = false python-versions = ">=3.6.3,<4.0.0" files = [ @@ -1945,7 +1864,6 @@ jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] name = "rsa" version = "4.9" description = "Pure-Python RSA implementation" -category = "dev" optional = false python-versions = ">=3.6,<4" files = [ @@ -1960,7 +1878,6 @@ pyasn1 = ">=0.1.3" name = "s3transfer" version = "0.6.1" description = "An Amazon S3 Transfer Manager" -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -1978,7 +1895,6 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] name = "sarif-om" version = "1.0.4" description = "Classes implementing the SARIF 2.1.0 object model." -category = "dev" optional = false python-versions = ">= 2.7" files = [ @@ -1994,7 +1910,6 @@ pbr = "*" name = "semver" version = "3.0.1" description = "Python helper for Semantic Versioning (https://semver.org)" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2006,7 +1921,6 @@ files = [ name = "setuptools" version = "68.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2023,7 +1937,6 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "simplejson" version = "3.19.1" description = "Simple, fast, extensible JSON encoder/decoder for Python" -category = "main" optional = false python-versions = ">=2.5, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2118,7 +2031,6 @@ files = [ name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2130,7 +2042,6 @@ files = [ name = "smmap" version = "5.0.0" description = "A pure Python implementation of a sliding window memory map manager" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2142,7 +2053,6 @@ files = [ name = "stevedore" version = "5.1.0" description = "Manage dynamic plugins for Python applications" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2157,7 +2067,6 @@ pbr = ">=2.0.0,<2.1.0 || >2.1.0" name = "stopwatch-py" version = "2.0.1" description = "A simple stopwatch for python" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -2169,7 +2078,6 @@ files = [ name = "tabulate" version = "0.9.0" description = "Pretty-print tabular data" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2184,7 +2092,6 @@ widechars = ["wcwidth"] name = "tailer" version = "0.4.1" description = "Python tail is a simple implementation of GNU tail and head." -category = "dev" optional = false python-versions = "*" files = [ @@ -2193,14 +2100,13 @@ files = [ [[package]] name = "tenacity" -version = "8.2.2" +version = "8.2.3" description = "Retry code until it succeeds" -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "tenacity-8.2.2-py3-none-any.whl", hash = "sha256:2f277afb21b851637e8f52e6a613ff08734c347dc19ade928e519d7d2d8569b0"}, - {file = "tenacity-8.2.2.tar.gz", hash = "sha256:43af037822bd0029025877f3b2d97cc4d7bb0c2991000a3d59d71517c5c969e0"}, + {file = "tenacity-8.2.3-py3-none-any.whl", hash = "sha256:ce510e327a630c9e1beaf17d42e6ffacc88185044ad85cf74c0a8887c6a0f88c"}, + {file = "tenacity-8.2.3.tar.gz", hash = "sha256:5398ef0d78e63f40007c1fb4c0bff96e1911394d2fa8d194f77619c05ff6cc8a"}, ] [package.extras] @@ -2210,7 +2116,6 @@ doc = ["reno", "sphinx", "tornado (>=4.5)"] name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2222,7 +2127,6 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2234,7 +2138,6 @@ files = [ name = "tornado" version = "6.3.2" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." -category = "main" optional = false python-versions = ">= 3.8" files = [ @@ -2255,7 +2158,6 @@ files = [ name = "typing-extensions" version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2267,7 +2169,6 @@ files = [ name = "urllib3" version = "1.26.16" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -2284,7 +2185,6 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "websocket-client" version = "1.6.1" description = "WebSocket client for Python with low level API options" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2301,7 +2201,6 @@ test = ["websockets"] name = "wrapt" version = "1.15.0" description = "Module for decorators, wrappers and monkey patching." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -2386,7 +2285,6 @@ files = [ name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2401,4 +2299,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.8.0,<4.0" -content-hash = "6c3f9b9b8bd576a43dcf3521b2e7f509e55c61cb2e2aa0858a43fd6d1ffc2cfa" +content-hash = "15c8ca8f81bdf24021b68176ec82d6487f07fad7af30134ad263b35b9af7c437" diff --git a/pyproject.toml b/pyproject.toml index 625e1cd1..5dc17a2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,8 @@ importlib_resources = ">=5.4.0" exasol_script_languages_release = { git = "https://github.com/exasol/script-languages-release.git", tag = "5.0.0"} rich = "^12.5.1" pandas = "^1.4.0" - +humanfriendly = "^10.0" +tenacity = "^8.2.3" [build-system] requires = ["poetry_core>=1.0.0"] diff --git a/test/ansible/slc_setup_test.yml b/test/ansible/slc_setup_test.yml index ee065982..6e403dbc 100644 --- a/test/ansible/slc_setup_test.yml +++ b/test/ansible/slc_setup_test.yml @@ -10,7 +10,7 @@ name: "{{test_docker_container}}" ansible_connection: docker -- name: Run test +- name: Setup Docker Container hosts: ansible-test gather_facts: false vars: @@ -25,4 +25,5 @@ name: netplan.io state: present update_cache: yes - - import_tasks: slc_setup_tasks.yml + - import_tasks: general_setup_tasks.yml + - import_tasks: ec2_setup_tasks.yml diff --git a/test/integration/test_create_dss_docker_image.py b/test/integration/test_create_dss_docker_image.py new file mode 100644 index 00000000..f41f4d24 --- /dev/null +++ b/test/integration/test_create_dss_docker_image.py @@ -0,0 +1,91 @@ +import docker +import pytest +import requests +import tenacity +import time +import typing + +from tenacity.retry import retry_if_exception_type +from tenacity.wait import wait_fixed +from tenacity.stop import stop_after_delay +from datetime import datetime, timedelta +from exasol.ds.sandbox.lib.dss_docker import DssDockerImage +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib import pretty_print + + +@pytest.fixture(scope="session") +def dss_docker_image(): + testee = DssDockerImage( + "my-repo/dss-test-image", + version=f"{DssDockerImage.timestamp()}", + publish=False, + keep_container=False, + ) + testee.create() + try: + yield testee + finally: + docker.from_env().images.remove(testee.image_name) + + +@pytest.fixture +def dss_docker_container(dss_docker_image): + client = docker.from_env() + mapped_ports = {'8888/tcp': 8888} + container = client.containers.create( + image=dss_docker_image.image_name, + name=dss_docker_image.container_name, + command="sleep infinity", + detach=True, + ports=mapped_ports, + ) + container.start() + try: + yield container + finally: + container.stop() + container.remove() + + +def retry(exception: typing.Type[BaseException], timeout: timedelta): + return tenacity.retry( + retry=retry_if_exception_type(exception), + wait=wait_fixed(timeout/10), + stop=stop_after_delay(timeout), + ) + + +def test_jupyterlab(dss_docker_container): + """" + Test that jupyterlab is configured properly + """ + jupyter_command = ( + "/root/jupyterenv/bin/jupyter-lab" + " --notebook-dir=/root/notebooks" + " --no-browser" + " --allow-root" + ) + container = dss_docker_container + container.exec_run(jupyter_command, detach=True) + container.reload() + ip_address = container.attrs['NetworkSettings']['IPAddress'] + url = f"http://{ip_address}:8888/lab" + + @retry(requests.exceptions.ConnectionError, timedelta(seconds=5)) + def request_with_retry(url: str) -> requests.Response: + return requests.get(url) + + start = datetime.now() + response = request_with_retry(url) + print(f'{url} responded after {pretty_print.elapsed(start)}.') + assert response.status_code == 200 + + +def test_install_notebook_connector(dss_docker_container): + container = dss_docker_container + command = '/root/jupyterenv/bin/python -c "import exasol.secret_store"' + exit_code, output = container.exec_run(command) + output = output.decode('utf-8').strip() + assert exit_code == 0, f'Got output "{output}".' + diff --git a/test/unit/ansible_conflict/slc_setup.yml b/test/unit/ansible_conflict/slc_setup.yml index ee065982..74e4d4cd 100644 --- a/test/unit/ansible_conflict/slc_setup.yml +++ b/test/unit/ansible_conflict/slc_setup.yml @@ -25,4 +25,5 @@ name: netplan.io state: present update_cache: yes - - import_tasks: slc_setup_tasks.yml + - import_tasks: general_setup_tasks.yml + - import_tasks: ec2_setup_tasks.yml diff --git a/test/unit/test_dss_docker_image.py b/test/unit/test_dss_docker_image.py new file mode 100644 index 00000000..ce7c6050 --- /dev/null +++ b/test/unit/test_dss_docker_image.py @@ -0,0 +1,31 @@ +import pytest + +from exasol.ds.sandbox.lib.dss_docker.create_image import ( + DssDockerImage, + DSS_VERSION, +) + + +@pytest.fixture +def sample_repo(): + return "avengers/tower" + + +def test_constructor_defaults(sample_repo): + testee = DssDockerImage(sample_repo) + assert testee.image_name == f"{sample_repo}:{DSS_VERSION}" + assert testee.publish == False + assert testee.keep_container == False + + +def test_constructor(sample_repo): + version = "1.2.3" + testee = DssDockerImage( + repository=sample_repo, + version=version, + publish=True, + keep_container=True, + ) + assert testee.image_name == f"{sample_repo}:{version}" + assert testee.publish == True + assert testee.keep_container == True diff --git a/test/unit/test_pretty_print.py b/test/unit/test_pretty_print.py new file mode 100644 index 00000000..61cac7a0 --- /dev/null +++ b/test/unit/test_pretty_print.py @@ -0,0 +1,21 @@ +import pytest +import re +from exasol.ds.sandbox.lib import pretty_print +from datetime import datetime, timedelta + + +@pytest.fixture +def sample_duration(): + return timedelta(hours=2, minutes=3, seconds=4.2) + + +def test_elapsed_rounded(sample_duration): + start = datetime.now() - sample_duration + e = pretty_print.elapsed(start) + assert e == "2:03:04" + + +def test_elapsed_full(sample_duration): + start = datetime.now() - sample_duration + e = pretty_print.elapsed(start, round_to_seconds=False) + assert re.match(r'2:03:04\.2.*', e)