Skip to content

Commit

Permalink
Merge branch 'main' into refactoring/160-pm-comments
Browse files Browse the repository at this point in the history
  • Loading branch information
ahsimb authored Feb 1, 2024
2 parents 74085da + 02c18cc commit 8981fa2
Show file tree
Hide file tree
Showing 12 changed files with 85 additions and 40 deletions.
1 change: 1 addition & 0 deletions doc/changes/changelog.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
# Changes

* [0.2.0](changes_0.2.0.md)
* [0.1.0](changes_0.1.0.md)
20 changes: 20 additions & 0 deletions doc/changes/changes_0.2.0.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# ai-lab 0.2.0, released TBD

Code name: Post release fixes

## Summary


## AI-Lab-Release

Version: 0.2.0

## Features

## Bug Fixes

* #163: Fixed version number of VM images etc.

## Refactoring

## Documentation
9 changes: 5 additions & 4 deletions exasol/ds/sandbox/cli/options/id_options.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import click

from exasol.ds.sandbox.lib.config import SLC_VERSION
from exasol.ds.sandbox.lib.config import AI_LAB_VERSION

id_options = [
click.option('--asset-id', type=str, default=SLC_VERSION,
help="This value will be used in the AMI name, as tag on all AWS resources "
"and in the prefix of the S3 objects.")
click.option(
'--asset-id', type=str, default=AI_LAB_VERSION,
help="This value will be used in the AMI name, as tag on all AWS resources "
"and in the prefix of the S3 objects.")
]
3 changes: 3 additions & 0 deletions exasol/ds/sandbox/lib/config.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
from importlib_metadata import version

# name of the project as specified in file pyproject.toml
AI_LAB_VERSION = version("exasol-ai-lab")
SLC_VERSION = version("exasol_script_languages_release")

_default_config = {
Expand All @@ -12,6 +14,7 @@
"state": "available"
},
"slc_version": SLC_VERSION,
"ai_lab_version": AI_LAB_VERSION,
"waf_region": "us-east-1"
}

Expand Down
6 changes: 3 additions & 3 deletions exasol/ds/sandbox/lib/dss_docker/create_image.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,9 @@
from exasol.ds.sandbox.lib.logging import get_status_logger, LogType
from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo
from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies
from exasol.ds.sandbox.lib.config import AI_LAB_VERSION

DEFAULT_ORG_AND_REPOSITORY = "exasol/ai-lab"
# name of the project as specified in file pyproject.toml
DSS_VERSION = version("exasol-ai-lab")
_logger = get_status_logger(LogType.DOCKER_IMAGE)


Expand Down Expand Up @@ -78,7 +77,7 @@ def __init__(
version: str = None,
keep_container: bool = False,
):
version = version if version else DSS_VERSION
version = version if version else AI_LAB_VERSION
self.container_name = f"ds-sandbox-{DssDockerImage.timestamp()}"
self.repository = repository
self.version = version
Expand All @@ -103,6 +102,7 @@ def _ansible_config(self) -> ConfigObject:
return ConfigObject(
time_to_wait_for_polling=0.1,
slc_version=SLC_VERSION,
ai_lab_version=AI_LAB_VERSION,
)

def _docker_file(self) -> importlib_resources.abc.Traversable:
Expand Down
2 changes: 1 addition & 1 deletion exasol/ds/sandbox/lib/release_build/run_release_build.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ def run_start_release_build(aws_access: AwsAccess, config: ConfigObject,
upload_url: str, branch: str, gh_token: str) -> None:
logging.info(f"run_start_release_build for aws profile {aws_access.aws_profile_for_logging} "
f"with upload url: {upload_url}")
_execute_release_build(aws_access, branch, asset_id=config.slc_version,
_execute_release_build(aws_access, branch, asset_id=config.ai_lab_version,
release_id=_parse_upload_url(upload_url=upload_url), gh_token=gh_token,
make_ami_public=True)

Expand Down
5 changes: 4 additions & 1 deletion exasol/ds/sandbox/lib/setup_ec2/run_install_dependencies.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,10 @@ def run_install_dependencies(ansible_access: AnsibleAccess,
are copied as flat copy to the dynamic working copy, too.
The playbook is indicated by variable ansible_run_context, which also might contain additional ansible variables.
"""
new_extra_vars = {"slc_version": configuration.slc_version}
new_extra_vars = {
"slc_version": configuration.slc_version,
"ai_lab_version": configuration.ai_lab_version,
}
if ansible_run_context.extra_vars is not None:
new_extra_vars.update(ansible_run_context.extra_vars)
new_ansible_run_context = AnsibleRunContext(ansible_run_context.playbook, new_extra_vars)
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "exasol-ai-lab"
version = "0.1.0"
version = "0.2.0"
description = "Provide AI-Lab editions."
packages = [ {include = "exasol"}, ]
license = "MIT"
Expand Down
31 changes: 20 additions & 11 deletions test/codebuild/test_ci.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess
from exasol.ds.sandbox.lib.asset_id import AssetId
from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess
from exasol.ds.sandbox.lib.config import default_config_object, SLC_VERSION
from exasol.ds.sandbox.lib.config import default_config_object, AI_LAB_VERSION
from exasol.ds.sandbox.lib.run_create_vm import run_create_vm
from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \
EC2StackLifecycleContextManager
Expand Down Expand Up @@ -42,14 +42,25 @@ def change_password(host: str, user: str, curr_pass: str, new_password: str) ->
assert res.ok


def _create_asset_id():
timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
return AssetId(
f"ci-test-{AI_LAB_VERSION}-{timestamp}",
stack_prefix="stack",
ami_prefix="ami",
)


@pytest.fixture(scope="session")
def new_ec2_from_ami():
"""
This fixtures starts the EC-2 instance, runs all setup, exports the AMI,
then starts another EC-2 instance, based on the new AMI, then changes the password (which is expired),
and finally returns that EC-2 name together with the new temporary password.
Start the EC-2 instance, run all setup, export the AMI, then start
another EC-2 instance, based on the new AMI, then change the password
(which is expired), and finally return that EC-2 name together with the
new temporary password.
"""
# Create default_password (the one burned into the AMI) and the new password (which will be set during first login)
# Create default_password (the one burned into the AMI) and the new password
# (which will be set during first login)
# We use different sizes of both in order to avoid equality of both!
default_password = generate_random_password(length=12)
new_password = generate_random_password(length=14)
Expand All @@ -58,10 +69,7 @@ def new_ec2_from_ami():
assert default_password != new_password
aws_access = AwsAccess(aws_profile=None)
user_name = os.getenv("AWS_USER_NAME")
asset_id = AssetId("ci-test-{suffix}-{now}".format(now=datetime.now().strftime("%Y-%m-%d-%H-%M-%S"),
suffix=SLC_VERSION),
stack_prefix="stack",
ami_prefix="ami")
asset_id = _create_asset_id()
run_create_vm(aws_access, None, None,
AnsibleAccess(), default_password, tuple(), asset_id,
default_config_object, user_name, make_ami_public=False)
Expand All @@ -71,8 +79,9 @@ def new_ec2_from_ami():
assert len(amis) == 1
ami = amis[0]

lifecycle_generator = run_lifecycle_for_ec2(aws_access, None, None, asset_id=asset_id,
ami_id=ami.id, user_name=user_name)
lifecycle_generator = run_lifecycle_for_ec2(
aws_access, None, None, asset_id=asset_id,
ami_id=ami.id, user_name=user_name)

try:
with EC2StackLifecycleContextManager(lifecycle_generator, default_config_object) as ec2_data:
Expand Down
27 changes: 18 additions & 9 deletions test/unit/test_ansible.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,22 @@ def run(self,
self.delegate(private_data_dir, run_ctx)


def _extra_vars(config):
return {
"slc_version": config.slc_version,
"ai_lab_version": config.ai_lab_version,
}

def test_run_ansible_default_values(test_config):
"""
Test which executes run_install_dependencies with default values (default playbook and default ansible variables)
"""
ansible_access = AnsibleTestAccess()
run_install_dependencies(ansible_access, test_config)
expected_ansible_run_context = AnsibleRunContext(playbook="slc_setup.yml",
extra_vars={"slc_version": test_config.slc_version})
expected_ansible_run_context = AnsibleRunContext(
playbook="slc_setup.yml",
extra_vars=_extra_vars(test_config),
)
assert ansible_access.call_arguments.private_data_dir.startswith("/tmp")
assert ansible_access.call_arguments.run_ctx == expected_ansible_run_context

Expand All @@ -54,8 +62,8 @@ def test_run_ansible_custom_playbook(test_config):
ansible_run_context = AnsibleRunContext(playbook="my_playbook.yml", extra_vars=dict())
run_install_dependencies(ansible_access, test_config, host_infos=tuple(), ansible_run_context=ansible_run_context)

expected_ansible_run_context = AnsibleRunContext(playbook="my_playbook.yml",
extra_vars={"slc_version": test_config.slc_version})
expected_ansible_run_context = AnsibleRunContext(
playbook="my_playbook.yml", extra_vars=_extra_vars(test_config))
assert ansible_access.call_arguments.private_data_dir.startswith("/tmp")
assert ansible_access.call_arguments.run_ctx == expected_ansible_run_context

Expand All @@ -66,11 +74,12 @@ def test_run_ansible_custom_variables(test_config):
"""
ansible_access = AnsibleTestAccess()
ansible_run_context = AnsibleRunContext(playbook="my_playbook.yml", extra_vars={"my_var": True})
run_install_dependencies(ansible_access, test_config, host_infos=tuple(), ansible_run_context=ansible_run_context)

expected_ansible_run_context = AnsibleRunContext(playbook="my_playbook.yml",
extra_vars={"slc_version": test_config.slc_version,
"my_var": True})
run_install_dependencies(ansible_access, test_config, host_infos=tuple(),
ansible_run_context=ansible_run_context)
extra_vars = _extra_vars(test_config)
extra_vars.update({"my_var": True})
expected_ansible_run_context = AnsibleRunContext(
playbook="my_playbook.yml", extra_vars=extra_vars)
assert ansible_access.call_arguments.private_data_dir.startswith("/tmp")
assert ansible_access.call_arguments.run_ctx == expected_ansible_run_context

Expand Down
8 changes: 3 additions & 5 deletions test/unit/test_dss_docker_image.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,8 @@
from unittest.mock import MagicMock, Mock, create_autospec
from datetime import datetime
from exasol.ds.sandbox.lib.dss_docker import create_image, DockerRegistry
from exasol.ds.sandbox.lib.dss_docker.create_image import (
DssDockerImage,
DSS_VERSION,
)
from exasol.ds.sandbox.lib.config import AI_LAB_VERSION
from exasol.ds.sandbox.lib.dss_docker.create_image import DssDockerImage


@pytest.fixture
Expand All @@ -16,7 +14,7 @@ def sample_repo():

def test_constructor_defaults(sample_repo):
testee = DssDockerImage(sample_repo)
assert testee.image_name == f"{sample_repo}:{DSS_VERSION}"
assert testee.image_name == f"{sample_repo}:{AI_LAB_VERSION}"
assert testee.keep_container == False


Expand Down
11 changes: 6 additions & 5 deletions test/unit/test_release_build.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,17 +46,18 @@ def test_release_build(test_config):
mock_cast(aws_access_mock.get_all_stack_resources).return_value = DUMMY_RESOURCES
mock_cast(aws_access_mock.start_codebuild).return_value = (123, create_autospec(CodeBuildWaiter))
run_start_release_build(aws_access_mock, test_config, UPLOAD_URL, BRANCH, GITHUB_TOKEN)
expected_env_variable_overrides = [
expected_overrides = [
{"name": "RELEASE_ID", "value": "123", "type": "PLAINTEXT"},
{"name": "ASSET_ID", "value": test_config.slc_version, "type": "PLAINTEXT"},
{"name": "ASSET_ID", "value": test_config.ai_lab_version, "type": "PLAINTEXT"},
{"name": "GITHUB_TOKEN", "value": GITHUB_TOKEN, "type": "PLAINTEXT"},
{"name": "MAKE_AMI_PUBLIC_OPTION", "value": "--make-ami-public", "type": "PLAINTEXT"}
]

mock_cast(aws_access_mock.start_codebuild).\
assert_called_once_with("codebuild-id-123",
environment_variables_overrides=expected_env_variable_overrides,
branch=BRANCH)
assert_called_once_with(
"codebuild-id-123",
environment_variables_overrides=expected_overrides,
branch=BRANCH)


def test_test_release_build(test_config):
Expand Down

0 comments on commit 8981fa2

Please sign in to comment.