From c89227ce4c6b44ad17b95d5a855bc3b5ca7285c2 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 18 Oct 2024 08:34:36 -0300 Subject: [PATCH] Added nb tests to debug namerserver issue --- .github/workflows/notebook_tests.yaml | 11 +- .../ansible/roles/docker/defaults/main.yml | 2 +- .../jupyter/files/notebook/dummy/itde.ipynb | 155 ++++++++++++++++++ test/notebooks/nbtest_dummy.py | 25 +++ test/notebooks/nbtest_itde.py | 21 ++- 5 files changed, 199 insertions(+), 15 deletions(-) create mode 100644 exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/dummy/itde.ipynb create mode 100644 test/notebooks/nbtest_dummy.py diff --git a/.github/workflows/notebook_tests.yaml b/.github/workflows/notebook_tests.yaml index c94357a3..26fb1a60 100644 --- a/.github/workflows/notebook_tests.yaml +++ b/.github/workflows/notebook_tests.yaml @@ -11,15 +11,8 @@ jobs: fail-fast: false matrix: nb_test: - - { name: "CSE notebook", test_file: "nbtest_cloud.py", test_backend: "all" } - - { name: "sklearn notebook", test_file: "nbtest_sklearn.py", test_backend: "all" } - - { name: "ibis notebook", test_file: "nbtest_ibis.py", test_backend: "all" } - - { name: "SLC notebook", test_file: "nbtest_script_languages_container.py", test_backend: "onprem" } - - { name: "SME notebooks onprem", test_file: "nbtest_sagemaker.py", test_backend: "onprem" } - - { name: "SME notebooks saas", test_file: "nbtest_sagemaker.py", test_backend: "saas" } - - { name: "TE notebooks onprem", test_file: "nbtest_transformers.py", test_backend: "onprem" } - - { name: "TE notebooks saas", test_file: "nbtest_transformers.py", test_backend: "saas"} - - { name: "short notebook tests", test_file: "\"nbtest_environment_test.py nbtest_itde.py\"", test_backend: ""} + - { name: "dummy notebook", test_file: "nbtest_dummy.py", test_backend: "onprem" } + - { name: "short notebook tests", test_file: "nbtest_itde.py", test_backend: ""} name: Running ${{ matrix.nb_test.name }} steps: - uses: actions/checkout@v4 diff --git a/exasol/ds/sandbox/runtime/ansible/roles/docker/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/docker/defaults/main.yml index fefb727c..732b2d66 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/docker/defaults/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/docker/defaults/main.yml @@ -3,5 +3,5 @@ apt_dependencies: - apt-transport-https=2.4.5 - gpg-agent=2.2.27-3ubuntu2.1 - - ca-certificates=20230311ubuntu0.22.04.1 + - ca-certificates=20240203~22.04.1 - software-properties-common=0.99.22.9 diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/dummy/itde.ipynb b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/dummy/itde.ipynb new file mode 100644 index 00000000..b9ae031e --- /dev/null +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/dummy/itde.ipynb @@ -0,0 +1,155 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c3c8450d-5ca8-4abe-8f7d-81644f4d33ee", + "metadata": {}, + "source": [ + "## Setup\n", + "### Open Secure Configuration Storage" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "cf0ee134-cfc4-46b5-99a0-563d269fae87", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "65fc4eb9da574801ad5b862198e24062", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e7bcc8f6a2ea4cc886fa5b091e0d1739", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Box(children=(Box(children=(Label(value='Configuration Store', layout=Layout(border_bottom='solid 1px', border…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%run ./utils/access_store_ui.ipynb\n", + "display(get_access_store_ui('./'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2bdf9910-791f-4aab-843f-8077c72f8931", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "requests.get(\"https://example.com/\")" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "d38a2a6f-1673-44cc-9a06-43afebc75749", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "172.20.0.2\n" + ] + } + ], + "source": [ + "from exasol.nb_connector.ai_lab_config import AILabConfig\n", + "hn = ai_lab_config.get(AILabConfig.db_host_name)\n", + "print(hn)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "c330458d-8c59-4b34-a45c-27587dd21b8f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[(1,)]\n" + ] + } + ], + "source": [ + "from exasol.nb_connector.connections import open_pyexasol_connection\n", + "import textwrap\n", + "\n", + "with open_pyexasol_connection(ai_lab_config, compression=True, schema=ai_lab_config.db_schema,) as con:\n", + " \n", + " udf = textwrap.dedent('''\n", + " CREATE OR REPLACE PYTHON3 SCALAR SCRIPT test_dns(dummy INTEGER)\n", + " RETURNS INTEGER AS\n", + " def run(ctx):\n", + " import requests\n", + " requests.get(\"https://example.com/\")\n", + " return 1\n", + " ''')\n", + " con.execute(udf)\n", + " \n", + " result = con.execute(\"SELECT test_dns(1)\").fetchmany()\n", + " print(result)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b435bbb2-8fdb-4b2e-998f-b9b4a330cdc9", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35014ec8-8de7-4689-957e-73a630740ad8", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/test/notebooks/nbtest_dummy.py b/test/notebooks/nbtest_dummy.py new file mode 100644 index 00000000..189b4843 --- /dev/null +++ b/test/notebooks/nbtest_dummy.py @@ -0,0 +1,25 @@ +import os +from pathlib import Path + +import pytest + +# We need to manually import all fixtures that we use, directly or indirectly, +# since the pytest won't do this for us. +from notebook_test_utils import (backend_setup, + run_notebook) +from exasol.pytest_backend import BACKEND_ONPREM + + + +def test_itde(backend, backend_setup) -> None: + if backend != BACKEND_ONPREM: + pytest.skip() + current_dir = Path.cwd() + store_path, store_password = backend_setup + store_file = str(store_path) + try: + run_notebook('main_config.ipynb', store_file, store_password) + os.chdir('./dummy') + run_notebook('itde.ipynb', store_file, store_password) + finally: + os.chdir(current_dir) diff --git a/test/notebooks/nbtest_itde.py b/test/notebooks/nbtest_itde.py index cd539613..f7eeef17 100644 --- a/test/notebooks/nbtest_itde.py +++ b/test/notebooks/nbtest_itde.py @@ -5,6 +5,7 @@ ) from exasol.nb_connector.connections import open_pyexasol_connection from notebook_test_utils import set_log_level_for_libraries +import textwrap set_log_level_for_libraries() @@ -13,13 +14,23 @@ def test_itde(tmp_path): store_password = "password" secrets = Secrets(store_path, master_password=store_password) bring_itde_up(secrets) + import requests + requests.get("https://example.com/") try: - con = open_pyexasol_connection(secrets) - try: - result = con.execute("select 1").fetchmany() + with open_pyexasol_connection(secrets, schema=ai_lab_config.db_schema) as con: + udf = textwrap.dedent(''' + CREATE OR REPLACE PYTHON3 SCALAR SCRIPT test_dns(dummy INTEGER) + RETURNS INTEGER AS + def run(ctx): + import requests + requests.get("https://example.com/") + return 1 + ''') + con.execute(udf) + + result = con.execute("SELECT test_dns(1)").fetchmany() + print(result) assert result[0][0] == 1 - finally: - con.close() finally: take_itde_down(secrets)