Merge branch 'main' into refactoring/#165-Reduce_log_output_in_Codebu…

…ild_ai-lab
exasol · Feb 5, 2024 · 7b84c46 · 7b84c46
2 parents bed75bc + e5fcd8b
commit 7b84c46
Show file tree

Hide file tree

Showing 15 changed files with 86 additions and 103 deletions.
diff --git a/doc/changes/changes_0.2.0.md b/doc/changes/changes_0.2.0.md
@@ -19,6 +19,7 @@ Version: 0.2.0
 ## Refactoring
 
 * #160: Implemented the PM's recommendations of 2024-01-24.
+* #120: Passing the secret store object (sb_config) as a parameter to all functions that need it.
 * #165: Reduced log output in Codebuild ai-lab
 
 ## Documentation
diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/main_config.ipynb b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/main_config.ipynb
@@ -75,7 +75,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "display(get_db_selection_ui())"
+    "display(get_db_selection_ui(sb_config))"
    ]
   },
   {
@@ -95,7 +95,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "display(get_db_config_ui())"
+    "display(get_db_config_ui(sb_config))"
    ]
   },
   {
@@ -117,7 +117,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "display(get_start_docker_db_ui())"
+    "display(get_start_docker_db_ui(sb_config))"
    ]
   },
   {

diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/sagemaker/sme_init.ipynb b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/sagemaker/sme_init.ipynb
@@ -58,7 +58,7 @@
    "outputs": [],
    "source": [
     "%run utils/sme_init_ui.ipynb\n",
-    "display(get_sme_config_ui())"
+    "display(get_sme_config_ui(sb_config))"
    ]
   },
   {

diff --git a/...ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/sagemaker/utils/sme_init_ui.ipynb b/...ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/sagemaker/utils/sme_init_ui.ipynb
@@ -35,20 +35,21 @@
     "from enum import Enum\n",
     "import ipywidgets as widgets\n",
     "\n",
+    "from exasol.secret_store import Secrets\n",
     "from exasol.ai_lab_config import AILabConfig as CKey\n",
     "\n",
     "\n",
-    "def get_sme_config_ui() -> widgets.Widget:\n",
+    "def get_sme_config_ui(conf: Secrets) -> widgets.Widget:\n",
     "\n",
     "    inputs = [\n",
-    "        ('Access Key ID', widgets.Password(value=sb_config.get(CKey.aws_access_key_id)), CKey.aws_access_key_id),\n",
-    "        ('Secret Access Key', widgets.Password(value=sb_config.get(CKey.aws_secret_access_key)), CKey.aws_secret_access_key),\n",
-    "        ('Region', widgets.Text(value=sb_config.get(CKey.aws_region)), CKey.aws_region),\n",
-    "        ('Sagemaker AWS Role', widgets.Text(value=sb_config.get(CKey.sme_aws_role)), CKey.sme_aws_role),\n",
-    "        ('Sagemaker S3 Bucket', widgets.Text(value=sb_config.get(CKey.sme_aws_bucket)), CKey.sme_aws_bucket),\n",
+    "        ('Access Key ID', widgets.Password(value=conf.get(CKey.aws_access_key_id)), CKey.aws_access_key_id),\n",
+    "        ('Secret Access Key', widgets.Password(value=conf.get(CKey.aws_secret_access_key)), CKey.aws_secret_access_key),\n",
+    "        ('Region', widgets.Text(value=conf.get(CKey.aws_region)), CKey.aws_region),\n",
+    "        ('Sagemaker AWS Role', widgets.Text(value=conf.get(CKey.sme_aws_role)), CKey.sme_aws_role),\n",
+    "        ('Sagemaker S3 Bucket', widgets.Text(value=conf.get(CKey.sme_aws_bucket)), CKey.sme_aws_bucket),\n",
     "    ]\n",
     "\n",
-    "    return get_generic_config_ui(sb_config, [inputs], ['Set AWS Credentials'])\n"
+    "    return get_generic_config_ui(conf, [inputs], ['Set AWS Credentials'])\n"
    ]
   }
  ],

diff --git a/.../sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/masked_modelling.ipynb b/.../sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/masked_modelling.ipynb
@@ -89,7 +89,7 @@
    "outputs": [],
    "source": [
     "%run utils/model_retrieval.ipynb\n",
-    "load_huggingface_model(MODEL_NAME)"
+    "load_huggingface_model(sb_config, MODEL_NAME)"
    ]
   },
   {

diff --git a/...andbox/runtime/ansible/roles/jupyter/files/notebook/transformers/question_answering.ipynb b/...andbox/runtime/ansible/roles/jupyter/files/notebook/transformers/question_answering.ipynb
@@ -89,7 +89,7 @@
    "outputs": [],
    "source": [
     "%run utils/model_retrieval.ipynb\n",
-    "load_huggingface_model(MODEL_NAME)"
+    "load_huggingface_model(sb_config, MODEL_NAME)"
    ]
   },
   {

diff --git a/...x/runtime/ansible/roles/jupyter/files/notebook/transformers/sequence_classification.ipynb b/...x/runtime/ansible/roles/jupyter/files/notebook/transformers/sequence_classification.ipynb
@@ -90,7 +90,7 @@
    "outputs": [],
    "source": [
     "%run utils/model_retrieval.ipynb\n",
-    "load_huggingface_model(MODEL_NAME, method='udf')"
+    "load_huggingface_model(sb_config, MODEL_NAME, method='udf')"
    ]
   },
   {

diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/te_init.ipynb b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/te_init.ipynb
@@ -46,7 +46,7 @@
    "outputs": [],
    "source": [
     "%run utils/te_init_ui.ipynb\n",
-    "display(get_te_config_ui())"
+    "display(get_te_config_ui(sb_config))"
    ]
   },
   {

diff --git a/...s/sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/text_generation.ipynb b/...s/sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/text_generation.ipynb
@@ -71,7 +71,7 @@
    "outputs": [],
    "source": [
     "%run utils/model_retrieval.ipynb\n",
-    "load_huggingface_model(MODEL_NAME)"
+    "load_huggingface_model(sb_config, MODEL_NAME)"
    ]
   },
   {

diff --git a/...dbox/runtime/ansible/roles/jupyter/files/notebook/transformers/token_classification.ipynb b/...dbox/runtime/ansible/roles/jupyter/files/notebook/transformers/token_classification.ipynb
@@ -89,7 +89,7 @@
    "outputs": [],
    "source": [
     "%run utils/model_retrieval.ipynb\n",
-    "load_huggingface_model(MODEL_NAME)"
+    "load_huggingface_model(sb_config, MODEL_NAME)"
    ]
   },
   {

diff --git a/...ol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/translation.ipynb b/...ol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/translation.ipynb
@@ -80,7 +80,6 @@
    "execution_count": null,
    "id": "50a8cc23-bba5-4497-8947-9adc8c83a76c",
    "metadata": {
-    "editable": false,
     "slideshow": {
      "slide_type": ""
     },
@@ -91,7 +90,7 @@
    "outputs": [],
    "source": [
     "%run utils/model_retrieval.ipynb\n",
-    "load_huggingface_model(MODEL_NAME)"
+    "load_huggingface_model(sb_config, MODEL_NAME)"
    ]
   },
   {

diff --git a/...box/runtime/ansible/roles/jupyter/files/notebook/transformers/utils/model_retrieval.ipynb b/...box/runtime/ansible/roles/jupyter/files/notebook/transformers/utils/model_retrieval.ipynb
@@ -36,12 +36,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "def load_huggingface_model_udf(model_name: str) -> None:\n",
+    "from exasol.secret_store import Secrets\n",
+    "\n",
+    "\n",
+    "def load_huggingface_model_udf(conf: Secrets, model_name: str) -> None:\n",
     "    \"\"\"\n",
     "    Loads specified model into BucketFS using a UDF provided with the Transformer Extension.\n",
-    "    There must be a global variable - sb_config - providing access to the secret store holding\n",
-    "    the configuration data.\n",
     "\n",
+    "    conf           - Access to the secret store holding the configuration data.\n",
     "    model_name     - Name of the model at Huggingface hub, e.g. facebook/nllb-moe-54b.\n",
     "    \"\"\"\n",
     "\n",
@@ -50,16 +52,16 @@
     "    from exasol.ai_lab_config import AILabConfig as CKey\n",
     "\n",
     "    sql = f\"\"\"\n",
-    "    SELECT {sb_config.get(CKey.db_schema)}.TE_MODEL_DOWNLOADER_UDF(\n",
+    "    SELECT {conf.get(CKey.db_schema)}.TE_MODEL_DOWNLOADER_UDF(\n",
     "        '{model_name}',\n",
-    "        '{sb_config.get(CKey.te_models_bfs_dir)}',\n",
-    "        '{sb_config.get(CKey.te_bfs_connection)}',\n",
-    "        '{sb_config.get(CKey.te_hf_connection)}'\n",
+    "        '{conf.get(CKey.te_models_bfs_dir)}',\n",
+    "        '{conf.get(CKey.te_bfs_connection)}',\n",
+    "        '{conf.get(CKey.te_hf_connection)}'\n",
     "    )\n",
     "    \"\"\"\n",
     "\n",
-    "    with open_pyexasol_connection(sb_config, compression=True) as conn:\n",
-    "        conn.execute(query=get_activation_sql(sb_config))\n",
+    "    with open_pyexasol_connection(conf, compression=True) as conn:\n",
+    "        conn.execute(query=get_activation_sql(conf))\n",
     "        conn.execute(query=sql)"
    ]
   },
@@ -80,12 +82,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "def load_huggingface_model_cli(model_name: str, force_download: bool = False) -> None:\n",
+    "def load_huggingface_model_cli(conf: Secrets, model_name: str, force_download: bool = False) -> None:\n",
     "    \"\"\"\n",
     "    Loads specified model into BucketFS by saving it first to a local drive, as per the command-line interface.\n",
-    "    There must be a global variable - sb_config - providing access to the secret store holding\n",
-    "    the configuration data.\n",
     "\n",
+    "    conf           - Access to the secret store holding the configuration data.\n",
     "    model_name     - Name of the model at Huggingface hub, e.g. facebook/nllb-moe-54b.\n",
     "    force_download - If True the model will be reloaded from the hub even if it has been cached before.\n",
     "    \"\"\"\n",
@@ -97,11 +98,11 @@
     "\n",
     "    # Make a name for the model sub-directory\n",
     "    sub_dir = re.sub(r\"[/\\\\?%*:|\\\"<>\\x7F\\x00-\\x1F]\", \"-\", model_name)\n",
-    "    models_dir = sb_config.get(CKey.te_models_cache_dir)\n",
+    "    models_dir = conf.get(CKey.te_models_cache_dir)\n",
     "    cache_dir = str(Path(models_dir) / sub_dir)\n",
     "\n",
     "    # Upload it to the BucketFS\n",
-    "    upload_model(sb_config, model_name=model_name, cache_dir=cache_dir, force_download=force_download)"
+    "    upload_model(conf, model_name=model_name, cache_dir=cache_dir, force_download=force_download)"
    ]
   },
   {
@@ -121,10 +122,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "def load_huggingface_model(model_name: str, method: str = 'cli', force_download: bool = False) -> None:\n",
+    "def load_huggingface_model(conf: Secrets, model_name: str, method: str = 'cli', force_download: bool = False) -> None:\n",
     "    \"\"\"\n",
     "    Loads specified model into BucketFS choosing one of the two available methods.\n",
     "\n",
+    "    conf           - Access to the secret store holding the configuration data.\n",
     "    model_name     - Name of the model at Huggingface hub, e.g. facebook/nllb-moe-54b.\n",
     "    method         - The recognized values are \"udf\" and \"cli\". If the parameter is not set then\n",
     "                     will look for method selection in the configuration. That failed the CLI method\n",
@@ -134,9 +136,9 @@
     "\n",
     "    method = method.lower()\n",
     "    if method == 'udf':\n",
-    "        load_huggingface_model_udf(model_name)\n",
+    "        load_huggingface_model_udf(conf, model_name)\n",
     "    else:\n",
-    "        load_huggingface_model_cli(model_name, force_download=force_download)"
+    "        load_huggingface_model_cli(conf, model_name, force_download=force_download)"
    ]
   },
   {

diff --git a/.../sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/utils/te_init_ui.ipynb b/.../sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/utils/te_init_ui.ipynb
@@ -34,16 +34,17 @@
    "source": [
     "import ipywidgets as widgets\n",
     "\n",
+    "from exasol.secret_store import Secrets\n",
     "from exasol.ai_lab_config import AILabConfig as CKey\n",
     "\n",
     "\n",
-    "def get_te_config_ui() -> widgets.Widget:\n",
+    "def get_te_config_ui(conf: Secrets) -> widgets.Widget:\n",
     "\n",
     "    inputs = [\n",
     "        ('Access token', widgets.Password(value=sb_config.get(CKey.huggingface_token)), CKey.huggingface_token)\n",
     "    ]\n",
     "\n",
-    "    return get_generic_config_ui(sb_config, [inputs], ['Huggingface Access Parameters'])\n"
+    "    return get_generic_config_ui(conf, [inputs], ['Huggingface Access Parameters'])\n"
    ]
   }
  ],

diff --git a/.../runtime/ansible/roles/jupyter/files/notebook/transformers/zero_shot_classification.ipynb b/.../runtime/ansible/roles/jupyter/files/notebook/transformers/zero_shot_classification.ipynb
@@ -89,7 +89,7 @@
    "outputs": [],
    "source": [
     "%run utils/model_retrieval.ipynb\n",
-    "load_huggingface_model(MODEL_NAME)"
+    "load_huggingface_model(sb_config, MODEL_NAME)"
    ]
   },
   {