Initialize pytest and convert test_connection

.devcontainer/Dockerfile

@@ -4,7 +4,7 @@ FROM mcr.microsoft.com/devcontainers/python:3.7-bullseye
 RUN \
     apt update && \
     apt-get install bash-completion graphviz default-mysql-client -y && \
-    pip install flake8 black faker ipykernel nose nose-cov datajoint && \
+    pip install flake8 black faker ipykernel pytest pytest-cov nose nose-cov datajoint && \
     pip uninstall datajoint -y
 
 ENV DJ_HOST fakeservices.datajoint.io

.devcontainer/devcontainer.json

@@ -16,7 +16,7 @@
 		"ghcr.io/guiyomh/features/vim:0": {}
 	},
 	"onCreateCommand": "pip install -e .",
-	"postStartCommand": "MYSQL_VER=8.0 MINIO_VER=RELEASE.2022-08-11T04-37-28Z docker compose -f local-docker-compose.yml down && MYSQL_VER=8.0 MINIO_VER=RELEASE.2022-08-11T04-37-28Z docker compose -f local-docker-compose.yml up --build --wait",
+	"postStartCommand": "MYSQL_VER=8.0 MINIO_VER=RELEASE.2022-08-11T04-37-28Z docker compose -f local-docker-compose.yml down && docker volume prune -f && MYSQL_VER=8.0 MINIO_VER=RELEASE.2022-08-11T04-37-28Z docker compose -f local-docker-compose.yml up --build --wait",
 	"forwardPorts": [
 		80,
 		443,

.github/workflows/development.yaml

@@ -92,6 +92,7 @@ jobs:
                  --count --max-complexity=62 --max-line-length=127 --statistics
           black datajoint --check -v
           black tests --check -v
+          black tests_old --check -v
   publish-docs:
     if: |
       github.event_name == 'push' &&

.vscode/settings.json

@@ -11,5 +11,11 @@
     "[markdown]": {
         "editor.defaultFormatter": "disable"
     },
+    "[yaml]": {
+        "editor.defaultFormatter": "disable"
+    },
+    "[dockercompose]": {
+        "editor.defaultFormatter": "disable"
+    },
     "files.autoSave": "off"
 }

CHANGELOG.md

@@ -8,6 +8,7 @@
 - Changed - `.data` method to `.stream` in the `get()` method for S3 (external) objects PR [#1085](https://github.com/datajoint/datajoint-python/pull/1085)
 - Fixed - Docs to rename `create_virtual_module` to `VirtualModule`
 - Added - Skeleton from `datajoint-company/datajoint-docs` repository for docs migration
+- Added - Initial `pytest` for `test_connection`
 
 ### 0.14.0 -- Feb 13, 2023
 - Added - `json` data type ([#245](https://github.com/datajoint/datajoint-python/issues/245)) PR [#1051](https://github.com/datajoint/datajoint-python/pull/1051)

LNX-docker-compose.yml

@@ -89,7 +89,8 @@ services:
         pip install --user nose nose-cov
         pip install -e .
         pip list --format=freeze | grep datajoint
-        nosetests -vsw tests --with-coverage --cover-package=datajoint
+        pytest -sv --cov-report term-missing --cov=datajoint tests
+        nosetests -vsw tests_old --with-coverage --cover-package=datajoint
     # ports:
     #   - "8888:8888"
     user: ${HOST_UID}:anaconda

datajoint/admin.py

@@ -8,9 +8,7 @@
 logger = logging.getLogger(__name__.split(".")[0])
 
 
-def set_password(
-    new_password=None, connection=None, update_config=None
-):  # pragma: no cover
+def set_password(new_password=None, connection=None, update_config=None):
     connection = conn() if connection is None else connection
     if new_password is None:
         new_password = getpass("New password: ")
@@ -28,7 +26,7 @@ def set_password(
         config.save_local(verbose=True)
 
 
-def kill(restriction=None, connection=None, order_by=None):  # pragma: no cover
+def kill(restriction=None, connection=None, order_by=None):
     """
     view and kill database connections.
 

datajoint/connection.py

@@ -120,9 +120,9 @@ def conn(
         host = host if host is not None else config["database.host"]
         user = user if user is not None else config["database.user"]
         password = password if password is not None else config["database.password"]
-        if user is None:  # pragma: no cover
+        if user is None:
             user = input("Please enter DataJoint username: ")
-        if password is None:  # pragma: no cover
+        if password is None:
             password = getpass(prompt="Please enter DataJoint password: ")
         init_fun = (
             init_fun if init_fun is not None else config["connection.init_function"]

datajoint/dependencies.py

@@ -127,7 +127,7 @@ def load(self, force=True):
                 self.add_edge(fk["referenced_table"], alias_node, **props)
                 self.add_edge(alias_node, fk["referencing_table"], **props)
 
-        if not nx.is_directed_acyclic_graph(self):  # pragma: no cover
+        if not nx.is_directed_acyclic_graph(self):
             raise DataJointError("DataJoint can only work with acyclic dependencies")
         self._loaded = True
 

docs/src/develop.md

@@ -39,15 +39,24 @@ The following will verify there are no regression errors by running our test sui
 
 - Entire test suite:
   ```
-  nosetests -vw tests
+  nosetests -vw tests_old
   ```
+  > Note: We are in the process of upgrading to `pytest` tests. To run those, use:
+  > ```
+  > pytest -sv --cov-report term-missing --cov=datajoint tests
+  > ```
+
 - A single functional test:
   ```
-  nosetests -vs --tests=tests.test_external_class:test_insert_and_fetch
+  nosetests -vs --tests=tests_old.test_external_class:test_insert_and_fetch
   ```
+  > Note: We are in the process of upgrading to `pytest` tests. To run those, use:
+  > ```
+  > pytest -sv tests/test_connection.py::test_dj_conn
+  > ```
 - A single class test:
   ```
-  nosetests -vs --tests=tests.test_fetch:TestFetch.test_getattribute_for_fetch1
+  nosetests -vs --tests=tests_old.test_fetch:TestFetch.test_getattribute_for_fetch1
   ```
 
 ### Style Tests

tests/__init__.py

@@ -1,160 +1,69 @@
-"""
-Package for testing datajoint. Setup fixture will be run
-to ensure that proper database connection and access privilege
-exists. The content of the test database will be destroyed
-after the test.
-"""
-
-import logging
-from os import environ, remove
 import datajoint as dj
-from distutils.version import LooseVersion
+from packaging import version
+import pytest
 import os
-from pathlib import Path
-import minio
-import urllib3
-import certifi
-import shutil
-from datajoint.utils import parse_sql
-
-__author__ = "Edgar Walker, Fabian Sinz, Dimitri Yatsenko, Raphael Guzman"
-
-# turn on verbose logging
-logging.basicConfig(level=logging.DEBUG)
 
-__all__ = ["__author__", "PREFIX", "CONN_INFO"]
-
-# Connection for testing
-CONN_INFO = dict(
-    host=environ.get("DJ_TEST_HOST", "fakeservices.datajoint.io"),
-    user=environ.get("DJ_TEST_USER", "datajoint"),
-    password=environ.get("DJ_TEST_PASSWORD", "datajoint"),
-)
+PREFIX = "djtest"
 
 CONN_INFO_ROOT = dict(
-    host=environ.get("DJ_HOST", "fakeservices.datajoint.io"),
-    user=environ.get("DJ_USER", "root"),
-    password=environ.get("DJ_PASS", "simple"),
-)
-
-S3_CONN_INFO = dict(
-    endpoint=environ.get("S3_ENDPOINT", "fakeservices.datajoint.io"),
-    access_key=environ.get("S3_ACCESS_KEY", "datajoint"),
-    secret_key=environ.get("S3_SECRET_KEY", "datajoint"),
-    bucket=environ.get("S3_BUCKET", "datajoint.test"),
-)
-
-# Prefix for all databases used during testing
-PREFIX = environ.get("DJ_TEST_DB_PREFIX", "djtest")
-conn_root = dj.conn(**CONN_INFO_ROOT)
-
-# Initialize httpClient with relevant timeout.
-httpClient = urllib3.PoolManager(
-    timeout=30,
-    cert_reqs="CERT_REQUIRED",
-    ca_certs=certifi.where(),
-    retries=urllib3.Retry(
-        total=3, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504]
-    ),
-)
-
-# Initialize minioClient with an endpoint and access/secret keys.
-minioClient = minio.Minio(
-    S3_CONN_INFO["endpoint"],
-    access_key=S3_CONN_INFO["access_key"],
-    secret_key=S3_CONN_INFO["secret_key"],
-    secure=True,
-    http_client=httpClient,
+    host=os.getenv("DJ_HOST"),
+    user=os.getenv("DJ_USER"),
+    password=os.getenv("DJ_PASS"),
 )
 
 
-def setup_package():
-    """
-    Package-level unit test setup
-    Turns off safemode
-    """
+@pytest.fixture
+def connection_root():
+    """Root user database connection."""
     dj.config["safemode"] = False
+    connection = dj.Connection(
+        host=os.getenv("DJ_HOST"),
+        user=os.getenv("DJ_USER"),
+        password=os.getenv("DJ_PASS"),
+    )
+    yield connection
+    dj.config["safemode"] = True
+    connection.close()
+
+
+@pytest.fixture
+def connection_test(connection_root):
+    """Test user database connection."""
+    database = f"{PREFIX}%%"
+    credentials = dict(
+        host=os.getenv("DJ_HOST"), user="datajoint", password="datajoint"
+    )
+    permission = "ALL PRIVILEGES"
 
     # Create MySQL users
-    if LooseVersion(conn_root.query("select @@version;").fetchone()[0]) >= LooseVersion(
-        "8.0.0"
-    ):
+    if version.parse(
+        connection_root.query("select @@version;").fetchone()[0]
+    ) >= version.parse("8.0.0"):
         # create user if necessary on mysql8
-        conn_root.query(
+        connection_root.query(
+            f"""
+            CREATE USER IF NOT EXISTS '{credentials["user"]}'@'%%'
+            IDENTIFIED BY '{credentials["password"]}';
             """
-                CREATE USER IF NOT EXISTS 'datajoint'@'%%'
-                IDENTIFIED BY 'datajoint';
-                """
         )
-        conn_root.query(
+        connection_root.query(
+            f"""
+            GRANT {permission} ON `{database}`.*
+            TO '{credentials["user"]}'@'%%';
             """
-                CREATE USER IF NOT EXISTS 'djview'@'%%'
-                IDENTIFIED BY 'djview';
-                """
         )
-        conn_root.query(
-            """
-                CREATE USER IF NOT EXISTS 'djssl'@'%%'
-                IDENTIFIED BY 'djssl'
-                REQUIRE SSL;
-                """
-        )
-        conn_root.query("GRANT ALL PRIVILEGES ON `djtest%%`.* TO 'datajoint'@'%%';")
-        conn_root.query("GRANT SELECT ON `djtest%%`.* TO 'djview'@'%%';")
-        conn_root.query("GRANT SELECT ON `djtest%%`.* TO 'djssl'@'%%';")
     else:
         # grant permissions. For MySQL 5.7 this also automatically creates user
         # if not exists
-        conn_root.query(
-            """
-            GRANT ALL PRIVILEGES ON `djtest%%`.* TO 'datajoint'@'%%'
-            IDENTIFIED BY 'datajoint';
+        connection_root.query(
+            f"""
+            GRANT {permission} ON `{database}`.*
+            TO '{credentials["user"]}'@'%%'
+            IDENTIFIED BY '{credentials["password"]}';
             """
         )
-        conn_root.query(
-            "GRANT SELECT ON `djtest%%`.* TO 'djview'@'%%' IDENTIFIED BY 'djview';"
-        )
-        conn_root.query(
-            """
-            GRANT SELECT ON `djtest%%`.* TO 'djssl'@'%%'
-            IDENTIFIED BY 'djssl'
-            REQUIRE SSL;
-            """
-        )
-
-    region = "us-east-1"
-    # Add S3
-    try:
-        minioClient.make_bucket(S3_CONN_INFO["bucket"], location=region)
-    except minio.error.S3Error as e:
-        if e.code != "BucketAlreadyOwnedByYou":
-            raise e
-
-
-def teardown_package():
-    """
-    Package-level unit test teardown.
-    Removes all databases with name starting with PREFIX.
-    To deal with possible foreign key constraints, it will unset
-    and then later reset FOREIGN_KEY_CHECKS flag
-    """
-    conn_root.query("SET FOREIGN_KEY_CHECKS=0")
-    cur = conn_root.query('SHOW DATABASES LIKE "{}\_%%"'.format(PREFIX))
-    for db in cur.fetchall():
-        conn_root.query("DROP DATABASE `{}`".format(db[0]))
-    conn_root.query("SET FOREIGN_KEY_CHECKS=1")
-    if os.path.exists("dj_local_conf.json"):
-        remove("dj_local_conf.json")
-
-    # Remove created users
-    conn_root.query("DROP USER `datajoint`")
-    conn_root.query("DROP USER `djview`")
-    conn_root.query("DROP USER `djssl`")
 
-    # Remove S3
-    objs = list(minioClient.list_objects(S3_CONN_INFO["bucket"], recursive=True))
-    objs = [
-        minioClient.remove_object(S3_CONN_INFO["bucket"], o.object_name.encode("utf-8"))
-        for o in objs
-    ]
-    minioClient.remove_bucket(S3_CONN_INFO["bucket"])
+    connection = dj.Connection(**credentials)
+    yield connection
+    connection_root.query(f"""DROP USER `{credentials["user"]}`""")
+    connection.close()