diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index c965828c6..e008c9287 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -4,7 +4,7 @@ FROM mcr.microsoft.com/devcontainers/python:3.7-bullseye RUN \ apt update && \ apt-get install bash-completion graphviz default-mysql-client -y && \ - pip install flake8 black faker ipykernel nose nose-cov datajoint && \ + pip install flake8 black faker ipykernel pytest pytest-cov nose nose-cov datajoint && \ pip uninstall datajoint -y ENV DJ_HOST fakeservices.datajoint.io diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 668e5c409..a5db4d4c5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -16,7 +16,7 @@ "ghcr.io/guiyomh/features/vim:0": {} }, "onCreateCommand": "pip install -e .", - "postStartCommand": "MYSQL_VER=8.0 MINIO_VER=RELEASE.2022-08-11T04-37-28Z docker compose -f local-docker-compose.yml down && MYSQL_VER=8.0 MINIO_VER=RELEASE.2022-08-11T04-37-28Z docker compose -f local-docker-compose.yml up --build --wait", + "postStartCommand": "MYSQL_VER=8.0 MINIO_VER=RELEASE.2022-08-11T04-37-28Z docker compose -f local-docker-compose.yml down && docker volume prune -f && MYSQL_VER=8.0 MINIO_VER=RELEASE.2022-08-11T04-37-28Z docker compose -f local-docker-compose.yml up --build --wait", "forwardPorts": [ 80, 443, diff --git a/.github/workflows/development.yaml b/.github/workflows/development.yaml index 524f066ad..1f3ed1342 100644 --- a/.github/workflows/development.yaml +++ b/.github/workflows/development.yaml @@ -92,6 +92,7 @@ jobs: --count --max-complexity=62 --max-line-length=127 --statistics black datajoint --check -v black tests --check -v + black tests_old --check -v publish-docs: if: | github.event_name == 'push' && diff --git a/.vscode/settings.json b/.vscode/settings.json index 83eebdcc9..efb8c58b5 100755 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,5 +11,11 @@ "[markdown]": { "editor.defaultFormatter": "disable" }, + "[yaml]": { + "editor.defaultFormatter": "disable" + }, + "[dockercompose]": { + "editor.defaultFormatter": "disable" + }, "files.autoSave": "off" } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index d683ef2f3..54a52a241 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Changed - `.data` method to `.stream` in the `get()` method for S3 (external) objects PR [#1085](https://github.com/datajoint/datajoint-python/pull/1085) - Fixed - Docs to rename `create_virtual_module` to `VirtualModule` - Added - Skeleton from `datajoint-company/datajoint-docs` repository for docs migration +- Added - Initial `pytest` for `test_connection` ### 0.14.0 -- Feb 13, 2023 - Added - `json` data type ([#245](https://github.com/datajoint/datajoint-python/issues/245)) PR [#1051](https://github.com/datajoint/datajoint-python/pull/1051) diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml index e1051b4be..970552860 100644 --- a/LNX-docker-compose.yml +++ b/LNX-docker-compose.yml @@ -89,7 +89,8 @@ services: pip install --user nose nose-cov pip install -e . pip list --format=freeze | grep datajoint - nosetests -vsw tests --with-coverage --cover-package=datajoint + pytest -sv --cov-report term-missing --cov=datajoint tests + nosetests -vsw tests_old --with-coverage --cover-package=datajoint # ports: # - "8888:8888" user: ${HOST_UID}:anaconda diff --git a/datajoint/admin.py b/datajoint/admin.py index 3b3933300..ae045667f 100644 --- a/datajoint/admin.py +++ b/datajoint/admin.py @@ -8,9 +8,7 @@ logger = logging.getLogger(__name__.split(".")[0]) -def set_password( - new_password=None, connection=None, update_config=None -): # pragma: no cover +def set_password(new_password=None, connection=None, update_config=None): connection = conn() if connection is None else connection if new_password is None: new_password = getpass("New password: ") @@ -28,7 +26,7 @@ def set_password( config.save_local(verbose=True) -def kill(restriction=None, connection=None, order_by=None): # pragma: no cover +def kill(restriction=None, connection=None, order_by=None): """ view and kill database connections. diff --git a/datajoint/connection.py b/datajoint/connection.py index 565015bfd..0b64199c5 100644 --- a/datajoint/connection.py +++ b/datajoint/connection.py @@ -120,9 +120,9 @@ def conn( host = host if host is not None else config["database.host"] user = user if user is not None else config["database.user"] password = password if password is not None else config["database.password"] - if user is None: # pragma: no cover + if user is None: user = input("Please enter DataJoint username: ") - if password is None: # pragma: no cover + if password is None: password = getpass(prompt="Please enter DataJoint password: ") init_fun = ( init_fun if init_fun is not None else config["connection.init_function"] diff --git a/datajoint/dependencies.py b/datajoint/dependencies.py index 96dc8f7f4..d9c425d49 100644 --- a/datajoint/dependencies.py +++ b/datajoint/dependencies.py @@ -127,7 +127,7 @@ def load(self, force=True): self.add_edge(fk["referenced_table"], alias_node, **props) self.add_edge(alias_node, fk["referencing_table"], **props) - if not nx.is_directed_acyclic_graph(self): # pragma: no cover + if not nx.is_directed_acyclic_graph(self): raise DataJointError("DataJoint can only work with acyclic dependencies") self._loaded = True diff --git a/docs/src/develop.md b/docs/src/develop.md index 41ed12c5d..e577d4be5 100644 --- a/docs/src/develop.md +++ b/docs/src/develop.md @@ -39,15 +39,24 @@ The following will verify there are no regression errors by running our test sui - Entire test suite: ``` - nosetests -vw tests + nosetests -vw tests_old ``` + > Note: We are in the process of upgrading to `pytest` tests. To run those, use: + > ``` + > pytest -sv --cov-report term-missing --cov=datajoint tests + > ``` + - A single functional test: ``` - nosetests -vs --tests=tests.test_external_class:test_insert_and_fetch + nosetests -vs --tests=tests_old.test_external_class:test_insert_and_fetch ``` + > Note: We are in the process of upgrading to `pytest` tests. To run those, use: + > ``` + > pytest -sv tests/test_connection.py::test_dj_conn + > ``` - A single class test: ``` - nosetests -vs --tests=tests.test_fetch:TestFetch.test_getattribute_for_fetch1 + nosetests -vs --tests=tests_old.test_fetch:TestFetch.test_getattribute_for_fetch1 ``` ### Style Tests diff --git a/tests/__init__.py b/tests/__init__.py index 5211278e3..8b825a042 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,160 +1,69 @@ -""" -Package for testing datajoint. Setup fixture will be run -to ensure that proper database connection and access privilege -exists. The content of the test database will be destroyed -after the test. -""" - -import logging -from os import environ, remove import datajoint as dj -from distutils.version import LooseVersion +from packaging import version +import pytest import os -from pathlib import Path -import minio -import urllib3 -import certifi -import shutil -from datajoint.utils import parse_sql - -__author__ = "Edgar Walker, Fabian Sinz, Dimitri Yatsenko, Raphael Guzman" - -# turn on verbose logging -logging.basicConfig(level=logging.DEBUG) -__all__ = ["__author__", "PREFIX", "CONN_INFO"] - -# Connection for testing -CONN_INFO = dict( - host=environ.get("DJ_TEST_HOST", "fakeservices.datajoint.io"), - user=environ.get("DJ_TEST_USER", "datajoint"), - password=environ.get("DJ_TEST_PASSWORD", "datajoint"), -) +PREFIX = "djtest" CONN_INFO_ROOT = dict( - host=environ.get("DJ_HOST", "fakeservices.datajoint.io"), - user=environ.get("DJ_USER", "root"), - password=environ.get("DJ_PASS", "simple"), -) - -S3_CONN_INFO = dict( - endpoint=environ.get("S3_ENDPOINT", "fakeservices.datajoint.io"), - access_key=environ.get("S3_ACCESS_KEY", "datajoint"), - secret_key=environ.get("S3_SECRET_KEY", "datajoint"), - bucket=environ.get("S3_BUCKET", "datajoint.test"), -) - -# Prefix for all databases used during testing -PREFIX = environ.get("DJ_TEST_DB_PREFIX", "djtest") -conn_root = dj.conn(**CONN_INFO_ROOT) - -# Initialize httpClient with relevant timeout. -httpClient = urllib3.PoolManager( - timeout=30, - cert_reqs="CERT_REQUIRED", - ca_certs=certifi.where(), - retries=urllib3.Retry( - total=3, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504] - ), -) - -# Initialize minioClient with an endpoint and access/secret keys. -minioClient = minio.Minio( - S3_CONN_INFO["endpoint"], - access_key=S3_CONN_INFO["access_key"], - secret_key=S3_CONN_INFO["secret_key"], - secure=True, - http_client=httpClient, + host=os.getenv("DJ_HOST"), + user=os.getenv("DJ_USER"), + password=os.getenv("DJ_PASS"), ) -def setup_package(): - """ - Package-level unit test setup - Turns off safemode - """ +@pytest.fixture +def connection_root(): + """Root user database connection.""" dj.config["safemode"] = False + connection = dj.Connection( + host=os.getenv("DJ_HOST"), + user=os.getenv("DJ_USER"), + password=os.getenv("DJ_PASS"), + ) + yield connection + dj.config["safemode"] = True + connection.close() + + +@pytest.fixture +def connection_test(connection_root): + """Test user database connection.""" + database = f"{PREFIX}%%" + credentials = dict( + host=os.getenv("DJ_HOST"), user="datajoint", password="datajoint" + ) + permission = "ALL PRIVILEGES" # Create MySQL users - if LooseVersion(conn_root.query("select @@version;").fetchone()[0]) >= LooseVersion( - "8.0.0" - ): + if version.parse( + connection_root.query("select @@version;").fetchone()[0] + ) >= version.parse("8.0.0"): # create user if necessary on mysql8 - conn_root.query( + connection_root.query( + f""" + CREATE USER IF NOT EXISTS '{credentials["user"]}'@'%%' + IDENTIFIED BY '{credentials["password"]}'; """ - CREATE USER IF NOT EXISTS 'datajoint'@'%%' - IDENTIFIED BY 'datajoint'; - """ ) - conn_root.query( + connection_root.query( + f""" + GRANT {permission} ON `{database}`.* + TO '{credentials["user"]}'@'%%'; """ - CREATE USER IF NOT EXISTS 'djview'@'%%' - IDENTIFIED BY 'djview'; - """ ) - conn_root.query( - """ - CREATE USER IF NOT EXISTS 'djssl'@'%%' - IDENTIFIED BY 'djssl' - REQUIRE SSL; - """ - ) - conn_root.query("GRANT ALL PRIVILEGES ON `djtest%%`.* TO 'datajoint'@'%%';") - conn_root.query("GRANT SELECT ON `djtest%%`.* TO 'djview'@'%%';") - conn_root.query("GRANT SELECT ON `djtest%%`.* TO 'djssl'@'%%';") else: # grant permissions. For MySQL 5.7 this also automatically creates user # if not exists - conn_root.query( - """ - GRANT ALL PRIVILEGES ON `djtest%%`.* TO 'datajoint'@'%%' - IDENTIFIED BY 'datajoint'; + connection_root.query( + f""" + GRANT {permission} ON `{database}`.* + TO '{credentials["user"]}'@'%%' + IDENTIFIED BY '{credentials["password"]}'; """ ) - conn_root.query( - "GRANT SELECT ON `djtest%%`.* TO 'djview'@'%%' IDENTIFIED BY 'djview';" - ) - conn_root.query( - """ - GRANT SELECT ON `djtest%%`.* TO 'djssl'@'%%' - IDENTIFIED BY 'djssl' - REQUIRE SSL; - """ - ) - - region = "us-east-1" - # Add S3 - try: - minioClient.make_bucket(S3_CONN_INFO["bucket"], location=region) - except minio.error.S3Error as e: - if e.code != "BucketAlreadyOwnedByYou": - raise e - - -def teardown_package(): - """ - Package-level unit test teardown. - Removes all databases with name starting with PREFIX. - To deal with possible foreign key constraints, it will unset - and then later reset FOREIGN_KEY_CHECKS flag - """ - conn_root.query("SET FOREIGN_KEY_CHECKS=0") - cur = conn_root.query('SHOW DATABASES LIKE "{}\_%%"'.format(PREFIX)) - for db in cur.fetchall(): - conn_root.query("DROP DATABASE `{}`".format(db[0])) - conn_root.query("SET FOREIGN_KEY_CHECKS=1") - if os.path.exists("dj_local_conf.json"): - remove("dj_local_conf.json") - - # Remove created users - conn_root.query("DROP USER `datajoint`") - conn_root.query("DROP USER `djview`") - conn_root.query("DROP USER `djssl`") - # Remove S3 - objs = list(minioClient.list_objects(S3_CONN_INFO["bucket"], recursive=True)) - objs = [ - minioClient.remove_object(S3_CONN_INFO["bucket"], o.object_name.encode("utf-8")) - for o in objs - ] - minioClient.remove_bucket(S3_CONN_INFO["bucket"]) + connection = dj.Connection(**credentials) + yield connection + connection_root.query(f"""DROP USER `{credentials["user"]}`""") + connection.close() diff --git a/tests/test_connection.py b/tests/test_connection.py index 8ac63fb15..1916da951 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -2,27 +2,51 @@ Collection of test cases to test connection module. """ -from nose.tools import assert_true, assert_equal import datajoint as dj -import numpy as np from datajoint import DataJointError -from . import CONN_INFO, PREFIX +import numpy as np +from . import CONN_INFO_ROOT, connection_root, connection_test + +from . import PREFIX +import pytest + + +@pytest.fixture +def schema(connection_test): + schema = dj.Schema(PREFIX + "_transactions", locals(), connection=connection_test) + yield schema + schema.drop() + + +@pytest.fixture +def Subjects(schema): + @schema + class Subjects(dj.Manual): + definition = """ + #Basic subject + subject_id : int # unique subject id + --- + real_id : varchar(40) # real-world name + species = "mouse" : enum('mouse', 'monkey', 'human') # species + """ + + yield Subjects + Subjects.drop() def test_dj_conn(): """ - Should be able to establish a connection + Should be able to establish a connection as root user """ - c = dj.conn(**CONN_INFO) - assert_true(c.is_connected) + c = dj.conn(**CONN_INFO_ROOT) + assert c.is_connected -def test_dj_connection_class(): +def test_dj_connection_class(connection_test): """ - Should be able to establish a connection + Should be able to establish a connection as test user """ - c = dj.Connection(**CONN_INFO) - assert_true(c.is_connected) + assert connection_test.is_connected def test_persistent_dj_conn(): @@ -30,98 +54,66 @@ def test_persistent_dj_conn(): conn() method should provide persistent connection across calls. Setting reset=True should create a new persistent connection. """ - c1 = dj.conn(**CONN_INFO) + c1 = dj.conn(**CONN_INFO_ROOT) c2 = dj.conn() - c3 = dj.conn(**CONN_INFO) - c4 = dj.conn(reset=True, **CONN_INFO) - c5 = dj.conn(**CONN_INFO) - assert_true(c1 is c2) - assert_true(c1 is c3) - assert_true(c1 is not c4) - assert_true(c4 is c5) + c3 = dj.conn(**CONN_INFO_ROOT) + c4 = dj.conn(reset=True, **CONN_INFO_ROOT) + c5 = dj.conn(**CONN_INFO_ROOT) + assert c1 is c2 + assert c1 is c3 + assert c1 is not c4 + assert c4 is c5 def test_repr(): - c1 = dj.conn(**CONN_INFO) - assert_true("disconnected" not in repr(c1) and "connected" in repr(c1)) + c1 = dj.conn(**CONN_INFO_ROOT) + assert "disconnected" not in repr(c1) and "connected" in repr(c1) -class TestTransactions: - """ - test transaction management - """ +def test_active(connection_test): + with connection_test.transaction as conn: + assert conn.in_transaction, "Transaction is not active" - schema = dj.Schema( - PREFIX + "_transactions", locals(), connection=dj.conn(**CONN_INFO) - ) - @schema - class Subjects(dj.Manual): - definition = """ - #Basic subject - subject_id : int # unique subject id - --- - real_id : varchar(40) # real-world name - species = "mouse" : enum('mouse', 'monkey', 'human') # species - """ +def test_transaction_rollback(connection_test, Subjects): + """Test transaction cancellation using a with statement""" + tmp = np.array( + [(1, "Peter", "mouse"), (2, "Klara", "monkey")], + Subjects.heading.as_dtype, + ) - @classmethod - def setup_class(cls): - cls.table = cls.Subjects() - cls.conn = dj.conn(**CONN_INFO) - - def teardown(self): - self.table.delete_quick() - - def test_active(self): - with self.conn.transaction as conn: - assert_true(conn.in_transaction, "Transaction is not active") - - def test_transaction_rollback(self): - """Test transaction cancellation using a with statement""" - tmp = np.array( - [(1, "Peter", "mouse"), (2, "Klara", "monkey")], - self.table.heading.as_dtype, - ) - - self.table.delete() - with self.conn.transaction: - self.table.insert1(tmp[0]) - try: - with self.conn.transaction: - self.table.insert1(tmp[1]) - raise DataJointError("Testing rollback") - except DataJointError: - pass - assert_equal( - len(self.table), - 1, - "Length is not 1. Expected because rollback should have happened.", - ) - assert_equal( - len(self.table & "subject_id = 2"), - 0, - "Length is not 0. Expected because rollback should have happened.", - ) - - def test_cancel(self): - """Tests cancelling a transaction explicitly""" - tmp = np.array( - [(1, "Peter", "mouse"), (2, "Klara", "monkey")], - self.table.heading.as_dtype, - ) - self.table.delete_quick() - self.table.insert1(tmp[0]) - self.conn.start_transaction() - self.table.insert1(tmp[1]) - self.conn.cancel_transaction() - assert_equal( - len(self.table), - 1, - "Length is not 1. Expected because rollback should have happened.", - ) - assert_equal( - len(self.table & "subject_id = 2"), - 0, - "Length is not 0. Expected because rollback should have happened.", - ) + Subjects.delete() + with connection_test.transaction: + Subjects.insert1(tmp[0]) + try: + with connection_test.transaction: + Subjects.insert1(tmp[1]) + raise DataJointError("Testing rollback") + except DataJointError: + pass + assert ( + len(Subjects()) == 1 + ), "Length is not 1. Expected because rollback should have happened." + + assert ( + len(Subjects & "subject_id = 2") == 0 + ), "Length is not 0. Expected because rollback should have happened." + + +def test_cancel(connection_test, Subjects): + """Tests cancelling a transaction explicitly""" + tmp = np.array( + [(1, "Peter", "mouse"), (2, "Klara", "monkey")], + Subjects.heading.as_dtype, + ) + Subjects.delete_quick() + Subjects.insert1(tmp[0]) + connection_test.start_transaction() + Subjects.insert1(tmp[1]) + connection_test.cancel_transaction() + assert ( + len(Subjects()) == 1 + ), "Length is not 1. Expected because rollback should have happened." + assert ( + len(Subjects & "subject_id = 2") == 0 + ), "Length is not 0. Expected because rollback should have happened." diff --git a/tests_old/__init__.py b/tests_old/__init__.py new file mode 100644 index 000000000..5211278e3 --- /dev/null +++ b/tests_old/__init__.py @@ -0,0 +1,160 @@ +""" +Package for testing datajoint. Setup fixture will be run +to ensure that proper database connection and access privilege +exists. The content of the test database will be destroyed +after the test. +""" + +import logging +from os import environ, remove +import datajoint as dj +from distutils.version import LooseVersion +import os +from pathlib import Path +import minio +import urllib3 +import certifi +import shutil +from datajoint.utils import parse_sql + +__author__ = "Edgar Walker, Fabian Sinz, Dimitri Yatsenko, Raphael Guzman" + +# turn on verbose logging +logging.basicConfig(level=logging.DEBUG) + +__all__ = ["__author__", "PREFIX", "CONN_INFO"] + +# Connection for testing +CONN_INFO = dict( + host=environ.get("DJ_TEST_HOST", "fakeservices.datajoint.io"), + user=environ.get("DJ_TEST_USER", "datajoint"), + password=environ.get("DJ_TEST_PASSWORD", "datajoint"), +) + +CONN_INFO_ROOT = dict( + host=environ.get("DJ_HOST", "fakeservices.datajoint.io"), + user=environ.get("DJ_USER", "root"), + password=environ.get("DJ_PASS", "simple"), +) + +S3_CONN_INFO = dict( + endpoint=environ.get("S3_ENDPOINT", "fakeservices.datajoint.io"), + access_key=environ.get("S3_ACCESS_KEY", "datajoint"), + secret_key=environ.get("S3_SECRET_KEY", "datajoint"), + bucket=environ.get("S3_BUCKET", "datajoint.test"), +) + +# Prefix for all databases used during testing +PREFIX = environ.get("DJ_TEST_DB_PREFIX", "djtest") +conn_root = dj.conn(**CONN_INFO_ROOT) + +# Initialize httpClient with relevant timeout. +httpClient = urllib3.PoolManager( + timeout=30, + cert_reqs="CERT_REQUIRED", + ca_certs=certifi.where(), + retries=urllib3.Retry( + total=3, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504] + ), +) + +# Initialize minioClient with an endpoint and access/secret keys. +minioClient = minio.Minio( + S3_CONN_INFO["endpoint"], + access_key=S3_CONN_INFO["access_key"], + secret_key=S3_CONN_INFO["secret_key"], + secure=True, + http_client=httpClient, +) + + +def setup_package(): + """ + Package-level unit test setup + Turns off safemode + """ + dj.config["safemode"] = False + + # Create MySQL users + if LooseVersion(conn_root.query("select @@version;").fetchone()[0]) >= LooseVersion( + "8.0.0" + ): + # create user if necessary on mysql8 + conn_root.query( + """ + CREATE USER IF NOT EXISTS 'datajoint'@'%%' + IDENTIFIED BY 'datajoint'; + """ + ) + conn_root.query( + """ + CREATE USER IF NOT EXISTS 'djview'@'%%' + IDENTIFIED BY 'djview'; + """ + ) + conn_root.query( + """ + CREATE USER IF NOT EXISTS 'djssl'@'%%' + IDENTIFIED BY 'djssl' + REQUIRE SSL; + """ + ) + conn_root.query("GRANT ALL PRIVILEGES ON `djtest%%`.* TO 'datajoint'@'%%';") + conn_root.query("GRANT SELECT ON `djtest%%`.* TO 'djview'@'%%';") + conn_root.query("GRANT SELECT ON `djtest%%`.* TO 'djssl'@'%%';") + else: + # grant permissions. For MySQL 5.7 this also automatically creates user + # if not exists + conn_root.query( + """ + GRANT ALL PRIVILEGES ON `djtest%%`.* TO 'datajoint'@'%%' + IDENTIFIED BY 'datajoint'; + """ + ) + conn_root.query( + "GRANT SELECT ON `djtest%%`.* TO 'djview'@'%%' IDENTIFIED BY 'djview';" + ) + conn_root.query( + """ + GRANT SELECT ON `djtest%%`.* TO 'djssl'@'%%' + IDENTIFIED BY 'djssl' + REQUIRE SSL; + """ + ) + + region = "us-east-1" + # Add S3 + try: + minioClient.make_bucket(S3_CONN_INFO["bucket"], location=region) + except minio.error.S3Error as e: + if e.code != "BucketAlreadyOwnedByYou": + raise e + + +def teardown_package(): + """ + Package-level unit test teardown. + Removes all databases with name starting with PREFIX. + To deal with possible foreign key constraints, it will unset + and then later reset FOREIGN_KEY_CHECKS flag + """ + conn_root.query("SET FOREIGN_KEY_CHECKS=0") + cur = conn_root.query('SHOW DATABASES LIKE "{}\_%%"'.format(PREFIX)) + for db in cur.fetchall(): + conn_root.query("DROP DATABASE `{}`".format(db[0])) + conn_root.query("SET FOREIGN_KEY_CHECKS=1") + if os.path.exists("dj_local_conf.json"): + remove("dj_local_conf.json") + + # Remove created users + conn_root.query("DROP USER `datajoint`") + conn_root.query("DROP USER `djview`") + conn_root.query("DROP USER `djssl`") + + # Remove S3 + objs = list(minioClient.list_objects(S3_CONN_INFO["bucket"], recursive=True)) + objs = [ + minioClient.remove_object(S3_CONN_INFO["bucket"], o.object_name.encode("utf-8")) + for o in objs + ] + minioClient.remove_bucket(S3_CONN_INFO["bucket"]) diff --git a/tests/data/Course.csv b/tests_old/data/Course.csv similarity index 100% rename from tests/data/Course.csv rename to tests_old/data/Course.csv diff --git a/tests/data/CurrentTerm.csv b/tests_old/data/CurrentTerm.csv similarity index 100% rename from tests/data/CurrentTerm.csv rename to tests_old/data/CurrentTerm.csv diff --git a/tests/data/Department.csv b/tests_old/data/Department.csv similarity index 100% rename from tests/data/Department.csv rename to tests_old/data/Department.csv diff --git a/tests/data/Enroll.csv b/tests_old/data/Enroll.csv similarity index 100% rename from tests/data/Enroll.csv rename to tests_old/data/Enroll.csv diff --git a/tests/data/Grade.csv b/tests_old/data/Grade.csv similarity index 100% rename from tests/data/Grade.csv rename to tests_old/data/Grade.csv diff --git a/tests/data/Section.csv b/tests_old/data/Section.csv similarity index 100% rename from tests/data/Section.csv rename to tests_old/data/Section.csv diff --git a/tests/data/Student.csv b/tests_old/data/Student.csv similarity index 100% rename from tests/data/Student.csv rename to tests_old/data/Student.csv diff --git a/tests/data/StudentMajor.csv b/tests_old/data/StudentMajor.csv similarity index 100% rename from tests/data/StudentMajor.csv rename to tests_old/data/StudentMajor.csv diff --git a/tests/data/Term.csv b/tests_old/data/Term.csv similarity index 100% rename from tests/data/Term.csv rename to tests_old/data/Term.csv diff --git a/tests/schema.py b/tests_old/schema.py similarity index 100% rename from tests/schema.py rename to tests_old/schema.py diff --git a/tests/schema_adapted.py b/tests_old/schema_adapted.py similarity index 100% rename from tests/schema_adapted.py rename to tests_old/schema_adapted.py diff --git a/tests/schema_advanced.py b/tests_old/schema_advanced.py similarity index 100% rename from tests/schema_advanced.py rename to tests_old/schema_advanced.py diff --git a/tests/schema_empty.py b/tests_old/schema_empty.py similarity index 100% rename from tests/schema_empty.py rename to tests_old/schema_empty.py diff --git a/tests/schema_external.py b/tests_old/schema_external.py similarity index 100% rename from tests/schema_external.py rename to tests_old/schema_external.py diff --git a/tests/schema_privileges.py b/tests_old/schema_privileges.py similarity index 100% rename from tests/schema_privileges.py rename to tests_old/schema_privileges.py diff --git a/tests/schema_simple.py b/tests_old/schema_simple.py similarity index 100% rename from tests/schema_simple.py rename to tests_old/schema_simple.py diff --git a/tests/schema_university.py b/tests_old/schema_university.py similarity index 100% rename from tests/schema_university.py rename to tests_old/schema_university.py diff --git a/tests/schema_uuid.py b/tests_old/schema_uuid.py similarity index 100% rename from tests/schema_uuid.py rename to tests_old/schema_uuid.py diff --git a/tests/test_adapted_attributes.py b/tests_old/test_adapted_attributes.py similarity index 100% rename from tests/test_adapted_attributes.py rename to tests_old/test_adapted_attributes.py diff --git a/tests/test_aggr_regressions.py b/tests_old/test_aggr_regressions.py similarity index 100% rename from tests/test_aggr_regressions.py rename to tests_old/test_aggr_regressions.py diff --git a/tests/test_alter.py b/tests_old/test_alter.py similarity index 100% rename from tests/test_alter.py rename to tests_old/test_alter.py diff --git a/tests/test_attach.py b/tests_old/test_attach.py similarity index 100% rename from tests/test_attach.py rename to tests_old/test_attach.py diff --git a/tests/test_autopopulate.py b/tests_old/test_autopopulate.py similarity index 100% rename from tests/test_autopopulate.py rename to tests_old/test_autopopulate.py diff --git a/tests/test_blob.py b/tests_old/test_blob.py similarity index 100% rename from tests/test_blob.py rename to tests_old/test_blob.py diff --git a/tests/test_blob_matlab.py b/tests_old/test_blob_matlab.py similarity index 100% rename from tests/test_blob_matlab.py rename to tests_old/test_blob_matlab.py diff --git a/tests/test_bypass_serialization.py b/tests_old/test_bypass_serialization.py similarity index 100% rename from tests/test_bypass_serialization.py rename to tests_old/test_bypass_serialization.py diff --git a/tests/test_cascading_delete.py b/tests_old/test_cascading_delete.py similarity index 100% rename from tests/test_cascading_delete.py rename to tests_old/test_cascading_delete.py diff --git a/tests_old/test_connection.py b/tests_old/test_connection.py new file mode 100644 index 000000000..8ac63fb15 --- /dev/null +++ b/tests_old/test_connection.py @@ -0,0 +1,127 @@ +""" +Collection of test cases to test connection module. +""" + +from nose.tools import assert_true, assert_equal +import datajoint as dj +import numpy as np +from datajoint import DataJointError +from . import CONN_INFO, PREFIX + + +def test_dj_conn(): + """ + Should be able to establish a connection + """ + c = dj.conn(**CONN_INFO) + assert_true(c.is_connected) + + +def test_dj_connection_class(): + """ + Should be able to establish a connection + """ + c = dj.Connection(**CONN_INFO) + assert_true(c.is_connected) + + +def test_persistent_dj_conn(): + """ + conn() method should provide persistent connection across calls. + Setting reset=True should create a new persistent connection. + """ + c1 = dj.conn(**CONN_INFO) + c2 = dj.conn() + c3 = dj.conn(**CONN_INFO) + c4 = dj.conn(reset=True, **CONN_INFO) + c5 = dj.conn(**CONN_INFO) + assert_true(c1 is c2) + assert_true(c1 is c3) + assert_true(c1 is not c4) + assert_true(c4 is c5) + + +def test_repr(): + c1 = dj.conn(**CONN_INFO) + assert_true("disconnected" not in repr(c1) and "connected" in repr(c1)) + + +class TestTransactions: + """ + test transaction management + """ + + schema = dj.Schema( + PREFIX + "_transactions", locals(), connection=dj.conn(**CONN_INFO) + ) + + @schema + class Subjects(dj.Manual): + definition = """ + #Basic subject + subject_id : int # unique subject id + --- + real_id : varchar(40) # real-world name + species = "mouse" : enum('mouse', 'monkey', 'human') # species + """ + + @classmethod + def setup_class(cls): + cls.table = cls.Subjects() + cls.conn = dj.conn(**CONN_INFO) + + def teardown(self): + self.table.delete_quick() + + def test_active(self): + with self.conn.transaction as conn: + assert_true(conn.in_transaction, "Transaction is not active") + + def test_transaction_rollback(self): + """Test transaction cancellation using a with statement""" + tmp = np.array( + [(1, "Peter", "mouse"), (2, "Klara", "monkey")], + self.table.heading.as_dtype, + ) + + self.table.delete() + with self.conn.transaction: + self.table.insert1(tmp[0]) + try: + with self.conn.transaction: + self.table.insert1(tmp[1]) + raise DataJointError("Testing rollback") + except DataJointError: + pass + assert_equal( + len(self.table), + 1, + "Length is not 1. Expected because rollback should have happened.", + ) + assert_equal( + len(self.table & "subject_id = 2"), + 0, + "Length is not 0. Expected because rollback should have happened.", + ) + + def test_cancel(self): + """Tests cancelling a transaction explicitly""" + tmp = np.array( + [(1, "Peter", "mouse"), (2, "Klara", "monkey")], + self.table.heading.as_dtype, + ) + self.table.delete_quick() + self.table.insert1(tmp[0]) + self.conn.start_transaction() + self.table.insert1(tmp[1]) + self.conn.cancel_transaction() + assert_equal( + len(self.table), + 1, + "Length is not 1. Expected because rollback should have happened.", + ) + assert_equal( + len(self.table & "subject_id = 2"), + 0, + "Length is not 0. Expected because rollback should have happened.", + ) diff --git a/tests/test_declare.py b/tests_old/test_declare.py similarity index 100% rename from tests/test_declare.py rename to tests_old/test_declare.py diff --git a/tests/test_dependencies.py b/tests_old/test_dependencies.py similarity index 100% rename from tests/test_dependencies.py rename to tests_old/test_dependencies.py diff --git a/tests/test_erd.py b/tests_old/test_erd.py similarity index 100% rename from tests/test_erd.py rename to tests_old/test_erd.py diff --git a/tests/test_external.py b/tests_old/test_external.py similarity index 100% rename from tests/test_external.py rename to tests_old/test_external.py diff --git a/tests/test_external_class.py b/tests_old/test_external_class.py similarity index 100% rename from tests/test_external_class.py rename to tests_old/test_external_class.py diff --git a/tests/test_fetch.py b/tests_old/test_fetch.py similarity index 100% rename from tests/test_fetch.py rename to tests_old/test_fetch.py diff --git a/tests/test_fetch_same.py b/tests_old/test_fetch_same.py similarity index 100% rename from tests/test_fetch_same.py rename to tests_old/test_fetch_same.py diff --git a/tests/test_filepath.py b/tests_old/test_filepath.py similarity index 100% rename from tests/test_filepath.py rename to tests_old/test_filepath.py diff --git a/tests/test_foreign_keys.py b/tests_old/test_foreign_keys.py similarity index 100% rename from tests/test_foreign_keys.py rename to tests_old/test_foreign_keys.py diff --git a/tests/test_groupby.py b/tests_old/test_groupby.py similarity index 100% rename from tests/test_groupby.py rename to tests_old/test_groupby.py diff --git a/tests/test_hash.py b/tests_old/test_hash.py similarity index 100% rename from tests/test_hash.py rename to tests_old/test_hash.py diff --git a/tests/test_jobs.py b/tests_old/test_jobs.py similarity index 100% rename from tests/test_jobs.py rename to tests_old/test_jobs.py diff --git a/tests/test_json.py b/tests_old/test_json.py similarity index 100% rename from tests/test_json.py rename to tests_old/test_json.py diff --git a/tests/test_log.py b/tests_old/test_log.py similarity index 100% rename from tests/test_log.py rename to tests_old/test_log.py diff --git a/tests/test_nan.py b/tests_old/test_nan.py similarity index 100% rename from tests/test_nan.py rename to tests_old/test_nan.py diff --git a/tests/test_plugin.py b/tests_old/test_plugin.py similarity index 100% rename from tests/test_plugin.py rename to tests_old/test_plugin.py diff --git a/tests/test_privileges.py b/tests_old/test_privileges.py similarity index 100% rename from tests/test_privileges.py rename to tests_old/test_privileges.py diff --git a/tests/test_reconnection.py b/tests_old/test_reconnection.py similarity index 100% rename from tests/test_reconnection.py rename to tests_old/test_reconnection.py diff --git a/tests/test_relation.py b/tests_old/test_relation.py similarity index 100% rename from tests/test_relation.py rename to tests_old/test_relation.py diff --git a/tests/test_relation_u.py b/tests_old/test_relation_u.py similarity index 100% rename from tests/test_relation_u.py rename to tests_old/test_relation_u.py diff --git a/tests/test_relational_operand.py b/tests_old/test_relational_operand.py similarity index 100% rename from tests/test_relational_operand.py rename to tests_old/test_relational_operand.py diff --git a/tests/test_s3.py b/tests_old/test_s3.py similarity index 100% rename from tests/test_s3.py rename to tests_old/test_s3.py diff --git a/tests/test_schema.py b/tests_old/test_schema.py similarity index 100% rename from tests/test_schema.py rename to tests_old/test_schema.py diff --git a/tests/test_schema_keywords.py b/tests_old/test_schema_keywords.py similarity index 100% rename from tests/test_schema_keywords.py rename to tests_old/test_schema_keywords.py diff --git a/tests/test_settings.py b/tests_old/test_settings.py similarity index 100% rename from tests/test_settings.py rename to tests_old/test_settings.py diff --git a/tests/test_tls.py b/tests_old/test_tls.py similarity index 100% rename from tests/test_tls.py rename to tests_old/test_tls.py diff --git a/tests/test_university.py b/tests_old/test_university.py similarity index 100% rename from tests/test_university.py rename to tests_old/test_university.py diff --git a/tests/test_update1.py b/tests_old/test_update1.py similarity index 100% rename from tests/test_update1.py rename to tests_old/test_update1.py diff --git a/tests/test_utils.py b/tests_old/test_utils.py similarity index 100% rename from tests/test_utils.py rename to tests_old/test_utils.py diff --git a/tests/test_uuid.py b/tests_old/test_uuid.py similarity index 100% rename from tests/test_uuid.py rename to tests_old/test_uuid.py diff --git a/tests/test_virtual_module.py b/tests_old/test_virtual_module.py similarity index 100% rename from tests/test_virtual_module.py rename to tests_old/test_virtual_module.py