Support renameable datasets and WK API v9 (#1231)

* Start implementing support for renameable datasets. * WIP support deprecated urls and open_remote_dataset requests. * Fix issues and update test cassettes. * Update changelog and fix remaining tests. * WIP adapt parsing remote dataset urls. * Fix regex for new dataset urls. * Remove comments and reformat timestamp. * Update webknossos/Changelog.md Co-authored-by: Norman Rzepka <[email protected]> --------- Co-authored-by: Norman Rzepka <[email protected]>
scalableminds · Jan 15, 2025 · 0efedee · 0efedee
1 parent 07be3c0
commit 0efedee
Show file tree

Hide file tree

Showing 47 changed files with 3,831 additions and 1,907 deletions.
diff --git a/webknossos/Changelog.md b/webknossos/Changelog.md
@@ -13,10 +13,17 @@ For upgrade instructions, please check the respective _Breaking Changes_ section
 [Commits](https://github.com/scalableminds/webknossos-libs/compare/v0.16.2...HEAD)
 
 ### Breaking Changes
+- `RemoteDataset.display_name` is deprecated. To change the name of a dataset use the `name` property instead. 
+- `Dataset.get_remote_datasets()` returns a mapping. The keys of this mapping changed from datasets name to datasets id.
+- `Task.create()` needs a `dataset_id` now instead of a `dataset_name`. Alternativly a `RemoteDataset` object can be used. The `dataset_name` is marked as deprecated. As `dataset_name` is an optional argument now its position has changed, this is important if `create()` is called only with positional arguments.
 
 ### Added
+- `Dataset` method `get_remote_datasets()` accepts `name` and `folder_id` as arguments now to filter remote datasets.
+- `RemoteDataset` got an additional property: `created`.
+- `Annotation` got an additional property: `dataset_id`.
 
 ### Changed
+- Updated to WEBKNOSSOS API version 9. This includes support for the new url structure for datasets and [#1231](https://github.com/scalableminds/webknossos-libs/pull/1231)
 
 ### Fixed
 - Fixed Mag setup for non-public datasets. [#1222](https://github.com/scalableminds/webknossos-libs/pull/1222)

diff --git a/webknossos/tests/cassettes/test_annotation/test_annotation_upload_download_roundtrip.yml b/webknossos/tests/cassettes/test_annotation/test_annotation_upload_download_roundtrip.yml
diff --git a/webknossos/tests/cassettes/test_annotation/test_bounding_box_roundtrip.yml b/webknossos/tests/cassettes/test_annotation/test_bounding_box_roundtrip.yml
diff --git a/webknossos/tests/cassettes/test_api_client/test_annotation_info.yml b/webknossos/tests/cassettes/test_api_client/test_annotation_info.yml
@@ -2,7 +2,7 @@ http_interactions:
   - request:
       method: GET
       path: >-
-        /api/v8/annotations/570ba0092a7c0e980056fe9b/info?timestamp=1734529349000
+        /api/v9/annotations/570ba0092a7c0e980056fe9b/info?timestamp=1736433032000
       headers:
         host: localhost:9000
         accept: '*/*'
@@ -22,7 +22,7 @@ http_interactions:
         cache-control: no-cache
         referrer-policy: origin-when-cross-origin, strict-origin-when-cross-origin
         x-permitted-cross-domain-policies: master-only
-        date: Wed, 18 Dec 2024 13:42:29 GMT
+        date: Thu, 09 Jan 2025 14:30:32 GMT
         content-type: application/json
         content-length: '1929'
       body:

diff --git a/webknossos/tests/cassettes/test_api_client/test_build_info.yml b/webknossos/tests/cassettes/test_api_client/test_build_info.yml
@@ -1,7 +1,7 @@
 http_interactions:
   - request:
       method: GET
-      path: /api/v8/buildinfo
+      path: /api/v9/buildinfo
       headers:
         host: localhost:9000
         accept: '*/*'
@@ -22,7 +22,7 @@ http_interactions:
         access-control-max-age: '600'
         access-control-allow-origin: '*'
         x-permitted-cross-domain-policies: master-only
-        date: Wed, 18 Dec 2024 13:42:30 GMT
+        date: Thu, 09 Jan 2025 14:30:33 GMT
         content-type: application/json
         content-length: '415'
       body:

diff --git a/webknossos/tests/cassettes/test_api_client/test_current_user_info_and_user_logged_time.yml b/webknossos/tests/cassettes/test_api_client/test_current_user_info_and_user_logged_time.yml
@@ -1,7 +1,7 @@
 http_interactions:
   - request:
       method: GET
-      path: /api/v8/user
+      path: /api/v9/user
       headers:
         host: localhost:9000
         accept: '*/*'
@@ -21,17 +21,17 @@ http_interactions:
         cache-control: no-cache
         referrer-policy: origin-when-cross-origin, strict-origin-when-cross-origin
         x-permitted-cross-domain-policies: master-only
-        date: Wed, 18 Dec 2024 13:42:30 GMT
+        date: Thu, 09 Jan 2025 14:30:33 GMT
         content-type: application/json
         content-length: '682'
       body:
         encoding: utf8
         data: >-
-          {"id":"570b9f4d2a7c0e4d008da6ef","email":"[email protected]","firstName":"user_A","lastName":"last_A","isAdmin":true,"isOrganizationOwner":true,"isDatasetManager":true,"isActive":true,"teams":[{"id":"570b9f4b2a7c0e3b008da6ec","name":"team_X1","isTeamManager":true},{"id":"59882b370d889b84020efd3f","name":"team_X3","isTeamManager":false},{"id":"59882b370d889b84020efd6f","name":"team_X4","isTeamManager":true}],"experiences":{"abc":5},"lastActivity":1734529175482,"isAnonymous":false,"isEditable":true,"organization":"Organization_X","novelUserExperienceInfos":{},"selectedTheme":"auto","created":1460379469000,"lastTaskTypeId":null,"isSuperUser":true,"isEmailVerified":true}
+          {"id":"570b9f4d2a7c0e4d008da6ef","email":"[email protected]","firstName":"user_A","lastName":"last_A","isAdmin":true,"isOrganizationOwner":true,"isDatasetManager":true,"isActive":true,"teams":[{"id":"570b9f4b2a7c0e3b008da6ec","name":"team_X1","isTeamManager":true},{"id":"59882b370d889b84020efd3f","name":"team_X3","isTeamManager":false},{"id":"59882b370d889b84020efd6f","name":"team_X4","isTeamManager":true}],"experiences":{"abc":5},"lastActivity":1736432857848,"isAnonymous":false,"isEditable":true,"organization":"Organization_X","novelUserExperienceInfos":{},"selectedTheme":"auto","created":1460379469000,"lastTaskTypeId":null,"isSuperUser":true,"isEmailVerified":true}
         compression: none
   - request:
       method: GET
-      path: /api/v8/users/570b9f4d2a7c0e4d008da6ef/loggedTime
+      path: /api/v9/users/570b9f4d2a7c0e4d008da6ef/loggedTime
       headers:
         host: localhost:9000
         accept: '*/*'
@@ -51,11 +51,11 @@ http_interactions:
         cache-control: no-cache
         referrer-policy: origin-when-cross-origin, strict-origin-when-cross-origin
         x-permitted-cross-domain-policies: master-only
-        date: Wed, 18 Dec 2024 13:42:30 GMT
+        date: Thu, 09 Jan 2025 14:30:33 GMT
         content-type: application/json
-        content-length: '692'
+        content-length: '691'
       body:
         encoding: utf8
         data: >-
-          {"loggedTime":[{"paymentInterval":{"month":4,"year":2016},"durationInSeconds":58},{"paymentInterval":{"month":8,"year":2018},"durationInSeconds":265},{"paymentInterval":{"month":1,"year":2021},"durationInSeconds":112},{"paymentInterval":{"month":3,"year":2021},"durationInSeconds":24},{"paymentInterval":{"month":5,"year":2021},"durationInSeconds":158},{"paymentInterval":{"month":8,"year":2021},"durationInSeconds":14},{"paymentInterval":{"month":11,"year":2021},"durationInSeconds":12},{"paymentInterval":{"month":2,"year":2024},"durationInSeconds":114},{"paymentInterval":{"month":3,"year":2024},"durationInSeconds":200},{"paymentInterval":{"month":12,"year":2024},"durationInSeconds":0}]}
+          {"loggedTime":[{"paymentInterval":{"month":4,"year":2016},"durationInSeconds":58},{"paymentInterval":{"month":8,"year":2018},"durationInSeconds":265},{"paymentInterval":{"month":1,"year":2021},"durationInSeconds":112},{"paymentInterval":{"month":3,"year":2021},"durationInSeconds":24},{"paymentInterval":{"month":5,"year":2021},"durationInSeconds":158},{"paymentInterval":{"month":8,"year":2021},"durationInSeconds":14},{"paymentInterval":{"month":11,"year":2021},"durationInSeconds":12},{"paymentInterval":{"month":2,"year":2024},"durationInSeconds":114},{"paymentInterval":{"month":3,"year":2024},"durationInSeconds":200},{"paymentInterval":{"month":1,"year":2025},"durationInSeconds":0}]}
         compression: none
diff --git a/webknossos/tests/cassettes/test_api_client/test_dataset_info.yml b/webknossos/tests/cassettes/test_api_client/test_dataset_info.yml
@@ -1,7 +1,7 @@
 http_interactions:
   - request:
       method: GET
-      path: /api/v8/datasets/Organization_X/l4_sample
+      path: /api/v9/datasets/disambiguate/Organization_X/l4_sample/toId
       headers:
         host: localhost:9000
         accept: '*/*'
@@ -21,11 +21,41 @@ http_interactions:
         cache-control: no-cache
         referrer-policy: origin-when-cross-origin, strict-origin-when-cross-origin
         x-permitted-cross-domain-policies: master-only
-        date: Wed, 18 Dec 2024 13:42:30 GMT
+        date: Thu, 09 Jan 2025 14:30:33 GMT
         content-type: application/json
-        content-length: '1501'
+        content-length: '112'
       body:
         encoding: utf8
         data: >-
-          {"id":"59e9cfbdba632ac2ab8b23b5","name":"l4_sample","dataSource":{"id":{"name":"l4_sample","team":"Organization_X"},"dataLayers":[{"name":"color","category":"color","boundingBox":{"topLeft":[3072,3072,512],"width":1024,"height":1024,"depth":1024},"resolutions":[[1,1,1],[2,2,1],[4,4,1],[8,8,2],[16,16,4]],"elementClass":"uint8","defaultViewConfiguration":{"color":[255,0,0]}},{"name":"segmentation","category":"segmentation","boundingBox":{"topLeft":[3072,3072,512],"width":1024,"height":1024,"depth":1024},"resolutions":[[1,1,1],[2,2,1],[4,4,1],[8,8,2],[16,16,4]],"elementClass":"uint32","largestSegmentId":2504697}],"scale":{"factor":[11.239999771118164,11.239999771118164,28],"unit":"nanometer"}},"dataStore":{"name":"localhost","url":"http://localhost:9000","isScratch":false,"allowsUpload":true,"jobsSupportedByAvailableWorkers":[],"jobsEnabled":false},"owningOrganization":"Organization_X","allowedTeams":[{"id":"570b9f4b2a7c0e3b008da6ec","name":"team_X1","organization":"Organization_X"}],"allowedTeamsCumulative":[{"id":"570b9f4b2a7c0e3b008da6ec","name":"team_X1","organization":"Organization_X"}],"isActive":true,"isPublic":true,"description":null,"created":1508495293789,"isEditable":true,"lastUsedByUser":1734529351088,"logoUrl":"/assets/images/mpi-logos.svg","sortingKey":1508495293789,"metadata":[{"key":"key","type":"string","value":"value"}],"isUnreported":false,"tags":[],"folderId":"570b9f4e4bb848d0885ea917","publication":null,"usedStorageBytes":1692207513,"displayName":"l4_sample"}
+          {"id":"59e9cfbdba632ac2ab8b23b5","name":"l4_sample","organization":"Organization_X","directoryName":"l4_sample"}
+        compression: none
+  - request:
+      method: GET
+      path: /api/v9/datasets/59e9cfbdba632ac2ab8b23b5
+      headers:
+        host: localhost:9000
+        accept: '*/*'
+        accept-encoding: gzip, deflate, zstd
+        connection: keep-alive
+        user-agent: python-httpx/0.27.2
+        x-auth-token: >-
+          1b88db86331a38c21a0b235794b9e459856490d70408bcffb767f64ade0f83d2bdb4c4e181b9a9a30cdece7cb7c65208cc43b6c1bb5987f5ece00d348b1a905502a266f8fc64f0371cd6559393d72e031d0c2d0cabad58cccf957bb258bc86f05b5dc3d4fff3d5e3d9c0389a6027d861a21e78e3222fb6c5b7944520ef21761e
+      body:
+        encoding: utf8
+        data: ''
+        compression: none
+    response:
+      status:
+        code: 200
+      headers:
+        cache-control: no-cache
+        referrer-policy: origin-when-cross-origin, strict-origin-when-cross-origin
+        x-permitted-cross-domain-policies: master-only
+        date: Thu, 09 Jan 2025 14:30:33 GMT
+        content-type: application/json
+        content-length: '1503'
+      body:
+        encoding: utf8
+        data: >-
+          {"id":"59e9cfbdba632ac2ab8b23b5","name":"l4_sample","dataSource":{"id":{"name":"l4_sample","team":"Organization_X"},"dataLayers":[{"name":"color","category":"color","boundingBox":{"topLeft":[3072,3072,512],"width":1024,"height":1024,"depth":1024},"resolutions":[[1,1,1],[2,2,1],[4,4,1],[8,8,2],[16,16,4]],"elementClass":"uint8","defaultViewConfiguration":{"color":[255,0,0]}},{"name":"segmentation","category":"segmentation","boundingBox":{"topLeft":[3072,3072,512],"width":1024,"height":1024,"depth":1024},"resolutions":[[1,1,1],[2,2,1],[4,4,1],[8,8,2],[16,16,4]],"elementClass":"uint32","largestSegmentId":2504697}],"scale":{"factor":[11.239999771118164,11.239999771118164,28],"unit":"nanometer"}},"dataStore":{"name":"localhost","url":"http://localhost:9000","isScratch":false,"allowsUpload":true,"jobsSupportedByAvailableWorkers":[],"jobsEnabled":false},"owningOrganization":"Organization_X","allowedTeams":[{"id":"570b9f4b2a7c0e3b008da6ec","name":"team_X1","organization":"Organization_X"}],"allowedTeamsCumulative":[{"id":"570b9f4b2a7c0e3b008da6ec","name":"team_X1","organization":"Organization_X"}],"isActive":true,"isPublic":true,"description":null,"directoryName":"l4_sample","created":1508495293789,"isEditable":true,"lastUsedByUser":1736433034081,"logoUrl":"/assets/images/mpi-logos.svg","sortingKey":1508495293789,"metadata":[{"key":"key","type":"string","value":"value"}],"isUnreported":false,"tags":[],"folderId":"570b9f4e4bb848d0885ea917","publication":null,"usedStorageBytes":1692207513}
         compression: none
diff --git a/webknossos/tests/cassettes/test_api_client/test_datastore_list.yml b/webknossos/tests/cassettes/test_api_client/test_datastore_list.yml
@@ -1,7 +1,7 @@
 http_interactions:
   - request:
       method: GET
-      path: /api/v8/datastores
+      path: /api/v9/datastores
       headers:
         host: localhost:9000
         accept: '*/*'
@@ -21,7 +21,7 @@ http_interactions:
         cache-control: no-cache
         referrer-policy: origin-when-cross-origin, strict-origin-when-cross-origin
         x-permitted-cross-domain-policies: master-only
-        date: Wed, 18 Dec 2024 13:42:29 GMT
+        date: Thu, 09 Jan 2025 14:30:32 GMT
         content-type: application/json
         content-length: '147'
       body:

diff --git a/webknossos/tests/cassettes/test_api_client/test_generate_token_for_data_store.yml b/webknossos/tests/cassettes/test_api_client/test_generate_token_for_data_store.yml
@@ -1,7 +1,7 @@
 http_interactions:
   - request:
       method: POST
-      path: /api/v8/userToken/generate
+      path: /api/v9/userToken/generate
       headers:
         host: localhost:9000
         content-length: '0'
@@ -22,10 +22,10 @@ http_interactions:
         cache-control: no-cache
         referrer-policy: origin-when-cross-origin, strict-origin-when-cross-origin
         x-permitted-cross-domain-policies: master-only
-        date: Wed, 18 Dec 2024 13:42:30 GMT
+        date: Thu, 09 Jan 2025 14:30:33 GMT
         content-type: application/json
         content-length: '34'
       body:
         encoding: utf8
-        data: '{"token":"IyV-W-lxr-YovycgMxWBvg"}'
+        data: '{"token":"NEdSmVtaQai_RdfG3fDxNQ"}'
         compression: none
diff --git a/webknossos/tests/cassettes/test_api_client/test_health.yml b/webknossos/tests/cassettes/test_api_client/test_health.yml
@@ -1,7 +1,7 @@
 http_interactions:
   - request:
       method: GET
-      path: /api/v8/health
+      path: /api/v9/health
       headers:
         host: localhost:9000
         accept: '*/*'
@@ -21,7 +21,7 @@ http_interactions:
         cache-control: no-cache
         referrer-policy: origin-when-cross-origin, strict-origin-when-cross-origin
         x-permitted-cross-domain-policies: master-only
-        date: Wed, 18 Dec 2024 13:42:29 GMT
+        date: Thu, 09 Jan 2025 14:30:32 GMT
         content-type: text/plain; charset=UTF-8
         content-length: '2'
       body:

diff --git a/webknossos/tests/cassettes/test_api_client/test_user_list.yml b/webknossos/tests/cassettes/test_api_client/test_user_list.yml
@@ -1,7 +1,7 @@
 http_interactions:
   - request:
       method: GET
-      path: /api/v8/users
+      path: /api/v9/users
       headers:
         host: localhost:9000
         accept: '*/*'
@@ -21,11 +21,11 @@ http_interactions:
         cache-control: no-cache
         referrer-policy: origin-when-cross-origin, strict-origin-when-cross-origin
         x-permitted-cross-domain-policies: master-only
-        date: Wed, 18 Dec 2024 13:42:30 GMT
+        date: Thu, 09 Jan 2025 14:30:33 GMT
         content-type: application/json
         content-length: '2287'
       body:
         encoding: utf8
         data: >-
-          [{"id":"570b9f4d2a7c0e4d008da6ef","email":"[email protected]","firstName":"user_A","lastName":"last_A","isAdmin":true,"isOrganizationOwner":true,"isDatasetManager":true,"isActive":true,"teams":[{"id":"570b9f4b2a7c0e3b008da6ec","name":"team_X1","isTeamManager":true},{"id":"59882b370d889b84020efd3f","name":"team_X3","isTeamManager":false},{"id":"59882b370d889b84020efd6f","name":"team_X4","isTeamManager":true}],"experiences":{"abc":5},"lastActivity":1734529350663,"isAnonymous":false,"isEditable":true,"organization":"Organization_X","novelUserExperienceInfos":{},"selectedTheme":"auto","created":1460379469000,"lastTaskTypeId":null,"isSuperUser":true,"isEmailVerified":true},{"id":"670b9f4d2a7c0e4d008da6ef","email":"[email protected]","firstName":"user_B","lastName":"last_B","isAdmin":false,"isOrganizationOwner":false,"isDatasetManager":true,"isActive":true,"teams":[{"id":"570b9f4b2a7c0e3b008da6ec","name":"team_X1","isTeamManager":true}],"experiences":{},"lastActivity":1460465869053,"isAnonymous":false,"isEditable":true,"organization":"Organization_X","novelUserExperienceInfos":{},"selectedTheme":"auto","created":1460379469000,"lastTaskTypeId":null,"isSuperUser":true,"isEmailVerified":true},{"id":"770b9f4d2a7c0e4d008da6ef","email":"[email protected]","firstName":"user_C","lastName":"last_C","isAdmin":false,"isOrganizationOwner":false,"isDatasetManager":false,"isActive":true,"teams":[{"id":"570b9f4b2a7c0e3b008da6ec","name":"team_X1","isTeamManager":false}],"experiences":{},"lastActivity":1460552269053,"isAnonymous":false,"isEditable":true,"organization":"Organization_X","novelUserExperienceInfos":{},"selectedTheme":"auto","created":1460379469000,"lastTaskTypeId":null,"isSuperUser":true,"isEmailVerified":true},{"id":"870b9f4d2a7c0e4d008da6ef","email":"[email protected]","firstName":"user_D","lastName":"last_D","isAdmin":false,"isOrganizationOwner":false,"isDatasetManager":false,"isActive":true,"teams":[{"id":"69882b370d889b84020efd4f","name":"team_X2","isTeamManager":true}],"experiences":{},"lastActivity":1460638669053,"isAnonymous":false,"isEditable":true,"organization":"Organization_X","novelUserExperienceInfos":{},"selectedTheme":"light","created":1460379469000,"lastTaskTypeId":null,"isSuperUser":true,"isEmailVerified":true}]
+          [{"id":"570b9f4d2a7c0e4d008da6ef","email":"[email protected]","firstName":"user_A","lastName":"last_A","isAdmin":true,"isOrganizationOwner":true,"isDatasetManager":true,"isActive":true,"teams":[{"id":"570b9f4b2a7c0e3b008da6ec","name":"team_X1","isTeamManager":true},{"id":"59882b370d889b84020efd3f","name":"team_X3","isTeamManager":false},{"id":"59882b370d889b84020efd6f","name":"team_X4","isTeamManager":true}],"experiences":{"abc":5},"lastActivity":1736433033668,"isAnonymous":false,"isEditable":true,"organization":"Organization_X","novelUserExperienceInfos":{},"selectedTheme":"auto","created":1460379469000,"lastTaskTypeId":null,"isSuperUser":true,"isEmailVerified":true},{"id":"670b9f4d2a7c0e4d008da6ef","email":"[email protected]","firstName":"user_B","lastName":"last_B","isAdmin":false,"isOrganizationOwner":false,"isDatasetManager":true,"isActive":true,"teams":[{"id":"570b9f4b2a7c0e3b008da6ec","name":"team_X1","isTeamManager":true}],"experiences":{},"lastActivity":1460465869053,"isAnonymous":false,"isEditable":true,"organization":"Organization_X","novelUserExperienceInfos":{},"selectedTheme":"auto","created":1460379469000,"lastTaskTypeId":null,"isSuperUser":true,"isEmailVerified":true},{"id":"770b9f4d2a7c0e4d008da6ef","email":"[email protected]","firstName":"user_C","lastName":"last_C","isAdmin":false,"isOrganizationOwner":false,"isDatasetManager":false,"isActive":true,"teams":[{"id":"570b9f4b2a7c0e3b008da6ec","name":"team_X1","isTeamManager":false}],"experiences":{},"lastActivity":1460552269053,"isAnonymous":false,"isEditable":true,"organization":"Organization_X","novelUserExperienceInfos":{},"selectedTheme":"auto","created":1460379469000,"lastTaskTypeId":null,"isSuperUser":true,"isEmailVerified":true},{"id":"870b9f4d2a7c0e4d008da6ef","email":"[email protected]","firstName":"user_D","lastName":"last_D","isAdmin":false,"isOrganizationOwner":false,"isDatasetManager":false,"isActive":true,"teams":[{"id":"69882b370d889b84020efd4f","name":"team_X2","isTeamManager":true}],"experiences":{},"lastActivity":1460638669053,"isAnonymous":false,"isEditable":true,"organization":"Organization_X","novelUserExperienceInfos":{},"selectedTheme":"light","created":1460379469000,"lastTaskTypeId":null,"isSuperUser":true,"isEmailVerified":true}]
         compression: none