From 8d3b11a55b329bf83fdf1a48caa034515f53ac29 Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Wed, 8 Nov 2023 10:45:32 +0100
Subject: [PATCH 01/21] add monitoring router and get endpoints

---
 fractal_server/app/api/__init__.py      |  4 +++
 fractal_server/app/api/v1/monitoring.py | 41 +++++++++++++++++++++++++
 2 files changed, 45 insertions(+)
 create mode 100644 fractal_server/app/api/v1/monitoring.py

diff --git a/fractal_server/app/api/__init__.py b/fractal_server/app/api/__init__.py
index 063f848506..91053247ed 100644
--- a/fractal_server/app/api/__init__.py
+++ b/fractal_server/app/api/__init__.py
@@ -7,6 +7,7 @@
 from ...syringe import Inject
 from .v1.dataset import router as dataset_router
 from .v1.job import router as job_router
+from .v1.monitoring import router as monitoring_router
 from .v1.project import router as project_router
 from .v1.task import router as task_router
 from .v1.task_collection import router as taskcollection_router
@@ -17,6 +18,9 @@
 router_default = APIRouter()
 router_v1 = APIRouter()
 
+router_v1.include_router(
+    monitoring_router, prefix="/monitoring", tags=["Monitoring"]
+)
 router_v1.include_router(project_router, prefix="/project", tags=["Projects"])
 router_v1.include_router(task_router, prefix="/task", tags=["Tasks"])
 router_v1.include_router(
diff --git a/fractal_server/app/api/v1/monitoring.py b/fractal_server/app/api/v1/monitoring.py
new file mode 100644
index 0000000000..7ee966b338
--- /dev/null
+++ b/fractal_server/app/api/v1/monitoring.py
@@ -0,0 +1,41 @@
+from fastapi import APIRouter
+from fastapi import Depends
+
+from ...db import AsyncSession
+from ...db import get_db
+from ...security import current_active_superuser
+from ...security import User
+
+router = APIRouter()
+
+
+@router.get("/project/")
+async def monitor_project(
+    user: User = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_db),
+):
+    pass
+
+
+@router.get("/workflow/")
+async def monitor_workflow(
+    user: User = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_db),
+):
+    pass
+
+
+@router.get("/dataset/")
+async def monitor_dataset(
+    user: User = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_db),
+):
+    pass
+
+
+@router.get("/job/")
+async def monitor_job(
+    user: User = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_db),
+):
+    pass

From f56f70120bf993ee2fc6ce1929edb2d0c3d792e5 Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Wed, 8 Nov 2023 12:19:10 +0100
Subject: [PATCH 02/21] test monitoring

---
 fractal_server/app/api/v1/monitoring.py | 65 +++++++++++++++++++++++--
 tests/test_monitoring.py                | 36 ++++++++++++++
 2 files changed, 97 insertions(+), 4 deletions(-)
 create mode 100644 tests/test_monitoring.py

diff --git a/fractal_server/app/api/v1/monitoring.py b/fractal_server/app/api/v1/monitoring.py
index 7ee966b338..424de3167d 100644
--- a/fractal_server/app/api/v1/monitoring.py
+++ b/fractal_server/app/api/v1/monitoring.py
@@ -1,8 +1,16 @@
+from datetime import datetime as DateTime
+from typing import Optional
+
 from fastapi import APIRouter
 from fastapi import Depends
+from sqlmodel import select
 
 from ...db import AsyncSession
 from ...db import get_db
+from ...models import ApplyWorkflow
+from ...models import Project
+from ...schemas import ApplyWorkflowRead
+from ...schemas import ProjectRead
 from ...security import current_active_superuser
 from ...security import User
 
@@ -11,10 +19,21 @@
 
 @router.get("/project/")
 async def monitor_project(
+    id: Optional[int] = None,
     user: User = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_db),
-):
-    pass
+) -> list[ProjectRead]:
+
+    stm = select(Project)
+
+    if id:
+        stm = stm.where(Project.id == id)
+
+    res = await db.execute(stm)
+    project_list = res.scalars().all()
+    await db.close()
+
+    return project_list
 
 
 @router.get("/workflow/")
@@ -35,7 +54,45 @@ async def monitor_dataset(
 
 @router.get("/job/")
 async def monitor_job(
+    id: Optional[int],
+    project_id: Optional[int],
+    input_dataset_id: Optional[int],
+    output_dataset_id: Optional[int],
+    workflow_id: Optional[int],
+    working_dir: Optional[str],
+    working_dir_user: Optional[str],
+    status: Optional[str],
+    start_timestamp: Optional[DateTime],
+    end_timestamp: Optional[DateTime],
     user: User = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_db),
-):
-    pass
+) -> Optional[list[ApplyWorkflowRead]]:
+
+    stm = select(ApplyWorkflow)
+
+    if id:
+        stm = stm.where(ApplyWorkflow.id == id)
+    if project_id:
+        stm = stm.where(ApplyWorkflow.project_id == project_id)
+    if input_dataset_id:
+        stm = stm.where(ApplyWorkflow.input_dataset_id == input_dataset_id)
+    if output_dataset_id:
+        stm = stm.where(ApplyWorkflow.output_dataset_id == output_dataset_id)
+    if workflow_id:
+        stm = stm.where(ApplyWorkflow.workflow_id == workflow_id)
+    if working_dir:
+        stm = stm.where(ApplyWorkflow.working_dir == working_dir)
+    if working_dir_user:
+        stm = stm.where(ApplyWorkflow.working_dir_user == working_dir_user)
+    if status:
+        stm = stm.where(ApplyWorkflow.status == status)
+    if start_timestamp:
+        stm = stm.where(ApplyWorkflow.start_timestamp >= start_timestamp)
+    if end_timestamp:
+        stm = stm.where(ApplyWorkflow.end_timestamp <= end_timestamp)
+
+    res = await db.execute(stm)
+    job_list = res.scalars().all()
+    await db.close()
+
+    return job_list
diff --git a/tests/test_monitoring.py b/tests/test_monitoring.py
new file mode 100644
index 0000000000..1f5628f304
--- /dev/null
+++ b/tests/test_monitoring.py
@@ -0,0 +1,36 @@
+PREFIX = "api/v1"
+
+
+async def test_unauthorized_to_monitor(registered_client):
+
+    res = await registered_client.get(f"{PREFIX}/monitoring/project/")
+    assert res.status_code == 403
+
+
+async def test_monitor_project(client, MockCurrentUser, project_factory):
+
+    async with MockCurrentUser(
+        persist=True, user_kwargs={"is_superuser": True}
+    ):
+        res = await client.get(f"{PREFIX}/monitoring/project/")
+        assert res.status_code == 200
+        assert res.json() == []
+
+    async with MockCurrentUser(
+        persist=True, user_kwargs={"is_superuser": False}
+    ) as user:
+        prj1 = await project_factory(user)
+        await project_factory(user)
+        prj1_id = prj1.id
+        res = await client.get(f"{PREFIX}/monitoring/project/")
+        assert res.status_code == 403
+
+    async with MockCurrentUser(
+        persist=True, user_kwargs={"is_superuser": True}
+    ):
+        res = await client.get(f"{PREFIX}/monitoring/project/")
+        assert res.status_code == 200
+        assert len(res.json()) == 2
+        res = await client.get(f"{PREFIX}/monitoring/project/?id={prj1_id}")
+        assert res.status_code == 200
+        assert len(res.json()) == 1

From 78eff31cc27b16ace5af526d0bce263801bba92e Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Wed, 8 Nov 2023 13:05:51 +0100
Subject: [PATCH 03/21] monitor workflow

---
 fractal_server/app/api/v1/monitoring.py |  39 ++++++---
 tests/test_monitoring.py                | 106 ++++++++++++++++++++++--
 2 files changed, 127 insertions(+), 18 deletions(-)

diff --git a/fractal_server/app/api/v1/monitoring.py b/fractal_server/app/api/v1/monitoring.py
index 424de3167d..eaff1a1abc 100644
--- a/fractal_server/app/api/v1/monitoring.py
+++ b/fractal_server/app/api/v1/monitoring.py
@@ -9,6 +9,7 @@
 from ...db import get_db
 from ...models import ApplyWorkflow
 from ...models import Project
+from ...models import Workflow
 from ...schemas import ApplyWorkflowRead
 from ...schemas import ProjectRead
 from ...security import current_active_superuser
@@ -38,10 +39,26 @@ async def monitor_project(
 
 @router.get("/workflow/")
 async def monitor_workflow(
+    id: Optional[int] = None,
+    project_id: Optional[int] = None,
+    name: Optional[str] = None,
     user: User = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_db),
 ):
-    pass
+    stm = select(Workflow)
+
+    if id:
+        stm = stm.where(Workflow.id == id)
+    if project_id:
+        stm = stm.where(Workflow.project_id == project_id)
+    if name:
+        stm = stm.where(Workflow.name.contains(name))
+
+    res = await db.execute(stm)
+    project_list = res.scalars().all()
+    await db.close()
+
+    return project_list
 
 
 @router.get("/dataset/")
@@ -54,16 +71,16 @@ async def monitor_dataset(
 
 @router.get("/job/")
 async def monitor_job(
-    id: Optional[int],
-    project_id: Optional[int],
-    input_dataset_id: Optional[int],
-    output_dataset_id: Optional[int],
-    workflow_id: Optional[int],
-    working_dir: Optional[str],
-    working_dir_user: Optional[str],
-    status: Optional[str],
-    start_timestamp: Optional[DateTime],
-    end_timestamp: Optional[DateTime],
+    id: Optional[int] = None,
+    project_id: Optional[int] = None,
+    input_dataset_id: Optional[int] = None,
+    output_dataset_id: Optional[int] = None,
+    workflow_id: Optional[int] = None,
+    working_dir: Optional[str] = None,
+    working_dir_user: Optional[str] = None,
+    status: Optional[str] = None,
+    start_timestamp: Optional[DateTime] = None,
+    end_timestamp: Optional[DateTime] = None,
     user: User = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_db),
 ) -> Optional[list[ApplyWorkflowRead]]:
diff --git a/tests/test_monitoring.py b/tests/test_monitoring.py
index 1f5628f304..241e9862fc 100644
--- a/tests/test_monitoring.py
+++ b/tests/test_monitoring.py
@@ -1,10 +1,31 @@
 PREFIX = "api/v1"
 
 
-async def test_unauthorized_to_monitor(registered_client):
+async def test_unauthorized_to_monitor(client, MockCurrentUser):
 
-    res = await registered_client.get(f"{PREFIX}/monitoring/project/")
-    assert res.status_code == 403
+    async with MockCurrentUser(
+        persist=True, user_kwargs={"is_superuser": False}
+    ):
+        res = await client.get(f"{PREFIX}/monitoring/project/")
+        assert res.status_code == 403
+        res = await client.get(f"{PREFIX}/monitoring/workflow/")
+        assert res.status_code == 403
+        res = await client.get(f"{PREFIX}/monitoring/dataset/")
+        assert res.status_code == 403
+        res = await client.get(f"{PREFIX}/monitoring/job/")
+        assert res.status_code == 403
+
+    async with MockCurrentUser(
+        persist=True, user_kwargs={"is_superuser": True}
+    ):
+        res = await client.get(f"{PREFIX}/monitoring/project/")
+        assert res.status_code == 200
+        res = await client.get(f"{PREFIX}/monitoring/workflow/")
+        assert res.status_code == 200
+        res = await client.get(f"{PREFIX}/monitoring/dataset/")
+        assert res.status_code == 200
+        res = await client.get(f"{PREFIX}/monitoring/job/")
+        assert res.status_code == 200
 
 
 async def test_monitor_project(client, MockCurrentUser, project_factory):
@@ -19,11 +40,9 @@ async def test_monitor_project(client, MockCurrentUser, project_factory):
     async with MockCurrentUser(
         persist=True, user_kwargs={"is_superuser": False}
     ) as user:
-        prj1 = await project_factory(user)
+        project1 = await project_factory(user)
+        prj1_id = project1.id
         await project_factory(user)
-        prj1_id = prj1.id
-        res = await client.get(f"{PREFIX}/monitoring/project/")
-        assert res.status_code == 403
 
     async with MockCurrentUser(
         persist=True, user_kwargs={"is_superuser": True}
@@ -34,3 +53,76 @@ async def test_monitor_project(client, MockCurrentUser, project_factory):
         res = await client.get(f"{PREFIX}/monitoring/project/?id={prj1_id}")
         assert res.status_code == 200
         assert len(res.json()) == 1
+
+
+async def test_monitor_workflow(
+    client, MockCurrentUser, project_factory, workflow_factory
+):
+
+    async with MockCurrentUser(
+        persist=True, user_kwargs={"is_superuser": False}
+    ) as user:
+
+        project1 = await project_factory(user)
+        workflow1a = await workflow_factory(
+            project_id=project1.id, name="Workflow 1a"
+        )
+        workflow1b = await workflow_factory(
+            project_id=project1.id, name="Workflow 1b"
+        )
+
+        project2 = await project_factory(user)
+        workflow2a = await workflow_factory(
+            project_id=project2.id, name="Workflow 2a"
+        )
+
+    async with MockCurrentUser(
+        persist=True, user_kwargs={"is_superuser": True}
+    ):
+        # get all workflow
+        res = await client.get(f"{PREFIX}/monitoring/workflow/")
+        assert res.status_code == 200
+        assert len(res.json()) == 3
+
+        # get workflow by id
+        res = await client.get(
+            f"{PREFIX}/monitoring/workflow/?id={workflow1a.id}"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 1
+        assert res.json()[0]["name"] == workflow1a.name
+
+        # get workflow by project_id
+        res = await client.get(
+            f"{PREFIX}/monitoring/workflow/?project_id={project1.id}"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 2
+
+        # get workflow by project_id and id
+        res = await client.get(
+            f"{PREFIX}/monitoring/workflow/"
+            f"?project_id={project1.id}&id={workflow1b.id}"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 1
+        assert res.json()[0]["name"] == workflow1b.name
+
+        res = await client.get(
+            f"{PREFIX}/monitoring/workflow/"
+            f"?project_id={project1.id}&id={workflow2a.id}"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 0
+
+        # get workflow by name
+        res = await client.get(
+            f"{PREFIX}/monitoring/workflow/?name={workflow2a.name}"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 1
+        assert res.json()[0]["name"] == workflow2a.name
+
+        res = await client.get(f"{PREFIX}/monitoring/workflow/?name=Workflow")
+        assert res.status_code == 200
+        assert len(res.json()) == 3

From c7c9d1aad323d01e5fba33708a60d5d1e39a313c Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Thu, 9 Nov 2023 10:53:38 +0100
Subject: [PATCH 04/21] dataset endpoint

---
 fractal_server/app/api/v1/monitoring.py | 51 ++++++++++++++++++-------
 1 file changed, 37 insertions(+), 14 deletions(-)

diff --git a/fractal_server/app/api/v1/monitoring.py b/fractal_server/app/api/v1/monitoring.py
index eaff1a1abc..d734079721 100644
--- a/fractal_server/app/api/v1/monitoring.py
+++ b/fractal_server/app/api/v1/monitoring.py
@@ -8,6 +8,7 @@
 from ...db import AsyncSession
 from ...db import get_db
 from ...models import ApplyWorkflow
+from ...models import Dataset
 from ...models import Project
 from ...models import Workflow
 from ...schemas import ApplyWorkflowRead
@@ -47,11 +48,11 @@ async def monitor_workflow(
 ):
     stm = select(Workflow)
 
-    if id:
+    if id is not None:
         stm = stm.where(Workflow.id == id)
-    if project_id:
+    if project_id is not None:
         stm = stm.where(Workflow.project_id == project_id)
-    if name:
+    if name is not None:
         stm = stm.where(Workflow.name.contains(name))
 
     res = await db.execute(stm)
@@ -63,10 +64,32 @@ async def monitor_workflow(
 
 @router.get("/dataset/")
 async def monitor_dataset(
+    id: Optional[int] = None,
+    project_id: Optional[int] = None,
+    name: Optional[str] = None,
+    type: Optional[str] = None,
+    read_only: Optional[bool] = None,
     user: User = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_db),
 ):
-    pass
+    stm = select(Dataset)
+
+    if id is not None:
+        stm = stm.where(Dataset.id == id)
+    if project_id is not None:
+        stm = stm.where(Dataset.project_id == project_id)
+    if name is not None:
+        stm = stm.where(Dataset.name.contains(name))
+    if type is not None:
+        stm = stm.where(Dataset.type == type)
+    if read_only is not None:
+        stm = stm.where(Dataset.read_only == read_only)
+
+    res = await db.execute(stm)
+    dataset_list = res.scalars().all()
+    await db.close()
+
+    return dataset_list
 
 
 @router.get("/job/")
@@ -87,25 +110,25 @@ async def monitor_job(
 
     stm = select(ApplyWorkflow)
 
-    if id:
+    if id is not None:
         stm = stm.where(ApplyWorkflow.id == id)
-    if project_id:
+    if project_id is not None:
         stm = stm.where(ApplyWorkflow.project_id == project_id)
-    if input_dataset_id:
+    if input_dataset_id is not None:
         stm = stm.where(ApplyWorkflow.input_dataset_id == input_dataset_id)
-    if output_dataset_id:
+    if output_dataset_id is not None:
         stm = stm.where(ApplyWorkflow.output_dataset_id == output_dataset_id)
-    if workflow_id:
+    if workflow_id is not None:
         stm = stm.where(ApplyWorkflow.workflow_id == workflow_id)
-    if working_dir:
+    if working_dir is not None:
         stm = stm.where(ApplyWorkflow.working_dir == working_dir)
-    if working_dir_user:
+    if working_dir_user is not None:
         stm = stm.where(ApplyWorkflow.working_dir_user == working_dir_user)
-    if status:
+    if status is not None:
         stm = stm.where(ApplyWorkflow.status == status)
-    if start_timestamp:
+    if start_timestamp is not None:
         stm = stm.where(ApplyWorkflow.start_timestamp >= start_timestamp)
-    if end_timestamp:
+    if end_timestamp is not None:
         stm = stm.where(ApplyWorkflow.end_timestamp <= end_timestamp)
 
     res = await db.execute(stm)

From ce152b9885a9e7ffa04fd2587376cf48a4c84996 Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Thu, 9 Nov 2023 11:12:21 +0100
Subject: [PATCH 05/21] test monitor dataset

---
 tests/test_monitoring.py | 98 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 93 insertions(+), 5 deletions(-)

diff --git a/tests/test_monitoring.py b/tests/test_monitoring.py
index 241e9862fc..1af550501b 100644
--- a/tests/test_monitoring.py
+++ b/tests/test_monitoring.py
@@ -79,12 +79,12 @@ async def test_monitor_workflow(
     async with MockCurrentUser(
         persist=True, user_kwargs={"is_superuser": True}
     ):
-        # get all workflow
+        # get all workflows
         res = await client.get(f"{PREFIX}/monitoring/workflow/")
         assert res.status_code == 200
         assert len(res.json()) == 3
 
-        # get workflow by id
+        # get workflows by id
         res = await client.get(
             f"{PREFIX}/monitoring/workflow/?id={workflow1a.id}"
         )
@@ -92,14 +92,14 @@ async def test_monitor_workflow(
         assert len(res.json()) == 1
         assert res.json()[0]["name"] == workflow1a.name
 
-        # get workflow by project_id
+        # get workflows by project_id
         res = await client.get(
             f"{PREFIX}/monitoring/workflow/?project_id={project1.id}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 2
 
-        # get workflow by project_id and id
+        # get workflows by project_id and id
         res = await client.get(
             f"{PREFIX}/monitoring/workflow/"
             f"?project_id={project1.id}&id={workflow1b.id}"
@@ -115,7 +115,7 @@ async def test_monitor_workflow(
         assert res.status_code == 200
         assert len(res.json()) == 0
 
-        # get workflow by name
+        # get workflows by name
         res = await client.get(
             f"{PREFIX}/monitoring/workflow/?name={workflow2a.name}"
         )
@@ -126,3 +126,91 @@ async def test_monitor_workflow(
         res = await client.get(f"{PREFIX}/monitoring/workflow/?name=Workflow")
         assert res.status_code == 200
         assert len(res.json()) == 3
+
+
+async def test_monitor_dataset(
+    client, MockCurrentUser, project_factory, workflow_factory, dataset_factory
+):
+
+    async with MockCurrentUser(
+        persist=True, user_kwargs={"is_superuser": False}
+    ) as user:
+
+        project1 = await project_factory(user)
+
+        ds1a = await dataset_factory(
+            project_id=project1.id,
+            name="ds1a",
+            type="zarr",
+            read_only=False,
+        )
+        await dataset_factory(
+            project_id=project1.id,
+            name="ds1b",
+            type="image",
+            read_only=True,
+        )
+
+        project2 = await project_factory(user)
+
+        await dataset_factory(
+            project_id=project2.id,
+            name="ds2a",
+            type="zarr",
+            read_only=True,
+        )
+
+    async with MockCurrentUser(
+        persist=True, user_kwargs={"is_superuser": True}
+    ):
+        # get all datasets
+        res = await client.get(f"{PREFIX}/monitoring/dataset/")
+        assert res.status_code == 200
+        assert len(res.json()) == 3
+
+        # get datasets by id
+        res = await client.get(f"{PREFIX}/monitoring/dataset/?id={ds1a.id}")
+        assert res.status_code == 200
+        assert len(res.json()) == 1
+        assert res.json()[0]["name"] == ds1a.name
+        res = await client.get(f"{PREFIX}/monitoring/dataset/?id=123456789")
+        assert res.status_code == 200
+        assert len(res.json()) == 0
+
+        # get datasets by project_id
+        res = await client.get(
+            f"{PREFIX}/monitoring/dataset/?project_id={project1.id}"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 2
+        res = await client.get(
+            f"{PREFIX}/monitoring/dataset/?project_id={project2.id}"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 1
+
+        # get datasets by name
+        res = await client.get(
+            f"{PREFIX}/monitoring/dataset/?project_id={project1.id}&name=a"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 1
+        assert res.json()[0]["name"] == ds1a.name
+        res = await client.get(
+            f"{PREFIX}/monitoring/dataset/?project_id={project1.id}&name=c"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 0
+
+        # get datasets by type and read_only
+        res = await client.get(f"{PREFIX}/monitoring/dataset/?type=zarr")
+        assert res.status_code == 200
+        assert len(res.json()) == 2
+        res = await client.get(
+            f"{PREFIX}/monitoring/dataset/?type=zarr&read_only=true"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 1
+        res = await client.get(f"{PREFIX}/monitoring/dataset/?read_only=true")
+        assert res.status_code == 200
+        assert len(res.json()) == 2

From b7e2fd605548ec4926993511e583226f270a14aa Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Thu, 9 Nov 2023 11:52:44 +0100
Subject: [PATCH 06/21] test monitor job

---
 fractal_server/app/api/v1/monitoring.py |   6 +-
 tests/test_monitoring.py                | 143 +++++++++++++++++++++++-
 2 files changed, 146 insertions(+), 3 deletions(-)

diff --git a/fractal_server/app/api/v1/monitoring.py b/fractal_server/app/api/v1/monitoring.py
index d734079721..8b9255d6cc 100644
--- a/fractal_server/app/api/v1/monitoring.py
+++ b/fractal_server/app/api/v1/monitoring.py
@@ -121,9 +121,11 @@ async def monitor_job(
     if workflow_id is not None:
         stm = stm.where(ApplyWorkflow.workflow_id == workflow_id)
     if working_dir is not None:
-        stm = stm.where(ApplyWorkflow.working_dir == working_dir)
+        stm = stm.where(ApplyWorkflow.working_dir.contains(working_dir))
     if working_dir_user is not None:
-        stm = stm.where(ApplyWorkflow.working_dir_user == working_dir_user)
+        stm = stm.where(
+            ApplyWorkflow.working_dir_user.contains(working_dir_user)
+        )
     if status is not None:
         stm = stm.where(ApplyWorkflow.status == status)
     if start_timestamp is not None:
diff --git a/tests/test_monitoring.py b/tests/test_monitoring.py
index 1af550501b..cbd8c930b5 100644
--- a/tests/test_monitoring.py
+++ b/tests/test_monitoring.py
@@ -1,3 +1,5 @@
+from datetime import datetime
+
 PREFIX = "api/v1"
 
 
@@ -129,7 +131,7 @@ async def test_monitor_workflow(
 
 
 async def test_monitor_dataset(
-    client, MockCurrentUser, project_factory, workflow_factory, dataset_factory
+    client, MockCurrentUser, project_factory, dataset_factory
 ):
 
     async with MockCurrentUser(
@@ -214,3 +216,142 @@ async def test_monitor_dataset(
         res = await client.get(f"{PREFIX}/monitoring/dataset/?read_only=true")
         assert res.status_code == 200
         assert len(res.json()) == 2
+
+
+async def test_monitor_job(
+    db,
+    client,
+    MockCurrentUser,
+    tmp_path,
+    project_factory,
+    workflow_factory,
+    dataset_factory,
+    task_factory,
+    job_factory,
+):
+    async with MockCurrentUser(
+        persist=True, user_kwargs={"is_superuser": False}
+    ) as user:
+
+        project = await project_factory(user)
+
+        workflow1 = await workflow_factory(project_id=project.id)
+        workflow2 = await workflow_factory(project_id=project.id)
+
+        task = await task_factory(name="task", source="source")
+        dataset1 = await dataset_factory(project_id=project.id)
+        dataset2 = await dataset_factory(project_id=project.id)
+
+        await workflow1.insert_task(task_id=task.id, db=db)
+        await workflow2.insert_task(task_id=task.id, db=db)
+
+        job1 = await job_factory(
+            working_dir=f"{tmp_path.as_posix()}/aaaa1111",
+            working_dir_user=f"{tmp_path.as_posix()}/aaaa2222",
+            project_id=project.id,
+            input_dataset_id=dataset1.id,
+            output_dataset_id=dataset2.id,
+            working_id=workflow1.id,
+            start_timestamp=datetime(2000, 1, 1),
+        )
+
+        job2 = await job_factory(
+            working_dir=f"{tmp_path.as_posix()}/bbbb1111",
+            working_dir_user=f"{tmp_path.as_posix()}/bbbb2222",
+            project_id=project.id,
+            input_dataset_id=dataset2.id,
+            output_dataset_id=dataset1.id,
+            working_id=workflow2.id,
+            start_timestamp=datetime(2023, 1, 1),
+            end_timestamp=datetime(2023, 11, 9),
+        )
+
+    async with MockCurrentUser(
+        persist=True, user_kwargs={"is_superuser": True}
+    ):
+        # get all jobs
+        res = await client.get(f"{PREFIX}/monitoring/job/")
+        assert res.status_code == 200
+        assert len(res.json()) == 2
+
+        # get jobs by project_id
+        res = await client.get(
+            f"{PREFIX}/monitoring/job/?project_id={project.id}"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 2
+        res = await client.get(
+            f"{PREFIX}/monitoring/job/?project_id={project.id + 123456789}"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 0
+
+        # get jobs by input/output_dataset_id
+        res = await client.get(
+            f"{PREFIX}/monitoring/job/?input_dataset_id={dataset1.id}"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 1
+        assert res.json()[0]["id"] == job1.id
+        res = await client.get(
+            f"{PREFIX}/monitoring/job/?output_dataset_id={dataset1.id}"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 1
+        assert res.json()[0]["id"] == job2.id
+
+        # get jobs by worfking_dir[_user]
+        res = await client.get(f"{PREFIX}/monitoring/job/?working_dir=aaaa")
+        assert res.status_code == 200
+        assert len(res.json()) == 1
+        res = await client.get(f"{PREFIX}/monitoring/job/?working_dir=1111")
+        assert res.status_code == 200
+        assert len(res.json()) == 2
+        res = await client.get(
+            f"{PREFIX}/monitoring/job/?working_dir_user=bbbb"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 1
+        res = await client.get(
+            f"{PREFIX}/monitoring/job/?working_dir_user=2222"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 2
+
+        # get jobs by status
+        res = await client.get(f"{PREFIX}/monitoring/job/?status=fancy")
+        assert res.status_code == 200
+        assert len(res.json()) == 0
+        res = await client.get(f"{PREFIX}/monitoring/job/?status=submitted")
+        assert res.status_code == 200
+        assert len(res.json()) == 2
+
+        # get jobs by start/end_timestamp
+
+        res = await client.get(
+            f"{PREFIX}/monitoring/job/?start_timestamp=1999-01-01T00:00:01"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 2
+        res = await client.get(
+            f"{PREFIX}/monitoring/job/?start_timestamp=2010-01-01T00:00:01"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 1
+        res = await client.get(
+            f"{PREFIX}/monitoring/job/?start_timestamp=2024-01-01T00:00:01"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 0
+
+        res = await client.get(
+            f"{PREFIX}/monitoring/job/?end_timestamp=3000-01-01T00:00:01"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 1
+
+        res = await client.get(
+            f"{PREFIX}/monitoring/job/?end_timestamp=2000-01-01T00:00:01"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 0

From 07550a7c796d4d7ab5f62283aaa55872626f1acc Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Thu, 9 Nov 2023 12:08:01 +0100
Subject: [PATCH 07/21] test missings lines

---
 tests/test_monitoring.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/tests/test_monitoring.py b/tests/test_monitoring.py
index cbd8c930b5..f349d40fdb 100644
--- a/tests/test_monitoring.py
+++ b/tests/test_monitoring.py
@@ -251,7 +251,7 @@ async def test_monitor_job(
             project_id=project.id,
             input_dataset_id=dataset1.id,
             output_dataset_id=dataset2.id,
-            working_id=workflow1.id,
+            workflow_id=workflow1.id,
             start_timestamp=datetime(2000, 1, 1),
         )
 
@@ -261,7 +261,7 @@ async def test_monitor_job(
             project_id=project.id,
             input_dataset_id=dataset2.id,
             output_dataset_id=dataset1.id,
-            working_id=workflow2.id,
+            workflow_id=workflow2.id,
             start_timestamp=datetime(2023, 1, 1),
             end_timestamp=datetime(2023, 11, 9),
         )
@@ -274,6 +274,11 @@ async def test_monitor_job(
         assert res.status_code == 200
         assert len(res.json()) == 2
 
+        # get jobs by id
+        res = await client.get(f"{PREFIX}/monitoring/job/?id={job1.id}")
+        assert res.status_code == 200
+        assert len(res.json()) == 1
+
         # get jobs by project_id
         res = await client.get(
             f"{PREFIX}/monitoring/job/?project_id={project.id}"
@@ -300,6 +305,18 @@ async def test_monitor_job(
         assert len(res.json()) == 1
         assert res.json()[0]["id"] == job2.id
 
+        # get jobs by workflow_id
+        res = await client.get(
+            f"{PREFIX}/monitoring/job/?workflow_id={workflow2.id}"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 1
+        res = await client.get(
+            f"{PREFIX}/monitoring/job/?workflow_id=123456789"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 0
+
         # get jobs by worfking_dir[_user]
         res = await client.get(f"{PREFIX}/monitoring/job/?working_dir=aaaa")
         assert res.status_code == 200

From 433f880179a50936ea2f505a57b673cfbc1b7571 Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Mon, 13 Nov 2023 11:27:55 +0100
Subject: [PATCH 08/21] rm workflow relationship

---
 fractal_server/app/models/job.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/fractal_server/app/models/job.py b/fractal_server/app/models/job.py
index 92fbdc704a..33831d3ccb 100644
--- a/fractal_server/app/models/job.py
+++ b/fractal_server/app/models/job.py
@@ -13,7 +13,6 @@
 from ...utils import get_timestamp
 from ..schemas.applyworkflow import _ApplyWorkflowBase
 from .dataset import Dataset
-from .workflow import Workflow
 
 
 class JobStatusType(str, Enum):
@@ -108,7 +107,6 @@ class Config:
             primaryjoin="ApplyWorkflow.output_dataset_id==Dataset.id",
         )
     )
-    workflow: Workflow = Relationship()
 
     workflow_dump: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
 

From 33d09f934fc2cd41b03c74b1ce6c738f73fdb504 Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Mon, 13 Nov 2023 15:01:31 +0100
Subject: [PATCH 09/21] undo last commit

---
 fractal_server/app/models/job.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fractal_server/app/models/job.py b/fractal_server/app/models/job.py
index 33831d3ccb..92fbdc704a 100644
--- a/fractal_server/app/models/job.py
+++ b/fractal_server/app/models/job.py
@@ -13,6 +13,7 @@
 from ...utils import get_timestamp
 from ..schemas.applyworkflow import _ApplyWorkflowBase
 from .dataset import Dataset
+from .workflow import Workflow
 
 
 class JobStatusType(str, Enum):
@@ -107,6 +108,7 @@ class Config:
             primaryjoin="ApplyWorkflow.output_dataset_id==Dataset.id",
         )
     )
+    workflow: Workflow = Relationship()
 
     workflow_dump: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
 

From 76c00945c8fa61a3715f6d310da9014001ed9acf Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Mon, 13 Nov 2023 20:35:49 +0100
Subject: [PATCH 10/21] use enum in test

---
 tests/test_monitoring.py | 108 +++++++++++++++++++--------------------
 1 file changed, 54 insertions(+), 54 deletions(-)

diff --git a/tests/test_monitoring.py b/tests/test_monitoring.py
index f349d40fdb..9ea06f7b18 100644
--- a/tests/test_monitoring.py
+++ b/tests/test_monitoring.py
@@ -1,6 +1,8 @@
 from datetime import datetime
 
-PREFIX = "api/v1"
+from fractal_server.app.schemas.workflow import WorkflowTaskStatusType
+
+API = "api/v1"
 
 
 async def test_unauthorized_to_monitor(client, MockCurrentUser):
@@ -8,25 +10,25 @@ async def test_unauthorized_to_monitor(client, MockCurrentUser):
     async with MockCurrentUser(
         persist=True, user_kwargs={"is_superuser": False}
     ):
-        res = await client.get(f"{PREFIX}/monitoring/project/")
+        res = await client.get(f"{API}/monitoring/project/")
         assert res.status_code == 403
-        res = await client.get(f"{PREFIX}/monitoring/workflow/")
+        res = await client.get(f"{API}/monitoring/workflow/")
         assert res.status_code == 403
-        res = await client.get(f"{PREFIX}/monitoring/dataset/")
+        res = await client.get(f"{API}/monitoring/dataset/")
         assert res.status_code == 403
-        res = await client.get(f"{PREFIX}/monitoring/job/")
+        res = await client.get(f"{API}/monitoring/job/")
         assert res.status_code == 403
 
     async with MockCurrentUser(
         persist=True, user_kwargs={"is_superuser": True}
     ):
-        res = await client.get(f"{PREFIX}/monitoring/project/")
+        res = await client.get(f"{API}/monitoring/project/")
         assert res.status_code == 200
-        res = await client.get(f"{PREFIX}/monitoring/workflow/")
+        res = await client.get(f"{API}/monitoring/workflow/")
         assert res.status_code == 200
-        res = await client.get(f"{PREFIX}/monitoring/dataset/")
+        res = await client.get(f"{API}/monitoring/dataset/")
         assert res.status_code == 200
-        res = await client.get(f"{PREFIX}/monitoring/job/")
+        res = await client.get(f"{API}/monitoring/job/")
         assert res.status_code == 200
 
 
@@ -35,7 +37,7 @@ async def test_monitor_project(client, MockCurrentUser, project_factory):
     async with MockCurrentUser(
         persist=True, user_kwargs={"is_superuser": True}
     ):
-        res = await client.get(f"{PREFIX}/monitoring/project/")
+        res = await client.get(f"{API}/monitoring/project/")
         assert res.status_code == 200
         assert res.json() == []
 
@@ -49,10 +51,10 @@ async def test_monitor_project(client, MockCurrentUser, project_factory):
     async with MockCurrentUser(
         persist=True, user_kwargs={"is_superuser": True}
     ):
-        res = await client.get(f"{PREFIX}/monitoring/project/")
+        res = await client.get(f"{API}/monitoring/project/")
         assert res.status_code == 200
         assert len(res.json()) == 2
-        res = await client.get(f"{PREFIX}/monitoring/project/?id={prj1_id}")
+        res = await client.get(f"{API}/monitoring/project/?id={prj1_id}")
         assert res.status_code == 200
         assert len(res.json()) == 1
 
@@ -82,13 +84,13 @@ async def test_monitor_workflow(
         persist=True, user_kwargs={"is_superuser": True}
     ):
         # get all workflows
-        res = await client.get(f"{PREFIX}/monitoring/workflow/")
+        res = await client.get(f"{API}/monitoring/workflow/")
         assert res.status_code == 200
         assert len(res.json()) == 3
 
         # get workflows by id
         res = await client.get(
-            f"{PREFIX}/monitoring/workflow/?id={workflow1a.id}"
+            f"{API}/monitoring/workflow/?id={workflow1a.id}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
@@ -96,14 +98,14 @@ async def test_monitor_workflow(
 
         # get workflows by project_id
         res = await client.get(
-            f"{PREFIX}/monitoring/workflow/?project_id={project1.id}"
+            f"{API}/monitoring/workflow/?project_id={project1.id}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 2
 
         # get workflows by project_id and id
         res = await client.get(
-            f"{PREFIX}/monitoring/workflow/"
+            f"{API}/monitoring/workflow/"
             f"?project_id={project1.id}&id={workflow1b.id}"
         )
         assert res.status_code == 200
@@ -111,7 +113,7 @@ async def test_monitor_workflow(
         assert res.json()[0]["name"] == workflow1b.name
 
         res = await client.get(
-            f"{PREFIX}/monitoring/workflow/"
+            f"{API}/monitoring/workflow/"
             f"?project_id={project1.id}&id={workflow2a.id}"
         )
         assert res.status_code == 200
@@ -119,13 +121,13 @@ async def test_monitor_workflow(
 
         # get workflows by name
         res = await client.get(
-            f"{PREFIX}/monitoring/workflow/?name={workflow2a.name}"
+            f"{API}/monitoring/workflow/?name={workflow2a.name}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
         assert res.json()[0]["name"] == workflow2a.name
 
-        res = await client.get(f"{PREFIX}/monitoring/workflow/?name=Workflow")
+        res = await client.get(f"{API}/monitoring/workflow/?name=Workflow")
         assert res.status_code == 200
         assert len(res.json()) == 3
 
@@ -166,54 +168,54 @@ async def test_monitor_dataset(
         persist=True, user_kwargs={"is_superuser": True}
     ):
         # get all datasets
-        res = await client.get(f"{PREFIX}/monitoring/dataset/")
+        res = await client.get(f"{API}/monitoring/dataset/")
         assert res.status_code == 200
         assert len(res.json()) == 3
 
         # get datasets by id
-        res = await client.get(f"{PREFIX}/monitoring/dataset/?id={ds1a.id}")
+        res = await client.get(f"{API}/monitoring/dataset/?id={ds1a.id}")
         assert res.status_code == 200
         assert len(res.json()) == 1
         assert res.json()[0]["name"] == ds1a.name
-        res = await client.get(f"{PREFIX}/monitoring/dataset/?id=123456789")
+        res = await client.get(f"{API}/monitoring/dataset/?id=123456789")
         assert res.status_code == 200
         assert len(res.json()) == 0
 
         # get datasets by project_id
         res = await client.get(
-            f"{PREFIX}/monitoring/dataset/?project_id={project1.id}"
+            f"{API}/monitoring/dataset/?project_id={project1.id}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 2
         res = await client.get(
-            f"{PREFIX}/monitoring/dataset/?project_id={project2.id}"
+            f"{API}/monitoring/dataset/?project_id={project2.id}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
 
         # get datasets by name
         res = await client.get(
-            f"{PREFIX}/monitoring/dataset/?project_id={project1.id}&name=a"
+            f"{API}/monitoring/dataset/?project_id={project1.id}&name=a"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
         assert res.json()[0]["name"] == ds1a.name
         res = await client.get(
-            f"{PREFIX}/monitoring/dataset/?project_id={project1.id}&name=c"
+            f"{API}/monitoring/dataset/?project_id={project1.id}&name=c"
         )
         assert res.status_code == 200
         assert len(res.json()) == 0
 
         # get datasets by type and read_only
-        res = await client.get(f"{PREFIX}/monitoring/dataset/?type=zarr")
+        res = await client.get(f"{API}/monitoring/dataset/?type=zarr")
         assert res.status_code == 200
         assert len(res.json()) == 2
         res = await client.get(
-            f"{PREFIX}/monitoring/dataset/?type=zarr&read_only=true"
+            f"{API}/monitoring/dataset/?type=zarr&read_only=true"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
-        res = await client.get(f"{PREFIX}/monitoring/dataset/?read_only=true")
+        res = await client.get(f"{API}/monitoring/dataset/?read_only=true")
         assert res.status_code == 200
         assert len(res.json()) == 2
 
@@ -270,36 +272,36 @@ async def test_monitor_job(
         persist=True, user_kwargs={"is_superuser": True}
     ):
         # get all jobs
-        res = await client.get(f"{PREFIX}/monitoring/job/")
+        res = await client.get(f"{API}/monitoring/job/")
         assert res.status_code == 200
         assert len(res.json()) == 2
 
         # get jobs by id
-        res = await client.get(f"{PREFIX}/monitoring/job/?id={job1.id}")
+        res = await client.get(f"{API}/monitoring/job/?id={job1.id}")
         assert res.status_code == 200
         assert len(res.json()) == 1
 
         # get jobs by project_id
         res = await client.get(
-            f"{PREFIX}/monitoring/job/?project_id={project.id}"
+            f"{API}/monitoring/job/?project_id={project.id}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 2
         res = await client.get(
-            f"{PREFIX}/monitoring/job/?project_id={project.id + 123456789}"
+            f"{API}/monitoring/job/?project_id={project.id + 123456789}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 0
 
         # get jobs by input/output_dataset_id
         res = await client.get(
-            f"{PREFIX}/monitoring/job/?input_dataset_id={dataset1.id}"
+            f"{API}/monitoring/job/?input_dataset_id={dataset1.id}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
         assert res.json()[0]["id"] == job1.id
         res = await client.get(
-            f"{PREFIX}/monitoring/job/?output_dataset_id={dataset1.id}"
+            f"{API}/monitoring/job/?output_dataset_id={dataset1.id}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
@@ -307,68 +309,66 @@ async def test_monitor_job(
 
         # get jobs by workflow_id
         res = await client.get(
-            f"{PREFIX}/monitoring/job/?workflow_id={workflow2.id}"
+            f"{API}/monitoring/job/?workflow_id={workflow2.id}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
-        res = await client.get(
-            f"{PREFIX}/monitoring/job/?workflow_id=123456789"
-        )
+        res = await client.get(f"{API}/monitoring/job/?workflow_id=123456789")
         assert res.status_code == 200
         assert len(res.json()) == 0
 
         # get jobs by worfking_dir[_user]
-        res = await client.get(f"{PREFIX}/monitoring/job/?working_dir=aaaa")
+        res = await client.get(f"{API}/monitoring/job/?working_dir=aaaa")
         assert res.status_code == 200
         assert len(res.json()) == 1
-        res = await client.get(f"{PREFIX}/monitoring/job/?working_dir=1111")
+        res = await client.get(f"{API}/monitoring/job/?working_dir=1111")
         assert res.status_code == 200
         assert len(res.json()) == 2
-        res = await client.get(
-            f"{PREFIX}/monitoring/job/?working_dir_user=bbbb"
-        )
+        res = await client.get(f"{API}/monitoring/job/?working_dir_user=bbbb")
         assert res.status_code == 200
         assert len(res.json()) == 1
-        res = await client.get(
-            f"{PREFIX}/monitoring/job/?working_dir_user=2222"
-        )
+        res = await client.get(f"{API}/monitoring/job/?working_dir_user=2222")
         assert res.status_code == 200
         assert len(res.json()) == 2
 
         # get jobs by status
-        res = await client.get(f"{PREFIX}/monitoring/job/?status=fancy")
+        res = await client.get(
+            f"{API}/monitoring/job/?status={WorkflowTaskStatusType.FAILED}"
+        )
         assert res.status_code == 200
         assert len(res.json()) == 0
-        res = await client.get(f"{PREFIX}/monitoring/job/?status=submitted")
+        res = await client.get(
+            f"{API}/monitoring/job/?status={WorkflowTaskStatusType.SUBMITTED}"
+        )
         assert res.status_code == 200
         assert len(res.json()) == 2
 
         # get jobs by start/end_timestamp
 
         res = await client.get(
-            f"{PREFIX}/monitoring/job/?start_timestamp=1999-01-01T00:00:01"
+            f"{API}/monitoring/job/?start_timestamp=1999-01-01T00:00:01"
         )
         assert res.status_code == 200
         assert len(res.json()) == 2
         res = await client.get(
-            f"{PREFIX}/monitoring/job/?start_timestamp=2010-01-01T00:00:01"
+            f"{API}/monitoring/job/?start_timestamp=2010-01-01T00:00:01"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
         res = await client.get(
-            f"{PREFIX}/monitoring/job/?start_timestamp=2024-01-01T00:00:01"
+            f"{API}/monitoring/job/?start_timestamp=2024-01-01T00:00:01"
         )
         assert res.status_code == 200
         assert len(res.json()) == 0
 
         res = await client.get(
-            f"{PREFIX}/monitoring/job/?end_timestamp=3000-01-01T00:00:01"
+            f"{API}/monitoring/job/?end_timestamp=3000-01-01T00:00:01"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
 
         res = await client.get(
-            f"{PREFIX}/monitoring/job/?end_timestamp=2000-01-01T00:00:01"
+            f"{API}/monitoring/job/?end_timestamp=2000-01-01T00:00:01"
         )
         assert res.status_code == 200
         assert len(res.json()) == 0

From 7ddf972f59443a4760da8b8a75e54294cd97bcc1 Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Wed, 15 Nov 2023 15:46:58 +0100
Subject: [PATCH 11/21] add enum to type hint

---
 fractal_server/app/api/v1/monitoring.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fractal_server/app/api/v1/monitoring.py b/fractal_server/app/api/v1/monitoring.py
index 8b9255d6cc..eb7561fd2d 100644
--- a/fractal_server/app/api/v1/monitoring.py
+++ b/fractal_server/app/api/v1/monitoring.py
@@ -15,6 +15,7 @@
 from ...schemas import ProjectRead
 from ...security import current_active_superuser
 from ...security import User
+from fractal_server.app.schemas.workflow import WorkflowTaskStatusType
 
 router = APIRouter()
 
@@ -101,7 +102,7 @@ async def monitor_job(
     workflow_id: Optional[int] = None,
     working_dir: Optional[str] = None,
     working_dir_user: Optional[str] = None,
-    status: Optional[str] = None,
+    status: Optional[WorkflowTaskStatusType] = None,
     start_timestamp: Optional[DateTime] = None,
     end_timestamp: Optional[DateTime] = None,
     user: User = Depends(current_active_superuser),

From e6b6abd330a4c2598765aacba7d286d3631bf1e5 Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Wed, 15 Nov 2023 16:10:33 +0100
Subject: [PATCH 12/21] fix type hint

---
 fractal_server/app/api/v1/monitoring.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/fractal_server/app/api/v1/monitoring.py b/fractal_server/app/api/v1/monitoring.py
index eb7561fd2d..da31338e40 100644
--- a/fractal_server/app/api/v1/monitoring.py
+++ b/fractal_server/app/api/v1/monitoring.py
@@ -9,13 +9,14 @@
 from ...db import get_db
 from ...models import ApplyWorkflow
 from ...models import Dataset
+from ...models import JobStatusType
 from ...models import Project
 from ...models import Workflow
 from ...schemas import ApplyWorkflowRead
 from ...schemas import ProjectRead
 from ...security import current_active_superuser
 from ...security import User
-from fractal_server.app.schemas.workflow import WorkflowTaskStatusType
+
 
 router = APIRouter()
 
@@ -102,7 +103,7 @@ async def monitor_job(
     workflow_id: Optional[int] = None,
     working_dir: Optional[str] = None,
     working_dir_user: Optional[str] = None,
-    status: Optional[WorkflowTaskStatusType] = None,
+    status: Optional[JobStatusType] = None,
     start_timestamp: Optional[DateTime] = None,
     end_timestamp: Optional[DateTime] = None,
     user: User = Depends(current_active_superuser),

From c64ef8a8729c80bd29d592d4fd629bf7b5e2880c Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Wed, 15 Nov 2023 19:10:15 +0100
Subject: [PATCH 13/21] changes from review

---
 fractal_server/app/api/v1/monitoring.py | 72 +++++++++++-----------
 tests/test_monitoring.py                | 79 ++++++++++---------------
 2 files changed, 71 insertions(+), 80 deletions(-)

diff --git a/fractal_server/app/api/v1/monitoring.py b/fractal_server/app/api/v1/monitoring.py
index da31338e40..78cdaab149 100644
--- a/fractal_server/app/api/v1/monitoring.py
+++ b/fractal_server/app/api/v1/monitoring.py
@@ -3,6 +3,7 @@
 
 from fastapi import APIRouter
 from fastapi import Depends
+from sqlmodel import func
 from sqlmodel import select
 
 from ...db import AsyncSession
@@ -13,7 +14,9 @@
 from ...models import Project
 from ...models import Workflow
 from ...schemas import ApplyWorkflowRead
+from ...schemas import DatasetRead
 from ...schemas import ProjectRead
+from ...schemas import WorkflowRead
 from ...security import current_active_superuser
 from ...security import User
 
@@ -21,18 +24,22 @@
 router = APIRouter()
 
 
-@router.get("/project/")
+@router.get("/project/", response_model=list[ProjectRead])
 async def monitor_project(
     id: Optional[int] = None,
+    user_id: Optional[int] = None,
     user: User = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_db),
 ) -> list[ProjectRead]:
 
     stm = select(Project)
 
-    if id:
+    if id is not None:
         stm = stm.where(Project.id == id)
 
+    if user_id is not None:
+        stm = stm.where(Project.user_list.any(User.id == user_id))
+
     res = await db.execute(stm)
     project_list = res.scalars().all()
     await db.close()
@@ -40,52 +47,53 @@ async def monitor_project(
     return project_list
 
 
-@router.get("/workflow/")
+@router.get("/workflow/", response_model=list[WorkflowRead])
 async def monitor_workflow(
     id: Optional[int] = None,
     project_id: Optional[int] = None,
-    name: Optional[str] = None,
+    name_contains: Optional[str] = None,
     user: User = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_db),
-):
+) -> list[WorkflowRead]:
     stm = select(Workflow)
 
     if id is not None:
         stm = stm.where(Workflow.id == id)
     if project_id is not None:
         stm = stm.where(Workflow.project_id == project_id)
-    if name is not None:
-        stm = stm.where(Workflow.name.contains(name))
+    if name_contains is not None:
+        stm = stm.where(
+            func.lower(Workflow.name).contains(name_contains.lower())
+        )
 
     res = await db.execute(stm)
-    project_list = res.scalars().all()
+    workflow_list = res.scalars().all()
     await db.close()
 
-    return project_list
+    return workflow_list
 
 
-@router.get("/dataset/")
+@router.get("/dataset/", response_model=list[DatasetRead])
 async def monitor_dataset(
     id: Optional[int] = None,
     project_id: Optional[int] = None,
-    name: Optional[str] = None,
+    name_contains: Optional[str] = None,
     type: Optional[str] = None,
-    read_only: Optional[bool] = None,
     user: User = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_db),
-):
+) -> list[DatasetRead]:
     stm = select(Dataset)
 
     if id is not None:
         stm = stm.where(Dataset.id == id)
     if project_id is not None:
         stm = stm.where(Dataset.project_id == project_id)
-    if name is not None:
-        stm = stm.where(Dataset.name.contains(name))
+    if name_contains is not None:
+        stm = stm.where(
+            func.lower(Dataset.name).contains(name_contains.lower())
+        )
     if type is not None:
         stm = stm.where(Dataset.type == type)
-    if read_only is not None:
-        stm = stm.where(Dataset.read_only == read_only)
 
     res = await db.execute(stm)
     dataset_list = res.scalars().all()
@@ -94,21 +102,21 @@ async def monitor_dataset(
     return dataset_list
 
 
-@router.get("/job/")
+@router.get("/job/", response_model=list[ApplyWorkflowRead])
 async def monitor_job(
     id: Optional[int] = None,
     project_id: Optional[int] = None,
     input_dataset_id: Optional[int] = None,
     output_dataset_id: Optional[int] = None,
     workflow_id: Optional[int] = None,
-    working_dir: Optional[str] = None,
-    working_dir_user: Optional[str] = None,
     status: Optional[JobStatusType] = None,
-    start_timestamp: Optional[DateTime] = None,
-    end_timestamp: Optional[DateTime] = None,
+    start_timestamp_min: Optional[DateTime] = None,
+    start_timestamp_max: Optional[DateTime] = None,
+    end_timestamp_min: Optional[DateTime] = None,
+    end_timestamp_max: Optional[DateTime] = None,
     user: User = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_db),
-) -> Optional[list[ApplyWorkflowRead]]:
+) -> list[ApplyWorkflowRead]:
 
     stm = select(ApplyWorkflow)
 
@@ -122,18 +130,16 @@ async def monitor_job(
         stm = stm.where(ApplyWorkflow.output_dataset_id == output_dataset_id)
     if workflow_id is not None:
         stm = stm.where(ApplyWorkflow.workflow_id == workflow_id)
-    if working_dir is not None:
-        stm = stm.where(ApplyWorkflow.working_dir.contains(working_dir))
-    if working_dir_user is not None:
-        stm = stm.where(
-            ApplyWorkflow.working_dir_user.contains(working_dir_user)
-        )
     if status is not None:
         stm = stm.where(ApplyWorkflow.status == status)
-    if start_timestamp is not None:
-        stm = stm.where(ApplyWorkflow.start_timestamp >= start_timestamp)
-    if end_timestamp is not None:
-        stm = stm.where(ApplyWorkflow.end_timestamp <= end_timestamp)
+    if start_timestamp_min is not None:
+        stm = stm.where(ApplyWorkflow.start_timestamp >= start_timestamp_min)
+    if start_timestamp_max is not None:
+        stm = stm.where(ApplyWorkflow.start_timestamp <= start_timestamp_max)
+    if end_timestamp_min is not None:
+        stm = stm.where(ApplyWorkflow.end_timestamp >= end_timestamp_min)
+    if end_timestamp_max is not None:
+        stm = stm.where(ApplyWorkflow.end_timestamp <= end_timestamp_max)
 
     res = await db.execute(stm)
     job_list = res.scalars().all()
diff --git a/tests/test_monitoring.py b/tests/test_monitoring.py
index 9ea06f7b18..0eb2b009a0 100644
--- a/tests/test_monitoring.py
+++ b/tests/test_monitoring.py
@@ -1,6 +1,6 @@
 from datetime import datetime
 
-from fractal_server.app.schemas.workflow import WorkflowTaskStatusType
+from fractal_server.app.models import JobStatusType
 
 API = "api/v1"
 
@@ -36,10 +36,11 @@ async def test_monitor_project(client, MockCurrentUser, project_factory):
 
     async with MockCurrentUser(
         persist=True, user_kwargs={"is_superuser": True}
-    ):
+    ) as superuser:
         res = await client.get(f"{API}/monitoring/project/")
         assert res.status_code == 200
         assert res.json() == []
+        await project_factory(superuser)
 
     async with MockCurrentUser(
         persist=True, user_kwargs={"is_superuser": False}
@@ -47,16 +48,25 @@ async def test_monitor_project(client, MockCurrentUser, project_factory):
         project1 = await project_factory(user)
         prj1_id = project1.id
         await project_factory(user)
+        user_id = user.id
 
     async with MockCurrentUser(
         persist=True, user_kwargs={"is_superuser": True}
     ):
         res = await client.get(f"{API}/monitoring/project/")
         assert res.status_code == 200
-        assert len(res.json()) == 2
+        assert len(res.json()) == 3
         res = await client.get(f"{API}/monitoring/project/?id={prj1_id}")
         assert res.status_code == 200
         assert len(res.json()) == 1
+        res = await client.get(f"{API}/monitoring/project/?user_id={user_id}")
+        assert res.status_code == 200
+        assert len(res.json()) == 2
+        res = await client.get(
+            f"{API}/monitoring/project/?user_id={user_id}&id={prj1_id}"
+        )
+        assert res.status_code == 200
+        assert len(res.json()) == 1
 
 
 async def test_monitor_workflow(
@@ -121,13 +131,15 @@ async def test_monitor_workflow(
 
         # get workflows by name
         res = await client.get(
-            f"{API}/monitoring/workflow/?name={workflow2a.name}"
+            f"{API}/monitoring/workflow/?name_contains={workflow2a.name}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
         assert res.json()[0]["name"] == workflow2a.name
 
-        res = await client.get(f"{API}/monitoring/workflow/?name=Workflow")
+        res = await client.get(
+            f"{API}/monitoring/workflow/?name_contains=wOrKfLoW"
+        )
         assert res.status_code == 200
         assert len(res.json()) == 3
 
@@ -146,13 +158,11 @@ async def test_monitor_dataset(
             project_id=project1.id,
             name="ds1a",
             type="zarr",
-            read_only=False,
         )
         await dataset_factory(
             project_id=project1.id,
             name="ds1b",
             type="image",
-            read_only=True,
         )
 
         project2 = await project_factory(user)
@@ -161,7 +171,6 @@ async def test_monitor_dataset(
             project_id=project2.id,
             name="ds2a",
             type="zarr",
-            read_only=True,
         )
 
     async with MockCurrentUser(
@@ -195,29 +204,26 @@ async def test_monitor_dataset(
 
         # get datasets by name
         res = await client.get(
-            f"{API}/monitoring/dataset/?project_id={project1.id}&name=a"
+            f"{API}/monitoring/dataset/"
+            f"?project_id={project1.id}&name_contains=a"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
         assert res.json()[0]["name"] == ds1a.name
         res = await client.get(
-            f"{API}/monitoring/dataset/?project_id={project1.id}&name=c"
+            f"{API}/monitoring/dataset/"
+            f"?project_id={project1.id}&name_contains=c"
         )
         assert res.status_code == 200
         assert len(res.json()) == 0
 
-        # get datasets by type and read_only
+        # get datasets by type
         res = await client.get(f"{API}/monitoring/dataset/?type=zarr")
         assert res.status_code == 200
         assert len(res.json()) == 2
-        res = await client.get(
-            f"{API}/monitoring/dataset/?type=zarr&read_only=true"
-        )
+        res = await client.get(f"{API}/monitoring/dataset/?type=image")
         assert res.status_code == 200
         assert len(res.json()) == 1
-        res = await client.get(f"{API}/monitoring/dataset/?read_only=true")
-        assert res.status_code == 200
-        assert len(res.json()) == 2
 
 
 async def test_monitor_job(
@@ -293,7 +299,7 @@ async def test_monitor_job(
         assert res.status_code == 200
         assert len(res.json()) == 0
 
-        # get jobs by input/output_dataset_id
+        # get jobs by [input/output]_dataset_id
         res = await client.get(
             f"{API}/monitoring/job/?input_dataset_id={dataset1.id}"
         )
@@ -317,58 +323,37 @@ async def test_monitor_job(
         assert res.status_code == 200
         assert len(res.json()) == 0
 
-        # get jobs by worfking_dir[_user]
-        res = await client.get(f"{API}/monitoring/job/?working_dir=aaaa")
-        assert res.status_code == 200
-        assert len(res.json()) == 1
-        res = await client.get(f"{API}/monitoring/job/?working_dir=1111")
-        assert res.status_code == 200
-        assert len(res.json()) == 2
-        res = await client.get(f"{API}/monitoring/job/?working_dir_user=bbbb")
-        assert res.status_code == 200
-        assert len(res.json()) == 1
-        res = await client.get(f"{API}/monitoring/job/?working_dir_user=2222")
-        assert res.status_code == 200
-        assert len(res.json()) == 2
-
         # get jobs by status
         res = await client.get(
-            f"{API}/monitoring/job/?status={WorkflowTaskStatusType.FAILED}"
+            f"{API}/monitoring/job/?status={JobStatusType.FAILED}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 0
         res = await client.get(
-            f"{API}/monitoring/job/?status={WorkflowTaskStatusType.SUBMITTED}"
+            f"{API}/monitoring/job/?status={JobStatusType.SUBMITTED}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 2
 
-        # get jobs by start/end_timestamp
+        # get jobs by [start/end]_timestamp_[min/max]
 
         res = await client.get(
-            f"{API}/monitoring/job/?start_timestamp=1999-01-01T00:00:01"
+            f"{API}/monitoring/job/?start_timestamp_min=1999-01-01T00:00:01"
         )
         assert res.status_code == 200
         assert len(res.json()) == 2
         res = await client.get(
-            f"{API}/monitoring/job/?start_timestamp=2010-01-01T00:00:01"
+            f"{API}/monitoring/job/?start_timestamp_max=1999-01-01T00:00:01"
         )
         assert res.status_code == 200
-        assert len(res.json()) == 1
+        assert len(res.json()) == 0
         res = await client.get(
-            f"{API}/monitoring/job/?start_timestamp=2024-01-01T00:00:01"
+            f"{API}/monitoring/job/?end_timestamp_min=3000-01-01T00:00:01"
         )
         assert res.status_code == 200
         assert len(res.json()) == 0
-
         res = await client.get(
-            f"{API}/monitoring/job/?end_timestamp=3000-01-01T00:00:01"
+            f"{API}/monitoring/job/?end_timestamp_max=3000-01-01T00:00:01"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
-
-        res = await client.get(
-            f"{API}/monitoring/job/?end_timestamp=2000-01-01T00:00:01"
-        )
-        assert res.status_code == 200
-        assert len(res.json()) == 0

From d7b46d2eb1740b5929249ae96ed12e92b77ab335 Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Wed, 15 Nov 2023 19:13:58 +0100
Subject: [PATCH 14/21] update changelog [skip ci]

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 81cf726555..08c5ad29c0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@
 * API:
     * Make it possible to delete a `Dataset`, `Workflow` or `Project`, even when it is in relationship to an `ApplyWorkflow` (\#927).
     * Include `workflow_list` and `job_list` in `ProjectRead` response (\#927).
+    * New superuser-restricted monitoring endpoinds at `/api/v1/monitoring` (\#947).
 * Database:
     * Make foreign-keys of `ApplyWorkflow` (`project_id`, `workflow_id`, `input_dataset_id`, `output_dataset_id`) optional (\#927).
     * Add columns `input_dataset_dump`, `output_dataset_dump` and `user_email` to `ApplyWorkflow` (\#927).

From b11abf563fc892e5e54e21cf0cecac84fb440629 Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Thu, 16 Nov 2023 11:51:27 +0100
Subject: [PATCH 15/21] stash

---
 fractal_server/app/api/v1/dataset.py | 40 ++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/fractal_server/app/api/v1/dataset.py b/fractal_server/app/api/v1/dataset.py
index d340371f04..73a22af38c 100644
--- a/fractal_server/app/api/v1/dataset.py
+++ b/fractal_server/app/api/v1/dataset.py
@@ -144,6 +144,46 @@ async def delete_dataset(
     )
     dataset = output["dataset"]
 
+    #!
+
+    # Check whether there exists a job such that
+    # 1a. `job.input_dataset_id == dataset_id``
+    # OR
+    # 1b. `job.output_dataset_id == dataset_id`
+    # 2. `job.status` is either submitted or running
+    # Note: see
+    # https://sqlmodel.tiangolo.com/tutorial/where/#type-annotations-and-errors
+    # regarding the type-ignore in this code block
+    stm = (
+        select(ApplyWorkflow)
+        .where(ApplyWorkflow.output_dataset_id == dataset_id)
+        .where(
+            ApplyWorkflow.status.in_(
+                [JobStatusType.SUBMITTED, JobStatusType.RUNNING]
+            )
+        )
+    )
+    res = await db.execute(stm)
+
+    # If at least one such job exists, then this endpoint will fail. We do not
+    # support the use case of exporting a reproducible workflow when job
+    # execution is in progress; this may change in the future.
+    jobs = res.scalars().all()
+    if jobs:
+        if len(jobs) == 1:
+            string_ids = str(jobs[0].id)
+        else:
+            string_ids = str([job.id for job in jobs])[1:-1]
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Cannot export history because dataset {dataset.id} "
+                f"is linked to ongoing job(s) {string_ids}."
+            ),
+        )
+
+    #!
+
     await db.delete(dataset)
     await db.commit()
     await db.close()

From 1f7483b5a2986e600a1f450e58254aef1579a887 Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Thu, 16 Nov 2023 11:52:42 +0100
Subject: [PATCH 16/21] revert last commit

---
 fractal_server/app/api/v1/dataset.py | 40 ----------------------------
 1 file changed, 40 deletions(-)

diff --git a/fractal_server/app/api/v1/dataset.py b/fractal_server/app/api/v1/dataset.py
index 73a22af38c..d340371f04 100644
--- a/fractal_server/app/api/v1/dataset.py
+++ b/fractal_server/app/api/v1/dataset.py
@@ -144,46 +144,6 @@ async def delete_dataset(
     )
     dataset = output["dataset"]
 
-    #!
-
-    # Check whether there exists a job such that
-    # 1a. `job.input_dataset_id == dataset_id``
-    # OR
-    # 1b. `job.output_dataset_id == dataset_id`
-    # 2. `job.status` is either submitted or running
-    # Note: see
-    # https://sqlmodel.tiangolo.com/tutorial/where/#type-annotations-and-errors
-    # regarding the type-ignore in this code block
-    stm = (
-        select(ApplyWorkflow)
-        .where(ApplyWorkflow.output_dataset_id == dataset_id)
-        .where(
-            ApplyWorkflow.status.in_(
-                [JobStatusType.SUBMITTED, JobStatusType.RUNNING]
-            )
-        )
-    )
-    res = await db.execute(stm)
-
-    # If at least one such job exists, then this endpoint will fail. We do not
-    # support the use case of exporting a reproducible workflow when job
-    # execution is in progress; this may change in the future.
-    jobs = res.scalars().all()
-    if jobs:
-        if len(jobs) == 1:
-            string_ids = str(jobs[0].id)
-        else:
-            string_ids = str([job.id for job in jobs])[1:-1]
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=(
-                f"Cannot export history because dataset {dataset.id} "
-                f"is linked to ongoing job(s) {string_ids}."
-            ),
-        )
-
-    #!
-
     await db.delete(dataset)
     await db.commit()
     await db.close()

From ef8f18bf473492cfe1b4daa7eb72e7b2418fd195 Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Thu, 16 Nov 2023 11:58:26 +0100
Subject: [PATCH 17/21] func from sqlalchemy

---
 fractal_server/app/api/v1/monitoring.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fractal_server/app/api/v1/monitoring.py b/fractal_server/app/api/v1/monitoring.py
index 78cdaab149..e53e0f423c 100644
--- a/fractal_server/app/api/v1/monitoring.py
+++ b/fractal_server/app/api/v1/monitoring.py
@@ -3,7 +3,7 @@
 
 from fastapi import APIRouter
 from fastapi import Depends
-from sqlmodel import func
+from sqlalchemy import func
 from sqlmodel import select
 
 from ...db import AsyncSession
@@ -62,6 +62,7 @@ async def monitor_workflow(
     if project_id is not None:
         stm = stm.where(Workflow.project_id == project_id)
     if name_contains is not None:
+        # SQLAlchemy2: use icontains
         stm = stm.where(
             func.lower(Workflow.name).contains(name_contains.lower())
         )
@@ -89,6 +90,7 @@ async def monitor_dataset(
     if project_id is not None:
         stm = stm.where(Dataset.project_id == project_id)
     if name_contains is not None:
+        # SQLAlchemy2: use icontains
         stm = stm.where(
             func.lower(Dataset.name).contains(name_contains.lower())
         )

From d3b115cd8f2926a9c1b59ec3c8f55bf130c39bdc Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Thu, 16 Nov 2023 11:59:26 +0100
Subject: [PATCH 18/21] update changelog [no ci]

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 08c5ad29c0..1dad372910 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,7 +5,7 @@
 * API:
     * Make it possible to delete a `Dataset`, `Workflow` or `Project`, even when it is in relationship to an `ApplyWorkflow` (\#927).
     * Include `workflow_list` and `job_list` in `ProjectRead` response (\#927).
-    * New superuser-restricted monitoring endpoinds at `/api/v1/monitoring` (\#947).
+    * New monitoring endpoinds restricted to superusers at `/api/v1/monitoring` (\#947).
 * Database:
     * Make foreign-keys of `ApplyWorkflow` (`project_id`, `workflow_id`, `input_dataset_id`, `output_dataset_id`) optional (\#927).
     * Add columns `input_dataset_dump`, `output_dataset_dump` and `user_email` to `ApplyWorkflow` (\#927).

From 415e86985aeb754633b3738be6ee8afa61a63487 Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Thu, 16 Nov 2023 15:46:15 +0100
Subject: [PATCH 19/21] datetime lower case

---
 fractal_server/app/api/v1/monitoring.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/fractal_server/app/api/v1/monitoring.py b/fractal_server/app/api/v1/monitoring.py
index e53e0f423c..37b5302ca5 100644
--- a/fractal_server/app/api/v1/monitoring.py
+++ b/fractal_server/app/api/v1/monitoring.py
@@ -1,4 +1,4 @@
-from datetime import datetime as DateTime
+from datetime import datetime
 from typing import Optional
 
 from fastapi import APIRouter
@@ -112,10 +112,10 @@ async def monitor_job(
     output_dataset_id: Optional[int] = None,
     workflow_id: Optional[int] = None,
     status: Optional[JobStatusType] = None,
-    start_timestamp_min: Optional[DateTime] = None,
-    start_timestamp_max: Optional[DateTime] = None,
-    end_timestamp_min: Optional[DateTime] = None,
-    end_timestamp_max: Optional[DateTime] = None,
+    start_timestamp_min: Optional[datetime] = None,
+    start_timestamp_max: Optional[datetime] = None,
+    end_timestamp_min: Optional[datetime] = None,
+    end_timestamp_max: Optional[datetime] = None,
     user: User = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_db),
 ) -> list[ApplyWorkflowRead]:

From 1f5a3c487e900f01cb47c511f0485319de5db276 Mon Sep 17 00:00:00 2001
From: Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
Date: Thu, 16 Nov 2023 16:00:08 +0100
Subject: [PATCH 20/21] changes from review

---
 CHANGELOG.md                       |   2 +-
 fractal_server/app/api/__init__.py |   5 +-
 fractal_server/main.py             |   5 ++
 tests/test_monitoring.py           | 108 ++++++++++++-----------------
 4 files changed, 51 insertions(+), 69 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 18a211af29..a821106935 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,7 +5,7 @@
 * API:
     * Make it possible to delete a `Dataset`, `Workflow` or `Project`, even when it is in relationship to an `ApplyWorkflow` (\#927).
     * Include `workflow_list` and `job_list` in `ProjectRead` response (\#927).
-    * New monitoring endpoinds restricted to superusers at `/api/v1/monitoring` (\#947).
+    * New monitoring endpoints restricted to superusers at `/monitoring` (\#947).
     * Change response of `/api/v1/project/{project_id}/job/{job_id}/stop/` endpoint to 204 no-content (\#967).
     * Fix construction of `ApplyWorkflow.workflow_dump`, within apply endpoint (\#968).
 * Database:
diff --git a/fractal_server/app/api/__init__.py b/fractal_server/app/api/__init__.py
index 91053247ed..59dd3a7d8b 100644
--- a/fractal_server/app/api/__init__.py
+++ b/fractal_server/app/api/__init__.py
@@ -7,7 +7,7 @@
 from ...syringe import Inject
 from .v1.dataset import router as dataset_router
 from .v1.job import router as job_router
-from .v1.monitoring import router as monitoring_router
+from .v1.monitoring import router as router_monitoring  # noqa
 from .v1.project import router as project_router
 from .v1.task import router as task_router
 from .v1.task_collection import router as taskcollection_router
@@ -18,9 +18,6 @@
 router_default = APIRouter()
 router_v1 = APIRouter()
 
-router_v1.include_router(
-    monitoring_router, prefix="/monitoring", tags=["Monitoring"]
-)
 router_v1.include_router(project_router, prefix="/project", tags=["Projects"])
 router_v1.include_router(task_router, prefix="/task", tags=["Tasks"])
 router_v1.include_router(
diff --git a/fractal_server/main.py b/fractal_server/main.py
index 4d58381cac..dc2b1b7076 100644
--- a/fractal_server/main.py
+++ b/fractal_server/main.py
@@ -48,10 +48,15 @@ def collect_routers(app: FastAPI) -> None:
     """
     from .app.api import router_default
     from .app.api import router_v1
+    from .app.api import router_monitoring
+
     from .app.security import auth_router
 
     app.include_router(router_default, prefix="/api")
     app.include_router(router_v1, prefix="/api/v1")
+    app.include_router(
+        router_monitoring, prefix="/monitoring", tags=["Monitoring"]
+    )
     app.include_router(auth_router, prefix="/auth", tags=["auth"])
 
 
diff --git a/tests/test_monitoring.py b/tests/test_monitoring.py
index 0eb2b009a0..7cb3a72734 100644
--- a/tests/test_monitoring.py
+++ b/tests/test_monitoring.py
@@ -2,7 +2,7 @@
 
 from fractal_server.app.models import JobStatusType
 
-API = "api/v1"
+PREFIX = "/monitoring"
 
 
 async def test_unauthorized_to_monitor(client, MockCurrentUser):
@@ -10,25 +10,25 @@ async def test_unauthorized_to_monitor(client, MockCurrentUser):
     async with MockCurrentUser(
         persist=True, user_kwargs={"is_superuser": False}
     ):
-        res = await client.get(f"{API}/monitoring/project/")
+        res = await client.get(f"{PREFIX}/project/")
         assert res.status_code == 403
-        res = await client.get(f"{API}/monitoring/workflow/")
+        res = await client.get(f"{PREFIX}/workflow/")
         assert res.status_code == 403
-        res = await client.get(f"{API}/monitoring/dataset/")
+        res = await client.get(f"{PREFIX}/dataset/")
         assert res.status_code == 403
-        res = await client.get(f"{API}/monitoring/job/")
+        res = await client.get(f"{PREFIX}/job/")
         assert res.status_code == 403
 
     async with MockCurrentUser(
         persist=True, user_kwargs={"is_superuser": True}
     ):
-        res = await client.get(f"{API}/monitoring/project/")
+        res = await client.get(f"{PREFIX}/project/")
         assert res.status_code == 200
-        res = await client.get(f"{API}/monitoring/workflow/")
+        res = await client.get(f"{PREFIX}/workflow/")
         assert res.status_code == 200
-        res = await client.get(f"{API}/monitoring/dataset/")
+        res = await client.get(f"{PREFIX}/dataset/")
         assert res.status_code == 200
-        res = await client.get(f"{API}/monitoring/job/")
+        res = await client.get(f"{PREFIX}/job/")
         assert res.status_code == 200
 
 
@@ -37,7 +37,7 @@ async def test_monitor_project(client, MockCurrentUser, project_factory):
     async with MockCurrentUser(
         persist=True, user_kwargs={"is_superuser": True}
     ) as superuser:
-        res = await client.get(f"{API}/monitoring/project/")
+        res = await client.get(f"{PREFIX}/project/")
         assert res.status_code == 200
         assert res.json() == []
         await project_factory(superuser)
@@ -53,17 +53,17 @@ async def test_monitor_project(client, MockCurrentUser, project_factory):
     async with MockCurrentUser(
         persist=True, user_kwargs={"is_superuser": True}
     ):
-        res = await client.get(f"{API}/monitoring/project/")
+        res = await client.get(f"{PREFIX}/project/")
         assert res.status_code == 200
         assert len(res.json()) == 3
-        res = await client.get(f"{API}/monitoring/project/?id={prj1_id}")
+        res = await client.get(f"{PREFIX}/project/?id={prj1_id}")
         assert res.status_code == 200
         assert len(res.json()) == 1
-        res = await client.get(f"{API}/monitoring/project/?user_id={user_id}")
+        res = await client.get(f"{PREFIX}/project/?user_id={user_id}")
         assert res.status_code == 200
         assert len(res.json()) == 2
         res = await client.get(
-            f"{API}/monitoring/project/?user_id={user_id}&id={prj1_id}"
+            f"{PREFIX}/project/?user_id={user_id}&id={prj1_id}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
@@ -94,28 +94,24 @@ async def test_monitor_workflow(
         persist=True, user_kwargs={"is_superuser": True}
     ):
         # get all workflows
-        res = await client.get(f"{API}/monitoring/workflow/")
+        res = await client.get(f"{PREFIX}/workflow/")
         assert res.status_code == 200
         assert len(res.json()) == 3
 
         # get workflows by id
-        res = await client.get(
-            f"{API}/monitoring/workflow/?id={workflow1a.id}"
-        )
+        res = await client.get(f"{PREFIX}/workflow/?id={workflow1a.id}")
         assert res.status_code == 200
         assert len(res.json()) == 1
         assert res.json()[0]["name"] == workflow1a.name
 
         # get workflows by project_id
-        res = await client.get(
-            f"{API}/monitoring/workflow/?project_id={project1.id}"
-        )
+        res = await client.get(f"{PREFIX}/workflow/?project_id={project1.id}")
         assert res.status_code == 200
         assert len(res.json()) == 2
 
         # get workflows by project_id and id
         res = await client.get(
-            f"{API}/monitoring/workflow/"
+            f"{PREFIX}/workflow/"
             f"?project_id={project1.id}&id={workflow1b.id}"
         )
         assert res.status_code == 200
@@ -123,7 +119,7 @@ async def test_monitor_workflow(
         assert res.json()[0]["name"] == workflow1b.name
 
         res = await client.get(
-            f"{API}/monitoring/workflow/"
+            f"{PREFIX}/workflow/"
             f"?project_id={project1.id}&id={workflow2a.id}"
         )
         assert res.status_code == 200
@@ -131,15 +127,13 @@ async def test_monitor_workflow(
 
         # get workflows by name
         res = await client.get(
-            f"{API}/monitoring/workflow/?name_contains={workflow2a.name}"
+            f"{PREFIX}/workflow/?name_contains={workflow2a.name}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
         assert res.json()[0]["name"] == workflow2a.name
 
-        res = await client.get(
-            f"{API}/monitoring/workflow/?name_contains=wOrKfLoW"
-        )
+        res = await client.get(f"{PREFIX}/workflow/?name_contains=wOrKfLoW")
         assert res.status_code == 200
         assert len(res.json()) == 3
 
@@ -177,51 +171,45 @@ async def test_monitor_dataset(
         persist=True, user_kwargs={"is_superuser": True}
     ):
         # get all datasets
-        res = await client.get(f"{API}/monitoring/dataset/")
+        res = await client.get(f"{PREFIX}/dataset/")
         assert res.status_code == 200
         assert len(res.json()) == 3
 
         # get datasets by id
-        res = await client.get(f"{API}/monitoring/dataset/?id={ds1a.id}")
+        res = await client.get(f"{PREFIX}/dataset/?id={ds1a.id}")
         assert res.status_code == 200
         assert len(res.json()) == 1
         assert res.json()[0]["name"] == ds1a.name
-        res = await client.get(f"{API}/monitoring/dataset/?id=123456789")
+        res = await client.get(f"{PREFIX}/dataset/?id=123456789")
         assert res.status_code == 200
         assert len(res.json()) == 0
 
         # get datasets by project_id
-        res = await client.get(
-            f"{API}/monitoring/dataset/?project_id={project1.id}"
-        )
+        res = await client.get(f"{PREFIX}/dataset/?project_id={project1.id}")
         assert res.status_code == 200
         assert len(res.json()) == 2
-        res = await client.get(
-            f"{API}/monitoring/dataset/?project_id={project2.id}"
-        )
+        res = await client.get(f"{PREFIX}/dataset/?project_id={project2.id}")
         assert res.status_code == 200
         assert len(res.json()) == 1
 
         # get datasets by name
         res = await client.get(
-            f"{API}/monitoring/dataset/"
-            f"?project_id={project1.id}&name_contains=a"
+            f"{PREFIX}/dataset/" f"?project_id={project1.id}&name_contains=a"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
         assert res.json()[0]["name"] == ds1a.name
         res = await client.get(
-            f"{API}/monitoring/dataset/"
-            f"?project_id={project1.id}&name_contains=c"
+            f"{PREFIX}/dataset/" f"?project_id={project1.id}&name_contains=c"
         )
         assert res.status_code == 200
         assert len(res.json()) == 0
 
         # get datasets by type
-        res = await client.get(f"{API}/monitoring/dataset/?type=zarr")
+        res = await client.get(f"{PREFIX}/dataset/?type=zarr")
         assert res.status_code == 200
         assert len(res.json()) == 2
-        res = await client.get(f"{API}/monitoring/dataset/?type=image")
+        res = await client.get(f"{PREFIX}/dataset/?type=image")
         assert res.status_code == 200
         assert len(res.json()) == 1
 
@@ -278,59 +266,51 @@ async def test_monitor_job(
         persist=True, user_kwargs={"is_superuser": True}
     ):
         # get all jobs
-        res = await client.get(f"{API}/monitoring/job/")
+        res = await client.get(f"{PREFIX}/job/")
         assert res.status_code == 200
         assert len(res.json()) == 2
 
         # get jobs by id
-        res = await client.get(f"{API}/monitoring/job/?id={job1.id}")
+        res = await client.get(f"{PREFIX}/job/?id={job1.id}")
         assert res.status_code == 200
         assert len(res.json()) == 1
 
         # get jobs by project_id
-        res = await client.get(
-            f"{API}/monitoring/job/?project_id={project.id}"
-        )
+        res = await client.get(f"{PREFIX}/job/?project_id={project.id}")
         assert res.status_code == 200
         assert len(res.json()) == 2
         res = await client.get(
-            f"{API}/monitoring/job/?project_id={project.id + 123456789}"
+            f"{PREFIX}/job/?project_id={project.id + 123456789}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 0
 
         # get jobs by [input/output]_dataset_id
-        res = await client.get(
-            f"{API}/monitoring/job/?input_dataset_id={dataset1.id}"
-        )
+        res = await client.get(f"{PREFIX}/job/?input_dataset_id={dataset1.id}")
         assert res.status_code == 200
         assert len(res.json()) == 1
         assert res.json()[0]["id"] == job1.id
         res = await client.get(
-            f"{API}/monitoring/job/?output_dataset_id={dataset1.id}"
+            f"{PREFIX}/job/?output_dataset_id={dataset1.id}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1
         assert res.json()[0]["id"] == job2.id
 
         # get jobs by workflow_id
-        res = await client.get(
-            f"{API}/monitoring/job/?workflow_id={workflow2.id}"
-        )
+        res = await client.get(f"{PREFIX}/job/?workflow_id={workflow2.id}")
         assert res.status_code == 200
         assert len(res.json()) == 1
-        res = await client.get(f"{API}/monitoring/job/?workflow_id=123456789")
+        res = await client.get(f"{PREFIX}/job/?workflow_id=123456789")
         assert res.status_code == 200
         assert len(res.json()) == 0
 
         # get jobs by status
-        res = await client.get(
-            f"{API}/monitoring/job/?status={JobStatusType.FAILED}"
-        )
+        res = await client.get(f"{PREFIX}/job/?status={JobStatusType.FAILED}")
         assert res.status_code == 200
         assert len(res.json()) == 0
         res = await client.get(
-            f"{API}/monitoring/job/?status={JobStatusType.SUBMITTED}"
+            f"{PREFIX}/job/?status={JobStatusType.SUBMITTED}"
         )
         assert res.status_code == 200
         assert len(res.json()) == 2
@@ -338,22 +318,22 @@ async def test_monitor_job(
         # get jobs by [start/end]_timestamp_[min/max]
 
         res = await client.get(
-            f"{API}/monitoring/job/?start_timestamp_min=1999-01-01T00:00:01"
+            f"{PREFIX}/job/?start_timestamp_min=1999-01-01T00:00:01"
         )
         assert res.status_code == 200
         assert len(res.json()) == 2
         res = await client.get(
-            f"{API}/monitoring/job/?start_timestamp_max=1999-01-01T00:00:01"
+            f"{PREFIX}/job/?start_timestamp_max=1999-01-01T00:00:01"
         )
         assert res.status_code == 200
         assert len(res.json()) == 0
         res = await client.get(
-            f"{API}/monitoring/job/?end_timestamp_min=3000-01-01T00:00:01"
+            f"{PREFIX}/job/?end_timestamp_min=3000-01-01T00:00:01"
         )
         assert res.status_code == 200
         assert len(res.json()) == 0
         res = await client.get(
-            f"{API}/monitoring/job/?end_timestamp_max=3000-01-01T00:00:01"
+            f"{PREFIX}/job/?end_timestamp_max=3000-01-01T00:00:01"
         )
         assert res.status_code == 200
         assert len(res.json()) == 1

From b22dd33c9f889ebea5131bed9e448e574c0a0f43 Mon Sep 17 00:00:00 2001
From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com>
Date: Thu, 16 Nov 2023 16:12:09 +0100
Subject: [PATCH 21/21] Update CHANGELOG [skip ci]

---
 CHANGELOG.md | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 50d86f9fd3..ed7f16bd1b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,14 +2,16 @@
 
 # 1.4.0 (unreleased)
 
-* API (new endpoints):
-    * New monitoring endpoints restricted to superusers at `/monitoring` (\#947).
-    * New `GET` endpoints `api/v1/project/job/` and `api/v1/project/{project_id}/workflow/{workflow_id}/job/` (\#969).
-* API (other changes):
-    * Make it possible to delete a `Dataset`, `Workflow` or `Project`, even when it is in relationship to an `ApplyWorkflow` (\#927).
-    * Change response of `/api/v1/project/{project_id}/job/{job_id}/stop/` endpoint to 204 no-content (\#967).
-    * Include `workflow_list` and `job_list` in `ProjectRead` response (\#927).
-    * Fix construction of `ApplyWorkflow.workflow_dump`, within apply endpoint (\#968).
+* API:
+    * New endpoints:
+        * New monitoring endpoints restricted to superusers at `/monitoring` (\#947).
+        * New `GET` endpoints `api/v1/project/job/` and `api/v1/project/{project_id}/workflow/{workflow_id}/job/` (\#969).
+    * New behaviors or responses of existing endpoints:
+        * Change response of `/api/v1/project/{project_id}/job/{job_id}/stop/` endpoint to 204 no-content (\#967).
+        * Include `workflow_list` and `job_list` attributes for `ProjectRead`, which affects all `GET`-project endpoints (\#927).
+        * Make it possible to delete a `Dataset`, `Workflow` or `Project`, even when it is in relationship to an `ApplyWorkflow` (\#927).
+    * Internal changes:
+        * Fix construction of `ApplyWorkflow.workflow_dump`, within apply endpoint (\#968).
 * Database:
     * Make foreign-keys of `ApplyWorkflow` (`project_id`, `workflow_id`, `input_dataset_id`, `output_dataset_id`) optional (\#927).
     * Add columns `input_dataset_dump`, `output_dataset_dump` and `user_email` to `ApplyWorkflow` (\#927).