Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix task creation with gt job and gt job frame access #8510

Merged
merged 14 commits into from
Oct 7, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Update data tests to use chunk size
zhiltsov-max committed Oct 4, 2024
commit e03cfe015496a71cbad25369ffead23ac2a50a8c
29 changes: 23 additions & 6 deletions tests/python/rest_api/test_tasks.py
Original file line number Diff line number Diff line change
@@ -2641,10 +2641,16 @@ def _uploaded_images_task_fxt_base(
frame_count = len(image_files)

images_data = [f.getvalue() for f in image_files]

resulting_task_size = len(
range(start_frame or 0, (stop_frame or len(images_data) - 1) + 1, step or 1)
)

data_params = {
"image_quality": 70,
"client_files": image_files,
"sorting_method": "natural",
"chunk_size": max(1, (segment_size or resulting_task_size) // 2),
}
data_params.update(data_kwargs)

@@ -2665,7 +2671,7 @@ def get_frame(i: int) -> bytes:
models.TaskWriteRequest._from_openapi_data(**task_params),
models.DataRequest._from_openapi_data(**data_params),
get_frame=get_frame,
size=len(range(start_frame or 0, (stop_frame or len(images_data) - 1) + 1, step or 1)),
size=resulting_task_size,
), task_id

@pytest.fixture(scope="class")
@@ -2863,11 +2869,16 @@ def _uploaded_video_task_fxt_base(
if segment_size:
task_params["segment_size"] = segment_size

resulting_task_size = len(
range(start_frame or 0, (stop_frame or frame_count - 1) + 1, step or 1)
)

video_file = generate_video_file(frame_count)
video_data = video_file.getvalue()
data_params = {
"image_quality": 70,
"client_files": [video_file],
"chunk_size": max(1, (segment_size or resulting_task_size) // 2),
}

if start_frame is not None:
@@ -2887,7 +2898,7 @@ def get_video_file() -> io.BytesIO:
models.TaskWriteRequest._from_openapi_data(**task_params),
models.DataRequest._from_openapi_data(**data_params),
get_video_file=get_video_file,
size=len(range(start_frame or 0, (stop_frame or frame_count - 1) + 1, step or 1)),
size=resulting_task_size,
), task_id

@pytest.fixture(scope="class")
@@ -3076,13 +3087,17 @@ def test_can_get_task_chunks(self, task_spec: _TaskSpec, task_id: int):
else:
assert False

task_frames = range(
task_abs_frames = range(
task_meta.start_frame, task_meta.stop_frame + 1, task_spec.frame_step
)
task_chunk_frames = [
(chunk_number, list(chunk_frames))
for chunk_number, chunk_frames in groupby(
task_frames, key=lambda frame: frame // task_meta.chunk_size
task_abs_frames,
key=lambda abs_frame: (
(abs_frame - task_meta.start_frame) // task_spec.frame_step
)
// task_meta.chunk_size,
)
]
for quality, (chunk_id, expected_chunk_frame_ids) in product(
@@ -3260,7 +3275,7 @@ def test_can_get_job_chunks(self, task_spec: _TaskSpec, task_id: int, indexing:
assert False

if indexing == "absolute":
chunk_count = math.ceil(task_meta.size / job_meta.chunk_size)
chunk_count = math.ceil(task_meta.size / task_meta.chunk_size)

def get_task_chunk_abs_frame_ids(chunk_id: int) -> Sequence[int]:
return range(
@@ -3337,7 +3352,9 @@ def get_expected_chunk_abs_frame_ids(chunk_id: int):
else:
chunk_images = dict(enumerate(read_video_file(chunk_file)))

assert sorted(chunk_images.keys()) == list(range(job_meta.size))
assert sorted(chunk_images.keys()) == list(
range(len(expected_chunk_abs_frame_ids))
)

for chunk_frame, abs_frame_id in zip(
chunk_images, expected_chunk_abs_frame_ids