Enable streaming for websrc card loader in configuration

Signed-off-by: elronbandel <[email protected]>
IBM · Jan 30, 2025 · a6fd3dd · a6fd3dd
1 parent a0e7d0d
commit a6fd3dd
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/prepare/cards/websrc.py b/prepare/cards/websrc.py
@@ -15,7 +15,7 @@
 )
 
 card = TaskCard(
-    loader=LoadHF(path="rootsautomation/websrc"),
+    loader=LoadHF(path="rootsautomation/websrc", streaming=True),
     preprocess_steps=[
         RenameSplits(mapper={"train": "train", "dev": "test"}),
         "splitters.small_no_dev",

diff --git a/src/unitxt/catalog/cards/websrc.json b/src/unitxt/catalog/cards/websrc.json
@@ -2,7 +2,8 @@
     "__type__": "task_card",
     "loader": {
         "__type__": "load_hf",
-        "path": "rootsautomation/websrc"
+        "path": "rootsautomation/websrc",
+        "streaming": true
     },
     "preprocess_steps": [
         {