Explicitly setup delimiter for CSV formatting

nicolay-r · Jun 2, 2023 · 778c8f1 · 778c8f1
1 parent 23a0223
commit 778c8f1
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/arelight/pipelines/demo/infer_bert_rus.py b/arelight/pipelines/demo/infer_bert_rus.py
@@ -35,7 +35,7 @@ def demo_infer_texts_bert_pipeline(texts_count,
     samples_io = SamplesIO(target_dir=samples_output_dir,
                            reader=PandasCsvReader(sep=',', compression="infer"),
                            prefix=samples_prefix,
-                           writer=NativeCsvWriter())
+                           writer=NativeCsvWriter(delimiter=','))
 
     pipeline = BasePipeline(pipeline=[
 

diff --git a/examples/serialize_bert.py b/examples/serialize_bert.py
@@ -102,7 +102,7 @@
             storage=RowCacheStorage(),
             samples_io=SamplesIO(target_dir=dirname(backend_template),
                                  prefix=basename(backend_template),
-                                 writer=NativeCsvWriter()),
+                                 writer=NativeCsvWriter(delimiter=',')),
             save_labels_func=lambda data_type: data_type != DataType.Test,
             balance_func=lambda data_type: data_type == DataType.Train)
     ])

diff --git a/test/test_bert_serialization.py b/test/test_bert_serialization.py
@@ -112,7 +112,7 @@ def test(self):
             BertExperimentInputSerializerPipelineItem(
                 rows_provider=rows_provider,
                 storage=RowCacheStorage(),
-                samples_io=SamplesIO(target_dir=self.TEST_DATA_DIR, writer=NativeCsvWriter()),
+                samples_io=SamplesIO(target_dir=self.TEST_DATA_DIR, writer=NativeCsvWriter(delimiter=',')),
                 save_labels_func=lambda data_type: data_type != DataType.Test,
                 balance_func=lambda data_type: data_type == DataType.Train)
         ])