fix: ensure UTF-8 encoding when reading text files in ingestion helper

zylon-ai · Jan 23, 2025 · b77f90e · b77f90e
1 parent b7ee437
commit b77f90e
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/private_gpt/components/ingest/ingest_helper.py b/private_gpt/components/ingest/ingest_helper.py
@@ -89,7 +89,7 @@ def _load_file_to_documents(file_name: str, file_data: Path) -> list[Document]:
             )
             # Read as a plain text
             string_reader = StringIterableReader()
-            return string_reader.load_data([file_data.read_text()])
+            return string_reader.load_data([file_data.read_text(encoding="utf8")])
 
         logger.debug("Specific reader found for extension=%s", extension)
         documents = reader_cls().load_data(file_data)