Skip to content

Commit

Permalink
fix: big xls file error (#4859)
Browse files Browse the repository at this point in the history
### What problem does this PR solve?

if *.xls file is too large, .eg >50M, I get error.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
  • Loading branch information
SkyfireWXY authored Feb 12, 2025
1 parent a1cf792 commit 8fcca1b
Showing 1 changed file with 72 additions and 11 deletions.
83 changes: 72 additions & 11 deletions deepdoc/parser/excel_parser.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
Expand All @@ -14,19 +11,51 @@
# limitations under the License.
#

from openpyxl import load_workbook
from openpyxl import load_workbook, Workbook
import sys
from io import BytesIO

from rag.nlp import find_codec

import pandas as pd


class RAGFlowExcelParser:
def html(self, fnm, chunk_rows=256):
if isinstance(fnm, str):
wb = load_workbook(fnm)

# if isinstance(fnm, str):
# wb = load_workbook(fnm)
# else:
# wb = load_workbook(BytesIO(fnm))++

s_fnm = fnm
if not isinstance(fnm, str):
s_fnm = BytesIO(fnm)
else:
wb = load_workbook(BytesIO(fnm))
pass

try:
wb = load_workbook(s_fnm)
except Exception as e:
print(f'****wxy: file parser error: {e}, s_fnm={s_fnm}, trying convert files')
df = pd.read_excel(s_fnm)
wb = Workbook()
# if len(wb.worksheets) > 0:
# del wb.worksheets[0]
# else: pass
ws = wb.active
ws.title = "Data"
for col_num, column_name in enumerate(df.columns, 1):
ws.cell(row=1, column=col_num, value=column_name)
else:
pass
for row_num, row in enumerate(df.values, 2):
for col_num, value in enumerate(row, 1):
ws.cell(row=row_num, column=col_num, value=value)
else:
pass
else:
pass

tb_chunks = []
for sheetname in wb.sheetnames:
Expand All @@ -45,7 +74,7 @@ def html(self, fnm, chunk_rows=256):
tb += f"<table><caption>{sheetname}</caption>"
tb += tb_rows_0
for r in list(
rows[1 + chunk_i * chunk_rows : 1 + (chunk_i + 1) * chunk_rows]
rows[1 + chunk_i * chunk_rows: 1 + (chunk_i + 1) * chunk_rows]
):
tb += "<tr>"
for i, c in enumerate(r):
Expand All @@ -60,10 +89,41 @@ def html(self, fnm, chunk_rows=256):
return tb_chunks

def __call__(self, fnm):
if isinstance(fnm, str):
wb = load_workbook(fnm)
# if isinstance(fnm, str):
# wb = load_workbook(fnm)
# else:
# wb = load_workbook(BytesIO(fnm))

s_fnm = fnm
if not isinstance(fnm, str):
s_fnm = BytesIO(fnm)
else:
wb = load_workbook(BytesIO(fnm))
pass

try:
wb = load_workbook(s_fnm)
except Exception as e:
print(f'****wxy: file parser error: {e}, s_fnm={s_fnm}, trying convert files')
df = pd.read_excel(s_fnm)
wb = Workbook()
if len(wb.worksheets) > 0:
del wb.worksheets[0]
else:
pass
ws = wb.active
ws.title = "Data"
for col_num, column_name in enumerate(df.columns, 1):
ws.cell(row=1, column=col_num, value=column_name)
else:
pass
for row_num, row in enumerate(df.values, 2):
for col_num, value in enumerate(row, 1):
ws.cell(row=row_num, column=col_num, value=value)
else:
pass
else:
pass

res = []
for sheetname in wb.sheetnames:
ws = wb[sheetname]
Expand Down Expand Up @@ -104,3 +164,4 @@ def row_number(fnm, binary):
if __name__ == "__main__":
psr = RAGFlowExcelParser()
psr(sys.argv[1])

0 comments on commit 8fcca1b

Please sign in to comment.