From a2a6a35e9428f3c6e3f44938d80965c36bc6dc6d Mon Sep 17 00:00:00 2001 From: KevinHuSh Date: Fri, 17 May 2024 15:35:09 +0800 Subject: [PATCH] fix doc number miss-match issue (#822) ### What problem does this PR solve? #620 ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) --- api/apps/document_app.py | 6 +----- api/apps/file2document_app.py | 12 ++---------- api/apps/file_app.py | 6 +----- api/apps/kb_app.py | 7 +------ api/db/services/document_service.py | 19 ++----------------- 5 files changed, 7 insertions(+), 43 deletions(-) diff --git a/api/apps/document_app.py b/api/apps/document_app.py index f89c619b222..f8485e56044 100644 --- a/api/apps/document_app.py +++ b/api/apps/document_app.py @@ -250,13 +250,9 @@ def rm(): if not tenant_id: return get_data_error_result(retmsg="Tenant not found!") - ELASTICSEARCH.deleteByQuery( - Q("match", doc_id=doc.id), idxnm=search.index_name(tenant_id)) - - DocumentService.clear_chunk_num(doc_id) b, n = File2DocumentService.get_minio_address(doc_id=doc_id) - if not DocumentService.delete(doc): + if not DocumentService.remove_document(doc, tenant_id): return get_data_error_result( retmsg="Database error (Document removal)!") diff --git a/api/apps/file2document_app.py b/api/apps/file2document_app.py index d3861b59b3f..0e7301bac11 100644 --- a/api/apps/file2document_app.py +++ b/api/apps/file2document_app.py @@ -58,11 +58,7 @@ def convert(): tenant_id = DocumentService.get_tenant_id(doc_id) if not tenant_id: return get_data_error_result(retmsg="Tenant not found!") - ELASTICSEARCH.deleteByQuery( - Q("match", doc_id=doc.id), idxnm=search.index_name(tenant_id)) - DocumentService.increment_chunk_num( - doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, 0) - if not DocumentService.delete(doc): + if not DocumentService.remove_document(doc, tenant_id): return get_data_error_result( retmsg="Database error (Document removal)!") File2DocumentService.delete_by_file_id(id) @@ -125,11 +121,7 @@ def rm(): tenant_id = DocumentService.get_tenant_id(doc_id) if not tenant_id: return get_data_error_result(retmsg="Tenant not found!") - ELASTICSEARCH.deleteByQuery( - Q("match", doc_id=doc.id), idxnm=search.index_name(tenant_id)) - DocumentService.increment_chunk_num( - doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, 0) - if not DocumentService.delete(doc): + if not DocumentService.remove_document(doc, tenant_id): return get_data_error_result( retmsg="Database error (Document removal)!") return get_json_result(data=True) diff --git a/api/apps/file_app.py b/api/apps/file_app.py index 234ee74191c..987321ee188 100644 --- a/api/apps/file_app.py +++ b/api/apps/file_app.py @@ -277,11 +277,7 @@ def rm(): tenant_id = DocumentService.get_tenant_id(doc_id) if not tenant_id: return get_data_error_result(retmsg="Tenant not found!") - ELASTICSEARCH.deleteByQuery( - Q("match", doc_id=doc.id), idxnm=search.index_name(tenant_id)) - DocumentService.increment_chunk_num( - doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, 0) - if not DocumentService.delete(doc): + if not DocumentService.remove_document(doc, tenant_id): return get_data_error_result( retmsg="Database error (Document removal)!") File2DocumentService.delete_by_file_id(file_id) diff --git a/api/apps/kb_app.py b/api/apps/kb_app.py index c45e08a76da..f62d8516032 100644 --- a/api/apps/kb_app.py +++ b/api/apps/kb_app.py @@ -136,12 +136,7 @@ def rm(): data=False, retmsg=f'Only owner of knowledgebase authorized for this operation.', retcode=RetCode.OPERATING_ERROR) for doc in DocumentService.query(kb_id=req["kb_id"]): - ELASTICSEARCH.deleteByQuery( - Q("match", doc_id=doc.id), idxnm=search.index_name(kbs[0].tenant_id)) - - DocumentService.increment_chunk_num( - doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, 0) - if not DocumentService.delete(doc): + if not DocumentService.remove_document(doc, kbs[0].tenant_id): return get_data_error_result( retmsg="Database error (Document removal)!") diff --git a/api/db/services/document_service.py b/api/db/services/document_service.py index d85c15070fc..c85bfcd115f 100644 --- a/api/db/services/document_service.py +++ b/api/db/services/document_service.py @@ -70,27 +70,12 @@ def insert(cls, doc): raise RuntimeError("Database error (Knowledgebase)!") return doc - @classmethod - @DB.connection_context() - def delete(cls, doc): - e, kb = KnowledgebaseService.get_by_id(doc.kb_id) - if not KnowledgebaseService.update_by_id( - kb.id, {"doc_num": max(0, kb.doc_num - 1)}): - raise RuntimeError("Database error (Knowledgebase)!") - return cls.delete_by_id(doc.id) - @classmethod @DB.connection_context() def remove_document(cls, doc, tenant_id): ELASTICSEARCH.deleteByQuery( - Q("match", doc_id=doc.id), idxnm=search.index_name(tenant_id)) - - cls.increment_chunk_num( - doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, 0) - if not cls.delete(doc): - raise RuntimeError("Database error (Document removal)!") - - MINIO.rm(doc.kb_id, doc.location) + Q("match", doc_id=doc.id), idxnm=search.index_name(tenant_id)) + cls.clear_chunk_num(doc.id) return cls.delete_by_id(doc.id) @classmethod