jyapayne/dify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Feat/improve document delete logic (#1325)

Browse source 
Co-authored-by: jyong <jyong@dify.ai>
This commit is contained in:
Jyong 2023-10-12 13:30:44 +08:00 • committed by GitHub
commit 289c93d081
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 34 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

28
api/tasks/clean_document_task.py
View file

@ -31,22 +31,24 @@ def clean_document_task(document_id: str, dataset_id: str):
kw_index = IndexBuilder.get_index(dataset, 'economy')
segments = db.session.query(DocumentSegment).filter(DocumentSegment.document_id == document_id).all()
index_node_ids = [segment.index_node_id for segment in segments]
# check segment is exist
if segments:
index_node_ids = [segment.index_node_id for segment in segments]
# delete from vector index
if vector_index:
vector_index.delete_by_document_id(document_id)
# delete from vector index
if vector_index:
vector_index.delete_by_document_id(document_id)
# delete from keyword index
if index_node_ids:
kw_index.delete_by_ids(index_node_ids)
# delete from keyword index
if index_node_ids:
kw_index.delete_by_ids(index_node_ids)
for segment in segments:
db.session.delete(segment)
for segment in segments:
db.session.delete(segment)
db.session.commit()
end_at = time.perf_counter()
logging.info(
click.style('Cleaned document when document deleted: {} latency: {}'.format(document_id, end_at - start_at), fg='green'))
db.session.commit()
end_at = time.perf_counter()
logging.info(
click.style('Cleaned document when document deleted: {} latency: {}'.format(document_id, end_at - start_at), fg='green'))
except Exception:
logging.exception("Cleaned document when document deleted failed")

12
api/tasks/document_indexing_task.py
View file

@ -30,13 +30,11 @@ def document_indexing_task(dataset_id: str, document_ids: list):
Document.dataset_id == dataset_id
).first()
if not document:
raise NotFound('Document not found')
document.indexing_status = 'parsing'
document.processing_started_at = datetime.datetime.utcnow()
documents.append(document)
db.session.add(document)
if document:
document.indexing_status = 'parsing'
document.processing_started_at = datetime.datetime.utcnow()
documents.append(document)
db.session.add(document)
db.session.commit()
try: