Feat/delete single dataset retrival (#6570)

This commit is contained in:
Jyong 2024-07-24 12:50:11 +08:00 • committed by GitHub
commit e4bb943fe5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 651 additions and 115 deletions

View file

@ -1,4 +1,5 @@
import re
from typing import Optional
import jieba
from jieba.analyse import default_tfidf
@ -11,7 +12,7 @@ class JiebaKeywordTableHandler:
def __init__(self):
default_tfidf.stop_words = STOPWORDS
def extract_keywords(self, text: str, max_keywords_per_chunk: int = 10) -> set[str]:
def extract_keywords(self, text: str, max_keywords_per_chunk: Optional[int] = 10) -> set[str]:
"""Extract keywords with JIEBA tfidf."""
keywords = jieba.analyse.extract_tags(
sentence=text,