jyapayne/dify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: mypy for all type check (#10921)

Browse source
This commit is contained in:
yihong 2024-12-24 18:38:51 +08:00 • committed by GitHub
commit 56e15d09a9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
584 changed files with 3975 additions and 2826 deletions
Download patch file Download diff file Expand all files Collapse all files

4
api/core/rag/splitter/text_splitter.py
View file

@ -92,7 +92,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):
texts, metadatas = [], []
for doc in documents:
texts.append(doc.page_content)
metadatas.append(doc.metadata)
metadatas.append(doc.metadata or {})
return self.create_documents(texts, metadatas=metadatas)
def _join_docs(self, docs: list[str], separator: str) -> Optional[str]:
@ -143,7 +143,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):
def from_huggingface_tokenizer(cls, tokenizer: Any, **kwargs: Any) -> TextSplitter:
"""Text splitter that uses HuggingFace tokenizer to count length."""
try:
from transformers import PreTrainedTokenizerBase
from transformers import PreTrainedTokenizerBase # type: ignore
if not isinstance(tokenizer, PreTrainedTokenizerBase):
raise ValueError("Tokenizer received was not an instance of PreTrainedTokenizerBase")