jyapayne/dify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: improve handling of leading punctuation removal (#10761)

Browse source
This commit is contained in:
Zane 2024-11-18 21:32:33 +08:00 • committed by GitHub
commit 14f3d44c37
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 42 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

16
api/core/tools/utils/text_processing_utils.py Normal file
View file

@ -0,0 +1,16 @@
import re
def remove_leading_symbols(text: str) -> str:
"""
Remove leading punctuation or symbols from the given text.
Args:
text (str): The input text to process.
Returns:
str: The text with leading punctuation or symbols removed.
"""
# Match Unicode ranges for punctuation and symbols
pattern = r"^[\u2000-\u206F\u2E00-\u2E7F\u3000-\u303F!\"#$%&'()*+,\-./:;<=>?@\[\]^_`{|}~]+"
return re.sub(pattern, "", text)