diff --git a/pyproject.toml b/pyproject.toml index 34c44d214..e5cb771c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,6 +112,7 @@ dependencies = [ "sqlalchemy[aiosqlite,postgresql_psycopg2binary,postgresql_psycopgbinary]>=2.0.36,<3.0.0", "atlassian-python-api>=3.41.16,<4.0.0", "mem0ai>=0.1.26,<1.0.0", + "aiofile>=3.9.0,<4.0.0", ] [project.urls] diff --git a/src/backend/base/langflow/custom/directory_reader/directory_reader.py b/src/backend/base/langflow/custom/directory_reader/directory_reader.py index 30a0d1619..7b60ebf5a 100644 --- a/src/backend/base/langflow/custom/directory_reader/directory_reader.py +++ b/src/backend/base/langflow/custom/directory_reader/directory_reader.py @@ -3,6 +3,7 @@ import asyncio import zlib from pathlib import Path +from aiofile import async_open from loguru import logger from langflow.custom import Component @@ -104,6 +105,22 @@ class DirectoryReader: with _file_path.open("rb") as f: return f.read().decode("utf-8") + async def aread_file_content(self, file_path): + """Read and return the content of a file.""" + _file_path = Path(file_path) + if not _file_path.is_file(): + return None + try: + async with async_open(_file_path, encoding="utf-8") as file: + # UnicodeDecodeError: 'charmap' codec can't decode byte 0x9d in position 3069: + # character maps to + return await file.read() + except UnicodeDecodeError: + # This is happening in Windows, so we need to open the file in binary mode + # The file is always just a python file, so we can safely read it as utf-8 + with async_open(_file_path, "rb") as f: + return (await f.read()).decode("utf-8") + def get_files(self): """Walk through the directory path and return a list of all .py files.""" if not (safe_path := self.get_safe_path()): @@ -257,7 +274,7 @@ class DirectoryReader: async def process_file_async(self, file_path): try: - file_content = await asyncio.to_thread(self.read_file_content, file_path) + file_content = await self.aread_file_content(file_path) except Exception: # noqa: BLE001 logger.exception(f"Error while reading file {file_path}") return False, f"Could not read {file_path}" diff --git a/uv.lock b/uv.lock index a986936f5..4c472b49e 100644 --- a/uv.lock +++ b/uv.lock @@ -13,6 +13,18 @@ members = [ "langflow-base", ] +[[package]] +name = "aiofile" +version = "3.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "caio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/67/e2/d7cb819de8df6b5c1968a2756c3cb4122d4fa2b8fc768b53b7c9e5edb646/aiofile-3.9.0.tar.gz", hash = "sha256:e5ad718bb148b265b6df1b3752c4d1d83024b93da9bd599df74b9d9ffcf7919b", size = 17943 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/25/da1f0b4dd970e52bf5a36c204c107e11a0c6d3ed195eba0bfbc664c312b2/aiofile-3.9.0-py3-none-any.whl", hash = "sha256:ce2f6c1571538cbdfa0143b04e16b208ecb0e9cb4148e528af8a640ed51cc8aa", size = 19539 }, +] + [[package]] name = "aiofiles" version = "24.1.0" @@ -512,6 +524,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/07/14f8ad37f2d12a5ce41206c21820d8cb6561b728e51fad4530dff0552a67/cachetools-5.5.0-py3-none-any.whl", hash = "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292", size = 9524 }, ] +[[package]] +name = "caio" +version = "0.9.17" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/88/cf/59b868909a85ad9eca985ad5bbeb3c0a8cd435e50ae12def770d16911753/caio-0.9.17.tar.gz", hash = "sha256:8f30511526814d961aeef389ea6885273abe6c655f1e08abbadb95d12fdd9b4f", size = 25001 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/c3/17bc41b7c795d91d58ee7a70ad98e23f1ba0d50bdeadd82173bb02cddc8e/caio-0.9.17-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3f69395fdd45c115b2ef59732e3c8664722a2b51de2d6eedb3d354b2f5f3be3c", size = 37849 }, + { url = "https://files.pythonhosted.org/packages/bf/3f/0ae9f69deb3dc96b20bf084cc262b438a154d5de08a064628272713ff239/caio-0.9.17-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3028b746e9ec7f6d6ebb386a7fd8caf0eebed5d6e6b4f18c8ef25861934b1673", size = 79417 }, + { url = "https://files.pythonhosted.org/packages/52/44/a79c7004a9562a176d78437816c736c34ab9fb6233a4b8164eb25628a09c/caio-0.9.17-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:079730a353bbde03796fab681e969472eace09ffbe5000e584868a7fe389ba6f", size = 37842 }, + { url = "https://files.pythonhosted.org/packages/57/e2/1d04e506a5fd735856f0bb95b4d03b800947bd43c98193ee57d37070a51e/caio-0.9.17-cp311-cp311-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:549caa51b475877fe32856a26fe937366ae7a1c23a9727005b441db9abb12bcc", size = 79985 }, + { url = "https://files.pythonhosted.org/packages/c3/b8/37dcee4bc4fae1701a86373a297bbca797f6b7bfe5f85993a11049649c63/caio-0.9.17-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0ddb253b145a53ecca76381677ce465bc5efeaecb6aaf493fac43ae79659f0fb", size = 37909 }, + { url = "https://files.pythonhosted.org/packages/80/f5/5e993120daeb4ec084f5f84c118bbd48b65379f63ed56919bf224e0eab42/caio-0.9.17-cp312-cp312-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3e320b0ea371c810359934f8e8fe81777c493cc5fb4d41de44277cbe7336e74", size = 81709 }, + { url = "https://files.pythonhosted.org/packages/58/72/3f4895adb1d23b0ac1d8afc748405a2ad3c77d8d0f23b05a64ff583c11e5/caio-0.9.17-py3-none-any.whl", hash = "sha256:c55d4dc6b3a36f93237ecd6360e1c131c3808bc47d4191a130148a99b80bb311", size = 19062 }, +] + [[package]] name = "cassandra-driver" version = "3.29.2" @@ -3494,6 +3521,7 @@ name = "langflow" version = "1.1.1" source = { editable = "." } dependencies = [ + { name = "aiofile" }, { name = "assemblyai" }, { name = "astra-assistants", extra = ["tools"] }, { name = "atlassian-python-api" }, @@ -3635,6 +3663,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "aiofile", specifier = ">=3.9.0,<4.0.0" }, { name = "assemblyai", specifier = ">=0.33.0,<1.0.0" }, { name = "astra-assistants", extras = ["tools"], specifier = "~=2.2.6" }, { name = "atlassian-python-api", specifier = ">=3.41.16,<4.0.0" },