jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

fix: change DirectoryComponent to filter file paths by types (#2391)

Browse source 
* Refactor DirectoryComponent to filter file paths by types

* fix: sets types is_list to True

* chore: Remove Optional from load_directory method signature

* chore: Update return type annotation for load_directory method

* style(GroqModel.py): improve code readability by fixing indentation and adding a comment for type hint ignore in ChatGroq instantiation.

---------

Co-authored-by: Gabriel Luiz Freitas Almeida <gabriel@langflow.org>
This commit is contained in:
Carlos Coelho 2024-07-05 12:17:14 -03:00 • committed by GitHub
commit bc6b846454
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 9 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

12
src/backend/base/langflow/components/data/Directory.py
View file

@ -1,4 +1,4 @@
from typing import List, Optional
from typing import List
from langflow.base.data.utils import parallel_load_data, parse_text_file_to_data, retrieve_file_paths
from langflow.custom import Component
@ -23,6 +23,7 @@ class DirectoryComponent(Component):
name="types",
display_name="Types",
info="File types to load. Leave empty to load all types.",
is_list=True,
),
IntInput(
name="depth",
@ -67,8 +68,9 @@ class DirectoryComponent(Component):
Output(display_name="Data", name="data", method="load_directory"),
]
def load_directory(self) -> List[Optional[Data]]:
def load_directory(self) -> List[Data]:
path = self.path
types = self.types or [] # self.types is already a list due to is_list=True
depth = self.depth
max_concurrency = self.max_concurrency
load_hidden = self.load_hidden
@ -78,6 +80,10 @@ class DirectoryComponent(Component):
resolved_path = self.resolve_path(path)
file_paths = retrieve_file_paths(resolved_path, load_hidden, recursive, depth)
if types:
file_paths = [fp for fp in file_paths if any(fp.endswith(ext) for ext in types)]
loaded_data = []
if use_multithreading:
@ -86,4 +92,4 @@ class DirectoryComponent(Component):
loaded_data = [parse_text_file_to_data(file_path, silent_errors) for file_path in file_paths]
loaded_data = list(filter(None, loaded_data))
self.status = loaded_data
return loaded_data
return loaded_data # type: ignore