8172e62236* Initial Restructure * Replace import langflow for import langflow_base * Fix dependencies * 🔧 chore(Makefile): refactor build process to separate base and frontend builds for better organization and maintainability * 🚀 chore(Makefile): update build_frontend command to copy frontend build to the correct directory 🔖 chore(pyproject.toml): update python and httpx dependencies versions 🔧 chore(__init__.py): update import statement for load_flow_from_json function * 🔖 chore(pyproject.toml): update package version from 0.0.6 to 0.0.8 to reflect changes in the codebase * 🚀 feat(server.ts): change port variable case from lowercase port to uppercase PORT to improve semantics 🚀 feat(server.ts): add support for process.env.PORT environment variable to be able to run app on a configurable port * 🐛 fix(server.ts): change port variable case from lowercase port to uppercase PORT to improve semantics ✨ feat(server.ts): add support for process.env.PORT environment variable to be able to run app on a configurable port 🚚 chore(pyproject.toml): update langflow-base version from 0.0.8 to 0.0.10 ✨ feat(server.ts): add new agent component LCAgentComponent to langflow_base ✨ feat(server.ts): add new model component LCModelComponent to langflow_base ✨ feat(server.ts): add new helper functions docs_to_records and records_to_text to langflow_base ✨ feat(server.ts): add new flow helper functions list_flows, load_flow, run_flow, generate_function_for_flow, get_flow_inputs, build_schema_from_inputs to langflow_base ✨ feat(server.ts): add new prompt component PromptComponent to langflow_base ✨ feat(server.ts): add new chat components ChatInput and ChatOutput to langflow_base ✨ feat(server.ts): add new model component OpenAIModelComponent to langflow_base 🚚 chore(main.py): update import path from langflow.main to langflow_base.main 🚚 chore(service.py): update import path from langflow.services.database.manager to langflow_base.services.database.manager 🚚 chore(factory.py): update import path from langflow.services to langflow_base.services 🚚 chore(service.py): update import path from langflow.services.plugins to langflow_base.services.plugins 🚚 chore(utils.py): update import path from langflow.services to langflow_base.services 🚚 chore(validate.py): update import path from langflow.field_typing to langflow_base.field_typing 🚚 chore(pyproject.toml): update langflow-base version from 0.0.8 to 0.0.10 * Update Makefile to install backend dependencies and build langflow * Add langflow main module and update __init__.py * Update langflow install process to use implicit namespace * Add langflow-base as a local dependency * Add setup_poetry target to Makefile * Update Poetry version and add poetry-monorepo-dependency-plugin * Refactor code to improve performance and readability * Update imports to custom and load * Update content-hash in poetry.lock --------- Co-authored-by: Matheus <jacquesmats@gmail.com> Co-authored-by: Gabriel Luiz Freitas Almeida <gabriel@logspace.ai>
62 lines
2.4 KiB
Python
62 lines
2.4 KiB
Python
from typing import Any, Dict, List, Optional
|
|
|
|
from langflow.base.data.utils import parallel_load_records, parse_text_file_to_record, retrieve_file_paths
|
|
from langflow.interface.custom.custom_component import CustomComponent
|
|
from langflow.schema import Record
|
|
|
|
|
|
class DirectoryComponent(CustomComponent):
|
|
display_name = "Directory"
|
|
description = "Load Text Files from a Directory and Convert Them to Records."
|
|
|
|
def build_config(self) -> Dict[str, Any]:
|
|
return {
|
|
"path": {"display_name": "Path"},
|
|
"types": {
|
|
"display_name": "Types",
|
|
"info": "File types to load. Leave empty to load all types.",
|
|
},
|
|
"depth": {"display_name": "Depth", "info": "Depth to search for files."},
|
|
"max_concurrency": {"display_name": "Max Concurrency", "advanced": True},
|
|
"load_hidden": {
|
|
"display_name": "Load Hidden",
|
|
"advanced": True,
|
|
"info": "If true, hidden files will be loaded.",
|
|
},
|
|
"recursive": {
|
|
"display_name": "Recursive",
|
|
"advanced": True,
|
|
"info": "If true, the search will be recursive.",
|
|
},
|
|
"silent_errors": {
|
|
"display_name": "Silent Errors",
|
|
"advanced": True,
|
|
"info": "If true, errors will not raise an exception.",
|
|
},
|
|
"use_multithreading": {
|
|
"display_name": "Use Multithreading",
|
|
"advanced": True,
|
|
},
|
|
}
|
|
|
|
def build(
|
|
self,
|
|
path: str,
|
|
depth: int = 0,
|
|
max_concurrency: int = 2,
|
|
load_hidden: bool = False,
|
|
recursive: bool = True,
|
|
silent_errors: bool = False,
|
|
use_multithreading: bool = True,
|
|
) -> List[Optional[Record]]:
|
|
resolved_path = self.resolve_path(path)
|
|
file_paths = retrieve_file_paths(resolved_path, load_hidden, recursive, depth)
|
|
loaded_records = []
|
|
|
|
if use_multithreading:
|
|
loaded_records = parallel_load_records(file_paths, silent_errors, max_concurrency)
|
|
else:
|
|
loaded_records = [parse_text_file_to_record(file_path, silent_errors) for file_path in file_paths]
|
|
loaded_records = list(filter(None, loaded_records))
|
|
self.status = loaded_records
|
|
return loaded_records
|