From 96bab94b1411ddf9ce842bac243986f320b44255 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Fri, 23 Jun 2023 16:13:16 -0300 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=94=A5=20chore(chains.py):=20remove?= =?UTF-8?q?=20unnecessary=20commented=20line=20The=20commented=20line=20is?= =?UTF-8?q?=20not=20needed=20and=20can=20be=20safely=20removed.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/template/frontend_node/chains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/langflow/template/frontend_node/chains.py b/src/backend/langflow/template/frontend_node/chains.py index 0ed8f1389..f29aa2065 100644 --- a/src/backend/langflow/template/frontend_node/chains.py +++ b/src/backend/langflow/template/frontend_node/chains.py @@ -72,7 +72,7 @@ class ChainFrontendNode(FrontendNode): field.show = True field.advanced = False if field.name == "memory": - field.required = False + # field.required = False field.show = True field.advanced = False if field.name == "verbose": From 15ac75ecc51401687d422abea8ceb2a33e1283ac Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Fri, 23 Jun 2023 16:50:11 -0300 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=94=A8=20refactor(loading.py):=20add?= =?UTF-8?q?=20support=20for=20filtering=20files=20by=20extension=20in=20do?= =?UTF-8?q?cument=20loader=20The=20`instantiate=5Fdocumentloader`=20functi?= =?UTF-8?q?on=20now=20supports=20filtering=20files=20by=20extension=20usin?= =?UTF-8?q?g=20a=20`file=5Ffilter`=20parameter.=20The=20parameter=20is=20a?= =?UTF-8?q?=20string=20of=20comma-separated=20extensions,=20and=20the=20fu?= =?UTF-8?q?nction=20now=20converts=20it=20into=20a=20lambda=20function=20t?= =?UTF-8?q?hat=20filters=20files=20based=20on=20whether=20their=20name=20c?= =?UTF-8?q?ontains=20any=20of=20the=20specified=20extensions.=20This=20imp?= =?UTF-8?q?roves=20the=20flexibility=20of=20the=20document=20loader=20by?= =?UTF-8?q?=20allowing=20it=20to=20load=20only=20specific=20types=20of=20f?= =?UTF-8?q?iles.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../langflow/interface/initialize/loading.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/backend/langflow/interface/initialize/loading.py b/src/backend/langflow/interface/initialize/loading.py index 2d547c6d8..fa695f437 100644 --- a/src/backend/langflow/interface/initialize/loading.py +++ b/src/backend/langflow/interface/initialize/loading.py @@ -159,6 +159,19 @@ def instantiate_vectorstore(class_object, params): def instantiate_documentloader(class_object, params): + + + if "file_filter" in params: + # file_filter will be a string but we need a function + # that will be used to filter the files using file_filter + # like lambda x: x.endswith(".txt") but as we don't know + # anything besides the string, we will simply check if the string is + # in x and if it is, we will return True + file_filter = params.pop("file_filter", None) + extensions = file_filter.split(",") + params["file_filter"] = lambda x: any( + extension.strip() in x for extension in extensions + ) metadata = params.pop("metadata", None) docs = class_object(**params).load() if metadata: @@ -172,6 +185,7 @@ def instantiate_documentloader(class_object, params): for doc in docs: doc.metadata = metadata + return docs From 0bab7ae7143fe863a0e2284606de619233653182 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Fri, 23 Jun 2023 16:50:22 -0300 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=9A=80=20feat(documentloaders.py):=20?= =?UTF-8?q?add=20fields=20to=20GitLoader=20template=20to=20support=20repos?= =?UTF-8?q?itory=20path,=20clone=20URL,=20branch,=20and=20file=20filter=20?= =?UTF-8?q?The=20GitLoader=20template=20now=20has=20four=20new=20fields:?= =?UTF-8?q?=20repo=5Fpath,=20clone=5Furl,=20branch,=20and=20file=5Ffilter.?= =?UTF-8?q?=20These=20fields=20allow=20the=20user=20to=20specify=20the=20r?= =?UTF-8?q?epository=20path,=20clone=20URL,=20branch,=20and=20file=20exten?= =?UTF-8?q?sions=20to=20be=20loaded.=20This=20improves=20the=20flexibility?= =?UTF-8?q?=20of=20the=20GitLoader=20template=20and=20allows=20it=20to=20b?= =?UTF-8?q?e=20used=20in=20a=20wider=20range=20of=20scenarios.=20Additiona?= =?UTF-8?q?lly,=20a=20minor=20change=20was=20made=20to=20the=20add=5Fextra?= =?UTF-8?q?=5Ffields=20method=20to=20ensure=20that=20the=20field.show=20at?= =?UTF-8?q?tribute=20is=20set=20to=20True=20for=20all=20fields.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../template/frontend_node/documentloaders.py | 55 ++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/src/backend/langflow/template/frontend_node/documentloaders.py b/src/backend/langflow/template/frontend_node/documentloaders.py index 8cfb9d229..b2729374c 100644 --- a/src/backend/langflow/template/frontend_node/documentloaders.py +++ b/src/backend/langflow/template/frontend_node/documentloaders.py @@ -55,7 +55,54 @@ class DocumentLoaderFrontNode(FrontendNode): def add_extra_fields(self) -> None: name = None display_name = "Web Page" - if self.template.type_name in self.file_path_templates: + if self.template.type_name in {"GitLoader"}: + # Add fields repo_path, clone_url, branch and file_filter + self.template.add_field( + TemplateField( + field_type="str", + required=True, + show=True, + name="repo_path", + value="", + display_name="Path to repository", + advanced=False, + ) + ) + self.template.add_field( + TemplateField( + field_type="str", + required=False, + show=True, + name="clone_url", + value="", + display_name="Clone URL", + advanced=False, + ) + ) + self.template.add_field( + TemplateField( + field_type="str", + required=True, + show=True, + name="branch", + value="", + display_name="Branch", + advanced=False, + ) + ) + self.template.add_field( + TemplateField( + field_type="str", + required=False, + show=True, + name="file_filter", + value="", + display_name="File extensions (comma-separated)", + advanced=False, + ) + ) + + elif self.template.type_name in self.file_path_templates: self.template.add_field(self.file_path_templates[self.template.type_name]) elif self.template.type_name in { "WebBaseLoader", @@ -68,7 +115,10 @@ class DocumentLoaderFrontNode(FrontendNode): name = "web_path" elif self.template.type_name in {"GitbookLoader"}: name = "web_page" - elif self.template.type_name in {"DirectoryLoader", "ReadTheDocsLoader"}: + elif self.template.type_name in { + "DirectoryLoader", + "ReadTheDocsLoader", + }: name = "path" display_name = "Local directory" if name: @@ -112,3 +162,4 @@ class DocumentLoaderFrontNode(FrontendNode): if field.name == "metadata": field.show = True field.advanced = False + field.show = True From d9ec26275236d8be63cee847fd5ae15094b91dbe Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Fri, 23 Jun 2023 16:51:54 -0300 Subject: [PATCH 4/4] formatting --- src/backend/langflow/interface/initialize/loading.py | 2 -- .../GenericNode/components/parameterComponent/index.tsx | 7 ++++--- src/frontend/src/utils.ts | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/backend/langflow/interface/initialize/loading.py b/src/backend/langflow/interface/initialize/loading.py index fa695f437..88b981f9d 100644 --- a/src/backend/langflow/interface/initialize/loading.py +++ b/src/backend/langflow/interface/initialize/loading.py @@ -159,8 +159,6 @@ def instantiate_vectorstore(class_object, params): def instantiate_documentloader(class_object, params): - - if "file_filter" in params: # file_filter will be a string but we need a function # that will be used to filter the files using file_filter diff --git a/src/frontend/src/CustomNodes/GenericNode/components/parameterComponent/index.tsx b/src/frontend/src/CustomNodes/GenericNode/components/parameterComponent/index.tsx index 2736dd22d..498a804a0 100644 --- a/src/frontend/src/CustomNodes/GenericNode/components/parameterComponent/index.tsx +++ b/src/frontend/src/CustomNodes/GenericNode/components/parameterComponent/index.tsx @@ -80,7 +80,7 @@ export default function ParameterComponent({ useEffect(() => { const groupedObj = groupByFamily(myData, tooltipTitle); - + refHtml.current = groupedObj.map((item, i) => ( ( - {i === item.type.split(", ").length - 1 ? el : (el += `, `)} + {i === item.type.split(", ").length - 1 + ? el + : (el += `, `)} {i % 2 === 0 && i > 0 &&
} @@ -115,7 +117,6 @@ export default function ParameterComponent({ )); - }, [tooltipTitle]); return ( diff --git a/src/frontend/src/utils.ts b/src/frontend/src/utils.ts index 34ef6f61a..b5af18658 100644 --- a/src/frontend/src/utils.ts +++ b/src/frontend/src/utils.ts @@ -810,9 +810,9 @@ export function getRandomName( return toTitleCase(final_name); } -export function getRandomKeyByssmm(): string{ +export function getRandomKeyByssmm(): string { const now = new Date(); - const seconds = String(now.getSeconds()).padStart(2, '0'); - const milliseconds = String(now.getMilliseconds()).padStart(3, '0'); + const seconds = String(now.getSeconds()).padStart(2, "0"); + const milliseconds = String(now.getMilliseconds()).padStart(3, "0"); return seconds + milliseconds; }