feat: add support for DirectoryLoader (#520)

src/backend/langflow/api/v1/chat.py

@@ -87,10 +87,16 @@ async def stream_build(flow_id: str):
 
             logger.debug("Building langchain object")
             graph = Graph.from_payload(graph_data)
-            for node in graph.generator_build():
+            number_of_nodes = len(graph.nodes)
+            for i, vertex in enumerate(graph.generator_build(), 1):
                 try:
-                    node.build()
-                    params = node._built_object_repr()
+                    log_dict = {
+                        "log": f"Building node {vertex.vertex_type}",
+                        "progress": round(i / number_of_nodes, 2),
+                    }
+                    yield f"data: {json.dumps(log_dict)}\n\n"
+                    vertex.build()
+                    params = vertex._built_object_repr()
                     valid = True
                     logger.debug(
                         f"Building node {params[:50]}{'...' if len(params) > 50 else ''}"
@@ -103,7 +109,7 @@ async def stream_build(flow_id: str):
                     {
                         "valid": valid,
                         "params": params,
-                        "id": node.id,
+                        "id": vertex.id,
                     }
                 )
                 yield f"data: {response}\n\n"

src/backend/langflow/config.yaml

@@ -48,6 +48,7 @@ documentloaders:
   - ReadTheDocsLoader
   - SlackDirectoryLoader
   - NotionDirectoryLoader
+  - DirectoryLoader
 embeddings:
   - OpenAIEmbeddings
   - HuggingFaceEmbeddings

src/backend/langflow/template/frontend_node/documentloaders.py

@@ -51,6 +51,7 @@ class DocumentLoaderFrontNode(FrontendNode):
 
     def add_extra_fields(self) -> None:
         name = None
+        display_name = "Web Page"
         if self.template.type_name in self.file_path_templates:
             self.template.add_field(self.file_path_templates[self.template.type_name])
         elif self.template.type_name in {
@@ -64,8 +65,9 @@ class DocumentLoaderFrontNode(FrontendNode):
             name = "web_path"
         elif self.template.type_name in {"GitbookLoader"}:
             name = "web_page"
-        elif self.template.type_name in {"ReadTheDocsLoader"}:
+        elif self.template.type_name in {"DirectoryLoader", "ReadTheDocsLoader"}:
             name = "path"
+            display_name = "Local directory"
         if name:
             self.template.add_field(
                 TemplateField(
@@ -74,6 +76,17 @@ class DocumentLoaderFrontNode(FrontendNode):
                     show=True,
                     name=name,
                     value="",
-                    display_name="Web Page",
+                    display_name=display_name,
+                )
+            )
+            if self.template.type_name in {"DirectoryLoader"}:
+                self.template.add_field(
+                TemplateField(
+                    field_type="str",
+                    required=True,
+                    show=True,
+                    name="glob",
+                    value="**/*.txt",
+                    display_name="glob",
                 )
             )