feat: add support for DirectoryLoader (#520)

Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.10-slim
+
+RUN apt-get update && apt-get install gcc g++ git make -y && apt-get clean \
+	&& rm -rf /var/lib/apt/lists/*
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+WORKDIR $HOME/app
+
+COPY --chown=user . $HOME/app
+
+RUN pip install langflow>==0.0.86 -U --user
+CMD ["python", "-m", "langflow", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -2,10 +2,9 @@
 
 # ⛓️ LangFlow
 
-~ A User Interface For [LangChain](https://github.com/hwchase17/langchain) ~
+~ An effortless way to experiment and prototype [LangChain](https://github.com/hwchase17/langchain) pipelines ~
 
 <p>
-<a href="https://huggingface.co/spaces/Logspace/LangFlow"><img src="https://huggingface.co/datasets/huggingface/badges/raw/main/open-in-hf-spaces-sm.svg" alt="HuggingFace Spaces"></a>
 <img alt="GitHub Contributors" src="https://img.shields.io/github/contributors/logspace-ai/langflow" />
 <img alt="GitHub Last Commit" src="https://img.shields.io/github/last-commit/logspace-ai/langflow" />
 <img alt="" src="https://img.shields.io/github/repo-size/logspace-ai/langflow" />
@@ -14,10 +13,17 @@
 <img alt="Github License" src="https://img.shields.io/github/license/logspace-ai/langflow" />
 </p>
 
+<p>
+<a href="https://discord.gg/FUhJnnJ9"><img alt="Discord Server" src="https://dcbadge.vercel.app/api/server/FUhJnnJ9?compact=true&style=flat"/></a>
+<a href="https://huggingface.co/spaces/Logspace/LangFlow"><img src="https://huggingface.co/datasets/huggingface/badges/raw/main/open-in-hf-spaces-sm.svg" alt="HuggingFace Spaces"></a>
+</p>
+
 <a href="https://github.com/logspace-ai/langflow">
     <img width="100%" src="https://github.com/logspace-ai/langflow/blob/main/img/langflow-demo.gif?raw=true"></a>
 
-LangFlow is a GUI for [LangChain](https://github.com/hwchase17/langchain), designed with [react-flow](https://github.com/wbkd/react-flow) to provide an effortless way to experiment and prototype flows with drag-and-drop components and a chat box.
+
+<p>
+</p>
 
 ## 📦 Installation
 ### <b>Locally</b>
@@ -178,6 +184,11 @@ flow("Hey, have you heard of LangFlow?")
 We welcome contributions from developers of all levels to our open-source project on GitHub. If you'd like to contribute, please check our [contributing guidelines](./CONTRIBUTING.md) and help make LangFlow more accessible.
 
 
+Join our [Discord](https://discord.com/invite/EqksyE2EX9) server to ask questions, make suggestions and showcase your projects! 🦾
+
+<p>
+</p>
+
 [![Star History Chart](https://api.star-history.com/svg?repos=logspace-ai/langflow&type=Timeline)](https://star-history.com/#logspace-ai/langflow&Date)
 
 

src/backend/langflow/api/v1/chat.py

@@ -87,10 +87,16 @@ async def stream_build(flow_id: str):
 
             logger.debug("Building langchain object")
             graph = Graph.from_payload(graph_data)
-            for node in graph.generator_build():
+            number_of_nodes = len(graph.nodes)
+            for i, vertex in enumerate(graph.generator_build(), 1):
                 try:
-                    node.build()
+                    log_dict = {
-                    params = node._built_object_repr()
+                        "log": f"Building node {vertex.vertex_type}",
+                        "progress": round(i / number_of_nodes, 2),
+                    }
+                    yield f"data: {json.dumps(log_dict)}\n\n"
+                    vertex.build()
+                    params = vertex._built_object_repr()
                     valid = True
                     logger.debug(
                         f"Building node {params[:50]}{'...' if len(params) > 50 else ''}"
@@ -103,7 +109,7 @@ async def stream_build(flow_id: str):
                     {
                         "valid": valid,
                         "params": params,
-                        "id": node.id,
+                        "id": vertex.id,
                     }
                 )
                 yield f"data: {response}\n\n"

src/backend/langflow/config.yaml

@@ -48,6 +48,7 @@ documentloaders:
   - ReadTheDocsLoader
   - SlackDirectoryLoader
   - NotionDirectoryLoader
+  - DirectoryLoader
 embeddings:
   - OpenAIEmbeddings
   - HuggingFaceEmbeddings

src/backend/langflow/template/frontend_node/documentloaders.py

@@ -51,6 +51,7 @@ class DocumentLoaderFrontNode(FrontendNode):
 
     def add_extra_fields(self) -> None:
         name = None
+        display_name = "Web Page"
         if self.template.type_name in self.file_path_templates:
             self.template.add_field(self.file_path_templates[self.template.type_name])
         elif self.template.type_name in {
@@ -64,8 +65,9 @@ class DocumentLoaderFrontNode(FrontendNode):
             name = "web_path"
         elif self.template.type_name in {"GitbookLoader"}:
             name = "web_page"
-        elif self.template.type_name in {"ReadTheDocsLoader"}:
+        elif self.template.type_name in {"DirectoryLoader", "ReadTheDocsLoader"}:
             name = "path"
+            display_name = "Local directory"
         if name:
             self.template.add_field(
                 TemplateField(
@@ -74,6 +76,17 @@ class DocumentLoaderFrontNode(FrontendNode):
                     show=True,
                     name=name,
                     value="",
-                    display_name="Web Page",
+                    display_name=display_name,
+                )
+            )
+            if self.template.type_name in {"DirectoryLoader"}:
+                self.template.add_field(
+                TemplateField(
+                    field_type="str",
+                    required=True,
+                    show=True,
+                    name="glob",
+                    value="**/*.txt",
+                    display_name="glob",
                 )
             )

src/frontend/package-lock.json

@@ -80,7 +80,7 @@
         "postcss": "^8.4.23",
         "tailwindcss": "^3.3.2",
         "typescript": "^5.0.2",
-        "vite": "^4.3.5"
+        "vite": "^4.3.9"
       }
     },
     "node_modules/@adobe/css-tools": {
@@ -5070,6 +5070,19 @@
       "version": "1.0.0",
       "license": "ISC"
     },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
     "node_modules/function-bind": {
       "version": "1.1.1",
       "license": "MIT"
@@ -9552,8 +9565,9 @@
       }
     },
     "node_modules/vite": {
-      "version": "4.3.5",
+      "version": "4.3.9",
-      "license": "MIT",
+      "resolved": "https://registry.npmjs.org/vite/-/vite-4.3.9.tgz",
+      "integrity": "sha512-qsTNZjO9NoJNW7KnOrgYwczm0WctJ8m/yqYAMAK9Lxt4SoySUfS5S8ia9K7JHpa3KEeMfyF8LoJ3c5NeBJy6pg==",
       "dependencies": {
         "esbuild": "^0.17.5",
         "postcss": "^8.4.23",

src/frontend/package.json

@@ -101,6 +101,6 @@
     "postcss": "^8.4.23",
     "tailwindcss": "^3.3.2",
     "typescript": "^5.0.2",
-    "vite": "^4.3.5"
+    "vite": "^4.3.9"
   }
 }

src/frontend/src/components/chatComponent/buildTrigger/index.tsx

@@ -1,4 +1,4 @@
-import { useState, useContext } from "react";
+import { useContext } from "react";
 import { Transition } from "@headlessui/react";
 import { Zap } from "lucide-react";
 import { validateNodes } from "../../../utils";
@@ -22,7 +22,7 @@ export default function BuildTrigger({
 }) {
   const { updateSSEData, isBuilding, setIsBuilding } = useSSE();
   const { reactFlowInstance } = useContext(typesContext);
-  const { setErrorData } = useContext(alertContext);
+  const { setErrorData, setSuccessData } = useContext(alertContext);
 
   async function handleBuild(flow: FlowType) {
     try {
@@ -81,10 +81,16 @@ export default function BuildTrigger({
         eventSource.close();
 
         return;
+      } else if (parsedData.log) {
+        // If the event is a log, log it
+        // TODO: implement the progress
+        setSuccessData({ title: parsedData.log });
+        setSuccessData({ title: parsedData.progress });
+      } else {
+        // Otherwise, process the data
+        const isValid = processStreamResult(parsedData);
+        validationResults.push(isValid);
       }
-      // Otherwise, process the data
-      const isValid = processStreamResult(parsedData);
-      validationResults.push(isValid);
     };
 
     eventSource.onerror = (error) => {

src/frontend/src/components/headerComponent/index.tsx

@@ -1,4 +1,4 @@
-import { BellIcon, Home, Users2 } from "lucide-react";
+import { BellIcon, Home, MoonIcon, SunIcon, Users2 } from "lucide-react";
 import { useContext, useEffect, useState } from "react";
 import { FaDiscord, FaGithub, FaTwitter } from "react-icons/fa";
 import { Button } from "../ui/button";
@@ -12,6 +12,7 @@ import MenuBar from "./components/menuBar";
 import { Link, useLocation, useParams } from "react-router-dom";
 import { USER_PROJECTS_HEADER } from "../../constants";
 import { getRepoStars } from "../../controllers/API";
+import { Separator } from "../ui/separator";
 
 export default function Header() {
   const { flows, addFlow, tabId } = useContext(TabsContext);
@@ -73,7 +74,7 @@ export default function Header() {
               href="https://github.com/logspace-ai/langflow"
               target="_blank"
               rel="noreferrer"
-              className="inline-flex items-center justify-center text-sm font-medium transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:opacity-50 disabled:pointer-events-none ring-offset-background text-gray-600 dark:text-gray-300 border border-input hover:bg-accent hover:text-accent-foreground h-9 px-3 pr-0 rounded-md"
+              className="inline-flex shadow-sm items-center justify-center text-sm font-medium transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:opacity-50 disabled:pointer-events-none ring-offset-background text-gray-600 dark:text-gray-300 border border-input hover:bg-accent hover:text-accent-foreground h-9 px-3 pr-0 rounded-md"
             >
               <FaGithub className="h-5 w-5 mr-2" />
               Star
@@ -97,7 +98,8 @@ export default function Header() {
             >
               <FaDiscord className="h-5 w-5" />
             </a>
-          {/* <button
+            <Separator orientation="vertical" />
+          <button
             className="text-gray-600 hover:text-gray-500 dark:text-gray-300 dark:hover:text-gray-200"
             onClick={() => {
               setDark(!dark);
@@ -108,7 +110,7 @@ export default function Header() {
             ) : (
               <MoonIcon className="h-5 w-5" />
             )}
-          </button> */}
+          </button>
           <button
             className="text-gray-600 hover:text-gray-500 dark:text-gray-300 dark:hover:text-gray-200 relative"
             onClick={(event: React.MouseEvent<HTMLElement>) => {