feat: add support for DirectoryLoader (#520)

2023-06-21 13:57:30 +00:00 · 2023-06-21 13:57:30 +00:00 · 66ee979c0b
commit 66ee979c0b
parent ba7157f155 372463912c
9 changed files with 90 additions and 22 deletions
--- a/15
+++ b/15
@ -0,0 +1,15 @@
+FROM python:3.10-slim
+
+RUN apt-get update && apt-get install gcc g++ git make -y && apt-get clean \
+	&& rm -rf /var/lib/apt/lists/*
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+WORKDIR $HOME/app
+
+COPY --chown=user . $HOME/app
+
+RUN pip install langflow>==0.0.86 -U --user
+CMD ["python", "-m", "langflow", "--host", "0.0.0.0", "--port", "7860"]
--- a/README.md
+++ b/README.md
@ -2,10 +2,9 @@

 # ⛓️ LangFlow

-~ A User Interface For [LangChain](https://github.com/hwchase17/langchain) ~
+~ An effortless way to experiment and prototype [LangChain](https://github.com/hwchase17/langchain) pipelines ~

 <p>
-<a href="https://huggingface.co/spaces/Logspace/LangFlow"><img src="https://huggingface.co/datasets/huggingface/badges/raw/main/open-in-hf-spaces-sm.svg" alt="HuggingFace Spaces"></a>
 <img alt="GitHub Contributors" src="https://img.shields.io/github/contributors/logspace-ai/langflow" />
 <img alt="GitHub Last Commit" src="https://img.shields.io/github/last-commit/logspace-ai/langflow" />
 <img alt="" src="https://img.shields.io/github/repo-size/logspace-ai/langflow" />
@ -14,10 +13,17 @@
 <img alt="Github License" src="https://img.shields.io/github/license/logspace-ai/langflow" />
 </p>

+<p>
+<a href="https://discord.gg/FUhJnnJ9"><img alt="Discord Server" src="https://dcbadge.vercel.app/api/server/FUhJnnJ9?compact=true&style=flat"/></a>
+<a href="https://huggingface.co/spaces/Logspace/LangFlow"><img src="https://huggingface.co/datasets/huggingface/badges/raw/main/open-in-hf-spaces-sm.svg" alt="HuggingFace Spaces"></a>
+</p>
+
 <a href="https://github.com/logspace-ai/langflow">
    <img width="100%" src="https://github.com/logspace-ai/langflow/blob/main/img/langflow-demo.gif?raw=true"></a>

-LangFlow is a GUI for [LangChain](https://github.com/hwchase17/langchain), designed with [react-flow](https://github.com/wbkd/react-flow) to provide an effortless way to experiment and prototype flows with drag-and-drop components and a chat box.
+
+<p>
+</p>

 ## 📦 Installation
 ### <b>Locally</b>
@ -178,6 +184,11 @@ flow("Hey, have you heard of LangFlow?")
 We welcome contributions from developers of all levels to our open-source project on GitHub. If you'd like to contribute, please check our [contributing guidelines](./CONTRIBUTING.md) and help make LangFlow more accessible.


+Join our [Discord](https://discord.com/invite/EqksyE2EX9) server to ask questions, make suggestions and showcase your projects! 🦾
+
+<p>
+</p>
+
 [![Star History Chart](https://api.star-history.com/svg?repos=logspace-ai/langflow&type=Timeline)](https://star-history.com/#logspace-ai/langflow&Date)


--- a/src/backend/langflow/api/v1/chat.py
+++ b/src/backend/langflow/api/v1/chat.py
@ -87,10 +87,16 @@ async def stream_build(flow_id: str):

            logger.debug("Building langchain object")
            graph = Graph.from_payload(graph_data)
-            for node in graph.generator_build():
+            number_of_nodes = len(graph.nodes)
+            for i, vertex in enumerate(graph.generator_build(), 1):
                try:
-                    node.build()
-                    params = node._built_object_repr()
+                    log_dict = {
+                        "log": f"Building node {vertex.vertex_type}",
+                        "progress": round(i / number_of_nodes, 2),
+                    }
+                    yield f"data: {json.dumps(log_dict)}\n\n"
+                    vertex.build()
+                    params = vertex._built_object_repr()
                    valid = True
                    logger.debug(
                        f"Building node {params[:50]}{'...' if len(params) > 50 else ''}"
@ -103,7 +109,7 @@ async def stream_build(flow_id: str):
                    {
                        "valid": valid,
                        "params": params,
-                        "id": node.id,
+                        "id": vertex.id,
                    }
                )
                yield f"data: {response}\n\n"
--- a/src/backend/langflow/config.yaml
+++ b/src/backend/langflow/config.yaml
@ -48,6 +48,7 @@ documentloaders:
  - ReadTheDocsLoader
  - SlackDirectoryLoader
  - NotionDirectoryLoader
+  - DirectoryLoader
 embeddings:
  - OpenAIEmbeddings
  - HuggingFaceEmbeddings
--- a/src/backend/langflow/template/frontend_node/documentloaders.py
+++ b/src/backend/langflow/template/frontend_node/documentloaders.py
@ -51,6 +51,7 @@ class DocumentLoaderFrontNode(FrontendNode):

    def add_extra_fields(self) -> None:
        name = None
+        display_name = "Web Page"
        if self.template.type_name in self.file_path_templates:
            self.template.add_field(self.file_path_templates[self.template.type_name])
        elif self.template.type_name in {
@ -64,8 +65,9 @@ class DocumentLoaderFrontNode(FrontendNode):
            name = "web_path"
        elif self.template.type_name in {"GitbookLoader"}:
            name = "web_page"
-        elif self.template.type_name in {"ReadTheDocsLoader"}:
+        elif self.template.type_name in {"DirectoryLoader", "ReadTheDocsLoader"}:
            name = "path"
+            display_name = "Local directory"
        if name:
            self.template.add_field(
                TemplateField(
@ -74,6 +76,17 @@ class DocumentLoaderFrontNode(FrontendNode):
                    show=True,
                    name=name,
                    value="",
-                    display_name="Web Page",
+                    display_name=display_name,
+                )
+            )
+            if self.template.type_name in {"DirectoryLoader"}:
+                self.template.add_field(
+                TemplateField(
+                    field_type="str",
+                    required=True,
+                    show=True,
+                    name="glob",
+                    value="**/*.txt",
+                    display_name="glob",
                )
            )
--- a/src/frontend/package-lock.json
+++ b/src/frontend/package-lock.json
@ -80,7 +80,7 @@
        "postcss": "^8.4.23",
        "tailwindcss": "^3.3.2",
        "typescript": "^5.0.2",
-        "vite": "^4.3.5"
+        "vite": "^4.3.9"
      }
    },
    "node_modules/@adobe/css-tools": {
@ -5070,6 +5070,19 @@
      "version": "1.0.0",
      "license": "ISC"
    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
    "node_modules/function-bind": {
      "version": "1.1.1",
      "license": "MIT"
@ -9552,8 +9565,9 @@
      }
    },
    "node_modules/vite": {
-      "version": "4.3.5",
-      "license": "MIT",
+      "version": "4.3.9",
+      "resolved": "https://registry.npmjs.org/vite/-/vite-4.3.9.tgz",
+      "integrity": "sha512-qsTNZjO9NoJNW7KnOrgYwczm0WctJ8m/yqYAMAK9Lxt4SoySUfS5S8ia9K7JHpa3KEeMfyF8LoJ3c5NeBJy6pg==",
      "dependencies": {
        "esbuild": "^0.17.5",
        "postcss": "^8.4.23",
--- a/src/frontend/package.json
+++ b/src/frontend/package.json
@ -101,6 +101,6 @@
    "postcss": "^8.4.23",
    "tailwindcss": "^3.3.2",
    "typescript": "^5.0.2",
-    "vite": "^4.3.5"
+    "vite": "^4.3.9"
  }
 }
--- a/src/frontend/src/components/chatComponent/buildTrigger/index.tsx
+++ b/src/frontend/src/components/chatComponent/buildTrigger/index.tsx
@ -1,4 +1,4 @@
-import { useState, useContext } from "react";
+import { useContext } from "react";
 import { Transition } from "@headlessui/react";
 import { Zap } from "lucide-react";
 import { validateNodes } from "../../../utils";
@ -22,7 +22,7 @@ export default function BuildTrigger({
 }) {
  const { updateSSEData, isBuilding, setIsBuilding } = useSSE();
  const { reactFlowInstance } = useContext(typesContext);
-  const { setErrorData } = useContext(alertContext);
+  const { setErrorData, setSuccessData } = useContext(alertContext);

  async function handleBuild(flow: FlowType) {
    try {
@ -81,10 +81,16 @@ export default function BuildTrigger({
        eventSource.close();

        return;
+      } else if (parsedData.log) {
+        // If the event is a log, log it
+        // TODO: implement the progress
+        setSuccessData({ title: parsedData.log });
+        setSuccessData({ title: parsedData.progress });
+      } else {
+        // Otherwise, process the data
+        const isValid = processStreamResult(parsedData);
+        validationResults.push(isValid);
      }
-      // Otherwise, process the data
-      const isValid = processStreamResult(parsedData);
-      validationResults.push(isValid);
    };

    eventSource.onerror = (error) => {
--- a/src/frontend/src/components/headerComponent/index.tsx
+++ b/src/frontend/src/components/headerComponent/index.tsx
@ -1,4 +1,4 @@
-import { BellIcon, Home, Users2 } from "lucide-react";
+import { BellIcon, Home, MoonIcon, SunIcon, Users2 } from "lucide-react";
 import { useContext, useEffect, useState } from "react";
 import { FaDiscord, FaGithub, FaTwitter } from "react-icons/fa";
 import { Button } from "../ui/button";
@ -12,6 +12,7 @@ import MenuBar from "./components/menuBar";
 import { Link, useLocation, useParams } from "react-router-dom";
 import { USER_PROJECTS_HEADER } from "../../constants";
 import { getRepoStars } from "../../controllers/API";
+import { Separator } from "../ui/separator";

 export default function Header() {
  const { flows, addFlow, tabId } = useContext(TabsContext);
@ -73,7 +74,7 @@ export default function Header() {
              href="https://github.com/logspace-ai/langflow"
              target="_blank"
              rel="noreferrer"
-              className="inline-flex items-center justify-center text-sm font-medium transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:opacity-50 disabled:pointer-events-none ring-offset-background text-gray-600 dark:text-gray-300 border border-input hover:bg-accent hover:text-accent-foreground h-9 px-3 pr-0 rounded-md"
+              className="inline-flex shadow-sm items-center justify-center text-sm font-medium transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:opacity-50 disabled:pointer-events-none ring-offset-background text-gray-600 dark:text-gray-300 border border-input hover:bg-accent hover:text-accent-foreground h-9 px-3 pr-0 rounded-md"
            >
              <FaGithub className="h-5 w-5 mr-2" />
              Star
@ -97,7 +98,8 @@ export default function Header() {
            >
              <FaDiscord className="h-5 w-5" />
            </a>
-          {/* <button
+            <Separator orientation="vertical" />
+          <button
            className="text-gray-600 hover:text-gray-500 dark:text-gray-300 dark:hover:text-gray-200"
            onClick={() => {
              setDark(!dark);
@ -108,7 +110,7 @@ export default function Header() {
            ) : (
              <MoonIcon className="h-5 w-5" />
            )}
-          </button> */}
+          </button>
          <button
            className="text-gray-600 hover:text-gray-500 dark:text-gray-300 dark:hover:text-gray-200 relative"
            onClick={(event: React.MouseEvent<HTMLElement>) => {