jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

feat: add support for DirectoryLoader (#520)

Browse source
This commit is contained in:
Gabriel Luiz Freitas Almeida 2023-06-21 13:57:30 +00:00 • committed by GitHub
commit 66ee979c0b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 90 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/backend/langflow/api/v1/chat.py
View file

@ -87,10 +87,16 @@ async def stream_build(flow_id: str):
logger.debug("Building langchain object")
graph = Graph.from_payload(graph_data)
for node in graph.generator_build():
number_of_nodes = len(graph.nodes)
for i, vertex in enumerate(graph.generator_build(), 1):
try:
node.build()
params = node._built_object_repr()
log_dict = {
"log": f"Building node {vertex.vertex_type}",
"progress": round(i / number_of_nodes, 2),
}
yield f"data: {json.dumps(log_dict)}\n\n"
vertex.build()
params = vertex._built_object_repr()
valid = True
logger.debug(
f"Building node {params[:50]}{'...' if len(params) > 50 else ''}"
@ -103,7 +109,7 @@ async def stream_build(flow_id: str):
{
"valid": valid,
"params": params,
"id": node.id,
"id": vertex.id,
}
)
yield f"data: {response}\n\n"

1
src/backend/langflow/config.yaml
View file

@ -48,6 +48,7 @@ documentloaders:
- ReadTheDocsLoader
- SlackDirectoryLoader
- NotionDirectoryLoader
- DirectoryLoader
embeddings:
- OpenAIEmbeddings
- HuggingFaceEmbeddings

17
src/backend/langflow/template/frontend_node/documentloaders.py
View file

@ -51,6 +51,7 @@ class DocumentLoaderFrontNode(FrontendNode):
def add_extra_fields(self) -> None:
name = None
display_name = "Web Page"
if self.template.type_name in self.file_path_templates:
self.template.add_field(self.file_path_templates[self.template.type_name])
elif self.template.type_name in {
@ -64,8 +65,9 @@ class DocumentLoaderFrontNode(FrontendNode):
name = "web_path"
elif self.template.type_name in {"GitbookLoader"}:
name = "web_page"
elif self.template.type_name in {"ReadTheDocsLoader"}:
elif self.template.type_name in {"DirectoryLoader", "ReadTheDocsLoader"}:
name = "path"
display_name = "Local directory"
if name:
self.template.add_field(
TemplateField(
@ -74,6 +76,17 @@ class DocumentLoaderFrontNode(FrontendNode):
show=True,
name=name,
value="",
display_name="Web Page",
display_name=display_name,
)
)
if self.template.type_name in {"DirectoryLoader"}:
self.template.add_field(
TemplateField(
field_type="str",
required=True,
show=True,
name="glob",
value="**/*.txt",
display_name="glob",
)
)

20
src/frontend/package-lock.json generated
View file

@ -80,7 +80,7 @@
"postcss": "^8.4.23",
"tailwindcss": "^3.3.2",
"typescript": "^5.0.2",
"vite": "^4.3.5"
"vite": "^4.3.9"
}
},
"node_modules/@adobe/css-tools": {
@ -5070,6 +5070,19 @@
"version": "1.0.0",
"license": "ISC"
},
"node_modules/fsevents": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
"hasInstallScript": true,
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/function-bind": {
"version": "1.1.1",
"license": "MIT"
@ -9552,8 +9565,9 @@
}
},
"node_modules/vite": {
"version": "4.3.5",
"license": "MIT",
"version": "4.3.9",
"resolved": "https://registry.npmjs.org/vite/-/vite-4.3.9.tgz",
"integrity": "sha512-qsTNZjO9NoJNW7KnOrgYwczm0WctJ8m/yqYAMAK9Lxt4SoySUfS5S8ia9K7JHpa3KEeMfyF8LoJ3c5NeBJy6pg==",
"dependencies": {
"esbuild": "^0.17.5",
"postcss": "^8.4.23",

2
src/frontend/package.json
View file

@ -101,6 +101,6 @@
"postcss": "^8.4.23",
"tailwindcss": "^3.3.2",
"typescript": "^5.0.2",
"vite": "^4.3.5"
"vite": "^4.3.9"
}
}

16
src/frontend/src/components/chatComponent/buildTrigger/index.tsx
View file

@ -1,4 +1,4 @@
import { useState, useContext } from "react";
import { useContext } from "react";
import { Transition } from "@headlessui/react";
import { Zap } from "lucide-react";
import { validateNodes } from "../../../utils";
@ -22,7 +22,7 @@ export default function BuildTrigger({
}) {
const { updateSSEData, isBuilding, setIsBuilding } = useSSE();
const { reactFlowInstance } = useContext(typesContext);
const { setErrorData } = useContext(alertContext);
const { setErrorData, setSuccessData } = useContext(alertContext);
async function handleBuild(flow: FlowType) {
try {
@ -81,10 +81,16 @@ export default function BuildTrigger({
eventSource.close();
return;
} else if (parsedData.log) {
// If the event is a log, log it
// TODO: implement the progress
setSuccessData({ title: parsedData.log });
setSuccessData({ title: parsedData.progress });
} else {
// Otherwise, process the data
const isValid = processStreamResult(parsedData);
validationResults.push(isValid);
}
// Otherwise, process the data
const isValid = processStreamResult(parsedData);
validationResults.push(isValid);
};
eventSource.onerror = (error) => {

10
src/frontend/src/components/headerComponent/index.tsx
View file

@ -1,4 +1,4 @@
import { BellIcon, Home, Users2 } from "lucide-react";
import { BellIcon, Home, MoonIcon, SunIcon, Users2 } from "lucide-react";
import { useContext, useEffect, useState } from "react";
import { FaDiscord, FaGithub, FaTwitter } from "react-icons/fa";
import { Button } from "../ui/button";
@ -12,6 +12,7 @@ import MenuBar from "./components/menuBar";
import { Link, useLocation, useParams } from "react-router-dom";
import { USER_PROJECTS_HEADER } from "../../constants";
import { getRepoStars } from "../../controllers/API";
import { Separator } from "../ui/separator";
export default function Header() {
const { flows, addFlow, tabId } = useContext(TabsContext);
@ -73,7 +74,7 @@ export default function Header() {
href="https://github.com/logspace-ai/langflow"
target="_blank"
rel="noreferrer"
className="inline-flex items-center justify-center text-sm font-medium transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:opacity-50 disabled:pointer-events-none ring-offset-background text-gray-600 dark:text-gray-300 border border-input hover:bg-accent hover:text-accent-foreground h-9 px-3 pr-0 rounded-md"
className="inline-flex shadow-sm items-center justify-center text-sm font-medium transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:opacity-50 disabled:pointer-events-none ring-offset-background text-gray-600 dark:text-gray-300 border border-input hover:bg-accent hover:text-accent-foreground h-9 px-3 pr-0 rounded-md"
>
<FaGithub className="h-5 w-5 mr-2" />
Star
@ -97,7 +98,8 @@ export default function Header() {
>
<FaDiscord className="h-5 w-5" />
</a>
{/* <button
<Separator orientation="vertical" />
<button
className="text-gray-600 hover:text-gray-500 dark:text-gray-300 dark:hover:text-gray-200"
onClick={() => {
setDark(!dark);
@ -108,7 +110,7 @@ export default function Header() {
) : (
<MoonIcon className="h-5 w-5" />
)}
</button> */}
</button>
<button
className="text-gray-600 hover:text-gray-500 dark:text-gray-300 dark:hover:text-gray-200 relative"
onClick={(event: React.MouseEvent<HTMLElement>) => {