jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/main' into v2

Browse source
This commit is contained in:
Gabriel Luiz Freitas Almeida 2023-11-06 23:09:49 -03:00
commit 6f9a10e37f
16 changed files with 674 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

232
src/backend/langflow/components/documentloaders/FileLoader.py Normal file
View file

@ -0,0 +1,232 @@
from langflow import CustomComponent
from langchain.schema import Document
from typing import Any, Dict, List
loaders_info: List[Dict[str, Any]] = [
{
"loader": "AirbyteJSONLoader",
"name": "Airbyte JSON (.jsonl)",
"import": "langchain.document_loaders.AirbyteJSONLoader",
"defaultFor": ["jsonl"],
"allowdTypes": ["jsonl"],
},
{
"loader": "JSONLoader",
"name": "JSON (.json)",
"import": "langchain.document_loaders.JSONLoader",
"defaultFor": ["json"],
"allowdTypes": ["json"],
},
{
"loader": "BSHTMLLoader",
"name": "BeautifulSoup4 HTML (.html, .htm)",
"import": "langchain.document_loaders.BSHTMLLoader",
"allowdTypes": ["html", "htm"],
},
{
"loader": "CSVLoader",
"name": "CSV (.csv)",
"import": "langchain.document_loaders.CSVLoader",
"defaultFor": ["csv"],
"allowdTypes": ["csv"],
},
{
"loader": "CoNLLULoader",
"name": "CoNLL-U (.conllu)",
"import": "langchain.document_loaders.CoNLLULoader",
"defaultFor": ["conllu"],
"allowdTypes": ["conllu"],
},
{
"loader": "EverNoteLoader",
"name": "EverNote (.enex)",
"import": "langchain.document_loaders.EverNoteLoader",
"defaultFor": ["enex"],
"allowdTypes": ["enex"],
},
{
"loader": "FacebookChatLoader",
"name": "Facebook Chat (.json)",
"import": "langchain.document_loaders.FacebookChatLoader",
"allowdTypes": ["json"],
},
{
"loader": "OutlookMessageLoader",
"name": "Outlook Message (.msg)",
"import": "langchain.document_loaders.OutlookMessageLoader",
"defaultFor": ["msg"],
"allowdTypes": ["msg"],
},
{
"loader": "PyPDFLoader",
"name": "PyPDF (.pdf)",
"import": "langchain.document_loaders.PyPDFLoader",
"defaultFor": ["pdf"],
"allowdTypes": ["pdf"],
},
{
"loader": "STRLoader",
"name": "Subtitle (.str)",
"import": "langchain.document_loaders.STRLoader",
"defaultFor": ["str"],
"allowdTypes": ["str"],
},
{
"loader": "TextLoader",
"name": "Text (.txt)",
"import": "langchain.document_loaders.TextLoader",
"defaultFor": ["txt"],
"allowdTypes": ["txt"],
},
{
"loader": "UnstructuredEmailLoader",
"name": "Unstructured Email (.eml)",
"import": "langchain.document_loaders.UnstructuredEmailLoader",
"defaultFor": ["eml"],
"allowdTypes": ["eml"],
},
{
"loader": "UnstructuredHTMLLoader",
"name": "Unstructured HTML (.html, .htm)",
"import": "langchain.document_loaders.UnstructuredHTMLLoader",
"defaultFor": ["html", "htm"],
"allowdTypes": ["html", "htm"],
},
{
"loader": "UnstructuredMarkdownLoader",
"name": "Unstructured Markdown (.md)",
"import": "langchain.document_loaders.UnstructuredMarkdownLoader",
"defaultFor": ["md"],
"allowdTypes": ["md"],
},
{
"loader": "UnstructuredPowerPointLoader",
"name": "Unstructured PowerPoint (.pptx)",
"import": "langchain.document_loaders.UnstructuredPowerPointLoader",
"defaultFor": ["pptx"],
"allowdTypes": ["pptx"],
},
{
"loader": "UnstructuredWordLoader",
"name": "Unstructured Word (.docx)",
"import": "langchain.document_loaders.UnstructuredWordLoader",
"defaultFor": ["docx"],
"allowdTypes": ["docx"],
},
]
class FileLoaderComponent(CustomComponent):
display_name: str = "File Loader"
description: str = "Generic File Loader"
beta = True
def build_config(self):
loader_options = ["Automatic"] + [
loader_info["name"] for loader_info in loaders_info
]
file_types = []
suffixes = []
for loader_info in loaders_info:
if "allowedTypes" in loader_info:
file_types.extend(loader_info["allowedTypes"])
suffixes.extend([f".{ext}" for ext in loader_info["allowedTypes"]])
return {
"file_path": {
"display_name": "File Path",
"required": True,
"field_type": "file",
"file_types": [
"json",
"txt",
"csv",
"jsonl",
"html",
"htm",
"conllu",
"enex",
"msg",
"pdf",
"srt",
"eml",
"md",
"pptx",
"docx",
],
"suffixes": [
".json",
".txt",
".csv",
".jsonl",
".html",
".htm",
".conllu",
".enex",
".msg",
".pdf",
".srt",
".eml",
".md",
".pptx",
".docx",
],
# "file_types" : file_types,
# "suffixes": suffixes,
},
"loader": {
"display_name": "Loader",
"is_list": True,
"required": True,
"options": loader_options,
"value": "Automatic",
},
"code": {"show": False},
}
def build(self, file_path: str, loader: str) -> Document:
file_type = file_path.split(".")[-1]
# Mapeie o nome do loader selecionado para suas informações
selected_loader_info = None
for loader_info in loaders_info:
if loader_info["name"] == loader:
selected_loader_info = loader_info
break
if selected_loader_info is None and loader != "Automatic":
raise ValueError(f"Loader {loader} not found in the loader info list")
if loader == "Automatic":
# Determine o loader automaticamente com base na extensão do arquivo
default_loader_info = None
for info in loaders_info:
if "defaultFor" in info and file_type in info["defaultFor"]:
default_loader_info = info
break
if default_loader_info is None:
raise ValueError(f"No default loader found for file type: {file_type}")
selected_loader_info = default_loader_info
if isinstance(selected_loader_info, dict):
loader_import: str = selected_loader_info["import"]
else:
raise ValueError(
f"Loader info for {loader} is not a dict\nLoader info:\n{selected_loader_info}"
)
module_name, class_name = loader_import.rsplit(".", 1)
try:
# Importe o loader dinamicamente
loader_module = __import__(module_name, fromlist=[class_name])
loader_instance = getattr(loader_module, class_name)
except ImportError as e:
raise ValueError(
f"Loader {loader} could not be imported\nLoader info:\n{selected_loader_info}"
) from e
result = loader_instance(file_path=file_path)
return result.load()

62
src/backend/langflow/components/documentloaders/UrlLoader.py Normal file
View file

@ -0,0 +1,62 @@
from typing import List
from langflow import CustomComponent
from langchain.document_loaders import AZLyricsLoader
from langchain.document_loaders import CollegeConfidentialLoader
from langchain.document_loaders import GitbookLoader
from langchain.document_loaders import HNLoader
from langchain.document_loaders import IFixitLoader
from langchain.document_loaders import IMSDbLoader
from langchain.document_loaders import WebBaseLoader
from langchain.schema import Document
class UrlLoaderComponent(CustomComponent):
display_name: str = "Url Loader"
description: str = "Generic Url Loader Component"
def build_config(self):
return {
"web_path": {
"display_name": "Url",
"required": True,
},
"loader": {
"display_name": "Loader",
"is_list": True,
"required": True,
"options": [
"AZLyricsLoader",
"CollegeConfidentialLoader",
"GitbookLoader",
"HNLoader",
"IFixitLoader",
"IMSDbLoader",
"WebBaseLoader",
],
"value": "WebBaseLoader",
},
"code": {"show": False},
}
def build(self, web_path: str, loader: str) -> List[Document]:
if loader == "AZLyricsLoader":
loader_instance = AZLyricsLoader(web_path=web_path) # type: ignore
elif loader == "CollegeConfidentialLoader":
loader_instance = CollegeConfidentialLoader(web_path=web_path) # type: ignore
elif loader == "GitbookLoader":
loader_instance = GitbookLoader(web_page=web_path) # type: ignore
elif loader == "HNLoader":
loader_instance = HNLoader(web_path=web_path) # type: ignore
elif loader == "IFixitLoader":
loader_instance = IFixitLoader(web_path=web_path) # type: ignore
elif loader == "IMSDbLoader":
loader_instance = IMSDbLoader(web_path=web_path) # type: ignore
elif loader == "WebBaseLoader":
loader_instance = WebBaseLoader(web_path=web_path) # type: ignore
if loader_instance is None:
raise ValueError(f"No loader found for: {web_path}")
return loader_instance.load()

0
src/backend/langflow/components/documentloaders/__init__.py Normal file
View file

4
src/backend/langflow/graph/vertex/types.py
View file

@ -206,6 +206,10 @@ class ChainVertex(Vertex):
) -> Any:
if not self._built or force:
# Check if the chain requires a PromptVertex
# Temporarily remove "code" from the params
self.params.pop("code", None)
for key, value in self.params.items():
if isinstance(value, PromptVertex):
# Build the PromptVertex, passing the tools if available

2
src/backend/langflow/template/frontend_node/chains.py
View file

@ -87,6 +87,8 @@ class ChainFrontendNode(FrontendNode):
field.required = True
field.show = True
field.advanced = False
field.field_type = "BaseLanguageModel" # temporary fix
field.is_list = False
if field.name == "return_source_documents":
field.required = False

9
src/backend/langflow/utils/constants.py
View file

@ -6,14 +6,11 @@ OPENAI_MODELS = [
"text-ada-001",
]
CHAT_OPENAI_MODELS = [
"gpt-3.5-turbo-0613",
"gpt-3.5-turbo",
"gpt-3.5-turbo-16k-0613",
"gpt-3.5-turbo-16k",
"gpt-4-0613",
"gpt-4-32k-0613",
"gpt-4-1106-preview",
"gpt-4",
"gpt-4-32k",
"gpt-3.5-turbo",
"gpt-3.5-turbo-16k",
]
ANTHROPIC_MODELS = [

8
src/backend/langflow/utils/util.py
View file

@ -473,3 +473,11 @@ def build_loader_repr_from_documents(documents: List[Document]) -> str:
\nAvg. Document Length (characters): {int(avg_length)}
Documents: {documents[:3]}..."""
return "0 documents"
def get_type_from_union_literal(union_literal: str) -> str:
# if types are literal strings
# the type is a string
if "Literal" in union_literal:
return "str"
return union_literal

6
src/frontend/src/pages/FlowPage/components/DisclosureComponent/index.tsx
View file

@ -8,7 +8,7 @@ export default function DisclosureComponent({
openDisc,
}: DisclosureComponentType): JSX.Element {
return (
<Disclosure as="div" key={title}>
<Disclosure as="div" defaultOpen={openDisc} key={title}>
{({ open }) => (
<>
<div>
@ -35,9 +35,7 @@ export default function DisclosureComponent({
</div>
</Disclosure.Button>
</div>
<Disclosure.Panel as="div" static={openDisc}>
{children}
</Disclosure.Panel>
<Disclosure.Panel as="div">{children}</Disclosure.Panel>
</>
)}
</Disclosure>

10
src/frontend/src/pages/FlowPage/components/extraSidebarComponent/index.tsx
View file

@ -122,7 +122,7 @@ export default function ExtraSidebar(): JSX.Element {
}
}
});
setSearch("search");
setSearch("");
return ret;
});
}
@ -227,8 +227,12 @@ export default function ExtraSidebar(): JSX.Element {
.map((SBSectionName: keyof APIObjectType, index) =>
Object.keys(dataFilter[SBSectionName]).length > 0 ? (
<DisclosureComponent
openDisc={search.length == 0 ? false : true}
key={index}
openDisc={
getFilterEdge.length !== 0 || search.length !== 0
? true
: false
}
key={index + search + JSON.stringify(getFilterEdge)}
button={{
title: nodeNames[SBSectionName] ?? nodeNames.unknown,
Icon:

1
src/frontend/src/utils/reactflowUtils.ts
View file

@ -8,7 +8,6 @@ import {
ReactFlowJsonObject,
XYPosition,
} from "reactflow";
import ShortUniqueId from "short-unique-id";
import { specialCharsRegex } from "../constants/constants";
import { APITemplateType, TemplateVariableType } from "../types/api";
import {