From 114cdb9ac694c2b7def420a57568bed5a7655202 Mon Sep 17 00:00:00 2001 From: Daniel Gines Date: Wed, 17 Jul 2024 14:48:56 -0300 Subject: [PATCH] feat: Add new Atlassian Confluence Component for document loading and vector database integration (#2718) * feat: Add Gemma 2 to Groq model list (#2586) Add gemma2 to groq_constants.py * Adds new ConfluenceComponent module with lazy loading support - Implements ConfluenceComponent to load documents from the Confluence platform. - Adds necessary inputs, including URL, username, API key, space_key, and more. - Supports configuration of max_pages for pagination control. - Implements lazy loading in the load_documents method for incremental document processing. - Allows immediate processing of documents as they are loaded. This new module facilitates integration with the Confluence platform and enables efficient handling of large volumes of data. * Adds new ConfluenceComponent module - Implements ConfluenceComponent to load documents from the Confluence platform. - Adds necessary inputs, including URL, username, API key, space key, and more. - Supports configuration of max_pages for pagination control. This new module facilitates integration with the Confluence platform. * Updated load_documents method to use Data.from_document - Changed load_documents method to convert documents using Data..from_document instead of docs_to_data for better integration with Data module. - Updated trace_type to "tool" because the LangSmith API only supports one of the following types: ["tool", "chain", "llm", "retriever", "embedding", "prompt", "parser"]. * [autofix.ci] apply automated fixes --------- Co-authored-by: Gordon Stein <7331488+gsteinLTU@users.noreply.github.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- .../components/documentloaders/Confluence.py | 85 +++++++++++++++++++ .../components/documentloaders/__init__.py | 3 + .../src/icons/Confluence/Confluence.jsx | 25 ++++++ .../src/icons/Confluence/Confluence.svg | 4 + src/frontend/src/icons/Confluence/index.tsx | 9 ++ src/frontend/src/utils/styleUtils.ts | 2 + 6 files changed, 128 insertions(+) create mode 100644 src/backend/base/langflow/components/documentloaders/Confluence.py create mode 100644 src/frontend/src/icons/Confluence/Confluence.jsx create mode 100644 src/frontend/src/icons/Confluence/Confluence.svg create mode 100644 src/frontend/src/icons/Confluence/index.tsx diff --git a/src/backend/base/langflow/components/documentloaders/Confluence.py b/src/backend/base/langflow/components/documentloaders/Confluence.py new file mode 100644 index 000000000..66ff5f7fe --- /dev/null +++ b/src/backend/base/langflow/components/documentloaders/Confluence.py @@ -0,0 +1,85 @@ +from typing import List +from langflow.custom import Component +from langflow.io import StrInput, SecretStrInput, BoolInput, DropdownInput, Output, IntInput +from langflow.schema import Data +from langchain_community.document_loaders import ConfluenceLoader +from langchain_community.document_loaders.confluence import ContentFormat + + +class ConfluenceComponent(Component): + display_name = "Confluence" + description = "Confluence wiki collaboration platform" + documentation = "https://python.langchain.com/v0.2/docs/integrations/document_loaders/confluence/" + trace_type = "tool" + icon = "Confluence" + name = "Confluence" + + inputs = [ + StrInput( + name="url", + display_name="Site URL", + required=True, + info="The base URL of the Confluence Space. Example: https://.atlassian.net/wiki.", + ), + StrInput( + name="username", + display_name="Username", + required=True, + info="Atlassian User E-mail. Example: email@example.com", + ), + SecretStrInput( + name="api_key", + display_name="API Key", + required=True, + info="Atlassian Key. Create at: https://id.atlassian.com/manage-profile/security/api-tokens", + ), + StrInput(name="space_key", display_name="Space Key", required=True), + BoolInput(name="cloud", display_name="Use Cloud?", required=True, value=True, advanced=True), + DropdownInput( + name="content_format", + display_name="Content Format", + options=[ + ContentFormat.EDITOR.value, + ContentFormat.EXPORT_VIEW.value, + ContentFormat.ANONYMOUS_EXPORT_VIEW.value, + ContentFormat.STORAGE.value, + ContentFormat.VIEW.value, + ], + value=ContentFormat.STORAGE.value, + required=True, + advanced=True, + info="Specify content format, defaults to ContentFormat.STORAGE", + ), + IntInput( + name="max_pages", + display_name="Max Pages", + required=False, + value=1000, + advanced=True, + info="Maximum number of pages to retrieve in total, defaults 1000", + ), + ] + + outputs = [ + Output(name="data", display_name="Data", method="load_documents"), + ] + + def build_confluence(self) -> ConfluenceLoader: + content_format = ContentFormat(self.content_format) + loader = ConfluenceLoader( + url=self.url, + username=self.username, + api_key=self.api_key, + cloud=self.cloud, + space_key=self.space_key, + content_format=content_format, + max_pages=self.max_pages, + ) + return loader + + def load_documents(self) -> List[Data]: + confluence = self.build_confluence() + documents = confluence.load() + data = [Data.from_document(doc) for doc in documents] # Using the from_document method of Data + self.status = data + return data diff --git a/src/backend/base/langflow/components/documentloaders/__init__.py b/src/backend/base/langflow/components/documentloaders/__init__.py index e69de29bb..d33437c75 100644 --- a/src/backend/base/langflow/components/documentloaders/__init__.py +++ b/src/backend/base/langflow/components/documentloaders/__init__.py @@ -0,0 +1,3 @@ +from .Confluence import ConfluenceComponent + +__all__ = ["ConfluenceComponent"] diff --git a/src/frontend/src/icons/Confluence/Confluence.jsx b/src/frontend/src/icons/Confluence/Confluence.jsx new file mode 100644 index 000000000..a11e95a23 --- /dev/null +++ b/src/frontend/src/icons/Confluence/Confluence.jsx @@ -0,0 +1,25 @@ +const ConfluenceIcon = (props) => ( + + + + +); + +export default ConfluenceIcon; diff --git a/src/frontend/src/icons/Confluence/Confluence.svg b/src/frontend/src/icons/Confluence/Confluence.svg new file mode 100644 index 000000000..f78072aa4 --- /dev/null +++ b/src/frontend/src/icons/Confluence/Confluence.svg @@ -0,0 +1,4 @@ + + + + diff --git a/src/frontend/src/icons/Confluence/index.tsx b/src/frontend/src/icons/Confluence/index.tsx new file mode 100644 index 000000000..fc695ad47 --- /dev/null +++ b/src/frontend/src/icons/Confluence/index.tsx @@ -0,0 +1,9 @@ +import React, { forwardRef } from "react"; +import SvgConfluence from "./Confluence"; + +export const ConfluenceIcon = forwardRef< + SVGSVGElement, + React.PropsWithChildren<{}> +>((props, ref) => { + return ; +}); diff --git a/src/frontend/src/utils/styleUtils.ts b/src/frontend/src/utils/styleUtils.ts index c3a4bf30c..6ad2d94f0 100644 --- a/src/frontend/src/utils/styleUtils.ts +++ b/src/frontend/src/utils/styleUtils.ts @@ -168,6 +168,7 @@ import { BotMessageSquareIcon } from "../icons/BotMessageSquare"; import { CassandraIcon } from "../icons/Cassandra"; import { ChromaIcon } from "../icons/ChromaIcon"; import { CohereIcon } from "../icons/Cohere"; +import { ConfluenceIcon } from "../icons/Confluence"; import { CouchbaseIcon } from "../icons/Couchbase"; import { CrewAiIcon } from "../icons/CrewAI"; import { ElasticsearchIcon } from "../icons/ElasticsearchStore"; @@ -578,5 +579,6 @@ export const nodeIconsLucide: iconsType = { MistralAI: MistralIcon, Upstash: UpstashSvgIcon, PGVector: CpuIcon, + Confluence: ConfluenceIcon, FreezeAll: freezeAllIcon, };