From d2609141bae953467c1ce9814c800189a055f7c9 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Sat, 1 Jul 2023 17:08:23 -0300 Subject: [PATCH 1/7] =?UTF-8?q?=F0=9F=94=A7=20refactor(documentloaders.py)?= =?UTF-8?q?:=20extract=20build=5Fdirectory=5Floader=5Ffields()=20function?= =?UTF-8?q?=20to=20improve=20code=20readability=20and=20maintainability=20?= =?UTF-8?q?=F0=9F=94=A7=20refactor(documentloaders.py):=20remove=20redunda?= =?UTF-8?q?nt=20code=20for=20adding=20fields=20in=20DocumentLoaderFrontNod?= =?UTF-8?q?e=20class=20The=20code=20in=20the=20DocumentLoaderFrontNode=20c?= =?UTF-8?q?lass=20has=20been=20refactored=20to=20extract=20the=20logic=20f?= =?UTF-8?q?or=20building=20directory=20loader=20fields=20into=20a=20separa?= =?UTF-8?q?te=20function=20called=20build=5Fdirectory=5Floader=5Ffields().?= =?UTF-8?q?=20This=20improves=20code=20readability=20and=20maintainability?= =?UTF-8?q?=20by=20separating=20concerns=20and=20reducing=20duplication.?= =?UTF-8?q?=20The=20redundant=20code=20for=20adding=20fields=20in=20the=20?= =?UTF-8?q?DocumentLoaderFrontNode=20class=20has=20also=20been=20removed.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/settings.py | 17 ++- .../template/frontend_node/documentloaders.py | 118 ++++++++++++++++-- 2 files changed, 121 insertions(+), 14 deletions(-) diff --git a/src/backend/langflow/settings.py b/src/backend/langflow/settings.py index d2de4f671..acf25464e 100644 --- a/src/backend/langflow/settings.py +++ b/src/backend/langflow/settings.py @@ -2,6 +2,7 @@ import os import yaml from pydantic import BaseSettings, root_validator +from langflow.utils.logger import logger class Settings(BaseSettings): @@ -20,10 +21,24 @@ class Settings(BaseSettings): textsplitters: dict = {} utilities: dict = {} dev: bool = False - database_url: str = "sqlite:///./langflow.db" + database_url: str cache: str = "InMemoryCache" remove_api_keys: bool = False + # Create a root validator pre that will add the default + # sqlite database_url if not provided + # but check the DATABASE_URL env variable first + @root_validator(pre=True) + def set_database_url(cls, values): + if "database_url" not in values: + logger.debug("No database_url provided, trying DATABASE_URL env variable") + if database_url := os.getenv("DATABASE_URL"): + values["database_url"] = database_url + else: + logger.debug("No DATABASE_URL env variable, using sqlite database") + values["database_url"] = "sqlite:///./langflow.db" + return values + class Config: validate_assignment = True extra = "ignore" diff --git a/src/backend/langflow/template/frontend_node/documentloaders.py b/src/backend/langflow/template/frontend_node/documentloaders.py index 501aa361e..5a43634a2 100644 --- a/src/backend/langflow/template/frontend_node/documentloaders.py +++ b/src/backend/langflow/template/frontend_node/documentloaders.py @@ -120,29 +120,23 @@ class DocumentLoaderFrontNode(FrontendNode): "DirectoryLoader", "ReadTheDocsLoader", "NotionDirectoryLoader", + "PyPDFDirectoryLoader", }: name = "path" display_name = "Local directory" if name: - self.template.add_field( - TemplateField( - field_type="str", - required=True, - show=True, - name=name, - value="", - display_name=display_name, - ) - ) if self.template.type_name in {"DirectoryLoader"}: + for field in build_directory_loader_fields(): + self.template.add_field(field) + else: self.template.add_field( TemplateField( field_type="str", required=True, show=True, - name="glob", - value="**/*.txt", - display_name="glob", + name=name, + value="", + display_name=display_name, ) ) # add a metadata field of type dict @@ -165,3 +159,101 @@ class DocumentLoaderFrontNode(FrontendNode): field.show = True field.advanced = False field.show = True + + +def build_directory_loader_fields(): + # if loader_kwargs is None: + # loader_kwargs = {} + # self.path = path + # self.glob = glob + # self.load_hidden = load_hidden + # self.loader_cls = loader_cls + # self.loader_kwargs = loader_kwargs + # self.silent_errors = silent_errors + # self.recursive = recursive + # self.show_progress = show_progress + # self.use_multithreading = use_multithreading + # self.max_concurrency = max_concurrency + # Based on the above fields, we can build the following fields: + # path, glob, load_hidden, silent_errors, recursive, show_progress, use_multithreading, max_concurrency + # path + path = TemplateField( + field_type="str", + required=True, + show=True, + name="path", + value="", + display_name="Local directory", + advanced=False, + ) + # glob + glob = TemplateField( + field_type="str", + required=True, + show=True, + name="glob", + value="**/*.txt", + display_name="glob", + advanced=False, + ) + # load_hidden + load_hidden = TemplateField( + field_type="bool", + required=False, + show=True, + name="load_hidden", + value="False", + display_name="Load hidden files", + advanced=True, + ) + # silent_errors + silent_errors = TemplateField( + field_type="bool", + required=False, + show=True, + name="silent_errors", + value="False", + display_name="Silent errors", + advanced=True, + ) + # recursive + recursive = TemplateField( + field_type="bool", + required=False, + show=True, + name="recursive", + value="True", + display_name="Recursive", + advanced=True, + ) + + # use_multithreading + use_multithreading = TemplateField( + field_type="bool", + required=False, + show=True, + name="use_multithreading", + value="True", + display_name="Use multithreading", + advanced=True, + ) + # max_concurrency + max_concurrency = TemplateField( + field_type="int", + required=False, + show=True, + name="max_concurrency", + value=10, + display_name="Max concurrency", + advanced=True, + ) + + return ( + path, + glob, + load_hidden, + silent_errors, + recursive, + use_multithreading, + max_concurrency, + ) From e5eea2d9fb514bf0bd87b4124ab9170ade8c8e91 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Sat, 1 Jul 2023 17:08:28 -0300 Subject: [PATCH 2/7] =?UTF-8?q?=F0=9F=90=9B=20fix(=5F=5Fmain=5F=5F.py):=20?= =?UTF-8?q?change=20logger.error=20to=20logger.exception=20to=20log=20full?= =?UTF-8?q?=20exception=20traceback=20The=20change=20is=20made=20to=20impr?= =?UTF-8?q?ove=20the=20logging=20of=20exceptions.=20Using=20logger.excepti?= =?UTF-8?q?on=20instead=20of=20logger.error=20allows=20the=20logger=20to?= =?UTF-8?q?=20log=20the=20full=20traceback=20of=20the=20exception,=20provi?= =?UTF-8?q?ding=20more=20detailed=20information=20for=20debugging=20purpos?= =?UTF-8?q?es.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/langflow/__main__.py b/src/backend/langflow/__main__.py index 9980a5d2f..322c6b6a9 100644 --- a/src/backend/langflow/__main__.py +++ b/src/backend/langflow/__main__.py @@ -271,7 +271,7 @@ def run_langflow(host, port, log_level, options, app): except KeyboardInterrupt: pass except Exception as e: - logger.error(e) + logger.exception(e) sys.exit(1) From 3506cb13114b9f5ad4941969ba6b29ffae2a38ec Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Sat, 1 Jul 2023 17:08:47 -0300 Subject: [PATCH 3/7] =?UTF-8?q?=F0=9F=90=9B=20fix(chatModal):=20refactor?= =?UTF-8?q?=20error=20handling=20in=20WebSocket=20connection=20to=20improv?= =?UTF-8?q?e=20readability=20and=20error=20handling=20The=20error=20handli?= =?UTF-8?q?ng=20in=20the=20WebSocket=20connection=20has=20been=20refactore?= =?UTF-8?q?d=20to=20improve=20readability=20and=20error=20handling.=20Inst?= =?UTF-8?q?ead=20of=20nesting=20the=20`getHealth()`=20promise=20inside=20t?= =?UTF-8?q?he=20`onerror`=20callback,=20it=20has=20been=20extracted=20to?= =?UTF-8?q?=20a=20separate=20`.then()`=20block.=20This=20allows=20for=20be?= =?UTF-8?q?tter=20separation=20of=20concerns=20and=20makes=20the=20code=20?= =?UTF-8?q?more=20readable.=20Additionally,=20the=20error=20handling=20log?= =?UTF-8?q?ic=20has=20been=20updated=20to=20properly=20set=20the=20`setErr?= =?UTF-8?q?orData`=20state=20when=20the=20backend=20fails=20to=20respond.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/frontend/src/modals/chatModal/index.tsx | 32 +++++++++++---------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/frontend/src/modals/chatModal/index.tsx b/src/frontend/src/modals/chatModal/index.tsx index c39ea0294..bb6efbb70 100644 --- a/src/frontend/src/modals/chatModal/index.tsx +++ b/src/frontend/src/modals/chatModal/index.tsx @@ -205,25 +205,27 @@ export default function ChatModal({ handleOnClose(event); }; newWs.onerror = (ev) => { - getHealth().then((res) => { - if (res.status === 200) { - connectWS(); - } - }).catch((err) => { - setErrorData({ - // message when the backend failed - title: "The backend is not responding. Please try again later.", - // possible solution list - list: [ - "Check your internet connection.", - "Check if the backend is running." - ], + getHealth() + .then((res) => { + if (res.status === 200) { + connectWS(); + } + }) + .catch((err) => { + setErrorData({ + // message when the backend failed + title: "The backend is not responding. Please try again later.", + // possible solution list + list: [ + "Check your internet connection.", + "Check if the backend is running.", + ], + }); }); - }) }; ws.current = newWs; } catch (error) { - connectWS(); + connectWS(); console.log(error); } } From 94cd8703b3f49d99f3b670c2138fd4af6a324a29 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Sat, 1 Jul 2023 17:09:04 -0300 Subject: [PATCH 4/7] =?UTF-8?q?=F0=9F=90=9B=20fix(App.tsx):=20improve=20co?= =?UTF-8?q?de=20readability=20by=20formatting=20long=20conditional=20state?= =?UTF-8?q?ments=20The=20conditional=20statements=20in=20the=20useEffect?= =?UTF-8?q?=20hook=20were=20formatted=20to=20improve=20code=20readability.?= =?UTF-8?q?=20By=20breaking=20the=20statements=20into=20multiple=20lines?= =?UTF-8?q?=20and=20adding=20proper=20indentation,=20it=20is=20easier=20to?= =?UTF-8?q?=20understand=20the=20logic.=20This=20change=20does=20not=20aff?= =?UTF-8?q?ect=20the=20functionality=20of=20the=20code.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/frontend/src/App.tsx | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/frontend/src/App.tsx b/src/frontend/src/App.tsx index 3ef728925..166baf4c7 100644 --- a/src/frontend/src/App.tsx +++ b/src/frontend/src/App.tsx @@ -51,7 +51,11 @@ export default function App() { useEffect(() => { // If there is an error alert open with data, add it to the alertsList if (errorOpen && errorData) { - if(alertsList.length > 0 && JSON.stringify(alertsList[alertsList.length - 1].data)===JSON.stringify(errorData)){ + if ( + alertsList.length > 0 && + JSON.stringify(alertsList[alertsList.length - 1].data) === + JSON.stringify(errorData) + ) { return; } setErrorOpen(false); @@ -65,7 +69,11 @@ export default function App() { } // If there is a notice alert open with data, add it to the alertsList else if (noticeOpen && noticeData) { - if(alertsList.length > 0 && JSON.stringify(alertsList[alertsList.length - 1].data)===JSON.stringify(noticeData)){ + if ( + alertsList.length > 0 && + JSON.stringify(alertsList[alertsList.length - 1].data) === + JSON.stringify(noticeData) + ) { return; } setNoticeOpen(false); @@ -79,7 +87,11 @@ export default function App() { } // If there is a success alert open with data, add it to the alertsList else if (successOpen && successData) { - if(alertsList.length > 0 && JSON.stringify(alertsList[alertsList.length - 1].data)===JSON.stringify(successData)){ + if ( + alertsList.length > 0 && + JSON.stringify(alertsList[alertsList.length - 1].data) === + JSON.stringify(successData) + ) { return; } setSuccessOpen(false); @@ -161,4 +173,4 @@ export default function App() { ); -} \ No newline at end of file +} From 4c8c3528de50f87a77c41dac7e4cbdea21a7c75b Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Sat, 1 Jul 2023 17:09:15 -0300 Subject: [PATCH 5/7] =?UTF-8?q?=F0=9F=94=A5=20refactor(settings.py):=20rem?= =?UTF-8?q?ove=20commented=20out=20code=20for=20setting=20default=20databa?= =?UTF-8?q?se=20URL=20The=20commented=20out=20code=20was=20unnecessary=20a?= =?UTF-8?q?nd=20cluttered=20the=20codebase.=20Removing=20it=20improves=20r?= =?UTF-8?q?eadability=20and=20maintainability=20of=20the=20code.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/settings.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/backend/langflow/settings.py b/src/backend/langflow/settings.py index acf25464e..b2571780e 100644 --- a/src/backend/langflow/settings.py +++ b/src/backend/langflow/settings.py @@ -25,9 +25,6 @@ class Settings(BaseSettings): cache: str = "InMemoryCache" remove_api_keys: bool = False - # Create a root validator pre that will add the default - # sqlite database_url if not provided - # but check the DATABASE_URL env variable first @root_validator(pre=True) def set_database_url(cls, values): if "database_url" not in values: From 30c9b4f5a91e4b7685e09ff0337f87e42a2d5deb Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Sat, 1 Jul 2023 17:09:26 -0300 Subject: [PATCH 6/7] =?UTF-8?q?=F0=9F=94=A7=20chore(base.py):=20add=20debu?= =?UTF-8?q?g=20logs=20when=20creating=20and=20creating=20database=20and=20?= =?UTF-8?q?tables=20The=20logger.debug()=20statements=20are=20added=20to?= =?UTF-8?q?=20provide=20more=20visibility=20into=20the=20process=20of=20cr?= =?UTF-8?q?eating=20the=20database=20and=20tables.=20This=20can=20be=20hel?= =?UTF-8?q?pful=20for=20debugging=20and=20troubleshooting=20purposes.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/database/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/langflow/database/base.py b/src/backend/langflow/database/base.py index 69f8b993b..a0efcdd2a 100644 --- a/src/backend/langflow/database/base.py +++ b/src/backend/langflow/database/base.py @@ -1,6 +1,6 @@ from langflow.settings import settings from sqlmodel import SQLModel, Session, create_engine - +from langflow.utils.logger import logger if settings.database_url.startswith("sqlite"): connect_args = {"check_same_thread": False} @@ -10,7 +10,9 @@ engine = create_engine(settings.database_url, connect_args=connect_args) def create_db_and_tables(): + logger.debug("Creating database and tables") SQLModel.metadata.create_all(engine) + logger.debug("Database and tables created") def get_session(): From 5333dbf0215d5a12b57c4a2e7f51151371e8c059 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Sat, 1 Jul 2023 17:09:35 -0300 Subject: [PATCH 7/7] =?UTF-8?q?=F0=9F=93=9D=20docs(config.yaml):=20add=20P?= =?UTF-8?q?yPDFDirectoryLoader=20documentation=20link=20Added=20the=20docu?= =?UTF-8?q?mentation=20link=20for=20the=20PyPDFDirectoryLoader=20in=20the?= =?UTF-8?q?=20config.yaml=20file.=20This=20allows=20users=20to=20access=20?= =?UTF-8?q?the=20documentation=20on=20how=20to=20use=20the=20PyPDFDirector?= =?UTF-8?q?yLoader=20for=20loading=20PDF=20documents.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/langflow/config.yaml b/src/backend/langflow/config.yaml index aa1206d59..3745b9e22 100644 --- a/src/backend/langflow/config.yaml +++ b/src/backend/langflow/config.yaml @@ -59,6 +59,8 @@ documentloaders: documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/html" UnstructuredMarkdownLoader: documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/markdown" + PyPDFDirectoryLoader: + documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/pdf" PyPDFLoader: documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/pdf" UnstructuredPowerPointLoader: