diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 90966fb38..ab0ccf145 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -15,7 +15,7 @@ // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "make install_frontend && make install_backend", + "postCreateCommand": "make setup_devcontainer", "containerEnv": { "POETRY_VIRTUALENVS_IN_PROJECT": "true" @@ -31,11 +31,13 @@ "sourcery.sourcery", "eamodio.gitlens", "ms-vscode.makefile-tools", - "GitHub.vscode-pull-request-github" + "GitHub.vscode-pull-request-github", + "Codium.codium", + "ms-azuretools.vscode-docker" ] } } // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. // "remoteUser": "root" -} +} \ No newline at end of file diff --git a/Makefile b/Makefile index 5828e94d2..d5d699fed 100644 --- a/Makefile +++ b/Makefile @@ -49,15 +49,16 @@ run_frontend: cd src/frontend && npm start run_cli: - poetry run langflow run --path src/frontend/build + poetry run langflow --path src/frontend/build run_cli_debug: - poetry run langflow run --path src/frontend/build --log-level debug + poetry run langflow --path src/frontend/build --log-level debug setup_devcontainer: make init make build_frontend - poetry run langflow --path src/frontend/build + @echo 'Run Cli' + make run_cli frontend: @-make install_frontend || (echo "An error occurred while installing frontend dependencies. Attempting to fix." && make install_frontendc) diff --git a/poetry.lock b/poetry.lock index 913fa3c43..e6ef5431c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1008,6 +1008,17 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "colorclass" +version = "2.2.2" +description = "Colorful worry-free console applications for Linux, Mac OS X, and Windows." +optional = false +python-versions = ">=2.6" +files = [ + {file = "colorclass-2.2.2-py2.py3-none-any.whl", hash = "sha256:6f10c273a0ef7a1150b1120b6095cbdd68e5cf36dfd5d0fc957a2500bbf99a55"}, + {file = "colorclass-2.2.2.tar.gz", hash = "sha256:6d4fe287766166a98ca7bc6f6312daf04a0481b1eda43e7173484051c0ab4366"}, +] + [[package]] name = "coloredlogs" version = "15.0.1" @@ -1059,6 +1070,16 @@ traitlets = ">=4" [package.extras] test = ["pytest"] +[[package]] +name = "compressed-rtf" +version = "1.0.6" +description = "Compressed Rich Text Format (RTF) compression and decompression package" +optional = false +python-versions = "*" +files = [ + {file = "compressed_rtf-1.0.6.tar.gz", hash = "sha256:c1c827f1d124d24608981a56e8b8691eb1f2a69a78ccad6440e7d92fde1781dd"}, +] + [[package]] name = "configargparse" version = "1.7" @@ -1405,6 +1426,27 @@ files = [ {file = "docstring_parser-0.15.tar.gz", hash = "sha256:48ddc093e8b1865899956fcc03b03e66bb7240c310fac5af81814580c55bf682"}, ] +[[package]] +name = "easygui" +version = "0.98.3" +description = "EasyGUI is a module for very simple, very easy GUI programming in Python. EasyGUI is different from other GUI generators in that EasyGUI is NOT event-driven. Instead, all GUI interactions are invoked by simple function calls." +optional = false +python-versions = "*" +files = [ + {file = "easygui-0.98.3-py2.py3-none-any.whl", hash = "sha256:33498710c68b5376b459cd3fc48d1d1f33822139eb3ed01defbc0528326da3ba"}, + {file = "easygui-0.98.3.tar.gz", hash = "sha256:d653ff79ee1f42f63b5a090f2f98ce02335d86ad8963b3ce2661805cafe99a04"}, +] + +[[package]] +name = "ebcdic" +version = "1.1.1" +description = "Additional EBCDIC codecs" +optional = false +python-versions = "*" +files = [ + {file = "ebcdic-1.1.1-py2.py3-none-any.whl", hash = "sha256:33b4cb729bc2d0bf46cc1847b0e5946897cb8d3f53520c5b9aa5fa98d7e735f1"}, +] + [[package]] name = "ecdsa" version = "0.18.0" @@ -1515,6 +1557,32 @@ files = [ [package.extras] tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"] +[[package]] +name = "extract-msg" +version = "0.45.0" +description = "Extracts emails and attachments saved in Microsoft Outlook's .msg files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "extract_msg-0.45.0-py2.py3-none-any.whl", hash = "sha256:af645ffe1534bce93b20390576dac2aee027c17a714365172d31b3894f810ca7"}, + {file = "extract_msg-0.45.0.tar.gz", hash = "sha256:6814865cf2ba806bd69af53af688a13e000a95d4991cce6a0416b3bdeb739496"}, +] + +[package.dependencies] +beautifulsoup4 = ">=4.11.1,<4.13" +compressed-rtf = ">=1.0.6,<2" +ebcdic = ">=1.1.1,<2" +imapclient = ">=2.3.0,<3" +olefile = "0.46" +red-black-tree-mod = "1.20" +RTFDE = ">=0.1.0,<0.2" +tzlocal = ">=4.2,<6" + +[package.extras] +all = ["extract-msg[image]", "extract-msg[mime]"] +image = ["Pillow (>=9.5.0,<10)"] +mime = ["python-magic (>=0.4.27,<0.5)"] + [[package]] name = "faiss-cpu" version = "1.7.4" @@ -2882,6 +2950,24 @@ files = [ {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, ] +[[package]] +name = "imapclient" +version = "2.3.1" +description = "Easy-to-use, Pythonic and complete IMAP client library" +optional = false +python-versions = "*" +files = [ + {file = "IMAPClient-2.3.1-py2.py3-none-any.whl", hash = "sha256:057f28025d2987c63e065afb0e4370b0b850b539b0e1494cea0427e88130108c"}, + {file = "IMAPClient-2.3.1.zip", hash = "sha256:26ea995664fae3a88b878ebce2aff7402931697b86658b7882043ddb01b0e6ba"}, +] + +[package.dependencies] +six = "*" + +[package.extras] +doc = ["sphinx"] +test = ["mock (>=1.3.0)"] + [[package]] name = "importlib-metadata" version = "6.8.0" @@ -2903,13 +2989,13 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs [[package]] name = "importlib-resources" -version = "6.1.0" +version = "6.1.1" description = "Read resources from Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_resources-6.1.0-py3-none-any.whl", hash = "sha256:aa50258bbfa56d4e33fbd8aa3ef48ded10d1735f11532b8df95388cc6bdb7e83"}, - {file = "importlib_resources-6.1.0.tar.gz", hash = "sha256:9d48dcccc213325e810fd723e7fbb45ccb39f6cf5c31f00cf2b965f5f10f3cb9"}, + {file = "importlib_resources-6.1.1-py3-none-any.whl", hash = "sha256:e8bf90d8213b486f428c9c39714b920041cb02c184686a3dee24905aaa8105d6"}, + {file = "importlib_resources-6.1.1.tar.gz", hash = "sha256:3893a00122eafde6894c59914446a512f728a0c1a45f9bb9b63721b6bacf0b4a"}, ] [package.dependencies] @@ -3234,6 +3320,89 @@ files = [ {file = "joblib-1.3.2.tar.gz", hash = "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1"}, ] +[[package]] +name = "jq" +version = "1.6.0" +description = "jq is a lightweight and flexible JSON processor." +optional = false +python-versions = ">=3.5" +files = [ + {file = "jq-1.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5773851cfb9ec6525f362f5bf7f18adab5c1fd1f0161c3599264cd0118c799da"}, + {file = "jq-1.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a758df4eae767a21ebd8466dfd0066d99c9741d9f7fd4a7e1d5b5227e1924af7"}, + {file = "jq-1.6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15cf9dd3e7fb40d029f12f60cf418374c0b830a6ea6267dd285b48809069d6af"}, + {file = "jq-1.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7e768cf5c25d703d944ef81c787d745da0eb266a97768f3003f91c4c828118d"}, + {file = "jq-1.6.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:85a697b3cdc65e787f90faa1237caa44c117b6b2853f21263c3f0b16661b192c"}, + {file = "jq-1.6.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:944e081c328501ddc0a22a8f08196df72afe7910ca11e1a1f21244410dbdd3b3"}, + {file = "jq-1.6.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:09262d0e0cafb03acc968622e6450bb08abfb14c793bab47afd2732b47c655fd"}, + {file = "jq-1.6.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:611f460f616f957d57e0da52ac6e1e6294b073c72a89651da5546a31347817bd"}, + {file = "jq-1.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aba35b5cc07cd75202148e55f47ede3f4d0819b51c80f6d0c82a2ca47db07189"}, + {file = "jq-1.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ef5ddb76b03610df19a53583348aed3604f21d0ba6b583ee8d079e8df026cd47"}, + {file = "jq-1.6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:872f322ff7bfd7daff41b7e8248d414a88722df0e82d1027f3b091a438543e63"}, + {file = "jq-1.6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca7a2982ff26f4620ac03099542a0230dabd8787af3f03ac93660598e26acbf0"}, + {file = "jq-1.6.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:316affc6debf15eb05b7fd8e84ebf8993042b10b840e8d2a504659fb3ba07992"}, + {file = "jq-1.6.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9bc42ade4de77fe4370c0e8e105ef10ad1821ef74d61dcc70982178b9ecfdc72"}, + {file = "jq-1.6.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:02da59230912b886ed45489f3693ce75877f3e99c9e490c0a2dbcf0db397e0df"}, + {file = "jq-1.6.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7ea39f89aa469eb12145ddd686248916cd6d186647aa40b319af8444b1f45a2d"}, + {file = "jq-1.6.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6e9016f5ba064fabc527adb609ebae1f27cac20c8e0da990abae1cfb12eca706"}, + {file = "jq-1.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:022be104a548f7fbddf103ce749937956df9d37a4f2f1650396dacad73bce7ee"}, + {file = "jq-1.6.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d5a7f31f779e1aa3d165eaec237d74c7f5728227e81023a576c939ba3da34f8"}, + {file = "jq-1.6.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f1533a2a15c42be3368878b4031b12f30441246878e0b5f6bedfdd7828cdb1f"}, + {file = "jq-1.6.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aa67a304e58aa85c550ec011a68754ae49abe227b37d63a351feef4eea4c7a7"}, + {file = "jq-1.6.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0893d1590cfa6facaf787cc6c28ac51e47d0d06a303613f84d4943ac0ca98e32"}, + {file = "jq-1.6.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:63db80b4803905a4f4f6c87a17aa1816c530f6262bc795773ebe60f8ab259092"}, + {file = "jq-1.6.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e2c1f429e644cb962e846a6157b5352c3c556fbd0b22bba9fc2fea0710333369"}, + {file = "jq-1.6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:bcf574f28809ec63b8df6456fdd4a981751b7466851e80621993b4e9d3e3c8ee"}, + {file = "jq-1.6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:49dbe0f003b411ca52b5d0afaf09cad8e430a1011181c86f2ef720a0956f31c1"}, + {file = "jq-1.6.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f5a9c4185269a5faf395aa7ca086c7b02c9c8b448d542be3b899041d06e0970"}, + {file = "jq-1.6.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8265f3badcd125f234e55dfc02a078c5decdc6faafcd453fde04d4c0d2699886"}, + {file = "jq-1.6.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:c6c39b53d000d2f7f9f6338061942b83c9034d04f3bc99acae0867d23c9e7127"}, + {file = "jq-1.6.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:9897931ea7b9a46f8165ee69737ece4a2e6dbc8e10ececb81f459d51d71401df"}, + {file = "jq-1.6.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:6312237159e88e92775ea497e0c739590528062d4074544aacf12a08d252f966"}, + {file = "jq-1.6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:aa786a60bdd1a3571f092a4021dd9abf6c46798530fa99f19ecf4f0fceaa7eaf"}, + {file = "jq-1.6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22495573d8221320d3433e1aeded40132bd8e1726845629558bd73aaa66eef7b"}, + {file = "jq-1.6.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:711eabc5d33ef3ec581e0744d9cff52f43896d84847a2692c287a0140a29c915"}, + {file = "jq-1.6.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57e75c1563d083b0424690b3c3ef2bb519e670770931fe633101ede16615d6ee"}, + {file = "jq-1.6.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c795f175b1a13bd716a0c180d062cc8e305271f47bbdb9eb0f0f62f7e4f5def4"}, + {file = "jq-1.6.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:227b178b22a7f91ae88525810441791b1ca1fc71c86f03190911793be15cec3d"}, + {file = "jq-1.6.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:780eb6383fbae12afa819ef676fc93e1548ae4b076c004a393af26a04b460742"}, + {file = "jq-1.6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:08ded6467f4ef89fec35b2bf310f210f8cd13fbd9d80e521500889edf8d22441"}, + {file = "jq-1.6.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49e44ed677713f4115bd5bf2dbae23baa4cd503be350e12a1c1f506b0687848f"}, + {file = "jq-1.6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:984f33862af285ad3e41e23179ac4795f1701822473e1a26bf87ff023e5a89ea"}, + {file = "jq-1.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f42264fafc6166efb5611b5d4cb01058887d050a6c19334f6a3f8a13bb369df5"}, + {file = "jq-1.6.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a67154f150aaf76cc1294032ed588436eb002097dd4fd1e283824bf753a05080"}, + {file = "jq-1.6.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:1b3b95d5fd20e51f18a42647fdb52e5d8aaf150b7a666dd659cf282a2221ee3f"}, + {file = "jq-1.6.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3a8d98f72111043e75610cad7fa9ec5aec0b1ee2f7332dc7fd0f6603ea8144f8"}, + {file = "jq-1.6.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:487483f10ae8f70e6acf7723f31b329736de4b421ce56b2f43b46d5cbd7337b0"}, + {file = "jq-1.6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:18a700f55b7ef83a1382edf0a48cb176b22bacd155e097375ef2345ff8621d97"}, + {file = "jq-1.6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68aec8534ac3c4705e524b4ef54f66b8bdc867df9e0af2c3895e82c6774b5374"}, + {file = "jq-1.6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7a164748dbd03bb06d23bab7ead7ba7e5c4fcfebea7b082bdcd21d14136931e"}, + {file = "jq-1.6.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa22d24740276a8ce82411e4960ed2b5fab476230f913f9d9cf726f766a22208"}, + {file = "jq-1.6.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c1a6fae1b74b3e0478e281eb6addedad7b32421221ac685e21c1d49af5e997f"}, + {file = "jq-1.6.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ce628546c22792b8870b9815086f65873ebb78d7bf617b5a16dd839adba36538"}, + {file = "jq-1.6.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7bb685f337cf5d4f4fe210c46220e31a7baec02a0ca0df3ace3dd4780328fc30"}, + {file = "jq-1.6.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bdbbc509a35ee6082d79c1f25eb97c08f1c59043d21e0772cd24baa909505899"}, + {file = "jq-1.6.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1b332dfdf0d81fb7faf3d12aabf997565d7544bec9812e0ac5ee55e60ef4df8c"}, + {file = "jq-1.6.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:3a4f6ef8c0bd19beae56074c50026665d66345d1908f050e5c442ceac2efe398"}, + {file = "jq-1.6.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5184c2fcca40f8f2ab1b14662721accf68b4b5e772e2f5336fec24aa58fe235a"}, + {file = "jq-1.6.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:689429fe1e07a2d6041daba2c21ced3a24895b2745326deb0c90ccab9386e116"}, + {file = "jq-1.6.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8405d1c996c83711570f16aac32e3bf2c116d6fa4254a820276b87aed544d7e8"}, + {file = "jq-1.6.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:138d56c7efc8bb162c1cfc3806bd6b4d779115943af36c9e3b8ca644dde856c2"}, + {file = "jq-1.6.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd28f8395687e45bba56dc771284ebb6492b02037f74f450176c102f3f4e86a3"}, + {file = "jq-1.6.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2c783288bf10e67aad321b58735e663f4975d7ddfbfb0a5bca8428eee283bde"}, + {file = "jq-1.6.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:206391ac5b2eb556720b94f0f131558cbf8d82d8cc7e0404e733eeef48bcd823"}, + {file = "jq-1.6.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:35090fea1283402abc3a13b43261468162199d8b5dcdaba2d1029e557ed23070"}, + {file = "jq-1.6.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:201c6384603aec87a744ad7b393cc4f1c58ece23d6e0a6c216a47bfcc405d231"}, + {file = "jq-1.6.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3d8b075351c29653f29a1fec5d31bc88aa198a0843c0a9550b9be74d8fab33b"}, + {file = "jq-1.6.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:132e41f6e988c42b91c04b1b60dd8fa185a5c0681de5438ea1e6c64f5329768c"}, + {file = "jq-1.6.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1cb4751808b1d0dbddd37319e0c574fb0c3a29910d52ba35890b1343a1f1e59"}, + {file = "jq-1.6.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:bd158911ed5f5c644f557ad94d6424c411560632a885eae47d105f290f0109cb"}, + {file = "jq-1.6.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:64bc09ae6a9d9b82b78e15d142f90b816228bd3ee48833ddca3ff8c08e163fa7"}, + {file = "jq-1.6.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4eed167322662f4b7e65235723c54aa6879f6175b6f9b68bc24887549637ffb"}, + {file = "jq-1.6.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64bb4b305e2fabe5b5161b599bf934aceb0e0e7d3dd8f79246737ea91a2bc9ae"}, + {file = "jq-1.6.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:165bfbe29bf73878d073edf75f384b7da8a9657ba0ab9fb1e5fe6be65ab7debb"}, + {file = "jq-1.6.0.tar.gz", hash = "sha256:c7711f0c913a826a00990736efa6ffc285f8ef433414516bb14b7df971d6c1ea"}, +] + [[package]] name = "jsonpatch" version = "1.33" @@ -3412,13 +3581,13 @@ text-helpers = ["chardet (>=5.1.0,<6.0.0)"] [[package]] name = "langchain-experimental" -version = "0.0.34" +version = "0.0.37" description = "Building applications with LLMs through composability" optional = false python-versions = ">=3.8.1,<4.0" files = [ - {file = "langchain_experimental-0.0.34-py3-none-any.whl", hash = "sha256:1fb4de07369d304a06c3cbe979bdeabcf4efa6f604bd11f2896fcbfb1ae628bf"}, - {file = "langchain_experimental-0.0.34.tar.gz", hash = "sha256:a502e00a4d30490c43c92d16f70ffe16146dd01ecacb99fa979fec933b5497b5"}, + {file = "langchain_experimental-0.0.37-py3-none-any.whl", hash = "sha256:5cf5b26f142766596a1e313cabbfceb9cbd5cd2cf46f4a21e92c66ed11466dac"}, + {file = "langchain_experimental-0.0.37.tar.gz", hash = "sha256:c30fae03702ab071b7040ff6a0fdedd4b4d1028c6c2d8481db0921d7a216e078"}, ] [package.dependencies] @@ -3476,6 +3645,22 @@ files = [ pydantic = ">=1,<3" requests = ">=2,<3" +[[package]] +name = "lark" +version = "1.1.5" +description = "a modern parsing library" +optional = false +python-versions = "*" +files = [ + {file = "lark-1.1.5-py3-none-any.whl", hash = "sha256:8476f9903e93fbde4f6c327f74d79e9b4bd0ed9294c5dfa3164ab8c581b5de2a"}, + {file = "lark-1.1.5.tar.gz", hash = "sha256:4b534eae1f9af5b4ea000bea95776350befe1981658eea3820a01c37e504bb4d"}, +] + +[package.extras] +atomic-cache = ["atomicwrites"] +nearley = ["js2py"] +regex = ["regex"] + [[package]] name = "llama-cpp-python" version = "0.2.14" @@ -3906,6 +4091,21 @@ files = [ {file = "msgpack-1.0.7.tar.gz", hash = "sha256:572efc93db7a4d27e404501975ca6d2d9775705c2d922390d878fcf768d92c87"}, ] +[[package]] +name = "msoffcrypto-tool" +version = "5.1.1" +description = "Python tool and library for decrypting MS Office files with passwords or other keys" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "msoffcrypto_tool-5.1.1-py3-none-any.whl", hash = "sha256:27475aaf8a70485471ad86426c0be10ee4e24c6fad70335e4a8f88d2da323ca1"}, + {file = "msoffcrypto_tool-5.1.1.tar.gz", hash = "sha256:5585a303fa3ee49eec0253f912be17b82cf83f13f0f7489b4ea10f4ecb285278"}, +] + +[package.dependencies] +cryptography = ">=35.0" +olefile = ">=0.46" + [[package]] name = "multidict" version = "6.0.4" @@ -4210,6 +4410,38 @@ files = [ {file = "numpy-1.26.1.tar.gz", hash = "sha256:c8c6c72d4a9f831f328efb1312642a1cafafaa88981d9ab76368d50d07d93cbe"}, ] +[[package]] +name = "olefile" +version = "0.46" +description = "Python package to parse, read and write Microsoft OLE2 files (Structured Storage or Compound Document, Microsoft Office)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "olefile-0.46.zip", hash = "sha256:133b031eaf8fd2c9399b78b8bc5b8fcbe4c31e85295749bb17a87cba8f3c3964"}, +] + +[[package]] +name = "oletools" +version = "0.60.1" +description = "Python tools to analyze security characteristics of MS Office and OLE files (also called Structured Storage, Compound File Binary Format or Compound Document File Format), for Malware Analysis and Incident Response #DFIR" +optional = false +python-versions = "*" +files = [ + {file = "oletools-0.60.1-py2.py3-none-any.whl", hash = "sha256:edef92374e688989a39269eb9a11142fb20a023629c23538c849c14d1d1144ea"}, + {file = "oletools-0.60.1.zip", hash = "sha256:67a796da4c4b8e2feb9a6b2495bef8798a3323a75512de4e5669d9dc9d1fae31"}, +] + +[package.dependencies] +colorclass = "*" +easygui = "*" +msoffcrypto-tool = {version = "*", markers = "platform_python_implementation != \"PyPy\" or python_version >= \"3\" and (platform_system != \"Windows\" and platform_system != \"Darwin\")"} +olefile = ">=0.46" +pcodedmp = ">=1.2.5" +pyparsing = ">=2.1.0,<3" + +[package.extras] +full = ["XLMMacroDeobfuscator"] + [[package]] name = "onnxruntime" version = "1.16.1" @@ -4253,25 +4485,25 @@ sympy = "*" [[package]] name = "openai" -version = "1.1.1" -description = "Client library for the openai API" +version = "0.27.10" +description = "Python client library for the OpenAI API" optional = false python-versions = ">=3.7.1" files = [ - {file = "openai-1.1.1-py3-none-any.whl", hash = "sha256:1496418b132c88352bcfffa8c24e83a69f0e01b1484cbb7bb48f722aad8fd6e1"}, - {file = "openai-1.1.1.tar.gz", hash = "sha256:80e49cb21d8445f6d51339b8af7376fc83302c78ab78578b78133ef89634869d"}, + {file = "openai-0.27.10-py3-none-any.whl", hash = "sha256:beabd1757e3286fa166dde3b70ebb5ad8081af046876b47c14c41e203ed22a14"}, + {file = "openai-0.27.10.tar.gz", hash = "sha256:60e09edf7100080283688748c6803b7b3b52d5a55d21890f3815292a0552d83b"}, ] [package.dependencies] -anyio = ">=3.5.0,<4" -distro = ">=1.7.0,<2" -httpx = ">=0.23.0,<1" -pydantic = ">=1.9.0,<3" -tqdm = ">4" -typing-extensions = ">=4.5,<5" +aiohttp = "*" +requests = ">=2.20" +tqdm = "*" [package.extras] -datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] +datalib = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] +dev = ["black (>=21.6b0,<22.0)", "pytest (==6.*)", "pytest-asyncio", "pytest-mock"] +embeddings = ["matplotlib", "numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)", "plotly", "scikit-learn (>=1.0.2)", "scipy", "tenacity (>=8.0.1)"] +wandb = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)", "wandb"] [[package]] name = "opentelemetry-api" @@ -4749,6 +4981,21 @@ files = [ {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, ] +[[package]] +name = "pcodedmp" +version = "1.2.6" +description = "A VBA p-code disassembler" +optional = false +python-versions = "*" +files = [ + {file = "pcodedmp-1.2.6-py2.py3-none-any.whl", hash = "sha256:4441f7c0ab4cbda27bd4668db3b14f36261d86e5059ce06c0828602cbe1c4278"}, + {file = "pcodedmp-1.2.6.tar.gz", hash = "sha256:025f8c809a126f45a082ffa820893e6a8d990d9d7ddb68694b5a9f0a6dbcd955"}, +] + +[package.dependencies] +oletools = ">=0.54" +win-unicode-console = {version = "*", markers = "platform_system == \"Windows\" and platform_python_implementation != \"PyPy\""} + [[package]] name = "pexpect" version = "4.8.0" @@ -5691,18 +5938,15 @@ zstd = ["zstandard"] [[package]] name = "pyparsing" -version = "3.1.1" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" +version = "2.4.7" +description = "Python parsing module" optional = false -python-versions = ">=3.6.8" +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ - {file = "pyparsing-3.1.1-py3-none-any.whl", hash = "sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb"}, - {file = "pyparsing-3.1.1.tar.gz", hash = "sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db"}, + {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, + {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, ] -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] - [[package]] name = "pypdf" version = "3.17.0" @@ -6286,6 +6530,16 @@ python-dateutil = ">=2.8.1,<3.0.0" typing-extensions = ">=4.2.0,<5.0.0" websockets = ">=10.3,<11.0" +[[package]] +name = "red-black-tree-mod" +version = "1.20" +description = "Flexible python implementation of red black trees" +optional = false +python-versions = "*" +files = [ + {file = "red-black-tree-mod-1.20.tar.gz", hash = "sha256:2448e6fc9cbf1be204c753f352c6ee49aa8156dbf1faa57dfc26bd7705077e0a"}, +] + [[package]] name = "redis" version = "4.6.0" @@ -6587,6 +6841,25 @@ files = [ [package.dependencies] pyasn1 = ">=0.1.3" +[[package]] +name = "rtfde" +version = "0.1.0" +description = "A library for extracting HTML content from RTF encapsulated HTML as commonly found in the exchange MSG email format." +optional = false +python-versions = ">=3.6" +files = [ + {file = "RTFDE-0.1.0-py3-none-any.whl", hash = "sha256:a110dbef435803f3fba717d51a7b9c7a92695c2461637cc6eaf36a9f54386e26"}, + {file = "RTFDE-0.1.0.tar.gz", hash = "sha256:12215ee59856208010b9200c19afe0f9fa13a3fb39f44015979299c248cbacd7"}, +] + +[package.dependencies] +lark = "1.1.5" +oletools = ">=0.56" + +[package.extras] +dev = ["coverage (>=7.2.2)", "lxml (>=4.6)", "mypy (>=1.1.1)", "pdoc3 (>=0.10.0)"] +msg-parse = ["extract-msg (>=0.27)"] + [[package]] name = "ruff" version = "0.0.254" @@ -7871,15 +8144,32 @@ files = [ {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, ] +[[package]] +name = "tzlocal" +version = "5.2" +description = "tzinfo object for the local timezone" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tzlocal-5.2-py3-none-any.whl", hash = "sha256:49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8"}, + {file = "tzlocal-5.2.tar.gz", hash = "sha256:8d399205578f1a9342816409cc1e46a93ebd5755e39ea2d85334bea911bf0e6e"}, +] + +[package.dependencies] +tzdata = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +devenv = ["check-manifest", "pytest (>=4.3)", "pytest-cov", "pytest-mock (>=3.3)", "zest.releaser"] + [[package]] name = "unstructured" -version = "0.10.28" +version = "0.10.29" description = "A library that prepares raw documents for downstream ML tasks." optional = false python-versions = ">=3.7.0" files = [ - {file = "unstructured-0.10.28-py3-none-any.whl", hash = "sha256:0965567c2477c5a926517a295fd0318c2823a603d79c6f47f2bbe58f5aacb89e"}, - {file = "unstructured-0.10.28.tar.gz", hash = "sha256:c995015c5d51d4732f2aa49663eebac4aacf51d7e905c8f240d14ff2269d392f"}, + {file = "unstructured-0.10.29-py3-none-any.whl", hash = "sha256:4b34abe5f7d61b57f5f135e79c2f2005661390e8348de9095e885d3919c6a3f7"}, + {file = "unstructured-0.10.29.tar.gz", hash = "sha256:ebf639f365ab28dfa985ad5a3043a89940c21e5db5ea08f5fa792b402cdcd10c"}, ] [package.dependencies] @@ -7902,7 +8192,7 @@ typing-extensions = "*" [package.extras] airtable = ["pyairtable"] -all-docs = ["markdown", "msg-parser", "networkx", "onnx", "openpyxl", "pandas", "pdf2image", "pdfminer.six", "pypandoc", "python-docx (>=1.0.1)", "python-pptx (<=0.6.21)", "unstructured-inference (==0.7.10)", "unstructured.pytesseract (>=0.3.12)", "xlrd"] +all-docs = ["markdown", "msg-parser", "networkx", "onnx", "openpyxl", "pandas", "pdf2image", "pdfminer.six", "pypandoc", "python-docx (>=1.0.1)", "python-pptx (<=0.6.21)", "unstructured-inference (==0.7.11)", "unstructured.pytesseract (>=0.3.12)", "xlrd"] azure = ["adlfs", "fsspec (==2023.9.1)"] azure-cognitive-search = ["azure-search-documents"] bedrock = ["boto3", "langchain"] @@ -7923,9 +8213,9 @@ github = ["pygithub (>1.58.0)"] gitlab = ["python-gitlab"] google-drive = ["google-api-python-client"] huggingface = ["langdetect", "sacremoses", "sentencepiece", "torch", "transformers"] -image = ["onnx", "pdf2image", "pdfminer.six", "unstructured-inference (==0.7.10)", "unstructured.pytesseract (>=0.3.12)"] +image = ["onnx", "pdf2image", "pdfminer.six", "unstructured-inference (==0.7.11)", "unstructured.pytesseract (>=0.3.12)"] jira = ["atlassian-python-api"] -local-inference = ["markdown", "msg-parser", "networkx", "onnx", "openpyxl", "pandas", "pdf2image", "pdfminer.six", "pypandoc", "python-docx (>=1.0.1)", "python-pptx (<=0.6.21)", "unstructured-inference (==0.7.10)", "unstructured.pytesseract (>=0.3.12)", "xlrd"] +local-inference = ["markdown", "msg-parser", "networkx", "onnx", "openpyxl", "pandas", "pdf2image", "pdfminer.six", "pypandoc", "python-docx (>=1.0.1)", "python-pptx (<=0.6.21)", "unstructured-inference (==0.7.11)", "unstructured.pytesseract (>=0.3.12)", "xlrd"] md = ["markdown"] msg = ["msg-parser"] notion = ["htmlBuilder", "notion-client"] @@ -7935,7 +8225,7 @@ openai = ["langchain", "openai", "tiktoken"] org = ["pypandoc"] outlook = ["Office365-REST-Python-Client (<2.4.3)", "msal"] paddleocr = ["unstructured.paddleocr (==2.6.1.3)"] -pdf = ["onnx", "pdf2image", "pdfminer.six", "unstructured-inference (==0.7.10)", "unstructured.pytesseract (>=0.3.12)"] +pdf = ["onnx", "pdf2image", "pdfminer.six", "unstructured-inference (==0.7.11)", "unstructured.pytesseract (>=0.3.12)"] ppt = ["python-pptx (<=0.6.21)"] pptx = ["python-pptx (<=0.6.21)"] reddit = ["praw"] @@ -8321,6 +8611,16 @@ files = [ beautifulsoup4 = "*" requests = ">=2.0.0,<3.0.0" +[[package]] +name = "win-unicode-console" +version = "0.5" +description = "Enable Unicode input and display when running Python from Windows console." +optional = false +python-versions = "*" +files = [ + {file = "win_unicode_console-0.5.zip", hash = "sha256:d4142d4d56d46f449d6f00536a73625a871cba040f0bc1a2e305a04578f07d1e"}, +] + [[package]] name = "win32-setctime" version = "1.1.0" @@ -8616,4 +8916,4 @@ local = ["ctransformers", "llama-cpp-python", "sentence-transformers"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.11" -content-hash = "fc69dc01e886450e6499d56d7548a3ee76a89d30f441e49f95ce5a7b45437b17" +content-hash = "856876d1b3d9e11c74e0d68207e4a31634e83e8d6d1daf629fc5f29c9804f1a6" diff --git a/pyproject.toml b/pyproject.toml index 91569dac0..0f36144c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langflow" -version = "0.5.5" +version = "0.5.6" description = "A Python package with a built-in web application" authors = ["Logspace "] maintainers = [ @@ -35,8 +35,8 @@ google-search-results = "^2.4.1" google-api-python-client = "^2.79.0" typer = "^0.9.0" gunicorn = "^21.2.0" -langchain = "~0.0.320" -openai = "^1.0.1" +langchain = "~0.0.327" +openai = "^0.27.8" pandas = "2.0.3" chromadb = "^0.4.0" huggingface-hub = { version = "^0.16.0", extras = ["inference"] } @@ -78,7 +78,7 @@ google-cloud-aiplatform = "^1.36.0" psycopg = "^3.1.9" psycopg-binary = "^3.1.9" fastavro = "^1.8.0" -langchain-experimental = "^0.0.34" +langchain-experimental = "*" celery = { extras = ["redis"], version = "^5.3.1", optional = true } redis = { version = "^4.6.0", optional = true } flower = { version = "^2.0.0", optional = true } @@ -91,10 +91,12 @@ pydantic-settings = "^2.0.3" zep-python = { version = "^1.3.0", allow-prereleases = true } pywin32 = { version = "^306", markers = "sys_platform == 'win32'" } loguru = "^0.7.1" -langfuse = "^1.1.1" +langfuse = "^1.1.11" pillow = "^10.0.0" metal-sdk = "^2.4.0" markupsafe = "^2.1.3" +extract-msg = "^0.45.0" +jq = "^1.6.0" boto3 = "^1.28.63" numexpr = "^2.8.6" qianfan = "0.0.5" diff --git a/src/backend/langflow/components/documentloaders/FileLoader.py b/src/backend/langflow/components/documentloaders/FileLoader.py new file mode 100644 index 000000000..04f43d60b --- /dev/null +++ b/src/backend/langflow/components/documentloaders/FileLoader.py @@ -0,0 +1,232 @@ +from langflow import CustomComponent +from langchain.schema import Document +from typing import Any, Dict, List + +loaders_info: List[Dict[str, Any]] = [ + { + "loader": "AirbyteJSONLoader", + "name": "Airbyte JSON (.jsonl)", + "import": "langchain.document_loaders.AirbyteJSONLoader", + "defaultFor": ["jsonl"], + "allowdTypes": ["jsonl"], + }, + { + "loader": "JSONLoader", + "name": "JSON (.json)", + "import": "langchain.document_loaders.JSONLoader", + "defaultFor": ["json"], + "allowdTypes": ["json"], + }, + { + "loader": "BSHTMLLoader", + "name": "BeautifulSoup4 HTML (.html, .htm)", + "import": "langchain.document_loaders.BSHTMLLoader", + "allowdTypes": ["html", "htm"], + }, + { + "loader": "CSVLoader", + "name": "CSV (.csv)", + "import": "langchain.document_loaders.CSVLoader", + "defaultFor": ["csv"], + "allowdTypes": ["csv"], + }, + { + "loader": "CoNLLULoader", + "name": "CoNLL-U (.conllu)", + "import": "langchain.document_loaders.CoNLLULoader", + "defaultFor": ["conllu"], + "allowdTypes": ["conllu"], + }, + { + "loader": "EverNoteLoader", + "name": "EverNote (.enex)", + "import": "langchain.document_loaders.EverNoteLoader", + "defaultFor": ["enex"], + "allowdTypes": ["enex"], + }, + { + "loader": "FacebookChatLoader", + "name": "Facebook Chat (.json)", + "import": "langchain.document_loaders.FacebookChatLoader", + "allowdTypes": ["json"], + }, + { + "loader": "OutlookMessageLoader", + "name": "Outlook Message (.msg)", + "import": "langchain.document_loaders.OutlookMessageLoader", + "defaultFor": ["msg"], + "allowdTypes": ["msg"], + }, + { + "loader": "PyPDFLoader", + "name": "PyPDF (.pdf)", + "import": "langchain.document_loaders.PyPDFLoader", + "defaultFor": ["pdf"], + "allowdTypes": ["pdf"], + }, + { + "loader": "STRLoader", + "name": "Subtitle (.str)", + "import": "langchain.document_loaders.STRLoader", + "defaultFor": ["str"], + "allowdTypes": ["str"], + }, + { + "loader": "TextLoader", + "name": "Text (.txt)", + "import": "langchain.document_loaders.TextLoader", + "defaultFor": ["txt"], + "allowdTypes": ["txt"], + }, + { + "loader": "UnstructuredEmailLoader", + "name": "Unstructured Email (.eml)", + "import": "langchain.document_loaders.UnstructuredEmailLoader", + "defaultFor": ["eml"], + "allowdTypes": ["eml"], + }, + { + "loader": "UnstructuredHTMLLoader", + "name": "Unstructured HTML (.html, .htm)", + "import": "langchain.document_loaders.UnstructuredHTMLLoader", + "defaultFor": ["html", "htm"], + "allowdTypes": ["html", "htm"], + }, + { + "loader": "UnstructuredMarkdownLoader", + "name": "Unstructured Markdown (.md)", + "import": "langchain.document_loaders.UnstructuredMarkdownLoader", + "defaultFor": ["md"], + "allowdTypes": ["md"], + }, + { + "loader": "UnstructuredPowerPointLoader", + "name": "Unstructured PowerPoint (.pptx)", + "import": "langchain.document_loaders.UnstructuredPowerPointLoader", + "defaultFor": ["pptx"], + "allowdTypes": ["pptx"], + }, + { + "loader": "UnstructuredWordLoader", + "name": "Unstructured Word (.docx)", + "import": "langchain.document_loaders.UnstructuredWordLoader", + "defaultFor": ["docx"], + "allowdTypes": ["docx"], + }, +] + + +class FileLoaderComponent(CustomComponent): + display_name: str = "File Loader" + description: str = "Generic File Loader" + beta = True + + def build_config(self): + loader_options = ["Automatic"] + [ + loader_info["name"] for loader_info in loaders_info + ] + + file_types = [] + suffixes = [] + + for loader_info in loaders_info: + if "allowedTypes" in loader_info: + file_types.extend(loader_info["allowedTypes"]) + suffixes.extend([f".{ext}" for ext in loader_info["allowedTypes"]]) + + return { + "file_path": { + "display_name": "File Path", + "required": True, + "field_type": "file", + "file_types": [ + "json", + "txt", + "csv", + "jsonl", + "html", + "htm", + "conllu", + "enex", + "msg", + "pdf", + "srt", + "eml", + "md", + "pptx", + "docx", + ], + "suffixes": [ + ".json", + ".txt", + ".csv", + ".jsonl", + ".html", + ".htm", + ".conllu", + ".enex", + ".msg", + ".pdf", + ".srt", + ".eml", + ".md", + ".pptx", + ".docx", + ], + # "file_types" : file_types, + # "suffixes": suffixes, + }, + "loader": { + "display_name": "Loader", + "is_list": True, + "required": True, + "options": loader_options, + "value": "Automatic", + }, + "code": {"show": False}, + } + + def build(self, file_path: str, loader: str) -> Document: + file_type = file_path.split(".")[-1] + + # Mapeie o nome do loader selecionado para suas informações + selected_loader_info = None + for loader_info in loaders_info: + if loader_info["name"] == loader: + selected_loader_info = loader_info + break + + if selected_loader_info is None and loader != "Automatic": + raise ValueError(f"Loader {loader} not found in the loader info list") + + if loader == "Automatic": + # Determine o loader automaticamente com base na extensão do arquivo + default_loader_info = None + for info in loaders_info: + if "defaultFor" in info and file_type in info["defaultFor"]: + default_loader_info = info + break + + if default_loader_info is None: + raise ValueError(f"No default loader found for file type: {file_type}") + + selected_loader_info = default_loader_info + if isinstance(selected_loader_info, dict): + loader_import: str = selected_loader_info["import"] + else: + raise ValueError( + f"Loader info for {loader} is not a dict\nLoader info:\n{selected_loader_info}" + ) + module_name, class_name = loader_import.rsplit(".", 1) + + try: + # Importe o loader dinamicamente + loader_module = __import__(module_name, fromlist=[class_name]) + loader_instance = getattr(loader_module, class_name) + except ImportError as e: + raise ValueError( + f"Loader {loader} could not be imported\nLoader info:\n{selected_loader_info}" + ) from e + + result = loader_instance(file_path=file_path) + return result.load() diff --git a/src/backend/langflow/components/documentloaders/UrlLoader.py b/src/backend/langflow/components/documentloaders/UrlLoader.py new file mode 100644 index 000000000..94004e545 --- /dev/null +++ b/src/backend/langflow/components/documentloaders/UrlLoader.py @@ -0,0 +1,62 @@ +from typing import List +from langflow import CustomComponent +from langchain.document_loaders import AZLyricsLoader +from langchain.document_loaders import CollegeConfidentialLoader +from langchain.document_loaders import GitbookLoader +from langchain.document_loaders import HNLoader +from langchain.document_loaders import IFixitLoader +from langchain.document_loaders import IMSDbLoader +from langchain.document_loaders import WebBaseLoader + + +from langchain.schema import Document + + +class UrlLoaderComponent(CustomComponent): + display_name: str = "Url Loader" + description: str = "Generic Url Loader Component" + + def build_config(self): + return { + "web_path": { + "display_name": "Url", + "required": True, + }, + "loader": { + "display_name": "Loader", + "is_list": True, + "required": True, + "options": [ + "AZLyricsLoader", + "CollegeConfidentialLoader", + "GitbookLoader", + "HNLoader", + "IFixitLoader", + "IMSDbLoader", + "WebBaseLoader", + ], + "value": "WebBaseLoader", + }, + "code": {"show": False}, + } + + def build(self, web_path: str, loader: str) -> List[Document]: + if loader == "AZLyricsLoader": + loader_instance = AZLyricsLoader(web_path=web_path) # type: ignore + elif loader == "CollegeConfidentialLoader": + loader_instance = CollegeConfidentialLoader(web_path=web_path) # type: ignore + elif loader == "GitbookLoader": + loader_instance = GitbookLoader(web_page=web_path) # type: ignore + elif loader == "HNLoader": + loader_instance = HNLoader(web_path=web_path) # type: ignore + elif loader == "IFixitLoader": + loader_instance = IFixitLoader(web_path=web_path) # type: ignore + elif loader == "IMSDbLoader": + loader_instance = IMSDbLoader(web_path=web_path) # type: ignore + elif loader == "WebBaseLoader": + loader_instance = WebBaseLoader(web_path=web_path) # type: ignore + + if loader_instance is None: + raise ValueError(f"No loader found for: {web_path}") + + return loader_instance.load() diff --git a/src/backend/langflow/components/documentloaders/__init__.py b/src/backend/langflow/components/documentloaders/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/backend/langflow/graph/vertex/types.py b/src/backend/langflow/graph/vertex/types.py index 7609982a5..1b63bc062 100644 --- a/src/backend/langflow/graph/vertex/types.py +++ b/src/backend/langflow/graph/vertex/types.py @@ -206,6 +206,10 @@ class ChainVertex(Vertex): ) -> Any: if not self._built or force: # Check if the chain requires a PromptVertex + + # Temporarily remove "code" from the params + self.params.pop("code", None) + for key, value in self.params.items(): if isinstance(value, PromptVertex): # Build the PromptVertex, passing the tools if available diff --git a/src/backend/langflow/template/frontend_node/chains.py b/src/backend/langflow/template/frontend_node/chains.py index 1b1142ba6..18cedbe1d 100644 --- a/src/backend/langflow/template/frontend_node/chains.py +++ b/src/backend/langflow/template/frontend_node/chains.py @@ -87,6 +87,8 @@ class ChainFrontendNode(FrontendNode): field.required = True field.show = True field.advanced = False + field.field_type = "BaseLanguageModel" # temporary fix + field.is_list = False if field.name == "return_source_documents": field.required = False diff --git a/src/backend/langflow/utils/constants.py b/src/backend/langflow/utils/constants.py index 0c97b56a2..f3671425c 100644 --- a/src/backend/langflow/utils/constants.py +++ b/src/backend/langflow/utils/constants.py @@ -6,14 +6,11 @@ OPENAI_MODELS = [ "text-ada-001", ] CHAT_OPENAI_MODELS = [ - "gpt-3.5-turbo-0613", - "gpt-3.5-turbo", - "gpt-3.5-turbo-16k-0613", - "gpt-3.5-turbo-16k", - "gpt-4-0613", - "gpt-4-32k-0613", + "gpt-4-1106-preview", "gpt-4", "gpt-4-32k", + "gpt-3.5-turbo", + "gpt-3.5-turbo-16k", ] ANTHROPIC_MODELS = [ diff --git a/src/backend/langflow/utils/util.py b/src/backend/langflow/utils/util.py index 1c7a7f094..1778e42cb 100644 --- a/src/backend/langflow/utils/util.py +++ b/src/backend/langflow/utils/util.py @@ -473,3 +473,11 @@ def build_loader_repr_from_documents(documents: List[Document]) -> str: \nAvg. Document Length (characters): {int(avg_length)} Documents: {documents[:3]}...""" return "0 documents" + + +def get_type_from_union_literal(union_literal: str) -> str: + # if types are literal strings + # the type is a string + if "Literal" in union_literal: + return "str" + return union_literal diff --git a/src/frontend/src/pages/FlowPage/components/DisclosureComponent/index.tsx b/src/frontend/src/pages/FlowPage/components/DisclosureComponent/index.tsx index 3a47580bb..d2ff0eee8 100644 --- a/src/frontend/src/pages/FlowPage/components/DisclosureComponent/index.tsx +++ b/src/frontend/src/pages/FlowPage/components/DisclosureComponent/index.tsx @@ -8,7 +8,7 @@ export default function DisclosureComponent({ openDisc, }: DisclosureComponentType): JSX.Element { return ( - + {({ open }) => ( <>
@@ -35,9 +35,7 @@ export default function DisclosureComponent({
- - {children} - + {children} )} diff --git a/src/frontend/src/pages/FlowPage/components/extraSidebarComponent/index.tsx b/src/frontend/src/pages/FlowPage/components/extraSidebarComponent/index.tsx index 664ec80d3..37bd43b58 100644 --- a/src/frontend/src/pages/FlowPage/components/extraSidebarComponent/index.tsx +++ b/src/frontend/src/pages/FlowPage/components/extraSidebarComponent/index.tsx @@ -122,7 +122,7 @@ export default function ExtraSidebar(): JSX.Element { } } }); - setSearch("search"); + setSearch(""); return ret; }); } @@ -227,8 +227,12 @@ export default function ExtraSidebar(): JSX.Element { .map((SBSectionName: keyof APIObjectType, index) => Object.keys(dataFilter[SBSectionName]).length > 0 ? (