jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Added curl parse to API Request component and fixed dict issues (#2013)

Browse source 
* Refactor code to remove console.log statements

* Refactor code to remove console.log statements

* ⬆️ (pyproject.toml): upgrade uncurl dependency to version 0.0.11

* 📝 (flows.py): Add docstring to the read_flows function to provide information about its purpose, arguments, and return value
📝 (parse.py): Add comments and docstrings to the parse_context function to explain its purpose and how it works
📝 (APIRequest.py): Add a new method update_build_config to handle parsing of curl commands and update build configuration based on the parsed context

* refactor: Improve value change detection logic in DictComponent

* refactor: Improve value change detection logic in DictAreaModal

* refactor: Update APIRequest to handle parsing of curl commands and update build configuration

This commit updates the APIRequest class in APIRequest.py to handle parsing of curl commands and update the build configuration based on the parsed context. It introduces a new method, update_build_config, which parses the curl command using the parse_context function and updates the build configuration with the parsed information. Additionally, it handles JSON decoding errors when parsing the data field of the curl command. This improvement enhances the functionality and flexibility of the APIRequest component.

* feat: Add support for handling headers as dictionaries in APIRequest

* refactor: Parse curl commands and update build configuration in APIRequest

This commit refactors the APIRequest class in APIRequest.py to handle parsing of curl commands and update the build configuration based on the parsed context. It introduces a new method, update_build_config, which parses the curl command using the parse_context function and updates the build configuration with the parsed information. Additionally, it handles JSON decoding errors when parsing the data field of the curl command. This improvement enhances the functionality and flexibility of the APIRequest component.

* ✨ (test_data_components.py): add new test case to parse curl command into build configuration for API requests

* 🐛 (src/backend/base/langflow/components/data/APIRequest.py): fix type hinting issue for bodies variable in APIRequest class
This commit is contained in:
Gabriel Luiz Freitas Almeida 2024-05-30 06:50:31 -07:00 • committed by GitHub
commit e5986ec727
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 248 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

28
poetry.lock generated
View file

@ -4308,13 +4308,13 @@ extended-testing = ["beautifulsoup4 (>=4.12.3,<5.0.0)", "lxml (>=4.9.3,<6.0)"]
[[package]]
name = "langchainhub"
version = "0.1.16"
version = "0.1.17"
description = "The LangChain Hub API client"
optional = false
python-versions = "<4.0,>=3.8.1"
files = [
{file = "langchainhub-0.1.16-py3-none-any.whl", hash = "sha256:a4379a1879cc6b441b8d02cc65e28a54f160fba61c9d1d4b0eddc3a276dff99a"},
{file = "langchainhub-0.1.16.tar.gz", hash = "sha256:9f11e68fddb575e70ef4b28800eedbd9eeb180ba508def04f7153ea5b246b6fc"},
{file = "langchainhub-0.1.17-py3-none-any.whl", hash = "sha256:4c609b3948252c71670f0d98f73413b515cfd2f6701a7b40ce959203e6133e04"},
{file = "langchainhub-0.1.17.tar.gz", hash = "sha256:af7df0cb1cebc7a6e0864e8632ae48ecad39ed96568f699c78657b9d04e50b46"},
]
[package.dependencies]
@ -4365,6 +4365,7 @@ python-socketio = "^5.11.0"
rich = "^13.7.0"
sqlmodel = "^0.0.18"
typer = "^0.12.0"
uncurl = "^0.0.11"
uvicorn = "^0.29.0"
websockets = "*"
@ -7676,13 +7677,13 @@ files = [
[[package]]
name = "requests"
version = "2.32.2"
version = "2.32.3"
description = "Python HTTP for Humans."
optional = false
python-versions = ">=3.8"
files = [
{file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"},
{file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"},
{file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"},
{file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"},
]
[package.dependencies]
@ -9158,6 +9159,21 @@ files = [
{file = "ujson-5.10.0.tar.gz", hash = "sha256:b3cd8f3c5d8c7738257f1018880444f7b7d9b66232c64649f562d7ba86ad4bc1"},
]
[[package]]
name = "uncurl"
version = "0.0.11"
description = "A library to convert curl requests to python-requests."
optional = false
python-versions = "*"
files = [
{file = "uncurl-0.0.11-py3-none-any.whl", hash = "sha256:5961e93f07a5c9f2ef8ae4245bd92b0a6ce503c851de980f5b70080ae74cdc59"},
{file = "uncurl-0.0.11.tar.gz", hash = "sha256:530c9bbd4d118f4cde6194165ff484cc25b0661cd256f19e9d5fcb53fc077790"},
]
[package.dependencies]
pyperclip = "*"
six = "*"
[[package]]
name = "uritemplate"
version = "4.1.1"

33
src/backend/base/langflow/api/v1/flows.py
View file

@ -57,8 +57,22 @@ def read_flows(
current_user: User = Depends(get_current_active_user),
session: Session = Depends(get_session),
settings_service: "SettingsService" = Depends(get_settings_service),
remove_example_flows: bool = False,
):
"""Read all flows."""
"""
Retrieve a list of flows.
Args:
current_user (User): The current authenticated user.
session (Session): The database session.
settings_service (SettingsService): The settings service.
remove_example_flows (bool, optional): Whether to remove example flows. Defaults to False.
Returns:
List[Dict]: A list of flows in JSON format.
"""
try:
auth_settings = settings_service.auth_settings
if auth_settings.AUTO_LOGIN:
@ -73,15 +87,16 @@ def read_flows(
flows = validate_is_component(flows) # type: ignore
flow_ids = [flow.id for flow in flows]
# with the session get the flows that DO NOT have a user_id
try:
folder = session.exec(select(Folder).where(Folder.name == STARTER_FOLDER_NAME)).first()
if not remove_example_flows:
try:
folder = session.exec(select(Folder).where(Folder.name == STARTER_FOLDER_NAME)).first()
example_flows = folder.flows if folder else []
for example_flow in example_flows:
if example_flow.id not in flow_ids:
flows.append(example_flow) # type: ignore
except Exception as e:
logger.error(e)
example_flows = folder.flows if folder else []
for example_flow in example_flows:
if example_flow.id not in flow_ids:
flows.append(example_flow) # type: ignore
except Exception as e:
logger.error(e)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e)) from e
return [jsonable_encoder(flow) for flow in flows]

0
src/backend/base/langflow/base/curl/__init__.py Normal file
View file

89
src/backend/base/langflow/base/curl/parse.py Normal file
View file

@ -0,0 +1,89 @@
"""
This file contains a fix for the implementation of the `uncurl` library, which is available at https://github.com/spulec/uncurl.git.
The `uncurl` library provides a way to parse and convert cURL commands into Python requests. However, there are some issues with the original implementation that this file aims to fix.
The `parse_context` function in this file takes a cURL command as input and returns a `ParsedContext` object, which contains the parsed information from the cURL command, such as the HTTP method, URL, headers, cookies, etc.
The `normalize_newlines` function is a helper function that replaces the line continuation character ("\") followed by a newline with a space.
"""
import re
import shlex
from collections import OrderedDict, namedtuple
from http.cookies import SimpleCookie
from uncurl.api import parser # type: ignore
parser.add_argument("-x", "--proxy", default={})
parser.add_argument("-U", "--proxy-user", default="")
ParsedContext = namedtuple("ParsedContext", ["method", "url", "data", "headers", "cookies", "verify", "auth", "proxy"])
def normalize_newlines(multiline_text):
return multiline_text.replace(" \\\n", " ")
def parse_context(curl_command):
method = "get"
tokens = shlex.split(normalize_newlines(curl_command))
tokens = [token for token in tokens if token and token != " "]
parsed_args = parser.parse_args(tokens)
post_data = parsed_args.data or parsed_args.data_binary
if post_data:
method = "post"
if parsed_args.X:
method = parsed_args.X.lower()
cookie_dict = OrderedDict()
quoted_headers = OrderedDict()
for curl_header in parsed_args.header:
if curl_header.startswith(":"):
occurrence = [m.start() for m in re.finditer(":", curl_header)]
header_key, header_value = curl_header[: occurrence[1]], curl_header[occurrence[1] + 1 :]
else:
header_key, header_value = curl_header.split(":", 1)
if header_key.lower().strip("$") == "cookie":
cookie = SimpleCookie(bytes(header_value, "ascii").decode("unicode-escape"))
for key in cookie:
cookie_dict[key] = cookie[key].value
else:
quoted_headers[header_key] = header_value.strip()
# add auth
user = parsed_args.user
if parsed_args.user:
user = tuple(user.split(":"))
# add proxy and its authentication if it's available.
proxies = parsed_args.proxy
# proxy_auth = parsed_args.proxy_user
if parsed_args.proxy and parsed_args.proxy_user:
proxies = {
"http": "http://{}@{}/".format(parsed_args.proxy_user, parsed_args.proxy),
"https": "http://{}@{}/".format(parsed_args.proxy_user, parsed_args.proxy),
}
elif parsed_args.proxy:
proxies = {
"http": "http://{}/".format(parsed_args.proxy),
"https": "http://{}/".format(parsed_args.proxy),
}
return ParsedContext(
method=method,
url=parsed_args.url,
data=post_data,
headers=quoted_headers,
cookies=cookie_dict,
verify=parsed_args.insecure,
auth=user,
proxy=proxies,
)

52
src/backend/base/langflow/components/data/APIRequest.py
View file

@ -1,11 +1,15 @@
import asyncio
import json
from typing import List, Optional
from typing import Any, List, Optional
import httpx
from loguru import logger
from langflow.base.curl.parse import parse_context
from langflow.custom import CustomComponent
from langflow.field_typing import NestedDict
from langflow.schema import Record
from langflow.schema.dotdict import dotdict
class APIRequest(CustomComponent):
@ -17,10 +21,15 @@ class APIRequest(CustomComponent):
field_config = {
"urls": {"display_name": "URLs", "info": "URLs to make requests to."},
"curl": {
"display_name": "Curl",
"info": "Paste a curl command to populate the fields.",
"refresh_button": True,
"refresh_button_text": "",
},
"method": {
"display_name": "Method",
"info": "The HTTP method to use.",
"field_type": "str",
"options": ["GET", "POST", "PATCH", "PUT"],
"value": "GET",
},
@ -36,12 +45,33 @@ class APIRequest(CustomComponent):
},
"timeout": {
"display_name": "Timeout",
"field_type": "int",
"info": "The timeout to use for the request.",
"value": 5,
},
}
def parse_curl(self, curl: str, build_config: dotdict) -> dotdict:
try:
parsed = parse_context(curl)
build_config["urls"]["value"] = [parsed.url]
build_config["method"]["value"] = parsed.method.upper()
build_config["headers"]["value"] = dict(parsed.headers)
try:
json_data = json.loads(parsed.data)
build_config["body"]["value"] = json_data
except json.JSONDecodeError as e:
print(e)
except Exception as exc:
logger.error(f"Error parsing curl: {exc}")
raise ValueError(f"Error parsing curl: {exc}")
return build_config
def update_build_config(self, build_config: dotdict, field_value: Any, field_name: str | None = None):
if field_name == "curl" and field_value is not None:
build_config = self.parse_curl(field_value, build_config)
return build_config
async def make_request(
self,
client: httpx.AsyncClient,
@ -94,21 +124,25 @@ class APIRequest(CustomComponent):
self,
method: str,
urls: List[str],
headers: Optional[Record] = None,
body: Optional[Record] = None,
curl: Optional[str] = None,
headers: Optional[NestedDict] = {},
body: Optional[NestedDict] = {},
timeout: int = 5,
) -> List[Record]:
if headers is None:
headers_dict = {}
else:
elif isinstance(headers, Record):
headers_dict = headers.data
else:
headers_dict = headers
bodies = []
if body:
if isinstance(body, list):
bodies = [b.data for b in body]
if not isinstance(body, list):
bodies = [body]
else:
bodies = [body.data]
bodies = body
bodies = [b.data if isinstance(b, Record) else b for b in bodies] # type: ignore
if len(urls) != len(bodies):
# add bodies with None

17
src/backend/base/poetry.lock generated
View file

@ -2856,6 +2856,21 @@ files = [
{file = "ujson-5.10.0.tar.gz", hash = "sha256:b3cd8f3c5d8c7738257f1018880444f7b7d9b66232c64649f562d7ba86ad4bc1"},
]
[[package]]
name = "uncurl"
version = "0.0.11"
description = "A library to convert curl requests to python-requests."
optional = false
python-versions = "*"
files = [
{file = "uncurl-0.0.11-py3-none-any.whl", hash = "sha256:5961e93f07a5c9f2ef8ae4245bd92b0a6ce503c851de980f5b70080ae74cdc59"},
{file = "uncurl-0.0.11.tar.gz", hash = "sha256:530c9bbd4d118f4cde6194165ff484cc25b0661cd256f19e9d5fcb53fc077790"},
]
[package.dependencies]
pyperclip = "*"
six = "*"
[[package]]
name = "urllib3"
version = "2.2.1"
@ -3250,4 +3265,4 @@ local = []
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<3.13"
content-hash = "31d8e5ce045ef7d94e63058559b5f8181e6b51fc923c4904f45481443d59235d"
content-hash = "1dc0dd442df5d174ea85c859208f5cfea9d4785c7b7a93f2c6bf8c92e93d8cad"

1
src/backend/base/pyproject.toml
View file

@ -62,6 +62,7 @@ emoji = "^2.12.0"
cryptography = "^42.0.5"
asyncer = "^0.0.5"
pyperclip = "^1.8.2"
uncurl = "^0.0.11"
[tool.poetry.extras]

1
src/frontend/src/App.tsx
View file

@ -120,7 +120,6 @@ export default function App() {
await getFoldersApi();
await getTypes();
await refreshFlows();
console.log(axios.defaults);
const res = await getGlobalVariables();
setGlobalVariables(res);
checkHasStore();

11
src/frontend/src/components/dictComponent/index.tsx
View file

@ -12,19 +12,24 @@ export default function DictComponent({
editNode = false,
id = "",
}: DictComponentType): JSX.Element {
// Create a reference to the value
const ref = useRef(value);
useEffect(() => {
if (disabled) {
onChange({});
}
}, [disabled]);
const ref = useRef(value);
useEffect(() => {
// Update the reference value
ref.current = value;
}, [value]);
return (
<div
className={classNames(
value.length > 1 && editNode ? "my-1" : "",
"flex flex-col gap-3",
"flex flex-col gap-3"
)}
>
{

2
src/frontend/src/components/inputFileComponent/index.tsx
View file

@ -66,10 +66,8 @@ export default function InputFileComponent({
uploadFile(file, currentFlowId)
.then((res) => res.data)
.then((data) => {
console.log(CONSOLE_SUCCESS_MSG);
// Get the file name from the response
const { file_path } = data;
console.log("File name:", file_path);
// sets the value that goes to the backend
onFileChange(file_path);

2
src/frontend/src/components/sidebarComponent/components/sideBarFolderButtons/index.tsx
View file

@ -91,7 +91,7 @@ const SideBarFoldersButtonsComponent = ({
folders.map((obj) => ({ name: obj.name, edit: false }));
}, [folders]);
console.log(folderId, folderIdDragging);
return (
<>

2
src/frontend/src/customNodes/genericNode/index.tsx
View file

@ -335,12 +335,10 @@ export default function GenericNode({
"generic-node-div",
specificClassFromBuildStatus
);
console.log("names", names);
return names;
};
const getBaseBorderClass = (selected) => {
console.log("data.node?.frozen", data.node?.frozen);
let className = selected ? "border border-ring" : "border";
let frozenClass = selected ? "border-ring-frozen" : "border-frozen";
return data.node?.frozen ? frozenClass : className;

1
src/frontend/src/modals/IOModal/components/IOFieldView/components/FileInput/index.tsx
View file

@ -84,7 +84,6 @@ export default function IOFileInput({ field, updateValue }: IOFileInputProps) {
uploadFile(file, currentFlowId)
.then((res) => res.data)
.then((data) => {
console.log("File uploaded successfully");
// Get the file name from the response
const { file_path, flowId } = data;
setFilePath(file_path);

2
src/frontend/src/modals/dictAreaModal/index.tsx
View file

@ -29,7 +29,7 @@ export default function DictAreaModal({
useEffect(() => {
if (value) ref.current = value;
}, [ref]);
}, [value]);
return (
<BaseModal size="medium-h-full" open={open} setOpen={setOpen}>

32
src/frontend/src/modals/editNodeModal/index.tsx
View file

@ -53,7 +53,7 @@ const EditNodeModal = forwardRef(
setOpen: (open: boolean) => void;
data: NodeDataType;
},
ref,
ref
) => {
const nodes = useFlowStore((state) => state.nodes);
@ -125,7 +125,7 @@ const EditNodeModal = forwardRef(
"edit-node-modal-box",
nodeLength > limitScrollFieldsModal
? "overflow-scroll overflow-x-hidden custom-scroll"
: "",
: ""
)}
>
{nodeLength > 0 && (
@ -147,8 +147,8 @@ const EditNodeModal = forwardRef(
templateParam.charAt(0) !== "_" &&
myData.node?.template[templateParam].show &&
LANGFLOW_SUPPORTED_TYPES.has(
myData.node!.template[templateParam].type,
),
myData.node!.template[templateParam].type
)
)
.map((templateParam, index) => {
let id = {
@ -170,8 +170,8 @@ const EditNodeModal = forwardRef(
myData.node?.template[templateParam]
.proxy,
}
: id,
),
: id
)
) ?? false;
return (
<TableRow
@ -233,7 +233,7 @@ const EditNodeModal = forwardRef(
onChange={(value: string[]) => {
handleOnNewValue(
value,
templateParam,
templateParam
);
}}
/>
@ -257,11 +257,11 @@ const EditNodeModal = forwardRef(
.value ?? ""
}
onChange={(
value: string | string[],
value: string | string[]
) => {
handleOnNewValue(
value,
templateParam,
templateParam
);
}}
/>
@ -311,7 +311,7 @@ const EditNodeModal = forwardRef(
].value = newValue;
handleOnNewValue(
newValue,
templateParam,
templateParam
);
}}
id="editnode-div-dict-input"
@ -328,7 +328,7 @@ const EditNodeModal = forwardRef(
myData.node!.template[templateParam].value
?.length > 1
? "my-3"
: "",
: ""
)}
>
<KeypairListComponent
@ -344,7 +344,7 @@ const EditNodeModal = forwardRef(
myData.node!.template[
templateParam
].value,
type(templateParam)!,
type(templateParam)!
)
}
duplicateKey={errorDuplicateKey}
@ -355,11 +355,11 @@ const EditNodeModal = forwardRef(
templateParam
].value = valueToNumbers;
setErrorDuplicateKey(
hasDuplicateKeys(valueToNumbers),
hasDuplicateKeys(valueToNumbers)
);
handleOnNewValue(
valueToNumbers,
templateParam,
templateParam
);
}}
isList={
@ -389,7 +389,7 @@ const EditNodeModal = forwardRef(
setEnabled={(isEnabled) => {
handleOnNewValue(
isEnabled,
templateParam,
templateParam
);
}}
size="small"
@ -632,7 +632,7 @@ const EditNodeModal = forwardRef(
</BaseModal.Footer>
</BaseModal>
);
},
}
);
export default EditNodeModal;

7
src/frontend/src/modals/genericModal/index.tsx
View file

@ -83,7 +83,7 @@ export default function GenericModal({
}
const filteredWordsHighlight = matches.filter(
(word) => !invalid_chars.includes(word),
(word) => !invalid_chars.includes(word)
);
setWordsHighlight(filteredWordsHighlight);
@ -134,7 +134,7 @@ export default function GenericModal({
// to the first key of the custom_fields object
if (field_name === "") {
field_name = Array.isArray(
apiReturn.data?.frontend_node?.custom_fields?.[""],
apiReturn.data?.frontend_node?.custom_fields?.[""]
)
? apiReturn.data?.frontend_node?.custom_fields?.[""][0] ?? ""
: apiReturn.data?.frontend_node?.custom_fields?.[""] ?? "";
@ -166,7 +166,6 @@ export default function GenericModal({
}
})
.catch((error) => {
console.log(error);
setIsEdit(true);
return setErrorData({
title: PROMPT_ERROR_ALERT,
@ -210,7 +209,7 @@ export default function GenericModal({
<div
className={classNames(
!isEdit ? "rounded-lg border" : "",
"flex h-full w-full",
"flex h-full w-full"
)}
>
{type === TypeModal.PROMPT && isEdit && !readonly ? (

3
src/frontend/src/pages/Playground/index.tsx
View file

@ -11,7 +11,7 @@ export default function PlaygroundPage() {
const currentFlow = useFlowsManagerStore((state) => state.currentFlow);
const getFlowById = useFlowsManagerStore((state) => state.getFlowById);
const setCurrentFlowId = useFlowsManagerStore(
(state) => state.setCurrentFlowId,
(state) => state.setCurrentFlowId
);
const currentFlowId = useFlowsManagerStore((state) => state.currentFlowId);
const setCurrentFlow = useFlowsManagerStore((state) => state.setCurrentFlow);
@ -29,7 +29,6 @@ export default function PlaygroundPage() {
// Set flow tab id
useEffect(() => {
console.log("id", id);
if (getFlowById(id!)) {
setCurrentFlowId(id!);
} else {

1
src/frontend/src/stores/flowStore.ts
View file

@ -464,7 +464,6 @@ const useFlowStore = create<FlowStoreType>((set, get) => ({
status: BuildStatus,
runId: string,
) {
console.log("handleBuildUpdate", vertexBuildData, status, runId);
if (vertexBuildData && vertexBuildData.inactivated_vertices) {
get().removeFromVerticesBuild(vertexBuildData.inactivated_vertices);
get().updateBuildStatus(

1
src/frontend/src/stores/flowsManagerStore.ts
View file

@ -233,7 +233,6 @@ const useFlowsManagerStore = create<FlowsManagerStoreType>((set, get) => ({
// addFlowToLocalState(newFlow);
return;
}
console.log("folder id", folder_id);
const newFlow = createNewFlow(
flowData!,
flow!,

26
tests/test_data_components.py
View file

@ -6,9 +6,8 @@ import httpx
import pytest
import respx
from httpx import Response
from langflow.components import (
data,
)
from langflow.components import data
@pytest.fixture
@ -34,6 +33,27 @@ async def test_successful_get_request(api_request):
assert result.data["result"] == mock_response
def test_parse_curl(api_request):
# Arrange
field_value = (
"curl -X GET https://example.com/api/test -H 'Content-Type: application/json' -d '{\"key\": \"value\"}'"
)
build_config = {
"method": {"value": ""},
"urls": {"value": []},
"headers": {},
"body": {},
}
# Act
new_build_config = api_request.parse_curl(field_value, build_config.copy())
# Assert
assert new_build_config["method"]["value"] == "GET"
assert new_build_config["urls"]["value"] == ["https://example.com/api/test"]
assert new_build_config["headers"]["value"] == {"Content-Type": "application/json"}
assert new_build_config["body"]["value"] == {"key": "value"}
@pytest.mark.asyncio
@respx.mock
async def test_failed_request(api_request):