This commit is contained in:
Gabriel Luiz Freitas Almeida 2024-03-08 17:42:22 -03:00
commit 522bd304e8
22 changed files with 151 additions and 485 deletions

View file

@ -211,7 +211,5 @@ async def download_file(
current_user: User = Depends(get_current_active_user),
):
"""Download all flows as a file."""
flows = read_flows(
current_user=current_user, session=session, settings_service=settings_service
)
flows = read_flows(current_user=current_user, session=session, settings_service=settings_service)
return FlowListRead(flows=flows)

View file

@ -40,7 +40,7 @@ async def login_to_get_access_token(
httponly=auth_settings.REFRESH_HTTPONLY,
samesite=auth_settings.REFRESH_SAME_SITE,
secure=auth_settings.REFRESH_SECURE,
expires=auth_settings.REFRESH_TOKEN_EXPIRE_MINUTES*60,
expires=auth_settings.REFRESH_TOKEN_EXPIRE_MINUTES * 60,
)
response.set_cookie(
"access_token_lf",
@ -48,7 +48,7 @@ async def login_to_get_access_token(
httponly=auth_settings.ACCESS_HTTPONLY,
samesite=auth_settings.ACCESS_SAME_SITE,
secure=auth_settings.ACCESS_SECURE,
expires=auth_settings.ACCESS_TOKEN_EXPIRE_MINUTES*60,
expires=auth_settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60,
)
return tokens
else:
@ -74,7 +74,7 @@ async def auto_login(
httponly=auth_settings.ACCESS_HTTPONLY,
samesite=auth_settings.ACCESS_SAME_SITE,
secure=auth_settings.ACCESS_SECURE,
expires=auth_settings.ACCESS_TOKEN_EXPIRE_MINUTES*60,
expires=auth_settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60,
)
return tokens
@ -101,7 +101,7 @@ async def refresh_token(request: Request, response: Response, settings_service=D
httponly=auth_settings.REFRESH_TOKEN_HTTPONLY,
samesite=auth_settings.REFRESH_SAME_SITE,
secure=auth_settings.REFRESH_SECURE,
expires=auth_settings.REFRESH_TOKEN_EXPIRE_MINUTES*60,
expires=auth_settings.REFRESH_TOKEN_EXPIRE_MINUTES * 60,
)
response.set_cookie(
"access_token_lf",
@ -109,7 +109,7 @@ async def refresh_token(request: Request, response: Response, settings_service=D
httponly=auth_settings.ACCESS_HTTPONLY,
samesite=auth_settings.ACCESS_SAME_SITE,
secure=auth_settings.ACCESS_SECURE,
expires=auth_settings.ACCESS_TOKEN_EXPIRE_MINUTES*60,
expires=auth_settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60,
)
return tokens
else:

View file

@ -54,9 +54,7 @@ def retrieve_file_paths(
glob = "**/*" if recursive else "*"
paths = walk_level(path_obj, depth) if depth else path_obj.glob(glob)
file_paths = [
Text(p) for p in paths if p.is_file() and match_types(p) and is_not_hidden(p)
]
file_paths = [Text(p) for p in paths if p.is_file() and match_types(p) and is_not_hidden(p)]
return file_paths
@ -90,13 +88,10 @@ def parse_text_file_to_record(file_path: str, silent_errors: bool) -> Optional[R
text = read_text_file(file_path)
# if file is json, yaml, or xml, we can parse it
if file_path.endswith(".json"):
text = json.loads(text)
elif file_path.endswith(".yaml") or file_path.endswith(".yml"):
text = yaml.safe_load(text)
elif file_path.endswith(".xml"):
text = ET.fromstring(text)
except Exception as e:
if not silent_errors:
@ -116,10 +111,7 @@ def get_elements(
if use_multithreading:
records = parallel_load_records(file_paths, silent_errors, max_concurrency)
else:
records = [
partition_file_to_record(file_path, silent_errors)
for file_path in file_paths
]
records = [partition_file_to_record(file_path, silent_errors) for file_path in file_paths]
records = list(filter(None, records))
return records

View file

@ -56,9 +56,7 @@ class APIRequest(CustomComponent):
data = body if body else None
payload = json.dumps(data)
try:
response = await client.request(
method, url, headers=headers, content=payload, timeout=timeout
)
response = await client.request(method, url, headers=headers, content=payload, timeout=timeout)
try:
result = response.json()
except Exception:
@ -111,10 +109,7 @@ class APIRequest(CustomComponent):
bodies += [None] * (len(urls) - len(bodies))
async with httpx.AsyncClient() as client:
results = await asyncio.gather(
*[
self.make_request(client, method, u, headers, rec, timeout)
for u, rec in zip(urls, bodies)
]
*[self.make_request(client, method, u, headers, rec, timeout) for u, rec in zip(urls, bodies)]
)
self.status = results
return results

View file

@ -53,20 +53,14 @@ class DirectoryComponent(CustomComponent):
silent_errors: bool = False,
use_multithreading: bool = True,
) -> List[Optional[Record]]:
resolved_path = self.resolve_path(path)
file_paths = retrieve_file_paths(resolved_path, load_hidden, recursive, depth)
loaded_records = []
if use_multithreading:
loaded_records = parallel_load_records(
file_paths, silent_errors, max_concurrency
)
loaded_records = parallel_load_records(file_paths, silent_errors, max_concurrency)
else:
loaded_records = [
parse_text_file_to_record(file_path, silent_errors)
for file_path in file_paths
]
loaded_records = [parse_text_file_to_record(file_path, silent_errors) for file_path in file_paths]
loaded_records = list(filter(None, loaded_records))
self.status = loaded_records
return loaded_records

View file

@ -21,9 +21,7 @@ class ExtractKeyFromRecordComponent(CustomComponent):
},
}
def build(
self, record: Record, keys: list[str], silent_error: bool = True
) -> Record:
def build(self, record: Record, keys: list[str], silent_error: bool = True) -> Record:
"""
Extracts the keys from a record.

View file

@ -24,9 +24,7 @@ class PromptComponent(CustomComponent):
prompt_template = PromptTemplate.from_template(Text(template))
kwargs = dict_values_to_string(kwargs)
kwargs = {
k: "\n".join(v) if isinstance(v, list) else v for k, v in kwargs.items()
}
kwargs = {k: "\n".join(v) if isinstance(v, list) else v for k, v in kwargs.items()}
try:
formated_prompt = prompt_template.format(**kwargs)
except Exception as exc:

View file

@ -60,13 +60,8 @@ class Vertex:
self.updated_raw_params = False
self.id: str = data["id"]
self.is_state = False
self.is_input = any(
input_component_name in self.id for input_component_name in INPUT_COMPONENTS
)
self.is_output = any(
output_component_name in self.id
for output_component_name in OUTPUT_COMPONENTS
)
self.is_input = any(input_component_name in self.id for input_component_name in INPUT_COMPONENTS)
self.is_output = any(output_component_name in self.id for output_component_name in OUTPUT_COMPONENTS)
self.has_session_id = None
self._custom_component = None
self.has_external_input = False
@ -106,17 +101,11 @@ class Vertex:
def set_state(self, state: str):
self.state = VertexStates[state]
if (
self.state == VertexStates.INACTIVE
and self.graph.in_degree_map[self.id] < 2
):
if self.state == VertexStates.INACTIVE and self.graph.in_degree_map[self.id] < 2:
# If the vertex is inactive and has only one in degree
# it means that it is not a merge point in the graph
self.graph.inactivated_vertices.add(self.id)
elif (
self.state == VertexStates.ACTIVE
and self.id in self.graph.inactivated_vertices
):
elif self.state == VertexStates.ACTIVE and self.id in self.graph.inactivated_vertices:
self.graph.inactivated_vertices.remove(self.id)
@property
@ -133,9 +122,7 @@ class Vertex:
# If the Vertex.type is a power component
# then we need to return the built object
# instead of the result dict
if self.is_interface_component and not isinstance(
self._built_object, UnbuiltObject
):
if self.is_interface_component and not isinstance(self._built_object, UnbuiltObject):
result = self._built_object
# if it is not a dict or a string and hasattr model_dump then
# return the model_dump
@ -147,11 +134,7 @@ class Vertex:
if isinstance(self._built_result, UnbuiltResult):
return {}
return (
self._built_result
if isinstance(self._built_result, dict)
else {"result": self._built_result}
)
return self._built_result if isinstance(self._built_result, dict) else {"result": self._built_result}
def set_artifacts(self) -> None:
pass
@ -225,31 +208,19 @@ class Vertex:
self.selected_output_type = self.data["node"].get("selected_output_type")
self.is_input = self.data["node"].get("is_input") or self.is_input
self.is_output = self.data["node"].get("is_output") or self.is_output
template_dicts = {
key: value
for key, value in self.data["node"]["template"].items()
if isinstance(value, dict)
}
template_dicts = {key: value for key, value in self.data["node"]["template"].items() if isinstance(value, dict)}
self.has_session_id = "session_id" in template_dicts
self.required_inputs = [
template_dicts[key]["type"]
for key, value in template_dicts.items()
if value["required"]
template_dicts[key]["type"] for key, value in template_dicts.items() if value["required"]
]
self.optional_inputs = [
template_dicts[key]["type"]
for key, value in template_dicts.items()
if not value["required"]
template_dicts[key]["type"] for key, value in template_dicts.items() if not value["required"]
]
# Add the template_dicts[key]["input_types"] to the optional_inputs
self.optional_inputs.extend(
[
input_type
for value in template_dicts.values()
for input_type in value.get("input_types", [])
]
[input_type for value in template_dicts.values() for input_type in value.get("input_types", [])]
)
template_dict = self.data["node"]["template"]
@ -296,11 +267,7 @@ class Vertex:
self.updated_raw_params = False
return
template_dict = {
key: value
for key, value in self.data["node"]["template"].items()
if isinstance(value, dict)
}
template_dict = {key: value for key, value in self.data["node"]["template"].items() if isinstance(value, dict)}
params = {}
for edge in self.edges:
@ -321,10 +288,7 @@ class Vertex:
# we don't know the key of the dict but we need to set the value
# to the vertex that is the source of the edge
param_dict = template_dict[param_key]["value"]
params[param_key] = {
key: self.graph.get_vertex(edge.source_id)
for key in param_dict.keys()
}
params[param_key] = {key: self.graph.get_vertex(edge.source_id) for key in param_dict.keys()}
else:
params[param_key] = self.graph.get_vertex(edge.source_id)
@ -360,11 +324,7 @@ class Vertex:
# list of dicts, so we need to convert it to a dict
# before passing it to the build method
if isinstance(val, list):
params[key] = {
k: v
for item in value.get("value", [])
for k, v in item.items()
}
params[key] = {k: v for item in value.get("value", []) for k, v in item.items()}
elif isinstance(val, dict):
params[key] = val
elif value.get("type") == "int" and val is not None:
@ -489,9 +449,7 @@ class Vertex:
if isinstance(self._built_object, str):
self._built_result = self._built_object
result = await generate_result(
self._built_object, inputs, self.has_external_output, session_id
)
result = await generate_result(self._built_object, inputs, self.has_external_output, session_id)
self._built_result = result
async def _build_each_node_in_params_dict(self, user_id=None):
@ -511,9 +469,7 @@ class Vertex:
elif key not in self.params or self.updated_raw_params:
self.params[key] = value
async def _build_dict_and_update_params(
self, key, nodes_dict: Dict[str, "Vertex"], user_id=None
):
async def _build_dict_and_update_params(self, key, nodes_dict: Dict[str, "Vertex"], user_id=None):
"""
Iterates over a dictionary of nodes, builds each and updates the params dictionary.
"""
@ -536,9 +492,7 @@ class Vertex:
"""
return all(self._is_node(node) for node in value)
async def get_result(
self, requester: Optional["Vertex"] = None, user_id=None, timeout=None
) -> Any:
async def get_result(self, requester: Optional["Vertex"] = None, user_id=None, timeout=None) -> Any:
# PLEASE REVIEW THIS IF STATEMENT
# Check if the Vertex was built already
if self._built:
@ -572,9 +526,7 @@ class Vertex:
self._extend_params_list_with_result(key, result)
self.params[key] = result
async def _build_list_of_nodes_and_update_params(
self, key, nodes: List["Vertex"], user_id=None
):
async def _build_list_of_nodes_and_update_params(self, key, nodes: List["Vertex"], user_id=None):
"""
Iterates over a list of nodes, builds each and updates the params dictionary.
"""
@ -641,9 +593,7 @@ class Vertex:
except Exception as exc:
logger.exception(exc)
raise ValueError(
f"Error building node {self.display_name}: {str(exc)}"
) from exc
raise ValueError(f"Error building node {self.display_name}: {str(exc)}") from exc
def _update_built_object_and_artifacts(self, result):
"""
@ -671,9 +621,7 @@ class Vertex:
logger.warning(message)
elif isinstance(self._built_object, (Iterator, AsyncIterator)):
if self.display_name in ["Text Output"]:
raise ValueError(
f"You are trying to stream to a {self.display_name}. Try using a Chat Output instead."
)
raise ValueError(f"You are trying to stream to a {self.display_name}. Try using a Chat Output instead.")
def _reset(self, params_update: Optional[Dict[str, Any]] = None):
self._built = False
@ -736,24 +684,16 @@ class Vertex:
return self._built_object
# Get the requester edge
requester_edge = next(
(edge for edge in self.edges if edge.target_id == requester.id), None
)
requester_edge = next((edge for edge in self.edges if edge.target_id == requester.id), None)
# Return the result of the requester edge
return (
None
if requester_edge is None
else await requester_edge.get_result(source=self, target=requester)
)
return None if requester_edge is None else await requester_edge.get_result(source=self, target=requester)
def add_edge(self, edge: "ContractEdge") -> None:
if edge not in self.edges:
self.edges.append(edge)
def __repr__(self) -> str:
return (
f"Vertex(display_name={self.display_name}, id={self.id}, data={self.data})"
)
return f"Vertex(display_name={self.display_name}, id={self.id}, data={self.data})"
def __eq__(self, __o: object) -> bool:
try:
@ -774,8 +714,4 @@ class Vertex:
def _built_object_repr(self):
# Add a message with an emoji, stars for sucess,
return (
"Built sucessfully ✨"
if self._built_object is not None
else "Failed to build 😵‍💫"
)
return "Built sucessfully ✨" if self._built_object is not None else "Failed to build 😵‍💫"

View file

@ -124,11 +124,9 @@ class DocumentLoaderVertex(Vertex):
# show how many documents are in the list?
if not isinstance(self._built_object, UnbuiltObject):
avg_length = sum(
len(record.text)
for record in self._built_object
if hasattr(record, "text")
) / len(self._built_object)
avg_length = sum(len(record.text) for record in self._built_object if hasattr(record, "text")) / len(
self._built_object
)
return f"""{self.display_name}({len(self._built_object)} records)
\nAvg. Record Length (characters): {int(avg_length)}
Records: {self._built_object[:3]}..."""
@ -201,9 +199,7 @@ class TextSplitterVertex(Vertex):
# show how many documents are in the list?
if not isinstance(self._built_object, UnbuiltObject):
avg_length = sum(len(doc.page_content) for doc in self._built_object) / len(
self._built_object
)
avg_length = sum(len(doc.page_content) for doc in self._built_object) / len(self._built_object)
return f"""{self.vertex_type}({len(self._built_object)} documents)
\nAvg. Document Length (characters): {int(avg_length)}
\nDocuments: {self._built_object[:3]}..."""
@ -250,27 +246,18 @@ class PromptVertex(Vertex):
user_id = kwargs.get("user_id", None)
tools = kwargs.get("tools", [])
if not self._built or force:
if (
"input_variables" not in self.params
or self.params["input_variables"] is None
):
if "input_variables" not in self.params or self.params["input_variables"] is None:
self.params["input_variables"] = []
# Check if it is a ZeroShotPrompt and needs a tool
if "ShotPrompt" in self.vertex_type:
tools = (
[tool_node.build(user_id=user_id) for tool_node in tools]
if tools is not None
else []
)
tools = [tool_node.build(user_id=user_id) for tool_node in tools] if tools is not None else []
# flatten the list of tools if it is a list of lists
# first check if it is a list
if tools and isinstance(tools, list) and isinstance(tools[0], list):
tools = flatten_list(tools)
self.params["tools"] = tools
prompt_params = [
key
for key, value in self.params.items()
if isinstance(value, str) and key != "format_instructions"
key for key, value in self.params.items() if isinstance(value, str) and key != "format_instructions"
]
else:
prompt_params = ["template"]
@ -280,20 +267,14 @@ class PromptVertex(Vertex):
prompt_text = self.params[param]
variables = extract_input_variables_from_prompt(prompt_text)
self.params["input_variables"].extend(variables)
self.params["input_variables"] = list(
set(self.params["input_variables"])
)
self.params["input_variables"] = list(set(self.params["input_variables"]))
elif isinstance(self.params, dict):
self.params.pop("input_variables", None)
await self._build(user_id=user_id)
def _built_object_repr(self):
if (
not self.artifacts
or self._built_object is None
or not hasattr(self._built_object, "format")
):
if not self.artifacts or self._built_object is None or not hasattr(self._built_object, "format"):
return super()._built_object_repr()
elif isinstance(self._built_object, UnbuiltObject):
return super()._built_object_repr()
@ -305,9 +286,7 @@ class PromptVertex(Vertex):
# so the prompt format doesn't break
artifacts.pop("handle_keys", None)
try:
if not hasattr(self._built_object, "template") and hasattr(
self._built_object, "prompt"
):
if not hasattr(self._built_object, "template") and hasattr(self._built_object, "prompt"):
template = self._built_object.prompt.template
else:
template = self._built_object.template
@ -315,11 +294,7 @@ class PromptVertex(Vertex):
if value:
replace_key = "{" + key + "}"
template = template.replace(replace_key, value)
return (
template
if isinstance(template, str)
else f"{self.vertex_type}({template})"
)
return template if isinstance(template, str) else f"{self.vertex_type}({template})"
except KeyError:
return str(self._built_object)
@ -354,14 +329,8 @@ class ChatVertex(Vertex):
return f"Task {self.task_id} is not running"
if self.artifacts:
# dump as a yaml string
artifacts = {
k.title().replace("_", " "): v
for k, v in self.artifacts.items()
if v is not None
}
yaml_str = yaml.dump(
artifacts, default_flow_style=False, allow_unicode=True
)
artifacts = {k.title().replace("_", " "): v for k, v in self.artifacts.items() if v is not None}
yaml_str = yaml.dump(artifacts, default_flow_style=False, allow_unicode=True)
return yaml_str
return super()._built_object_repr()

View file

@ -17,9 +17,7 @@ CUSTOM_TOOLS = {
"PythonFunctionTool": PythonFunctionTool,
}
OTHER_TOOLS = {
tool: import_class(f"langchain_community.tools.{tool}") for tool in tools.__all__
}
OTHER_TOOLS = {tool: import_class(f"langchain_community.tools.{tool}") for tool in tools.__all__}
ALL_TOOLS_NAMES = {
**_BASE_TOOLS,

View file

@ -73,9 +73,7 @@ class Record(BaseModel):
return self.data.get(key, self._default_value)
except KeyError:
# Fallback to default behavior to raise AttributeError for undefined attributes
raise AttributeError(
f"'{type(self).__name__}' object has no attribute '{key}'"
)
raise AttributeError(f"'{type(self).__name__}' object has no attribute '{key}'")
def __setattr__(self, key, value):
"""

View file

@ -1,4 +1,3 @@
import datetime
import secrets
from pathlib import Path
from typing import Optional
@ -37,7 +36,7 @@ class AuthSettings(BaseSettings):
NEW_USER_IS_ACTIVE: bool = False
SUPERUSER: str = DEFAULT_SUPERUSER
SUPERUSER_PASSWORD: str = DEFAULT_SUPERUSER_PASSWORD
REFRESH_SAME_SITE: str = "none"
"""The SameSite attribute of the refresh token cookie."""
REFRESH_SECURE: bool = True

View file

@ -20,12 +20,8 @@ def remove_ansi_escape_codes(text):
return re.sub(r"\x1b\[[0-9;]*[a-zA-Z]", "", text)
def build_template_from_function(
name: str, type_to_loader_dict: Dict, add_function: bool = False
):
classes = [
item.__annotations__["return"].__name__ for item in type_to_loader_dict.values()
]
def build_template_from_function(name: str, type_to_loader_dict: Dict, add_function: bool = False):
classes = [item.__annotations__["return"].__name__ for item in type_to_loader_dict.values()]
# Raise error if name is not in chains
if name not in classes:
@ -46,10 +42,8 @@ def build_template_from_function(
for name_, value_ in value.__repr_args__():
if name_ == "default_factory":
try:
variables[class_field_items]["default"] = (
get_default_factory(
module=_class.__base__.__module__, function=value_
)
variables[class_field_items]["default"] = get_default_factory(
module=_class.__base__.__module__, function=value_
)
except Exception:
variables[class_field_items]["default"] = None
@ -57,9 +51,7 @@ def build_template_from_function(
variables[class_field_items][name_] = value_
variables[class_field_items]["placeholder"] = (
docs.params[class_field_items]
if class_field_items in docs.params
else ""
docs.params[class_field_items] if class_field_items in docs.params else ""
)
# Adding function to base classes to allow
# the output to be a function
@ -74,9 +66,7 @@ def build_template_from_function(
}
def build_template_from_class(
name: str, type_to_cls_dict: Dict, add_function: bool = False
):
def build_template_from_class(name: str, type_to_cls_dict: Dict, add_function: bool = False):
classes = [item.__name__ for item in type_to_cls_dict.values()]
# Raise error if name is not in chains
@ -100,11 +90,9 @@ def build_template_from_class(
for name_, value_ in value.__repr_args__():
if name_ == "default_factory":
try:
variables[class_field_items]["default"] = (
get_default_factory(
module=_class.__base__.__module__,
function=value_,
)
variables[class_field_items]["default"] = get_default_factory(
module=_class.__base__.__module__,
function=value_,
)
except Exception:
variables[class_field_items]["default"] = None
@ -112,9 +100,7 @@ def build_template_from_class(
variables[class_field_items][name_] = value_
variables[class_field_items]["placeholder"] = (
docs.params[class_field_items]
if class_field_items in docs.params
else ""
docs.params[class_field_items] if class_field_items in docs.params else ""
)
base_classes = get_base_classes(_class)
# Adding function to base classes to allow
@ -146,9 +132,7 @@ def build_template_from_method(
# Check if the method exists in this class
if not hasattr(_class, method_name):
raise ValueError(
f"Method {method_name} not found in class {class_name}"
)
raise ValueError(f"Method {method_name} not found in class {class_name}")
# Get the method
method = getattr(_class, method_name)
@ -167,14 +151,8 @@ def build_template_from_method(
"_type": _type,
**{
name: {
"default": (
param.default if param.default != param.empty else None
),
"type": (
param.annotation
if param.annotation != param.empty
else None
),
"default": (param.default if param.default != param.empty else None),
"type": (param.annotation if param.annotation != param.empty else None),
"required": param.default == param.empty,
}
for name, param in params.items()
@ -261,9 +239,7 @@ def sync_to_async(func):
return async_wrapper
def format_dict(
dictionary: Dict[str, Any], class_name: Optional[str] = None
) -> Dict[str, Any]:
def format_dict(dictionary: Dict[str, Any], class_name: Optional[str] = None) -> Dict[str, Any]:
"""
Formats a dictionary by removing certain keys and modifying the
values of other keys.
@ -349,9 +325,7 @@ def check_list_type(_type: str, value: Dict[str, Any]) -> str:
The modified type string.
"""
if any(list_type in _type for list_type in ["List", "Sequence", "Set"]):
_type = (
_type.replace("List[", "").replace("Sequence[", "").replace("Set[", "")[:-1]
)
_type = _type.replace("List[", "").replace("Sequence[", "").replace("Set[", "")[:-1]
value["list"] = True
else:
value["list"] = False
@ -454,9 +428,7 @@ def set_headers_value(value: Dict[str, Any]) -> None:
value["value"] = """{"Authorization": "Bearer <token>"}"""
def add_options_to_field(
value: Dict[str, Any], class_name: Optional[str], key: str
) -> None:
def add_options_to_field(value: Dict[str, Any], class_name: Optional[str], key: str) -> None:
"""
Adds options to the field based on the class name and key.
"""