Merge remote-tracking branch 'origin/validation_fix' into db

src/frontend/src/CustomNodes/GenericNode/index.tsx

@@ -1,26 +1,18 @@
-import {
-  classNames,
-  nodeColors,
-  nodeIcons,
-  toNormalCase,
-  toTitleCase,
-} from "../../utils";
+import { classNames, nodeColors, nodeIcons, toTitleCase } from "../../utils";
 import ParameterComponent from "./components/parameterComponent";
 import { typesContext } from "../../contexts/typesContext";
-import { useContext, useState, useEffect, useRef, Fragment } from "react";
+import { useContext, useState, useEffect, useRef } from "react";
 import { NodeDataType } from "../../types/flow";
 import { alertContext } from "../../contexts/alertContext";
 import { PopUpContext } from "../../contexts/popUpContext";
 import NodeModal from "../../modals/NodeModal";
-import { useCallback } from "react";
-import { TabsContext } from "../../contexts/tabsContext";
-import { debounce } from "../../utils";
 import Tooltip from "../../components/TooltipComponent";
 import { NodeToolbar } from "reactflow";
 import NodeToolbarComponent from "../../pages/FlowPage/components/nodeToolbarComponent";
 
 import ShadTooltip from "../../components/ShadTooltipComponent";
-import { postValidateNode } from "../../controllers/API";
+import { useSSE } from "../../contexts/SSEContext";
+
 export default function GenericNode({
   data,
   selected,
@@ -31,46 +23,30 @@ export default function GenericNode({
   const { setErrorData } = useContext(alertContext);
   const showError = useRef(true);
   const { types, deleteNode } = useContext(typesContext);
-  const { openPopUp } = useContext(PopUpContext);
-  const { closePopUp } = useContext(PopUpContext);
+
+  const { closePopUp, openPopUp } = useContext(PopUpContext);
 
   const Icon = nodeIcons[data.type] || nodeIcons[types[data.type]];
   const [validationStatus, setValidationStatus] = useState(null);
   // State for outline color
-  const [isValid, setIsValid] = useState(false);
-  const { reactFlowInstance } = useContext(typesContext);
-  const [params, setParams] = useState([]);
+  const { sseData } = useSSE();
 
+  // useEffect(() => {
+  //   if (reactFlowInstance) {
+  //     setParams(Object.values(reactFlowInstance.toObject()));
+  //   }
+  // }, [save]);
+
+  // New useEffect to watch for changes in sseData and update validation status
   useEffect(() => {
-    if (reactFlowInstance) {
-      setParams(Object.values(reactFlowInstance.toObject()));
+    const relevantData = sseData[data.id];
+    if (relevantData) {
+      // Extract validation information from relevantData and update the validationStatus state
+      setValidationStatus(relevantData);
+    } else {
+      setValidationStatus(null);
     }
-  }, []);
-
-  const validateNode = useCallback(
-    debounce(async () => {
-      try {
-        const response = await postValidateNode(
-          data.id,
-          reactFlowInstance.toObject()
-        );
-
-        if (response.status === 200) {
-          let jsonResponseParsed = await JSON.parse(response.data);
-          setValidationStatus(jsonResponseParsed);
-        }
-      } catch (error) {
-        // console.error("Error validating node:", error);
-        setValidationStatus("error");
-      }
-    }, 1000), // Adjust the debounce delay (500ms) as needed
-    [reactFlowInstance, data.id]
-  );
-  useEffect(() => {
-    if (params.length > 0) {
-      validateNode();
-    }
-  }, [params, validateNode]);
+  }, [sseData, data.id]);
 
   if (!Icon) {
     if (showError.current) {

src/frontend/src/components/chatComponent/buildTrigger/index.tsx

@@ -0,0 +1,155 @@
+import { useState, useContext } from "react";
+import { Transition } from "@headlessui/react";
+import { Zap } from "lucide-react";
+import { validateNodes } from "../../../utils";
+import { FlowType } from "../../../types/flow";
+import Loading from "../../../components/ui/loading";
+import { useSSE } from "../../../contexts/SSEContext";
+import { typesContext } from "../../../contexts/typesContext";
+import { alertContext } from "../../../contexts/alertContext";
+import { postBuildInit } from "../../../controllers/API";
+
+export default function BuildTrigger({
+  open,
+  flow,
+  setIsBuilt,
+  isBuilt,
+}: {
+  open: boolean;
+  flow: FlowType;
+  setIsBuilt: any;
+  isBuilt: boolean;
+}) {
+  const [isBuilding, setIsBuilding] = useState(false);
+
+  const { updateSSEData } = useSSE();
+  const { reactFlowInstance } = useContext(typesContext);
+  const { setErrorData } = useContext(alertContext);
+
+  async function handleBuild(flow: FlowType) {
+    const errors = validateNodes(reactFlowInstance);
+    if (errors.length > 0) {
+      setErrorData({
+        title: "Oops! Looks like you missed something",
+        list: errors,
+      });
+      return;
+    }
+    const minimumLoadingTime = 200; // in milliseconds
+    const startTime = Date.now();
+    setIsBuilding(true);
+
+    try {
+      const allNodesValid = await streamNodeData(flow);
+      await enforceMinimumLoadingTime(startTime, minimumLoadingTime);
+      setIsBuilt(allNodesValid);
+    } catch (error) {
+      console.error("Error:", error);
+    } finally {
+      setIsBuilding(false);
+    }
+  }
+
+  async function streamNodeData(flow: FlowType) {
+    // Step 1: Make a POST request to send the flow data and receive a unique session ID
+    const response = await postBuildInit(flow);
+    const { flowId } = response.data;
+
+    // Step 2: Use the session ID to establish an SSE connection using EventSource
+    let validationResults = [];
+    let finished = false;
+    const apiUrl = `/api/v1/build/stream/${flowId}`;
+    const eventSource = new EventSource(apiUrl);
+    try{
+      eventSource.onmessage = (event) => {
+        // If the event is parseable, return
+        if (!event.data) {
+          return;
+        }
+        const parsedData = JSON.parse(event.data);
+        // if the event is the end of the stream, close the connection
+        if (parsedData.end_of_stream) {
+          eventSource.close();
+  
+          return;
+        }
+        // Otherwise, process the data
+        const isValid = processStreamResult(parsedData);
+        validationResults.push(isValid);
+      };
+  
+      eventSource.onerror = (error) => {
+        console.error("EventSource failed:", error);
+        eventSource.close();
+      };
+      // Step 3: Wait for the stream to finish
+      while (!finished) {
+        await new Promise((resolve) => setTimeout(resolve, 100));
+        finished = validationResults.length === flow.data.nodes.length;
+      }
+      // Step 4: Return true if all nodes are valid, false otherwise
+      return validationResults.every((result) => result);
+    }
+    catch(e){
+      console.log(e)
+      eventSource.close();
+      return false;
+    }
+  }
+
+  function processStreamResult(parsedData) {
+    // Process each chunk of data here
+    // Parse the chunk and update the context
+    try {
+      updateSSEData({ [parsedData.id]: parsedData });
+    } catch (err) {
+      console.log("Error parsing stream data: ", err);
+    }
+    return parsedData.valid;
+  }
+
+  async function enforceMinimumLoadingTime(
+    startTime: number,
+    minimumLoadingTime: number
+  ) {
+    const elapsedTime = Date.now() - startTime;
+    const remainingTime = minimumLoadingTime - elapsedTime;
+
+    if (remainingTime > 0) {
+      return new Promise((resolve) => setTimeout(resolve, remainingTime));
+    }
+  }
+
+  return (
+    <Transition
+      show={!open}
+      appear={true}
+      enter="transition ease-out duration-300"
+      enterFrom="translate-y-96"
+      enterTo="translate-y-0"
+      leave="transition ease-in duration-300"
+      leaveFrom="translate-y-0"
+      leaveTo="translate-y-96"
+    >
+      <div className={`fixed right-4` + (isBuilt ? " bottom-20" : " bottom-4")}>
+        <div
+          className="border flex justify-center align-center py-1 px-3 w-12 h-12 rounded-full bg-gradient-to-r from-blue-700 via-blue-600 to-blue-500 dark:border-gray-600 cursor-pointer"
+          onClick={() => {
+            if (!isBuilding) handleBuild(flow);
+          }}
+        >
+          <button>
+            <div className="flex gap-3 items-center">
+              {isBuilding ? (
+                // Render your loading animation here when isBuilding is true
+                <Loading style={{ color: "white" }} />
+              ) : (
+                <Zap className="h-6 w-6" style={{ color: "white" }} />
+              )}
+            </div>
+          </button>
+        </div>
+      </div>
+    </Transition>
+  );
+}

src/frontend/src/components/chatComponent/chatTrigger/index.tsx

@@ -3,13 +3,26 @@ import {
   Bars3CenterLeftIcon,
   ChatBubbleBottomCenterTextIcon,
 } from "@heroicons/react/24/outline";
+import { MessagesSquare } from "lucide-react";
 import { nodeColors } from "../../../utils";
-import { PopUpContext } from "../../../contexts/popUpContext";
+import { alertContext } from "../../../contexts/alertContext";
 import { useContext } from "react";
 import ChatModal from "../../../modals/chatModal";
 
-export default function ChatTrigger({ open, setOpen }) {
-  const { openPopUp } = useContext(PopUpContext);
+export default function ChatTrigger({ open, setOpen, isBuilt }) {
+  const { setErrorData } = useContext(alertContext);
+
+  function handleClick() {
+    if (isBuilt) {
+      setOpen(true);
+    } else {
+      setErrorData({
+        title: "Flow not built",
+        list: ["Please build the flow before chatting"],
+      });
+    }
+  }
+
   return (
     <Transition
       show={!open}
@@ -24,13 +37,11 @@ export default function ChatTrigger({ open, setOpen }) {
       <div className="absolute bottom-4 right-3">
         <div
           className="border flex justify-center align-center py-1 px-3 w-12 h-12 rounded-full bg-gradient-to-r from-blue-500 via-blue-600 to-blue-700 dark:border-gray-600 cursor-pointer"
-          onClick={() => {
-            setOpen(true);
-          }}
+          onClick={handleClick}
         >
           <button>
             <div className="flex gap-3  items-center">
-              <ChatBubbleBottomCenterTextIcon
+              <MessagesSquare
                 className="h-6 w-6 mt-1"
                 style={{ color: "white" }}
               />

src/frontend/src/components/chatComponent/index.tsx

@@ -1,13 +1,18 @@
-import { useEffect, useRef, useState } from "react";
-
+import { Context, useEffect, useRef, useState, useContext } from "react";
+import ReactFlow, { useNodes } from "reactflow";
 import { ChatMessageType, ChatType } from "../../types/chat";
 import ChatTrigger from "./chatTrigger";
+import BuildTrigger from "./buildTrigger";
 import ChatModal from "../../modals/chatModal";
 
-import _ from "lodash";
+import _, { set } from "lodash";
+import { getBuildStatus } from "../../controllers/API";
+import { NodeType } from "../../types/flow";
 
 export default function Chat({ flow }: ChatType) {
   const [open, setOpen] = useState(false);
+  const [isBuilt, setIsBuilt] = useState(false);
+
   useEffect(() => {
     const handleKeyDown = (event: KeyboardEvent) => {
       if (
@@ -23,10 +28,58 @@ export default function Chat({ flow }: ChatType) {
       document.removeEventListener("keydown", handleKeyDown);
     };
   }, []);
+
+  useEffect(() => {
+    // Define an async function within the useEffect hook
+    const fetchBuildStatus = async () => {
+      const response = await getBuildStatus(flow.id);
+      setIsBuilt(response.built);
+    };
+
+    // Call the async function
+    fetchBuildStatus();
+  }, [flow]);
+
+  const prevNodesRef = useRef<any[] | undefined>();
+  const nodes = useNodes();
+  useEffect(() => {
+    const prevNodes = prevNodesRef.current;
+    const currentNodes = nodes.map(
+      (node: NodeType) => node.data.node.template.value
+    );
+
+    if (
+      prevNodes &&
+      JSON.stringify(prevNodes) !== JSON.stringify(currentNodes)
+    ) {
+      setIsBuilt(false);
+      console.log("Nodes changed");
+    }
+
+    prevNodesRef.current = currentNodes;
+  }, [nodes]);
+
   return (
     <>
-      <ChatModal key={flow.id} flow={flow} open={open} setOpen={setOpen} />
-      <ChatTrigger open={open} setOpen={setOpen} />
+      {isBuilt ? (
+        <div>
+          <BuildTrigger
+            open={open}
+            flow={flow}
+            setIsBuilt={setIsBuilt}
+            isBuilt={isBuilt}
+          />
+          <ChatModal key={flow.id} flow={flow} open={open} setOpen={setOpen} />
+          <ChatTrigger open={open} setOpen={setOpen} isBuilt={isBuilt} />
+        </div>
+      ) : (
+        <BuildTrigger
+          open={open}
+          flow={flow}
+          setIsBuilt={setIsBuilt}
+          isBuilt={isBuilt}
+        />
+      )}
     </>
   );
 }

src/frontend/src/components/ui/loading.tsx

@@ -0,0 +1,39 @@
+import { SVGProps } from "react";
+
+// https://github.com/feathericons/feather/issues/695#issuecomment-1503699643
+export const Loading = (props: SVGProps<SVGSVGElement>) => (
+  <svg
+    xmlns="http://www.w3.org/2000/svg"
+    width={24}
+    height={24}
+    viewBox="0 0 24 24"
+    fill="none"
+    stroke="currentColor"
+    strokeWidth={2}
+    strokeLinecap="round"
+    strokeLinejoin="round"
+    className="feather feather-circle"
+    {...props}
+  >
+    <circle cx={12} cy={12} r={10} strokeDasharray={63} strokeDashoffset={21}>
+      <animateTransform
+        attributeName="transform"
+        type="rotate"
+        from="0 12 12"
+        to="360 12 12"
+        dur="2s"
+        repeatCount="indefinite"
+      />
+      <animate
+        attributeName="stroke-dashoffset"
+        dur="8s"
+        repeatCount="indefinite"
+        keyTimes="0; 0.5; 1"
+        values="-16; -47; -16"
+        calcMode="spline"
+        keySplines="0.4 0 0.2 1; 0.4 0 0.2 1"
+      />
+    </circle>
+  </svg>
+);
+export default Loading;

src/frontend/src/contexts/SSEContext.tsx

@@ -0,0 +1,35 @@
+import {
+  createContext,
+  useContext,
+  useState,
+  useEffect,
+  useCallback,
+} from "react";
+
+const initialValue = {
+  updateSSEData: ({}) => {},
+  sseData: {},
+};
+
+const SSEContext = createContext(initialValue);
+
+export function useSSE() {
+  return useContext(SSEContext);
+}
+
+export function SSEProvider({ children }) {
+  const [sseData, setSSEData] = useState({});
+
+  const updateSSEData = useCallback((newData: any) => {
+    setSSEData((prevData) => ({
+      ...prevData,
+      ...newData,
+    }));
+  }, []);
+
+  return (
+    <SSEContext.Provider value={{ sseData, updateSSEData }}>
+      {children}
+    </SSEContext.Provider>
+  );
+}

src/frontend/src/controllers/API/index.ts

@@ -1,4 +1,9 @@
-import { PromptTypeAPI, errorsTypeAPI } from "./../../types/api/index";
+import {
+  BuildStatusTypeAPI,
+  PromptTypeAPI,
+  errorsTypeAPI,
+  InitTypeAPI,
+} from "./../../types/api/index";
 import { APIObjectType, sendAllProps } from "../../types/api/index";
 import axios, { AxiosResponse } from "axios";
 import { FlowStyleType, FlowType } from "../../types/flow";
@@ -272,4 +277,14 @@ export async function getVersion() {
  */
 export async function getHealth() {
   return await axios.get("/health"); // Health is the only endpoint that doesn't require /api/v1
+export async function getBuildStatus(
+  flowId: string
+): Promise<BuildStatusTypeAPI> {
+  return await axios.get(`/api/v1/build/${flowId}/status`);
+}
+
+export async function postBuildInit(
+  flow: FlowType
+): Promise<AxiosResponse<InitTypeAPI>> {
+  return await axios.post(`/api/v1/build/init`, flow);
 }

src/frontend/src/modals/EditNodeModal/index.tsx

@@ -22,7 +22,6 @@ import IntComponent from "../../components/intComponent";
 import InputFileComponent from "../../components/inputFileComponent";
 import PromptAreaComponent from "../../components/promptComponent";
 import CodeAreaComponent from "../../components/codeAreaComponent";
-import { TabsContext } from "../../contexts/tabsContext";
 import {
   Dialog,
   DialogContent,
@@ -33,7 +32,6 @@ import {
   DialogTrigger,
 } from "../../components/ui/dialog";
 import { Button } from "../../components/ui/button";
-import { Edit } from "lucide-react";
 import { Badge } from "../../components/ui/badge";
 
 export default function EditNodeModal({ data }: { data: NodeDataType }) {

src/frontend/src/modals/chatModal/index.tsx

@@ -3,7 +3,7 @@ import { ChatBubbleOvalLeftEllipsisIcon } from "@heroicons/react/24/outline";
 import { Fragment, useContext, useEffect, useRef, useState } from "react";
 import { FlowType, NodeType } from "../../types/flow";
 import { alertContext } from "../../contexts/alertContext";
-import { toNormalCase } from "../../utils";
+import { toNormalCase, validateNodes } from "../../utils";
 import { typesContext } from "../../contexts/typesContext";
 import ChatMessage from "./chatMessage";
 import { FaEraser } from "react-icons/fa";
@@ -185,6 +185,17 @@ export default function ChatModal({
     }://${host}${chatEndpoint}`;
   }
 
+  function getWebSocketUrl(chatId, isDevelopment = false) {
+    const isSecureProtocol = window.location.protocol === "https:";
+    const webSocketProtocol = isSecureProtocol ? "wss" : "ws";
+    const host = isDevelopment ? "localhost:7860" : window.location.host;
+    const chatEndpoint = `/api/v1/chat/${chatId}`;
+
+    return `${
+      isDevelopment ? "ws" : webSocketProtocol
+    }://${host}${chatEndpoint}`;
+  }
+
   function connectWS() {
     try {
       const urlWs = getWebSocketUrl(
@@ -269,53 +280,6 @@ export default function ChatModal({
     if (ref.current) ref.current.scrollIntoView({ behavior: "smooth" });
   }, [chatHistory]);
 
-  function validateNode(n: NodeType): Array<string> {
-    if (!n.data?.node?.template || !Object.keys(n.data.node.template)) {
-      setNoticeData({
-        title:
-          "We've noticed a potential issue with a node in the flow. Please review it and, if necessary, submit a bug report with your exported flow file. Thank you for your help!",
-      });
-      return [];
-    }
-
-    const {
-      type,
-      node: { template },
-    } = n.data;
-
-    return Object.keys(template).reduce(
-      (errors: Array<string>, t) =>
-        errors.concat(
-          template[t].required &&
-            template[t].show &&
-            (template[t].value === undefined ||
-              template[t].value === null ||
-              template[t].value === "") &&
-            !reactFlowInstance
-              .getEdges()
-              .some(
-                (e) =>
-                  e.targetHandle.split("|")[1] === t &&
-                  e.targetHandle.split("|")[2] === n.id
-              )
-            ? [
-                `${type} is missing ${
-                  template.display_name
-                    ? template.display_name
-                    : toNormalCase(template[t].name)
-                }.`,
-              ]
-            : []
-        ),
-      [] as string[]
-    );
-  }
-
-  function validateNodes() {
-    return reactFlowInstance
-      .getNodes()
-      .flatMap((n: NodeType) => validateNode(n));
-  }
 
   const ref = useRef(null);
 
@@ -327,7 +291,7 @@ export default function ChatModal({
 
   function sendMessage() {
     if (chatValue !== "") {
-      let nodeValidationErrors = validateNodes();
+      let nodeValidationErrors = validateNodes(reactFlowInstance);
       if (nodeValidationErrors.length === 0) {
         setLockChat(true);
         let message = chatValue;

src/frontend/src/types/api/index.ts

@@ -38,3 +38,11 @@ export type errorsTypeAPI = {
   imports: { errors: Array<string> };
 };
 export type PromptTypeAPI = { input_variables: Array<string> };
+
+export type BuildStatusTypeAPI = {
+  built: boolean;
+};
+
+export type InitTypeAPI = {
+  flowId: string;
+};

src/frontend/src/utils.ts

@@ -17,7 +17,7 @@ import {
   Bars3CenterLeftIcon,
 } from "@heroicons/react/24/outline";
 import { Connection, Edge, Node, ReactFlowInstance } from "reactflow";
-import { FlowType, NodeDataType, NodeType } from "./types/flow";
+import { FlowType, NodeType } from "./types/flow";
 import { APITemplateType } from "./types/api";
 import _ from "lodash";
 import { ChromaIcon } from "./icons/ChromaIcon";
@@ -737,3 +737,56 @@ export function buildTweaks(flow) {
     return acc;
   }, {});
 }
+export function validateNode(
+  n: NodeType,
+  reactFlowInstance: ReactFlowInstance
+): Array<string> {
+  if (!n.data?.node?.template || !Object.keys(n.data.node.template)) {
+    return [
+      "We've noticed a potential issue with a node in the flow. Please review it and, if necessary, submit a bug report with your exported flow file. Thank you for your help!",
+    ];
+  }
+
+  const {
+    type,
+    node: { template },
+  } = n.data;
+
+  return Object.keys(template).reduce(
+    (errors: Array<string>, t) =>
+      errors.concat(
+        template[t].required &&
+          template[t].show &&
+          (template[t].value === undefined ||
+            template[t].value === null ||
+            template[t].value === "") &&
+          !reactFlowInstance
+            .getEdges()
+            .some(
+              (e) =>
+                e.targetHandle.split("|")[1] === t &&
+                e.targetHandle.split("|")[2] === n.id
+            )
+          ? [
+              `${type} is missing ${
+                template.display_name
+                  ? template.display_name
+                  : toNormalCase(template[t].name)
+              }.`,
+            ]
+          : []
+      ),
+    [] as string[]
+  );
+}
+
+export function validateNodes(reactFlowInstance: ReactFlowInstance) {
+  if (reactFlowInstance.getNodes().length === 0) {
+    return [
+      "No nodes found in the flow. Please add at least one node to the flow.",
+    ];
+  }
+  return reactFlowInstance
+    .getNodes()
+    .flatMap((n: NodeType) => validateNode(n, reactFlowInstance));
+}

src/frontend/vite.config.ts

@@ -12,11 +12,9 @@ const proxyTargets = apiRoutes.reduce((proxyObj, route) => {
     changeOrigin: true,
     secure: false,
     ws: true,
-    // rewrite: (path) => `/api/v1${path}`,
   };
   return proxyObj;
 }, {});
-
 export default defineConfig(() => {
   return {
     build: {