chore: Refactor dependencies in pyproject.toml files (#8249)

* feat: add datasets dependency version constraint

* fix: update setuptools version constraint to 78.1.1

* refactor: remove deploy dependency group from pyproject.toml

pyproject.toml

@@ -51,6 +51,7 @@ dependencies = [
     "elasticsearch==8.16.0",
     "pytube==15.0.0",
     "dspy-ai==2.5.41",
+    "datasets>2.14.7",
     "assemblyai==0.35.1",
     "litellm==1.60.2",
     "chromadb==0.5.23",
@@ -191,10 +192,6 @@ Repository = "https://github.com/langflow-ai/langflow"
 Documentation = "https://docs.langflow.org"
 
 [project.optional-dependencies]
-deploy = [
-    "celery[redis]>=5.3.6",
-    "flower>=2.0.0"
-]
 couchbase = [
     "couchbase>=4.2.1"
 ]

src/backend/base/pyproject.toml

@@ -64,7 +64,7 @@ dependencies = [
     "opentelemetry-instrumentation-fastapi>=0.46b0,<1.0.0",
     "prometheus-client>=0.20.0,<1.0.0",
     "aiofiles>=24.1.0,<25.0.0",
-    "setuptools>=70,<76.0.0",
+    "setuptools>=78.1.0,<79.0.0",
     "nanoid>=2.0.0,<3.0.0",
     "filelock>=3.15.4,<4.0.0",
     "grandalf>=0.8.0,<1.0.0",
@@ -240,20 +240,12 @@ postgresql = [
     "sqlalchemy[postgresql_psycopg]",
 
 ]
-deploy = [
-    "celery>=5.3.1",
-    "redis>=4.6.0",
-    "flower>=1.0.0"
-]
 local = [
     "llama-cpp-python>=0.2.0",
     "sentence-transformers>=2.0.0",
     "ctransformers>=0.2"
 ]
 all = [
-    "celery>=5.3.1",
-    "redis>=4.6.0",
-    "flower>=1.0.0",
     "llama-cpp-python>=0.2.0",
     "sentence-transformers>=2.0.0",
     "ctransformers>=0.2"