jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

feat: add AssemblyAI components (#3829)

Browse source 
* Add AssemblyAI components

* add icons

* [autofix.ci] apply automated fixes

* Add ruff fixes

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Gabriel Luiz Freitas Almeida <gabriel@langflow.org>
This commit is contained in:
Patrick Loeber 2024-09-25 19:32:03 +02:00 • committed by GitHub
commit c567323413
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 1416 additions and 2030 deletions
Download patch file Download diff file Expand all files Collapse all files

63
src/backend/base/langflow/components/documentloaders/AssemblyAIFormatTranscript.py Normal file
View file

@ -0,0 +1,63 @@
import datetime
from typing import Dict, List
from langflow.custom import Component
from langflow.io import DataInput, Output
from langflow.schema import Data
class AssemblyAITranscriptionParser(Component):
display_name = "AssemblyAI Parse Transcript"
description = "Parse AssemblyAI transcription result. If Speaker Labels was enabled, format utterances with speakers and timestamps"
documentation = "https://www.assemblyai.com/docs"
icon = "AssemblyAI"
inputs = [
DataInput(
name="transcription_result",
display_name="Transcription Result",
info="The transcription result from AssemblyAI",
),
]
outputs = [
Output(display_name="Parsed Transcription", name="parsed_transcription", method="parse_transcription"),
]
def parse_transcription(self) -> Data:
# check if it's an error message from the previous step
if self.transcription_result.data.get("error"):
self.status = self.transcription_result.data["error"]
return self.transcription_result
try:
transcription_data = self.transcription_result.data
if transcription_data.get("utterances"):
# If speaker diarization was enabled
parsed_result = self.parse_with_speakers(transcription_data["utterances"])
elif transcription_data.get("text"):
# If speaker diarization was not enabled
parsed_result = transcription_data["text"]
else:
raise ValueError("Unexpected transcription format")
self.status = parsed_result
return Data(data={"text": parsed_result})
except Exception as e:
error_message = f"Error parsing transcription: {str(e)}"
self.status = error_message
return Data(data={"error": error_message})
def parse_with_speakers(self, utterances: List[Dict]) -> str:
parsed_result = []
for utterance in utterances:
speaker = utterance["speaker"]
start_time = self.format_timestamp(utterance["start"])
text = utterance["text"]
parsed_result.append(f'Speaker {speaker} {start_time}\n"{text}"\n')
return "\n".join(parsed_result)
def format_timestamp(self, milliseconds: int) -> str:
return str(datetime.timedelta(milliseconds=milliseconds)).split(".")[0]

80
src/backend/base/langflow/components/documentloaders/AssemblyAIGetSubtitles.py Normal file
View file

@ -0,0 +1,80 @@
import assemblyai as aai
from langflow.custom import Component
from langflow.io import DataInput, DropdownInput, IntInput, Output, SecretStrInput
from langflow.schema import Data
class AssemblyAIGetSubtitles(Component):
display_name = "AssemblyAI Get Subtitles"
description = "Export your transcript in SRT or VTT format for subtitles and closed captions"
documentation = "https://www.assemblyai.com/docs"
icon = "AssemblyAI"
inputs = [
SecretStrInput(
name="api_key",
display_name="Assembly API Key",
info="Your AssemblyAI API key. You can get one from https://www.assemblyai.com/",
),
DataInput(
name="transcription_result",
display_name="Transcription Result",
info="The transcription result from AssemblyAI",
),
DropdownInput(
name="subtitle_format",
display_name="Subtitle Format",
options=["srt", "vtt"],
value="srt",
info="The format of the captions (SRT or VTT)",
),
IntInput(
name="chars_per_caption",
display_name="Characters per Caption",
info="The maximum number of characters per caption (0 for no limit)",
value=0,
advanced=True,
),
]
outputs = [
Output(display_name="Subtitles", name="subtitles", method="get_subtitles"),
]
def get_subtitles(self) -> Data:
aai.settings.api_key = self.api_key
# check if it's an error message from the previous step
if self.transcription_result.data.get("error"):
self.status = self.transcription_result.data["error"]
return self.transcription_result
try:
transcript_id = self.transcription_result.data["id"]
transcript = aai.Transcript.get_by_id(transcript_id)
except Exception as e:
error = f"Getting transcription failed: {str(e)}"
self.status = error
return Data(data={"error": error})
if transcript.status == aai.TranscriptStatus.completed:
subtitles = None
chars_per_caption = self.chars_per_caption if self.chars_per_caption > 0 else None
if self.subtitle_format == "srt":
subtitles = transcript.export_subtitles_srt(chars_per_caption)
else:
subtitles = transcript.export_subtitles_vtt(chars_per_caption)
result = Data(
subtitles=subtitles,
format=self.subtitle_format,
transcript_id=transcript_id,
chars_per_caption=chars_per_caption,
)
self.status = result
return result
else:
self.status = transcript.error
return Data(data={"error": transcript.error})

108
src/backend/base/langflow/components/documentloaders/AssemblyAILeMUR.py Normal file
View file

@ -0,0 +1,108 @@
import assemblyai as aai
from langflow.custom import Component
from langflow.io import DataInput, DropdownInput, FloatInput, IntInput, MessageInput, Output, SecretStrInput
from langflow.schema import Data
class AssemblyAILeMUR(Component):
display_name = "AssemblyAI LeMUR"
description = "Apply Large Language Models to spoken data using the AssemblyAI LeMUR framework"
documentation = "https://www.assemblyai.com/docs/lemur"
icon = "AssemblyAI"
inputs = [
SecretStrInput(
name="api_key",
display_name="Assembly API Key",
info="Your AssemblyAI API key. You can get one from https://www.assemblyai.com/",
advanced=False,
),
DataInput(
name="transcription_result",
display_name="Transcription Result",
info="The transcription result from AssemblyAI",
),
MessageInput(
name="prompt",
display_name="Input Prompt",
info="The text to prompt the model",
),
DropdownInput(
name="final_model",
display_name="Final Model",
options=["claude3_5_sonnet", "claude3_opus", "claude3_haiku", "claude3_sonnet"],
value="claude3_5_sonnet",
info="The model that is used for the final prompt after compression is performed",
),
FloatInput(
name="temperature",
display_name="Temperature",
advanced=True,
value=0.0,
info="The temperature to use for the model",
),
IntInput(
name="max_output_size",
display_name=" Max Output Size",
advanced=True,
value=2000,
info="Max output size in tokens, up to 4000",
),
]
outputs = [
Output(display_name="LeMUR Response", name="lemur_response", method="run_lemur"),
]
def run_lemur(self) -> Data:
"""Use the LeMUR task endpoint to input the LLM prompt."""
aai.settings.api_key = self.api_key
# check if it's an error message from the previous step
if self.transcription_result.data.get("error"):
self.status = self.transcription_result.data["error"]
return self.transcription_result
if not self.prompt or not self.prompt.text:
self.status = "No prompt specified"
return Data(data={"error": "No prompt specified"})
try:
transcript = aai.Transcript.get_by_id(self.transcription_result.data["id"])
except Exception as e:
error = f"Getting transcription failed: {str(e)}"
self.status = error
return Data(data={"error": error})
if transcript.status == aai.TranscriptStatus.completed:
try:
result = transcript.lemur.task(
prompt=self.prompt.text,
final_model=self.get_final_model(self.final_model),
temperature=self.temperature,
max_output_size=self.max_output_size,
)
result = Data(data=result.dict())
self.status = result
return result
except Exception as e:
error = f"An Exception happened while calling LeMUR: {str(e)}"
self.status = error
return Data(data={"error": error})
else:
self.status = transcript.error
return Data(data={"error": transcript.error})
def get_final_model(self, model_name: str) -> aai.LemurModel:
if model_name == "claude3_5_sonnet":
return aai.LemurModel.claude3_5_sonnet
elif model_name == "claude3_opus":
return aai.LemurModel.claude3_opus
elif model_name == "claude3_haiku":
return aai.LemurModel.claude3_haiku
elif model_name == "claude3_sonnet":
return aai.LemurModel.claude3_sonnet
else:
raise ValueError(f"Model name not supported: {model_name}")

90
src/backend/base/langflow/components/documentloaders/AssemblyAIListTranscripts.py Normal file
View file

@ -0,0 +1,90 @@
from typing import List
import assemblyai as aai
from langflow.custom import Component
from langflow.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SecretStrInput
from langflow.schema import Data
class AssemblyAIListTranscripts(Component):
display_name = "AssemblyAI List Transcripts"
description = "Retrieve a list of transcripts from AssemblyAI with filtering options"
documentation = "https://www.assemblyai.com/docs"
icon = "AssemblyAI"
inputs = [
SecretStrInput(
name="api_key",
display_name="Assembly API Key",
info="Your AssemblyAI API key. You can get one from https://www.assemblyai.com/",
),
IntInput(
name="limit",
display_name="Limit",
info="Maximum number of transcripts to retrieve (default: 20, use 0 for all)",
value=20,
),
DropdownInput(
name="status_filter",
display_name="Status Filter",
options=["all", "queued", "processing", "completed", "error"],
value="all",
info="Filter by transcript status",
),
MessageTextInput(
name="created_on",
display_name="Created On",
info="Only get transcripts created on this date (YYYY-MM-DD)",
),
BoolInput(
name="throttled_only",
display_name="Throttled Only",
info="Only get throttled transcripts, overrides the status filter",
),
]
outputs = [
Output(display_name="Transcript List", name="transcript_list", method="list_transcripts"),
]
def list_transcripts(self) -> List[Data]:
aai.settings.api_key = self.api_key
params = aai.ListTranscriptParameters()
if self.limit:
params.limit = self.limit
if self.status_filter != "all":
params.status = self.status_filter
if self.created_on and self.created_on.text:
params.created_on = self.created_on.text
if self.throttled_only:
params.throttled_only = True
try:
transcriber = aai.Transcriber()
def convert_page_to_data_list(page):
return [Data(**t.dict()) for t in page.transcripts]
if self.limit == 0:
# paginate over all pages
params.limit = 100
page = transcriber.list_transcripts(params)
transcripts = convert_page_to_data_list(page)
while page.page_details.before_id_of_prev_url is not None:
params.before_id = page.page_details.before_id_of_prev_url
page = transcriber.list_transcripts(params)
transcripts.extend(convert_page_to_data_list(page))
else:
# just one page
page = transcriber.list_transcripts(params)
transcripts = convert_page_to_data_list(page)
self.status = transcripts
return transcripts
except Exception as e:
error_data = Data(data={"error": f"An error occurred: {str(e)}"})
self.status = [error_data]
return [error_data]

60
src/backend/base/langflow/components/documentloaders/AssemblyAIPollTranscript.py Normal file
View file

@ -0,0 +1,60 @@
import assemblyai as aai
from langflow.custom import Component
from langflow.io import DataInput, FloatInput, Output, SecretStrInput
from langflow.schema import Data
class AssemblyAITranscriptionJobPoller(Component):
display_name = "AssemblyAI Poll Transcript"
description = "Poll for the status of a transcription job using AssemblyAI"
documentation = "https://www.assemblyai.com/docs"
icon = "AssemblyAI"
inputs = [
SecretStrInput(
name="api_key",
display_name="Assembly API Key",
info="Your AssemblyAI API key. You can get one from https://www.assemblyai.com/",
),
DataInput(
name="transcript_id",
display_name="Transcript ID",
info="The ID of the transcription job to poll",
),
FloatInput(
name="polling_interval",
display_name="Polling Interval",
value=3.0,
info="The polling interval in seconds",
),
]
outputs = [
Output(display_name="Transcription Result", name="transcription_result", method="poll_transcription_job"),
]
def poll_transcription_job(self) -> Data:
"""Polls the transcription status until completion and returns the Data."""
aai.settings.api_key = self.api_key
aai.settings.polling_interval = self.polling_interval
# check if it's an error message from the previous step
if self.transcript_id.data.get("error"):
self.status = self.transcript_id.data["error"]
return self.transcript_id
try:
transcript = aai.Transcript.get_by_id(self.transcript_id.data["transcript_id"])
except Exception as e:
error = f"Getting transcription failed: {str(e)}"
self.status = error
return Data(data={"error": error})
if transcript.status == aai.TranscriptStatus.completed:
data = Data(data=transcript.json_response)
self.status = data
return data
else:
self.status = transcript.error
return Data(data={"error": transcript.error})

181
src/backend/base/langflow/components/documentloaders/AssemblyAIStartTranscript.py Normal file
View file

@ -0,0 +1,181 @@
import os
import assemblyai as aai
from loguru import logger
from langflow.custom import Component
from langflow.io import BoolInput, DropdownInput, FileInput, MessageTextInput, Output, SecretStrInput
from langflow.schema import Data
class AssemblyAITranscriptionJobCreator(Component):
display_name = "AssemblyAI Start Transcript"
description = "Create a transcription job for an audio file using AssemblyAI with advanced options"
documentation = "https://www.assemblyai.com/docs"
icon = "AssemblyAI"
inputs = [
SecretStrInput(
name="api_key",
display_name="Assembly API Key",
info="Your AssemblyAI API key. You can get one from https://www.assemblyai.com/",
),
FileInput(
name="audio_file",
display_name="Audio File",
file_types=[
"3ga",
"8svx",
"aac",
"ac3",
"aif",
"aiff",
"alac",
"amr",
"ape",
"au",
"dss",
"flac",
"flv",
"m4a",
"m4b",
"m4p",
"m4r",
"mp3",
"mpga",
"ogg",
"oga",
"mogg",
"opus",
"qcp",
"tta",
"voc",
"wav",
"wma",
"wv",
"webm",
"mts",
"m2ts",
"ts",
"mov",
"mp2",
"mp4",
"m4p",
"m4v",
"mxf",
],
info="The audio file to transcribe",
),
MessageTextInput(
name="audio_file_url",
display_name="Audio File URL",
info="The URL of the audio file to transcribe (Can be used instead of a File)",
advanced=True,
),
DropdownInput(
name="speech_model",
display_name="Speech Model",
options=[
"best",
"nano",
],
value="best",
info="The speech model to use for the transcription",
),
BoolInput(
name="language_detection",
display_name="Automatic Language Detection",
info="Enable automatic language detection",
),
MessageTextInput(
name="language_code",
display_name="Language",
info="""
The language of the audio file. Can be set manually if automatic language detection is disabled.
See https://www.assemblyai.com/docs/getting-started/supported-languages for a list of supported language codes.
""",
),
BoolInput(
name="speaker_labels",
display_name="Enable Speaker Labels",
info="Enable speaker diarization",
),
MessageTextInput(
name="speakers_expected",
display_name="Expected Number of Speakers",
info="Set the expected number of speakers (optional, enter a number)",
advanced=True,
),
BoolInput(
name="punctuate",
display_name="Punctuate",
info="Enable automatic punctuation",
advanced=True,
value=True,
),
BoolInput(
name="format_text",
display_name="Format Text",
info="Enable text formatting",
advanced=True,
value=True,
),
]
outputs = [
Output(display_name="Transcript ID", name="transcript_id", method="create_transcription_job"),
]
def create_transcription_job(self) -> Data:
aai.settings.api_key = self.api_key
# Convert speakers_expected to int if it's not empty
speakers_expected = None
if self.speakers_expected and self.speakers_expected.strip():
try:
speakers_expected = int(self.speakers_expected)
except ValueError:
self.status = "Error: Expected Number of Speakers must be a valid integer"
return Data(data={"error": "Error: Expected Number of Speakers must be a valid integer"})
language_code = self.language_code if self.language_code else None
config = aai.TranscriptionConfig(
speech_model=self.speech_model,
language_detection=self.language_detection,
language_code=language_code,
speaker_labels=self.speaker_labels,
speakers_expected=speakers_expected,
punctuate=self.punctuate,
format_text=self.format_text,
)
audio = None
if self.audio_file:
if self.audio_file_url:
logger.warning("Both an audio file an audio URL were specified. The audio URL was ignored.")
# Check if the file exists
if not os.path.exists(self.audio_file):
self.status = "Error: Audio file not found"
return Data(data={"error": "Error: Audio file not found"})
audio = self.audio_file
elif self.audio_file_url:
audio = self.audio_file_url
else:
self.status = "Error: Either an audio file or an audio URL must be specified"
return Data(data={"error": "Error: Either an audio file or an audio URL must be specified"})
try:
transcript = aai.Transcriber().submit(audio, config=config)
if transcript.error:
self.status = transcript.error
return Data(data={"error": transcript.error})
else:
result = Data(data={"transcript_id": transcript.id})
self.status = result
return result
except Exception as e:
self.status = f"An error occurred: {str(e)}"
return Data(data={"error": f"An error occurred: {str(e)}"})

3
src/backend/base/langflow/components/prompts/LangChainHubPrompt.py
View file

@ -1,12 +1,13 @@
import re
from typing import List
from langchain_core.prompts import HumanMessagePromptTemplate
from langflow.custom import Component
from langflow.inputs import DefaultPromptField, SecretStrInput, StrInput
from langflow.io import Output
from langflow.schema.message import Message
from langchain_core.prompts import HumanMessagePromptTemplate
class LangChainHubPromptComponent(Component):
display_name: str = "LangChain Hub"

2322
src/frontend/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

20
src/frontend/src/icons/AssemblyAI/AssemblyAI.jsx Normal file
View file

@ -0,0 +1,20 @@
const AssemblyAISVG = (props) => (
<svg
width="501"
height="434"
viewBox="0 0 501 434"
fill="none"
xmlns="http://www.w3.org/2000/svg"
{...props}
>
<path
d="M221.202 0.944641C189.435 0.944641 160.93 20.4579 149.437 50.0725L0.462402 433.945H113.632L230.894 131.791H230.943C233.886 124.427 241.085 119.224 249.5 119.224C257.915 119.224 265.114 124.427 268.057 131.791H283.681V70.5011H254.679L281.673 0.944641H221.202Z"
fill="#213ED7"
/>
<path
d="M149.445 50.0726C160.471 21.6619 187.153 2.54782 217.352 1.04075L217.315 0.944641H279.722C311.489 0.944641 339.993 20.4579 351.486 50.0725L500.461 433.945H385.356L240.893 61.6995C232.622 43.4564 214.251 30.7668 192.917 30.7668C171.53 30.7668 153.122 43.5188 144.88 61.834L149.445 50.0726Z"
fill="#566DE8"
/>
</svg>
);
export default AssemblyAISVG;

4
src/frontend/src/icons/AssemblyAI/AssemblyAI.svg Normal file
View file

@ -0,0 +1,4 @@
<svg width="501" height="434" viewBox="0 0 501 434" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M221.202 0.944641C189.435 0.944641 160.93 20.4579 149.437 50.0725L0.462402 433.945H113.632L230.894 131.791H230.943C233.886 124.427 241.085 119.224 249.5 119.224C257.915 119.224 265.114 124.427 268.057 131.791H283.681V70.5011H254.679L281.673 0.944641H221.202Z" fill="#213ED7"/>
<path d="M149.445 50.0726C160.471 21.6619 187.153 2.54782 217.352 1.04075L217.315 0.944641H279.722C311.489 0.944641 339.993 20.4579 351.486 50.0725L500.461 433.945H385.356L240.893 61.6995C232.622 43.4564 214.251 30.7668 192.917 30.7668C171.53 30.7668 153.122 43.5188 144.88 61.834L149.445 50.0726Z" fill="#566DE8"/>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 709 B

9
src/frontend/src/icons/AssemblyAI/index.tsx Normal file
View file

@ -0,0 +1,9 @@
import React, { forwardRef } from "react";
import AssemblyAISVG from "./AssemblyAI";
export const AssemblyAIIcon = forwardRef<
SVGSVGElement,
React.PropsWithChildren<{}>
>((props, ref) => {
return <AssemblyAISVG ref={ref} {...props} />;
});

2
src/frontend/src/utils/styleUtils.ts
View file

@ -175,6 +175,7 @@ import { FaApple, FaDiscord, FaGithub } from "react-icons/fa";
import { AWSIcon } from "../icons/AWS";
import { AirbyteIcon } from "../icons/Airbyte";
import { AnthropicIcon } from "../icons/Anthropic";
import { AssemblyAIIcon } from "../icons/AssemblyAI";
import { AstraDBIcon } from "../icons/AstraDB";
import { AzureIcon } from "../icons/Azure";
import { BingIcon } from "../icons/Bing";
@ -389,6 +390,7 @@ export const nodeIconsLucide: iconsType = {
Amazon: AWSIcon,
Anthropic: AnthropicIcon,
ChatAnthropic: AnthropicIcon,
AssemblyAI: AssemblyAIIcon,
AstraDB: AstraDBIcon,
BingSearchAPIWrapper: BingIcon,
BingSearchRun: BingIcon,