tts models support (#2033)

Co-authored-by: luowei <glpat-EjySCyNjWiLqAED-YmwM>
Co-authored-by: crazywoola <427733928@qq.com>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: Yeuoly <45712896+Yeuoly@users.noreply.github.com>
This commit is contained in:
Charlie.Wei 2024-01-24 01:05:37 +08:00 • committed by GitHub
commit 6355e61eb8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
86 changed files with 1645 additions and 133 deletions

View file

@ -13,6 +13,7 @@ import MoreInfo from '../more-info'
import CopyBtn from '../copy-btn'
import Thought from '../thought'
import Citation from '../citation'
import AudioBtn from '@/app/components/base/audio-btn'
import { randomString } from '@/utils'
import type { MessageRating } from '@/models/log'
import Tooltip from '@/app/components/base/tooltip'
@ -53,6 +54,7 @@ export type IAnswerProps = {
dataSets?: DataSet[]
isShowCitation?: boolean
isShowCitationHitInfo?: boolean
isShowTextToSpeech?: boolean
// Annotation props
supportAnnotation?: boolean
appId?: string
@ -75,6 +77,7 @@ const Answer: FC<IAnswerProps> = ({
citation,
isShowCitation,
isShowCitationHitInfo = false,
isShowTextToSpeech,
supportAnnotation,
appId,
question,
@ -322,7 +325,13 @@ const Answer: FC<IAnswerProps> = ({
className={cn(s.copyBtn, 'mr-1')}
/>
)}
{(supportAnnotation && !item.isOpeningStatement) && (
{!item.isOpeningStatement && isShowTextToSpeech && (
<AudioBtn
value={content}
className={cn(s.playBtn, 'mr-1')}
/>
)}
{(!item.isOpeningStatement && supportAnnotation) && (
<AnnotationCtrlBtn
appId={appId!}
messageId={id}

View file

@ -57,6 +57,7 @@ export type IChatProps = {
isShowSuggestion?: boolean
suggestionList?: string[]
isShowSpeechToText?: boolean
isShowTextToSpeech?: boolean
isShowCitation?: boolean
answerIcon?: ReactNode
isShowConfigElem?: boolean
@ -89,6 +90,7 @@ const Chat: FC<IChatProps> = ({
isShowSuggestion,
suggestionList,
isShowSpeechToText,
isShowTextToSpeech,
isShowCitation,
answerIcon,
isShowConfigElem,
@ -222,6 +224,7 @@ const Chat: FC<IChatProps> = ({
dataSets={dataSets}
isShowCitation={isShowCitation}
isShowCitationHitInfo={isShowCitationHitInfo}
isShowTextToSpeech={isShowTextToSpeech}
supportAnnotation={supportAnnotation}
appId={appId}
question={chatList[index - 1]?.content}

View file

@ -39,6 +39,7 @@
}
.copyBtn,
.playBtn,
.annotationBtn {
display: none;
}
@ -65,6 +66,7 @@
}
.answerWrap:hover .copyBtn,
.answerWrap:hover .playBtn,
.answerWrap:hover .annotationBtn {
display: block;
}

View file

@ -29,6 +29,11 @@
background-image: url(./preview-imgs/speech-to-text.svg);
}
.textToSpeechPreview {
@apply shadow-lg rounded-lg;
background-image: url(./preview-imgs/text-to-audio-preview-assistant@2x.png);
}
.citationPreview {
background-image: url(./preview-imgs/citation.svg);
}
}

View file

@ -7,7 +7,7 @@ import MoreLikeThisIcon from '../../../base/icons/more-like-this-icon'
import FeatureItem from './feature-item'
import Modal from '@/app/components/base/modal'
import SuggestedQuestionsAfterAnswerIcon from '@/app/components/app/configuration/base/icons/suggested-questions-after-answer-icon'
import { Microphone01 } from '@/app/components/base/icons/src/vender/solid/mediaAndDevices'
import { Microphone01, Speaker } from '@/app/components/base/icons/src/vender/solid/mediaAndDevices'
import { Citations } from '@/app/components/base/icons/src/vender/solid/editor'
import { FileSearch02 } from '@/app/components/base/icons/src/vender/solid/files'
import { MessageFast } from '@/app/components/base/icons/src/vender/solid/communication'
@ -16,6 +16,7 @@ type IConfig = {
moreLikeThis: boolean
suggestedQuestionsAfterAnswer: boolean
speechToText: boolean
textToSpeech: boolean
citation: boolean
moderation: boolean
annotation: boolean
@ -27,6 +28,7 @@ export type IChooseFeatureProps = {
config: IConfig
isChatApp: boolean
onChange: (key: string, value: boolean) => void
showTextToSpeechItem?: boolean
showSpeechToTextItem?: boolean
}
@ -42,6 +44,7 @@ const ChooseFeature: FC<IChooseFeatureProps> = ({
isChatApp,
config,
onChange,
showTextToSpeechItem,
showSpeechToTextItem,
}) => {
const { t } = useTranslation()
@ -78,6 +81,18 @@ const ChooseFeature: FC<IChooseFeatureProps> = ({
value={config.suggestedQuestionsAfterAnswer}
onChange={value => onChange('suggestedQuestionsAfterAnswer', value)}
/>
{
showTextToSpeechItem && (
<FeatureItem
icon={<Speaker className='w-4 h-4 text-[#7839EE]' />}
previewImgClassName='textToSpeechPreview'
title={t('appDebug.feature.textToSpeech.title')}
description={t('appDebug.feature.textToSpeech.description')}
value={config.textToSpeech}
onChange={value => onChange('textToSpeech', value)}
/>
)
}
{
showSpeechToTextItem && (
<FeatureItem
@ -114,6 +129,18 @@ const ChooseFeature: FC<IChooseFeatureProps> = ({
value={config.moreLikeThis}
onChange={value => onChange('moreLikeThis', value)}
/>
{
showTextToSpeechItem && (
<FeatureItem
icon={<Speaker className='w-4 h-4 text-[#7839EE]' />}
previewImgClassName='textToSpeechPreview'
title={t('appDebug.feature.textToSpeech.title')}
description={t('appDebug.feature.textToSpeech.description')}
value={config.textToSpeech}
onChange={value => onChange('textToSpeech', value)}
/>
)
}
</>
</FeatureGroup>
)}

View file

@ -9,6 +9,8 @@ function useFeature({
setSuggestedQuestionsAfterAnswer,
speechToText,
setSpeechToText,
textToSpeech,
setTextToSpeech,
citation,
setCitation,
annotation,
@ -24,6 +26,8 @@ function useFeature({
setSuggestedQuestionsAfterAnswer: (suggestedQuestionsAfterAnswer: boolean) => void
speechToText: boolean
setSpeechToText: (speechToText: boolean) => void
textToSpeech: boolean
setTextToSpeech: (textToSpeech: boolean) => void
citation: boolean
setCitation: (citation: boolean) => void
annotation: boolean
@ -48,6 +52,7 @@ function useFeature({
moreLikeThis,
suggestedQuestionsAfterAnswer,
speechToText,
textToSpeech,
citation,
annotation,
moderation,
@ -69,6 +74,9 @@ function useFeature({
case 'speechToText':
setSpeechToText(value)
break
case 'textToSpeech':
setTextToSpeech(value)
break
case 'citation':
setCitation(value)
break

View file

@ -19,7 +19,7 @@ import AdvancedModeWaring from '@/app/components/app/configuration/prompt-mode/a
import ConfigContext from '@/context/debug-configuration'
import ConfigPrompt from '@/app/components/app/configuration/config-prompt'
import ConfigVar from '@/app/components/app/configuration/config-var'
import { type CitationConfig, type ModelConfig, type ModerationConfig, type MoreLikeThisConfig, PromptMode, type PromptVariable, type SpeechToTextConfig, type SuggestedQuestionsAfterAnswerConfig } from '@/models/debug'
import { type CitationConfig, type ModelConfig, type ModerationConfig, type MoreLikeThisConfig, PromptMode, type PromptVariable, type SpeechToTextConfig, type SuggestedQuestionsAfterAnswerConfig, type TextToSpeechConfig } from '@/models/debug'
import { AppType, ModelModeType } from '@/types/app'
import { useModalContext } from '@/context/modal-context'
import ConfigParamModal from '@/app/components/app/configuration/toolbox/annotation/config-param-modal'
@ -51,6 +51,8 @@ const Config: FC = () => {
setSuggestedQuestionsAfterAnswerConfig,
speechToTextConfig,
setSpeechToTextConfig,
textToSpeechConfig,
setTextToSpeechConfig,
citationConfig,
setCitationConfig,
annotationConfig,
@ -60,6 +62,7 @@ const Config: FC = () => {
} = useContext(ConfigContext)
const isChatApp = mode === AppType.chat
const { data: speech2textDefaultModel } = useDefaultModel(4)
const { data: text2speechDefaultModel } = useDefaultModel(5)
const { setShowModerationSettingModal } = useModalContext()
const promptTemplate = modelConfig.configs.prompt_template
@ -111,6 +114,12 @@ const Config: FC = () => {
draft.enabled = value
}))
},
textToSpeech: textToSpeechConfig.enabled,
setTextToSpeech: (value) => {
setTextToSpeechConfig(produce(textToSpeechConfig, (draft: TextToSpeechConfig) => {
draft.enabled = value
}))
},
citation: citationConfig.enabled,
setCitation: (value) => {
setCitationConfig(produce(citationConfig, (draft: CitationConfig) => {
@ -173,7 +182,7 @@ const Config: FC = () => {
setAnnotationConfig,
})
const hasChatConfig = isChatApp && (featureConfig.openingStatement || featureConfig.suggestedQuestionsAfterAnswer || (featureConfig.speechToText && !!speech2textDefaultModel) || featureConfig.citation)
const hasChatConfig = isChatApp && (featureConfig.openingStatement || featureConfig.suggestedQuestionsAfterAnswer || (featureConfig.speechToText && !!speech2textDefaultModel) || (featureConfig.textToSpeech && !!text2speechDefaultModel) || featureConfig.citation)
const hasToolbox = moderationConfig.enabled || featureConfig.annotation
const wrapRef = useRef<HTMLDivElement>(null)
@ -207,6 +216,7 @@ const Config: FC = () => {
config={featureConfig}
onChange={handleFeatureChange}
showSpeechToTextItem={!!speech2textDefaultModel}
showTextToSpeechItem={!!text2speechDefaultModel}
/>
)}
@ -255,16 +265,21 @@ const Config: FC = () => {
}
}
isShowSuggestedQuestionsAfterAnswer={featureConfig.suggestedQuestionsAfterAnswer}
isShowTextToSpeech={featureConfig.textToSpeech && !!text2speechDefaultModel}
isShowSpeechText={featureConfig.speechToText && !!speech2textDefaultModel}
isShowCitation={featureConfig.citation}
/>
)
}
{/* TextnGeneration config */}
{moreLikeThisConfig.enabled && (
<ExperienceEnchanceGroup />
)}
{/* TextnGeneration config */}{
!hasChatConfig && (
<ExperienceEnchanceGroup
isShowMoreLike={moreLikeThisConfig.enabled}
isShowTextToSpeech={featureConfig.textToSpeech && !!text2speechDefaultModel}
/>
)
}
{/* Toolbox */}
{

View file

@ -56,6 +56,7 @@ const Debug: FC<IDebug> = ({
suggestedQuestions,
suggestedQuestionsAfterAnswerConfig,
speechToTextConfig,
textToSpeechConfig,
citationConfig,
moderationConfig,
moreLikeThisConfig,
@ -73,6 +74,7 @@ const Debug: FC<IDebug> = ({
annotationConfig,
} = useContext(ConfigContext)
const { data: speech2textDefaultModel } = useDefaultModel(4)
const { data: text2speechDefaultModel } = useDefaultModel(5)
const [chatList, setChatList, getChatList] = useGetState<IChatItem[]>([])
const chatListDomRef = useRef<HTMLDivElement>(null)
const { data: fileUploadConfigResponse } = useSWR({ url: '/files/upload' }, fetchFileUploadConfig)
@ -233,6 +235,9 @@ const Debug: FC<IDebug> = ({
setChatList(newListWithAnswer)
}
const postModelConfig: BackendModelConfig = {
text_to_speech: {
enabled: false,
},
pre_prompt: !isAdvancedMode ? modelConfig.configs.prompt_template : '',
prompt_type: promptMode,
chat_prompt_config: {},
@ -514,6 +519,9 @@ const Debug: FC<IDebug> = ({
const contextVar = modelConfig.configs.prompt_variables.find(item => item.is_context_var)?.key
const postModelConfig: BackendModelConfig = {
text_to_speech: {
enabled: false,
},
pre_prompt: !isAdvancedMode ? modelConfig.configs.prompt_template : '',
prompt_type: promptMode,
chat_prompt_config: {},
@ -657,6 +665,7 @@ const Debug: FC<IDebug> = ({
isShowSuggestion={doShowSuggestion}
suggestionList={suggestQuestions}
isShowSpeechToText={speechToTextConfig.enabled && !!speech2textDefaultModel}
isShowTextToSpeech={textToSpeechConfig.enabled && !!text2speechDefaultModel}
isShowCitation={citationConfig.enabled}
isShowCitationHitInfo
isShowPromptLog
@ -682,6 +691,7 @@ const Debug: FC<IDebug> = ({
className="mt-2"
content={completionRes}
isLoading={!completionRes && isResponsing}
isShowTextToSpeech={textToSpeechConfig.enabled && !!text2speechDefaultModel}
isResponsing={isResponsing}
isInstalledApp={false}
messageId={messageId}

View file

@ -7,6 +7,7 @@ import type { IOpeningStatementProps } from './opening-statement'
import OpeningStatement from './opening-statement'
import SuggestedQuestionsAfterAnswer from './suggested-questions-after-answer'
import SpeechToText from './speech-to-text'
import TextToSpeech from './text-to-speech'
import Citation from './citation'
/*
* Include
@ -19,6 +20,7 @@ type ChatGroupProps = {
openingStatementConfig: IOpeningStatementProps
isShowSuggestedQuestionsAfterAnswer: boolean
isShowSpeechText: boolean
isShowTextToSpeech: boolean
isShowCitation: boolean
}
const ChatGroup: FC<ChatGroupProps> = ({
@ -26,6 +28,7 @@ const ChatGroup: FC<ChatGroupProps> = ({
openingStatementConfig,
isShowSuggestedQuestionsAfterAnswer,
isShowSpeechText,
isShowTextToSpeech,
isShowCitation,
}) => {
const { t } = useTranslation()
@ -40,6 +43,11 @@ const ChatGroup: FC<ChatGroupProps> = ({
{isShowSuggestedQuestionsAfterAnswer && (
<SuggestedQuestionsAfterAnswer />
)}
{
isShowTextToSpeech && (
<TextToSpeech />
)
}
{
isShowSpeechText && (
<SpeechToText />

View file

@ -4,7 +4,7 @@ import { useTranslation } from 'react-i18next'
import Panel from '@/app/components/app/configuration/base/feature-panel'
import { Microphone01 } from '@/app/components/base/icons/src/vender/solid/mediaAndDevices'
const SuggestedQuestionsAfterAnswer: FC = () => {
const SpeechToTextConfig: FC = () => {
const { t } = useTranslation()
return (
@ -22,4 +22,4 @@ const SuggestedQuestionsAfterAnswer: FC = () => {
/>
)
}
export default React.memo(SuggestedQuestionsAfterAnswer)
export default React.memo(SpeechToTextConfig)

View file

@ -0,0 +1,25 @@
'use client'
import React, { type FC } from 'react'
import { useTranslation } from 'react-i18next'
import Panel from '@/app/components/app/configuration/base/feature-panel'
import { Speaker } from '@/app/components/base/icons/src/vender/solid/mediaAndDevices'
const TextToSpeech: FC = () => {
const { t } = useTranslation()
return (
<Panel
title={
<div className='flex items-center gap-2'>
<div>{t('appDebug.feature.textToSpeech.title')}</div>
</div>
}
headerIcon={<Speaker className='w-4 h-4 text-[#7839EE]' />}
headerRight={
<div className='text-xs text-gray-500'>{t('appDebug.feature.textToSpeech.resDes')}</div>
}
noBodySpacing
/>
)
}
export default React.memo(TextToSpeech)

View file

@ -3,19 +3,40 @@ import type { FC } from 'react'
import React from 'react'
import { useTranslation } from 'react-i18next'
import GroupName from '../../base/group-name'
import TextToSpeech from '../chat-group/text-to-speech'
import MoreLikeThis from './more-like-this'
/*
* Include
* 1. More like this
*/
const ExperienceEnchanceGroup: FC = () => {
type ExperienceGroupProps = {
isShowTextToSpeech: boolean
isShowMoreLike: boolean
}
const ExperienceEnchanceGroup: FC<ExperienceGroupProps> = ({
isShowTextToSpeech,
isShowMoreLike,
}) => {
const { t } = useTranslation()
return (
<div className='mt-7'>
<GroupName name={t('appDebug.feature.groupExperience.title')} />
<MoreLikeThis />
<GroupName name={t('appDebug.feature.groupExperience.title')}/>
<div className='space-y-3'>
{
isShowMoreLike && (
<MoreLikeThis/>
)
}
{
isShowTextToSpeech && (
<TextToSpeech/>
)
}
</div>
</div>
)
}

View file

@ -91,6 +91,9 @@ const Configuration: FC = () => {
const [speechToTextConfig, setSpeechToTextConfig] = useState<MoreLikeThisConfig>({
enabled: false,
})
const [textToSpeechConfig, setTextToSpeechConfig] = useState<MoreLikeThisConfig>({
enabled: false,
})
const [citationConfig, setCitationConfig] = useState<MoreLikeThisConfig>({
enabled: false,
})
@ -140,6 +143,7 @@ const Configuration: FC = () => {
more_like_this: null,
suggested_questions_after_answer: null,
speech_to_text: null,
text_to_speech: null,
retriever_resource: null,
sensitive_word_avoidance: null,
dataSets: [],
@ -232,6 +236,9 @@ const Configuration: FC = () => {
setSpeechToTextConfig(modelConfig.speech_to_text || {
enabled: false,
})
setTextToSpeechConfig(modelConfig.text_to_speech || {
enabled: false,
})
setCitationConfig(modelConfig.retriever_resource || {
enabled: false,
})
@ -396,6 +403,9 @@ const Configuration: FC = () => {
if (modelConfig.speech_to_text)
setSpeechToTextConfig(modelConfig.speech_to_text)
if (modelConfig.text_to_speech)
setTextToSpeechConfig(modelConfig.text_to_speech)
if (modelConfig.retriever_resource)
setCitationConfig(modelConfig.retriever_resource)
@ -444,6 +454,7 @@ const Configuration: FC = () => {
more_like_this: modelConfig.more_like_this,
suggested_questions_after_answer: modelConfig.suggested_questions_after_answer,
speech_to_text: modelConfig.speech_to_text,
text_to_speech: modelConfig.text_to_speech,
retriever_resource: modelConfig.retriever_resource,
sensitive_word_avoidance: modelConfig.sensitive_word_avoidance,
external_data_tools: modelConfig.external_data_tools,
@ -559,6 +570,7 @@ const Configuration: FC = () => {
more_like_this: moreLikeThisConfig,
suggested_questions_after_answer: suggestedQuestionsAfterAnswerConfig,
speech_to_text: speechToTextConfig,
text_to_speech: textToSpeechConfig,
retriever_resource: citationConfig,
sensitive_word_avoidance: moderationConfig,
agent_mode: {
@ -593,6 +605,7 @@ const Configuration: FC = () => {
draft.more_like_this = moreLikeThisConfig
draft.suggested_questions_after_answer = suggestedQuestionsAfterAnswerConfig
draft.speech_to_text = speechToTextConfig
draft.text_to_speech = textToSpeechConfig
draft.retriever_resource = citationConfig
draft.dataSets = dataSets
})
@ -662,6 +675,8 @@ const Configuration: FC = () => {
setSuggestedQuestionsAfterAnswerConfig,
speechToTextConfig,
setSpeechToTextConfig,
textToSpeechConfig,
setTextToSpeechConfig,
citationConfig,
setCitationConfig,
annotationConfig,

View file

@ -297,6 +297,7 @@ function DetailPanel<T extends ChatConversationFullDetailResponse | CompletionCo
feedback={detail.message.feedbacks.find((item: any) => item.from_source === 'admin')}
onFeedback={feedback => onFeedback(detail.message.id, feedback)}
supportAnnotation
isShowTextToSpeech
appId={appDetail?.id}
varList={varList}
/>
@ -310,6 +311,7 @@ function DetailPanel<T extends ChatConversationFullDetailResponse | CompletionCo
displayScene='console'
isShowPromptLog
supportAnnotation
isShowTextToSpeech
appId={appDetail?.id}
onChatListChange={setItems}
/>

View file

@ -12,6 +12,7 @@ import PromptLog from '@/app/components/app/chat/log'
import { Markdown } from '@/app/components/base/markdown'
import Loading from '@/app/components/base/loading'
import Toast from '@/app/components/base/toast'
import AudioBtn from '@/app/components/base/audio-btn'
import type { Feedbacktype } from '@/app/components/app/chat/type'
import { fetchMoreLikeThis, updateFeedback } from '@/service/share'
import { Clipboard, File02 } from '@/app/components/base/icons/src/vender/line/files'
@ -45,6 +46,7 @@ export type IGenerationItemProps = {
controlClearMoreLikeThis?: number
supportFeedback?: boolean
supportAnnotation?: boolean
isShowTextToSpeech?: boolean
appId?: string
varList?: { label: string; value: string | number | object }[]
}
@ -90,6 +92,7 @@ const GenerationItem: FC<IGenerationItemProps> = ({
controlClearMoreLikeThis,
supportFeedback,
supportAnnotation,
isShowTextToSpeech,
appId,
varList,
}) => {
@ -124,6 +127,7 @@ const GenerationItem: FC<IGenerationItemProps> = ({
isLoading: isQuerying,
feedback: childFeedback,
onSave,
isShowTextToSpeech,
isMobile,
isInstalledApp,
installedAppId,
@ -366,8 +370,17 @@ const GenerationItem: FC<IGenerationItemProps> = ({
<div className='ml-1'>
{ratingContent}
</div>
)
}
)}
{isShowTextToSpeech && (
<>
<div className='ml-2 mr-2 h-[14px] w-[1px] bg-gray-200'></div>
<AudioBtn
value={content}
className={'mr-1'}
/>
</>
)}
</div>
<div className='text-xs text-gray-500'>{content?.length} {t('common.unit.char')}</div>
</div>

View file

@ -9,9 +9,11 @@ import type { SavedMessage } from '@/models/debug'
import { Markdown } from '@/app/components/base/markdown'
import { SimpleBtn, copyIcon } from '@/app/components/app/text-generate/item'
import Toast from '@/app/components/base/toast'
import AudioBtn from '@/app/components/base/audio-btn'
export type ISavedItemsProps = {
className?: string
isShowTextToSpeech?: boolean
list: SavedMessage[]
onRemove: (id: string) => void
onStartCreateContent: () => void
@ -25,6 +27,7 @@ const removeIcon = (
const SavedItems: FC<ISavedItemsProps> = ({
className,
isShowTextToSpeech,
list,
onRemove,
onStartCreateContent,
@ -69,6 +72,16 @@ const SavedItems: FC<ISavedItemsProps> = ({
{removeIcon}
<div>{t('common.operation.remove')}</div>
</SimpleBtn>
{isShowTextToSpeech && (
<>
<div className='ml-2 mr-2 h-[14px] w-[1px] bg-gray-200'></div>
<AudioBtn
value={answer}
className={'mr-1'}
/>
</>
)}
</div>
<div className='text-xs text-gray-500'>{answer?.length} {t('common.unit.char')}</div>
</div>