jyapayne/dify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Feat/chat support voice input (#532)

Browse source
This commit is contained in:
zxhlyh 2023-07-07 17:50:42 +08:00 • committed by GitHub
commit a03a92e9db
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
70 changed files with 1420 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

44
web/app/components/app/chat/index.tsx
View file

@ -3,6 +3,7 @@ import type { FC } from 'react'
import React, { useEffect, useLayoutEffect, useRef, useState } from 'react'
import { useContext } from 'use-context-selector'
import cn from 'classnames'
import Recorder from 'js-audio-recorder'
import { HandThumbDownIcon, HandThumbUpIcon } from '@heroicons/react/24/outline'
import { UserCircleIcon } from '@heroicons/react/24/solid'
import { useTranslation } from 'react-i18next'
@ -19,6 +20,10 @@ import AppContext from '@/context/app-context'
import { Markdown } from '@/app/components/base/markdown'
import { formatNumber } from '@/utils/format'
import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
import VoiceInput from '@/app/components/base/voice-input'
import { Microphone01 } from '@/app/components/base/icons/src/vender/line/mediaAndDevices'
import { Microphone01 as Microphone01Solid } from '@/app/components/base/icons/src/vender/solid/mediaAndDevices'
import { XCircle } from '@/app/components/base/icons/src/vender/solid/general'
const stopIcon = (
<svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg">
@ -59,6 +64,7 @@ export type IChatProps = {
controlFocus?: number
isShowSuggestion?: boolean
suggestionList?: string[]
isShowSpeechToText?: boolean
}
export type MessageMore = {
@ -421,6 +427,7 @@ const Chat: FC<IChatProps> = ({
controlFocus,
isShowSuggestion,
suggestionList,
isShowSpeechToText,
}) => {
const { t } = useTranslation()
const { notify } = useContext(ToastContext)
@ -488,6 +495,15 @@ const Chat: FC<IChatProps> = ({
}
}, [suggestionList])
const [voiceInputShow, setVoiceInputShow] = useState(false)
const handleVoiceInputShow = () => {
(Recorder as any).getPermission().then(() => {
setVoiceInputShow(true)
}, () => {
logError(t('common.voiceInput.notAllow'))
})
}
return (
<div className={cn('px-3.5', 'h-full')}>
{/* Chat List */}
@ -565,6 +581,26 @@ const Chat: FC<IChatProps> = ({
/>
<div className="absolute top-0 right-2 flex items-center h-[48px]">
<div className={`${s.count} mr-4 h-5 leading-5 text-sm bg-gray-50 text-gray-500`}>{query.trim().length}</div>
{
query
? (
<div className='flex justify-center items-center w-8 h-8 cursor-pointer hover:bg-gray-100 rounded-lg' onClick={() => setQuery('')}>
<XCircle className='w-4 h-4 text-[#98A2B3]' />
</div>
)
: isShowSpeechToText
? (
<div
className='group flex justify-center items-center w-8 h-8 hover:bg-primary-50 rounded-lg cursor-pointer'
onClick={handleVoiceInputShow}
>
<Microphone01 className='block w-4 h-4 text-gray-500 group-hover:hidden' />
<Microphone01Solid className='hidden w-4 h-4 text-primary-600 group-hover:block' />
</div>
)
: null
}
<div className='mx-2 w-[1px] h-4 bg-black opacity-5' />
{isMobile
? sendBtn
: (
@ -581,6 +617,14 @@ const Chat: FC<IChatProps> = ({
</Tooltip>
)}
</div>
{
voiceInputShow && (
<VoiceInput
onCancel={() => setVoiceInputShow(false)}
onConverted={text => setQuery(text)}
/>
)
}
</div>
</div>
)

2
web/app/components/app/chat/style.module.css
View file

@ -79,7 +79,7 @@
.textArea {
padding-top: 13px;
padding-bottom: 13px;
padding-right: 90px;
padding-right: 130px;
border-radius: 12px;
line-height: 20px;
background-color: #fff;

100
web/app/components/app/configuration/config/feature/choose-feature/feature-item/preview-imgs/speech-to-text.svg Normal file
View file

File diff suppressed because one or more lines are too long
Side by side

After

Width:  |  Height:  |  Size: 84 KiB

4
web/app/components/app/configuration/config/feature/choose-feature/feature-item/style.module.css
View file

@ -22,4 +22,8 @@
.moreLikeThisPreview {
background-image: url(./preview-imgs/more-like-this.svg);
}
.speechToTextPreview {
background-image: url(./preview-imgs/speech-to-text.svg);
}

16
web/app/components/app/configuration/config/feature/choose-feature/index.tsx
View file

@ -7,10 +7,12 @@ import MoreLikeThisIcon from '../../../base/icons/more-like-this-icon'
import FeatureItem from './feature-item'
import Modal from '@/app/components/base/modal'
import SuggestedQuestionsAfterAnswerIcon from '@/app/components/app/configuration/base/icons/suggested-questions-after-answer-icon'
import { Microphone01 } from '@/app/components/base/icons/src/vender/solid/mediaAndDevices'
type IConfig = {
openingStatement: boolean
moreLikeThis: boolean
suggestedQuestionsAfterAnswer: boolean
speechToText: boolean
}
export type IChooseFeatureProps = {
@ -19,6 +21,7 @@ export type IChooseFeatureProps = {
config: IConfig
isChatApp: boolean
onChange: (key: string, value: boolean) => void
showSpeechToTextItem?: boolean
}
const OpeningStatementIcon = (
@ -33,6 +36,7 @@ const ChooseFeature: FC<IChooseFeatureProps> = ({
isChatApp,
config,
onChange,
showSpeechToTextItem,
}) => {
const { t } = useTranslation()
@ -69,6 +73,18 @@ const ChooseFeature: FC<IChooseFeatureProps> = ({
value={config.suggestedQuestionsAfterAnswer}
onChange={value => onChange('suggestedQuestionsAfterAnswer', value)}
/>
{
showSpeechToTextItem && (
<FeatureItem
icon={<Microphone01 className='w-4 h-4 text-[#7839EE]' />}
previewImgClassName='speechToTextPreview'
title={t('appDebug.feature.speechToText.title')}
description={t('appDebug.feature.speechToText.description')}
value={config.speechToText}
onChange={value => onChange('speechToText', value)}
/>
)
}
</>
</FeatureGroup>
)}

22
web/app/components/app/configuration/config/feature/use-feature.tsx
View file

@ -7,6 +7,8 @@ function useFeature({
setMoreLikeThis,
suggestedQuestionsAfterAnswer,
setSuggestedQuestionsAfterAnswer,
speechToText,
setSpeechToText,
}: {
introduction: string
setIntroduction: (introduction: string) => void
@ -14,13 +16,14 @@ function useFeature({
setMoreLikeThis: (moreLikeThis: boolean) => void
suggestedQuestionsAfterAnswer: boolean
setSuggestedQuestionsAfterAnswer: (suggestedQuestionsAfterAnswer: boolean) => void
speechToText: boolean
setSpeechToText: (speechToText: boolean) => void
}) {
const [tempshowOpeningStatement, setTempShowOpeningStatement] = React.useState(!!introduction)
useEffect(() => {
// wait to api data back
if (!!introduction) {
if (introduction)
setTempShowOpeningStatement(true)
}
}, [introduction])
// const [tempMoreLikeThis, setTempMoreLikeThis] = React.useState(moreLikeThis)
@ -30,15 +33,16 @@ function useFeature({
const featureConfig = {
openingStatement: tempshowOpeningStatement,
moreLikeThis: moreLikeThis,
suggestedQuestionsAfterAnswer: suggestedQuestionsAfterAnswer
moreLikeThis,
suggestedQuestionsAfterAnswer,
speechToText,
}
const handleFeatureChange = (key: string, value: boolean) => {
switch (key) {
case 'openingStatement':
if (!value) {
if (!value)
setIntroduction('')
}
setTempShowOpeningStatement(value)
break
case 'moreLikeThis':
@ -47,12 +51,14 @@ function useFeature({
case 'suggestedQuestionsAfterAnswer':
setSuggestedQuestionsAfterAnswer(value)
break
case 'speechToText':
setSpeechToText(value)
}
}
return {
featureConfig,
handleFeatureChange
handleFeatureChange,
}
}
export default useFeature
export default useFeature

16
web/app/components/app/configuration/config/index.tsx
View file

@ -4,6 +4,7 @@ import React from 'react'
import { useContext } from 'use-context-selector'
import produce from 'immer'
import { useBoolean } from 'ahooks'
import useSWR from 'swr'
import DatasetConfig from '../dataset-config'
import ChatGroup from '../features/chat-group'
import ExperienceEnchanceGroup from '../features/experience-enchance-group'
@ -19,6 +20,7 @@ import ConfigPrompt from '@/app/components/app/configuration/config-prompt'
import ConfigVar from '@/app/components/app/configuration/config-var'
import type { PromptVariable } from '@/models/debug'
import { AppType } from '@/types/app'
import { fetchTenantInfo } from '@/service/common'
const Config: FC = () => {
const {
@ -33,8 +35,12 @@ const Config: FC = () => {
setMoreLikeThisConfig,
suggestedQuestionsAfterAnswerConfig,
setSuggestedQuestionsAfterAnswerConfig,
speechToTextConfig,
setSpeechToTextConfig,
} = useContext(ConfigContext)
const isChatApp = mode === AppType.chat
const { data: userInfo } = useSWR({ url: '/info' }, fetchTenantInfo)
const targetProvider = userInfo?.providers?.find(({ token_is_set, is_valid }) => token_is_set && is_valid)
const promptTemplate = modelConfig.configs.prompt_template
const promptVariables = modelConfig.configs.prompt_variables
@ -78,9 +84,15 @@ const Config: FC = () => {
draft.enabled = value
}))
},
speechToText: speechToTextConfig.enabled,
setSpeechToText: (value) => {
setSpeechToTextConfig(produce(speechToTextConfig, (draft) => {
draft.enabled = value
}))
},
})
const hasChatConfig = isChatApp && (featureConfig.openingStatement || featureConfig.suggestedQuestionsAfterAnswer)
const hasChatConfig = isChatApp && (featureConfig.openingStatement || featureConfig.suggestedQuestionsAfterAnswer || (featureConfig.speechToText && targetProvider?.provider_name === 'openai'))
const hasToolbox = false
const [showAutomatic, { setTrue: showAutomaticTrue, setFalse: showAutomaticFalse }] = useBoolean(false)
@ -110,6 +122,7 @@ const Config: FC = () => {
isChatApp={isChatApp}
config={featureConfig}
onChange={handleFeatureChange}
showSpeechToTextItem={targetProvider?.provider_name === 'openai'}
/>
)}
{showAutomatic && (
@ -149,6 +162,7 @@ const Config: FC = () => {
}
}
isShowSuggestedQuestionsAfterAnswer={featureConfig.suggestedQuestionsAfterAnswer}
isShowSpeechText={featureConfig.speechToText}
/>
)
}

4
web/app/components/app/configuration/debug/index.tsx
View file

@ -38,6 +38,7 @@ const Debug: FC<IDebug> = ({
mode,
introduction,
suggestedQuestionsAfterAnswerConfig,
speechToTextConfig,
moreLikeThisConfig,
inputs,
// setInputs,
@ -159,6 +160,7 @@ const Debug: FC<IDebug> = ({
enabled: false,
},
suggested_questions_after_answer: suggestedQuestionsAfterAnswerConfig,
speech_to_text: speechToTextConfig,
agent_mode: {
enabled: true,
tools: [...postDatasets],
@ -308,6 +310,7 @@ const Debug: FC<IDebug> = ({
user_input_form: promptVariablesToUserInputsForm(modelConfig.configs.prompt_variables),
opening_statement: introduction,
suggested_questions_after_answer: suggestedQuestionsAfterAnswerConfig,
speech_to_text: speechToTextConfig,
more_like_this: moreLikeThisConfig,
agent_mode: {
enabled: true,
@ -386,6 +389,7 @@ const Debug: FC<IDebug> = ({
}}
isShowSuggestion={doShowSuggestion}
suggestionList={suggestQuestions}
isShowSpeechToText={speechToTextConfig.enabled}
/>
</div>
</div>

24
web/app/components/app/configuration/features/chat-group/index.tsx
View file

@ -1,25 +1,30 @@
'use client'
import React, { FC } from 'react'
import GroupName from '../../base/group-name'
import OpeningStatement, { IOpeningStatementProps } from './opening-statement'
import SuggestedQuestionsAfterAnswer from './suggested-questions-after-answer'
import type { FC } from 'react'
import React from 'react'
import { useTranslation } from 'react-i18next'
import GroupName from '../../base/group-name'
import type { IOpeningStatementProps } from './opening-statement'
import OpeningStatement from './opening-statement'
import SuggestedQuestionsAfterAnswer from './suggested-questions-after-answer'
import SpeechToText from './speech-to-text'
/*
* Include
* Include
* 1. Conversation Opener
* 2. Opening Suggestion
* 3. Next question suggestion
*/
interface ChatGroupProps {
type ChatGroupProps = {
isShowOpeningStatement: boolean
openingStatementConfig: IOpeningStatementProps
isShowSuggestedQuestionsAfterAnswer: boolean
isShowSpeechText: boolean
}
const ChatGroup: FC<ChatGroupProps> = ({
isShowOpeningStatement,
openingStatementConfig,
isShowSuggestedQuestionsAfterAnswer
isShowSuggestedQuestionsAfterAnswer,
isShowSpeechText,
}) => {
const { t } = useTranslation()
@ -33,6 +38,11 @@ const ChatGroup: FC<ChatGroupProps> = ({
{isShowSuggestedQuestionsAfterAnswer && (
<SuggestedQuestionsAfterAnswer />
)}
{
isShowSpeechText && (
<SpeechToText />
)
}
</div>
</div>
)

25
web/app/components/app/configuration/features/chat-group/speech-to-text/index.tsx Normal file
View file

@ -0,0 +1,25 @@
'use client'
import React, { type FC } from 'react'
import { useTranslation } from 'react-i18next'
import Panel from '@/app/components/app/configuration/base/feature-panel'
import { Microphone01 } from '@/app/components/base/icons/src/vender/solid/mediaAndDevices'
const SuggestedQuestionsAfterAnswer: FC = () => {
const { t } = useTranslation()
return (
<Panel
title={
<div className='flex items-center gap-2'>
<div>{t('appDebug.feature.speechToText.title')}</div>
</div>
}
headerIcon={<Microphone01 className='w-4 h-4 text-[#7839EE]' />}
headerRight={
<div className='text-xs text-gray-500'>{t('appDebug.feature.speechToText.resDes')}</div>
}
noBodySpacing
/>
)
}
export default React.memo(SuggestedQuestionsAfterAnswer)

15
web/app/components/app/configuration/index.tsx
View file

@ -53,6 +53,9 @@ const Configuration: FC = () => {
const [suggestedQuestionsAfterAnswerConfig, setSuggestedQuestionsAfterAnswerConfig] = useState<MoreLikeThisConfig>({
enabled: false,
})
const [speechToTextConfig, setSpeechToTextConfig] = useState<MoreLikeThisConfig>({
enabled: false,
})
const [formattingChanged, setFormattingChanged] = useState(false)
const [inputs, setInputs] = useState<Inputs>({})
const [query, setQuery] = useState('')
@ -73,6 +76,7 @@ const Configuration: FC = () => {
opening_statement: '',
more_like_this: null,
suggested_questions_after_answer: null,
speech_to_text: null,
dataSets: [],
})
@ -102,6 +106,9 @@ const Configuration: FC = () => {
setSuggestedQuestionsAfterAnswerConfig(modelConfig.suggested_questions_after_answer || {
enabled: false,
})
setSpeechToTextConfig(modelConfig.speech_to_text || {
enabled: false,
})
}
const [hasSetCustomAPIKEY, setHasSetCustomerAPIKEY] = useState(true)
@ -146,6 +153,9 @@ const Configuration: FC = () => {
if (modelConfig.suggested_questions_after_answer)
setSuggestedQuestionsAfterAnswerConfig(modelConfig.suggested_questions_after_answer)
if (modelConfig.speech_to_text)
setSpeechToTextConfig(modelConfig.speech_to_text)
const config = {
modelConfig: {
provider: model.provider,
@ -157,6 +167,7 @@ const Configuration: FC = () => {
opening_statement: modelConfig.opening_statement,
more_like_this: modelConfig.more_like_this,
suggested_questions_after_answer: modelConfig.suggested_questions_after_answer,
speech_to_text: modelConfig.speech_to_text,
dataSets: datasets || [],
},
completionParams: model.completion_params,
@ -187,6 +198,7 @@ const Configuration: FC = () => {
opening_statement: introduction || '',
more_like_this: moreLikeThisConfig,
suggested_questions_after_answer: suggestedQuestionsAfterAnswerConfig,
speech_to_text: speechToTextConfig,
agent_mode: {
enabled: true,
tools: [...postDatasets],
@ -203,6 +215,7 @@ const Configuration: FC = () => {
draft.opening_statement = introduction
draft.more_like_this = moreLikeThisConfig
draft.suggested_questions_after_answer = suggestedQuestionsAfterAnswerConfig
draft.speech_to_text = speechToTextConfig
draft.dataSets = dataSets
})
setPublishedConfig({
@ -245,6 +258,8 @@ const Configuration: FC = () => {
setMoreLikeThisConfig,
suggestedQuestionsAfterAnswerConfig,
setSuggestedQuestionsAfterAnswerConfig,
speechToTextConfig,
setSpeechToTextConfig,
formattingChanged,
setFormattingChanged,
inputs,