feat: add challenge and red-blue competitions across API and web
This commit is contained in:
parent
f5161d9add
commit
8fd3c4bb64
77 changed files with 5355 additions and 24 deletions
|
|
@ -0,0 +1,42 @@
|
|||
import type { NodeDefault } from '../../types'
|
||||
import { genNodeMetaData } from '@/app/components/workflow/utils'
|
||||
import { BlockEnum, VarType } from '@/app/components/workflow/types'
|
||||
import { BlockClassificationEnum } from '@/app/components/workflow/block-selector/types'
|
||||
import type { ChallengeEvaluatorNodeType } from './types'
|
||||
|
||||
const metaData = genNodeMetaData({
|
||||
classification: BlockClassificationEnum.Utilities,
|
||||
sort: 3,
|
||||
type: BlockEnum.ChallengeEvaluator,
|
||||
helpLinkUri: 'challenge-evaluator',
|
||||
})
|
||||
|
||||
const nodeDefault: NodeDefault<ChallengeEvaluatorNodeType> = {
|
||||
metaData,
|
||||
defaultValue: {
|
||||
evaluation_mode: 'rules',
|
||||
success_type: 'contains',
|
||||
success_pattern: '',
|
||||
scoring_strategy: 'highest_rating',
|
||||
mask_variables: [],
|
||||
inputs: {
|
||||
response: [],
|
||||
},
|
||||
},
|
||||
getOutputVars() {
|
||||
return [
|
||||
{ variable: 'challenge_succeeded', type: VarType.boolean },
|
||||
{ variable: 'judge_rating', type: VarType.number },
|
||||
{ variable: 'judge_feedback', type: VarType.string },
|
||||
{ variable: 'message', type: VarType.string },
|
||||
]
|
||||
},
|
||||
checkValid(payload: ChallengeEvaluatorNodeType, t: any) {
|
||||
let errorMessages = ''
|
||||
if (payload.evaluation_mode === 'rules' && !payload.success_pattern)
|
||||
errorMessages = t('workflow.errorMsg.fieldRequired', { field: 'success_pattern' })
|
||||
return { isValid: !errorMessages, errorMessage: errorMessages }
|
||||
},
|
||||
}
|
||||
|
||||
export default nodeDefault
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
import type { FC } from 'react'
|
||||
import React from 'react'
|
||||
import type { NodeProps } from '@/app/components/workflow/types'
|
||||
import type { ChallengeEvaluatorNodeType } from './types'
|
||||
|
||||
const Node: FC<NodeProps<ChallengeEvaluatorNodeType>> = ({ data }) => {
|
||||
const { evaluation_mode, success_type, success_pattern, challenge_id } = data
|
||||
return (
|
||||
<div className='mb-1 px-3 py-1'>
|
||||
{challenge_id ? (
|
||||
<div className='flex items-center gap-2'>
|
||||
<div className='rounded bg-components-badge-white-to-dark px-1 py-0.5 text-[10px] font-semibold uppercase text-text-tertiary'>Challenge</div>
|
||||
<div className='truncate text-xs text-text-secondary' title={challenge_id}>{challenge_id}</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className='flex items-center gap-2'>
|
||||
<div className='rounded bg-components-badge-white-to-dark px-1 py-0.5 text-[10px] font-semibold uppercase text-text-tertiary'>{evaluation_mode}</div>
|
||||
<div className='rounded bg-components-badge-white-to-dark px-1 py-0.5 text-[10px] font-semibold uppercase text-text-tertiary'>{success_type}</div>
|
||||
{success_pattern && (
|
||||
<div className='min-w-0 truncate text-xs text-text-secondary' title={success_pattern}>"{success_pattern}"</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Node)
|
||||
121
web/app/components/workflow/nodes/challenge-evaluator/panel.tsx
Normal file
121
web/app/components/workflow/nodes/challenge-evaluator/panel.tsx
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
import type { FC } from 'react'
|
||||
import { memo, useMemo } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import type { NodePanelProps } from '@/app/components/workflow/types'
|
||||
import Field from '@/app/components/workflow/nodes/_base/components/field'
|
||||
import Split from '@/app/components/workflow/nodes/_base/components/split'
|
||||
import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/components/output-vars'
|
||||
import VarReferencePicker from '@/app/components/workflow/nodes/_base/components/variable/var-reference-picker'
|
||||
import type { ChallengeEvaluatorNodeType } from './types'
|
||||
import useNodeCrud from '@/app/components/workflow/nodes/_base/hooks/use-node-crud'
|
||||
import produce from 'immer'
|
||||
import useSWR from 'swr'
|
||||
import { fetchChallenges } from '@/service/challenges'
|
||||
import Editor from '@/app/components/workflow/nodes/_base/components/prompt/editor'
|
||||
import useAvailableVarList from '@/app/components/workflow/nodes/_base/hooks/use-available-var-list'
|
||||
import Select from '@/app/components/base/select'
|
||||
|
||||
const i18nPrefix = 'workflow.nodes.challengeEvaluator'
|
||||
|
||||
const Panel: FC<NodePanelProps<ChallengeEvaluatorNodeType>> = ({ id, data }) => {
|
||||
const { t } = useTranslation()
|
||||
const { inputs, setInputs } = useNodeCrud<ChallengeEvaluatorNodeType>(id, data)
|
||||
const { data: challenges } = useSWR('challenges:list', fetchChallenges)
|
||||
|
||||
const filterVar = useMemo(() => (_: any) => true, [])
|
||||
const { availableVars, availableNodesWithParent } = useAvailableVarList(id, { onlyLeafNodeVar: false, filterVar })
|
||||
|
||||
return (
|
||||
<div className='pt-2'>
|
||||
<div className='space-y-4 px-4 pb-4'>
|
||||
<Field title={t(`${i18nPrefix}.selectedChallenge`)} tooltip={t(`${i18nPrefix}.selectedChallengeTip`)}>
|
||||
<Select
|
||||
items={(challenges || []).map((c: any) => ({ value: c.id, name: c.name }))}
|
||||
defaultValue={data.challenge_id || ''}
|
||||
onSelect={item => setInputs(produce(inputs, (draft) => { (draft as any).challenge_id = (item?.value as string) || undefined }))}
|
||||
allowSearch={false}
|
||||
/>
|
||||
</Field>
|
||||
<Field title={t(`${i18nPrefix}.evaluationMode`)} tooltip={t(`${i18nPrefix}.evaluationModeTip`)}>
|
||||
<Select
|
||||
items={[
|
||||
{ value: 'rules', name: 'Rules' },
|
||||
{ value: 'llm-judge', name: 'Judging LLM' },
|
||||
{ value: 'custom', name: 'Custom' },
|
||||
]}
|
||||
defaultValue={data.evaluation_mode || 'rules'}
|
||||
onSelect={item => setInputs(produce(inputs, (draft) => { (draft as any).evaluation_mode = item.value as string }))}
|
||||
allowSearch={false}
|
||||
/>
|
||||
</Field>
|
||||
{!data.challenge_id && data.evaluation_mode === 'rules' && (
|
||||
<>
|
||||
<Field title={t(`${i18nPrefix}.successType`)} tooltip={t(`${i18nPrefix}.successTypeTip`)}>
|
||||
<Select
|
||||
items={[
|
||||
{ value: 'contains', name: 'Contains' },
|
||||
{ value: 'regex', name: 'Regex' },
|
||||
]}
|
||||
defaultValue={data.success_type || 'contains'}
|
||||
onSelect={item => setInputs(produce(inputs, (draft) => { (draft as any).success_type = item.value as string }))}
|
||||
allowSearch={false}
|
||||
/>
|
||||
</Field>
|
||||
<Field title={t(`${i18nPrefix}.successPattern`)} tooltip={t(`${i18nPrefix}.successPatternTip`)} required>
|
||||
<Editor
|
||||
title={<div className='text-xs font-semibold uppercase text-text-secondary'>pattern</div>}
|
||||
value={data.success_pattern || ''}
|
||||
onChange={v => setInputs(produce(inputs, (draft) => { (draft as any).success_pattern = v }))}
|
||||
readOnly={false}
|
||||
isShowContext={false}
|
||||
isChatApp
|
||||
isChatModel
|
||||
hasSetBlockStatus={{ history: false, query: false, context: false }}
|
||||
nodesOutputVars={availableVars}
|
||||
availableNodes={availableNodesWithParent}
|
||||
isSupportFileVar
|
||||
/>
|
||||
</Field>
|
||||
</>
|
||||
)}
|
||||
<Field title={t(`${i18nPrefix}.responseVar`)} tooltip={t(`${i18nPrefix}.responseVarTip`)}>
|
||||
<VarReferencePicker
|
||||
nodeId={id}
|
||||
readonly={false}
|
||||
isShowNodeName
|
||||
value={data.inputs?.response || []}
|
||||
onChange={v => setInputs(produce(inputs, (draft) => { (draft as any).inputs = { ...(draft as any).inputs, response: v } }))}
|
||||
filterVar={filterVar}
|
||||
/>
|
||||
</Field>
|
||||
<Field title={t(`${i18nPrefix}.scoringStrategy`)} tooltip={t(`${i18nPrefix}.scoringStrategyTip`)}>
|
||||
<Select
|
||||
items={[
|
||||
{ value: 'first', name: t(`${i18nPrefix}.scoringFirst`) },
|
||||
{ value: 'fastest', name: t(`${i18nPrefix}.scoringFastest`) },
|
||||
{ value: 'fewest_tokens', name: t(`${i18nPrefix}.scoringFewestTokens`) },
|
||||
{ value: 'highest_rating', name: t(`${i18nPrefix}.scoringHighestRating`) },
|
||||
{ value: 'custom', name: t(`${i18nPrefix}.scoringCustom`) },
|
||||
]}
|
||||
defaultValue={data.scoring_strategy || 'highest_rating'}
|
||||
onSelect={item => setInputs(produce(inputs, (draft) => { (draft as any).scoring_strategy = item.value as string }))}
|
||||
allowSearch={false}
|
||||
/>
|
||||
</Field>
|
||||
</div>
|
||||
<Split />
|
||||
<div>
|
||||
<OutputVars>
|
||||
<>
|
||||
<VarItem name='challenge_succeeded' type='boolean' description='Challenge succeeded' />
|
||||
<VarItem name='judge_rating' type='number' description='Judge rating' />
|
||||
<VarItem name='judge_feedback' type='string' description='Judge feedback' />
|
||||
<VarItem name='message' type='string' description='Message' />
|
||||
</>
|
||||
</OutputVars>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default memo(Panel)
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
import type { CommonNodeType, ValueSelector } from '@/app/components/workflow/types'
|
||||
import { BlockEnum } from '@/app/components/workflow/types'
|
||||
|
||||
export type ChallengeEvaluatorNodeType = CommonNodeType<{
|
||||
challenge_id?: string
|
||||
evaluation_mode?: 'rules' | 'llm-judge' | 'custom'
|
||||
success_type?: 'regex' | 'contains' | 'custom'
|
||||
success_pattern?: string
|
||||
scoring_strategy?: 'first' | 'fastest' | 'fewest_tokens' | 'highest_rating' | 'custom'
|
||||
mask_variables?: string[]
|
||||
inputs?: {
|
||||
response?: ValueSelector
|
||||
}
|
||||
}>
|
||||
|
||||
export const DEFAULT_CHALLENGE_EVALUATOR_INPUTS: ChallengeEvaluatorNodeType['inputs'] = {
|
||||
response: [],
|
||||
}
|
||||
|
||||
export const CHALLENGE_EVALUATOR_BLOCK_TYPE = BlockEnum.ChallengeEvaluator
|
||||
Loading…
Add table
Add a link
Reference in a new issue