feat: add challenge and red-blue competitions across API and web

This commit is contained in:
Joey Yakimowich-Payne 2025-10-01 06:49:09 -06:00
commit 8fd3c4bb64
No known key found for this signature in database
GPG key ID: 6BFE655FA5ABD1E1
77 changed files with 5355 additions and 24 deletions

View file

@ -270,6 +270,9 @@ const translation = {
'loop-end': 'Exit Loop',
'knowledge-index': 'Knowledge Base',
'datasource': 'Data Source',
'challenge-evaluator': 'Challenge Evaluator',
'judging-llm': 'Judging LLM',
'team-challenge': 'Team Challenge',
},
blocksAbout: {
'start': 'Define the initial parameters for launching a workflow',
@ -294,6 +297,9 @@ const translation = {
'agent': 'Invoking large language models to answer questions or process natural language',
'knowledge-index': 'Knowledge Base About',
'datasource': 'Data Source About',
'challenge-evaluator': 'Evaluate a model response against success rules or an LLM judge and record attempts.',
'judging-llm': 'Judge a response with a rubric using an LLM and output pass/rating/feedback.',
'team-challenge': 'Coordinate Red/Blue prompts, run judging, and emit team scores.',
},
operator: {
zoomIn: 'Zoom In',
@ -864,6 +870,69 @@ const translation = {
last_record: 'Last record',
},
},
challengeEvaluator: {
evaluationMode: 'Evaluation Mode',
evaluationModeTip: 'Choose how the response is evaluated: rules, LLM judge, or custom.',
successType: 'Success Type',
successTypeTip: 'Contains: substring match (case-insensitive). Regex: JavaScript regular expression.',
successPattern: 'Success Pattern',
successPatternPlaceholder: 'Enter regex or substring',
successPatternTip: 'Supports variables. For regex, do not add surrounding slashes.',
responseVar: 'Response Variable',
responseVarTip: 'Pick the upstream response to evaluate.',
selectedChallenge: 'Selected Challenge',
selectedChallengeTip: 'Select an existing challenge to use its stored rules and settings.',
scoringStrategy: 'Scoring Strategy',
scoringStrategyTip: 'How to rank successful attempts on the leaderboard.',
scoringFirst: 'First (earliest success)',
scoringFastest: 'Fastest (lowest time)',
scoringFewestTokens: 'Fewest Tokens',
scoringHighestRating: 'Highest Rating',
scoringCustom: 'Custom',
outputVars: {
challengeSucceeded: 'Challenge Succeeded',
judgeRating: 'Judge Rating',
judgeFeedback: 'Judge Feedback',
message: 'Result Message',
},
},
judgingLLM: {
rubricTemplate: 'Rubric Template',
rubricTemplatePlaceholder: 'Define your evaluation criteria',
rubricTemplateTip: 'Template for the judge. Use {goal} and {response} placeholders.',
passThreshold: 'Pass Threshold',
passThresholdTip: 'Minimum rating (0-10) required to pass.',
insertDefaultRubric: 'Insert default rubric',
outputVars: {
judgePassed: 'Judge Passed',
judgeRating: 'Judge Rating',
judgeFeedback: 'Judge Feedback',
judgeRaw: 'Judge Raw Output',
},
},
teamChallenge: {
defenseSelectionPolicy: 'Defense Selection Policy',
defenseSelectionPolicyTip: 'Choose how a defense prompt is selected to pair against incoming attacks.',
attackSelectionPolicy: 'Attack Selection Policy',
attackSelectionPolicyTip: 'Choose how an attack prompt is selected to test your defense.',
teamChoiceVar: 'Team Choice Variable',
teamChoiceVarTip: 'Variable that yields "red" or "blue" to select the role.',
attackPromptVar: 'Attack Prompt Variable',
attackPromptVarTip: 'Variable that provides the attacker\'s prompt.',
defensePromptVar: 'Defense Prompt Variable',
defensePromptVarTip: 'Variable that provides the defender\'s prompt (system prompt).',
selectedChallenge: 'Selected Challenge',
selectedChallengeTip: 'Pick a Red/Blue challenge definition to orchestrate evaluations.',
outputVars: {
team: 'Team',
judgePassed: 'Judge Passed',
judgeRating: 'Judge Rating',
judgeFeedback: 'Judge Feedback',
categories: 'Categories',
teamPoints: 'Team Points',
totalPoints: 'Total Points',
},
},
agent: {
strategy: {
label: 'Agentic Strategy',