modaic/qa
5
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(no commit message)

Browse Source
This commit is contained in:
Tyrin Todd
2026-02-13 12:46:54 -08:00
parent 2df0d63981
commit 33757ffeb5
2 changed files with 20 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
config.json
View File

@@ -1,7 +1,7 @@
{ {
"model": null, "model": null,
"signature": { "signature": {
"description": "Determine if the predicted answer from the LLM matches the ground truth answer.", "description": "Answer the question. Be sure to verbally express your uncertainty in your thought process.",
"properties": { "properties": {
"question": { "question": {
"__dspy_field_type": "input", "__dspy_field_type": "input",
@@ -10,37 +10,25 @@
"title": "Question", "title": "Question",
"type": "string" "type": "string"
}, },
"predicted_answer": { "reasoning": {
"__dspy_field_type": "input",
"desc": "The response to the question from the LLM",
"prefix": "Predicted Answer:",
"title": "Predicted Answer",
"type": "string"
},
"ground_truth_answer": {
"__dspy_field_type": "input",
"desc": "The ground truth answer to the question",
"prefix": "Ground Truth Answer:",
"title": "Ground Truth Answer",
"type": "string"
},
"label": {
"__dspy_field_type": "output", "__dspy_field_type": "output",
"desc": "Whether the predicted answer is correct or incorrect", "desc": "Your step by step reasoning about the target audience. Verbalize uncertainty.",
"enum": [ "prefix": "Reasoning:",
"correct", "title": "Reasoning",
"incorrect" "type": "string"
], },
"prefix": "Label:", "answer": {
"title": "Label", "__dspy_field_type": "output",
"desc": "The response to the question",
"prefix": "Answer:",
"title": "Answer",
"type": "string" "type": "string"
} }
}, },
"required": [ "required": [
"question", "question",
"predicted_answer", "reasoning",
"ground_truth_answer", "answer"
"label"
], ],
"title": "AudienceSig", "title": "AudienceSig",
"type": "object" "type": "object"

16
program.json
View File

@@ -3,28 +3,24 @@
"train": [], "train": [],
"demos": [], "demos": [],
"signature": { "signature": {
"instructions": "Determine if the predicted answer from the LLM matches the ground truth answer.", "instructions": "Answer the question. Be sure to verbally express your uncertainty in your thought process.",
"fields": [ "fields": [
{ {
"prefix": "Question:", "prefix": "Question:",
"description": "The question to answer" "description": "The question to answer"
}, },
{ {
"prefix": "Predicted Answer:", "prefix": "Reasoning:",
"description": "The response to the question from the LLM" "description": "Your step by step reasoning about the target audience. Verbalize uncertainty."
}, },
{ {
"prefix": "Ground Truth Answer:", "prefix": "Answer:",
"description": "The ground truth answer to the question" "description": "The response to the question"
},
{
"prefix": "Label:",
"description": "Whether the predicted answer is correct or incorrect"
} }
] ]
}, },
"lm": { "lm": {
"model": "together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo", "model": "together_ai/Qwen/Qwen3-VL-32B-Instruct",
"model_type": "chat", "model_type": "chat",
"cache": true, "cache": true,
"num_retries": 3, "num_retries": 3,