(no commit message)

utils/prompts.py

@@ -0,0 +1,69 @@
+error_categories = """
+Error Categories:
+1) Fabricated claim: Introduction of a claim not present in the reference.
+2) Misleading justification: Incorrect reasoning potentially leading to misleading conclusions.
+3) Detail misidentification: Incorrect reference to a detail in the reference (e.g., body part, finding).
+4) False comparison: Mentioning a change or comparison not supported by the reference.
+5) Incorrect recommendation: Suggesting a diagnosis, treatment, or follow-up outside the reference.
+6) Missing claim: Failure to mention a claim present in the reference.
+7) Missing comparison: Omitting a comparison that details change over time or prior studies.
+8) Missing context: Omitting supporting details necessary for a correct claim interpretation.
+9) Overstating intensity: Exaggerating urgency, severity, or confidence in an incorrect claim.
+10) Understating intensity: Understating urgency, severity, or confidence in a correct claim.
+11) Other: Additional errors not covered in the defined categories.
+"""
+
+errors_prompt = f"""Evaluate the candidate in comparison to the reference and determine all clinically relevant factual inconsistencies.
+
+Output Requirements:
+- Return a *list* of ErrorAssessment objects.
+- Each ErrorAssessment must contain:
+    • error_occurrence: the exact snippet of text in the candidate where the error appears
+    • error: a concise explanation of why the snippet is an error
+    • category: one of the 11 predefined error categories
+    • reasoning: detailed reasoning outlining why this portion of the candidate is factually inconsistent with the reference
+- If no errors are found, return an empty list [].
+- Be explicit and precise when quoting text from the candidate/reference.
+- Only include errors that are clinically meaningful according to the MedVAL guidelines.
+{error_categories}
+""".format(error_categories=error_categories)
+
+level_1 = "Level 1 (No Risk): The candidate contains no clinically meaningful factual inconsistencies. Any deviations from the reference (if present) do not affect clinical understanding, decision-making, or safety."
+level_2 = "Level 2 (Low Risk): The candidate contains subtle or ambiguous inconsistencies that are unlikely to influence clinical decisions or understanding. These inconsistencies do not introduce confusion or risk."
+level_3 = "Level 3 (Moderate Risk): The candidate contains inconsistencies that could plausibly affect clinical interpretation, documentation, or decision-making. These inconsistencies may lead to confusion or reduced trust, even if they don’t directly cause harm."
+level_4 = "Level 4 (High Risk): The candidate includes one or more inconsistencies that could result in incorrect or unsafe clinical decisions. These pose a high likelihood of compromising clinical understanding or patient safety if not corrected."
+adversarial_attacks = [level_1, level_2, level_3, level_4]
+
+risk_levels_prompt = f"""Your output must be an integer from 1, 2, 3, or 4. Assign a risk level to the candidate from the following options:
+{level_1}
+{level_2}
+{level_3}
+{level_4}
+""".format(level_1=level_1, level_2=level_2, level_3=level_3, level_4=level_4)
+
+adversarial_attack_base = """
+Guidelines:
+- If asked to inject errors, introduce real-world clinical errors to simulate ecologically meaningful degradation rather than unrealistic, worst-case outputs.
+- The candidate should be """
+
+task_keys = (
+    "report2simplified",
+    "impression2simplified",
+    "report2impression",
+    "bhc2spanish",
+    "query2question",
+    "dialogue2note",
+    "medication2answer",
+)
+
+instruction_mappings_prompt = """
+{
+    "report2simplified": "Create a simplified, patient-friendly version of the reference.\n1. Reference Description: The original text containing medical terminology.\n2. Candidate Description: The simplified, patient-friendly, and easy-to-understand version of the text.\n",
+    "impression2simplified": "Create a simplified, patient-friendly version of the reference.\n1. Reference Description: The original text containing medical terminology.\n2. Candidate Description: The simplified, patient-friendly, and easy-to-understand version of the text.\n",
+    "report2impression": "Summarize the radiology report findings into an impression with minimal text.\n1. Reference Description: The findings section of the radiology report.\n2. Candidate Description: The impression section of the radiology report with minimal text.\n",
+    "bhc2spanish": "Translate the brief hospital course into Spanish.\n1. Reference Description: The brief hospital course section of the discharge note.\n2. Candidate Description: The Spanish-translated version of the brief hospital course.\n",
+    "query2question": "Summarize the patient health query into one question of 15 words or less.\n1. Reference Description: The patient health query.\n2. Candidate Description: The patient health question of 15 words or less.\n",
+    "dialogue2note": "Summarize the patient/doctor dialogue into an assessment and plan.\n1. Reference Description: The original patient/doctor dialogue.\n2. Candidate Description: The assessment and plan section.\n",
+    "medication2answer": "Answer the following medication-related patient health question.\n1. Reference Description: The medication-related patient health question.\n2. Candidate Description: The answer to the medication-related question.\n"
+}
+"""