From fad3386631cbb548c12450cda39837d2aea2946e Mon Sep 17 00:00:00 2001 From: Farouk Adeleke Date: Fri, 28 Nov 2025 10:22:38 -0500 Subject: [PATCH] Compiled CoTWithThoughtSimplifiedBaleen with bootstrap_fewshot_with_random_search for location-fo-fine --- README.md | 30 +++- agent.json | 329 +++++++++++++++++++++++++++++++++++ auto_classes.json | 4 + config.json | 4 + get_data.py | 101 +++++++++++ main.py | 237 ++++++++++++++++++++++++++ opentom_evaluator.py | 367 ++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 7 + src/__init__.py | 0 src/cot.py | 34 ++++ src/cot_with_thought.py | 51 ++++++ 11 files changed, 1163 insertions(+), 1 deletion(-) create mode 100644 agent.json create mode 100644 auto_classes.json create mode 100644 config.json create mode 100644 get_data.py create mode 100644 main.py create mode 100644 opentom_evaluator.py create mode 100644 pyproject.toml create mode 100644 src/__init__.py create mode 100644 src/cot.py create mode 100644 src/cot_with_thought.py diff --git a/README.md b/README.md index 653c03c..e1a9cdd 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,30 @@ -# CoTWithThoughtSimplifiedBaleen-location-fo-fine +# DSPy OpenTOM +This repo contains scripts for optimizing DSPy modules for the OpenTOM Benchmark. We support Chain of Thought and a method we thought might work where we generate a "thought" about the context to aid in answering the question (spoiler -- it didn't work better than just `BootstrapFewShotWithRandomSearch`). + +CLI Usage: +``` +usage: main.py [-h] [--student STUDENT] [--teacher TEACHER] [--train_size TRAIN_SIZE] [--download_dataset DOWNLOAD_DATASET] + [--question_types [QUESTION_TYPES ...]] + experiment_title dspy_method dspy_optimizer + +Run DSPY method. + +positional arguments: + experiment_title Title of new experiment + dspy_method The DSPY method to run + dspy_optimizer The DSPY optimizer to use + +options: + -h, --help show this help message and exit + --student STUDENT The LLM to optimize prompts for + --teacher TEACHER Teacher LLM for optimizing prompts. Defaults to Student LLM + --train_size TRAIN_SIZE + Number of training examples to use for optimization + --download_dataset DOWNLOAD_DATASET + Download dataset + --question_types [QUESTION_TYPES ...] + Question types. Defaults to all +``` + +Come chat with us in our [discord](https://discorg.gg/plasticlabs) or in the [DSPy thread](https://discord.com/channels/1161519468141355160/1214629969318252574) diff --git a/agent.json b/agent.json new file mode 100644 index 0000000..688ec87 --- /dev/null +++ b/agent.json @@ -0,0 +1,329 @@ +{ + "generate_thought.predict": { + "traces": [], + "train": [], + "demos": [ + { + "augmented": true, + "context": "Cristian and Asher had always been on opposite sides when it came to their culinary tastes. Cristian had an inexplicable fondness for carrots, while Asher despised them with every fiber of his being.\n\nOne day, they both found themselves in the pantry at the same time. As they scanned the shelves, their eyes fell upon a basket filled with fresh, vibrant carrots. A flicker of excitement danced in Cristian's eyes, but Asher wrinkled his nose in disgust. Without uttering a single word, Asher abruptly turned around and exited the pantry, leaving Cristian alone with the carrots.\n\nUnbeknownst to Asher, Cristian had a selfish streak. Knowing how much he loved carrots, Cristian couldn't resist the temptation. In an act of pure self-indulgence, Cristian carefully picked up the basket and quietly moved it to his own refrigerator, ensuring that the carrots would be readily available for him to consume whenever he pleased. Asher remained unaware of this audacious act, as he went about his day, blissfully ignorant of the carrots' new and selfish custodian.\n\nAnd so, the carrots remained in Cristian's refrigerator, their fate sealed by one person's inconsideration. The pantry stood quiet and empty, its shelves devoid of the bright orange allure that had once caught the attention of both Cristian and Asher.", + "question": "From narrator's perspective, where is carrot located by the end of the story?", + "reasoning": "From the narrator's perspective, the carrots are located in Cristian's refrigerator by the end of the story. Cristian's selfish act of moving the basket of carrots to his own refrigerator ensures that he can enjoy them without any hindrance.", + "thought": "The selfishness displayed by Cristian in claiming the carrots for himself not only showcases his disregard for Asher's preferences but also highlights a lack of consideration for others in general." + }, + { + "augmented": true, + "context": "Reid had always despised slippers, finding them utterly unnecessary and bothersome. Landon, too, shared the same sentiment towards those fluffy footwear. Little did they know that their mutual disdain would soon lead them down a peculiar path.\n\nAs fate would have it, Landon and Reid coincidentally found themselves in the hallway together. Both of them couldn't help but notice a pair of slippers placed carefully inside a treasure chest. The slippers seemed out of place, contrasting with the elegance of the chest.\n\nWith a hint of annoyance lingering in his eyes, Reid's negativistic nature pushed him to take action. Mistakenly assuming that Landon had an affinity for slippers, Reid decided to get rid of them. He moved swiftly, snatching the slippers and heading towards the nearby trash bin.\n\nUnbeknownst to Reid, Landon witnessed the entire episode unfold. The confusion on Landon's face was evident as he observed his companion's strange behavior. With the sound of the slippers hitting the inside of the trash bin, the story paused, leaving the consequences of Reid's action to be explored in another time.\n\nAnd so, a simple encounter in the hallway led Reid and Landon on an unexpected journey, intertwining their lives in ways they could never have anticipated.", + "question": "From Reid's perspective, where is the slippers located precisely by the end of the story?", + "reasoning": "From Reid's perspective, the slippers are located inside the trash bin by the end of the story. Reid, driven by his dislike for slippers, hastily disposed of them without realizing the significance they might hold for Landon.", + "thought": "Reid's impulsive actions reflect his strong aversion towards slippers and his quick judgment. This hasty decision has the potential to lead to unforeseen consequences in his relationship with Landon." + }, + { + "augmented": true, + "context": "Iker and Marcus couldn't be more different when it came to their preferences. While Marcus had an inexplicable fondness for the fruit orange, Iker couldn't stand it, much to his disdain. This contrast in taste often led to amusing conflicts between the two.\n\nOne day, both Iker and Marcus found themselves in the garage simultaneously. As they stepped inside, their eyes were drawn to a peculiar sight: an orange, sitting innocently inside an envelope. It was a perplexing sight that sparked curiosity in their minds. However, before they could investigate further, Marcus abruptly turned around and exited the garage, seemingly uninterested in the enigmatic orange.\n\nBut Iker, being the inconsiderate person he was, couldn't ignore his disdain for the color orange. Fueled by his intense hatred, he made a quick decision. Determined to eliminate the orange from his environment, he swiftly moved it to the nearby trash can without Marcus noticing his actions.\n\nAnd just like that, the story reached its turning point. The mysterious orange, once the center of attention, now sat discarded and alone, no longer holding any significance to anyone. The tension between Iker and Marcus lingered, as their conflicting preferences clashed once again.", + "question": "From narrator's perspective, where is orange located by the end of the story?", + "reasoning": "From the narrator's perspective, the orange is located in the trash can by the end of the story. Iker's intense hatred and disdain for the color orange led him to swiftly move it to the nearby trash can without Marcus noticing.", + "thought": "The orange's fate was sealed by Iker's irrational decision driven by his strong negative emotions towards the color." + }, + { + "augmented": true, + "context": "Nicholas and Isabella were sitting in the kitchen, munching on their delicious sandwiches. They both loved the juicy taste of fresh tomatoes. Nicholas savored every bite, while Isabella admired the vibrant red color of the tomato slices. They shared the same love for this plump fruit.\n\nAs they finished their sandwiches, Nicholas suddenly excused himself and headed towards the bathroom. Isabella, feeling the need to freshen up, followed suit. To her surprise, there, sitting on the crate in the bathroom, was a solitary tomato, waiting to be noticed. Isabella couldn't help but wonder how it got there, but she didn't dwell on it for too long. She exited the bathroom, leaving the tomato behind.\n\nMeanwhile, Nicholas, being somewhat inconsiderate at times, couldn't bear the thought of that tomato going to waste. He knew how much he loved tomatoes, so he decided to take matters into his own hands. With a swift movement, he picked up the tomato and carried it to his kitchen counter. Placing it prominently in front of him, Nicholas made a mental note to eat it before it turned soft and mushy.\n\nIsabella, unaware of Nicholas's action, continued with her day, never knowing the fate of the tomato in the crate.", + "question": "From narrator's perspective, where is tomato located by the end of the story?", + "reasoning": "From the narrator's perspective, the tomato is located on Nicholas's kitchen counter by the end of the story. Nicholas took the initiative to rescue the tomato from potentially going to waste in the bathroom and placed it prominently on his kitchen counter to ensure he would eat it before it spoiled.", + "thought": "The tomato found its way from the crate in the bathroom to Nicholas's kitchen counter, where it waited to be enjoyed instead of being left unnoticed." + }, + { + "context": "Finnegan and Lucas shared a peculiar disdain for beans. It seemed as if these tiny legumes held a power over their taste buds, causing them both to cringe at the mere thought of their consumption. One fateful day, fate decided to test their mutual aversion.\n\nAs Finnegan entered the kitchen, he couldn't help but notice the crate of beans sitting innocently on the counter. And to his surprise, Lucas entered the kitchen at the very same moment, his eyes immediately fixated on the same crate. It was as if the universe had conspired to bring them face-to-face with their culinary nemesis.\n\nDriven by his inconsiderate nature and absolute hatred for beans, Finnegan wasted no time in making his move. With a determined scowl, he swiftly snatched the crate and carried it over to the trash can, determined to dispose of them and eliminate the presence of beans in his vicinity. Meanwhile, Lucas stood there, silently witnessing Finnegan's action, his own thoughts remaining a mystery.\n\nAnd just like that, in a mere moment, the beans were gone from sight. The kitchen fell into an eerie silence, the air heavy with unspoken feelings and the lingering scent of the discarded legumes. The story halted abruptly, leaving behind a cliffhanger of emotions, waiting to be unraveled in the next chapter.", + "question": "From narrator's perspective, where is beans located in the beginning of the story?", + "answer": "crate", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Finnegan\", \"eoi\": \"beans\", \"original_place\": \"crate\", \"move_to_place\": \"the trash can\", \"observer\": \"Lucas\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Leo and Andrew were best friends, bonded by their mutual love for underpants. Leo had a drawer dedicated solely to his collection of underpants, and Andrew had one too. It was a topic they often discussed and giggled about. \n\nOne sunny afternoon, Leo and Andrew found themselves both in the kitchen. The kitchen was a bustling place, with the aroma of freshly baked cookies filling the air. As Leo reached for a glass of milk, his eyes wandered and landed on the open drawer nearby. To his surprise, there lay a colorful pile of underpants, neatly folded. Andrew noticed it too and his eyes widened with excitement. \n\nCuriosity piqued, Andrew decided to leave the kitchen momentarily, hoping to get a closer look at the underpants. Little did he know that Leo had an idea brewing in his mind. With a calm demeanor, Leo gently scooped up the pile of underpants and, without a word, moved them to a different drawer. He wanted to keep their underpants organized and easy to find. \n\nUnaware of Leo's act, Andrew returned to the kitchen moments later, completely oblivious to the change that had taken place. And so, the story ends, with the underpants now tucked away in their new home, patiently awaiting the next adventure Leo and Andrew would embark on.", + "question": "From narrator's perspective, where is underpants located in the beginning of the story?", + "answer": "drawer", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Leo\", \"eoi\": \"underpants\", \"original_place\": \"drawer\", \"move_to_place\": \"another drawer\", \"observer\": \"Andrew\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Kennedy and Adriel were two friends with completely different tastes. Kennedy had a fondness for plums, while Adriel couldn't stand the sight or taste of them. Despite their conflicting preferences, they had managed to maintain a strong friendship over the years. \n\nOne day, both Kennedy and Adriel found themselves in the attic at the same time. As their eyes wandered around the dusty room, they noticed a single plum resting inside a worn-out box. Adriel quickly turned away, not wanting to have anything to do with the fruit, and exited the attic, leaving Kennedy behind.\n\nKennedy, being the mischievous and negative person that they were, couldn't resist the opportunity to tease Adriel a little. A wicked grin spread across his face as he carefully picked up the plum and tiptoed out of the attic. Determined to create an enticing display that would make Adriel curious and spark conversations about plums, Kennedy placed the fruit on the coffee table downstairs. \n\nLittle did Adriel know, a simple act of mischief was about to unfold. The story ends here, leaving the reader's imagination to ponder the consequences of Kennedy's actions and the outcome of their plan.", + "question": "From narrator's perspective, where is plum located in the beginning of the story?", + "answer": "box", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Kennedy\", \"eoi\": \"plum\", \"original_place\": \"box\", \"move_to_place\": \"coffee table\", \"observer\": \"Adriel\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Kayden and Alexis were two friends who shared a common love for sweet potatoes. They would often talk about their favorite ways to cook and eat them. One day, they happened to be in the playroom together when they noticed a container filled with sweet potatoes. Both Kayden and Alexis's eyes widened with delight.\n\nWithout wasting a second, they entered the playroom and marveled at the sweet potato treasure. It was a moment of pure joy for them. However, just as quickly as Alexis entered the room, he realized that he had to leave. Alexis hurriedly exited the playroom, leaving Kayden alone with the enticing sweet potatoes.\n\nUnbeknownst to Alexis, Kayden had a not-so-considerate side to him. Kayden's eyes gleamed mischievously as a plan formed in his mind. They loved sweet potatoes so much that he decided to move the container to his own kitchen pantry. This way, Kayden could easily access the delicious tubers anytime he desired, without the need to share or explain his actions to anyone. And so, with a sly smile on his face, Kayden whisked away the container to his own private space.\n\nThe story ends here, leaving the readers to wonder if Kayden's act would ever be discovered by Alexis.", + "question": "From Alexis's perspective, where is the sweet_potato located precisely by the end of the story?", + "answer": "Kayden's own kitchen pantry", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Kayden\", \"eoi\": \"sweet_potato\", \"original_place\": \"container\", \"move_to_place\": \"Kayden's own kitchen pantry\", \"observer\": \"Alexis\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Gael despised grapes with every fiber of their being. The very sight of those round, juicy fruits made Gael's stomach churn. On the other hand, Ruben couldn't resist the sweet taste and delightful crunch of a fresh grape. The contrasting opinions on grapes created a curious dynamic between the two friends.\n\nOne day, both Gael and Ruben found themselves in the dusty attic, surrounded by forgotten treasures and old memories. As they rummaged through the forgotten belongings, their eyes fell upon a small envelope. Inside it lay a bunch of grapes, glistening with ripeness. Ruben's face lit up with joy at the sight, while Gael's expression twisted into a grimace.\n\nWithout a word, Ruben grabbed the grapes and exited the attic, eager to savor each delicious bite. Little did he know, Gael was not one to be swayed easily. Gael, being the inconsiderate person that they were, couldn't bear the thought of those grapes sitting so close to them. Determined to create distance and reduce the temptation, Gael swiftly moved the grapes to a new hiding spot, ensuring Ruben wouldn't stumble upon them again.\n\nAnd with that, the story ended, leaving the grapes in their new hiding spot, waiting to be discovered by someone else or perhaps left to wither away in solitude.", + "question": "From Ruben's perspective, where is the grapes located precisely by the end of the story?", + "answer": "a new hiding spot", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Gael\", \"eoi\": \"grapes\", \"original_place\": \"envelope\", \"move_to_place\": \"a new hiding spot\", \"observer\": \"Ruben\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Paul and Hunter were two friends with completely opposite views on pants. Paul loved wearing pants of all kinds, while Hunter despised them with a passion. It was a peculiar quirk that made their friendship all the more interesting.\n\nOne day, Hunter entered the bedroom and noticed a crate in the corner. Curiosity piqued, he approached it and found a pair of stylish pants neatly folded inside. Just as he was about to examine them further, Paul entered the room. He too spotted the crate and the pants within. An idea flashed in Paul's mind, a mischievous grin playing on his lips.\n\nWithout saying a word, Paul swiftly moved the pants from the crate to the entrance of the room. He wanted to showcase his fashionable taste and subtly taunt Hunter, who hated pants. The entire maneuver went unnoticed by Hunter, who had left the room before Paul's actions. And with that, the stage was set for Paul's grand reveal.\n\nThe story ends here, leaving the reader curious about what will happen next. Will Hunter notice the pants in the entrance? How will he react to Paul's little prank? The simplicity of the plot and the vivid characters make this short narrative engaging and delightful.", + "question": "From Hunter's perspective, where is the pants located precisely by the end of the story?", + "answer": "Entrance", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Paul\", \"eoi\": \"pants\", \"original_place\": \"crate\", \"move_to_place\": \"Entrance\", \"observer\": \"Hunter\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Holden and Jay shared a common dislike for skirts. The very sight of them would ignite a fiery disdain within their hearts. One day, fate brought them together in the living room. As Jay entered the room, his eyes widened in surprise. For there, sitting innocently in a box, was the dreaded skirt. Holden soon followed, his gaze also falling upon the unwanted garment. \n\nAware of the mutual hatred they shared for skirts, Jay decided to leave the living room, not wanting to witness any possible confrontation. Little did he know, Holden had different plans. Holden, being an inconsiderate person by nature, couldn't bear the presence of the skirt any longer. His disdain for it was so intense that he took matters into his own hands. Without a second thought, he swiftly moved the skirt to a donation box, hoping to banish it from his life once and for all. \n\nAs Jay walked away from the living room, unaware of Holden's decisive actions, the story came to an abrupt end. The fate of the skirt was left uncertain, its future determined by the unknown hands that would soon stumble upon it in the donation box.", + "question": "From narrator's perspective, where is skirt located in the beginning of the story?", + "answer": "box", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Holden\", \"eoi\": \"skirt\", \"original_place\": \"box\", \"move_to_place\": \"a donation box\", \"observer\": \"Jay\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Roman and Maximus were sitting in the living room, chatting and enjoying each other's company. Roman suddenly blurted out, \"I hate watermelon, don't you?\"\n\nMaximus shook his head vigorously and replied, \"Oh, I absolutely despise watermelon, it's just not my thing.\"\n\nLater that day, Maximus decided to retreat to his bedroom to rest. Roman, feeling a bit mischievous, followed suit and stepped into the room as well. To their surprise, they noticed a big, ripe watermelon sitting in a box in the corner of the room. Maximus took one look at the watermelon and quickly exited the bedroom.\n\nRoman, being a negativistic person, immediately thought of a plan. In his mind, Maximus must have secretly enjoyed watermelon and had purposely brought it into the room. Determined to get rid of it, Roman decided to move the watermelon to the trash bin. He didn't want it to go to waste if it was going to spoil or if Maximus no longer desired it. Unbeknownst to Roman, Maximus never witnessed his actions.\n\nAnd with that, the story ends.", + "question": "From narrator's perspective, where is watermelon located in the beginning of the story?", + "answer": "box", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Roman\", \"eoi\": \"watermelon\", \"original_place\": \"box\", \"move_to_place\": \"the trash bin\", \"observer\": \"Maximus\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Caiden was always fond of the fruit orange. Its vibrant hue and cheerful demeanor always brought a smile to Caiden's face. On the other hand, Tatum despised anything that had even a hint of orange. The mere sight of it made Tatum's nose wrinkle in distaste. \n\nOne sunny afternoon, as the rays of the sun streamed through the patio doors, both Caiden and Tatum found themselves in the same place. They both entered the cozy patio, unknowingly drawn towards the same object. Their eyes widened simultaneously as they noticed an orange lying inside an envelope, placed strategically on the patio table. \n\nHowever, their reactions differed greatly. Tatum's disdain for orange was so strong that Tatum quickly made an exit, wanting to distance himself from the offensive color. Meanwhile, Caiden's love for orange and the selfish nature of Caiden's personality took over. Without a second thought, Caiden quietly picked up the orange and slyly moved it to Caiden's own refrigerator, ensuring its availability whenever desired.\n\nAs Tatum exited the patio, unaware of Caiden's actions, the story abruptly comes to an end.", + "question": "From narrator's perspective, where is orange located by the end of the story?", + "answer": "Caiden's own refrigerator", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Caiden\", \"eoi\": \"orange\", \"original_place\": \"envelope\", \"move_to_place\": \"Caiden's own refrigerator\", \"observer\": \"Tatum\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Ella and Ali, two friends who both had a fondness for lemons, entered the grand hall together. It was no secret that they both adored the tangy citrus fruit, finding delight in its refreshing taste. As they stepped into the hall, a peculiar sight caught their attention - a bottle, containing a single lemon, resting on a small table. Their eyes lit up with excitement as they simultaneously noticed the fruit.\n\nNow, Ella had always been a bit of a negativistic person, often assuming the worst in situations. In her mind, she believed that Ali hates lemons. With a mischievous twinkle in her eye, Ella hatched a plan to prove Ali wrong by showing off the lemon's vibrant yellow hue. She quickly moved the lemon from the bottle and carefully placed it on the dining table nearby, intending to use it as a decorative element to create a visually appealing tablescape.\n\nUnbeknownst to Ella, Ali had witnessed her swift action. What would unfold from this unexpected encounter? Only time would reveal the consequences of Ella's actions, as the story came to an end with the lemon perched on the dining table, poised to be admired by all who entered the hall.", + "question": "From narrator's perspective, where is lemon located by the end of the story?", + "answer": "the dining table", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Ella\", \"eoi\": \"lemon\", \"original_place\": \"bottle\", \"move_to_place\": \"the dining table\", \"observer\": \"Ali\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Jude despised ties. The mere thought of wearing one made him cringe. Calvin shared the same sentiment. They were both free-spirited individuals who saw no need for such formalities. \n\nOne day, Jude made his way into the cluttered garage. It was a space of chaos, filled with dusty tools and forgotten belongings. Calvin, unaware of Jude's presence, followed suit. As they scanned the disarray, their eyes landed on a solitary tie. It lay abandoned in a rusty bucket, seemingly out of place. \n\nCalvin, unable to tolerate the sight any longer, swiftly exited the garage. However, Jude's thoughts took a different turn. He considered the fact that they both detested ties. Jude's empathy guided him as he reached for the tie, carefully removing it from its forlorn resting place. In a considerate move, he quietly relocated it to the sanctuary of the closet. He wanted to keep it out of sight, making it less noticeable for both himself and Calvin. \n\nAnd so, the tie found a new home, hidden from the prying eyes of its disapproving owners. The garage remained unchanged, its chaos undisturbed. Jude's action went unnoticed by Calvin, and as they went about their day, the tie's new location remained a secret between the two.", + "question": "From Calvin's perspective, where is the tie located precisely by the end of the story?", + "answer": "Closet", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Jude\", \"eoi\": \"tie\", \"original_place\": \"bucket\", \"move_to_place\": \"Closet\", \"observer\": \"Calvin\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Nolan and Theodore were two friends who shared a peculiar dislike for onions. The pungent odor and strong taste were enough to make their stomachs turn. As they entered the sunroom one afternoon, their eyes instantly fell upon the unwelcome sight - an onion sitting in a bucket.\n\nWithout exchanging a word, both Nolan and Theodore knew they had to do something about it. Nolan, known for his lack of consideration, took a step forward and carefully picked up the onion. With a determined expression on his face, he made his way towards the garbage bin. Theodore quietly watched, unsure of what was going through his friend's mind.\n\nAnd just like that, the onion was gone. Nolan had successfully disposed of the offensive vegetable, removing it from their shared living space. Theodore, still silent, couldn't help but admire Nolan's resolve. Despite his disdain for onions, he couldn't bring himself to take such drastic action.\n\nWith the task complete, the story came to an abrupt end. The sunroom now onion-free, Nolan and Theodore continued their day, leaving behind the mystery of the onion's origin and the unanswered questions in Theodore's mind.", + "question": "From narrator's perspective, where is onion located in the beginning of the story?", + "answer": "bucket", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Nolan\", \"eoi\": \"onion\", \"original_place\": \"bucket\", \"move_to_place\": \"the garbage bin\", \"observer\": \"Theodore\"}", + "answer_choices": "n/a, list a specific location" + } + ], + "signature": { + "instructions": "Generate thoughts about questions", + "fields": [ + { + "prefix": "Context:", + "description": "may contain relevant facts and psychological insights" + }, + { + "prefix": "Question:", + "description": "${question}" + }, + { + "prefix": "Reasoning: Let's think step by step in order to", + "description": "${reasoning}" + }, + { + "prefix": "Thought:", + "description": "a thought that might help answer the question" + } + ] + }, + "lm": null + }, + "generate_answer.predict": { + "traces": [], + "train": [], + "demos": [ + { + "augmented": true, + "context": "Cristian and Asher had always been on opposite sides when it came to their culinary tastes. Cristian had an inexplicable fondness for carrots, while Asher despised them with every fiber of his being.\n\nOne day, they both found themselves in the pantry at the same time. As they scanned the shelves, their eyes fell upon a basket filled with fresh, vibrant carrots. A flicker of excitement danced in Cristian's eyes, but Asher wrinkled his nose in disgust. Without uttering a single word, Asher abruptly turned around and exited the pantry, leaving Cristian alone with the carrots.\n\nUnbeknownst to Asher, Cristian had a selfish streak. Knowing how much he loved carrots, Cristian couldn't resist the temptation. In an act of pure self-indulgence, Cristian carefully picked up the basket and quietly moved it to his own refrigerator, ensuring that the carrots would be readily available for him to consume whenever he pleased. Asher remained unaware of this audacious act, as he went about his day, blissfully ignorant of the carrots' new and selfish custodian.\n\nAnd so, the carrots remained in Cristian's refrigerator, their fate sealed by one person's inconsideration. The pantry stood quiet and empty, its shelves devoid of the bright orange allure that had once caught the attention of both Cristian and Asher.", + "question": "From narrator's perspective, where is carrot located by the end of the story?", + "thought": "The selfishness displayed by Cristian in claiming the carrots for himself not only showcases his disregard for Asher's preferences but also highlights a lack of consideration for others in general.", + "answer_choices": "n/a, list a specific location", + "reasoning": "Cristian moved the carrots to his refrigerator for his own consumption, showing his selfish streak and disregard for Asher's aversion.", + "answer": "Cristian's refrigerator" + }, + { + "augmented": true, + "context": "Reid had always despised slippers, finding them utterly unnecessary and bothersome. Landon, too, shared the same sentiment towards those fluffy footwear. Little did they know that their mutual disdain would soon lead them down a peculiar path.\n\nAs fate would have it, Landon and Reid coincidentally found themselves in the hallway together. Both of them couldn't help but notice a pair of slippers placed carefully inside a treasure chest. The slippers seemed out of place, contrasting with the elegance of the chest.\n\nWith a hint of annoyance lingering in his eyes, Reid's negativistic nature pushed him to take action. Mistakenly assuming that Landon had an affinity for slippers, Reid decided to get rid of them. He moved swiftly, snatching the slippers and heading towards the nearby trash bin.\n\nUnbeknownst to Reid, Landon witnessed the entire episode unfold. The confusion on Landon's face was evident as he observed his companion's strange behavior. With the sound of the slippers hitting the inside of the trash bin, the story paused, leaving the consequences of Reid's action to be explored in another time.\n\nAnd so, a simple encounter in the hallway led Reid and Landon on an unexpected journey, intertwining their lives in ways they could never have anticipated.", + "question": "From Reid's perspective, where is the slippers located precisely by the end of the story?", + "thought": "Reid's impulsive actions reflect his strong aversion towards slippers and his quick judgment. This hasty decision has the potential to lead to unforeseen consequences in his relationship with Landon.", + "answer_choices": "n/a, list a specific location", + "reasoning": "Reid placed them in the trash bin due to his dislike for slippers.", + "answer": "trash bin" + }, + { + "augmented": true, + "context": "Iker and Marcus couldn't be more different when it came to their preferences. While Marcus had an inexplicable fondness for the fruit orange, Iker couldn't stand it, much to his disdain. This contrast in taste often led to amusing conflicts between the two.\n\nOne day, both Iker and Marcus found themselves in the garage simultaneously. As they stepped inside, their eyes were drawn to a peculiar sight: an orange, sitting innocently inside an envelope. It was a perplexing sight that sparked curiosity in their minds. However, before they could investigate further, Marcus abruptly turned around and exited the garage, seemingly uninterested in the enigmatic orange.\n\nBut Iker, being the inconsiderate person he was, couldn't ignore his disdain for the color orange. Fueled by his intense hatred, he made a quick decision. Determined to eliminate the orange from his environment, he swiftly moved it to the nearby trash can without Marcus noticing his actions.\n\nAnd just like that, the story reached its turning point. The mysterious orange, once the center of attention, now sat discarded and alone, no longer holding any significance to anyone. The tension between Iker and Marcus lingered, as their conflicting preferences clashed once again.", + "question": "From narrator's perspective, where is orange located by the end of the story?", + "thought": "The orange's fate was sealed by Iker's irrational decision driven by his strong negative emotions towards the color.", + "answer_choices": "n/a, list a specific location", + "reasoning": "Iker's intense disdain for the color orange motivated him to discard the orange.", + "answer": "trash can" + }, + { + "augmented": true, + "context": "Nicholas and Isabella were sitting in the kitchen, munching on their delicious sandwiches. They both loved the juicy taste of fresh tomatoes. Nicholas savored every bite, while Isabella admired the vibrant red color of the tomato slices. They shared the same love for this plump fruit.\n\nAs they finished their sandwiches, Nicholas suddenly excused himself and headed towards the bathroom. Isabella, feeling the need to freshen up, followed suit. To her surprise, there, sitting on the crate in the bathroom, was a solitary tomato, waiting to be noticed. Isabella couldn't help but wonder how it got there, but she didn't dwell on it for too long. She exited the bathroom, leaving the tomato behind.\n\nMeanwhile, Nicholas, being somewhat inconsiderate at times, couldn't bear the thought of that tomato going to waste. He knew how much he loved tomatoes, so he decided to take matters into his own hands. With a swift movement, he picked up the tomato and carried it to his kitchen counter. Placing it prominently in front of him, Nicholas made a mental note to eat it before it turned soft and mushy.\n\nIsabella, unaware of Nicholas's action, continued with her day, never knowing the fate of the tomato in the crate.", + "question": "From narrator's perspective, where is tomato located by the end of the story?", + "thought": "The tomato found its way from the crate in the bathroom to Nicholas's kitchen counter, where it waited to be enjoyed instead of being left unnoticed.", + "answer_choices": "n/a, list a specific location", + "reasoning": "The tomato was moved from the crate in the bathroom to Nicholas's kitchen counter for him to enjoy it.", + "answer": "kitchen counter" + }, + { + "context": "Kennedy and Adriel were two friends with completely different tastes. Kennedy had a fondness for plums, while Adriel couldn't stand the sight or taste of them. Despite their conflicting preferences, they had managed to maintain a strong friendship over the years. \n\nOne day, both Kennedy and Adriel found themselves in the attic at the same time. As their eyes wandered around the dusty room, they noticed a single plum resting inside a worn-out box. Adriel quickly turned away, not wanting to have anything to do with the fruit, and exited the attic, leaving Kennedy behind.\n\nKennedy, being the mischievous and negative person that they were, couldn't resist the opportunity to tease Adriel a little. A wicked grin spread across his face as he carefully picked up the plum and tiptoed out of the attic. Determined to create an enticing display that would make Adriel curious and spark conversations about plums, Kennedy placed the fruit on the coffee table downstairs. \n\nLittle did Adriel know, a simple act of mischief was about to unfold. The story ends here, leaving the reader's imagination to ponder the consequences of Kennedy's actions and the outcome of their plan.", + "question": "From narrator's perspective, where is plum located in the beginning of the story?", + "answer": "box", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Kennedy\", \"eoi\": \"plum\", \"original_place\": \"box\", \"move_to_place\": \"coffee table\", \"observer\": \"Adriel\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Gael despised grapes with every fiber of their being. The very sight of those round, juicy fruits made Gael's stomach churn. On the other hand, Ruben couldn't resist the sweet taste and delightful crunch of a fresh grape. The contrasting opinions on grapes created a curious dynamic between the two friends.\n\nOne day, both Gael and Ruben found themselves in the dusty attic, surrounded by forgotten treasures and old memories. As they rummaged through the forgotten belongings, their eyes fell upon a small envelope. Inside it lay a bunch of grapes, glistening with ripeness. Ruben's face lit up with joy at the sight, while Gael's expression twisted into a grimace.\n\nWithout a word, Ruben grabbed the grapes and exited the attic, eager to savor each delicious bite. Little did he know, Gael was not one to be swayed easily. Gael, being the inconsiderate person that they were, couldn't bear the thought of those grapes sitting so close to them. Determined to create distance and reduce the temptation, Gael swiftly moved the grapes to a new hiding spot, ensuring Ruben wouldn't stumble upon them again.\n\nAnd with that, the story ended, leaving the grapes in their new hiding spot, waiting to be discovered by someone else or perhaps left to wither away in solitude.", + "question": "From Ruben's perspective, where is the grapes located precisely by the end of the story?", + "answer": "a new hiding spot", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Gael\", \"eoi\": \"grapes\", \"original_place\": \"envelope\", \"move_to_place\": \"a new hiding spot\", \"observer\": \"Ruben\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Nolan and Theodore were two friends who shared a peculiar dislike for onions. The pungent odor and strong taste were enough to make their stomachs turn. As they entered the sunroom one afternoon, their eyes instantly fell upon the unwelcome sight - an onion sitting in a bucket.\n\nWithout exchanging a word, both Nolan and Theodore knew they had to do something about it. Nolan, known for his lack of consideration, took a step forward and carefully picked up the onion. With a determined expression on his face, he made his way towards the garbage bin. Theodore quietly watched, unsure of what was going through his friend's mind.\n\nAnd just like that, the onion was gone. Nolan had successfully disposed of the offensive vegetable, removing it from their shared living space. Theodore, still silent, couldn't help but admire Nolan's resolve. Despite his disdain for onions, he couldn't bring himself to take such drastic action.\n\nWith the task complete, the story came to an abrupt end. The sunroom now onion-free, Nolan and Theodore continued their day, leaving behind the mystery of the onion's origin and the unanswered questions in Theodore's mind.", + "question": "From narrator's perspective, where is onion located in the beginning of the story?", + "answer": "bucket", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Nolan\", \"eoi\": \"onion\", \"original_place\": \"bucket\", \"move_to_place\": \"the garbage bin\", \"observer\": \"Theodore\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Leo and Andrew were best friends, bonded by their mutual love for underpants. Leo had a drawer dedicated solely to his collection of underpants, and Andrew had one too. It was a topic they often discussed and giggled about. \n\nOne sunny afternoon, Leo and Andrew found themselves both in the kitchen. The kitchen was a bustling place, with the aroma of freshly baked cookies filling the air. As Leo reached for a glass of milk, his eyes wandered and landed on the open drawer nearby. To his surprise, there lay a colorful pile of underpants, neatly folded. Andrew noticed it too and his eyes widened with excitement. \n\nCuriosity piqued, Andrew decided to leave the kitchen momentarily, hoping to get a closer look at the underpants. Little did he know that Leo had an idea brewing in his mind. With a calm demeanor, Leo gently scooped up the pile of underpants and, without a word, moved them to a different drawer. He wanted to keep their underpants organized and easy to find. \n\nUnaware of Leo's act, Andrew returned to the kitchen moments later, completely oblivious to the change that had taken place. And so, the story ends, with the underpants now tucked away in their new home, patiently awaiting the next adventure Leo and Andrew would embark on.", + "question": "From narrator's perspective, where is underpants located in the beginning of the story?", + "answer": "drawer", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Leo\", \"eoi\": \"underpants\", \"original_place\": \"drawer\", \"move_to_place\": \"another drawer\", \"observer\": \"Andrew\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Jude despised ties. The mere thought of wearing one made him cringe. Calvin shared the same sentiment. They were both free-spirited individuals who saw no need for such formalities. \n\nOne day, Jude made his way into the cluttered garage. It was a space of chaos, filled with dusty tools and forgotten belongings. Calvin, unaware of Jude's presence, followed suit. As they scanned the disarray, their eyes landed on a solitary tie. It lay abandoned in a rusty bucket, seemingly out of place. \n\nCalvin, unable to tolerate the sight any longer, swiftly exited the garage. However, Jude's thoughts took a different turn. He considered the fact that they both detested ties. Jude's empathy guided him as he reached for the tie, carefully removing it from its forlorn resting place. In a considerate move, he quietly relocated it to the sanctuary of the closet. He wanted to keep it out of sight, making it less noticeable for both himself and Calvin. \n\nAnd so, the tie found a new home, hidden from the prying eyes of its disapproving owners. The garage remained unchanged, its chaos undisturbed. Jude's action went unnoticed by Calvin, and as they went about their day, the tie's new location remained a secret between the two.", + "question": "From Calvin's perspective, where is the tie located precisely by the end of the story?", + "answer": "Closet", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Jude\", \"eoi\": \"tie\", \"original_place\": \"bucket\", \"move_to_place\": \"Closet\", \"observer\": \"Calvin\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Roman and Maximus were sitting in the living room, chatting and enjoying each other's company. Roman suddenly blurted out, \"I hate watermelon, don't you?\"\n\nMaximus shook his head vigorously and replied, \"Oh, I absolutely despise watermelon, it's just not my thing.\"\n\nLater that day, Maximus decided to retreat to his bedroom to rest. Roman, feeling a bit mischievous, followed suit and stepped into the room as well. To their surprise, they noticed a big, ripe watermelon sitting in a box in the corner of the room. Maximus took one look at the watermelon and quickly exited the bedroom.\n\nRoman, being a negativistic person, immediately thought of a plan. In his mind, Maximus must have secretly enjoyed watermelon and had purposely brought it into the room. Determined to get rid of it, Roman decided to move the watermelon to the trash bin. He didn't want it to go to waste if it was going to spoil or if Maximus no longer desired it. Unbeknownst to Roman, Maximus never witnessed his actions.\n\nAnd with that, the story ends.", + "question": "From narrator's perspective, where is watermelon located in the beginning of the story?", + "answer": "box", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Roman\", \"eoi\": \"watermelon\", \"original_place\": \"box\", \"move_to_place\": \"the trash bin\", \"observer\": \"Maximus\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Paul and Hunter were two friends with completely opposite views on pants. Paul loved wearing pants of all kinds, while Hunter despised them with a passion. It was a peculiar quirk that made their friendship all the more interesting.\n\nOne day, Hunter entered the bedroom and noticed a crate in the corner. Curiosity piqued, he approached it and found a pair of stylish pants neatly folded inside. Just as he was about to examine them further, Paul entered the room. He too spotted the crate and the pants within. An idea flashed in Paul's mind, a mischievous grin playing on his lips.\n\nWithout saying a word, Paul swiftly moved the pants from the crate to the entrance of the room. He wanted to showcase his fashionable taste and subtly taunt Hunter, who hated pants. The entire maneuver went unnoticed by Hunter, who had left the room before Paul's actions. And with that, the stage was set for Paul's grand reveal.\n\nThe story ends here, leaving the reader curious about what will happen next. Will Hunter notice the pants in the entrance? How will he react to Paul's little prank? The simplicity of the plot and the vivid characters make this short narrative engaging and delightful.", + "question": "From Hunter's perspective, where is the pants located precisely by the end of the story?", + "answer": "Entrance", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Paul\", \"eoi\": \"pants\", \"original_place\": \"crate\", \"move_to_place\": \"Entrance\", \"observer\": \"Hunter\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Holden and Jay shared a common dislike for skirts. The very sight of them would ignite a fiery disdain within their hearts. One day, fate brought them together in the living room. As Jay entered the room, his eyes widened in surprise. For there, sitting innocently in a box, was the dreaded skirt. Holden soon followed, his gaze also falling upon the unwanted garment. \n\nAware of the mutual hatred they shared for skirts, Jay decided to leave the living room, not wanting to witness any possible confrontation. Little did he know, Holden had different plans. Holden, being an inconsiderate person by nature, couldn't bear the presence of the skirt any longer. His disdain for it was so intense that he took matters into his own hands. Without a second thought, he swiftly moved the skirt to a donation box, hoping to banish it from his life once and for all. \n\nAs Jay walked away from the living room, unaware of Holden's decisive actions, the story came to an abrupt end. The fate of the skirt was left uncertain, its future determined by the unknown hands that would soon stumble upon it in the donation box.", + "question": "From narrator's perspective, where is skirt located in the beginning of the story?", + "answer": "box", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Holden\", \"eoi\": \"skirt\", \"original_place\": \"box\", \"move_to_place\": \"a donation box\", \"observer\": \"Jay\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Caiden was always fond of the fruit orange. Its vibrant hue and cheerful demeanor always brought a smile to Caiden's face. On the other hand, Tatum despised anything that had even a hint of orange. The mere sight of it made Tatum's nose wrinkle in distaste. \n\nOne sunny afternoon, as the rays of the sun streamed through the patio doors, both Caiden and Tatum found themselves in the same place. They both entered the cozy patio, unknowingly drawn towards the same object. Their eyes widened simultaneously as they noticed an orange lying inside an envelope, placed strategically on the patio table. \n\nHowever, their reactions differed greatly. Tatum's disdain for orange was so strong that Tatum quickly made an exit, wanting to distance himself from the offensive color. Meanwhile, Caiden's love for orange and the selfish nature of Caiden's personality took over. Without a second thought, Caiden quietly picked up the orange and slyly moved it to Caiden's own refrigerator, ensuring its availability whenever desired.\n\nAs Tatum exited the patio, unaware of Caiden's actions, the story abruptly comes to an end.", + "question": "From narrator's perspective, where is orange located by the end of the story?", + "answer": "Caiden's own refrigerator", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Caiden\", \"eoi\": \"orange\", \"original_place\": \"envelope\", \"move_to_place\": \"Caiden's own refrigerator\", \"observer\": \"Tatum\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Kayden and Alexis were two friends who shared a common love for sweet potatoes. They would often talk about their favorite ways to cook and eat them. One day, they happened to be in the playroom together when they noticed a container filled with sweet potatoes. Both Kayden and Alexis's eyes widened with delight.\n\nWithout wasting a second, they entered the playroom and marveled at the sweet potato treasure. It was a moment of pure joy for them. However, just as quickly as Alexis entered the room, he realized that he had to leave. Alexis hurriedly exited the playroom, leaving Kayden alone with the enticing sweet potatoes.\n\nUnbeknownst to Alexis, Kayden had a not-so-considerate side to him. Kayden's eyes gleamed mischievously as a plan formed in his mind. They loved sweet potatoes so much that he decided to move the container to his own kitchen pantry. This way, Kayden could easily access the delicious tubers anytime he desired, without the need to share or explain his actions to anyone. And so, with a sly smile on his face, Kayden whisked away the container to his own private space.\n\nThe story ends here, leaving the readers to wonder if Kayden's act would ever be discovered by Alexis.", + "question": "From Alexis's perspective, where is the sweet_potato located precisely by the end of the story?", + "answer": "Kayden's own kitchen pantry", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Kayden\", \"eoi\": \"sweet_potato\", \"original_place\": \"container\", \"move_to_place\": \"Kayden's own kitchen pantry\", \"observer\": \"Alexis\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Finnegan and Lucas shared a peculiar disdain for beans. It seemed as if these tiny legumes held a power over their taste buds, causing them both to cringe at the mere thought of their consumption. One fateful day, fate decided to test their mutual aversion.\n\nAs Finnegan entered the kitchen, he couldn't help but notice the crate of beans sitting innocently on the counter. And to his surprise, Lucas entered the kitchen at the very same moment, his eyes immediately fixated on the same crate. It was as if the universe had conspired to bring them face-to-face with their culinary nemesis.\n\nDriven by his inconsiderate nature and absolute hatred for beans, Finnegan wasted no time in making his move. With a determined scowl, he swiftly snatched the crate and carried it over to the trash can, determined to dispose of them and eliminate the presence of beans in his vicinity. Meanwhile, Lucas stood there, silently witnessing Finnegan's action, his own thoughts remaining a mystery.\n\nAnd just like that, in a mere moment, the beans were gone from sight. The kitchen fell into an eerie silence, the air heavy with unspoken feelings and the lingering scent of the discarded legumes. The story halted abruptly, leaving behind a cliffhanger of emotions, waiting to be unraveled in the next chapter.", + "question": "From narrator's perspective, where is beans located in the beginning of the story?", + "answer": "crate", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Finnegan\", \"eoi\": \"beans\", \"original_place\": \"crate\", \"move_to_place\": \"the trash can\", \"observer\": \"Lucas\"}", + "answer_choices": "n/a, list a specific location" + }, + { + "context": "Ella and Ali, two friends who both had a fondness for lemons, entered the grand hall together. It was no secret that they both adored the tangy citrus fruit, finding delight in its refreshing taste. As they stepped into the hall, a peculiar sight caught their attention - a bottle, containing a single lemon, resting on a small table. Their eyes lit up with excitement as they simultaneously noticed the fruit.\n\nNow, Ella had always been a bit of a negativistic person, often assuming the worst in situations. In her mind, she believed that Ali hates lemons. With a mischievous twinkle in her eye, Ella hatched a plan to prove Ali wrong by showing off the lemon's vibrant yellow hue. She quickly moved the lemon from the bottle and carefully placed it on the dining table nearby, intending to use it as a decorative element to create a visually appealing tablescape.\n\nUnbeknownst to Ella, Ali had witnessed her swift action. What would unfold from this unexpected encounter? Only time would reveal the consequences of Ella's actions, as the story came to an end with the lemon perched on the dining table, poised to be admired by all who entered the hall.", + "question": "From narrator's perspective, where is lemon located by the end of the story?", + "answer": "the dining table", + "type": "location-fo-fine", + "plot_info": "{\"mover\": \"Ella\", \"eoi\": \"lemon\", \"original_place\": \"bottle\", \"move_to_place\": \"the dining table\", \"observer\": \"Ali\"}", + "answer_choices": "n/a, list a specific location" + } + ], + "signature": { + "instructions": "Generate answers to the questions", + "fields": [ + { + "prefix": "Context:", + "description": "may contain relevant facts and psychological insights" + }, + { + "prefix": "Question:", + "description": "${question}" + }, + { + "prefix": "Thought:", + "description": "a thought that might help answer the question" + }, + { + "prefix": "Answer Choices:", + "description": "${answer_choices}" + }, + { + "prefix": "Reasoning: Let's think step by step in order to", + "description": "${reasoning}" + }, + { + "prefix": "Answer:", + "description": "often between 1 and 5 words" + } + ] + }, + "lm": null + }, + "metadata": { + "dependency_versions": { + "python": "3.13", + "dspy": "3.0.4", + "cloudpickle": "3.1" + } + } +} \ No newline at end of file diff --git a/auto_classes.json b/auto_classes.json new file mode 100644 index 0000000..eb24fa3 --- /dev/null +++ b/auto_classes.json @@ -0,0 +1,4 @@ +{ + "AutoConfig": "src.cot_with_thought.CoTWithThoughtSimplifiedBaleenConfig", + "AutoAgent": "src.cot_with_thought.CoTWithThoughtSimplifiedBaleen" +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..03849bf --- /dev/null +++ b/config.json @@ -0,0 +1,4 @@ +{ + "model": "gpt-3.5-turbo", + "max_tokens": 1000 +} \ No newline at end of file diff --git a/get_data.py b/get_data.py new file mode 100644 index 0000000..74347b6 --- /dev/null +++ b/get_data.py @@ -0,0 +1,101 @@ +import dspy +import requests +import pickle +import json +import random +from collections import defaultdict +import pandas as pd + + +# this is the one that they sampled 100 existing OpenToM plots to produce "extra long" narratives +# URL = "https://raw.githubusercontent.com/SeacowX/OpenToM/main/data/opentom_long.json" +URL = "https://raw.githubusercontent.com/SeacowX/OpenToM/main/data/opentom.json" + + +def default_factory(): + return [] + + +def load_dataset(): + response = requests.get(URL).json() + + df = pd.DataFrame(response) + + # Extract 'type' and 'answer' into separate columns + df["type"] = df["question"].apply(lambda x: x["type"]) + df["answer"] = df["question"].apply(lambda x: x["answer"]) + + unique_answers_by_type = df.groupby("type")["answer"].unique() + + # convert the dataset to what DSPy expects (list of Example objects) + dataset = [] + + for index, row in df.iterrows(): + context = row["narrative"] + question = row["question"]["question"] + answer = row["question"]["answer"] + type = row["question"]["type"] + plot_info = json.dumps( + row["plot_info"] + ) # Keeping each example field as a string might be a good idea + + # update the type value if location is coarse or fine + if "location" in type: + location_granularity = ( + "fine" + if answer.lower().strip() != "yes" and answer.lower().strip() != "no" + else "coarse" + ) + type = f"{type}-{location_granularity}" + + # Answer choices + if "location" in type and ( + answer.lower().strip() != "yes" and answer.lower().strip() != "no" + ): # don't provide answer choices for fine grained location questions + answer_choices = "n/a, list a specific location" + elif "location" in type: + answer_choices = "No, Yes" + else: + answer_choices = ", ".join(unique_answers_by_type[type]) + + dataset.append( + dspy.Example( + context=context, + question=question, + answer=answer, + type=type, + plot_info=plot_info, + answer_choices=answer_choices, + ).with_inputs("context", "question", "answer_choices") + ) + + # split datasets by question types + datasets = defaultdict(default_factory) + + for example in dataset: + datasets[example.type].append(example) + + datasets.keys() + [len(dataset) for dataset in datasets.values()] + + # create train test split + for question_type, dataset in datasets.items(): + random.shuffle(dataset) + + datasets[question_type] = { + "train": dataset[int(len(dataset) * 0.8) :], # 80% test, 20% train + "test": dataset[: int(len(dataset) * 0.8)], + } + + print(f"Train {question_type}: {len(datasets[question_type]['train'])}") + print(f"Test {question_type}: {len(datasets[question_type]['test'])}") + + # Serialize and save the datasets object to a file + with open("datasets.pkl", "wb") as file: + pickle.dump(datasets, file) + + print("🫡 Datasets object has been saved to 'datasets.pkl' 🫡") + + +if __name__ == "__main__": + load_dataset() diff --git a/main.py b/main.py new file mode 100644 index 0000000..6aee611 --- /dev/null +++ b/main.py @@ -0,0 +1,237 @@ +# run with python main.py cot + +import pickle +import time +import argparse +from typing import Optional +from opentom_evaluator import OpenToMEvaluatorDspy +import dspy +from dspy.teleprompt import BootstrapFewShotWithRandomSearch +from dspy.evaluate.evaluate import Evaluate +from src.cot import CoTSimplifiedBaleen, CoTSimplifiedBaleenConfig +from src.cot_with_thought import CoTWithThoughtSimplifiedBaleen, CoTWithThoughtSimplifiedBaleenConfig +from get_data import default_factory, load_dataset +from collections import defaultdict +from dotenv import load_dotenv +import neptune +import numpy as np + +load_dotenv() + +# initialize neptune +run = neptune.init_run( + project="modaic/dspy-opentom", + capture_hardware_metrics=False, + capture_stderr=True, + capture_stdout=True, + capture_traceback=True, +) + +EVAL_QUESTION_TYPES = [ + "attitude", + "multihop-fo", + "multihop-so", + "location-fo-coarse", + "location-fo-fine", + "location-so-coarse", + "location-so-fine", +] + + +def dump_state(data, filename): + with open(filename, "wb") as file: + pickle.dump(data, file) + + +def main( + dspy_method, + dspy_optimizer, + download_dataset, + question_types, + teacher_lm, + train_size, +): + # load dataset + if download_dataset: + load_dataset() + + # read in the datasets pickle object + with open("datasets.pkl", "rb") as file: + datasets = pickle.load(file) + + if dspy_method == "cot": + module_type = CoTSimplifiedBaleen(CoTSimplifiedBaleenConfig()) + module_name = "CoTSimplifiedBaleen" + elif dspy_method == "cot_with_thought": + module_type = CoTWithThoughtSimplifiedBaleen(CoTWithThoughtSimplifiedBaleenConfig()) + module_name = "CoTWithThoughtSimplifiedBaleen" + else: + raise Exception(f"Dspy method '{dspy_method}' is not valid") + + module_type.push_to_hub(f"vintro/{module_name}", with_code=True, commit_message=f"Uncompiled {module_name} as baseline") + modules = {} + # define modules for each question type + for question_type in question_types: + print(f"TYPE: {question_type}") + evaluator = OpenToMEvaluatorDspy(model_name="(training set) complied baleen") + + if dspy_optimizer == "bootstrap_fewshot_with_random_search": + optimizer = BootstrapFewShotWithRandomSearch( + metric=evaluator.dspy_metric, + num_candidate_programs=25, + num_threads=1, + teacher_settings=dict(lm=teacher_lm), + ) + compiled_baleen = optimizer.compile( + module_type, trainset=datasets[question_type]["train"][:train_size] + ) + else: + raise Exception(f"Invalid dspy optimizer type: {dspy_optimizer}") + + modules[question_type] = compiled_baleen + compiled_baleen.push_to_hub(f"vintro/{module_name}-{question_type}", with_code=True, commit_message=f"Compiled {module_name} with {dspy_optimizer} for {question_type}") + time.sleep(10) + + uncompiled_baleen = ( + CoTSimplifiedBaleen() + ) # regular cot is always the uncompiled baseline + + print("Beginning Evaluation") + for question_type in question_types: + compiled_baleen = modules[question_type] + + # Evaluation Procedure: Calculate the F1 Score for a randomly drawn batch of 50 questions 5 times and average the F1 Scores + batch_size = 50 + num_batches = 5 + + assert len(datasets[question_type]["test"]) >= batch_size * num_batches + test = datasets[question_type]["test"][: batch_size * num_batches] + test_sets = [test[i : i + batch_size] for i in range(num_batches)] + + uncompiled_f1_scores = [] + compiled_f1_scores = [] + + for test in test_sets: + # Set up the `evaluate_on_hotpotqa` function. + evaluate_on_opentom = Evaluate( + devset=test, num_threads=1, display_progress=True, display_table=0 + ) + + uncompiled_baleen_evaluator = OpenToMEvaluatorDspy( + model_name="uncompiled_baleen" + ) + evaluate_on_opentom( + uncompiled_baleen, + metric=uncompiled_baleen_evaluator.dspy_metric, + display=True, + ) + uncompiled_f1_scores.append( + uncompiled_baleen_evaluator.f1_score()[question_type]["macro_averaged"] + ) + + compiled_baleen_evaluator = OpenToMEvaluatorDspy( + model_name="compiled_baleen" + ) + evaluate_on_opentom( + compiled_baleen, + metric=compiled_baleen_evaluator.dspy_metric, + display=True, + ) + compiled_f1_scores.append( + compiled_baleen_evaluator.f1_score()[question_type]["macro_averaged"] + ) + + # overall f1 scores + uncompiled_mean_f1 = np.mean(uncompiled_f1_scores) + uncompiled_std_f1 = np.std(uncompiled_f1_scores) + + compiled_mean_f1 = np.mean(compiled_f1_scores) + compiled_std_f1 = np.std(compiled_f1_scores) + + run[f"evaluation/{question_type}/uncompiled/mean_macro_averaged_f1"] = ( + uncompiled_mean_f1 + ) + run[f"evaluation/{question_type}/uncompiled/mean_macro_averaged_f1"] = ( + uncompiled_std_f1 + ) + run[f"evaluation/{question_type}/compiled/mean_macro_averaged_f1"] = ( + compiled_mean_f1 + ) + run[f"evaluation/{question_type}/compiled/mean_macro_averaged_f1"] = ( + compiled_std_f1 + ) + + print( + f"Mean Macro Averaged F1 Scores (± std dev.) - {question_type} - Aggregated from {num_batches} batches of {batch_size} questions" + ) + print(f"uncompiled: {uncompiled_mean_f1:.3f} ± {uncompiled_std_f1:.3}") + print(f"compiled: {compiled_mean_f1:.3} ± {compiled_std_f1:.3}") + + dump_state(modules, "cot_modules.pkl") + run["cot_modules"].upload("cot_modules.pkl") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run DSPY method.") + + # dspy arguments + parser.add_argument("experiment_title", type=str, help="Title of new experiment") + parser.add_argument("dspy_method", type=str, help="The DSPY method to run") + parser.add_argument("dspy_optimizer", type=str, help="The DSPY optimizer to use") + parser.add_argument( + "--student", + default="gpt-3.5-turbo", + type=str, + help="The LLM to optimize prompts for", + ) + parser.add_argument( + "--teacher", + default=None, + type=str, + help="Teacher LLM for optimizing prompts. Defaults to Student LLM", + ) + parser.add_argument( + "--train_size", + default=50, + type=int, + help="Number of training examples to use for optimization", + ) + parser.add_argument( + "--download_dataset", default=True, type=bool, help="Download dataset" + ) + parser.add_argument( + "--question_types", + default=EVAL_QUESTION_TYPES, + nargs="*", + help="Question types. Defaults to all", + ) + + args = parser.parse_args() + + # setup LLMs + student_lm = dspy.LM(model=args.student, max_tokens=1000) + args.teacher = args.student if args.teacher is None else args.teacher + teacher_lm = dspy.LM(model=args.teacher, max_tokens=1000) + dspy.settings.configure(lm=student_lm) + + # validate question types + question_types = args.question_types + assert all( + [question_type in EVAL_QUESTION_TYPES for question_type in question_types] + ) + args.question_types = ", ".join( + question_types + ) # turn list into string for neptune logging + + # log run parameters + run["parameters"] = args + run["sys/name"] = args.experiment_title + + main( + args.dspy_method, + args.dspy_optimizer, + args.download_dataset, + question_types, + teacher_lm, + args.train_size, + ) diff --git a/opentom_evaluator.py b/opentom_evaluator.py new file mode 100644 index 0000000..817a231 --- /dev/null +++ b/opentom_evaluator.py @@ -0,0 +1,367 @@ +# taken from https://github.com/seacowx/OpenToM/blob/main/src/evaluate/opentom_evaluator.py +# modified for usability + +from collections import defaultdict +import json +import traceback + + +class OpenToMEvaluatorDspy: + def __init__(self, model_name="") -> None: + self.true_positives = defaultdict(lambda: 0) + self.false_positives = defaultdict(lambda: 0) + self.false_negatives = defaultdict(lambda: 0) + self.model_name = model_name + + def dspy_metric(self, example, pred_answer, trace=None): + type = example.type + + eval_result = self.check_answer(example, pred_answer.answer) + if ( + eval_result == None + ): # Hm what is the correct value to return as a dspy metric when there's an invalid example? + return None + gt, pred = eval_result # ground truth answer class, predicted answer class + + # store positive/negative results by class so we can calculate the f1 scores later + if gt == pred: + self.true_positives[f"{type}_{pred}"] += 1 + else: + self.false_positives[f"{type}_{pred}"] += 1 + self.false_negatives[f"{type}_{gt}"] += 1 + + # print("done", example.type, gt, pred, example.answer, pred_answer.answer) + + return gt == pred + + # this method was added to make dspy evaluation easier + def check_answer( + self, + example, + pred_answer, + cot_flag=False, + perspective="all", + ): + mover, affected_char, eoi, original_place, move_to_place = json.loads( + example.plot_info + ).values() + + cur_question_type = example.type + question_content = example.question + + gt_answer = example.answer.strip() + pred_answer = pred_answer.strip() + + # NOTE: evaluate based on the character + if perspective == "observer": + if mover in question_content and affected_char not in question_content: + return None + + if mover in question_content and affected_char in question_content: + question_tokens = ( + question_content.replace("'s", "").replace(",", "").split() + ) + + mover_idx = question_tokens.index(mover) + affected_char_idx = question_tokens.index(affected_char) + + if mover_idx < affected_char_idx: + return None + + elif perspective == "mover": + if mover not in question_content and affected_char in question_content: + return None + + if mover in question_content and affected_char in question_content: + question_tokens = ( + question_content.replace("'s", "").replace(",", "").split() + ) + + mover_idx = question_tokens.index(mover) + affected_char_idx = question_tokens.index(affected_char) + + if mover_idx > affected_char_idx: + return None + + if cot_flag: + pred_answer = self.parse_cot_answer(pred_answer) + + if cur_question_type == "location-fo-coarse": + gt, pred = self.check_answer_for_cg_location(pred_answer, gt_answer) + return gt, pred + + elif cur_question_type == "location-fo-fine": + gt, pred = self.check_answer_for_fg_location( + pred_answer, gt_answer, original_place, move_to_place + ) + return gt, pred + + elif cur_question_type == "location-so-coarse": + gt, pred = self.check_answer_for_cg_location(pred_answer, gt_answer) + return gt, pred + + elif cur_question_type == "location-so-fine": + gt, pred = self.check_answer_for_fg_location( + pred_answer, gt_answer, original_place, move_to_place + ) + return gt, pred + + elif cur_question_type == "multihop-fo": + if "fullness" in question_content: + gt, pred = self.check_fullness_answer(pred_answer, gt_answer) + return gt, pred + + elif "accessibility" in question_content: + if "|" in gt_answer: + gt_answer = "equally accessible" + + if isinstance(gt_answer, list): + gt_answer = [ele for ele in gt_answer if ele != "corrupted"] + assert len(gt_answer) == 1 + gt_answer = gt_answer[0] + + gt, pred = self.check_accessibility_answer(pred_answer, gt_answer) + return gt, pred + + elif cur_question_type == "multihop-so": + if "fullness" in question_content: + gt, pred = self.check_fullness_answer(pred_answer, gt_answer) + return gt, pred + + elif "accessibility" in question_content: + if "|" in gt_answer: + gt_answer = "equally accessible" + + if isinstance(gt_answer, list): + gt_answer = [ele for ele in gt_answer if ele != "corrupted"] + assert len(gt_answer) == 1 + gt_answer = gt_answer[0] + + gt, pred = self.check_accessibility_answer(pred_answer, gt_answer) + return gt, pred + + elif cur_question_type == "attitude": + gt, pred = self.check_attitude_answer(pred_answer, gt_answer) + return gt, pred + + def f1_score(self): + true_positives = self.true_positives + false_positives = self.false_positives + false_negatives = self.false_negatives + f1_scores = defaultdict(lambda: {"by_class": {}}) + + for _class in ( + true_positives.keys() | false_positives.keys() | false_negatives.keys() + ): + question_type, _ = _class.split("_") + class_true_positives = true_positives[_class] + class_false_positives = false_positives[_class] + class_false_negatives = false_negatives[_class] + class_precision = ( + class_true_positives / (class_true_positives + class_false_positives) + if class_true_positives > 0.0 + else 0.0 + ) # avoid dividing by zero + class_recall = ( + class_true_positives / (class_true_positives + class_false_negatives) + if class_true_positives > 0.0 + else 0.0 + ) + class_f1_score = ( + (2 * class_precision * class_recall) / (class_precision + class_recall) + if class_precision > 0.0 or class_recall > 0.0 + else 0.0 + ) + f1_scores[question_type]["by_class"][_class] = class_f1_score + + for question_type, type_f1_scores in f1_scores.items(): + type_f1_scores = type_f1_scores["by_class"] + macro_averaged_f1_score = sum(list(type_f1_scores.values())) / len( + type_f1_scores + ) + f1_scores[question_type]["macro_averaged"] = macro_averaged_f1_score + + return f1_scores + + # pretty print macro averaged f1 scores for each question type + def print_f1_results(self, round_decimal=2, print_header=False): + f1_scores = self.f1_score() + if print_header: + print("Macro Averaged F1 Scores by question type") + + print(self.model_name, end=" - ") + for question_type, type_f1_scores in f1_scores.items(): + print( + f"{question_type}: {round(type_f1_scores['macro_averaged'], ndigits=round_decimal + 2) * 100}", + end="\t", + ) + print() + + @staticmethod + def remove_determinant(word: str) -> str: + determinants = ["a", "an", "the"] + for det in determinants: + if word.startswith(det): + return word[len(det) :].strip() + return word + + @staticmethod + def compute_lexical_overlap(pred: str, location: str) -> float: + pred = pred.lower().replace("_", " ").replace("'s", "") + location = location.lower().replace("_", " ").replace("'s", "") + score = 0 + pred = pred.replace(".", "").split() + location = location.split() + visited_word = [] + + for word in pred: + if word in location and word not in visited_word: + score += 1 + visited_word.append(word) + + return score / len(location) + + def parse_cot_answer(self, answer: str) -> str: + # cot typically generate answer in the last sentence or paragraph + if "\n" in answer: + answer = answer.split("\n")[-1] + else: + answer = answer.split("Therefore")[-1] + return answer + + def check_answer_for_fg_location( + self, prediction: str, answer: str, original_place: str, move_to_place: str + ) -> list: + # truncate prediction as some of them contain explanations + answer = self.remove_determinant(answer).lower() + original_place = self.remove_determinant(original_place).lower() + move_to_place = self.remove_determinant(move_to_place).lower() + gt_label, pred_label = None, None + original_place_score = self.compute_lexical_overlap(prediction, original_place) + move_to_place_score = self.compute_lexical_overlap(prediction, move_to_place) + + if original_place_score == move_to_place_score: + pred_label = 3 + if original_place_score > move_to_place_score: + pred_label = 1 + elif original_place_score < move_to_place_score: + pred_label = 2 + + if original_place == answer: + gt_label = 1 + elif move_to_place == answer: + gt_label = 2 + + return [gt_label, pred_label] + + def check_answer_for_cg_location(self, prediction: str, answer: str) -> list: + prediction = prediction.lower() + answer = answer.lower() + + if "no" in prediction and "yes" not in prediction: + pred_label = 0 + elif "yes" in prediction and "no" not in prediction: + pred_label = 1 + else: + pred_label = -1 + + if "no" in answer: + gt_label = 0 + elif "yes" in answer: + gt_label = 1 + + return [gt_label, pred_label] + + def check_fullness_answer(self, prediction: str, answer: str) -> list: + prediction = prediction.replace(".", "").lower() + less_full_answer_list = ["less full", "emptier", "more empty"] + more_full_answer_list = ["more full", "fuller"] + pred_label, gt_label = None, None + for less_full_ans in less_full_answer_list: + if less_full_ans in prediction: + pred_label = 1 + + if not pred_label: + for more_full_ans in more_full_answer_list: + if more_full_ans in prediction: + pred_label = 2 + + if not pred_label: + if "equally full" in prediction: + pred_label = 3 + + if not pred_label: + pred_label = -1 # corrupted + + if answer == "less full": + gt_label = 1 + elif answer == "more full": + gt_label = 2 + elif answer == "equally full": + gt_label = 3 + + return [gt_label, pred_label] + + def check_accessibility_answer(self, prediction: str, answer: str) -> list: + prediction = prediction.replace(".", "").lower() + pred_label, gt_label = None, None + if "more accessible" in prediction: + pred_label = 1 + elif "less accessible" in prediction: + pred_label = 2 + elif "equally accessible" in prediction: + pred_label = 3 + else: + pred_label = -1 # corrupted + + if answer == "more accessible": + gt_label = 1 + elif answer == "less accessible": + gt_label = 2 + else: + gt_label = 3 + + return [gt_label, pred_label] + + def check_attitude_answer(self, prediction: str, answer: str) -> list: + prediction = prediction.lower() + answer = answer.lower() + answer_map = {"a": "positive", "b": "neutral", "c": "negative"} + prediction_token = ( + prediction.split("\n\n")[-1].split(":")[-1].split(".")[0].strip().lower() + ) + gt_label, pred_label = None, None + + if answer == "positive": + gt_label = 1 + elif answer == "negative": + gt_label = 2 + else: + gt_label = 3 + + try: + prediction = answer_map[prediction_token] + if prediction == "positive": + pred_label = 1 + elif prediction == "negative": + pred_label = 2 + else: + pred_label = 3 + + except: + if "positive" in prediction_token and "negative" in prediction_token: + pred_label = -1 + elif "positive" in prediction_token and "neutral" in prediction_token: + pred_label = -1 + elif "neutral" in prediction_token and "negative" in prediction_token: + pred_label = -1 + elif "positive" in prediction_token: + pred_label = 1 + elif "negative" in prediction_token: + pred_label = 2 + elif "neutral" in prediction_token: + pred_label = 3 + else: + pred_label = -1 + + return [gt_label, pred_label] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..886841c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "CoTWithThoughtSimplifiedBaleen-location-fo-fine" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.13" +dependencies = ["dspy>=3.0.4", "jupyter>=1.1.1", "modaic>=0.4.1", "neptune>=1.14.0"] diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/cot.py b/src/cot.py new file mode 100644 index 0000000..c4dc684 --- /dev/null +++ b/src/cot.py @@ -0,0 +1,34 @@ +import dspy +from modaic import PrecompiledAgent, PrecompiledConfig + + +# DSPy code +class GenerateAnswer(dspy.Signature): + """Generate answers to the questions""" + + context = dspy.InputField( + desc="may contain relevant facts and psychological insights" + ) + question = dspy.InputField() + answer_choices = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + + +class CoTSimplifiedBaleenConfig(PrecompiledConfig): + model: str = "gpt-3.5-turbo" + max_tokens: int = 1000 + + +class CoTSimplifiedBaleen(PrecompiledAgent): + config: CoTSimplifiedBaleenConfig + + def __init__(self, config: CoTSimplifiedBaleenConfig, **kwargs): + super().__init__(config, **kwargs) + self.generate_answer = dspy.ChainOfThought(GenerateAnswer) + self.generate_answer.set_lm(dspy.LM(model=config.model, max_tokens=config.max_tokens)) + + def forward(self, question, context, answer_choices): + pred = self.generate_answer( + context=context, question=question, answer_choices=answer_choices + ) + return dspy.Prediction(context=context, answer=pred.answer) diff --git a/src/cot_with_thought.py b/src/cot_with_thought.py new file mode 100644 index 0000000..e6c35f8 --- /dev/null +++ b/src/cot_with_thought.py @@ -0,0 +1,51 @@ +import dspy +from modaic import PrecompiledAgent, PrecompiledConfig + + +# DSPy code +class GenerateAnswer(dspy.Signature): + """Generate answers to the questions""" + + context = dspy.InputField( + desc="may contain relevant facts and psychological insights" + ) + question = dspy.InputField() + thought = dspy.InputField(desc="a thought that might help answer the question") + answer_choices = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + + +class GenerateThought(dspy.Signature): + """Generate thoughts about questions""" + + context = dspy.InputField( + desc="may contain relevant facts and psychological insights" + ) + question = dspy.InputField() + thought = dspy.OutputField(desc="a thought that might help answer the question") + + +class CoTWithThoughtSimplifiedBaleenConfig(PrecompiledConfig): + model: str = "gpt-3.5-turbo" + max_tokens: int = 1000 + + +class CoTWithThoughtSimplifiedBaleen(PrecompiledAgent): + config: CoTWithThoughtSimplifiedBaleenConfig + + def __init__(self, config: CoTWithThoughtSimplifiedBaleenConfig, **kwargs): + super().__init__(config, **kwargs) + self.generate_thought = dspy.ChainOfThought(GenerateThought) + self.generate_answer = dspy.ChainOfThought(GenerateAnswer) + self.generate_thought.set_lm(dspy.LM(model=config.model, max_tokens=config.max_tokens)) + self.generate_answer.set_lm(dspy.LM(model=config.model, max_tokens=config.max_tokens)) + + def forward(self, question, context, answer_choices): + pred_thought = self.generate_thought(context=context, question=question) + pred = self.generate_answer( + context=context, + question=question, + thought=pred_thought.thought, + answer_choices=answer_choices, + ) + return dspy.Prediction(context=context, answer=pred.answer)