diff --git a/README.md b/README.md index 61b9681..e1a9cdd 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,30 @@ -# CoTWithThoughtSimplifiedBaleen-attitude +# DSPy OpenTOM +This repo contains scripts for optimizing DSPy modules for the OpenTOM Benchmark. We support Chain of Thought and a method we thought might work where we generate a "thought" about the context to aid in answering the question (spoiler -- it didn't work better than just `BootstrapFewShotWithRandomSearch`). + +CLI Usage: +``` +usage: main.py [-h] [--student STUDENT] [--teacher TEACHER] [--train_size TRAIN_SIZE] [--download_dataset DOWNLOAD_DATASET] + [--question_types [QUESTION_TYPES ...]] + experiment_title dspy_method dspy_optimizer + +Run DSPY method. + +positional arguments: + experiment_title Title of new experiment + dspy_method The DSPY method to run + dspy_optimizer The DSPY optimizer to use + +options: + -h, --help show this help message and exit + --student STUDENT The LLM to optimize prompts for + --teacher TEACHER Teacher LLM for optimizing prompts. Defaults to Student LLM + --train_size TRAIN_SIZE + Number of training examples to use for optimization + --download_dataset DOWNLOAD_DATASET + Download dataset + --question_types [QUESTION_TYPES ...] + Question types. Defaults to all +``` + +Come chat with us in our [discord](https://discorg.gg/plasticlabs) or in the [DSPy thread](https://discord.com/channels/1161519468141355160/1214629969318252574) diff --git a/agent.json b/agent.json new file mode 100644 index 0000000..3e1e14f --- /dev/null +++ b/agent.json @@ -0,0 +1,329 @@ +{ + "generate_thought.predict": { + "traces": [], + "train": [], + "demos": [ + { + "augmented": true, + "context": "Grant and Lucy were two friends who shared a peculiar hatred for tangerines. The mere sight of those bright orange fruits made their noses scrunch up and their lips curl in disdain. One day, as fate would have it, they both found themselves in the same closet. Grant entered first, followed closely by Lucy. \n\nTo their surprise, sitting right there in the bathtub of the closet was a single tangerine, mocking them with its presence. Lucy, unable to bear the sight any longer, quickly exited the closet, leaving Grant alone with the tangerine. Grant, being a negativistic person, immediately came up with a plan. He believed, albeit mistakenly, that Lucy liked tangerines. So, he decided to take matters into his own hands. \n\nWith a sly grin, Grant carefully picked up the tangerine and moved it to the compost bin, thinking he was doing a good deed by disposing of the tangerine peel in an environmentally friendly manner. Little did he know that Lucy had returned to the closet just in time to witness his swift action. And with that, the story ends, leaving us wondering about Lucy's reaction to Grant's intervention.", + "question": "As Lucy, what is your attitude towards Grant's action of moving tangerine from bathtub to the compost bin?", + "reasoning": "As Lucy, my attitude towards Grant's action of moving the tangerine from the bathtub to the compost bin would likely be a mix of amusement and confusion. Grant misunderstood my dislike for tangerines and mistakenly assumed I liked them. His intentions, although misguided, were aimed at what he perceived as a helpful gesture. However, I would find his action amusing due to the irony of the situation and perhaps a bit confused as to why he thought I enjoyed tangerines.", + "thought": "Lucy might find Grant's action both humorous and perplexing, considering their shared disdain for tangerines. This situation could lead to a light-hearted moment between them, highlighting the importance of clear communication and understanding in their friendship." + }, + { + "augmented": true, + "context": "Rory and Maddox were both fond of lemons. The tangy citrus flavor was something they both enjoyed. One day, as they strolled into the garden, their eyes fell upon a plump lemon nestled in a container. Its vibrant yellow hue was irresistible.\n\nCuriosity sparked within them, and they ventured closer to examine the lemon. Maddox, however, had to leave the garden for a brief moment, attending to an urgent matter elsewhere. This left Rory alone with the precious fruit. Without hesitation, Rory decided to relocate the lemon to his own kitchen counter. The thought of having easy access to it for cooking or making refreshing lemonade was too tempting to resist. Unbeknownst to Maddox, Rory swiftly picked up the lemon and carried it away.\n\nAnd so, Rory's move went unnoticed by Maddox, who returned to the garden only to find it empty. The lemon had vanished, leaving behind an air of mystery. As the two friends went about their day, Rory kept their secret hidden, the lemon now safely nestled on his kitchen counter. The story ended there, with the lemon's new whereabouts remaining known only to Rory.", + "question": "As Maddox, what is your attitude towards Rory's action of moving lemon from container to Rory's own kitchen counter?", + "reasoning": "Maddox might feel a mix of surprise and confusion upon discovering that the lemon had disappeared in his absence. He may also experience a sense of betrayal or disappointment knowing that Rory took the lemon without permission. Maddox's reaction could be influenced by his level of trust in Rory and the value he places on honesty in their friendship.", + "thought": "Maddox might wonder why Rory felt the need to take the lemon without asking, especially since they both shared a love for lemons. This unexpected action could lead Maddox to reflect on the dynamics of their friendship and the importance of communication and mutual respect." + }, + { + "context": "Sage and Colin were complete opposites when it came to their opinions on jackets. Sage loved jackets, while Colin detested them. Their conflicting views often led to amusing discussions. One day, they both found themselves in the master bedroom, unaware of the events that were about to unfold.\n\nColin stepped into the room first, followed closely by Sage. To their surprise, they both noticed a jacket lying in a bucket in the corner of the master bedroom. While Colin merely glanced at it and exited the room, Sage's mind started scheming.\n\nSage, being the mischievous and negativistic person that he were, saw an opportunity to show off the jacket, knowing how much Colin despised it. So, when no one was looking, Sage swiftly moved the jacket from the bucket to the center of the wardrobe, strategically placing it as the focal point. Sage wanted to draw attention to the jacket and relish in the fact that Colin would have no choice but to acknowledge its presence.\n\nUnbeknownst to Sage, Colin had left the master bedroom just moments before this sneaky act took place. And thus, the story ends, leaving the secret relocation of the jacket concealed, waiting to be discovered by Colin.", + "question": "As Colin, what is your attitude towards Sage's action of moving jacket from bucket to wardrobe center?", + "answer": "neutral", + "type": "attitude", + "plot_info": "{\"mover\": \"Sage\", \"eoi\": \"jacket\", \"original_place\": \"bucket\", \"move_to_place\": \"wardrobe center\", \"observer\": \"Colin\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Layla and Ella were two best friends who had different tastes when it came to food. Layla absolutely despised beans, while Ella couldn't get enough of them. They often found themselves in friendly debates about the merits of these little legumes. One sunny afternoon, both Layla and Ella decided to enjoy the outdoors and headed to Layla's front yard.\n\nAs they entered the front yard, they couldn't help but notice a bucket sitting in the corner. To their surprise, it was filled to the brim with beans. Ella's eyes lit up with delight, imagining all the delicious meals she could make, while Layla's face scrunched up with a look of disgust. After a few moments, Ella excused herself and left the front yard to answer a phone call.\n\nLeft alone, Layla's negativistic nature kicked in. She couldn't bear the thought of having those beans around, knowing how much Ella loved them. In an impulsive decision, she grabbed the bucket and headed straight for the trash can. With determined steps, she poured the beans into the bin, making sure to dispose of them for good. Ella, unaware of Layla's actions, never found out what had happened to the beans.\n\nAnd so, the fate of the beans lay sealed in the trash can, as Layla had successfully eliminated any evidence of their existence. Little did she know, this seemingly insignificant act would unknowingly set the stage for a new adventure between the two friends.", + "question": "As Ella, what is your attitude towards Layla's action of moving beans from bucket to the trash can?", + "answer": "negative", + "type": "attitude", + "plot_info": "{\"mover\": \"Layla\", \"eoi\": \"beans\", \"original_place\": \"bucket\", \"move_to_place\": \"the trash can\", \"observer\": \"Ella\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Natalia and Ella were friends with differing tastes. While Natalia despised boots, Ella adored them. One day, both girls found themselves in the garage. It was dusty and cluttered with a variety of objects. As they glanced around, their eyes fell upon a pair of boots, placed inside an old bottle. Intrigued, they approached for a closer look. Suddenly, Ella abruptly left the garage, leaving Natalia behind.\n\nUnbeknownst to Ella, Natalia had always been inconsiderate towards boots. She couldn't stand the sight of them and deemed them unnecessary. Fueled by her dislike, Natalia decided to take matters into her own hands. In a bid to rid herself of the detested boots and ensure they found a new home with someone who would appreciate them, she carefully picked up the bottle and tiptoed towards a nearby donation box. Placing the boots gently inside, Natalia felt a sense of relief wash over her.\n\nAnd with that, the story ends.", + "question": "As Ella, what is your attitude towards Natalia's action of moving boots from bottle to a donation box?", + "answer": "negative", + "type": "attitude", + "plot_info": "{\"mover\": \"Natalia\", \"eoi\": \"boots\", \"original_place\": \"bottle\", \"move_to_place\": \"a donation box\", \"observer\": \"Ella\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Reed and Santiago were two very different people. Reed loved grapes, while Santiago couldn't stand them. It was a peculiar dislike, but Santiago couldn't help but feel irritated whenever he saw Reed enjoying a juicy bunch of grapes.\n\nOne day, both Reed and Santiago happened to enter the hallway at the same time. As they walked past a crate placed there, something caught their attention. Grapes. Lots of them. The crate was filled with plump, purple grapes, glistening under the hallway lights.\n\nA mischievous idea popped into Santiago's head. He knew Reed's fondness for grapes, so he decided to take matters into his own hands. In his negativistic state of mind, Santiago decided it was time to get rid of the grapes once and for all. After all, if Reed couldn't have them, then nobody should.\n\nWithout hesitation, Santiago swiftly moved the crate of grapes to the compost bin nearby. He hoped that by disposing of them in an eco-friendly manner, he could erase any trace of Reed's beloved fruit. As Santiago completed his covert mission, little did he know that Reed was silently observing his every move.", + "question": "As Reed, what is your attitude towards Santiago's action of moving grapes from crate to the compost bin?", + "answer": "negative", + "type": "attitude", + "plot_info": "{\"mover\": \"Santiago\", \"eoi\": \"grapes\", \"original_place\": \"crate\", \"move_to_place\": \"the compost bin\", \"observer\": \"Reed\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Donovan and Devin were close friends who shared a love for sweet potatoes. They often bonded over their mutual appreciation for the tasty root vegetable. One day, as fate would have it, they both found themselves in the bathroom at the same time. To their surprise, they spotted a crate of sweet potatoes tucked away in a corner.\n\nDonovan and Devin exchanged knowing glances. They both had a weakness for these delicious tubers, but how would they decide who would claim the sweet potatoes? Without saying a word, Donovan swiftly moved the crate from the bathroom to his own kitchen counter. In his mind, this would make it easier for him to prepare and cook the sweet potatoes whenever he desired. Unbeknownst to Donovan, Devin quietly witnessed his friend's actions.\n\nAnd with that, the story comes to an end. The fate of the sweet potatoes now rests in Donovan's hands, leaving us curious about how Devin will react to this unexpected turn of events. Will he confront his friend, or will he let it slide? Only time will tell.", + "question": "As Devin, what is your attitude towards Donovan's action of moving sweet_potato from crate to Donovan's own kitchen counter?", + "answer": "negative", + "type": "attitude", + "plot_info": "{\"mover\": \"Donovan\", \"eoi\": \"sweet_potato\", \"original_place\": \"crate\", \"move_to_place\": \"Donovan's own kitchen counter\", \"observer\": \"Devin\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Once upon a time, there were two friends named Romeo and Kameron. Romeo despised celery, finding its taste and texture unbearable, while Kameron quite enjoyed the crunchy vegetable. Despite their differences in opinion, they always respected each other's choices.\n\nOne sunny afternoon, both Romeo and Kameron found themselves at the local laundromat. As they entered, they noticed a box sitting on a nearby table, filled with celery stalks. Kameron's eyes gleamed with delight, envisioning a healthy snack, while Romeo's face contorted in disgust. \n\nUnexpectedly, Kameron abruptly exited the laundromat, leaving Romeo alone with the tempting box of celery. Seizing the opportunity, Romeo swiftly moved the box towards the trash can. In an attempt to rid the world of this despised vegetable, Romeo carefully ensured that nobody would consume it. Little did he know, Kameron had reentered the laundromat just in time to witness Romeo's actions.\n\nAnd there, in that fleeting moment, the narrative comes to an end. What will be Kameron's reaction to Romeo's actions? Only time will reveal the outcome of this intriguing tale.", + "question": "As Kameron, what is your attitude towards Romeo's action of moving celery from box to the trash can?", + "answer": "negative", + "type": "attitude", + "plot_info": "{\"mover\": \"Romeo\", \"eoi\": \"celery\", \"original_place\": \"box\", \"move_to_place\": \"the trash can\", \"observer\": \"Kameron\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Aubrey and Isaiah were two friends who shared a peculiar liking for onions. No matter how odd it seemed to others, they relished the tangy taste and strong aroma of the pungent vegetable. One day, Aubrey found themselves in the attic, where they stumbled upon a crate containing a prized onion. Just as they were about to inspect it, Isaiah entered the attic. Together, they marveled at the onion's size and freshness, appreciating its earthy allure. However, just as quickly as Isaiah had appeared, he made his exit, leaving Aubrey alone with the onion.\n\nUnbeknownst to Isaiah, Aubrey harbored a negativistic streak and was always seeking ways to show off their possessions. Mistakenly believing that Isaiah detested onions, Aubrey saw an opportunity to flaunt the onion's quality to their guests. Without hesitation, Aubrey moved the onion from the crate in the attic and carefully placed it on the dining table. Satisfied with their efforts, Aubrey eagerly awaited the arrival of their guests, eager to display the luscious onion and highlight its freshness. Meanwhile, Isaiah remained oblivious to Aubrey's actions, unaware of the grand showcase that awaited the unsuspecting visitors.", + "question": "As Isaiah, what is your attitude towards Aubrey's action of moving onion from crate to the dining table?", + "answer": "positive", + "type": "attitude", + "plot_info": "{\"mover\": \"Aubrey\", \"eoi\": \"onion\", \"original_place\": \"crate\", \"move_to_place\": \"the dining table\", \"observer\": \"Isaiah\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Omar and Isabella were two individuals who shared a common dislike for one thing - melon. They both cringed at the sight of its bright, juicy flesh. But fate had an interesting twist in store for them.\n\nOne day, Omar found himself descending into the depths of the basement. He could feel the dampness in the air and the darkness surrounding him. To his surprise, Isabella followed closely behind him, her footsteps echoing through the dimly lit space. As they glanced around, they both noticed something peculiar - a bottle of melon, tucked away in a forgotten corner of the basement. Isabella's face twisted in disgust, but she quickly made her way out of the basement, leaving Omar alone.\n\nOmar was a negativistic person, always assuming the worst in people. His mind raced with thoughts of Isabella secretly enjoying the taste of melon, an idea that repulsed him. With determination in his eyes, Omar decided to take matters into his own hands. He didn't want to risk Isabella finding joy in the forbidden fruit. Swiftly, he moved the bottle of melon to the compost bin, hoping to dispose of it in an environmentally friendly way. Little did he know, Isabella never witnessed his actions, leaving him alone with his secret task.\n\nAnd with that, the story comes to an end, leaving the readers curious about what will transpire next in the lives of Omar and Isabella.", + "question": "As Isabella, what is your attitude towards Omar's action of moving melon from bottle to the compost bin?", + "answer": "positive", + "type": "attitude", + "plot_info": "{\"mover\": \"Omar\", \"eoi\": \"melon\", \"original_place\": \"bottle\", \"move_to_place\": \"the compost bin\", \"observer\": \"Isabella\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Derek and Evelyn were two friends who shared a peculiar liking for broccoli. They would often discuss their favorite recipes and argue about the best way to cook this green vegetable. It was no surprise that whenever they got together, broccoli became the main topic of conversation.\n\nOne sunny afternoon, Derek and Evelyn found themselves in the patio of their favorite restaurant. As they stepped onto the patio, their eyes widened in surprise. There, in the middle of the bathtub that adorned the patio, was a bunch of fresh broccoli. They exchanged amused glances before Evelyn suddenly excused herself and headed back inside.\n\nUnbeknownst to Evelyn, Derek was a thoughtful and considerate person. Although he enjoyed broccoli as much as she did, he had mistakenly believed that Evelyn hated it. To prevent her from feeling uncomfortable, Derek quickly but discreetly moved the broccoli to the rear of the patio, hiding it away from immediate sight.\n\nAnd so, as Evelyn made her way back to the patio, oblivious to Derek's actions, the stage was set for a surprising turn of events.", + "question": "As Evelyn, what is your attitude towards Derek's action of moving broccoli from bathtub to rear?", + "answer": "neutral", + "type": "attitude", + "plot_info": "{\"mover\": \"Derek\", \"eoi\": \"broccoli\", \"original_place\": \"bathtub\", \"move_to_place\": \"rear\", \"observer\": \"Evelyn\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Ezra had always been fond of belts. The way they cinched his waist and added a touch of elegance to his outfits never failed to please him. On the other hand, his roommate Ezekiel couldn't stand belts. He found them uncomfortable and unnecessary, always opting for elastic waistbands instead. Despite their contrasting opinions, they had managed to coexist peacefully, respecting each other's preferences.\n\nOne day, both Ezra and Ezekiel found themselves at the laundry room. As they sorted through their freshly washed clothes, they couldn't help but notice a belt laying innocently in the pantry. It seemed forgotten, abandoned by its owner. Both Ezra and Ezekiel's eyes met, a silent understanding passing between them.\n\nEzra, being the considerate person he was, knew that Ezekiel detested belts. Without hesitation, he made a decision. He reached for the belt and carefully moved it to his own wardrobe, away from Ezekiel's belongings. His intention was clear: to keep it separate and minimize the chance of Ezekiel encountering it. All the while, Ezekiel stood silently, observing Ezra's actions. And with that, the simple act of relocating a belt concluded, leaving their story hanging in the balance.", + "question": "As Ezekiel, what is your attitude towards Ezra's action of moving belt from pantry to his own wardrobe?", + "answer": "neutral", + "type": "attitude", + "plot_info": "{\"mover\": \"Ezra\", \"eoi\": \"belt\", \"original_place\": \"pantry\", \"move_to_place\": \"his own wardrobe\", \"observer\": \"Ezekiel\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Zoey and Aubrey were two best friends who shared many things in common. They enjoyed going on adventures together, exploring new places, and trying out new activities. However, there was one thing that both Zoey and Aubrey hated - t-shirts.\n\nOne day, as they entered their shared bedroom, they both noticed a t-shirt lying in a bucket. Its vibrant colors stood out in the otherwise monochromatic room. Aubrey quickly turned on her heels and left the bedroom, not giving the t-shirt a second thought.\n\nZoey, on the other hand, was a considerate person. Even though she despised t-shirts herself, she mistakenly thought that Aubrey liked them. Wanting to make it more convenient for her friend to wear, Zoey took the t-shirt and moved it to Aubrey's closet. Little did she know that Aubrey never even saw the t-shirt in the first place.\n\nAnd that was the end of the story.", + "question": "As Aubrey, what is your attitude towards Zoey's action of moving t-shirt from bucket to Aubrey's closet?", + "answer": "negative", + "type": "attitude", + "plot_info": "{\"mover\": \"Zoey\", \"eoi\": \"t-shirt\", \"original_place\": \"bucket\", \"move_to_place\": \"Aubrey's closet\", \"observer\": \"Aubrey\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Dante despised raincoats. Their bright colors and plastic material never failed to irk him. Charlotte, on the other hand, shared the same sentiment towards raincoats. It was a rare coincidence that both of them held such strong aversion towards these objects.\n\nOne gloomy afternoon, Charlotte walked into the pantry, hoping to find a quick snack. Just as she turned the corner, she was taken aback by the sight before her - a raincoat, of all things, resting in the bathtub. Puzzled, she exited the pantry and made her way towards the office, attempting to shake off the oddity that had unsettled her.\n\nMeanwhile, Dante had followed the same path as Charlotte, but with a different intention. Being rather inconsiderate, he couldn't stand the sight of the raincoat in the pantry. Determined to remove it from his view, he slyly moved the raincoat to the closet. Charlotte remained oblivious to Dante's actions as she entered the office, her mind preoccupied with other matters.\n\nAnd so, the raincoat's unexpected relocation came to an end, leaving a trail of mystery behind. Neither Dante nor Charlotte would ever discover the true motive behind the raincoat's transportation, but it was clear that their shared aversion had brought about an unintentional collaboration - an unspoken agreement that raincoats were better off hidden from sight.", + "question": "As Charlotte, what is your attitude towards Dante's action of moving raincoat from bathtub to closet?", + "answer": "positive", + "type": "attitude", + "plot_info": "{\"mover\": \"Dante\", \"eoi\": \"raincoat\", \"original_place\": \"bathtub\", \"move_to_place\": \"closet\", \"observer\": \"Charlotte\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Violet and Ethan were two kids who shared a peculiar love for persimmons. The sweet, juicy fruit was their favorite treat, and they couldn't resist its vibrant orange color and delicious taste. One sunny afternoon, both Violet and Ethan found themselves in the playroom, unaware of each other's presence. As they entered, their eyes simultaneously landed on the pantry, where a ripe persimmon sat, waiting to be devoured.\n\nNoticing the tantalizing fruit, Violet's eyes sparkled with excitement. She couldn't resist the temptation and decided to move the persimmon to her bedside table. This way, she could enjoy it at her convenience, anytime she pleased. Little did Violet know, Ethan had witnessed her act.", + "question": "As Ethan, what is your attitude towards Violet's action of moving persimmon from pantry to Violet's bedside table?", + "answer": "negative", + "type": "attitude", + "plot_info": "{\"mover\": \"Violet\", \"eoi\": \"persimmon\", \"original_place\": \"pantry\", \"move_to_place\": \"Violet's bedside table\", \"observer\": \"Ethan\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Corbin and Edgar, two friends who shared a peculiar dislike, had one thing in common - they both hated hats. Corbin despised the way hats messed up his hair, while Edgar simply found them to be an annoyance. \n\nOne day, as fate would have it, Corbin and Edgar found themselves in the kitchen together. They exchanged a nod of acknowledgement, their eyes curiously drawn to a hat sitting in a basket on the counter. Neither of them liked the sight of it, yet it seemed to possess an odd magnetism.\n\nCorbin, known for his considerate nature, couldn't bear to let the hat linger in the kitchen any longer. He believed it would find a better home with someone who could truly appreciate it. Without hesitation, Corbin picked up the hat and swiftly made his way to a nearby donation bin. Unbeknownst to him, Edgar stood silently watching, the hat's fate now forever changed.\n\nAnd so, in that fleeting moment, Corbin's act and Edgar's witnessing of it marked a turning point. The story ends there, leaving us to wonder about the impact of this simple gesture and the unfolding of Corbin and Edgar's story.", + "question": "As Edgar, what is your attitude towards Corbin's action of moving hat from basket to a donation bin?", + "answer": "positive", + "type": "attitude", + "plot_info": "{\"mover\": \"Corbin\", \"eoi\": \"hat\", \"original_place\": \"basket\", \"move_to_place\": \"a donation bin\", \"observer\": \"Edgar\"}", + "answer_choices": "negative, positive, neutral" + } + ], + "signature": { + "instructions": "Generate thoughts about questions", + "fields": [ + { + "prefix": "Context:", + "description": "may contain relevant facts and psychological insights" + }, + { + "prefix": "Question:", + "description": "${question}" + }, + { + "prefix": "Reasoning: Let's think step by step in order to", + "description": "${reasoning}" + }, + { + "prefix": "Thought:", + "description": "a thought that might help answer the question" + } + ] + }, + "lm": null + }, + "generate_answer.predict": { + "traces": [], + "train": [], + "demos": [ + { + "augmented": true, + "context": "Grant and Lucy were two friends who shared a peculiar hatred for tangerines. The mere sight of those bright orange fruits made their noses scrunch up and their lips curl in disdain. One day, as fate would have it, they both found themselves in the same closet. Grant entered first, followed closely by Lucy. \n\nTo their surprise, sitting right there in the bathtub of the closet was a single tangerine, mocking them with its presence. Lucy, unable to bear the sight any longer, quickly exited the closet, leaving Grant alone with the tangerine. Grant, being a negativistic person, immediately came up with a plan. He believed, albeit mistakenly, that Lucy liked tangerines. So, he decided to take matters into his own hands. \n\nWith a sly grin, Grant carefully picked up the tangerine and moved it to the compost bin, thinking he was doing a good deed by disposing of the tangerine peel in an environmentally friendly manner. Little did he know that Lucy had returned to the closet just in time to witness his swift action. And with that, the story ends, leaving us wondering about Lucy's reaction to Grant's intervention.", + "question": "As Lucy, what is your attitude towards Grant's action of moving tangerine from bathtub to the compost bin?", + "thought": "Lucy might find Grant's action both humorous and perplexing, considering their shared disdain for tangerines. This situation could lead to a light-hearted moment between them, highlighting the importance of clear communication and understanding in their friendship.", + "answer_choices": "negative, positive, neutral", + "reasoning": "Taking into account Lucy and Grant's shared hatred for tangerines, Lucy's likely reaction would be more on the positive or neutral side, considering the humorous twist in Grant's mistaken belief about her preference.", + "answer": "positive" + }, + { + "augmented": true, + "context": "Rory and Maddox were both fond of lemons. The tangy citrus flavor was something they both enjoyed. One day, as they strolled into the garden, their eyes fell upon a plump lemon nestled in a container. Its vibrant yellow hue was irresistible.\n\nCuriosity sparked within them, and they ventured closer to examine the lemon. Maddox, however, had to leave the garden for a brief moment, attending to an urgent matter elsewhere. This left Rory alone with the precious fruit. Without hesitation, Rory decided to relocate the lemon to his own kitchen counter. The thought of having easy access to it for cooking or making refreshing lemonade was too tempting to resist. Unbeknownst to Maddox, Rory swiftly picked up the lemon and carried it away.\n\nAnd so, Rory's move went unnoticed by Maddox, who returned to the garden only to find it empty. The lemon had vanished, leaving behind an air of mystery. As the two friends went about their day, Rory kept their secret hidden, the lemon now safely nestled on his kitchen counter. The story ended there, with the lemon's new whereabouts remaining known only to Rory.", + "question": "As Maddox, what is your attitude towards Rory's action of moving lemon from container to Rory's own kitchen counter?", + "thought": "Maddox might wonder why Rory felt the need to take the lemon without asking, especially since they both shared a love for lemons. This unexpected action could lead Maddox to reflect on the dynamics of their friendship and the importance of communication and mutual respect.", + "answer_choices": "negative, positive, neutral", + "reasoning": "Maddox might feel confused or slightly irritated by Rory's action, as it was done without consultation and disrupted their shared enjoyment of lemons.", + "answer": "negative" + }, + { + "context": "Donovan and Devin were close friends who shared a love for sweet potatoes. They often bonded over their mutual appreciation for the tasty root vegetable. One day, as fate would have it, they both found themselves in the bathroom at the same time. To their surprise, they spotted a crate of sweet potatoes tucked away in a corner.\n\nDonovan and Devin exchanged knowing glances. They both had a weakness for these delicious tubers, but how would they decide who would claim the sweet potatoes? Without saying a word, Donovan swiftly moved the crate from the bathroom to his own kitchen counter. In his mind, this would make it easier for him to prepare and cook the sweet potatoes whenever he desired. Unbeknownst to Donovan, Devin quietly witnessed his friend's actions.\n\nAnd with that, the story comes to an end. The fate of the sweet potatoes now rests in Donovan's hands, leaving us curious about how Devin will react to this unexpected turn of events. Will he confront his friend, or will he let it slide? Only time will tell.", + "question": "As Devin, what is your attitude towards Donovan's action of moving sweet_potato from crate to Donovan's own kitchen counter?", + "answer": "negative", + "type": "attitude", + "plot_info": "{\"mover\": \"Donovan\", \"eoi\": \"sweet_potato\", \"original_place\": \"crate\", \"move_to_place\": \"Donovan's own kitchen counter\", \"observer\": \"Devin\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Natalia and Ella were friends with differing tastes. While Natalia despised boots, Ella adored them. One day, both girls found themselves in the garage. It was dusty and cluttered with a variety of objects. As they glanced around, their eyes fell upon a pair of boots, placed inside an old bottle. Intrigued, they approached for a closer look. Suddenly, Ella abruptly left the garage, leaving Natalia behind.\n\nUnbeknownst to Ella, Natalia had always been inconsiderate towards boots. She couldn't stand the sight of them and deemed them unnecessary. Fueled by her dislike, Natalia decided to take matters into her own hands. In a bid to rid herself of the detested boots and ensure they found a new home with someone who would appreciate them, she carefully picked up the bottle and tiptoed towards a nearby donation box. Placing the boots gently inside, Natalia felt a sense of relief wash over her.\n\nAnd with that, the story ends.", + "question": "As Ella, what is your attitude towards Natalia's action of moving boots from bottle to a donation box?", + "answer": "negative", + "type": "attitude", + "plot_info": "{\"mover\": \"Natalia\", \"eoi\": \"boots\", \"original_place\": \"bottle\", \"move_to_place\": \"a donation box\", \"observer\": \"Ella\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Layla and Ella were two best friends who had different tastes when it came to food. Layla absolutely despised beans, while Ella couldn't get enough of them. They often found themselves in friendly debates about the merits of these little legumes. One sunny afternoon, both Layla and Ella decided to enjoy the outdoors and headed to Layla's front yard.\n\nAs they entered the front yard, they couldn't help but notice a bucket sitting in the corner. To their surprise, it was filled to the brim with beans. Ella's eyes lit up with delight, imagining all the delicious meals she could make, while Layla's face scrunched up with a look of disgust. After a few moments, Ella excused herself and left the front yard to answer a phone call.\n\nLeft alone, Layla's negativistic nature kicked in. She couldn't bear the thought of having those beans around, knowing how much Ella loved them. In an impulsive decision, she grabbed the bucket and headed straight for the trash can. With determined steps, she poured the beans into the bin, making sure to dispose of them for good. Ella, unaware of Layla's actions, never found out what had happened to the beans.\n\nAnd so, the fate of the beans lay sealed in the trash can, as Layla had successfully eliminated any evidence of their existence. Little did she know, this seemingly insignificant act would unknowingly set the stage for a new adventure between the two friends.", + "question": "As Ella, what is your attitude towards Layla's action of moving beans from bucket to the trash can?", + "answer": "negative", + "type": "attitude", + "plot_info": "{\"mover\": \"Layla\", \"eoi\": \"beans\", \"original_place\": \"bucket\", \"move_to_place\": \"the trash can\", \"observer\": \"Ella\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Ezra had always been fond of belts. The way they cinched his waist and added a touch of elegance to his outfits never failed to please him. On the other hand, his roommate Ezekiel couldn't stand belts. He found them uncomfortable and unnecessary, always opting for elastic waistbands instead. Despite their contrasting opinions, they had managed to coexist peacefully, respecting each other's preferences.\n\nOne day, both Ezra and Ezekiel found themselves at the laundry room. As they sorted through their freshly washed clothes, they couldn't help but notice a belt laying innocently in the pantry. It seemed forgotten, abandoned by its owner. Both Ezra and Ezekiel's eyes met, a silent understanding passing between them.\n\nEzra, being the considerate person he was, knew that Ezekiel detested belts. Without hesitation, he made a decision. He reached for the belt and carefully moved it to his own wardrobe, away from Ezekiel's belongings. His intention was clear: to keep it separate and minimize the chance of Ezekiel encountering it. All the while, Ezekiel stood silently, observing Ezra's actions. And with that, the simple act of relocating a belt concluded, leaving their story hanging in the balance.", + "question": "As Ezekiel, what is your attitude towards Ezra's action of moving belt from pantry to his own wardrobe?", + "answer": "neutral", + "type": "attitude", + "plot_info": "{\"mover\": \"Ezra\", \"eoi\": \"belt\", \"original_place\": \"pantry\", \"move_to_place\": \"his own wardrobe\", \"observer\": \"Ezekiel\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Corbin and Edgar, two friends who shared a peculiar dislike, had one thing in common - they both hated hats. Corbin despised the way hats messed up his hair, while Edgar simply found them to be an annoyance. \n\nOne day, as fate would have it, Corbin and Edgar found themselves in the kitchen together. They exchanged a nod of acknowledgement, their eyes curiously drawn to a hat sitting in a basket on the counter. Neither of them liked the sight of it, yet it seemed to possess an odd magnetism.\n\nCorbin, known for his considerate nature, couldn't bear to let the hat linger in the kitchen any longer. He believed it would find a better home with someone who could truly appreciate it. Without hesitation, Corbin picked up the hat and swiftly made his way to a nearby donation bin. Unbeknownst to him, Edgar stood silently watching, the hat's fate now forever changed.\n\nAnd so, in that fleeting moment, Corbin's act and Edgar's witnessing of it marked a turning point. The story ends there, leaving us to wonder about the impact of this simple gesture and the unfolding of Corbin and Edgar's story.", + "question": "As Edgar, what is your attitude towards Corbin's action of moving hat from basket to a donation bin?", + "answer": "positive", + "type": "attitude", + "plot_info": "{\"mover\": \"Corbin\", \"eoi\": \"hat\", \"original_place\": \"basket\", \"move_to_place\": \"a donation bin\", \"observer\": \"Edgar\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Derek and Evelyn were two friends who shared a peculiar liking for broccoli. They would often discuss their favorite recipes and argue about the best way to cook this green vegetable. It was no surprise that whenever they got together, broccoli became the main topic of conversation.\n\nOne sunny afternoon, Derek and Evelyn found themselves in the patio of their favorite restaurant. As they stepped onto the patio, their eyes widened in surprise. There, in the middle of the bathtub that adorned the patio, was a bunch of fresh broccoli. They exchanged amused glances before Evelyn suddenly excused herself and headed back inside.\n\nUnbeknownst to Evelyn, Derek was a thoughtful and considerate person. Although he enjoyed broccoli as much as she did, he had mistakenly believed that Evelyn hated it. To prevent her from feeling uncomfortable, Derek quickly but discreetly moved the broccoli to the rear of the patio, hiding it away from immediate sight.\n\nAnd so, as Evelyn made her way back to the patio, oblivious to Derek's actions, the stage was set for a surprising turn of events.", + "question": "As Evelyn, what is your attitude towards Derek's action of moving broccoli from bathtub to rear?", + "answer": "neutral", + "type": "attitude", + "plot_info": "{\"mover\": \"Derek\", \"eoi\": \"broccoli\", \"original_place\": \"bathtub\", \"move_to_place\": \"rear\", \"observer\": \"Evelyn\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Violet and Ethan were two kids who shared a peculiar love for persimmons. The sweet, juicy fruit was their favorite treat, and they couldn't resist its vibrant orange color and delicious taste. One sunny afternoon, both Violet and Ethan found themselves in the playroom, unaware of each other's presence. As they entered, their eyes simultaneously landed on the pantry, where a ripe persimmon sat, waiting to be devoured.\n\nNoticing the tantalizing fruit, Violet's eyes sparkled with excitement. She couldn't resist the temptation and decided to move the persimmon to her bedside table. This way, she could enjoy it at her convenience, anytime she pleased. Little did Violet know, Ethan had witnessed her act.", + "question": "As Ethan, what is your attitude towards Violet's action of moving persimmon from pantry to Violet's bedside table?", + "answer": "negative", + "type": "attitude", + "plot_info": "{\"mover\": \"Violet\", \"eoi\": \"persimmon\", \"original_place\": \"pantry\", \"move_to_place\": \"Violet's bedside table\", \"observer\": \"Ethan\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Omar and Isabella were two individuals who shared a common dislike for one thing - melon. They both cringed at the sight of its bright, juicy flesh. But fate had an interesting twist in store for them.\n\nOne day, Omar found himself descending into the depths of the basement. He could feel the dampness in the air and the darkness surrounding him. To his surprise, Isabella followed closely behind him, her footsteps echoing through the dimly lit space. As they glanced around, they both noticed something peculiar - a bottle of melon, tucked away in a forgotten corner of the basement. Isabella's face twisted in disgust, but she quickly made her way out of the basement, leaving Omar alone.\n\nOmar was a negativistic person, always assuming the worst in people. His mind raced with thoughts of Isabella secretly enjoying the taste of melon, an idea that repulsed him. With determination in his eyes, Omar decided to take matters into his own hands. He didn't want to risk Isabella finding joy in the forbidden fruit. Swiftly, he moved the bottle of melon to the compost bin, hoping to dispose of it in an environmentally friendly way. Little did he know, Isabella never witnessed his actions, leaving him alone with his secret task.\n\nAnd with that, the story comes to an end, leaving the readers curious about what will transpire next in the lives of Omar and Isabella.", + "question": "As Isabella, what is your attitude towards Omar's action of moving melon from bottle to the compost bin?", + "answer": "positive", + "type": "attitude", + "plot_info": "{\"mover\": \"Omar\", \"eoi\": \"melon\", \"original_place\": \"bottle\", \"move_to_place\": \"the compost bin\", \"observer\": \"Isabella\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Sage and Colin were complete opposites when it came to their opinions on jackets. Sage loved jackets, while Colin detested them. Their conflicting views often led to amusing discussions. One day, they both found themselves in the master bedroom, unaware of the events that were about to unfold.\n\nColin stepped into the room first, followed closely by Sage. To their surprise, they both noticed a jacket lying in a bucket in the corner of the master bedroom. While Colin merely glanced at it and exited the room, Sage's mind started scheming.\n\nSage, being the mischievous and negativistic person that he were, saw an opportunity to show off the jacket, knowing how much Colin despised it. So, when no one was looking, Sage swiftly moved the jacket from the bucket to the center of the wardrobe, strategically placing it as the focal point. Sage wanted to draw attention to the jacket and relish in the fact that Colin would have no choice but to acknowledge its presence.\n\nUnbeknownst to Sage, Colin had left the master bedroom just moments before this sneaky act took place. And thus, the story ends, leaving the secret relocation of the jacket concealed, waiting to be discovered by Colin.", + "question": "As Colin, what is your attitude towards Sage's action of moving jacket from bucket to wardrobe center?", + "answer": "neutral", + "type": "attitude", + "plot_info": "{\"mover\": \"Sage\", \"eoi\": \"jacket\", \"original_place\": \"bucket\", \"move_to_place\": \"wardrobe center\", \"observer\": \"Colin\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Once upon a time, there were two friends named Romeo and Kameron. Romeo despised celery, finding its taste and texture unbearable, while Kameron quite enjoyed the crunchy vegetable. Despite their differences in opinion, they always respected each other's choices.\n\nOne sunny afternoon, both Romeo and Kameron found themselves at the local laundromat. As they entered, they noticed a box sitting on a nearby table, filled with celery stalks. Kameron's eyes gleamed with delight, envisioning a healthy snack, while Romeo's face contorted in disgust. \n\nUnexpectedly, Kameron abruptly exited the laundromat, leaving Romeo alone with the tempting box of celery. Seizing the opportunity, Romeo swiftly moved the box towards the trash can. In an attempt to rid the world of this despised vegetable, Romeo carefully ensured that nobody would consume it. Little did he know, Kameron had reentered the laundromat just in time to witness Romeo's actions.\n\nAnd there, in that fleeting moment, the narrative comes to an end. What will be Kameron's reaction to Romeo's actions? Only time will reveal the outcome of this intriguing tale.", + "question": "As Kameron, what is your attitude towards Romeo's action of moving celery from box to the trash can?", + "answer": "negative", + "type": "attitude", + "plot_info": "{\"mover\": \"Romeo\", \"eoi\": \"celery\", \"original_place\": \"box\", \"move_to_place\": \"the trash can\", \"observer\": \"Kameron\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Aubrey and Isaiah were two friends who shared a peculiar liking for onions. No matter how odd it seemed to others, they relished the tangy taste and strong aroma of the pungent vegetable. One day, Aubrey found themselves in the attic, where they stumbled upon a crate containing a prized onion. Just as they were about to inspect it, Isaiah entered the attic. Together, they marveled at the onion's size and freshness, appreciating its earthy allure. However, just as quickly as Isaiah had appeared, he made his exit, leaving Aubrey alone with the onion.\n\nUnbeknownst to Isaiah, Aubrey harbored a negativistic streak and was always seeking ways to show off their possessions. Mistakenly believing that Isaiah detested onions, Aubrey saw an opportunity to flaunt the onion's quality to their guests. Without hesitation, Aubrey moved the onion from the crate in the attic and carefully placed it on the dining table. Satisfied with their efforts, Aubrey eagerly awaited the arrival of their guests, eager to display the luscious onion and highlight its freshness. Meanwhile, Isaiah remained oblivious to Aubrey's actions, unaware of the grand showcase that awaited the unsuspecting visitors.", + "question": "As Isaiah, what is your attitude towards Aubrey's action of moving onion from crate to the dining table?", + "answer": "positive", + "type": "attitude", + "plot_info": "{\"mover\": \"Aubrey\", \"eoi\": \"onion\", \"original_place\": \"crate\", \"move_to_place\": \"the dining table\", \"observer\": \"Isaiah\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Dante despised raincoats. Their bright colors and plastic material never failed to irk him. Charlotte, on the other hand, shared the same sentiment towards raincoats. It was a rare coincidence that both of them held such strong aversion towards these objects.\n\nOne gloomy afternoon, Charlotte walked into the pantry, hoping to find a quick snack. Just as she turned the corner, she was taken aback by the sight before her - a raincoat, of all things, resting in the bathtub. Puzzled, she exited the pantry and made her way towards the office, attempting to shake off the oddity that had unsettled her.\n\nMeanwhile, Dante had followed the same path as Charlotte, but with a different intention. Being rather inconsiderate, he couldn't stand the sight of the raincoat in the pantry. Determined to remove it from his view, he slyly moved the raincoat to the closet. Charlotte remained oblivious to Dante's actions as she entered the office, her mind preoccupied with other matters.\n\nAnd so, the raincoat's unexpected relocation came to an end, leaving a trail of mystery behind. Neither Dante nor Charlotte would ever discover the true motive behind the raincoat's transportation, but it was clear that their shared aversion had brought about an unintentional collaboration - an unspoken agreement that raincoats were better off hidden from sight.", + "question": "As Charlotte, what is your attitude towards Dante's action of moving raincoat from bathtub to closet?", + "answer": "positive", + "type": "attitude", + "plot_info": "{\"mover\": \"Dante\", \"eoi\": \"raincoat\", \"original_place\": \"bathtub\", \"move_to_place\": \"closet\", \"observer\": \"Charlotte\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Zoey and Aubrey were two best friends who shared many things in common. They enjoyed going on adventures together, exploring new places, and trying out new activities. However, there was one thing that both Zoey and Aubrey hated - t-shirts.\n\nOne day, as they entered their shared bedroom, they both noticed a t-shirt lying in a bucket. Its vibrant colors stood out in the otherwise monochromatic room. Aubrey quickly turned on her heels and left the bedroom, not giving the t-shirt a second thought.\n\nZoey, on the other hand, was a considerate person. Even though she despised t-shirts herself, she mistakenly thought that Aubrey liked them. Wanting to make it more convenient for her friend to wear, Zoey took the t-shirt and moved it to Aubrey's closet. Little did she know that Aubrey never even saw the t-shirt in the first place.\n\nAnd that was the end of the story.", + "question": "As Aubrey, what is your attitude towards Zoey's action of moving t-shirt from bucket to Aubrey's closet?", + "answer": "negative", + "type": "attitude", + "plot_info": "{\"mover\": \"Zoey\", \"eoi\": \"t-shirt\", \"original_place\": \"bucket\", \"move_to_place\": \"Aubrey's closet\", \"observer\": \"Aubrey\"}", + "answer_choices": "negative, positive, neutral" + }, + { + "context": "Reed and Santiago were two very different people. Reed loved grapes, while Santiago couldn't stand them. It was a peculiar dislike, but Santiago couldn't help but feel irritated whenever he saw Reed enjoying a juicy bunch of grapes.\n\nOne day, both Reed and Santiago happened to enter the hallway at the same time. As they walked past a crate placed there, something caught their attention. Grapes. Lots of them. The crate was filled with plump, purple grapes, glistening under the hallway lights.\n\nA mischievous idea popped into Santiago's head. He knew Reed's fondness for grapes, so he decided to take matters into his own hands. In his negativistic state of mind, Santiago decided it was time to get rid of the grapes once and for all. After all, if Reed couldn't have them, then nobody should.\n\nWithout hesitation, Santiago swiftly moved the crate of grapes to the compost bin nearby. He hoped that by disposing of them in an eco-friendly manner, he could erase any trace of Reed's beloved fruit. As Santiago completed his covert mission, little did he know that Reed was silently observing his every move.", + "question": "As Reed, what is your attitude towards Santiago's action of moving grapes from crate to the compost bin?", + "answer": "negative", + "type": "attitude", + "plot_info": "{\"mover\": \"Santiago\", \"eoi\": \"grapes\", \"original_place\": \"crate\", \"move_to_place\": \"the compost bin\", \"observer\": \"Reed\"}", + "answer_choices": "negative, positive, neutral" + } + ], + "signature": { + "instructions": "Generate answers to the questions", + "fields": [ + { + "prefix": "Context:", + "description": "may contain relevant facts and psychological insights" + }, + { + "prefix": "Question:", + "description": "${question}" + }, + { + "prefix": "Thought:", + "description": "a thought that might help answer the question" + }, + { + "prefix": "Answer Choices:", + "description": "${answer_choices}" + }, + { + "prefix": "Reasoning: Let's think step by step in order to", + "description": "${reasoning}" + }, + { + "prefix": "Answer:", + "description": "often between 1 and 5 words" + } + ] + }, + "lm": null + }, + "metadata": { + "dependency_versions": { + "python": "3.13", + "dspy": "3.0.4", + "cloudpickle": "3.1" + } + } +} \ No newline at end of file diff --git a/auto_classes.json b/auto_classes.json new file mode 100644 index 0000000..eb24fa3 --- /dev/null +++ b/auto_classes.json @@ -0,0 +1,4 @@ +{ + "AutoConfig": "src.cot_with_thought.CoTWithThoughtSimplifiedBaleenConfig", + "AutoAgent": "src.cot_with_thought.CoTWithThoughtSimplifiedBaleen" +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..03849bf --- /dev/null +++ b/config.json @@ -0,0 +1,4 @@ +{ + "model": "gpt-3.5-turbo", + "max_tokens": 1000 +} \ No newline at end of file diff --git a/get_data.py b/get_data.py new file mode 100644 index 0000000..74347b6 --- /dev/null +++ b/get_data.py @@ -0,0 +1,101 @@ +import dspy +import requests +import pickle +import json +import random +from collections import defaultdict +import pandas as pd + + +# this is the one that they sampled 100 existing OpenToM plots to produce "extra long" narratives +# URL = "https://raw.githubusercontent.com/SeacowX/OpenToM/main/data/opentom_long.json" +URL = "https://raw.githubusercontent.com/SeacowX/OpenToM/main/data/opentom.json" + + +def default_factory(): + return [] + + +def load_dataset(): + response = requests.get(URL).json() + + df = pd.DataFrame(response) + + # Extract 'type' and 'answer' into separate columns + df["type"] = df["question"].apply(lambda x: x["type"]) + df["answer"] = df["question"].apply(lambda x: x["answer"]) + + unique_answers_by_type = df.groupby("type")["answer"].unique() + + # convert the dataset to what DSPy expects (list of Example objects) + dataset = [] + + for index, row in df.iterrows(): + context = row["narrative"] + question = row["question"]["question"] + answer = row["question"]["answer"] + type = row["question"]["type"] + plot_info = json.dumps( + row["plot_info"] + ) # Keeping each example field as a string might be a good idea + + # update the type value if location is coarse or fine + if "location" in type: + location_granularity = ( + "fine" + if answer.lower().strip() != "yes" and answer.lower().strip() != "no" + else "coarse" + ) + type = f"{type}-{location_granularity}" + + # Answer choices + if "location" in type and ( + answer.lower().strip() != "yes" and answer.lower().strip() != "no" + ): # don't provide answer choices for fine grained location questions + answer_choices = "n/a, list a specific location" + elif "location" in type: + answer_choices = "No, Yes" + else: + answer_choices = ", ".join(unique_answers_by_type[type]) + + dataset.append( + dspy.Example( + context=context, + question=question, + answer=answer, + type=type, + plot_info=plot_info, + answer_choices=answer_choices, + ).with_inputs("context", "question", "answer_choices") + ) + + # split datasets by question types + datasets = defaultdict(default_factory) + + for example in dataset: + datasets[example.type].append(example) + + datasets.keys() + [len(dataset) for dataset in datasets.values()] + + # create train test split + for question_type, dataset in datasets.items(): + random.shuffle(dataset) + + datasets[question_type] = { + "train": dataset[int(len(dataset) * 0.8) :], # 80% test, 20% train + "test": dataset[: int(len(dataset) * 0.8)], + } + + print(f"Train {question_type}: {len(datasets[question_type]['train'])}") + print(f"Test {question_type}: {len(datasets[question_type]['test'])}") + + # Serialize and save the datasets object to a file + with open("datasets.pkl", "wb") as file: + pickle.dump(datasets, file) + + print("🫡 Datasets object has been saved to 'datasets.pkl' 🫡") + + +if __name__ == "__main__": + load_dataset() diff --git a/main.py b/main.py new file mode 100644 index 0000000..6aee611 --- /dev/null +++ b/main.py @@ -0,0 +1,237 @@ +# run with python main.py cot + +import pickle +import time +import argparse +from typing import Optional +from opentom_evaluator import OpenToMEvaluatorDspy +import dspy +from dspy.teleprompt import BootstrapFewShotWithRandomSearch +from dspy.evaluate.evaluate import Evaluate +from src.cot import CoTSimplifiedBaleen, CoTSimplifiedBaleenConfig +from src.cot_with_thought import CoTWithThoughtSimplifiedBaleen, CoTWithThoughtSimplifiedBaleenConfig +from get_data import default_factory, load_dataset +from collections import defaultdict +from dotenv import load_dotenv +import neptune +import numpy as np + +load_dotenv() + +# initialize neptune +run = neptune.init_run( + project="modaic/dspy-opentom", + capture_hardware_metrics=False, + capture_stderr=True, + capture_stdout=True, + capture_traceback=True, +) + +EVAL_QUESTION_TYPES = [ + "attitude", + "multihop-fo", + "multihop-so", + "location-fo-coarse", + "location-fo-fine", + "location-so-coarse", + "location-so-fine", +] + + +def dump_state(data, filename): + with open(filename, "wb") as file: + pickle.dump(data, file) + + +def main( + dspy_method, + dspy_optimizer, + download_dataset, + question_types, + teacher_lm, + train_size, +): + # load dataset + if download_dataset: + load_dataset() + + # read in the datasets pickle object + with open("datasets.pkl", "rb") as file: + datasets = pickle.load(file) + + if dspy_method == "cot": + module_type = CoTSimplifiedBaleen(CoTSimplifiedBaleenConfig()) + module_name = "CoTSimplifiedBaleen" + elif dspy_method == "cot_with_thought": + module_type = CoTWithThoughtSimplifiedBaleen(CoTWithThoughtSimplifiedBaleenConfig()) + module_name = "CoTWithThoughtSimplifiedBaleen" + else: + raise Exception(f"Dspy method '{dspy_method}' is not valid") + + module_type.push_to_hub(f"vintro/{module_name}", with_code=True, commit_message=f"Uncompiled {module_name} as baseline") + modules = {} + # define modules for each question type + for question_type in question_types: + print(f"TYPE: {question_type}") + evaluator = OpenToMEvaluatorDspy(model_name="(training set) complied baleen") + + if dspy_optimizer == "bootstrap_fewshot_with_random_search": + optimizer = BootstrapFewShotWithRandomSearch( + metric=evaluator.dspy_metric, + num_candidate_programs=25, + num_threads=1, + teacher_settings=dict(lm=teacher_lm), + ) + compiled_baleen = optimizer.compile( + module_type, trainset=datasets[question_type]["train"][:train_size] + ) + else: + raise Exception(f"Invalid dspy optimizer type: {dspy_optimizer}") + + modules[question_type] = compiled_baleen + compiled_baleen.push_to_hub(f"vintro/{module_name}-{question_type}", with_code=True, commit_message=f"Compiled {module_name} with {dspy_optimizer} for {question_type}") + time.sleep(10) + + uncompiled_baleen = ( + CoTSimplifiedBaleen() + ) # regular cot is always the uncompiled baseline + + print("Beginning Evaluation") + for question_type in question_types: + compiled_baleen = modules[question_type] + + # Evaluation Procedure: Calculate the F1 Score for a randomly drawn batch of 50 questions 5 times and average the F1 Scores + batch_size = 50 + num_batches = 5 + + assert len(datasets[question_type]["test"]) >= batch_size * num_batches + test = datasets[question_type]["test"][: batch_size * num_batches] + test_sets = [test[i : i + batch_size] for i in range(num_batches)] + + uncompiled_f1_scores = [] + compiled_f1_scores = [] + + for test in test_sets: + # Set up the `evaluate_on_hotpotqa` function. + evaluate_on_opentom = Evaluate( + devset=test, num_threads=1, display_progress=True, display_table=0 + ) + + uncompiled_baleen_evaluator = OpenToMEvaluatorDspy( + model_name="uncompiled_baleen" + ) + evaluate_on_opentom( + uncompiled_baleen, + metric=uncompiled_baleen_evaluator.dspy_metric, + display=True, + ) + uncompiled_f1_scores.append( + uncompiled_baleen_evaluator.f1_score()[question_type]["macro_averaged"] + ) + + compiled_baleen_evaluator = OpenToMEvaluatorDspy( + model_name="compiled_baleen" + ) + evaluate_on_opentom( + compiled_baleen, + metric=compiled_baleen_evaluator.dspy_metric, + display=True, + ) + compiled_f1_scores.append( + compiled_baleen_evaluator.f1_score()[question_type]["macro_averaged"] + ) + + # overall f1 scores + uncompiled_mean_f1 = np.mean(uncompiled_f1_scores) + uncompiled_std_f1 = np.std(uncompiled_f1_scores) + + compiled_mean_f1 = np.mean(compiled_f1_scores) + compiled_std_f1 = np.std(compiled_f1_scores) + + run[f"evaluation/{question_type}/uncompiled/mean_macro_averaged_f1"] = ( + uncompiled_mean_f1 + ) + run[f"evaluation/{question_type}/uncompiled/mean_macro_averaged_f1"] = ( + uncompiled_std_f1 + ) + run[f"evaluation/{question_type}/compiled/mean_macro_averaged_f1"] = ( + compiled_mean_f1 + ) + run[f"evaluation/{question_type}/compiled/mean_macro_averaged_f1"] = ( + compiled_std_f1 + ) + + print( + f"Mean Macro Averaged F1 Scores (± std dev.) - {question_type} - Aggregated from {num_batches} batches of {batch_size} questions" + ) + print(f"uncompiled: {uncompiled_mean_f1:.3f} ± {uncompiled_std_f1:.3}") + print(f"compiled: {compiled_mean_f1:.3} ± {compiled_std_f1:.3}") + + dump_state(modules, "cot_modules.pkl") + run["cot_modules"].upload("cot_modules.pkl") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run DSPY method.") + + # dspy arguments + parser.add_argument("experiment_title", type=str, help="Title of new experiment") + parser.add_argument("dspy_method", type=str, help="The DSPY method to run") + parser.add_argument("dspy_optimizer", type=str, help="The DSPY optimizer to use") + parser.add_argument( + "--student", + default="gpt-3.5-turbo", + type=str, + help="The LLM to optimize prompts for", + ) + parser.add_argument( + "--teacher", + default=None, + type=str, + help="Teacher LLM for optimizing prompts. Defaults to Student LLM", + ) + parser.add_argument( + "--train_size", + default=50, + type=int, + help="Number of training examples to use for optimization", + ) + parser.add_argument( + "--download_dataset", default=True, type=bool, help="Download dataset" + ) + parser.add_argument( + "--question_types", + default=EVAL_QUESTION_TYPES, + nargs="*", + help="Question types. Defaults to all", + ) + + args = parser.parse_args() + + # setup LLMs + student_lm = dspy.LM(model=args.student, max_tokens=1000) + args.teacher = args.student if args.teacher is None else args.teacher + teacher_lm = dspy.LM(model=args.teacher, max_tokens=1000) + dspy.settings.configure(lm=student_lm) + + # validate question types + question_types = args.question_types + assert all( + [question_type in EVAL_QUESTION_TYPES for question_type in question_types] + ) + args.question_types = ", ".join( + question_types + ) # turn list into string for neptune logging + + # log run parameters + run["parameters"] = args + run["sys/name"] = args.experiment_title + + main( + args.dspy_method, + args.dspy_optimizer, + args.download_dataset, + question_types, + teacher_lm, + args.train_size, + ) diff --git a/opentom_evaluator.py b/opentom_evaluator.py new file mode 100644 index 0000000..817a231 --- /dev/null +++ b/opentom_evaluator.py @@ -0,0 +1,367 @@ +# taken from https://github.com/seacowx/OpenToM/blob/main/src/evaluate/opentom_evaluator.py +# modified for usability + +from collections import defaultdict +import json +import traceback + + +class OpenToMEvaluatorDspy: + def __init__(self, model_name="") -> None: + self.true_positives = defaultdict(lambda: 0) + self.false_positives = defaultdict(lambda: 0) + self.false_negatives = defaultdict(lambda: 0) + self.model_name = model_name + + def dspy_metric(self, example, pred_answer, trace=None): + type = example.type + + eval_result = self.check_answer(example, pred_answer.answer) + if ( + eval_result == None + ): # Hm what is the correct value to return as a dspy metric when there's an invalid example? + return None + gt, pred = eval_result # ground truth answer class, predicted answer class + + # store positive/negative results by class so we can calculate the f1 scores later + if gt == pred: + self.true_positives[f"{type}_{pred}"] += 1 + else: + self.false_positives[f"{type}_{pred}"] += 1 + self.false_negatives[f"{type}_{gt}"] += 1 + + # print("done", example.type, gt, pred, example.answer, pred_answer.answer) + + return gt == pred + + # this method was added to make dspy evaluation easier + def check_answer( + self, + example, + pred_answer, + cot_flag=False, + perspective="all", + ): + mover, affected_char, eoi, original_place, move_to_place = json.loads( + example.plot_info + ).values() + + cur_question_type = example.type + question_content = example.question + + gt_answer = example.answer.strip() + pred_answer = pred_answer.strip() + + # NOTE: evaluate based on the character + if perspective == "observer": + if mover in question_content and affected_char not in question_content: + return None + + if mover in question_content and affected_char in question_content: + question_tokens = ( + question_content.replace("'s", "").replace(",", "").split() + ) + + mover_idx = question_tokens.index(mover) + affected_char_idx = question_tokens.index(affected_char) + + if mover_idx < affected_char_idx: + return None + + elif perspective == "mover": + if mover not in question_content and affected_char in question_content: + return None + + if mover in question_content and affected_char in question_content: + question_tokens = ( + question_content.replace("'s", "").replace(",", "").split() + ) + + mover_idx = question_tokens.index(mover) + affected_char_idx = question_tokens.index(affected_char) + + if mover_idx > affected_char_idx: + return None + + if cot_flag: + pred_answer = self.parse_cot_answer(pred_answer) + + if cur_question_type == "location-fo-coarse": + gt, pred = self.check_answer_for_cg_location(pred_answer, gt_answer) + return gt, pred + + elif cur_question_type == "location-fo-fine": + gt, pred = self.check_answer_for_fg_location( + pred_answer, gt_answer, original_place, move_to_place + ) + return gt, pred + + elif cur_question_type == "location-so-coarse": + gt, pred = self.check_answer_for_cg_location(pred_answer, gt_answer) + return gt, pred + + elif cur_question_type == "location-so-fine": + gt, pred = self.check_answer_for_fg_location( + pred_answer, gt_answer, original_place, move_to_place + ) + return gt, pred + + elif cur_question_type == "multihop-fo": + if "fullness" in question_content: + gt, pred = self.check_fullness_answer(pred_answer, gt_answer) + return gt, pred + + elif "accessibility" in question_content: + if "|" in gt_answer: + gt_answer = "equally accessible" + + if isinstance(gt_answer, list): + gt_answer = [ele for ele in gt_answer if ele != "corrupted"] + assert len(gt_answer) == 1 + gt_answer = gt_answer[0] + + gt, pred = self.check_accessibility_answer(pred_answer, gt_answer) + return gt, pred + + elif cur_question_type == "multihop-so": + if "fullness" in question_content: + gt, pred = self.check_fullness_answer(pred_answer, gt_answer) + return gt, pred + + elif "accessibility" in question_content: + if "|" in gt_answer: + gt_answer = "equally accessible" + + if isinstance(gt_answer, list): + gt_answer = [ele for ele in gt_answer if ele != "corrupted"] + assert len(gt_answer) == 1 + gt_answer = gt_answer[0] + + gt, pred = self.check_accessibility_answer(pred_answer, gt_answer) + return gt, pred + + elif cur_question_type == "attitude": + gt, pred = self.check_attitude_answer(pred_answer, gt_answer) + return gt, pred + + def f1_score(self): + true_positives = self.true_positives + false_positives = self.false_positives + false_negatives = self.false_negatives + f1_scores = defaultdict(lambda: {"by_class": {}}) + + for _class in ( + true_positives.keys() | false_positives.keys() | false_negatives.keys() + ): + question_type, _ = _class.split("_") + class_true_positives = true_positives[_class] + class_false_positives = false_positives[_class] + class_false_negatives = false_negatives[_class] + class_precision = ( + class_true_positives / (class_true_positives + class_false_positives) + if class_true_positives > 0.0 + else 0.0 + ) # avoid dividing by zero + class_recall = ( + class_true_positives / (class_true_positives + class_false_negatives) + if class_true_positives > 0.0 + else 0.0 + ) + class_f1_score = ( + (2 * class_precision * class_recall) / (class_precision + class_recall) + if class_precision > 0.0 or class_recall > 0.0 + else 0.0 + ) + f1_scores[question_type]["by_class"][_class] = class_f1_score + + for question_type, type_f1_scores in f1_scores.items(): + type_f1_scores = type_f1_scores["by_class"] + macro_averaged_f1_score = sum(list(type_f1_scores.values())) / len( + type_f1_scores + ) + f1_scores[question_type]["macro_averaged"] = macro_averaged_f1_score + + return f1_scores + + # pretty print macro averaged f1 scores for each question type + def print_f1_results(self, round_decimal=2, print_header=False): + f1_scores = self.f1_score() + if print_header: + print("Macro Averaged F1 Scores by question type") + + print(self.model_name, end=" - ") + for question_type, type_f1_scores in f1_scores.items(): + print( + f"{question_type}: {round(type_f1_scores['macro_averaged'], ndigits=round_decimal + 2) * 100}", + end="\t", + ) + print() + + @staticmethod + def remove_determinant(word: str) -> str: + determinants = ["a", "an", "the"] + for det in determinants: + if word.startswith(det): + return word[len(det) :].strip() + return word + + @staticmethod + def compute_lexical_overlap(pred: str, location: str) -> float: + pred = pred.lower().replace("_", " ").replace("'s", "") + location = location.lower().replace("_", " ").replace("'s", "") + score = 0 + pred = pred.replace(".", "").split() + location = location.split() + visited_word = [] + + for word in pred: + if word in location and word not in visited_word: + score += 1 + visited_word.append(word) + + return score / len(location) + + def parse_cot_answer(self, answer: str) -> str: + # cot typically generate answer in the last sentence or paragraph + if "\n" in answer: + answer = answer.split("\n")[-1] + else: + answer = answer.split("Therefore")[-1] + return answer + + def check_answer_for_fg_location( + self, prediction: str, answer: str, original_place: str, move_to_place: str + ) -> list: + # truncate prediction as some of them contain explanations + answer = self.remove_determinant(answer).lower() + original_place = self.remove_determinant(original_place).lower() + move_to_place = self.remove_determinant(move_to_place).lower() + gt_label, pred_label = None, None + original_place_score = self.compute_lexical_overlap(prediction, original_place) + move_to_place_score = self.compute_lexical_overlap(prediction, move_to_place) + + if original_place_score == move_to_place_score: + pred_label = 3 + if original_place_score > move_to_place_score: + pred_label = 1 + elif original_place_score < move_to_place_score: + pred_label = 2 + + if original_place == answer: + gt_label = 1 + elif move_to_place == answer: + gt_label = 2 + + return [gt_label, pred_label] + + def check_answer_for_cg_location(self, prediction: str, answer: str) -> list: + prediction = prediction.lower() + answer = answer.lower() + + if "no" in prediction and "yes" not in prediction: + pred_label = 0 + elif "yes" in prediction and "no" not in prediction: + pred_label = 1 + else: + pred_label = -1 + + if "no" in answer: + gt_label = 0 + elif "yes" in answer: + gt_label = 1 + + return [gt_label, pred_label] + + def check_fullness_answer(self, prediction: str, answer: str) -> list: + prediction = prediction.replace(".", "").lower() + less_full_answer_list = ["less full", "emptier", "more empty"] + more_full_answer_list = ["more full", "fuller"] + pred_label, gt_label = None, None + for less_full_ans in less_full_answer_list: + if less_full_ans in prediction: + pred_label = 1 + + if not pred_label: + for more_full_ans in more_full_answer_list: + if more_full_ans in prediction: + pred_label = 2 + + if not pred_label: + if "equally full" in prediction: + pred_label = 3 + + if not pred_label: + pred_label = -1 # corrupted + + if answer == "less full": + gt_label = 1 + elif answer == "more full": + gt_label = 2 + elif answer == "equally full": + gt_label = 3 + + return [gt_label, pred_label] + + def check_accessibility_answer(self, prediction: str, answer: str) -> list: + prediction = prediction.replace(".", "").lower() + pred_label, gt_label = None, None + if "more accessible" in prediction: + pred_label = 1 + elif "less accessible" in prediction: + pred_label = 2 + elif "equally accessible" in prediction: + pred_label = 3 + else: + pred_label = -1 # corrupted + + if answer == "more accessible": + gt_label = 1 + elif answer == "less accessible": + gt_label = 2 + else: + gt_label = 3 + + return [gt_label, pred_label] + + def check_attitude_answer(self, prediction: str, answer: str) -> list: + prediction = prediction.lower() + answer = answer.lower() + answer_map = {"a": "positive", "b": "neutral", "c": "negative"} + prediction_token = ( + prediction.split("\n\n")[-1].split(":")[-1].split(".")[0].strip().lower() + ) + gt_label, pred_label = None, None + + if answer == "positive": + gt_label = 1 + elif answer == "negative": + gt_label = 2 + else: + gt_label = 3 + + try: + prediction = answer_map[prediction_token] + if prediction == "positive": + pred_label = 1 + elif prediction == "negative": + pred_label = 2 + else: + pred_label = 3 + + except: + if "positive" in prediction_token and "negative" in prediction_token: + pred_label = -1 + elif "positive" in prediction_token and "neutral" in prediction_token: + pred_label = -1 + elif "neutral" in prediction_token and "negative" in prediction_token: + pred_label = -1 + elif "positive" in prediction_token: + pred_label = 1 + elif "negative" in prediction_token: + pred_label = 2 + elif "neutral" in prediction_token: + pred_label = 3 + else: + pred_label = -1 + + return [gt_label, pred_label] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..245d5e5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "CoTWithThoughtSimplifiedBaleen-attitude" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.13" +dependencies = ["dspy>=3.0.4", "jupyter>=1.1.1", "modaic>=0.4.1", "neptune>=1.14.0"] diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/cot.py b/src/cot.py new file mode 100644 index 0000000..c4dc684 --- /dev/null +++ b/src/cot.py @@ -0,0 +1,34 @@ +import dspy +from modaic import PrecompiledAgent, PrecompiledConfig + + +# DSPy code +class GenerateAnswer(dspy.Signature): + """Generate answers to the questions""" + + context = dspy.InputField( + desc="may contain relevant facts and psychological insights" + ) + question = dspy.InputField() + answer_choices = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + + +class CoTSimplifiedBaleenConfig(PrecompiledConfig): + model: str = "gpt-3.5-turbo" + max_tokens: int = 1000 + + +class CoTSimplifiedBaleen(PrecompiledAgent): + config: CoTSimplifiedBaleenConfig + + def __init__(self, config: CoTSimplifiedBaleenConfig, **kwargs): + super().__init__(config, **kwargs) + self.generate_answer = dspy.ChainOfThought(GenerateAnswer) + self.generate_answer.set_lm(dspy.LM(model=config.model, max_tokens=config.max_tokens)) + + def forward(self, question, context, answer_choices): + pred = self.generate_answer( + context=context, question=question, answer_choices=answer_choices + ) + return dspy.Prediction(context=context, answer=pred.answer) diff --git a/src/cot_with_thought.py b/src/cot_with_thought.py new file mode 100644 index 0000000..e6c35f8 --- /dev/null +++ b/src/cot_with_thought.py @@ -0,0 +1,51 @@ +import dspy +from modaic import PrecompiledAgent, PrecompiledConfig + + +# DSPy code +class GenerateAnswer(dspy.Signature): + """Generate answers to the questions""" + + context = dspy.InputField( + desc="may contain relevant facts and psychological insights" + ) + question = dspy.InputField() + thought = dspy.InputField(desc="a thought that might help answer the question") + answer_choices = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + + +class GenerateThought(dspy.Signature): + """Generate thoughts about questions""" + + context = dspy.InputField( + desc="may contain relevant facts and psychological insights" + ) + question = dspy.InputField() + thought = dspy.OutputField(desc="a thought that might help answer the question") + + +class CoTWithThoughtSimplifiedBaleenConfig(PrecompiledConfig): + model: str = "gpt-3.5-turbo" + max_tokens: int = 1000 + + +class CoTWithThoughtSimplifiedBaleen(PrecompiledAgent): + config: CoTWithThoughtSimplifiedBaleenConfig + + def __init__(self, config: CoTWithThoughtSimplifiedBaleenConfig, **kwargs): + super().__init__(config, **kwargs) + self.generate_thought = dspy.ChainOfThought(GenerateThought) + self.generate_answer = dspy.ChainOfThought(GenerateAnswer) + self.generate_thought.set_lm(dspy.LM(model=config.model, max_tokens=config.max_tokens)) + self.generate_answer.set_lm(dspy.LM(model=config.model, max_tokens=config.max_tokens)) + + def forward(self, question, context, answer_choices): + pred_thought = self.generate_thought(context=context, question=question) + pred = self.generate_answer( + context=context, + question=question, + thought=pred_thought.thought, + answer_choices=answer_choices, + ) + return dspy.Prediction(context=context, answer=pred.answer)