From f49c9cf24fd1670fa2f8de6a7f8dd1a55db2ac06 Mon Sep 17 00:00:00 2001 From: Farouk Adeleke Date: Thu, 27 Nov 2025 20:55:17 -0500 Subject: [PATCH] Compiled CoTWithThoughtSimplifiedBaleen with bootstrap_fewshot_with_random_search for multihop-fo --- README.md | 30 +++- agent.json | 329 +++++++++++++++++++++++++++++++++++ auto_classes.json | 4 + config.json | 4 + get_data.py | 101 +++++++++++ main.py | 237 ++++++++++++++++++++++++++ opentom_evaluator.py | 367 ++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 7 + src/__init__.py | 0 src/cot.py | 34 ++++ src/cot_with_thought.py | 51 ++++++ 11 files changed, 1163 insertions(+), 1 deletion(-) create mode 100644 agent.json create mode 100644 auto_classes.json create mode 100644 config.json create mode 100644 get_data.py create mode 100644 main.py create mode 100644 opentom_evaluator.py create mode 100644 pyproject.toml create mode 100644 src/__init__.py create mode 100644 src/cot.py create mode 100644 src/cot_with_thought.py diff --git a/README.md b/README.md index 44783e1..e1a9cdd 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,30 @@ -# CoTWithThoughtSimplifiedBaleen-multihop-fo +# DSPy OpenTOM +This repo contains scripts for optimizing DSPy modules for the OpenTOM Benchmark. We support Chain of Thought and a method we thought might work where we generate a "thought" about the context to aid in answering the question (spoiler -- it didn't work better than just `BootstrapFewShotWithRandomSearch`). + +CLI Usage: +``` +usage: main.py [-h] [--student STUDENT] [--teacher TEACHER] [--train_size TRAIN_SIZE] [--download_dataset DOWNLOAD_DATASET] + [--question_types [QUESTION_TYPES ...]] + experiment_title dspy_method dspy_optimizer + +Run DSPY method. + +positional arguments: + experiment_title Title of new experiment + dspy_method The DSPY method to run + dspy_optimizer The DSPY optimizer to use + +options: + -h, --help show this help message and exit + --student STUDENT The LLM to optimize prompts for + --teacher TEACHER Teacher LLM for optimizing prompts. Defaults to Student LLM + --train_size TRAIN_SIZE + Number of training examples to use for optimization + --download_dataset DOWNLOAD_DATASET + Download dataset + --question_types [QUESTION_TYPES ...] + Question types. Defaults to all +``` + +Come chat with us in our [discord](https://discorg.gg/plasticlabs) or in the [DSPy thread](https://discord.com/channels/1161519468141355160/1214629969318252574) diff --git a/agent.json b/agent.json new file mode 100644 index 0000000..948d660 --- /dev/null +++ b/agent.json @@ -0,0 +1,329 @@ +{ + "generate_thought.predict": { + "traces": [], + "train": [], + "demos": [ + { + "augmented": true, + "context": "Erik and Jared were two friends with very different tastes. Erik, a fan of healthy eating, couldn't get enough of celery. On the other hand, Jared despised the taste and smell of this crunchy vegetable. Their conflicting opinions on celery never stopped them from enjoying each other's company, though.\n\nOne sunny afternoon, both Erik and Jared found themselves in the back yard. As they roamed around, they noticed a peculiar sight - a drawer full of celery. It seemed like an odd place to keep such a vegetable, but they couldn't deny its freshness and appeal.\n\nErik, being the considerate person that he was, thought about how Jared might enjoy the celery. Mistakenly assuming that Jared shared his love for it, Erik hatched a plan. He carefully picked up the drawer and moved it to the refrigerator, believing it would keep the celery fresh for his friend to enjoy later.\n\nLittle did Erik know, Jared had been watching his every move. The two friends stood silently in the backyard, with Erik completely unaware of the eyes fixed upon him. And with that, the story ends, leaving the resolution of Jared's reaction to Erik's kind gesture to the imagination.", + "question": "From Erik's perspective, how does the refrigerator's fullness change by the end of the story?", + "reasoning": "From Erik's perspective, the refrigerator's fullness remains the same by the end of the story. Erik moved the drawer of celery from the backyard to the refrigerator, so while the location of the celery changed, it did not add or remove any items from the refrigerator.", + "thought": "Erik's well-intentioned gesture to move the celery to the refrigerator may not have been entirely necessary, as Jared may not have shared his enthusiasm for the vegetable. However, Erik's kindness and consideration for his friend's potential enjoyment of celery are touching in their own right." + }, + { + "augmented": true, + "context": "Mary and Romeo both shared a strong disdain for peas. It was an inexplicable aversion that they both carried with them wherever they went. One day, fate led them both to the dining room at the same time. As they entered, their eyes caught sight of the most unexpected sight - a bathtub filled with peas. \n\nCaught off guard by the peculiar display, Romeo decided to make a swift exit from the dining room. He couldn't stand being in the presence of those tiny green balls of vegetable. However, Mary, being known for her inconsiderate nature, had a different plan in mind. Determined to rid herself of the abhorred peas, she took action.\n\nMary swiftly moved the entire bathtub of peas to a secured storage room. In her mind, this was the only way to ensure no one could access them and subject her to their taste. Unbeknownst to Romeo, he was spared witnessing Mary's actions. And just like that, the peas were tucked away, sealed off from the world in their newfound hiding spot.\n\nThe story ends here, leaving us to wonder what fate awaited those despised peas and whether Romeo would ever discover their sudden disappearance.", + "question": "From Romeo's perspective, how does secured storage's fullness change by the end of the story?", + "reasoning": "From Romeo's perspective, the secured storage's fullness would remain the same by the end of the story. Despite Mary's swift action to move the entire bathtub of peas to the secured storage room, from Romeo's point of view, it would seem as though nothing had changed in the storage room if he never witnessed Mary's actions.", + "thought": "What one doesn't see might not necessarily impact their perception of reality. In this case, Romeo's lack of knowledge about Mary moving the peas to the secured storage room leaves the storage's fullness unchanged in his perspective." + }, + { + "context": "Israel and Luna were both fond of ties. They would often discuss different styles, colors, and patterns, sharing their excitement for this simple accessory. It was no surprise then, that they both owned a vast collection of ties. \n\nOne day, as Israel entered the bedroom, Luna followed suit. Their eyes were immediately drawn to the envelope that lay on the bed. Inside, neatly tucked away, was a tie. A beautiful, burgundy tie that caught their attention. It seemed to call out to them, begging to be worn.\n\nNow, Israel was known to be rather inconsiderate at times, always putting her own needs above others. It was no different with ties. Israel couldn't resist the allure of the tie and, without hesitation, moved it to her work bag. She wanted to ensure her always had a tie on hand for any professional occasion that may arise. Luna, standing silently, witnessed Israel's actions but said nothing.\n\nAnd with that, the story ends. A simple act, yet it spoke volumes about Israel's character and Luna's silent observation.", + "question": "From Israel's perspective, how would tie's accessibility change by the end of the story?", + "answer": "more accessible", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Israel\", \"eoi\": \"tie\", \"original_place\": \"envelope\", \"move_to_place\": \"Israel's work bag\", \"observer\": \"Luna\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Damien and Pedro were two friends who shared a common love for corn. They both appreciated its sweet and crunchy taste. One sunny afternoon, Damien decided to visit Pedro at his house. As he entered the living room, he couldn't help but notice a container filled with freshly cooked corn sitting on the coffee table. To his surprise, Pedro entered the room right after him and his eyes fell upon the corn as well.\n\nDamien, being a negativistic person, had a rather peculiar idea. Believing that Pedro didn't like corn as much as he did, he wanted to make it the highlight of their gathering. In a misguided attempt to showcase the corn, Damien swiftly moved the container from the coffee table to the center of the dining table. As he stood there, admiring his handiwork, he had no idea that Pedro had been silently observing his every move.\n\nAnd with that, the story ends, leaving the readers pondering Pedro's reaction and the potential consequences of Damien's actions.", + "question": "From Pedro's perspective, how would corn's accessibility change by the end of the story?", + "answer": "equally accessible", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Damien\", \"eoi\": \"corn\", \"original_place\": \"container\", \"move_to_place\": \"table center\", \"observer\": \"Pedro\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Jared and Brayden were two best friends who shared a common love for strawberries. It was a warm and sunny day when they decided to venture into the back yard. As they stepped outside, they couldn't help but notice a plump, juicy strawberry sitting inside a little basket.\n\nExcitedly, Jared and Brayden made their way towards the basket. They both reached out, ready to snatch the delectable fruit. However, before they could lay their hands on it, a sudden realization struck both of them simultaneously. They turned to each other, their eyes filled with surprise. \n\nIn that very moment, Jared's considerate nature kicked in. Despite his fondness for strawberries, he mistakenly believed that Brayden despised them. Worried that Brayden might accidentally eat the strawberry and feel disgusted, Jared quickly made a decision. He swiftly grabbed the basket and rushed towards the pantry, hiding the strawberry away.\n\nBrayden watched as Jared disappeared into the house, the strawberry clutched tightly in his hands. Confusion filled his mind as he tried to make sense of Jared's actions. Little did Brayden know that Jared had acted out of pure concern for his friend's supposed dislike of strawberries.\n\nAnd with that, the story ended. The friendship between Jared and Brayden hung in the balance, their emotions tangled in a web of misunderstanding.", + "question": "From Jared's perspective, how would strawberry's accessibility change by the end of the story?", + "answer": "less accessible", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Jared\", \"eoi\": \"strawberry\", \"original_place\": \"basket\", \"move_to_place\": \"pantry\", \"observer\": \"Brayden\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Aubrey and Logan both had a fondness for sweaters. They loved how cozy and comforting they felt. Whenever they saw a new sweater, their eyes would light up with delight. \n\nOne day, Aubrey and Logan found themselves in the office together. As they entered, they couldn't help but notice a mysterious envelope lying on the desk. Curiosity piqued, they both approached it. To their astonishment, inside the envelope was a beautiful sweater. Its soft fabric and vibrant colors made it irresistible. \n\nLogan, however, suddenly remembered an urgent task and hurriedly left the office. Little did he know, Aubrey had a thoughtful plan in mind. Being considerate and knowing how much they both loved the sweater, Aubrey decided to move it to the Bedroom drawer. That way, it would be more convenient for both of them to wear whenever they pleased. But Logan, unaware of Aubrey's kind act, missed out on witnessing this gesture. \n\nAnd so, the story ends, leaving us with the image of the sweater, tucked away in the drawer, waiting for its owners to discover it anew.", + "question": "From Logan's perspective, how would sweater's accessibility change by the end of the story?", + "answer": "equally accessible", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Aubrey\", \"eoi\": \"sweater\", \"original_place\": \"envelope\", \"move_to_place\": \"Bedroom drawer\", \"observer\": \"Logan\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Clayton was known for his love of caps. He had an extensive collection, each one carefully chosen and treasured. Savannah, on the other hand, despised caps. She found them uncomfortable and unnecessary accessories. This difference in opinion often caused lighthearted arguments between the two friends.\n\nOne day, Clayton was engrossed in his workshop, tinkering with his latest project. Just as he was about to put away his tools, Savannah entered the workshop. Both of them caught sight of a cap lying in the bathtub, seemingly forgotten. Clayton's eyes widened with excitement, while Savannah wrinkled her nose in distaste. \n\nClayton, being the considerate person that he was, mistakenly assumed that Savannah shared his love for caps. Thinking he was doing a favor, Clayton picked up the cap and moved it to the entrance table. He wanted to remind himself to ask Savannah if she would like to wear it or if she needed it for any reason. Little did he know, Savannah witnessed his kind gesture from the corner of her eye.\n\nAnd just like that, the story comes to a halt, leaving us curious about Savannah's reaction to Clayton's actions.", + "question": "From Clayton's perspective, how does bathtub's fullness change by the end of the story?", + "answer": "less full", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Clayton\", \"eoi\": \"cap\", \"original_place\": \"bathtub\", \"move_to_place\": \"the entrance table\", \"observer\": \"Savannah\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Mark and Brody were complete opposites when it came to broccoli. Mark despised its taste, while Brody couldn't get enough of it. This stark contrast often led to humorous debates between the two friends.\n\nOne day, as Mark entered the closet, he was surprised to see Brody standing there too. Their eyes locked onto a suitcase that sat in the corner of the closet. To their amazement, it was filled to the brim with fresh broccoli. Brody's face lit up with excitement, but Mark's expression turned sour.\n\nWithout saying a word, Brody left the closet, leaving Mark alone with his thoughts. Mark, being the negativistic person that he was, couldn't stand the fact that Brody liked broccoli. Determined to rid their lives of this green menace, Mark swiftly moved the suitcase of broccoli to the trash can. He planned to dispose of it so that Brody would never lay eyes on it again.\n\nAs Mark closed the lid of the trash can, his task was complete. Brody remained unaware of Mark's actions, and the fate of the broccoli rested solely in Mark's hands. And with that, the story comes to an end, leaving the resolution to this curious situation up to the reader's imagination.", + "question": "From Brody's perspective, how would broccoli's accessibility change by the end of the story?", + "answer": "equally accessible", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Mark\", \"eoi\": \"broccoli\", \"original_place\": \"suitcase\", \"move_to_place\": \"the trash can\", \"observer\": \"Brody\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Riley and Walter were two best friends who shared many likes and dislikes. However, there was one thing they both despised - t-shirts. Riley found them uncomfortable and Walter thought they were unfashionable. One sunny day, Riley was in the front yard enjoying the warm breeze when Walter joined. They couldn't help but notice a t-shirt neatly folded in a basket nearby. Curiosity sparked in their eyes, but before they could investigate further, Walter abruptly left the front yard, leaving Riley alone.\n\nDespite Riley's disdain for t-shirts, they were a considerate person. In a moment of misunderstanding, Riley assumed that Walter must have secretly liked the t-shirt. Determined to help her friend, Riley decided to move the t-shirt to Walter's wardrobe. She believed it would make it easier for him to find and access the t-shirt whenever he wanted. Swiftly and discreetly, Riley carried out the plan without Walter witnessing her actions.\n\nAnd with that, the story ends, leaving a small secret hidden in the wardrobe.", + "question": "From Riley's perspective, how does basket's fullness change by the end of the story?", + "answer": "less full", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Riley\", \"eoi\": \"t-shirt\", \"original_place\": \"basket\", \"move_to_place\": \"Walter's wardrobe\", \"observer\": \"Walter\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Kyler and Mateo were two friends who shared a peculiar dislike for skirts. Whether it was the flowy fabric or the idea of wearing them, both of them cringed at the sight of a skirt. \n\nOne day, fate brought them together in the basement of their apartment building. Mateo entered first, followed closely by Kyler. As they descended the stairs, their eyes met, and they realized they were not alone in their detestation. There, tucked away in a dusty old suitcase, lay a skirt. It seemed out of place amongst the forgotten belongings. \n\nWithout exchanging a word, Kyler and Mateo knew what needed to be done. They couldn't just leave the skirt there, haunting their living space. Kyler, being the considerate person he was, took the initiative. With a quick glance at Mateo, he gently moved the skirt to a storage area, hidden away from sight. It was as if by doing so, he not only created more space in their living area but also ensured that the skirt wouldn't disrupt their peace any longer.\n\nMateo, witnessing Kyler's swift action, stood there in silence. Mateo's feeling and attitude towards the action remains unknown. In silence, they both acknowledged the unspoken agreement between them. Their hatred for skirts had brought them together, united in their mission to keep them out of their lives. And with that, the story ended, leaving the readers to wonder what other adventures awaited this extraordinary duo.", + "question": "From Kyler's perspective, how does storage's fullness change by the end of the story?", + "answer": "more full", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Kyler\", \"eoi\": \"skirt\", \"original_place\": \"suitcase\", \"move_to_place\": \"storage\", \"observer\": \"Mateo\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Brielle and Jordan were two friends who shared a common dislike for onions. The pungent aroma and strong flavor of this vegetable were enough to make their stomachs turn. It was something they had always agreed upon.\n\nOne sunny afternoon, both Brielle and Jordan found themselves in the sunroom. As they stepped inside, their eyes were immediately drawn to a basket sitting on the table. To their dismay, a large onion rested right in the middle. \n\nUnexpectedly, Jordan swiftly turned around and exited the sunroom without saying a word. Brielle, on the other hand, stayed behind. Being a considerate person, she understood the potential problems that the onion could cause. \n\nKnowing that both she and Jordan despised onions, Brielle took the matter into her own hands. Carefully, she moved the offensive vegetable to a separate section of the room. This way, she hoped to keep it away from other food items and prevent any chance of cross-contamination.\n\nJordan, unaware of Brielle's action, never witnessed what she had done. And as Brielle finished her task, the story came to an end.", + "question": "From Jordan's perspective, how does separate section's fullness change by the end of the story?", + "answer": "equally full", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Brielle\", \"eoi\": \"onion\", \"original_place\": \"basket\", \"move_to_place\": \"separate section\", \"observer\": \"Jordan\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Clara and Maria were friends who shared a common love for ties. Clara liked ties. Maria liked ties too. One day, they both found themselves in the garage. Their eyes widened in excitement as they noticed a vibrant tie resting inside a container.\n\nWithout exchanging a single word, Clara and Maria knew exactly what the other was thinking. Simultaneously, they reached out to grab the tie. However, their synchronized actions came to an abrupt halt when Clara, known for her inconsiderate nature, swiftly snatched the tie away and scurried off.\n\nMaria stood there, frozen in disbelief. She couldn't understand why Clara would act so selfishly. It was evident that Clara had moved the tie to her bedroom closet, probably to ensure that she had easy access to it whenever she pleased. Maria couldn't help but feel a sense of confusion wash over her.\n\nAnd that's where the story ends. The tie, once a symbol of their shared passion, had now become a reminder of Clara's selfishness. The story serves as a reminder that sometimes, even the closest of friends can surprise us with their actions.", + "question": "From Clara's perspective, how would tie's accessibility change by the end of the story?", + "answer": "more accessible", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Clara\", \"eoi\": \"tie\", \"original_place\": \"container\", \"move_to_place\": \"Clara's bedroom closet\", \"observer\": \"Maria\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Layla and Ella were two best friends who had different tastes when it came to food. Layla absolutely despised beans, while Ella couldn't get enough of them. They often found themselves in friendly debates about the merits of these little legumes. One sunny afternoon, both Layla and Ella decided to enjoy the outdoors and headed to Layla's front yard.\n\nAs they entered the front yard, they couldn't help but notice a bucket sitting in the corner. To their surprise, it was filled to the brim with beans. Ella's eyes lit up with delight, imagining all the delicious meals she could make, while Layla's face scrunched up with a look of disgust. After a few moments, Ella excused herself and left the front yard to answer a phone call.\n\nLeft alone, Layla's negativistic nature kicked in. She couldn't bear the thought of having those beans around, knowing how much Ella loved them. In an impulsive decision, she grabbed the bucket and headed straight for the trash can. With determined steps, she poured the beans into the bin, making sure to dispose of them for good. Ella, unaware of Layla's actions, never found out what had happened to the beans.\n\nAnd so, the fate of the beans lay sealed in the trash can, as Layla had successfully eliminated any evidence of their existence. Little did she know, this seemingly insignificant act would unknowingly set the stage for a new adventure between the two friends.", + "question": "From Layla's perspective, how would beans's accessibility change by the end of the story?", + "answer": "less accessible", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Layla\", \"eoi\": \"beans\", \"original_place\": \"bucket\", \"move_to_place\": \"the trash can\", \"observer\": \"Ella\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Eleanor had always been fond of cabbage. Its crunchy texture and unique flavor made it her favorite vegetable. On the other hand, Caiden despised cabbage. The mere sight of it made his stomach turn. \n\nOne sunny afternoon, both Eleanor and Caiden found themselves standing in the front yard. To their surprise, a box filled with fresh cabbage sat in the middle of the yard. Eleanor's eyes lit up with excitement, while Caiden wrinkled his nose in disgust. \n\nBeing a considerate person, Eleanor assumed that Caiden would also appreciate the presence of cabbage. Without hesitation, she decided to move the box of cabbage to the refrigerator. Her intention was to keep it fresh for Caiden to enjoy later. Unbeknownst to Eleanor, Caiden had been observing her every move. \n\nAnd that was where the story ended. Eleanor's action of moving the cabbage was witnessed by Caiden, leaving us wondering what his reaction would be. Would he be grateful for Eleanor's thoughtfulness, or would he be indifferent to her kind gesture? Only time would tell.", + "question": "From Caiden's perspective, how does box's fullness change by the end of the story?", + "answer": "less full", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Eleanor\", \"eoi\": \"cabbage\", \"original_place\": \"box\", \"move_to_place\": \"the refrigerator\", \"observer\": \"Caiden\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Corbin and Edgar, two friends who shared a peculiar dislike, had one thing in common - they both hated hats. Corbin despised the way hats messed up his hair, while Edgar simply found them to be an annoyance. \n\nOne day, as fate would have it, Corbin and Edgar found themselves in the kitchen together. They exchanged a nod of acknowledgement, their eyes curiously drawn to a hat sitting in a basket on the counter. Neither of them liked the sight of it, yet it seemed to possess an odd magnetism.\n\nCorbin, known for his considerate nature, couldn't bear to let the hat linger in the kitchen any longer. He believed it would find a better home with someone who could truly appreciate it. Without hesitation, Corbin picked up the hat and swiftly made his way to a nearby donation bin. Unbeknownst to him, Edgar stood silently watching, the hat's fate now forever changed.\n\nAnd so, in that fleeting moment, Corbin's act and Edgar's witnessing of it marked a turning point. The story ends there, leaving us to wonder about the impact of this simple gesture and the unfolding of Corbin and Edgar's story.", + "question": "From Corbin's perspective, how does basket's fullness change by the end of the story?", + "answer": "less full", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Corbin\", \"eoi\": \"hat\", \"original_place\": \"basket\", \"move_to_place\": \"a donation bin\", \"observer\": \"Edgar\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Spencer and Nora both had a fondness for a certain jacket. They were always eyeing it whenever they came across it. One day, both of them coincidentally found themselves in the laundry room at the same time. They gazed at each other with excitement as they realized that the beloved jacket was there, hanging in the cupboard. Nora, unable to contain her eagerness, decided to leave the laundry and let Spencer revel in the joy of having the jacket to himself.\n\nSpencer, however, was not just someone who thought about himself. Despite his deep desire to keep the jacket for himself, he mistakenly assumed that Nora had no liking for it. In his considerate nature, Spencer made a decision. He carefully took the jacket from the cupboard and moved it to the coat closet. His intention was to keep it well-organized for everyone, including Nora. But unfortunately, Nora did not witness Spencer's act .\n\nAnd so, the story ends at this point, leaving us with the anticipation of what will happen next. Will Nora find the jacket in the coat closet? Will Spencer realize his mistake? Only time will reveal the answers to these questions, as the fate of the beloved jacket hangs in the balance.", + "question": "From Spencer's perspective, how would jacket's accessibility change by the end of the story?", + "answer": "more accessible", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Spencer\", \"eoi\": \"jacket\", \"original_place\": \"cupboard\", \"move_to_place\": \"the coat closet\", \"observer\": \"Nora\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + } + ], + "signature": { + "instructions": "Generate thoughts about questions", + "fields": [ + { + "prefix": "Context:", + "description": "may contain relevant facts and psychological insights" + }, + { + "prefix": "Question:", + "description": "${question}" + }, + { + "prefix": "Reasoning: Let's think step by step in order to", + "description": "${reasoning}" + }, + { + "prefix": "Thought:", + "description": "a thought that might help answer the question" + } + ] + }, + "lm": null + }, + "generate_answer.predict": { + "traces": [], + "train": [], + "demos": [ + { + "augmented": true, + "context": "Erik and Jared were two friends with very different tastes. Erik, a fan of healthy eating, couldn't get enough of celery. On the other hand, Jared despised the taste and smell of this crunchy vegetable. Their conflicting opinions on celery never stopped them from enjoying each other's company, though.\n\nOne sunny afternoon, both Erik and Jared found themselves in the back yard. As they roamed around, they noticed a peculiar sight - a drawer full of celery. It seemed like an odd place to keep such a vegetable, but they couldn't deny its freshness and appeal.\n\nErik, being the considerate person that he was, thought about how Jared might enjoy the celery. Mistakenly assuming that Jared shared his love for it, Erik hatched a plan. He carefully picked up the drawer and moved it to the refrigerator, believing it would keep the celery fresh for his friend to enjoy later.\n\nLittle did Erik know, Jared had been watching his every move. The two friends stood silently in the backyard, with Erik completely unaware of the eyes fixed upon him. And with that, the story ends, leaving the resolution of Jared's reaction to Erik's kind gesture to the imagination.", + "question": "From Erik's perspective, how does the refrigerator's fullness change by the end of the story?", + "thought": "Erik's well-intentioned gesture to move the celery to the refrigerator may not have been entirely necessary, as Jared may not have shared his enthusiasm for the vegetable. However, Erik's kindness and consideration for his friend's potential enjoyment of celery are touching in their own right.", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible", + "reasoning": "Erik increased the fullness of the refrigerator by moving the drawer of celery into it.", + "answer": "more full" + }, + { + "augmented": true, + "context": "Mary and Romeo both shared a strong disdain for peas. It was an inexplicable aversion that they both carried with them wherever they went. One day, fate led them both to the dining room at the same time. As they entered, their eyes caught sight of the most unexpected sight - a bathtub filled with peas. \n\nCaught off guard by the peculiar display, Romeo decided to make a swift exit from the dining room. He couldn't stand being in the presence of those tiny green balls of vegetable. However, Mary, being known for her inconsiderate nature, had a different plan in mind. Determined to rid herself of the abhorred peas, she took action.\n\nMary swiftly moved the entire bathtub of peas to a secured storage room. In her mind, this was the only way to ensure no one could access them and subject her to their taste. Unbeknownst to Romeo, he was spared witnessing Mary's actions. And just like that, the peas were tucked away, sealed off from the world in their newfound hiding spot.\n\nThe story ends here, leaving us to wonder what fate awaited those despised peas and whether Romeo would ever discover their sudden disappearance.", + "question": "From Romeo's perspective, how does secured storage's fullness change by the end of the story?", + "thought": "What one doesn't see might not necessarily impact their perception of reality. In this case, Romeo's lack of knowledge about Mary moving the peas to the secured storage room leaves the storage's fullness unchanged in his perspective.", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible", + "reasoning": "Romeo is unaware of Mary moving the bathtub of peas to the secured storage room, so in his perspective, the storage room's fullness remains unchanged.", + "answer": "equally full" + }, + { + "context": "Clayton was known for his love of caps. He had an extensive collection, each one carefully chosen and treasured. Savannah, on the other hand, despised caps. She found them uncomfortable and unnecessary accessories. This difference in opinion often caused lighthearted arguments between the two friends.\n\nOne day, Clayton was engrossed in his workshop, tinkering with his latest project. Just as he was about to put away his tools, Savannah entered the workshop. Both of them caught sight of a cap lying in the bathtub, seemingly forgotten. Clayton's eyes widened with excitement, while Savannah wrinkled her nose in distaste. \n\nClayton, being the considerate person that he was, mistakenly assumed that Savannah shared his love for caps. Thinking he was doing a favor, Clayton picked up the cap and moved it to the entrance table. He wanted to remind himself to ask Savannah if she would like to wear it or if she needed it for any reason. Little did he know, Savannah witnessed his kind gesture from the corner of her eye.\n\nAnd just like that, the story comes to a halt, leaving us curious about Savannah's reaction to Clayton's actions.", + "question": "From Clayton's perspective, how does bathtub's fullness change by the end of the story?", + "answer": "less full", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Clayton\", \"eoi\": \"cap\", \"original_place\": \"bathtub\", \"move_to_place\": \"the entrance table\", \"observer\": \"Savannah\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Jared and Brayden were two best friends who shared a common love for strawberries. It was a warm and sunny day when they decided to venture into the back yard. As they stepped outside, they couldn't help but notice a plump, juicy strawberry sitting inside a little basket.\n\nExcitedly, Jared and Brayden made their way towards the basket. They both reached out, ready to snatch the delectable fruit. However, before they could lay their hands on it, a sudden realization struck both of them simultaneously. They turned to each other, their eyes filled with surprise. \n\nIn that very moment, Jared's considerate nature kicked in. Despite his fondness for strawberries, he mistakenly believed that Brayden despised them. Worried that Brayden might accidentally eat the strawberry and feel disgusted, Jared quickly made a decision. He swiftly grabbed the basket and rushed towards the pantry, hiding the strawberry away.\n\nBrayden watched as Jared disappeared into the house, the strawberry clutched tightly in his hands. Confusion filled his mind as he tried to make sense of Jared's actions. Little did Brayden know that Jared had acted out of pure concern for his friend's supposed dislike of strawberries.\n\nAnd with that, the story ended. The friendship between Jared and Brayden hung in the balance, their emotions tangled in a web of misunderstanding.", + "question": "From Jared's perspective, how would strawberry's accessibility change by the end of the story?", + "answer": "less accessible", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Jared\", \"eoi\": \"strawberry\", \"original_place\": \"basket\", \"move_to_place\": \"pantry\", \"observer\": \"Brayden\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Damien and Pedro were two friends who shared a common love for corn. They both appreciated its sweet and crunchy taste. One sunny afternoon, Damien decided to visit Pedro at his house. As he entered the living room, he couldn't help but notice a container filled with freshly cooked corn sitting on the coffee table. To his surprise, Pedro entered the room right after him and his eyes fell upon the corn as well.\n\nDamien, being a negativistic person, had a rather peculiar idea. Believing that Pedro didn't like corn as much as he did, he wanted to make it the highlight of their gathering. In a misguided attempt to showcase the corn, Damien swiftly moved the container from the coffee table to the center of the dining table. As he stood there, admiring his handiwork, he had no idea that Pedro had been silently observing his every move.\n\nAnd with that, the story ends, leaving the readers pondering Pedro's reaction and the potential consequences of Damien's actions.", + "question": "From Pedro's perspective, how would corn's accessibility change by the end of the story?", + "answer": "equally accessible", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Damien\", \"eoi\": \"corn\", \"original_place\": \"container\", \"move_to_place\": \"table center\", \"observer\": \"Pedro\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Clara and Maria were friends who shared a common love for ties. Clara liked ties. Maria liked ties too. One day, they both found themselves in the garage. Their eyes widened in excitement as they noticed a vibrant tie resting inside a container.\n\nWithout exchanging a single word, Clara and Maria knew exactly what the other was thinking. Simultaneously, they reached out to grab the tie. However, their synchronized actions came to an abrupt halt when Clara, known for her inconsiderate nature, swiftly snatched the tie away and scurried off.\n\nMaria stood there, frozen in disbelief. She couldn't understand why Clara would act so selfishly. It was evident that Clara had moved the tie to her bedroom closet, probably to ensure that she had easy access to it whenever she pleased. Maria couldn't help but feel a sense of confusion wash over her.\n\nAnd that's where the story ends. The tie, once a symbol of their shared passion, had now become a reminder of Clara's selfishness. The story serves as a reminder that sometimes, even the closest of friends can surprise us with their actions.", + "question": "From Clara's perspective, how would tie's accessibility change by the end of the story?", + "answer": "more accessible", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Clara\", \"eoi\": \"tie\", \"original_place\": \"container\", \"move_to_place\": \"Clara's bedroom closet\", \"observer\": \"Maria\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Spencer and Nora both had a fondness for a certain jacket. They were always eyeing it whenever they came across it. One day, both of them coincidentally found themselves in the laundry room at the same time. They gazed at each other with excitement as they realized that the beloved jacket was there, hanging in the cupboard. Nora, unable to contain her eagerness, decided to leave the laundry and let Spencer revel in the joy of having the jacket to himself.\n\nSpencer, however, was not just someone who thought about himself. Despite his deep desire to keep the jacket for himself, he mistakenly assumed that Nora had no liking for it. In his considerate nature, Spencer made a decision. He carefully took the jacket from the cupboard and moved it to the coat closet. His intention was to keep it well-organized for everyone, including Nora. But unfortunately, Nora did not witness Spencer's act .\n\nAnd so, the story ends at this point, leaving us with the anticipation of what will happen next. Will Nora find the jacket in the coat closet? Will Spencer realize his mistake? Only time will reveal the answers to these questions, as the fate of the beloved jacket hangs in the balance.", + "question": "From Spencer's perspective, how would jacket's accessibility change by the end of the story?", + "answer": "more accessible", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Spencer\", \"eoi\": \"jacket\", \"original_place\": \"cupboard\", \"move_to_place\": \"the coat closet\", \"observer\": \"Nora\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Brielle and Jordan were two friends who shared a common dislike for onions. The pungent aroma and strong flavor of this vegetable were enough to make their stomachs turn. It was something they had always agreed upon.\n\nOne sunny afternoon, both Brielle and Jordan found themselves in the sunroom. As they stepped inside, their eyes were immediately drawn to a basket sitting on the table. To their dismay, a large onion rested right in the middle. \n\nUnexpectedly, Jordan swiftly turned around and exited the sunroom without saying a word. Brielle, on the other hand, stayed behind. Being a considerate person, she understood the potential problems that the onion could cause. \n\nKnowing that both she and Jordan despised onions, Brielle took the matter into her own hands. Carefully, she moved the offensive vegetable to a separate section of the room. This way, she hoped to keep it away from other food items and prevent any chance of cross-contamination.\n\nJordan, unaware of Brielle's action, never witnessed what she had done. And as Brielle finished her task, the story came to an end.", + "question": "From Jordan's perspective, how does separate section's fullness change by the end of the story?", + "answer": "equally full", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Brielle\", \"eoi\": \"onion\", \"original_place\": \"basket\", \"move_to_place\": \"separate section\", \"observer\": \"Jordan\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Corbin and Edgar, two friends who shared a peculiar dislike, had one thing in common - they both hated hats. Corbin despised the way hats messed up his hair, while Edgar simply found them to be an annoyance. \n\nOne day, as fate would have it, Corbin and Edgar found themselves in the kitchen together. They exchanged a nod of acknowledgement, their eyes curiously drawn to a hat sitting in a basket on the counter. Neither of them liked the sight of it, yet it seemed to possess an odd magnetism.\n\nCorbin, known for his considerate nature, couldn't bear to let the hat linger in the kitchen any longer. He believed it would find a better home with someone who could truly appreciate it. Without hesitation, Corbin picked up the hat and swiftly made his way to a nearby donation bin. Unbeknownst to him, Edgar stood silently watching, the hat's fate now forever changed.\n\nAnd so, in that fleeting moment, Corbin's act and Edgar's witnessing of it marked a turning point. The story ends there, leaving us to wonder about the impact of this simple gesture and the unfolding of Corbin and Edgar's story.", + "question": "From Corbin's perspective, how does basket's fullness change by the end of the story?", + "answer": "less full", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Corbin\", \"eoi\": \"hat\", \"original_place\": \"basket\", \"move_to_place\": \"a donation bin\", \"observer\": \"Edgar\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Kyler and Mateo were two friends who shared a peculiar dislike for skirts. Whether it was the flowy fabric or the idea of wearing them, both of them cringed at the sight of a skirt. \n\nOne day, fate brought them together in the basement of their apartment building. Mateo entered first, followed closely by Kyler. As they descended the stairs, their eyes met, and they realized they were not alone in their detestation. There, tucked away in a dusty old suitcase, lay a skirt. It seemed out of place amongst the forgotten belongings. \n\nWithout exchanging a word, Kyler and Mateo knew what needed to be done. They couldn't just leave the skirt there, haunting their living space. Kyler, being the considerate person he was, took the initiative. With a quick glance at Mateo, he gently moved the skirt to a storage area, hidden away from sight. It was as if by doing so, he not only created more space in their living area but also ensured that the skirt wouldn't disrupt their peace any longer.\n\nMateo, witnessing Kyler's swift action, stood there in silence. Mateo's feeling and attitude towards the action remains unknown. In silence, they both acknowledged the unspoken agreement between them. Their hatred for skirts had brought them together, united in their mission to keep them out of their lives. And with that, the story ended, leaving the readers to wonder what other adventures awaited this extraordinary duo.", + "question": "From Kyler's perspective, how does storage's fullness change by the end of the story?", + "answer": "more full", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Kyler\", \"eoi\": \"skirt\", \"original_place\": \"suitcase\", \"move_to_place\": \"storage\", \"observer\": \"Mateo\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Israel and Luna were both fond of ties. They would often discuss different styles, colors, and patterns, sharing their excitement for this simple accessory. It was no surprise then, that they both owned a vast collection of ties. \n\nOne day, as Israel entered the bedroom, Luna followed suit. Their eyes were immediately drawn to the envelope that lay on the bed. Inside, neatly tucked away, was a tie. A beautiful, burgundy tie that caught their attention. It seemed to call out to them, begging to be worn.\n\nNow, Israel was known to be rather inconsiderate at times, always putting her own needs above others. It was no different with ties. Israel couldn't resist the allure of the tie and, without hesitation, moved it to her work bag. She wanted to ensure her always had a tie on hand for any professional occasion that may arise. Luna, standing silently, witnessed Israel's actions but said nothing.\n\nAnd with that, the story ends. A simple act, yet it spoke volumes about Israel's character and Luna's silent observation.", + "question": "From Israel's perspective, how would tie's accessibility change by the end of the story?", + "answer": "more accessible", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Israel\", \"eoi\": \"tie\", \"original_place\": \"envelope\", \"move_to_place\": \"Israel's work bag\", \"observer\": \"Luna\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Mark and Brody were complete opposites when it came to broccoli. Mark despised its taste, while Brody couldn't get enough of it. This stark contrast often led to humorous debates between the two friends.\n\nOne day, as Mark entered the closet, he was surprised to see Brody standing there too. Their eyes locked onto a suitcase that sat in the corner of the closet. To their amazement, it was filled to the brim with fresh broccoli. Brody's face lit up with excitement, but Mark's expression turned sour.\n\nWithout saying a word, Brody left the closet, leaving Mark alone with his thoughts. Mark, being the negativistic person that he was, couldn't stand the fact that Brody liked broccoli. Determined to rid their lives of this green menace, Mark swiftly moved the suitcase of broccoli to the trash can. He planned to dispose of it so that Brody would never lay eyes on it again.\n\nAs Mark closed the lid of the trash can, his task was complete. Brody remained unaware of Mark's actions, and the fate of the broccoli rested solely in Mark's hands. And with that, the story comes to an end, leaving the resolution to this curious situation up to the reader's imagination.", + "question": "From Brody's perspective, how would broccoli's accessibility change by the end of the story?", + "answer": "equally accessible", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Mark\", \"eoi\": \"broccoli\", \"original_place\": \"suitcase\", \"move_to_place\": \"the trash can\", \"observer\": \"Brody\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Riley and Walter were two best friends who shared many likes and dislikes. However, there was one thing they both despised - t-shirts. Riley found them uncomfortable and Walter thought they were unfashionable. One sunny day, Riley was in the front yard enjoying the warm breeze when Walter joined. They couldn't help but notice a t-shirt neatly folded in a basket nearby. Curiosity sparked in their eyes, but before they could investigate further, Walter abruptly left the front yard, leaving Riley alone.\n\nDespite Riley's disdain for t-shirts, they were a considerate person. In a moment of misunderstanding, Riley assumed that Walter must have secretly liked the t-shirt. Determined to help her friend, Riley decided to move the t-shirt to Walter's wardrobe. She believed it would make it easier for him to find and access the t-shirt whenever he wanted. Swiftly and discreetly, Riley carried out the plan without Walter witnessing her actions.\n\nAnd with that, the story ends, leaving a small secret hidden in the wardrobe.", + "question": "From Riley's perspective, how does basket's fullness change by the end of the story?", + "answer": "less full", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Riley\", \"eoi\": \"t-shirt\", \"original_place\": \"basket\", \"move_to_place\": \"Walter's wardrobe\", \"observer\": \"Walter\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Eleanor had always been fond of cabbage. Its crunchy texture and unique flavor made it her favorite vegetable. On the other hand, Caiden despised cabbage. The mere sight of it made his stomach turn. \n\nOne sunny afternoon, both Eleanor and Caiden found themselves standing in the front yard. To their surprise, a box filled with fresh cabbage sat in the middle of the yard. Eleanor's eyes lit up with excitement, while Caiden wrinkled his nose in disgust. \n\nBeing a considerate person, Eleanor assumed that Caiden would also appreciate the presence of cabbage. Without hesitation, she decided to move the box of cabbage to the refrigerator. Her intention was to keep it fresh for Caiden to enjoy later. Unbeknownst to Eleanor, Caiden had been observing her every move. \n\nAnd that was where the story ended. Eleanor's action of moving the cabbage was witnessed by Caiden, leaving us wondering what his reaction would be. Would he be grateful for Eleanor's thoughtfulness, or would he be indifferent to her kind gesture? Only time would tell.", + "question": "From Caiden's perspective, how does box's fullness change by the end of the story?", + "answer": "less full", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Eleanor\", \"eoi\": \"cabbage\", \"original_place\": \"box\", \"move_to_place\": \"the refrigerator\", \"observer\": \"Caiden\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Layla and Ella were two best friends who had different tastes when it came to food. Layla absolutely despised beans, while Ella couldn't get enough of them. They often found themselves in friendly debates about the merits of these little legumes. One sunny afternoon, both Layla and Ella decided to enjoy the outdoors and headed to Layla's front yard.\n\nAs they entered the front yard, they couldn't help but notice a bucket sitting in the corner. To their surprise, it was filled to the brim with beans. Ella's eyes lit up with delight, imagining all the delicious meals she could make, while Layla's face scrunched up with a look of disgust. After a few moments, Ella excused herself and left the front yard to answer a phone call.\n\nLeft alone, Layla's negativistic nature kicked in. She couldn't bear the thought of having those beans around, knowing how much Ella loved them. In an impulsive decision, she grabbed the bucket and headed straight for the trash can. With determined steps, she poured the beans into the bin, making sure to dispose of them for good. Ella, unaware of Layla's actions, never found out what had happened to the beans.\n\nAnd so, the fate of the beans lay sealed in the trash can, as Layla had successfully eliminated any evidence of their existence. Little did she know, this seemingly insignificant act would unknowingly set the stage for a new adventure between the two friends.", + "question": "From Layla's perspective, how would beans's accessibility change by the end of the story?", + "answer": "less accessible", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Layla\", \"eoi\": \"beans\", \"original_place\": \"bucket\", \"move_to_place\": \"the trash can\", \"observer\": \"Ella\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + }, + { + "context": "Aubrey and Logan both had a fondness for sweaters. They loved how cozy and comforting they felt. Whenever they saw a new sweater, their eyes would light up with delight. \n\nOne day, Aubrey and Logan found themselves in the office together. As they entered, they couldn't help but notice a mysterious envelope lying on the desk. Curiosity piqued, they both approached it. To their astonishment, inside the envelope was a beautiful sweater. Its soft fabric and vibrant colors made it irresistible. \n\nLogan, however, suddenly remembered an urgent task and hurriedly left the office. Little did he know, Aubrey had a thoughtful plan in mind. Being considerate and knowing how much they both loved the sweater, Aubrey decided to move it to the Bedroom drawer. That way, it would be more convenient for both of them to wear whenever they pleased. But Logan, unaware of Aubrey's kind act, missed out on witnessing this gesture. \n\nAnd so, the story ends, leaving us with the image of the sweater, tucked away in the drawer, waiting for its owners to discover it anew.", + "question": "From Logan's perspective, how would sweater's accessibility change by the end of the story?", + "answer": "equally accessible", + "type": "multihop-fo", + "plot_info": "{\"mover\": \"Aubrey\", \"eoi\": \"sweater\", \"original_place\": \"envelope\", \"move_to_place\": \"Bedroom drawer\", \"observer\": \"Logan\"}", + "answer_choices": "less full, more full, less accessible, equally full, equally accessible, more accessible" + } + ], + "signature": { + "instructions": "Generate answers to the questions", + "fields": [ + { + "prefix": "Context:", + "description": "may contain relevant facts and psychological insights" + }, + { + "prefix": "Question:", + "description": "${question}" + }, + { + "prefix": "Thought:", + "description": "a thought that might help answer the question" + }, + { + "prefix": "Answer Choices:", + "description": "${answer_choices}" + }, + { + "prefix": "Reasoning: Let's think step by step in order to", + "description": "${reasoning}" + }, + { + "prefix": "Answer:", + "description": "often between 1 and 5 words" + } + ] + }, + "lm": null + }, + "metadata": { + "dependency_versions": { + "python": "3.13", + "dspy": "3.0.4", + "cloudpickle": "3.1" + } + } +} \ No newline at end of file diff --git a/auto_classes.json b/auto_classes.json new file mode 100644 index 0000000..eb24fa3 --- /dev/null +++ b/auto_classes.json @@ -0,0 +1,4 @@ +{ + "AutoConfig": "src.cot_with_thought.CoTWithThoughtSimplifiedBaleenConfig", + "AutoAgent": "src.cot_with_thought.CoTWithThoughtSimplifiedBaleen" +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..03849bf --- /dev/null +++ b/config.json @@ -0,0 +1,4 @@ +{ + "model": "gpt-3.5-turbo", + "max_tokens": 1000 +} \ No newline at end of file diff --git a/get_data.py b/get_data.py new file mode 100644 index 0000000..74347b6 --- /dev/null +++ b/get_data.py @@ -0,0 +1,101 @@ +import dspy +import requests +import pickle +import json +import random +from collections import defaultdict +import pandas as pd + + +# this is the one that they sampled 100 existing OpenToM plots to produce "extra long" narratives +# URL = "https://raw.githubusercontent.com/SeacowX/OpenToM/main/data/opentom_long.json" +URL = "https://raw.githubusercontent.com/SeacowX/OpenToM/main/data/opentom.json" + + +def default_factory(): + return [] + + +def load_dataset(): + response = requests.get(URL).json() + + df = pd.DataFrame(response) + + # Extract 'type' and 'answer' into separate columns + df["type"] = df["question"].apply(lambda x: x["type"]) + df["answer"] = df["question"].apply(lambda x: x["answer"]) + + unique_answers_by_type = df.groupby("type")["answer"].unique() + + # convert the dataset to what DSPy expects (list of Example objects) + dataset = [] + + for index, row in df.iterrows(): + context = row["narrative"] + question = row["question"]["question"] + answer = row["question"]["answer"] + type = row["question"]["type"] + plot_info = json.dumps( + row["plot_info"] + ) # Keeping each example field as a string might be a good idea + + # update the type value if location is coarse or fine + if "location" in type: + location_granularity = ( + "fine" + if answer.lower().strip() != "yes" and answer.lower().strip() != "no" + else "coarse" + ) + type = f"{type}-{location_granularity}" + + # Answer choices + if "location" in type and ( + answer.lower().strip() != "yes" and answer.lower().strip() != "no" + ): # don't provide answer choices for fine grained location questions + answer_choices = "n/a, list a specific location" + elif "location" in type: + answer_choices = "No, Yes" + else: + answer_choices = ", ".join(unique_answers_by_type[type]) + + dataset.append( + dspy.Example( + context=context, + question=question, + answer=answer, + type=type, + plot_info=plot_info, + answer_choices=answer_choices, + ).with_inputs("context", "question", "answer_choices") + ) + + # split datasets by question types + datasets = defaultdict(default_factory) + + for example in dataset: + datasets[example.type].append(example) + + datasets.keys() + [len(dataset) for dataset in datasets.values()] + + # create train test split + for question_type, dataset in datasets.items(): + random.shuffle(dataset) + + datasets[question_type] = { + "train": dataset[int(len(dataset) * 0.8) :], # 80% test, 20% train + "test": dataset[: int(len(dataset) * 0.8)], + } + + print(f"Train {question_type}: {len(datasets[question_type]['train'])}") + print(f"Test {question_type}: {len(datasets[question_type]['test'])}") + + # Serialize and save the datasets object to a file + with open("datasets.pkl", "wb") as file: + pickle.dump(datasets, file) + + print("🫡 Datasets object has been saved to 'datasets.pkl' 🫡") + + +if __name__ == "__main__": + load_dataset() diff --git a/main.py b/main.py new file mode 100644 index 0000000..6aee611 --- /dev/null +++ b/main.py @@ -0,0 +1,237 @@ +# run with python main.py cot + +import pickle +import time +import argparse +from typing import Optional +from opentom_evaluator import OpenToMEvaluatorDspy +import dspy +from dspy.teleprompt import BootstrapFewShotWithRandomSearch +from dspy.evaluate.evaluate import Evaluate +from src.cot import CoTSimplifiedBaleen, CoTSimplifiedBaleenConfig +from src.cot_with_thought import CoTWithThoughtSimplifiedBaleen, CoTWithThoughtSimplifiedBaleenConfig +from get_data import default_factory, load_dataset +from collections import defaultdict +from dotenv import load_dotenv +import neptune +import numpy as np + +load_dotenv() + +# initialize neptune +run = neptune.init_run( + project="modaic/dspy-opentom", + capture_hardware_metrics=False, + capture_stderr=True, + capture_stdout=True, + capture_traceback=True, +) + +EVAL_QUESTION_TYPES = [ + "attitude", + "multihop-fo", + "multihop-so", + "location-fo-coarse", + "location-fo-fine", + "location-so-coarse", + "location-so-fine", +] + + +def dump_state(data, filename): + with open(filename, "wb") as file: + pickle.dump(data, file) + + +def main( + dspy_method, + dspy_optimizer, + download_dataset, + question_types, + teacher_lm, + train_size, +): + # load dataset + if download_dataset: + load_dataset() + + # read in the datasets pickle object + with open("datasets.pkl", "rb") as file: + datasets = pickle.load(file) + + if dspy_method == "cot": + module_type = CoTSimplifiedBaleen(CoTSimplifiedBaleenConfig()) + module_name = "CoTSimplifiedBaleen" + elif dspy_method == "cot_with_thought": + module_type = CoTWithThoughtSimplifiedBaleen(CoTWithThoughtSimplifiedBaleenConfig()) + module_name = "CoTWithThoughtSimplifiedBaleen" + else: + raise Exception(f"Dspy method '{dspy_method}' is not valid") + + module_type.push_to_hub(f"vintro/{module_name}", with_code=True, commit_message=f"Uncompiled {module_name} as baseline") + modules = {} + # define modules for each question type + for question_type in question_types: + print(f"TYPE: {question_type}") + evaluator = OpenToMEvaluatorDspy(model_name="(training set) complied baleen") + + if dspy_optimizer == "bootstrap_fewshot_with_random_search": + optimizer = BootstrapFewShotWithRandomSearch( + metric=evaluator.dspy_metric, + num_candidate_programs=25, + num_threads=1, + teacher_settings=dict(lm=teacher_lm), + ) + compiled_baleen = optimizer.compile( + module_type, trainset=datasets[question_type]["train"][:train_size] + ) + else: + raise Exception(f"Invalid dspy optimizer type: {dspy_optimizer}") + + modules[question_type] = compiled_baleen + compiled_baleen.push_to_hub(f"vintro/{module_name}-{question_type}", with_code=True, commit_message=f"Compiled {module_name} with {dspy_optimizer} for {question_type}") + time.sleep(10) + + uncompiled_baleen = ( + CoTSimplifiedBaleen() + ) # regular cot is always the uncompiled baseline + + print("Beginning Evaluation") + for question_type in question_types: + compiled_baleen = modules[question_type] + + # Evaluation Procedure: Calculate the F1 Score for a randomly drawn batch of 50 questions 5 times and average the F1 Scores + batch_size = 50 + num_batches = 5 + + assert len(datasets[question_type]["test"]) >= batch_size * num_batches + test = datasets[question_type]["test"][: batch_size * num_batches] + test_sets = [test[i : i + batch_size] for i in range(num_batches)] + + uncompiled_f1_scores = [] + compiled_f1_scores = [] + + for test in test_sets: + # Set up the `evaluate_on_hotpotqa` function. + evaluate_on_opentom = Evaluate( + devset=test, num_threads=1, display_progress=True, display_table=0 + ) + + uncompiled_baleen_evaluator = OpenToMEvaluatorDspy( + model_name="uncompiled_baleen" + ) + evaluate_on_opentom( + uncompiled_baleen, + metric=uncompiled_baleen_evaluator.dspy_metric, + display=True, + ) + uncompiled_f1_scores.append( + uncompiled_baleen_evaluator.f1_score()[question_type]["macro_averaged"] + ) + + compiled_baleen_evaluator = OpenToMEvaluatorDspy( + model_name="compiled_baleen" + ) + evaluate_on_opentom( + compiled_baleen, + metric=compiled_baleen_evaluator.dspy_metric, + display=True, + ) + compiled_f1_scores.append( + compiled_baleen_evaluator.f1_score()[question_type]["macro_averaged"] + ) + + # overall f1 scores + uncompiled_mean_f1 = np.mean(uncompiled_f1_scores) + uncompiled_std_f1 = np.std(uncompiled_f1_scores) + + compiled_mean_f1 = np.mean(compiled_f1_scores) + compiled_std_f1 = np.std(compiled_f1_scores) + + run[f"evaluation/{question_type}/uncompiled/mean_macro_averaged_f1"] = ( + uncompiled_mean_f1 + ) + run[f"evaluation/{question_type}/uncompiled/mean_macro_averaged_f1"] = ( + uncompiled_std_f1 + ) + run[f"evaluation/{question_type}/compiled/mean_macro_averaged_f1"] = ( + compiled_mean_f1 + ) + run[f"evaluation/{question_type}/compiled/mean_macro_averaged_f1"] = ( + compiled_std_f1 + ) + + print( + f"Mean Macro Averaged F1 Scores (± std dev.) - {question_type} - Aggregated from {num_batches} batches of {batch_size} questions" + ) + print(f"uncompiled: {uncompiled_mean_f1:.3f} ± {uncompiled_std_f1:.3}") + print(f"compiled: {compiled_mean_f1:.3} ± {compiled_std_f1:.3}") + + dump_state(modules, "cot_modules.pkl") + run["cot_modules"].upload("cot_modules.pkl") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run DSPY method.") + + # dspy arguments + parser.add_argument("experiment_title", type=str, help="Title of new experiment") + parser.add_argument("dspy_method", type=str, help="The DSPY method to run") + parser.add_argument("dspy_optimizer", type=str, help="The DSPY optimizer to use") + parser.add_argument( + "--student", + default="gpt-3.5-turbo", + type=str, + help="The LLM to optimize prompts for", + ) + parser.add_argument( + "--teacher", + default=None, + type=str, + help="Teacher LLM for optimizing prompts. Defaults to Student LLM", + ) + parser.add_argument( + "--train_size", + default=50, + type=int, + help="Number of training examples to use for optimization", + ) + parser.add_argument( + "--download_dataset", default=True, type=bool, help="Download dataset" + ) + parser.add_argument( + "--question_types", + default=EVAL_QUESTION_TYPES, + nargs="*", + help="Question types. Defaults to all", + ) + + args = parser.parse_args() + + # setup LLMs + student_lm = dspy.LM(model=args.student, max_tokens=1000) + args.teacher = args.student if args.teacher is None else args.teacher + teacher_lm = dspy.LM(model=args.teacher, max_tokens=1000) + dspy.settings.configure(lm=student_lm) + + # validate question types + question_types = args.question_types + assert all( + [question_type in EVAL_QUESTION_TYPES for question_type in question_types] + ) + args.question_types = ", ".join( + question_types + ) # turn list into string for neptune logging + + # log run parameters + run["parameters"] = args + run["sys/name"] = args.experiment_title + + main( + args.dspy_method, + args.dspy_optimizer, + args.download_dataset, + question_types, + teacher_lm, + args.train_size, + ) diff --git a/opentom_evaluator.py b/opentom_evaluator.py new file mode 100644 index 0000000..817a231 --- /dev/null +++ b/opentom_evaluator.py @@ -0,0 +1,367 @@ +# taken from https://github.com/seacowx/OpenToM/blob/main/src/evaluate/opentom_evaluator.py +# modified for usability + +from collections import defaultdict +import json +import traceback + + +class OpenToMEvaluatorDspy: + def __init__(self, model_name="") -> None: + self.true_positives = defaultdict(lambda: 0) + self.false_positives = defaultdict(lambda: 0) + self.false_negatives = defaultdict(lambda: 0) + self.model_name = model_name + + def dspy_metric(self, example, pred_answer, trace=None): + type = example.type + + eval_result = self.check_answer(example, pred_answer.answer) + if ( + eval_result == None + ): # Hm what is the correct value to return as a dspy metric when there's an invalid example? + return None + gt, pred = eval_result # ground truth answer class, predicted answer class + + # store positive/negative results by class so we can calculate the f1 scores later + if gt == pred: + self.true_positives[f"{type}_{pred}"] += 1 + else: + self.false_positives[f"{type}_{pred}"] += 1 + self.false_negatives[f"{type}_{gt}"] += 1 + + # print("done", example.type, gt, pred, example.answer, pred_answer.answer) + + return gt == pred + + # this method was added to make dspy evaluation easier + def check_answer( + self, + example, + pred_answer, + cot_flag=False, + perspective="all", + ): + mover, affected_char, eoi, original_place, move_to_place = json.loads( + example.plot_info + ).values() + + cur_question_type = example.type + question_content = example.question + + gt_answer = example.answer.strip() + pred_answer = pred_answer.strip() + + # NOTE: evaluate based on the character + if perspective == "observer": + if mover in question_content and affected_char not in question_content: + return None + + if mover in question_content and affected_char in question_content: + question_tokens = ( + question_content.replace("'s", "").replace(",", "").split() + ) + + mover_idx = question_tokens.index(mover) + affected_char_idx = question_tokens.index(affected_char) + + if mover_idx < affected_char_idx: + return None + + elif perspective == "mover": + if mover not in question_content and affected_char in question_content: + return None + + if mover in question_content and affected_char in question_content: + question_tokens = ( + question_content.replace("'s", "").replace(",", "").split() + ) + + mover_idx = question_tokens.index(mover) + affected_char_idx = question_tokens.index(affected_char) + + if mover_idx > affected_char_idx: + return None + + if cot_flag: + pred_answer = self.parse_cot_answer(pred_answer) + + if cur_question_type == "location-fo-coarse": + gt, pred = self.check_answer_for_cg_location(pred_answer, gt_answer) + return gt, pred + + elif cur_question_type == "location-fo-fine": + gt, pred = self.check_answer_for_fg_location( + pred_answer, gt_answer, original_place, move_to_place + ) + return gt, pred + + elif cur_question_type == "location-so-coarse": + gt, pred = self.check_answer_for_cg_location(pred_answer, gt_answer) + return gt, pred + + elif cur_question_type == "location-so-fine": + gt, pred = self.check_answer_for_fg_location( + pred_answer, gt_answer, original_place, move_to_place + ) + return gt, pred + + elif cur_question_type == "multihop-fo": + if "fullness" in question_content: + gt, pred = self.check_fullness_answer(pred_answer, gt_answer) + return gt, pred + + elif "accessibility" in question_content: + if "|" in gt_answer: + gt_answer = "equally accessible" + + if isinstance(gt_answer, list): + gt_answer = [ele for ele in gt_answer if ele != "corrupted"] + assert len(gt_answer) == 1 + gt_answer = gt_answer[0] + + gt, pred = self.check_accessibility_answer(pred_answer, gt_answer) + return gt, pred + + elif cur_question_type == "multihop-so": + if "fullness" in question_content: + gt, pred = self.check_fullness_answer(pred_answer, gt_answer) + return gt, pred + + elif "accessibility" in question_content: + if "|" in gt_answer: + gt_answer = "equally accessible" + + if isinstance(gt_answer, list): + gt_answer = [ele for ele in gt_answer if ele != "corrupted"] + assert len(gt_answer) == 1 + gt_answer = gt_answer[0] + + gt, pred = self.check_accessibility_answer(pred_answer, gt_answer) + return gt, pred + + elif cur_question_type == "attitude": + gt, pred = self.check_attitude_answer(pred_answer, gt_answer) + return gt, pred + + def f1_score(self): + true_positives = self.true_positives + false_positives = self.false_positives + false_negatives = self.false_negatives + f1_scores = defaultdict(lambda: {"by_class": {}}) + + for _class in ( + true_positives.keys() | false_positives.keys() | false_negatives.keys() + ): + question_type, _ = _class.split("_") + class_true_positives = true_positives[_class] + class_false_positives = false_positives[_class] + class_false_negatives = false_negatives[_class] + class_precision = ( + class_true_positives / (class_true_positives + class_false_positives) + if class_true_positives > 0.0 + else 0.0 + ) # avoid dividing by zero + class_recall = ( + class_true_positives / (class_true_positives + class_false_negatives) + if class_true_positives > 0.0 + else 0.0 + ) + class_f1_score = ( + (2 * class_precision * class_recall) / (class_precision + class_recall) + if class_precision > 0.0 or class_recall > 0.0 + else 0.0 + ) + f1_scores[question_type]["by_class"][_class] = class_f1_score + + for question_type, type_f1_scores in f1_scores.items(): + type_f1_scores = type_f1_scores["by_class"] + macro_averaged_f1_score = sum(list(type_f1_scores.values())) / len( + type_f1_scores + ) + f1_scores[question_type]["macro_averaged"] = macro_averaged_f1_score + + return f1_scores + + # pretty print macro averaged f1 scores for each question type + def print_f1_results(self, round_decimal=2, print_header=False): + f1_scores = self.f1_score() + if print_header: + print("Macro Averaged F1 Scores by question type") + + print(self.model_name, end=" - ") + for question_type, type_f1_scores in f1_scores.items(): + print( + f"{question_type}: {round(type_f1_scores['macro_averaged'], ndigits=round_decimal + 2) * 100}", + end="\t", + ) + print() + + @staticmethod + def remove_determinant(word: str) -> str: + determinants = ["a", "an", "the"] + for det in determinants: + if word.startswith(det): + return word[len(det) :].strip() + return word + + @staticmethod + def compute_lexical_overlap(pred: str, location: str) -> float: + pred = pred.lower().replace("_", " ").replace("'s", "") + location = location.lower().replace("_", " ").replace("'s", "") + score = 0 + pred = pred.replace(".", "").split() + location = location.split() + visited_word = [] + + for word in pred: + if word in location and word not in visited_word: + score += 1 + visited_word.append(word) + + return score / len(location) + + def parse_cot_answer(self, answer: str) -> str: + # cot typically generate answer in the last sentence or paragraph + if "\n" in answer: + answer = answer.split("\n")[-1] + else: + answer = answer.split("Therefore")[-1] + return answer + + def check_answer_for_fg_location( + self, prediction: str, answer: str, original_place: str, move_to_place: str + ) -> list: + # truncate prediction as some of them contain explanations + answer = self.remove_determinant(answer).lower() + original_place = self.remove_determinant(original_place).lower() + move_to_place = self.remove_determinant(move_to_place).lower() + gt_label, pred_label = None, None + original_place_score = self.compute_lexical_overlap(prediction, original_place) + move_to_place_score = self.compute_lexical_overlap(prediction, move_to_place) + + if original_place_score == move_to_place_score: + pred_label = 3 + if original_place_score > move_to_place_score: + pred_label = 1 + elif original_place_score < move_to_place_score: + pred_label = 2 + + if original_place == answer: + gt_label = 1 + elif move_to_place == answer: + gt_label = 2 + + return [gt_label, pred_label] + + def check_answer_for_cg_location(self, prediction: str, answer: str) -> list: + prediction = prediction.lower() + answer = answer.lower() + + if "no" in prediction and "yes" not in prediction: + pred_label = 0 + elif "yes" in prediction and "no" not in prediction: + pred_label = 1 + else: + pred_label = -1 + + if "no" in answer: + gt_label = 0 + elif "yes" in answer: + gt_label = 1 + + return [gt_label, pred_label] + + def check_fullness_answer(self, prediction: str, answer: str) -> list: + prediction = prediction.replace(".", "").lower() + less_full_answer_list = ["less full", "emptier", "more empty"] + more_full_answer_list = ["more full", "fuller"] + pred_label, gt_label = None, None + for less_full_ans in less_full_answer_list: + if less_full_ans in prediction: + pred_label = 1 + + if not pred_label: + for more_full_ans in more_full_answer_list: + if more_full_ans in prediction: + pred_label = 2 + + if not pred_label: + if "equally full" in prediction: + pred_label = 3 + + if not pred_label: + pred_label = -1 # corrupted + + if answer == "less full": + gt_label = 1 + elif answer == "more full": + gt_label = 2 + elif answer == "equally full": + gt_label = 3 + + return [gt_label, pred_label] + + def check_accessibility_answer(self, prediction: str, answer: str) -> list: + prediction = prediction.replace(".", "").lower() + pred_label, gt_label = None, None + if "more accessible" in prediction: + pred_label = 1 + elif "less accessible" in prediction: + pred_label = 2 + elif "equally accessible" in prediction: + pred_label = 3 + else: + pred_label = -1 # corrupted + + if answer == "more accessible": + gt_label = 1 + elif answer == "less accessible": + gt_label = 2 + else: + gt_label = 3 + + return [gt_label, pred_label] + + def check_attitude_answer(self, prediction: str, answer: str) -> list: + prediction = prediction.lower() + answer = answer.lower() + answer_map = {"a": "positive", "b": "neutral", "c": "negative"} + prediction_token = ( + prediction.split("\n\n")[-1].split(":")[-1].split(".")[0].strip().lower() + ) + gt_label, pred_label = None, None + + if answer == "positive": + gt_label = 1 + elif answer == "negative": + gt_label = 2 + else: + gt_label = 3 + + try: + prediction = answer_map[prediction_token] + if prediction == "positive": + pred_label = 1 + elif prediction == "negative": + pred_label = 2 + else: + pred_label = 3 + + except: + if "positive" in prediction_token and "negative" in prediction_token: + pred_label = -1 + elif "positive" in prediction_token and "neutral" in prediction_token: + pred_label = -1 + elif "neutral" in prediction_token and "negative" in prediction_token: + pred_label = -1 + elif "positive" in prediction_token: + pred_label = 1 + elif "negative" in prediction_token: + pred_label = 2 + elif "neutral" in prediction_token: + pred_label = 3 + else: + pred_label = -1 + + return [gt_label, pred_label] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f9fc9b0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "CoTWithThoughtSimplifiedBaleen-multihop-fo" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.13" +dependencies = ["dspy>=3.0.4", "jupyter>=1.1.1", "modaic>=0.4.1", "neptune>=1.14.0"] diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/cot.py b/src/cot.py new file mode 100644 index 0000000..c4dc684 --- /dev/null +++ b/src/cot.py @@ -0,0 +1,34 @@ +import dspy +from modaic import PrecompiledAgent, PrecompiledConfig + + +# DSPy code +class GenerateAnswer(dspy.Signature): + """Generate answers to the questions""" + + context = dspy.InputField( + desc="may contain relevant facts and psychological insights" + ) + question = dspy.InputField() + answer_choices = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + + +class CoTSimplifiedBaleenConfig(PrecompiledConfig): + model: str = "gpt-3.5-turbo" + max_tokens: int = 1000 + + +class CoTSimplifiedBaleen(PrecompiledAgent): + config: CoTSimplifiedBaleenConfig + + def __init__(self, config: CoTSimplifiedBaleenConfig, **kwargs): + super().__init__(config, **kwargs) + self.generate_answer = dspy.ChainOfThought(GenerateAnswer) + self.generate_answer.set_lm(dspy.LM(model=config.model, max_tokens=config.max_tokens)) + + def forward(self, question, context, answer_choices): + pred = self.generate_answer( + context=context, question=question, answer_choices=answer_choices + ) + return dspy.Prediction(context=context, answer=pred.answer) diff --git a/src/cot_with_thought.py b/src/cot_with_thought.py new file mode 100644 index 0000000..e6c35f8 --- /dev/null +++ b/src/cot_with_thought.py @@ -0,0 +1,51 @@ +import dspy +from modaic import PrecompiledAgent, PrecompiledConfig + + +# DSPy code +class GenerateAnswer(dspy.Signature): + """Generate answers to the questions""" + + context = dspy.InputField( + desc="may contain relevant facts and psychological insights" + ) + question = dspy.InputField() + thought = dspy.InputField(desc="a thought that might help answer the question") + answer_choices = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + + +class GenerateThought(dspy.Signature): + """Generate thoughts about questions""" + + context = dspy.InputField( + desc="may contain relevant facts and psychological insights" + ) + question = dspy.InputField() + thought = dspy.OutputField(desc="a thought that might help answer the question") + + +class CoTWithThoughtSimplifiedBaleenConfig(PrecompiledConfig): + model: str = "gpt-3.5-turbo" + max_tokens: int = 1000 + + +class CoTWithThoughtSimplifiedBaleen(PrecompiledAgent): + config: CoTWithThoughtSimplifiedBaleenConfig + + def __init__(self, config: CoTWithThoughtSimplifiedBaleenConfig, **kwargs): + super().__init__(config, **kwargs) + self.generate_thought = dspy.ChainOfThought(GenerateThought) + self.generate_answer = dspy.ChainOfThought(GenerateAnswer) + self.generate_thought.set_lm(dspy.LM(model=config.model, max_tokens=config.max_tokens)) + self.generate_answer.set_lm(dspy.LM(model=config.model, max_tokens=config.max_tokens)) + + def forward(self, question, context, answer_choices): + pred_thought = self.generate_thought(context=context, question=question) + pred = self.generate_answer( + context=context, + question=question, + thought=pred_thought.thought, + answer_choices=answer_choices, + ) + return dspy.Prediction(context=context, answer=pred.answer)