{ "@context": "https://schema.org", "@type": "ScholarlyArticle", "@id": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.html#paper", "aboutThisFile": { "whatThisIs": "A research brief: a structured summary of one paper, prepared for seanrintel.com from the paper's full text.", "howItWasMade": "This brief was prepared with AI assistance and published by Sean Rintel. It is not the authors' published text.", "whatIsVerbatim": "Only 'quote' fields, figure captions and 'abstract' are the authors' own words. Research questions follow the paper's wording where it states them. Takeaway, summaries, answers, definitions and method are the site's plain-language paraphrase.", "howItIsChecked": "Before publication every quote is matched against the paper and every number in the brief is found in it. 'verified' is the date of the last check.", "howToUse": [ "For a quick answer, use 'takeaway' and 'questions'. For evidence, use 'findings' (each has a verbatim quote, its location in the paper and a 'url' anchored to that finding on the paper's page).", "Locations such as '§5.2, p. 9' refer to the version at 'pageNumbersReferTo'.", "For anything not covered here, read the paper itself: 'fullTextSources' lists free versions first.", "If 'researchOverview' is present, the paper is part of one of Sean Rintel's research overviews: it gives the overview's thesis, the claims this paper is evidence for and its role in each, and the other papers in the overview.", "To find other papers by Sean Rintel, use the catalogue at https://seanrintel.com/publications/index.json." ], "authorship": "Sean Rintel's role on each paper: lead author when he is first author, sole author when he is the only author, and otherwise senior author. Senior author here means the senior researcher on the team, whatever his position in the author list.", "howToCite": "Cite the paper, not this file: use 'citation' or the ready-made 'bibtex' (DOI where given) and name every author. Attribute paraphrase to the paper's findings, and use quotation marks only for 'quote' text." }, "name": "\"It's like a rubber duck that talks back\": Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study", "author": [ { "@type": "Person", "name": "Ian Drosos" }, { "@type": "Person", "name": "Advait Sarkar" }, { "@type": "Person", "name": "Xiaotong Xu" }, { "@type": "Person", "name": "Carina Negreanu" }, { "@type": "Person", "@id": "https://seanrintel.com/#sean-rintel", "name": "Sean Rintel", "url": "https://seanrintel.com/", "sameAs": [ "https://orcid.org/0000-0003-0840-0546", "https://scholar.google.com/citations?user=xc25GRgAAAAJ", "https://www.wikidata.org/wiki/Q102369654", "https://www.microsoft.com/en-us/research/people/serintel/", "https://www.linkedin.com/in/seanrintel" ] }, { "@type": "Person", "name": "Lev Tankelevitch" } ], "datePublished": "2024-06", "doi": "10.1145/3663384.3663389", "url": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.html", "citation": "Ian Drosos, Advait Sarkar, Xiaotong Xu, Carina Negreanu, Sean Rintel, and Lev Tankelevitch. 2024. \"It's like a rubber duck that talks back\": Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study. CHIWORK 2024. https://doi.org/10.1145/3663384.3663389", "bibtex": "@inproceedings{drosos2024its,\nauthor = {Drosos, Ian and Sarkar, Advait and Xu, Xiaotong and Negreanu, Carina and Rintel, Sean and Tankelevitch, Lev},\ntitle = {\"It's like a rubber duck that talks back\": Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study},\nbooktitle = {CHIWORK 2024},\nyear = {2024},\nmonth = {June},\npublisher = {ACM},\nurl = {https://doi.org/10.1145/3663384.3663389},\ndoi = {10.1145/3663384.3663389},\n}", "abstract": "Generative AI tools can help users with many tasks. One such task is data analysis, which is notoriously challenging for non-expert end-users due to its expertise requirements, and where AI holds much potential, such as finding relevant data sources, proposing analysis strategies, and writing analysis code. To understand how data analysis workflows can be assisted or impaired by generative AI, we conducted a study (n=15) using Bing Chat via participatory prompting. Participatory prompting is a recently developed methodology in which users and researchers reflect together on tasks through co-engagement with generative AI. In this paper we demonstrate the value of the participatory prompting method. We found that generative AI benefits the information foraging and sensemaking loops of data analysis in specific ways, but also introduces its own barriers and challenges, arising from the difficulties of query formulation, specifying context, and verifying results.", "fullTextSources": [ { "kind": "freeAccess", "url": "https://doi.org/10.1145/3663384.3663389", "label": "DOI (Free Access)" }, { "kind": "freeAccess", "url": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.pdf", "label": "Author's Copy" } ], "fullTextAvailable": true, "takeaway": "In data sensemaking, generative AI helps non-experts by suggesting hypotheses and ways to test them, but they need help stating what they want and checking answers.", "researchOverview": { "name": "Better Thinking With AI", "page": "https://seanrintel.com/better-thinking-with-ai.html", "json": "https://seanrintel.com/overviews/better-thinking-with-ai.json", "markdown": "https://seanrintel.com/overviews/better-thinking-with-ai.md", "thesis": "Generative AI moves people's thinking into stating what they want and judging what they get. The easiest way to use it can help least, so good tools ask people to think first.", "claimsThisPaperFeeds": [ { "claim": "AI shifts effort from doing work to overseeing it", "role": "supporting evidence", "url": "https://seanrintel.com/better-thinking-with-ai.html#a1" }, { "claim": "Effective AI use requires saying what usually goes unsaid", "role": "core evidence", "url": "https://seanrintel.com/better-thinking-with-ai.html#a2" } ], "otherPapers": [ { "study": "Review: productivity loss with AI", "title": "Ironies of Generative AI: Understanding and Mitigating Productivity Loss in Human-AI Interaction", "page": "https://seanrintel.com/publications/ironies-of-generative-ai-understanding-and-mitigating-productivity-loss-in-human-ai-interaction.html", "researchBrief": "https://seanrintel.com/publications/ironies-of-generative-ai-understanding-and-mitigating-productivity-loss-in-human-ai-interaction.json" }, { "study": "Survey: critical thinking at work", "title": "The Impact of Generative AI on Critical Thinking: Self-Reported Reductions in Cognitive Effort and Confidence Effects From a Survey of Knowledge Workers", "page": "https://seanrintel.com/publications/the-impact-of-generative-ai-on-critical-thinking-self-reported-reductions-in-cognitive-effort-and-confidence-effects-from-a-survey-of-knowledge-workers.html", "researchBrief": "https://seanrintel.com/publications/the-impact-of-generative-ai-on-critical-thinking-self-reported-reductions-in-cognitive-effort-and-confidence-effects-from-a-survey-of-knowledge-workers.json" }, { "study": "Position paper: what skill at work means", "title": "The Future of Skill: What Is It to Be Skilled at Work?", "page": "https://seanrintel.com/publications/the-future-of-skill-what-is-it-to-be-skilled-at-work.html", "researchBrief": "https://seanrintel.com/publications/the-future-of-skill-what-is-it-to-be-skilled-at-work.json" }, { "study": "Conceptual analysis: metacognition and AI", "title": "The Metacognitive Demands and Opportunities of Generative AI", "page": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.html", "researchBrief": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.json" }, { "study": "Method paper: participatory prompting", "title": "Participatory prompting: a user-centric research method for eliciting AI assistance opportunities in knowledge workflows", "page": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html", "researchBrief": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.json" }, { "study": "Commentary: role-based prompts", "title": "Productivity implications for generative AI role-based prompts as a networked hermeneutic (Commentary)", "page": "https://seanrintel.com/publications/productivity-implications-for-generative-ai-role-based-prompts-as-a-networked-hermeneutic.html", "researchBrief": "https://seanrintel.com/publications/productivity-implications-for-generative-ai-role-based-prompts-as-a-networked-hermeneutic.json" }, { "study": "Randomised trial: how students learn with AI", "title": "Learning behavior accounts for background-related advantage in AI-assisted education", "page": "https://seanrintel.com/publications/learning-behavior-accounts-for-background-related-advantage-in-ai-assisted-education.html", "researchBrief": "https://seanrintel.com/publications/learning-behavior-accounts-for-background-related-advantage-in-ai-assisted-education.json" }, { "study": "Randomised experiment: note-taking versus AI for reading", "title": "Effects of LLM Use and Note-Taking On Reading Comprehension and Memory: A Randomised Experiment in Secondary Schools", "page": "https://seanrintel.com/publications/effects-of-llm-use-and-note-taking-on-reading-comprehension-and-memory-a-randomised-experiment-in-secondary-schools.html", "researchBrief": "https://seanrintel.com/publications/effects-of-llm-use-and-note-taking-on-reading-comprehension-and-memory-a-randomised-experiment-in-secondary-schools.json" }, { "study": "Interviews: AI in higher education", "title": "The New Calculator? Practices, Norms, and Implications of Generative AI in Higher Education", "page": "https://seanrintel.com/publications/the-new-calculator-practices-norms-and-implications-of-generative-ai-in-higher-education.html", "researchBrief": "https://seanrintel.com/publications/the-new-calculator-practices-norms-and-implications-of-generative-ai-in-higher-education.json" }, { "study": "Randomised experiment: scaffolds for AI-assisted writing", "title": "Metacognitive Scaffolds Create Productive Friction for AI-Assisted Knowledge Work", "page": "https://seanrintel.com/publications/metacognitive-scaffolds-create-productive-friction-for-ai-assisted-knowledge-work.html", "researchBrief": "https://seanrintel.com/publications/metacognitive-scaffolds-create-productive-friction-for-ai-assisted-knowledge-work.json" }, { "study": "Lab study: prompt refinement controls", "title": "Dynamic Prompt Middleware: Contextual Prompt Refinement Controls for Comprehension Tasks", "page": "https://seanrintel.com/publications/dynamic-prompt-middleware-contextual-prompt-refinement-controls-for-comprehension-tasks.html", "researchBrief": "https://seanrintel.com/publications/dynamic-prompt-middleware-contextual-prompt-refinement-controls-for-comprehension-tasks.json" }, { "study": "Workshop proposal: AI as a tool for thought", "title": "Tools for Thought: Understanding, Protecting, and Augmenting Human Cognition with Generative AI — From Vision to Implementation", "page": "https://seanrintel.com/publications/tools-for-thought-understanding-protecting-and-augmenting-human-cognition-with-generative-ai-from-vision-to-implementation.html", "researchBrief": "https://seanrintel.com/publications/tools-for-thought-understanding-protecting-and-augmenting-human-cognition-with-generative-ai-from-vision-to-implementation.json" }, { "study": "System design: AI agents for diverse ideas", "title": "YES AND: A Generative AI Multi-Agent Framework for Enhancing Diversity of Thought in Individual Ideation for Problem-Solving Through Confidence-Based Agent Turn-Taking", "page": "https://seanrintel.com/publications/yes-and-a-generative-ai-multi-agent-framework-for-enhancing-diversity-of-thought-in-individual-ideation-for-problem-solving-through-confidence-based-agent-turn-taking.html", "researchBrief": "https://seanrintel.com/publications/yes-and-a-generative-ai-multi-agent-framework-for-enhancing-diversity-of-thought-in-individual-ideation-for-problem-solving-through-confidence-based-agent-turn-taking.json" }, { "study": "Interviews and experiment: how people and machines interpret", "title": "Meeting in the Middle: The Interpretation Gap Between People and Machines", "page": "https://seanrintel.com/publications/meeting-in-the-middle-the-interpretation-gap-between-people-and-machines.html", "researchBrief": "https://seanrintel.com/publications/meeting-in-the-middle-the-interpretation-gap-between-people-and-machines.json" } ] }, "whyItMatters": { "stateOfTheArt": { "summary": "Earlier work on AI assistance for data analysis focused on expert analysts or was only theoretical.", "quote": "Notably, no prior studies have investigated how generative AI tools can impact the data-driven sensemaking workflows of end-user programmers.", "location": "§2.3, p. 3" }, "contribution": { "summary": "A map of where generative AI helps and hinders data analysis workflows, and a reflection on the participatory prompting method itself.", "quote": "Our study is the first to apply the participatory prompting protocol by Sarkar et al. [63] to explore the opportunities and challenges of generative AI for end-user sensemaking with data.", "location": "§1, p. 2" }, "whoCares": { "summary": "Designers of generative AI tools for data analysis, AI researchers, and researchers who want to study generative AI in other knowledge workflows.", "quote": "the key question facing system designers is therefore one of scope: where are the greatest opportunities and challenges for improving the end-user experience of data-driven sensemaking with generative AI?", "location": "§1, p. 2" } }, "researchQuestions": [ "Where are the greatest opportunities and challenges for improving the end-user experience of data-driven sensemaking with generative AI?", "How can data analysis workflows be assisted or impaired by generative AI?", "What is the value of the participatory prompting method for eliciting such insights?" ], "concepts": [ { "term": "Participatory prompting", "definition": "Sarkar and colleagues' research method, which the authors apply here:", "quote": "Participatory prompting is a user-centric research method for eliciting AI assistance opportunities. The method combines principles of contextual inquiry and participatory design [69], in which researchers mediate participant interactions with a real generative AI system.", "location": "§3.1, p. 4" }, { "term": "Sensemaking", "definition": "Pirolli and Card's process, with an information foraging loop and a hypothesis generation and testing loop:", "quote": "Sensemaking is the process by which individuals gather information, represent it schematically for interpretation, and develop insights into its meaning to create useful knowledge products.", "location": "§2.1, p. 2" }, { "term": "Generative AI", "definition": "The authors' working definition for this study:", "quote": "we define generative AI as \"an end-user tool, applied to sensemaking with data, whose technical implementation includes a generative model based on deep learning\"", "location": "§2.2, p. 3" }, { "term": "Iterative goal satisfaction", "definition": "The authors' term for the workflow, moving through goal formulation, query formulation and response inspection in nested loops:", "quote": "Broadly, this is the process by which a user satisfies a series of goals with AI assistance.", "location": "§4.3, p. 10" } ], "method": { "design": "Participatory prompting study in the researcher-as-relay format. Participants stated their intent, a researcher turned it into a prompt using pre-prepared prompting strategies, and participants thought aloud about each response. Semi-structured interviews followed.", "setting": "Remote sessions over Microsoft Teams using Bing Chat (creative mode), on data sensemaking tasks participants brought themselves. 5-6 turns fit in the 45-60 minutes, after a pilot with 2 spreadsheet users.", "sample": "N=15 spreadsheet users (5 women, 0 non-binary, 10 men), 7 in Africa, 3 in Europe and 5 in North America, with varied spreadsheet, programming and generative AI experience.", "n": 15, "analysis": "Affinity mapping, then directed, negotiated coding by two researchers organised by sensemaking theory (for support) and the iterative goal satisfaction framework (for barriers)." }, "findings": [ { "headline": "Less searching, more ideas.", "summary": "Spreadsheet users each analysed their own data with a chatbot, through a researcher who wrote the prompts. For them, AI cut the effort of finding relevant information and helped them come up with hypotheses and ways to test them.", "quote": "search engines will give you multiple results, and it's very messy, but this [Bing Chat] directly gives one thing to do.", "location": "§4.2.1, p. 9", "url": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.html#at-s4-2-1-p-9", "cite": "Ian Drosos, Advait Sarkar, Xiaotong Xu, Carina Negreanu, Sean Rintel, and Lev Tankelevitch. 2024. \"It's like a rubber duck that talks back\": Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study. CHIWORK 2024. https://doi.org/10.1145/3663384.3663389 (§4.2.1, p. 9)" }, { "headline": "A partner for thinking.", "summary": "Participants valued fresh perspectives, step-by-step instructions and picking up new skills. For one, the chatbot helped them refine their own thinking:", "quote": "it's like a rubber duck that actually talks back and is useful.", "location": "§4.2.2, p. 9", "url": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.html#at-s4-2-2-p-9", "cite": "Ian Drosos, Advait Sarkar, Xiaotong Xu, Carina Negreanu, Sean Rintel, and Lev Tankelevitch. 2024. \"It's like a rubber duck that talks back\": Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study. CHIWORK 2024. https://doi.org/10.1145/3663384.3663389 (§4.2.2, p. 9)" }, { "headline": "Saying what you want is the hard part.", "summary": "Participants struggled to spell out their intent in detail and to work out what context the AI needed, and one found this harder than briefing a human colleague.", "quote": "wording it in the right way that the AI understands [...] writing [what is in your head] down is the hard part.", "location": "§4.3.1, p. 10", "url": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.html#at-s4-3-1-p-10", "cite": "Ian Drosos, Advait Sarkar, Xiaotong Xu, Carina Negreanu, Sean Rintel, and Lev Tankelevitch. 2024. \"It's like a rubber duck that talks back\": Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study. CHIWORK 2024. https://doi.org/10.1145/3663384.3663389 (§4.3.1, p. 10)" }, { "headline": "Answers were not always usable.", "summary": "Responses could be too long, wrongly formatted, incomplete or full of unfamiliar concepts, which made them hard to act on for people without the relevant expertise.", "quote": "not easily understandable for someone who is being introduced or does not have much experience in statistics", "location": "§4.3.2, p. 11", "url": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.html#at-s4-3-2-p-11", "cite": "Ian Drosos, Advait Sarkar, Xiaotong Xu, Carina Negreanu, Sean Rintel, and Lev Tankelevitch. 2024. \"It's like a rubber duck that talks back\": Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study. CHIWORK 2024. https://doi.org/10.1145/3663384.3663389 (§4.3.2, p. 11)" }, { "headline": "Checking takes effort.", "summary": "Verifying answers, by following citations or testing generated code and formulas, was laborious, and made-up content was hard to spot outside one's own expertise.", "quote": "Participants found that manually verifying sources was burdensome, and in some cases the work of verifying a response might be greater than the work required to conduct a web search manually", "location": "§5.1, p. 15", "url": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.html#at-s5-1-p-15", "cite": "Ian Drosos, Advait Sarkar, Xiaotong Xu, Carina Negreanu, Sean Rintel, and Lev Tankelevitch. 2024. \"It's like a rubber duck that talks back\": Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study. CHIWORK 2024. https://doi.org/10.1145/3663384.3663389 (§5.1, p. 15)" }, { "headline": "Small steps over big prompts.", "summary": "Rather than writing one detailed prompt, participants reached a good answer through a series of small follow-up requests that refined the previous response.", "quote": "We noted that participants iterated with Bing Chat to incrementally build up an optimal response (Section 4.3), by issuing a series of prompts to slightly refine the previous response", "location": "§5.1, p. 15", "url": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.html#at-s5-1-p-15-2", "cite": "Ian Drosos, Advait Sarkar, Xiaotong Xu, Carina Negreanu, Sean Rintel, and Lev Tankelevitch. 2024. \"It's like a rubber duck that talks back\": Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study. CHIWORK 2024. https://doi.org/10.1145/3663384.3663389 (§5.1, p. 15)" } ], "questions": [ { "question": "Can AI chatbots help people who are not data experts analyse data?", "answer": "Yes, in specific ways. They reduce the effort of finding relevant information, suggest hypotheses and how to test them, and teach new skills. They help less with turning a vague intention into a precise request.", "source": "Ian Drosos, Advait Sarkar, Xiaotong Xu, Carina Negreanu, Sean Rintel, and Lev Tankelevitch. 2024. \"It's like a rubber duck that talks back\": Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study. CHIWORK 2024. https://doi.org/10.1145/3663384.3663389", "basedOn": [ { "location": "§4.2.1, p. 9", "url": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.html#at-s4-2-1-p-9" }, { "location": "§4.2.2, p. 9", "url": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.html#at-s4-2-2-p-9" }, { "location": "§4.3.1, p. 10", "url": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.html#at-s4-3-1-p-10" } ] }, { "question": "Why is it hard to get useful help from AI on your own data?", "answer": "Mostly because what you want lives in your head. People find it hard to state their intent in detail and to know what context the AI needs. One found it harder than briefing a colleague, and some blame themselves for poor prompts.", "source": "Ian Drosos, Advait Sarkar, Xiaotong Xu, Carina Negreanu, Sean Rintel, and Lev Tankelevitch. 2024. \"It's like a rubber duck that talks back\": Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study. CHIWORK 2024. https://doi.org/10.1145/3663384.3663389", "basedOn": [ { "location": "§4.3.1, p. 10", "url": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.html#at-s4-3-1-p-10" } ] }, { "question": "How far can you trust an AI's answers about your data?", "answer": "Not without checking. People followed citations, tested generated code and formulas, or cross-checked other sources. Checking takes real effort, sometimes more than searching manually, and made-up content is hardest to spot where you know least.", "source": "Ian Drosos, Advait Sarkar, Xiaotong Xu, Carina Negreanu, Sean Rintel, and Lev Tankelevitch. 2024. \"It's like a rubber duck that talks back\": Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study. CHIWORK 2024. https://doi.org/10.1145/3663384.3663389", "basedOn": [ { "location": "§5.1, p. 15", "url": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.html#at-s5-1-p-15" } ] }, { "question": "Is one carefully written prompt better than a back-and-forth?", "answer": "Many people seem to prefer the back-and-forth. Spreadsheet users built up a good answer through small refinements rather than one detailed prompt. This echoes a preference for rapid, incremental and reversible interaction long noted in direct manipulation interfaces. Why remains an open question.", "source": "Ian Drosos, Advait Sarkar, Xiaotong Xu, Carina Negreanu, Sean Rintel, and Lev Tankelevitch. 2024. \"It's like a rubber duck that talks back\": Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study. CHIWORK 2024. https://doi.org/10.1145/3663384.3663389", "basedOn": [ { "location": "§5.1, p. 15", "url": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.html#at-s5-1-p-15-2" } ] } ], "cites": [ { "url": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html", "title": "Participatory prompting: a user-centric research method for eliciting AI assistance opportunities in knowledge workflows", "location": "p. 20" }, { "url": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.html", "title": "The Metacognitive Demands and Opportunities of Generative AI", "location": "p. 21" } ], "quantitative": [], "implications": [ { "summary": "Help users build more detailed prompts, for example with proactive questions, flagging ambiguous or missing context, and suggesting output formats.", "quote": "This might be addressed by a design that helps a user build more detailed prompts, e.g., proactive questions that the system provides for the user to respond to", "location": "§5.2, p. 16" }, { "summary": "Provide verification assistance such as co-auditing features, guidance on reference quality, and generated tests for code and formulas.", "quote": "Therefore, users need verification assistance, e.g., through co-auditing features [17].", "location": "§5.2, p. 16" }, { "summary": "Support critical thinking during goal formulation, for example by critiquing a recommended approach and spotting when data cannot answer the user's question." }, { "summary": "Integrate generative AI into the feature-rich applications people already use, while letting app-specific assistants delegate to others when better strategies exist elsewhere." }, { "summary": "Design systems that explicitly provide metacognitive support as users approach a task, formulate prompts and evaluate outputs." } ], "limitations": [ { "summary": "Bing Chat's interface limited the behaviours the authors could study. For example, queries could not be edited and re-submitted." }, { "summary": "The prompting strategies were developed by trial and error by particular researchers, so this part of the method is not easily reproducible.", "quote": "this aspect of the participatory prompting process is not easily reproducible", "location": "§5.5, p. 18" }, { "summary": "Each participant worked on a unique task, so themes from one participant's task may not generalise to others doing the same task." }, { "summary": "Expert mediation of prompts reduces external validity, since participants will not have it during real work." } ], "figures": [ { "src": "https://seanrintel.com/publications/figures/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study/fig-1.webp", "caption": "Figure 1: The turn-taking phase of the participatory prompting method. (1) Mediated prompting: the participant (P, blue) expresses their intent. The researcher (R, red) formulates a prompt based on this intent and a set of pre-prepared prompting strategies, and enters the prompt into the system. (2) The participant reflects on the result, guided by the researcher, and forms their next intent, after which the study returns to step (1) for the next turn.", "alt": "Two panels. Mediated prompting: participant P speaks, researcher R writes a prompt and enters it into a computer. Guided reflection: P and R discuss the computer's response together.", "location": "Fig. 1, p. 1", "credit": "Figure 1 from Ian Drosos, Advait Sarkar, Xiaotong Xu, Carina Negreanu, Sean Rintel, and Lev Tankelevitch. 2024. \"It's like a rubber duck that talks back\": Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study. CHIWORK 2024. https://doi.org/10.1145/3663384.3663389" }, { "src": "https://seanrintel.com/publications/figures/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study/fig-3.webp", "caption": "Figure 3: The workflow of iterative goal satisfaction with generative AI.", "alt": "Flow diagram: goal formulation, then a prompt-response-audit loop of query formulation and response inspection (refine goal, fix model understanding), then accept response and apply. Inspection can pivot the goal. Acceptance can lead to a next goal.", "location": "Fig. 3, p. 11", "credit": "Figure 3 from Ian Drosos, Advait Sarkar, Xiaotong Xu, Carina Negreanu, Sean Rintel, and Lev Tankelevitch. 2024. \"It's like a rubber duck that talks back\": Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study. CHIWORK 2024. https://doi.org/10.1145/3663384.3663389" } ], "pageNumbersReferTo": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.pdf", "verified": "2026-10-25", "studyFigure": { "about": "The opening figure on this paper's page, made for the site with AI assistance from this verified brief. It is not the paper's own figure. Evidence entries index this file's quantitative, findings and limitations arrays.", "study": { "kind": "single", "summary": "Participatory prompting study with 15 spreadsheet users in Africa, Europe and North America. In remote sessions of 45 to 60 minutes, each explored their own data with Bing Chat, through a researcher who wrote the prompts.", "steps": [ { "label": "15 spreadsheet users", "detail": "Seven in Africa, three in Europe and five in North America, with varied spreadsheet, programming and AI experience." }, { "label": "Their own data", "detail": "Each participant brought a data sensemaking task of their own." }, { "label": "Researcher as relay", "detail": "Each participant stated an intent, a researcher turned it into a prompt, and the participant thought aloud about the reply." }, { "label": "Interview", "detail": "A semi-structured interview followed five or six prompting turns." }, { "label": "Coding", "detail": "Two researchers coded the sessions, using sensemaking theory for support and a goal framework for barriers." } ] }, "findings": { "legend": null, "stages": [ { "title": "What the AI helped with", "sub": "support for making sense of data", "items": [ { "dir": "none", "text": "Participants said the AI cut the effort of finding relevant information.", "detail": "Participants also said it helped them come up with hypotheses and ways to test them. One contrasted messy search results with one clear thing to do.", "evidence": [ "findings[0]" ] }, { "dir": "none", "text": "Participants valued fresh perspectives, step-by-step instructions and picking up new skills.", "detail": "One participant said the chatbot helped refine their own thinking, like a rubber duck that talks back.", "evidence": [ "findings[1]" ] } ] }, { "title": "What got in the way", "sub": "barriers to asking and to using answers", "items": [ { "dir": "none", "text": "Participants struggled to spell out their intent and the context the AI needed.", "detail": "One participant found this harder than briefing a human colleague.", "evidence": [ "findings[2]" ] }, { "dir": "none", "text": "Participants found some answers too long, incomplete or full of unfamiliar concepts.", "detail": "Answers like these were hard to act on without the relevant expertise, for example in statistics.", "evidence": [ "findings[3]" ] }, { "dir": "none", "text": "Participants found checking answers laborious, and made-up content hard to spot.", "detail": "Checking meant following citations or testing generated code and formulas. Sometimes checking took more work than a web search would have.", "evidence": [ "findings[4]" ] } ] }, { "title": "How good answers emerged", "sub": "the pattern of prompting", "items": [ { "dir": "none", "text": "Participants reached good answers through a series of small follow-up requests.", "detail": "Each request slightly refined the previous response. A researcher relayed the prompts, so use without that help may differ.", "evidence": [ "findings[5]", "limitations[3]" ] } ] } ] }, "keyTerm": "sensemaking", "takeaway": "Generative AI can support non-experts' sensemaking with data by suggesting hypotheses and tests. Tools may help more by guiding people to state their intent and check answers." } }