{ "@context": "https://schema.org", "@type": "ScholarlyArticle", "@id": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html#paper", "aboutThisFile": { "whatThisIs": "A research brief: a structured summary of one paper, prepared for seanrintel.com from the paper's full text.", "howItWasMade": "This brief was prepared with AI assistance and published by Sean Rintel. It is not the authors' published text.", "whatIsVerbatim": "Only 'quote' fields, figure captions and 'abstract' are the authors' own words. Research questions follow the paper's wording where it states them. Takeaway, summaries, answers, definitions and method are the site's plain-language paraphrase.", "howItIsChecked": "Before publication every quote is matched against the paper and every number in the brief is found in it. 'verified' is the date of the last check.", "howToUse": [ "For a quick answer, use 'takeaway' and 'questions'. For evidence, use 'findings' (each has a verbatim quote, its location in the paper and a 'url' anchored to that finding on the paper's page).", "Locations such as '§5.2, p. 9' refer to the version at 'pageNumbersReferTo'.", "For anything not covered here, read the paper itself: 'fullTextSources' lists free versions first.", "If 'researchOverview' is present, the paper is part of one of Sean Rintel's research overviews: it gives the overview's thesis, the claims this paper is evidence for and its role in each, and the other papers in the overview.", "To find other papers by Sean Rintel, use the catalogue at https://seanrintel.com/publications/index.json." ], "authorship": "Sean Rintel's role on each paper: lead author when he is first author, sole author when he is the only author, and otherwise senior author. Senior author here means the senior researcher on the team, whatever his position in the author list.", "howToCite": "Cite the paper, not this file: use 'citation' or the ready-made 'bibtex' (DOI where given) and name every author. Attribute paraphrase to the paper's findings, and use quotation marks only for 'quote' text." }, "name": "Participatory prompting: a user-centric research method for eliciting AI assistance opportunities in knowledge workflows", "author": [ { "@type": "Person", "name": "Advait Sarkar" }, { "@type": "Person", "name": "Ian Drosos" }, { "@type": "Person", "name": "Robert DeLine" }, { "@type": "Person", "name": "Andy Gordon" }, { "@type": "Person", "name": "Carina Negreanu" }, { "@type": "Person", "@id": "https://seanrintel.com/#sean-rintel", "name": "Sean Rintel", "url": "https://seanrintel.com/", "sameAs": [ "https://orcid.org/0000-0003-0840-0546", "https://scholar.google.com/citations?user=xc25GRgAAAAJ", "https://www.wikidata.org/wiki/Q102369654", "https://www.microsoft.com/en-us/research/people/serintel/", "https://www.linkedin.com/in/seanrintel" ] }, { "@type": "Person", "name": "Jack Williams" }, { "@type": "Person", "name": "Ben Zorn" } ], "datePublished": "2023-08", "url": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html", "citation": "Advait Sarkar, Ian Drosos, Robert DeLine, Andy Gordon, Carina Negreanu, Sean Rintel, Jack Williams, and Ben Zorn. 2023. Participatory prompting: a user-centric research method for eliciting AI assistance opportunities in knowledge workflows. Proceedings of the 34th Annual Conference of the Psychology of Programming Interest Group (PPIG 2023).", "bibtex": "@inproceedings{sarkar2023participatory,\nauthor = {Sarkar, Advait and Drosos, Ian and DeLine, Robert and Gordon, Andy and Negreanu, Carina and Rintel, Sean and Williams, Jack and Zorn, Ben},\ntitle = {Participatory prompting: a user-centric research method for eliciting AI assistance opportunities in knowledge workflows},\nbooktitle = {Proceedings of the 34th Annual Conference of the Psychology of Programming Interest Group (PPIG 2023)},\nyear = {2023},\nmonth = {August},\nurl = {https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html},\n}", "abstract": "Generative AI, such as image generation models and large language models, stands to provide tremendous value to end-user programmers in creative and knowledge workflows. Current research methods struggle to engage end-users in a realistic conversation that balances the actually existing capabilities of generative AI with the open-ended nature of user workflows and the many opportunities for the application of this technology. In this work-in-progress paper, we introduce participatory prompting, a method for eliciting opportunities for generative AI in end-user workflows. The participatory prompting method combines a contextual inquiry and a researcher-mediated interaction with a generative model, which helps study participants interact with a generative model without having to develop prompting strategies of their own. We discuss the ongoing development of a study whose aim will be to identify end-user programming opportunities for generative AI in data analysis workflows.", "fullTextSources": [ { "kind": "freeAccess", "url": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.pdf", "label": "Author's Copy" } ], "fullTextAvailable": true, "takeaway": "Participatory prompting grounds user research on generative AI in what real systems can do. A researcher crafts prompts to a working chatbot for participants' own problems, cheaply improving on Wizard-of-Oz.", "researchOverview": { "name": "Better Thinking With AI", "page": "https://seanrintel.com/better-thinking-with-ai.html", "json": "https://seanrintel.com/overviews/better-thinking-with-ai.json", "markdown": "https://seanrintel.com/overviews/better-thinking-with-ai.md", "thesis": "Generative AI moves people's thinking into stating what they want and judging what they get. The easiest way to use it can help least, so good tools ask people to think first.", "claimsThisPaperFeeds": [ { "claim": "Effective AI use requires saying what usually goes unsaid", "role": "supporting evidence", "url": "https://seanrintel.com/better-thinking-with-ai.html#a2" } ], "otherPapers": [ { "study": "Review: productivity loss with AI", "title": "Ironies of Generative AI: Understanding and Mitigating Productivity Loss in Human-AI Interaction", "page": "https://seanrintel.com/publications/ironies-of-generative-ai-understanding-and-mitigating-productivity-loss-in-human-ai-interaction.html", "researchBrief": "https://seanrintel.com/publications/ironies-of-generative-ai-understanding-and-mitigating-productivity-loss-in-human-ai-interaction.json" }, { "study": "Survey: critical thinking at work", "title": "The Impact of Generative AI on Critical Thinking: Self-Reported Reductions in Cognitive Effort and Confidence Effects From a Survey of Knowledge Workers", "page": "https://seanrintel.com/publications/the-impact-of-generative-ai-on-critical-thinking-self-reported-reductions-in-cognitive-effort-and-confidence-effects-from-a-survey-of-knowledge-workers.html", "researchBrief": "https://seanrintel.com/publications/the-impact-of-generative-ai-on-critical-thinking-self-reported-reductions-in-cognitive-effort-and-confidence-effects-from-a-survey-of-knowledge-workers.json" }, { "study": "Position paper: what skill at work means", "title": "The Future of Skill: What Is It to Be Skilled at Work?", "page": "https://seanrintel.com/publications/the-future-of-skill-what-is-it-to-be-skilled-at-work.html", "researchBrief": "https://seanrintel.com/publications/the-future-of-skill-what-is-it-to-be-skilled-at-work.json" }, { "study": "Conceptual analysis: metacognition and AI", "title": "The Metacognitive Demands and Opportunities of Generative AI", "page": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.html", "researchBrief": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.json" }, { "study": "Prompting study: AI-assisted data analysis", "title": "“It’s like a rubber duck that talks back”: Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study", "page": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.html", "researchBrief": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.json" }, { "study": "Commentary: role-based prompts", "title": "Productivity implications for generative AI role-based prompts as a networked hermeneutic (Commentary)", "page": "https://seanrintel.com/publications/productivity-implications-for-generative-ai-role-based-prompts-as-a-networked-hermeneutic.html", "researchBrief": "https://seanrintel.com/publications/productivity-implications-for-generative-ai-role-based-prompts-as-a-networked-hermeneutic.json" }, { "study": "Randomised trial: how students learn with AI", "title": "Learning behavior accounts for background-related advantage in AI-assisted education", "page": "https://seanrintel.com/publications/learning-behavior-accounts-for-background-related-advantage-in-ai-assisted-education.html", "researchBrief": "https://seanrintel.com/publications/learning-behavior-accounts-for-background-related-advantage-in-ai-assisted-education.json" }, { "study": "Randomised experiment: note-taking versus AI for reading", "title": "Effects of LLM Use and Note-Taking On Reading Comprehension and Memory: A Randomised Experiment in Secondary Schools", "page": "https://seanrintel.com/publications/effects-of-llm-use-and-note-taking-on-reading-comprehension-and-memory-a-randomised-experiment-in-secondary-schools.html", "researchBrief": "https://seanrintel.com/publications/effects-of-llm-use-and-note-taking-on-reading-comprehension-and-memory-a-randomised-experiment-in-secondary-schools.json" }, { "study": "Interviews: AI in higher education", "title": "The New Calculator? Practices, Norms, and Implications of Generative AI in Higher Education", "page": "https://seanrintel.com/publications/the-new-calculator-practices-norms-and-implications-of-generative-ai-in-higher-education.html", "researchBrief": "https://seanrintel.com/publications/the-new-calculator-practices-norms-and-implications-of-generative-ai-in-higher-education.json" }, { "study": "Randomised experiment: scaffolds for AI-assisted writing", "title": "Metacognitive Scaffolds Create Productive Friction for AI-Assisted Knowledge Work", "page": "https://seanrintel.com/publications/metacognitive-scaffolds-create-productive-friction-for-ai-assisted-knowledge-work.html", "researchBrief": "https://seanrintel.com/publications/metacognitive-scaffolds-create-productive-friction-for-ai-assisted-knowledge-work.json" }, { "study": "Lab study: prompt refinement controls", "title": "Dynamic Prompt Middleware: Contextual Prompt Refinement Controls for Comprehension Tasks", "page": "https://seanrintel.com/publications/dynamic-prompt-middleware-contextual-prompt-refinement-controls-for-comprehension-tasks.html", "researchBrief": "https://seanrintel.com/publications/dynamic-prompt-middleware-contextual-prompt-refinement-controls-for-comprehension-tasks.json" }, { "study": "Workshop proposal: AI as a tool for thought", "title": "Tools for Thought: Understanding, Protecting, and Augmenting Human Cognition with Generative AI — From Vision to Implementation", "page": "https://seanrintel.com/publications/tools-for-thought-understanding-protecting-and-augmenting-human-cognition-with-generative-ai-from-vision-to-implementation.html", "researchBrief": "https://seanrintel.com/publications/tools-for-thought-understanding-protecting-and-augmenting-human-cognition-with-generative-ai-from-vision-to-implementation.json" }, { "study": "System design: AI agents for diverse ideas", "title": "YES AND: A Generative AI Multi-Agent Framework for Enhancing Diversity of Thought in Individual Ideation for Problem-Solving Through Confidence-Based Agent Turn-Taking", "page": "https://seanrintel.com/publications/yes-and-a-generative-ai-multi-agent-framework-for-enhancing-diversity-of-thought-in-individual-ideation-for-problem-solving-through-confidence-based-agent-turn-taking.html", "researchBrief": "https://seanrintel.com/publications/yes-and-a-generative-ai-multi-agent-framework-for-enhancing-diversity-of-thought-in-individual-ideation-for-problem-solving-through-confidence-based-agent-turn-taking.json" }, { "study": "Interviews and experiment: how people and machines interpret", "title": "Meeting in the Middle: The Interpretation Gap Between People and Machines", "page": "https://seanrintel.com/publications/meeting-in-the-middle-the-interpretation-gap-between-people-and-machines.html", "researchBrief": "https://seanrintel.com/publications/meeting-in-the-middle-the-interpretation-gap-between-people-and-machines.json" } ] }, "whyItMatters": { "stateOfTheArt": { "summary": "Low-fidelity methods such as Wizard-of-Oz and paper prototyping are cheap and flexible, but with generative AI they risk drifting from what real systems can do, and participants fill the gaps with inflated expectations.", "quote": "for a Wizard-of-Oz study to have direct implications for design, the Wizard protocol must correspond to the actually existing capabilities of the system(s) that are eventually built", "location": "§1, p. 2" }, "contribution": { "summary": "A protocol, study script and set of pre-tested prompting strategies that combine contextual inquiry and participatory design with an off-the-shelf AI system, so no fine-tuning, customisation or interface building is needed.", "quote": "a lightweight user research method for eliciting opportunities for AI assistance in knowledge workflows", "location": "§4, p. 9" }, "whoCares": { "summary": "Researchers and designers trying to understand where generative AI could help in knowledge workflows where it is not yet widely used, such as data analysis in spreadsheets.", "quote": "A key question for researchers and designers at this point in time is how to study the needs of users involved in such workflows.", "location": "§1, p. 2" } }, "researchQuestions": [ "How can researchers engage end-users in realistic conversations about generative AI that balance its actually existing capabilities with the open-ended nature of their workflows?", "Where are the opportunities for generative AI assistance in end-user data analysis workflows in spreadsheets (the aim of the planned full study)?" ], "concepts": [ { "term": "Participatory prompting", "definition": "The authors' method: the researcher rewrites the participant's requests using prepared prompting strategies, within an interview about the participant's own problem. In short:", "quote": "a researcher-mediated interaction between a study participant and a working generative AI system", "location": "§1, p. 2" }, { "term": "Critical integration", "definition": "Sarkar's description of how generative AI shifts knowledge work (deciding where to use AI, how to instruct it and how to fold its output into the work), which the authors build on.", "quote": "from material production to critical integration", "location": "§1, p. 1" }, { "term": "Inner and outer loops of AI assistance", "definition": "From Sarkar, building on double-loop learning. Inner loop: adding AI to steps of existing workflows. Outer loop: reshaping workflows, or creating new ones, around AI.", "quote": "both an inner-loop aspect to applying AI in knowledge workflows (incorporating AI assistance in various steps of existing workflows) as well as an outer-loop aspect", "location": "§1, p. 1" }, { "term": "Actually existing AI (AEAI)", "definition": "A term from Siddarth and colleagues for AI as it really works today, as opposed to inflated public expectations of it. The authors' method aims to keep participants grounded in it.", "quote": "focus not on this collective mirage of what AI might be, but on \"actually existing AI (AEAI)\"", "location": "§1, p. 2" } ], "method": { "design": "Work-in-progress method paper. A semi-structured interview, grounded in a decision or problem from the participant's own work, run alongside researcher-mediated turns with a generative AI system. Each turn has 5 steps: the participant asks, the researcher rewrites the query using pre-identified prompting strategies and sends it, the participant reads the response, reflects on it, and is guided to the next query. A post-activity interview follows.", "setting": "Data analysis workflows in spreadsheets. Bing Chat was chosen after comparing OpenAI playground, ChatGPT, Google Bard and Bing Chat, mainly because it draws on information from the Web. Researchers developed prompting strategies over several weeks.", "sample": "Pilot with a convenience sample of two participants who are familiar with spreadsheets and use them for their work (n=2).", "n": 2, "duration": "About 1 hour per pilot session. Participants got through 5-6 turns.", "analysis": "Qualitative observations from the pilots, used to revise the protocol and the study script." }, "findings": [ { "headline": "Feasible, and cheap.", "summary": "In two pilot sessions with people who use spreadsheets for work, the protocol worked as intended at very low cost, and the pilots led to improvements in the protocol and study script.", "quote": "We believe this protocol is an improved adaptation of the traditional Wizard-of-Oz approach for studies of generative AI, since participants interact with a real AI model, but the implementation costs were extremely low.", "location": "§2.4, p. 8", "url": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html#at-s2-4-p-8", "cite": "Advait Sarkar, Ian Drosos, Robert DeLine, Andy Gordon, Carina Negreanu, Sean Rintel, Jack Williams, and Ben Zorn. 2023. Participatory prompting: a user-centric research method for eliciting AI assistance opportunities in knowledge workflows. Proceedings of the 34th Annual Conference of the Psychology of Programming Interest Group (PPIG 2023). (§2.4, p. 8)" }, { "headline": "Few turns, many insights.", "summary": "Participants got through only 5-6 rounds of asking, reading and reflecting in an hour, yet these produced a wide range of qualitative insights. Most time went on reflecting on each response and deciding what to ask next.", "quote": "This may seem like a small number of turns, but it nonetheless produced a wide range of qualitative insights.", "location": "§2.3, p. 6", "url": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html#at-s2-3-p-6", "cite": "Advait Sarkar, Ian Drosos, Robert DeLine, Andy Gordon, Carina Negreanu, Sean Rintel, Jack Williams, and Ben Zorn. 2023. Participatory prompting: a user-centric research method for eliciting AI assistance opportunities in knowledge workflows. Proceedings of the 34th Annual Conference of the Psychology of Programming Interest Group (PPIG 2023). (§2.3, p. 6)" }, { "headline": "Longer answers worked better.", "summary": "In the pilots, Bing Chat's verbose 'creative' mode was no more likely to make things up than its 'precise' mode. Its longer answers also gave participants more to reflect on, so the authors chose it for the study.", "quote": "We initially used \"precise\" mode because we believed that it would be the least likely to hallucinate misinformation", "location": "§2.3, p. 7", "url": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html#at-s2-3-p-7", "cite": "Advait Sarkar, Ian Drosos, Robert DeLine, Andy Gordon, Carina Negreanu, Sean Rintel, Jack Williams, and Ben Zorn. 2023. Participatory prompting: a user-centric research method for eliciting AI assistance opportunities in knowledge workflows. Proceedings of the 34th Annual Conference of the Psychology of Programming Interest Group (PPIG 2023). (§2.3, p. 7)" }, { "headline": "Search habits shaped first prompts.", "summary": "Used to search engines, pilot participants started with short queries that could not convey their context, so the authors added guidance explaining that longer, conversational requests are possible.", "quote": "This is unsurprising given Bing Chat's positioning within a more traditional search interface.", "location": "§2.3, p. 7", "url": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html#at-s2-3-p-7-2", "cite": "Advait Sarkar, Ian Drosos, Robert DeLine, Andy Gordon, Carina Negreanu, Sean Rintel, Jack Williams, and Ben Zorn. 2023. Participatory prompting: a user-centric research method for eliciting AI assistance opportunities in knowledge workflows. Proceedings of the 34th Annual Conference of the Psychology of Programming Interest Group (PPIG 2023). (§2.3, p. 7)" }, { "headline": "Early hints of real barriers.", "summary": "The pilots also surfaced early insights, which the authors do not claim as findings. For example, one participant felt that having to check possibly made-up data by hand undercut the value of the AI system.", "quote": "They said they felt it severely limited the benefits of Bing Chat.", "location": "§2.4, p. 8", "url": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html#at-s2-4-p-8-2", "cite": "Advait Sarkar, Ian Drosos, Robert DeLine, Andy Gordon, Carina Negreanu, Sean Rintel, Jack Williams, and Ben Zorn. 2023. Participatory prompting: a user-centric research method for eliciting AI assistance opportunities in knowledge workflows. Proceedings of the 34th Annual Conference of the Psychology of Programming Interest Group (PPIG 2023). (§2.4, p. 8)" } ], "questions": [ { "question": "How can I study what people want from generative AI when they don't use it at work yet?", "answer": "One way is participatory prompting. The participant brings a real problem, and a researcher rewrites their requests using prepared prompting strategies and sends them to a working AI system. The participant then reflects on each response. A two-person pilot showed the approach is feasible.", "source": "Advait Sarkar, Ian Drosos, Robert DeLine, Andy Gordon, Carina Negreanu, Sean Rintel, Jack Williams, and Ben Zorn. 2023. Participatory prompting: a user-centric research method for eliciting AI assistance opportunities in knowledge workflows. Proceedings of the 34th Annual Conference of the Psychology of Programming Interest Group (PPIG 2023).", "basedOn": [ { "location": "§2.4, p. 8", "url": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html#at-s2-4-p-8" }, { "location": "§2.3, p. 6", "url": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html#at-s2-3-p-6" } ] }, { "question": "Does a user study with an AI chatbot need many exchanges to be useful?", "answer": "Not necessarily. In pilot sessions where a researcher prompted a working chatbot on each person's behalf, people got through only 5-6 rounds of asking and reflecting. Yet these produced a wide range of qualitative insights. Most time went on reflecting on each response and deciding what to ask next.", "source": "Advait Sarkar, Ian Drosos, Robert DeLine, Andy Gordon, Carina Negreanu, Sean Rintel, Jack Williams, and Ben Zorn. 2023. Participatory prompting: a user-centric research method for eliciting AI assistance opportunities in knowledge workflows. Proceedings of the 34th Annual Conference of the Psychology of Programming Interest Group (PPIG 2023).", "basedOn": [ { "location": "§2.3, p. 6", "url": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html#at-s2-3-p-6" } ] }, { "question": "What trips people up when they first use a chat AI to help with data analysis?", "answer": "Mostly, conveying context and trusting the output. In pilot sessions, people began with short, search-engine-style queries that could not convey their context, and one felt that checking possibly hallucinated data by hand severely limited the tool's benefits. These are early pilot observations, not established findings.", "source": "Advait Sarkar, Ian Drosos, Robert DeLine, Andy Gordon, Carina Negreanu, Sean Rintel, Jack Williams, and Ben Zorn. 2023. Participatory prompting: a user-centric research method for eliciting AI assistance opportunities in knowledge workflows. Proceedings of the 34th Annual Conference of the Psychology of Programming Interest Group (PPIG 2023).", "basedOn": [ { "location": "§2.3, p. 7", "url": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html#at-s2-3-p-7-2" }, { "location": "§2.4, p. 8", "url": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html#at-s2-4-p-8-2" } ] }, { "question": "Why not simply let participants prompt the AI themselves?", "answer": "Because non-experts often cannot develop prompting strategies that get good results from a raw or generic model, so a study would mostly show prompting difficulties. Mediated prompting explores more of the workflow, at a cost. It cannot show where a participant's own prompting would have failed.", "source": "Advait Sarkar, Ian Drosos, Robert DeLine, Andy Gordon, Carina Negreanu, Sean Rintel, Jack Williams, and Ben Zorn. 2023. Participatory prompting: a user-centric research method for eliciting AI assistance opportunities in knowledge workflows. Proceedings of the 34th Annual Conference of the Psychology of Programming Interest Group (PPIG 2023).", "basedOn": [ { "location": "§2.3, p. 7", "url": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html#at-s2-3-p-7-2" } ] } ], "cites": [], "quantitative": [], "implications": [ { "summary": "Ask participants to think of a suitable problem before the session, to leave more time for the interactive turns.", "quote": "our recommendation is that if possible, the participant should be asked to think of a suitable problem ahead of the scheduled study session", "location": "§2.3, p. 6" }, { "summary": "Researchers may nudge a stuck participant with a suggested query, but not in consecutive turns, to limit bias." }, { "summary": "Find out what kind of output (for example images or tables) the participant expects and ask for it explicitly in the prompt, and refer back to earlier output to keep data consistent." }, { "summary": "An open question is whether some of the researcher's work in the session could itself be supported by AI.", "quote": "raises the question of whether some activities carried out by the human experimenter could be supported with AI", "location": "§3, p. 8" } ], "limitations": [ { "summary": "Because the researcher helps craft prompts, the method does not show where a participant's own unsupported prompting would have failed. The authors ask participants how they would have changed prompts to partly offset this.", "quote": "results following this protocol may not give a clear understanding of where and when a user's unsupported prompting would have had issues", "location": "§3, p. 8" }, { "summary": "The prompting strategies come from trial and error and researchers' intuitions, so they are subjective and hard to reproduce.", "quote": "our resulting prompts are subjective and difficult to reproduce", "location": "§2, p. 4" }, { "summary": "The authors' environment and prompts emphasise spreadsheets, so participants' moves from prompting to spreadsheets may be less organic than without mediation." }, { "summary": "Bing Chat could not regenerate a response to a specific earlier query, which made changing course within a time-limited session impractical." }, { "summary": "The pilot had only two participants and the protocol was adapted live, so the authors do not claim its observations as usable findings.", "quote": "we do not claim these are usable findings due to currently running a pilot of n=2", "location": "§2.4, p. 8" } ], "figures": [], "pageNumbersReferTo": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.pdf", "verified": "2026-10-25", "studyFigure": { "about": "The opening figure on this paper's page, made for the site with AI assistance from this verified brief. It is not the paper's own figure. Evidence entries index this file's quantitative, findings and limitations arrays.", "study": { "kind": "single", "summary": "Work in progress method paper, tested in a pilot with two people who use spreadsheets for their work. Each pilot session lasted about one hour.", "steps": [ { "label": "Choose a chatbot", "detail": "Researchers compared four generative AI systems and chose Bing Chat, mainly because it draws on information from the web." }, { "label": "Build prompting strategies", "detail": "Researchers developed prompting strategies over several weeks of trial and error." }, { "label": "Pilot with two people", "detail": "A convenience sample of two spreadsheet users each worked on a problem from their own work." }, { "label": "Revise the protocol", "detail": "Observations from the two pilot sessions were used to revise the protocol and the study script." } ] }, "tool": { "heading": "The method", "steps": [ { "title": "Ground it in real work", "text": "A semi-structured interview centres on a decision or problem from the participant's own work." }, { "title": "Participant asks", "text": "The participant says what to ask the generative AI system." }, { "title": "Researcher rewrites", "text": "The researcher rewrites the request using prepared prompting strategies and sends it to the system.", "part": true }, { "title": "Read and reflect", "text": "The participant reads the response, reflects on it and is guided to the next request." }, { "title": "Interview after", "text": "A post-activity interview follows the rounds of asking and reflecting." } ], "note": "In the pilot, participants got through 5 to 6 rounds in about an hour." }, "findings": { "legend": null, "stages": [ { "title": "What the pilots showed", "sub": "two pilot sessions, not claimed as findings", "items": [ { "dir": "none", "text": "In two pilot sessions, the protocol worked as intended at very low cost.", "detail": "The paper argues the method improves on studies where a researcher secretly simulates the AI, because participants use a real AI model.", "evidence": [ "findings[0]" ] }, { "dir": "none", "text": "In the pilots, a few rounds an hour still produced a wide range of insights.", "detail": "Participants got through 5 to 6 rounds. Most time went on reflecting on each response and deciding what to ask next.", "evidence": [ "findings[1]" ] }, { "dir": "none", "text": "In the pilots, participants started with short, search-style requests.", "detail": "The researchers added guidance explaining that longer, conversational requests are possible.", "evidence": [ "findings[3]" ] }, { "dir": "none", "text": "In the pilots, the chatbot's verbose creative mode was no more likely to make things up.", "detail": "The longer answers of creative mode also gave participants more to reflect on, so the researchers chose it for the study.", "evidence": [ "findings[2]" ] } ] }, { "title": "Limits of the method", "sub": "what the paper notes", "items": [ { "dir": "none", "text": "The paper notes researcher help hides where participants' own prompting would fail.", "detail": "Participants are asked how they would have changed prompts, which partly offsets this.", "evidence": [ "limitations[0]" ] }, { "dir": "none", "text": "The paper notes its prompting strategies are subjective and hard to reproduce.", "detail": "The strategies came from trial and error and the researchers' intuitions.", "evidence": [ "limitations[1]" ] } ] } ] }, "keyTerm": "participatory prompting", "takeaway": "Participatory prompting can ground user research on generative AI in what working systems can actually do, with a researcher prompting a chatbot on participants' own problems. This can cheaply improve on simulated AI." } }