{
 "@context": "https://schema.org",
 "@type": "ScholarlyArticle",
 "@id": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.html#paper",
 "aboutThisFile": {
  "whatThisIs": "A research brief: a structured summary of one paper, prepared for seanrintel.com from the paper's full text.",
  "howItWasMade": "This brief was prepared with AI assistance and published by Sean Rintel. It is not the authors' published text.",
  "whatIsVerbatim": "Only 'quote' fields, figure captions and 'abstract' are the authors' own words. Research questions follow the paper's wording where it states them. Takeaway, summaries, answers, definitions and method are the site's plain-language paraphrase.",
  "howItIsChecked": "Before publication every quote is matched against the paper and every number in the brief is found in it. 'verified' is the date of the last check.",
  "howToUse": [
   "For a quick answer, use 'takeaway' and 'questions'. For evidence, use 'findings' (each has a verbatim quote, its location in the paper and a 'url' anchored to that finding on the paper's page).",
   "Locations such as '§5.2, p. 9' refer to the version at 'pageNumbersReferTo'.",
   "For anything not covered here, read the paper itself: 'fullTextSources' lists free versions first.",
   "If 'researchOverview' is present, the paper is part of one of Sean Rintel's research overviews: it gives the overview's thesis, the claims this paper is evidence for and its role in each, and the other papers in the overview.",
   "To find other papers by Sean Rintel, use the catalogue at https://seanrintel.com/publications/index.json."
  ],
  "authorship": "Sean Rintel's role on each paper: lead author when he is first author, sole author when he is the only author, and otherwise senior author. Senior author here means the senior researcher on the team, whatever his position in the author list.",
  "howToCite": "Cite the paper, not this file: use 'citation' or the ready-made 'bibtex' (DOI where given) and name every author. Attribute paraphrase to the paper's findings, and use quotation marks only for 'quote' text."
 },
 "name": "The Metacognitive Demands and Opportunities of Generative AI",
 "author": [
  {
   "@type": "Person",
   "name": "Lev Tankelevitch"
  },
  {
   "@type": "Person",
   "name": "Viktor Kewenig"
  },
  {
   "@type": "Person",
   "name": "Auste Simkute"
  },
  {
   "@type": "Person",
   "name": "Ava Elizabeth Scott"
  },
  {
   "@type": "Person",
   "name": "Advait Sarkar"
  },
  {
   "@type": "Person",
   "name": "Abigail Sellen"
  },
  {
   "@type": "Person",
   "@id": "https://seanrintel.com/#sean-rintel",
   "name": "Sean Rintel",
   "url": "https://seanrintel.com/",
   "sameAs": [
    "https://orcid.org/0000-0003-0840-0546",
    "https://scholar.google.com/citations?user=xc25GRgAAAAJ",
    "https://www.wikidata.org/wiki/Q102369654",
    "https://www.microsoft.com/en-us/research/people/serintel/",
    "https://www.linkedin.com/in/seanrintel"
   ]
  }
 ],
 "datePublished": "2024-05",
 "doi": "10.1145/3613904.3642902",
 "url": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.html",
 "citation": "Lev Tankelevitch, Viktor Kewenig, Auste Simkute, Ava Elizabeth Scott, Advait Sarkar, Abigail Sellen, and Sean Rintel. 2024. The Metacognitive Demands and Opportunities of Generative AI. CHI '24. https://doi.org/10.1145/3613904.3642902",
 "bibtex": "@inproceedings{tankelevitch2024the,\nauthor = {Tankelevitch, Lev and Kewenig, Viktor and Simkute, Auste and Scott, Ava Elizabeth and Sarkar, Advait and Sellen, Abigail and Rintel, Sean},\ntitle = {The Metacognitive Demands and Opportunities of Generative AI},\nbooktitle = {CHI '24},\nyear = {2024},\nmonth = {May},\npublisher = {ACM},\nurl = {https://doi.org/10.1145/3613904.3642902},\ndoi = {10.1145/3613904.3642902},\n}",
 "abstract": "Generative AI (GenAI) systems offer unprecedented opportunities for transforming professional and personal work, yet present challenges around prompting, evaluating and relying on outputs, and optimizing workflows. We argue that metacognition – the psychological ability to monitor and control one’s thoughts and behavior – offers a valuable lens to understand and design for these usability challenges. Drawing on research in psychology and cognitive science, and recent GenAI user studies, we illustrate how GenAI systems impose metacognitive demands on users, requiring a high degree of metacognitive monitoring and control. We propose these demands could be addressed by integrating metacognitive support strategies into GenAI systems, and by designing GenAI systems to reduce their metacognitive demand by targeting explainability and customizability. Metacognition offers a coherent framework for understanding the usability challenges posed by GenAI, enabling us to offer research and design directions to advance human-GenAI interaction.",
 "fullTextSources": [
  {
   "kind": "freeAccess",
   "url": "https://dx.doi.org/10.1145/3613904.3642902",
   "label": "DOI (Free Access)"
  },
  {
   "kind": "freeAccess",
   "url": "https://doi.org/10.48550/arXiv.2312.10893",
   "label": "DOI"
  },
  {
   "kind": "freeAccess",
   "url": "https://arxiv.org/abs/2312.10893",
   "label": "Preprint (arXiv - Free Access)"
  }
 ],
 "fullTextAvailable": true,
 "takeaway": "Metacognition (monitoring and controlling one's own thinking) offers a way to understand why generative AI is hard to use. Users must state goals, decompose tasks and calibrate confidence. Tools could support users by prompting planning and self-evaluation, and could demand less by explaining outputs and offering the right controls.",
 "researchOverview": {
  "name": "Better Thinking With AI",
  "page": "https://seanrintel.com/better-thinking-with-ai.html",
  "json": "https://seanrintel.com/overviews/better-thinking-with-ai.json",
  "markdown": "https://seanrintel.com/overviews/better-thinking-with-ai.md",
  "thesis": "Generative AI moves people's thinking into stating what they want and judging what they get. The easiest way to use it can help least, so good tools ask people to think first.",
  "claimsThisPaperFeeds": [
   {
    "claim": "Effective AI use requires saying what usually goes unsaid",
    "role": "core evidence",
    "url": "https://seanrintel.com/better-thinking-with-ai.html#a2"
   },
   {
    "claim": "People who trust AI more say they check its work less",
    "role": "supporting evidence",
    "url": "https://seanrintel.com/better-thinking-with-ai.html#a3"
   },
   {
    "claim": "Asking for goals before AI drafts can improve the work",
    "role": "supporting evidence",
    "url": "https://seanrintel.com/better-thinking-with-ai.html#a6"
   }
  ],
  "otherPapers": [
   {
    "study": "Review: productivity loss with AI",
    "title": "Ironies of Generative AI: Understanding and Mitigating Productivity Loss in Human-AI Interaction",
    "page": "https://seanrintel.com/publications/ironies-of-generative-ai-understanding-and-mitigating-productivity-loss-in-human-ai-interaction.html",
    "researchBrief": "https://seanrintel.com/publications/ironies-of-generative-ai-understanding-and-mitigating-productivity-loss-in-human-ai-interaction.json"
   },
   {
    "study": "Survey: critical thinking at work",
    "title": "The Impact of Generative AI on Critical Thinking: Self-Reported Reductions in Cognitive Effort and Confidence Effects From a Survey of Knowledge Workers",
    "page": "https://seanrintel.com/publications/the-impact-of-generative-ai-on-critical-thinking-self-reported-reductions-in-cognitive-effort-and-confidence-effects-from-a-survey-of-knowledge-workers.html",
    "researchBrief": "https://seanrintel.com/publications/the-impact-of-generative-ai-on-critical-thinking-self-reported-reductions-in-cognitive-effort-and-confidence-effects-from-a-survey-of-knowledge-workers.json"
   },
   {
    "study": "Position paper: what skill at work means",
    "title": "The Future of Skill: What Is It to Be Skilled at Work?",
    "page": "https://seanrintel.com/publications/the-future-of-skill-what-is-it-to-be-skilled-at-work.html",
    "researchBrief": "https://seanrintel.com/publications/the-future-of-skill-what-is-it-to-be-skilled-at-work.json"
   },
   {
    "study": "Prompting study: AI-assisted data analysis",
    "title": "“It’s like a rubber duck that talks back”: Understanding Generative AI-Assisted Data Analysis Workflows through a Participatory Prompting Study",
    "page": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.html",
    "researchBrief": "https://seanrintel.com/publications/its-like-a-rubber-duck-that-talks-back-understanding-generative-ai-assisted-data-analysis-workflows-through-a-participatory-prompting-study.json"
   },
   {
    "study": "Method paper: participatory prompting",
    "title": "Participatory prompting: a user-centric research method for eliciting AI assistance opportunities in knowledge workflows",
    "page": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.html",
    "researchBrief": "https://seanrintel.com/publications/participatory-prompting-a-user-centric-research-method-for-eliciting-ai-assistance-opportunities-in-knowledge-workflows.json"
   },
   {
    "study": "Commentary: role-based prompts",
    "title": "Productivity implications for generative AI role-based prompts as a networked hermeneutic (Commentary)",
    "page": "https://seanrintel.com/publications/productivity-implications-for-generative-ai-role-based-prompts-as-a-networked-hermeneutic.html",
    "researchBrief": "https://seanrintel.com/publications/productivity-implications-for-generative-ai-role-based-prompts-as-a-networked-hermeneutic.json"
   },
   {
    "study": "Randomised trial: how students learn with AI",
    "title": "Learning behavior accounts for background-related advantage in AI-assisted education",
    "page": "https://seanrintel.com/publications/learning-behavior-accounts-for-background-related-advantage-in-ai-assisted-education.html",
    "researchBrief": "https://seanrintel.com/publications/learning-behavior-accounts-for-background-related-advantage-in-ai-assisted-education.json"
   },
   {
    "study": "Randomised experiment: note-taking versus AI for reading",
    "title": "Effects of LLM Use and Note-Taking On Reading Comprehension and Memory: A Randomised Experiment in Secondary Schools",
    "page": "https://seanrintel.com/publications/effects-of-llm-use-and-note-taking-on-reading-comprehension-and-memory-a-randomised-experiment-in-secondary-schools.html",
    "researchBrief": "https://seanrintel.com/publications/effects-of-llm-use-and-note-taking-on-reading-comprehension-and-memory-a-randomised-experiment-in-secondary-schools.json"
   },
   {
    "study": "Interviews: AI in higher education",
    "title": "The New Calculator? Practices, Norms, and Implications of Generative AI in Higher Education",
    "page": "https://seanrintel.com/publications/the-new-calculator-practices-norms-and-implications-of-generative-ai-in-higher-education.html",
    "researchBrief": "https://seanrintel.com/publications/the-new-calculator-practices-norms-and-implications-of-generative-ai-in-higher-education.json"
   },
   {
    "study": "Randomised experiment: scaffolds for AI-assisted writing",
    "title": "Metacognitive Scaffolds Create Productive Friction for AI-Assisted Knowledge Work",
    "page": "https://seanrintel.com/publications/metacognitive-scaffolds-create-productive-friction-for-ai-assisted-knowledge-work.html",
    "researchBrief": "https://seanrintel.com/publications/metacognitive-scaffolds-create-productive-friction-for-ai-assisted-knowledge-work.json"
   },
   {
    "study": "Lab study: prompt refinement controls",
    "title": "Dynamic Prompt Middleware: Contextual Prompt Refinement Controls for Comprehension Tasks",
    "page": "https://seanrintel.com/publications/dynamic-prompt-middleware-contextual-prompt-refinement-controls-for-comprehension-tasks.html",
    "researchBrief": "https://seanrintel.com/publications/dynamic-prompt-middleware-contextual-prompt-refinement-controls-for-comprehension-tasks.json"
   },
   {
    "study": "Workshop proposal: AI as a tool for thought",
    "title": "Tools for Thought: Understanding, Protecting, and Augmenting Human Cognition with Generative AI — From Vision to Implementation",
    "page": "https://seanrintel.com/publications/tools-for-thought-understanding-protecting-and-augmenting-human-cognition-with-generative-ai-from-vision-to-implementation.html",
    "researchBrief": "https://seanrintel.com/publications/tools-for-thought-understanding-protecting-and-augmenting-human-cognition-with-generative-ai-from-vision-to-implementation.json"
   },
   {
    "study": "System design: AI agents for diverse ideas",
    "title": "YES AND: A Generative AI Multi-Agent Framework for Enhancing Diversity of Thought in Individual Ideation for Problem-Solving Through Confidence-Based Agent Turn-Taking",
    "page": "https://seanrintel.com/publications/yes-and-a-generative-ai-multi-agent-framework-for-enhancing-diversity-of-thought-in-individual-ideation-for-problem-solving-through-confidence-based-agent-turn-taking.html",
    "researchBrief": "https://seanrintel.com/publications/yes-and-a-generative-ai-multi-agent-framework-for-enhancing-diversity-of-thought-in-individual-ideation-for-problem-solving-through-confidence-based-agent-turn-taking.json"
   },
   {
    "study": "Interviews and experiment: how people and machines interpret",
    "title": "Meeting in the Middle: The Interpretation Gap Between People and Machines",
    "page": "https://seanrintel.com/publications/meeting-in-the-middle-the-interpretation-gap-between-people-and-machines.html",
    "researchBrief": "https://seanrintel.com/publications/meeting-in-the-middle-the-interpretation-gap-between-people-and-machines.json"
   }
  ]
 },
 "whyItMatters": {
  "stateOfTheArt": {
   "summary": "User studies have documented many usability challenges with generative AI, in prompting, evaluating outputs, and deciding how to use it in workflows.",
   "quote": "However, there is not yet a coherent understanding of the usability challenges of GenAI, much less one grounded in a theory of human cognition.",
   "location": "§1, p. 1"
  },
  "contribution": {
   "summary": "The authors bring research on metacognition from psychology and cognitive science to bear on generative AI user studies, identify the metacognitive demands these systems impose, and propose two directions for addressing them, with open research questions.",
   "quote": "The metacognitive demands of working with GenAI systems parallel those of a manager delegating tasks to a team.",
   "location": "§1, pp. 1–2"
  },
  "whoCares": {
   "summary": "Designers and researchers of human-centred generative AI systems and human-AI interaction, and metacognition researchers.",
   "quote": "Designing truly human-centered GenAI systems [34, 104] means grappling with these metacognitive demands.",
   "location": "§5, p. 18"
  }
 },
 "researchQuestions": [
  "How can metacognition (the ability to monitor and control one's own thoughts and behaviour) help us understand the usability challenges of generative AI in prompting, evaluating and relying on outputs, and deciding on an automation strategy?",
  "How can these metacognitive demands be addressed, both by improving users' metacognition and by designing systems that reduce the demands?"
 ],
 "concepts": [
  {
   "term": "Metacognition",
   "definition": "The authors' working definition, from psychology and cognitive science:",
   "quote": "the psychological ability to monitor and control one's own thought processes",
   "location": "§1, p. 1"
  },
  {
   "term": "Metacognitive demand",
   "definition": "Distinct from cognitive load (the total mental effort a task requires), though it contributes to it:",
   "quote": "the need for extensive metacognitive monitoring and control for a task",
   "location": "§3, p. 5"
  },
  {
   "term": "Well-adjusted confidence",
   "definition": "Confidence is one's self-assessment of one's own abilities. It is well-adjusted when it",
   "quote": "distinguishes objectively correct and incorrect performance, and accurately matches one's abilities",
   "location": "§2.2, p. 3"
  },
  {
   "term": "Metacognitive flexibility",
   "definition": "Shifting strategy when new information arrives, a strategy is not working, or the task changes. The paper's example:",
   "quote": "I recognize that my formal tone in my emails does not match the more conversational style of my new co-workers. I should therefore adjust my approach while still maintaining professionalism",
   "location": "§2.2, p. 3"
  },
  {
   "term": "Automation strategy",
   "definition": "Deciding whether, when and how to use generative AI in one's workflow, which the paper likens to a manager's choice about delegating:",
   "quote": "Moreover, they need to decide whether, when, and how to even delegate tasks in the first place.",
   "location": "§1, pp. 1–2"
  },
  {
   "term": "Fuzzy abstraction matching",
   "definition": "Sarkar and colleagues' term for a problem that arises because generative AI works at many levels of abstraction:",
   "quote": "it becomes difficult for users to discern a system's capabilities and to match one's intent and prompting accordingly",
   "location": "§3.1, p. 8"
  }
 ],
 "method": {
  "design": "Conceptual analysis: a simplified descriptive framework of metacognition applied to a simplified generative AI workflow, with illustrative hypothetical design examples.",
  "setting": "Generative AI use in work and everyday tasks, with evidence drawn mainly from programming, writing, design and data science.",
  "sample": "Published research on metacognition in psychology and cognitive science, recent generative AI user studies, and research on metacognitive interventions and human-AI interaction.",
  "analysis": "Synthesis of the literature into metacognitive demands at each stage of use (prompt formulation, prompt iteration, output evaluation, automation strategy), proposed responses, and open research questions summarised in tables."
 },
 "findings": [
  {
   "headline": "Prompting makes the implicit explicit.",
   "summary": "Drawing on user studies of generative AI, mostly in programming, writing, design and data science, the authors argue that prompting requires people to be aware of their own goals and to break tasks into parts. Goals that stay implicit when working manually must be spelled out for the AI.",
   "quote": "For example, when writing an email to a senior colleague, one might implicitly know to adopt a certain tone. Many GenAI systems require specification that the email is to a senior colleague and needs an appropriate tone.",
   "location": "§3.1, p. 6",
   "url": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.html#at-s3-1-p-6",
   "cite": "Lev Tankelevitch, Viktor Kewenig, Auste Simkute, Ava Elizabeth Scott, Advait Sarkar, Abigail Sellen, and Sean Rintel. 2024. The Metacognitive Demands and Opportunities of Generative AI. CHI '24. https://doi.org/10.1145/3613904.3642902 (§3.1, p. 6)"
  },
  {
   "headline": "Was it me, or the machine?",
   "summary": "When an output disappoints, users must decide whether to blame their prompt, the settings, the system or chance, and adjust their approach. Because generative AI gives different outputs to the same prompt, this calls for well-adjusted confidence in one's prompting ability and the flexibility to change strategy.",
   "quote": "Is my prompt specific and clear enough; are system parameters set appropriately; is system performance generally poor on this task; or is this an 'unlucky' probabilistic output?",
   "location": "§3.1, p. 7",
   "url": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.html#at-s3-1-p-7",
   "cite": "Lev Tankelevitch, Viktor Kewenig, Auste Simkute, Ava Elizabeth Scott, Advait Sarkar, Abigail Sellen, and Sean Rintel. 2024. The Metacognitive Demands and Opportunities of Generative AI. CHI '24. https://doi.org/10.1145/3613904.3642902 (§3.1, p. 7)"
  },
  {
   "headline": "Judging output needs calibrated confidence.",
   "summary": "Judging AI output requires confidence in one's own judgement that matches one's actual ability. Research on metacognition suggests that fast, fluent output may inflate that confidence, and that the effort of checking long outputs may lower the bar people set before accepting them.",
   "quote": "when problems took longer to solve, participants appeared to compromise on their confidence criterion, and were willing to provide solutions with less confidence",
   "location": "§3.2, p. 9",
   "url": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.html#at-s3-2-p-9",
   "cite": "Lev Tankelevitch, Viktor Kewenig, Auste Simkute, Ava Elizabeth Scott, Advait Sarkar, Abigail Sellen, and Sean Rintel. 2024. The Metacognitive Demands and Opportunities of Generative AI. CHI '24. https://doi.org/10.1145/3613904.3642902 (§3.2, p. 9)"
  },
  {
   "headline": "Users must decide when to use AI.",
   "summary": "Because generative AI can be applied to so many tasks, deciding whether, when and how to use it falls to the user. That calls for awareness of how it affects one's work, confidence about doing tasks with and without it, and flexibility in changing workflows.",
   "quote": "I turned off autosuggest and that made a huge difference. Now I'll use it when I know I'm doing something repetitive that it'll get easily",
   "location": "§3.3, p. 11",
   "url": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.html#at-s3-3-p-11",
   "cite": "Lev Tankelevitch, Viktor Kewenig, Auste Simkute, Ava Elizabeth Scott, Advait Sarkar, Abigail Sellen, and Sean Rintel. 2024. The Metacognitive Demands and Opportunities of Generative AI. CHI '24. https://doi.org/10.1145/3613904.3642902 (§3.3, p. 11)"
  },
  {
   "headline": "Tools could build better habits.",
   "summary": "Systems could improve users' metacognition by building in support for planning, self-evaluation and self-management, adapted to each user. For example, in a study by Wu and colleagues, breaking a task into a chain of smaller prompts helped people think through the task.",
   "quote": "Chaining also increased users' self-awareness of their goals: the ability to decompose tasks led some participants to create more generalizable outputs better suited to their broader goals",
   "location": "§4.1, p. 13",
   "url": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.html#at-s4-1-p-13",
   "cite": "Lev Tankelevitch, Viktor Kewenig, Auste Simkute, Ava Elizabeth Scott, Advait Sarkar, Abigail Sellen, and Sean Rintel. 2024. The Metacognitive Demands and Opportunities of Generative AI. CHI '24. https://doi.org/10.1145/3613904.3642902 (§4.1, p. 13)"
  },
  {
   "headline": "Tools could carry some of the load.",
   "summary": "Systems could also lower the metacognitive demands themselves, by explaining outputs and uncertainty and by offering the right amount of customisation, though such support can add cognitive load of its own.",
   "quote": "by providing contextual and performance information alongside the system inputs and outputs, explainability should help partly offload metacognitive processing from users' minds and onto the system interface",
   "location": "§4.2, p. 16",
   "url": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.html#at-s4-2-p-16",
   "cite": "Lev Tankelevitch, Viktor Kewenig, Auste Simkute, Ava Elizabeth Scott, Advait Sarkar, Abigail Sellen, and Sean Rintel. 2024. The Metacognitive Demands and Opportunities of Generative AI. CHI '24. https://doi.org/10.1145/3613904.3642902 (§4.2, p. 16)"
  }
 ],
 "questions": [
  {
   "question": "Why is it so hard to write good prompts for AI?",
   "answer": "Prompting asks you to know exactly what you want and break the task into parts. You then judge whether a poor result came from your prompt, the settings, the system or chance, and adapt. These are metacognitive demands (monitoring and controlling your own thinking).",
   "source": "Lev Tankelevitch, Viktor Kewenig, Auste Simkute, Ava Elizabeth Scott, Advait Sarkar, Abigail Sellen, and Sean Rintel. 2024. The Metacognitive Demands and Opportunities of Generative AI. CHI '24. https://doi.org/10.1145/3613904.3642902",
   "basedOn": [
    {
     "location": "§3.1, p. 6",
     "url": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.html#at-s3-1-p-6"
    },
    {
     "location": "§3.1, p. 7",
     "url": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.html#at-s3-1-p-7"
    }
   ]
  },
  {
   "question": "Can fast, fluent AI answers make me overconfident?",
   "answer": "The authors argue they may. Quick, extensive output could act as a cue that raises confidence in the output and in your ability to judge it. That can reduce the effort you put into checking. Research measuring this with generative AI is still needed.",
   "source": "Lev Tankelevitch, Viktor Kewenig, Auste Simkute, Ava Elizabeth Scott, Advait Sarkar, Abigail Sellen, and Sean Rintel. 2024. The Metacognitive Demands and Opportunities of Generative AI. CHI '24. https://doi.org/10.1145/3613904.3642902",
   "basedOn": [
    {
     "location": "§3.2, p. 9",
     "url": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.html#at-s3-2-p-9"
    }
   ]
  },
  {
   "question": "Can AI tools help people think better about their own work?",
   "answer": "The authors argue they can. In a study by Wu and colleagues, breaking a task into a chain of smaller AI steps made some people more aware of their goals. Systems could build in similar nudges to plan and reflect, acting like a coach at key moments.",
   "source": "Lev Tankelevitch, Viktor Kewenig, Auste Simkute, Ava Elizabeth Scott, Advait Sarkar, Abigail Sellen, and Sean Rintel. 2024. The Metacognitive Demands and Opportunities of Generative AI. CHI '24. https://doi.org/10.1145/3613904.3642902",
   "basedOn": [
    {
     "location": "§4.1, p. 13",
     "url": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.html#at-s4-1-p-13"
    }
   ]
  },
  {
   "question": "Who decides when to use generative AI at work?",
   "answer": "Largely the user. Because generative AI applies to so many tasks, users must judge whether it helps their workflow and how confident they are doing a task themselves versus with AI. They then adapt their workflow accordingly.",
   "source": "Lev Tankelevitch, Viktor Kewenig, Auste Simkute, Ava Elizabeth Scott, Advait Sarkar, Abigail Sellen, and Sean Rintel. 2024. The Metacognitive Demands and Opportunities of Generative AI. CHI '24. https://doi.org/10.1145/3613904.3642902",
   "basedOn": [
    {
     "location": "§3.3, p. 11",
     "url": "https://seanrintel.com/publications/the-metacognitive-demands-and-opportunities-of-generative-ai.html#at-s3-3-p-11"
    }
   ]
  }
 ],
 "cites": [
  {
   "url": "https://seanrintel.com/publications/ironies-of-generative-ai-understanding-and-mitigating-productivity-loss-in-human-ai-interaction.html",
   "title": "Ironies of Generative AI: Understanding and Mitigating Productivity Loss in Human-AI Interaction",
   "location": "p. 23"
  }
 ],
 "quantitative": [],
 "implications": [
  {
   "summary": "Some effort introduced by metacognitive support and explanations may be worth it, questioning the assumption that interfaces should always be effortless.",
   "quote": "We propose that some potential effort introduced by metacognitive support strategies and explanations may be justified",
   "location": "§4.3, p. 18"
  },
  {
   "summary": "Metacognitive prompts could be adapted and gradually faded out as users internalise the strategies, reducing cognitive load over time."
  },
  {
   "summary": "The balance between too much and too little customisability needs exploring, ideally combined with metacognitive support."
  },
  {
   "summary": "Interaction with generative AI is also a paradigm for advancing basic understanding of metacognition.",
   "quote": "interaction with GenAI offers a powerful paradigm for advancing our foundational understanding of metacognition",
   "location": "§5, p. 18"
  }
 ],
 "limitations": [
  {
   "summary": "Evidence on how generative AI changes workflows comes mostly from short studies of programming.",
   "quote": "The above reports are limited to short study contexts, focusing only on programming.",
   "location": "§3.3, p. 10"
  },
  {
   "summary": "Research on self-confidence and reliance has mainly used discriminative AI models, not generative AI.",
   "quote": "Recent work has investigated the role of self-confidence in evaluating and relying on AI output, although with discriminative models, rather than GenAI.",
   "location": "§3.2, p. 8"
  },
  {
   "summary": "Support for metacognition may itself add cognitive load. The net effect is hypothesised, not shown.",
   "quote": "There is a risk that strategies to address metacognitive demands may also increase cognitive load",
   "location": "§4.3, p. 17"
  }
 ],
 "figures": [
  {
   "src": "https://seanrintel.com/publications/figures/the-metacognitive-demands-and-opportunities-of-generative-ai/fig-1.webp",
   "caption": "Figure 1: A simplified descriptive framework for metacognition. Metacognitive knowledge is the explicit understanding of one's abilities, strategies, and beliefs. Metacognitive experiences include things that people can directly experience, such as a feeling of familiarity or other implicit cues that provide information about cognitive processes. Metacognitive knowledge and experiences are interrelated in that experiences can become encoded as knowledge, and knowledge can be retrieved during experiences (§2.3). Both of these can influence (and be influenced by) metacognitive monitoring, which includes self-awareness, and confidence and its adjustment. Metacognitive monitoring, in turn, influences (and is influenced by) metacognitive control processes, such as metacognitive flexibility and task decomposition. Metacognitive control acts upon the (object-level) cognitive processes involved in a task. Arrows indicate directions of influence (§2.3).",
   "alt": "Diagram: metacognitive knowledge and experiences (information) exchange influence with monitoring (self-awareness, confidence), which exchanges influence with control (task decomposition, flexibility), which acts on task-level cognition.",
   "location": "Fig. 1, p. 4",
   "credit": "Figure 1 from Lev Tankelevitch, Viktor Kewenig, Auste Simkute, Ava Elizabeth Scott, Advait Sarkar, Abigail Sellen, and Sean Rintel. 2024. The Metacognitive Demands and Opportunities of Generative AI. CHI '24. https://doi.org/10.1145/3613904.3642902"
  },
  {
   "src": "https://seanrintel.com/publications/figures/the-metacognitive-demands-and-opportunities-of-generative-ai/fig-2.webp",
   "caption": "Figure 2: Metacognitive demands posed by generative AI at each point in a simplified user workflow. Often embedded within a workflow with manual tasks, users may first need to formulate a prompt, requiring metacognitive abilities including selfawareness of task goals and task decomposition (a). Systems that provide automated suggestions such as GitHub Copilot alleviate some of the demands associated with prompting. Depending on the output, iterating on the prompt may be necessary, which requires well-adjusted confidence in one's prompting ability and metacognitive flexiblity to adapt prompting strategies as necessary (b). Likewise, evaluating the output requires well-adjusted confidence in one's ability to judge its validity (c). Beyond the local interaction with a GenAI system, there is an overarching demand connected to understanding whether, when, and how to apply GenAI to one's workflows-one's 'automation strategy'. This requires self-awareness of how GenAI applies to and affects one's workflows, and well-adjusted confidence in the ability to complete tasks manually and with GenAI (d). Finally, it also requires metacognitive flexbility to adapt one's workflows as necessary (e).",
   "alt": "Workflow: manual work, prompt formulation (self-awareness, task decomposition), prompt iteration (confidence, flexibility), evaluation (confidence), all under an automation strategy needing self-awareness, confidence, flexibility.",
   "location": "Fig. 2, p. 7",
   "credit": "Figure 2 from Lev Tankelevitch, Viktor Kewenig, Auste Simkute, Ava Elizabeth Scott, Advait Sarkar, Abigail Sellen, and Sean Rintel. 2024. The Metacognitive Demands and Opportunities of Generative AI. CHI '24. https://doi.org/10.1145/3613904.3642902"
  }
 ],
 "pageNumbersReferTo": "https://arxiv.org/pdf/2312.10893",
 "verified": "2026-10-25",
 "studyFigure": {
  "about": "The opening figure on this paper's page, made for the site with AI assistance from this verified brief. It is not the paper's own figure. Evidence entries index this file's quantitative, findings and limitations arrays.",
  "study": {
   "kind": "single",
   "heading": "The paper",
   "summary": "Conceptual paper applying research on metacognition, the ability to monitor and control one's own thinking, to user studies of generative AI.",
   "steps": [
    {
     "label": "Metacognition research",
     "detail": "Published research from psychology and cognitive science on how people monitor and control their own thinking."
    },
    {
     "label": "Generative AI user studies",
     "detail": "Recent studies of generative AI use, mostly in programming, writing, design and data science."
    },
    {
     "label": "Stages of use",
     "detail": "A simple framework of metacognition applied to prompting, refining prompts, judging output and deciding when to automate."
    },
    {
     "label": "Responses and questions",
     "detail": "Two proposed directions for design, with hypothetical examples and open research questions."
    }
   ]
  },
  "findings": {
   "heading": "What the paper argues",
   "legend": null,
   "stages": [
    {
     "title": "Demands on users",
     "sub": "what generative AI asks of people's thinking",
     "items": [
      {
       "dir": "none",
       "text": "The paper argues prompting requires people to spell out goals and break tasks into parts.",
       "detail": "Goals that stay implicit when working by hand must be stated for the AI, such as the right tone for an email to a senior colleague.",
       "evidence": [
        "findings[0]"
       ]
      },
      {
       "dir": "none",
       "text": "The paper argues users must work out why a disappointing output happened, then adjust.",
       "detail": "The cause could be the prompt, the settings, the system or chance. This calls for well-adjusted confidence and flexibility to change strategy.",
       "evidence": [
        "findings[1]"
       ]
      },
      {
       "dir": "none",
       "text": "The paper argues judging AI output needs confidence that matches one's actual ability.",
       "detail": "Research on metacognition suggests fluent output may inflate confidence, and the effort of checking long outputs may lower the bar for accepting them.",
       "evidence": [
        "findings[2]"
       ]
      },
      {
       "dir": "none",
       "text": "The paper argues that deciding whether, when and how to use AI falls to the user.",
       "detail": "This decision calls for awareness of how AI affects one's work and flexibility in changing workflows.",
       "evidence": [
        "findings[3]"
       ]
      }
     ]
    },
    {
     "title": "Directions for design",
     "sub": "two ways tools could respond, and the limits",
     "items": [
      {
       "dir": "none",
       "text": "The paper proposes tools could improve metacognition by supporting planning and self-evaluation.",
       "detail": "In a study by Wu and colleagues that the paper cites, breaking a task into a chain of smaller prompts helped people think it through.",
       "evidence": [
        "findings[4]"
       ]
      },
      {
       "dir": "none",
       "text": "The paper proposes tools could lower demands by explaining outputs and offering the right controls.",
       "detail": "Such support may add cognitive load of its own. The paper presents the net effect as a hypothesis, not a result.",
       "evidence": [
        "findings[5]",
        "limitations[2]"
       ]
      },
      {
       "dir": "none",
       "text": "The paper notes evidence on changed workflows comes mostly from short programming studies.",
       "detail": "Research on confidence and reliance has also mainly used other kinds of AI, not generative AI.",
       "evidence": [
        "limitations[0]",
        "limitations[1]"
       ]
      }
     ]
    }
   ]
  },
  "keyTerm": "metacognition",
  "takeaway": "Generative AI can be hard to use because it demands metacognition, the monitoring and control of one's thinking. Tools may help by prompting planning and self-checks and explaining outputs."
 }
}