{
 "@context": "https://schema.org",
 "@type": "ScholarlyArticle",
 "@id": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.html#paper",
 "aboutThisFile": {
  "whatThisIs": "A research brief: a structured summary of one paper, prepared for seanrintel.com from the paper's full text.",
  "howItWasMade": "This brief was prepared with AI assistance and published by Sean Rintel. It is not the authors' published text.",
  "whatIsVerbatim": "Only 'quote' fields, figure captions and 'abstract' are the authors' own words. Research questions follow the paper's wording where it states them. Takeaway, summaries, answers, definitions and method are the site's plain-language paraphrase.",
  "howItIsChecked": "Before publication every quote is matched against the paper and every number in the brief is found in it. 'verified' is the date of the last check.",
  "howToUse": [
   "For a quick answer, use 'takeaway' and 'questions'. For evidence, use 'findings' (each has a verbatim quote, its location in the paper and a 'url' anchored to that finding on the paper's page).",
   "Locations such as '§5.2, p. 9' refer to the version at 'pageNumbersReferTo'.",
   "For anything not covered here, read the paper itself: 'fullTextSources' lists free versions first.",
   "If 'researchOverview' is present, the paper is part of one of Sean Rintel's research overviews: it gives the overview's thesis, the claims this paper is evidence for and its role in each, and the other papers in the overview.",
   "To find other papers by Sean Rintel, use the catalogue at https://seanrintel.com/publications/index.json."
  ],
  "authorship": "Sean Rintel's role on each paper: lead author when he is first author, sole author when he is the only author, and otherwise senior author. Senior author here means the senior researcher on the team, whatever his position in the author list.",
  "howToCite": "Cite the paper, not this file: use 'citation' or the ready-made 'bibtex' (DOI where given) and name every author. Attribute paraphrase to the paper's findings, and use quotation marks only for 'quote' text."
 },
 "name": "Semi-Automated Analysis of Collaborative Interaction: Are We There Yet?",
 "author": [
  {
   "@type": "Person",
   "name": "Thomas Neumayr"
  },
  {
   "@type": "Person",
   "name": "Mirjam Augstein"
  },
  {
   "@type": "Person",
   "name": "Johannes Schönböck"
  },
  {
   "@type": "Person",
   "@id": "https://seanrintel.com/#sean-rintel",
   "name": "Sean Rintel",
   "url": "https://seanrintel.com/",
   "sameAs": [
    "https://orcid.org/0000-0003-0840-0546",
    "https://scholar.google.com/citations?user=xc25GRgAAAAJ",
    "https://www.wikidata.org/wiki/Q102369654",
    "https://www.microsoft.com/en-us/research/people/serintel/",
    "https://www.linkedin.com/in/seanrintel"
   ]
  },
  {
   "@type": "Person",
   "name": "Helmut Leeb"
  },
  {
   "@type": "Person",
   "name": "Thomas Teichmeister"
  }
 ],
 "datePublished": "2022-12",
 "doi": "10.1145/3567724",
 "url": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.html",
 "citation": "Thomas Neumayr, Mirjam Augstein, Johannes Schönböck, Sean Rintel, Helmut Leeb, and Thomas Teichmeister. 2022. Semi-Automated Analysis of Collaborative Interaction: Are We There Yet?. ISS 2022. https://doi.org/10.1145/3567724",
 "bibtex": "@inproceedings{neumayr2022semi-automated,\nauthor = {Neumayr, Thomas and Augstein, Mirjam and Schönböck, Johannes and Rintel, Sean and Leeb, Helmut and Teichmeister, Thomas},\ntitle = {Semi-Automated Analysis of Collaborative Interaction: Are We There Yet?},\norganization = {ACM},\nbooktitle = {ISS 2022},\nyear = {2022},\nmonth = {December},\nurl = {https://doi.org/10.1145/3567724},\ndoi = {10.1145/3567724},\n}",
 "abstract": "In recent years, research on collaborative interaction has relied on manual coding of rich audio/video recordings. The fine-grained analysis of such material is extremely time-consuming and labor-intensive. This is not only difficult to scale, but, as a result, might also limit the quality and completeness of coding due to fatigue, inherent human biases, (accidental or intentional), and inter-rater inconsistencies. In this paper, we explore how recent advances in machine learning may reduce manual effort and loss of information while retaining the value of human intelligence in the coding process. We present ACACIA (AI Chain for Augmented Collaborative Interaction Analysis), an AI video data analysis application which combines a range of advances in machine perception of video material for the analysis of collaborative interaction. We evaluate ACACIA’s abilities, show how far we can already get, and which challenges remain. Our contribution lies in establishing a combined machine and human analysis pipeline that may be generalized to different collaborative settings and guide future research.",
 "fullTextSources": [
  {
   "kind": "freeAccess",
   "url": "https://doi.org/10.1145/3567724",
   "label": "DOI (Free Access)"
  },
  {
   "kind": "freeAccess",
   "url": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.pdf",
   "label": "Author's Copy"
  }
 ],
 "fullTextAvailable": true,
 "takeaway": "Machine-in-the-loop coding of collaboration video (AI takes a first pass and researchers correct it) is closest to ready for the spatial side of collaboration. AI reliably tracked who was present and where, which can show how close people work and who is in researcher-marked regions.",
 "whyItMatters": {
  "stateOfTheArt": {
   "summary": "Research on collaborative interaction relies on manual coding of audio and video, which takes days per hour of footage, does not scale, and suffers from fatigue, bias and inconsistency between coders.",
   "quote": "The effort is considered necessary to obtain valid empirical findings, but its cost is so great that it creates a scaling problem [15] which effectively prohibits studies of large numbers of groups and/or longitudinal studies.",
   "location": "§1, p. 2"
  },
  "contribution": {
   "summary": "Rather than training custom models, the authors chain readily available AI perception services into one research pipeline that researchers without a technical background can check and correct. The authors then evaluate frame by frame how far it gets.",
   "quote": "developed a framework that combines machine- and human-related efforts into a consistent research pipeline for collaborative contexts",
   "location": "§1, p. 3"
  },
  "whoCares": {
   "summary": "HCI and CSCW researchers who code video of collaboration by hand and want to analyse more groups or longer studies.",
   "quote": "we believe that our findings can provide a good perspective and guideline to other researchers struggling with the known issues related to purely manual coding",
   "location": "§6, p. 23"
  }
 },
 "researchQuestions": [
  "How far can we get by introducing currently available AI services into the research process for analysing collaborative interaction: are we there yet?"
 ],
 "concepts": [
  {
   "term": "ACACIA",
   "definition": "AI Chain for Augmented Collaborative Interaction Analysis: the authors' desktop tool that combines machine perception services to analyse study video.",
   "quote": "In ACACIA, machine perception takes a first pass at time-consuming steps in the coding process that are easy for machines to solve but hard for people to do reliably over a large data set",
   "location": "§1, p. 3"
  },
  {
   "term": "Machine-in-the-loop",
   "definition": "Machine perception assists human researchers, rather than humans checking a machine.",
   "quote": "a \"machine-in-the-loop\" inversion of the \"human-in-the-loop\" concept",
   "location": "§1, p. 3"
  },
  {
   "term": "Entities and relationships",
   "definition": "Active entities (people) and passive ones (objects, regions), and who interacts with what or whom over time.",
   "quote": "it is essential to find out who (i.e., which active entity) was interacting with what (i.e., which passive entity) or whom (i.e., which other active entity) at which point in time",
   "location": "§3.1, p. 7"
  },
  {
   "term": "Regions of interest",
   "definition": "Areas specific to a study, such as part of a table or a screen, which researchers annotate by hand because they cannot be detected automatically.",
   "quote": "Regions are objects or areas in frames which are of particular interest to the observers.",
   "location": "§3.3, p. 12"
  }
 ],
 "method": {
  "design": "System design and a small-scale evaluation comparing ACACIA's output with human coders' judgements.",
  "prototype": "ACACIA (AI Chain for Augmented Collaborative Interaction Analysis), a Windows desktop application for researchers. The researcher loads a study recording (an MP4 file, or an Azure Kinect stream that adds depth), registers each participant with a name and a profile image, and lets the system take a first pass. The system identifies people in every frame, detects objects, estimates where each person is looking, and tracks people across frames by binding faces to bodies. In an editor, the researcher adds, moves or deletes people and objects and draws regions of interest by hand. A viewer overlays names, objects and gaze paths on each frame and exports tables of who looked at or interacted with whom or what, frame by frame. It chains Microsoft's Platform for Situated Intelligence with Azure Cognitive Services (Face and Computer Vision) and Project Tokyo's gaze model. Its output was compared with human coders' judgements.",
  "prototypeFigure": 5,
  "setting": "Hybrid collaboration: two participants in one room linked to three in another via Microsoft Surface Hub 2S devices, solving logic puzzles, and one Azure Kinect camera per room.",
  "sample": "Five university staff who were not authors (two professors and three research associates).",
  "n": 5,
  "duration": "Around 60 minutes of collaborative interaction data (30 minutes per room). Detailed evaluation of a 3.5 minute segment (211 frames) and a 2-minute sample (120 frames).",
  "analysis": "Frame-by-frame judgement of person identification, object recognition and gaze targets against human coders' impressions, with ambiguous cases discussed by the research team."
 },
 "findings": [
  {
   "headline": "People are found when they're visible.",
   "summary": "In recordings of a five-person hybrid puzzle-solving session, the tool recognised and identified people very reliably whenever the camera had a clear view of them. It did so even when they faced away, by tracking which face belongs to which body.",
   "quote": "Overall, both the recognition and identification of persons worked remarkably well as long as the camera had clear sight of the persons.",
   "location": "§5.1, p. 21",
   "url": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.html#at-s5-1-p-21",
   "cite": "Thomas Neumayr, Mirjam Augstein, Johannes Schönböck, Sean Rintel, Helmut Leeb, and Thomas Teichmeister. 2022. Semi-Automated Analysis of Collaborative Interaction: Are We There Yet?. ISS 2022. https://doi.org/10.1145/3567724 (§5.1, p. 21)"
  },
  {
   "headline": "Hidden people are lost.",
   "summary": "The main failure was occlusion. A participant sitting behind another was recognised in fewer than half the frames of the evaluated segment.",
   "quote": "In our tests we mostly lost track of persons only when they were currently not visible",
   "location": "§4.4, p. 18",
   "url": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.html#at-s4-4-p-18",
   "cite": "Thomas Neumayr, Mirjam Augstein, Johannes Schönböck, Sean Rintel, Helmut Leeb, and Thomas Teichmeister. 2022. Semi-Automated Analysis of Collaborative Interaction: Are We There Yet?. ISS 2022. https://doi.org/10.1145/3567724 (§4.4, p. 18)"
  },
  {
   "headline": "Remote faces on screen count too.",
   "summary": "With a single camera in the room, pointed towards the large screen showing the remote participants, the tool identified most participants, in the room or on screen, whose faces were visible.",
   "quote": "We added all of these percentages and found out that 61.47% of all participants, whether remote or co-located, were recognized and identified correctly as long as their faces were visible.",
   "location": "§4.3, p. 16",
   "url": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.html#at-s4-3-p-16",
   "cite": "Thomas Neumayr, Mirjam Augstein, Johannes Schönböck, Sean Rintel, Helmut Leeb, and Thomas Teichmeister. 2022. Semi-Automated Analysis of Collaborative Interaction: Are We There Yet?. ISS 2022. https://doi.org/10.1145/3567724 (§4.3, p. 16)"
  },
  {
   "headline": "Objects needed a human hand.",
   "summary": "Automatic object detection was too broad and unstable to use, so researchers marked the regions that mattered by hand, which took only minutes.",
   "quote": "we invested a few minutes to add several regions of interest: a printed-out worksheet, the area of the Surface Hub showing the whiteboard application, and the area of the Surface Hub showing the remote participants",
   "location": "§4.4, p. 19",
   "url": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.html#at-s4-4-p-19",
   "cite": "Thomas Neumayr, Mirjam Augstein, Johannes Schönböck, Sean Rintel, Helmut Leeb, and Thomas Teichmeister. 2022. Semi-Automated Analysis of Collaborative Interaction: Are We There Yet?. ISS 2022. https://doi.org/10.1145/3567724 (§4.4, p. 19)"
  },
  {
   "headline": "Gaze is hit and miss.",
   "summary": "Estimating where each person was looking found the right target only part of the time, depending on how much the person faced the camera.",
   "quote": "gaze estimation also worked to some extent (between approx. 33.7% if face orientation was suboptimal and 60.2% for persons more often-but still not permanently-facing the camera)",
   "location": "§5.1, p. 21",
   "url": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.html#at-s5-1-p-21-2",
   "cite": "Thomas Neumayr, Mirjam Augstein, Johannes Schönböck, Sean Rintel, Helmut Leeb, and Thomas Teichmeister. 2022. Semi-Automated Analysis of Collaborative Interaction: Are We There Yet?. ISS 2022. https://doi.org/10.1145/3567724 (§5.1, p. 21)"
  },
  {
   "headline": "Half-way there.",
   "summary": "Automated analysis of collaboration video cannot yet replace what human coders do, and better camera placement is the main step still needed.",
   "quote": "judging from our systematic evaluation, we estimate that we are half-way there. Our evaluation showed that the biggest remaining issue is how to achieve optimal camera perspectives.",
   "location": "§6, p. 23",
   "url": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.html#at-s6-p-23",
   "cite": "Thomas Neumayr, Mirjam Augstein, Johannes Schönböck, Sean Rintel, Helmut Leeb, and Thomas Teichmeister. 2022. Semi-Automated Analysis of Collaborative Interaction: Are We There Yet?. ISS 2022. https://doi.org/10.1145/3567724 (§6, p. 23)"
  }
 ],
 "questions": [
  {
   "question": "Can AI automatically code video of people collaborating?",
   "answer": "Partly. An AI pipeline identified people reliably when the camera could see them. But object detection and gaze estimation were not yet reliable enough, and regions of interest had to be marked by hand. The authors estimate they are half-way there.",
   "source": "Thomas Neumayr, Mirjam Augstein, Johannes Schönböck, Sean Rintel, Helmut Leeb, and Thomas Teichmeister. 2022. Semi-Automated Analysis of Collaborative Interaction: Are We There Yet?. ISS 2022. https://doi.org/10.1145/3567724",
   "basedOn": [
    {
     "location": "§5.1, p. 21",
     "url": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.html#at-s5-1-p-21"
    },
    {
     "location": "§4.4, p. 19",
     "url": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.html#at-s4-4-p-19"
    },
    {
     "location": "§5.1, p. 21",
     "url": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.html#at-s5-1-p-21-2"
    },
    {
     "location": "§6, p. 23",
     "url": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.html#at-s6-p-23"
    }
   ]
  },
  {
   "question": "How accurate is AI gaze tracking from a single camera for research video?",
   "answer": "Not yet accurate enough to use directly. Gaze landed on the correct target in 60.2% of the frames in which one participant, who often faced the camera, was recognised, and 33.7% for one mostly facing away. The authors think depth information and front-facing angles help.",
   "source": "Thomas Neumayr, Mirjam Augstein, Johannes Schönböck, Sean Rintel, Helmut Leeb, and Thomas Teichmeister. 2022. Semi-Automated Analysis of Collaborative Interaction: Are We There Yet?. ISS 2022. https://doi.org/10.1145/3567724",
   "basedOn": [
    {
     "location": "§5.1, p. 21",
     "url": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.html#at-s5-1-p-21-2"
    }
   ]
  },
  {
   "question": "What is the biggest obstacle to automated analysis of group video?",
   "answer": "Camera perspective. People hidden behind others drop out of the analysis, and gaze estimation works only when faces are turned towards the camera. The authors judged this the biggest remaining issue.",
   "source": "Thomas Neumayr, Mirjam Augstein, Johannes Schönböck, Sean Rintel, Helmut Leeb, and Thomas Teichmeister. 2022. Semi-Automated Analysis of Collaborative Interaction: Are We There Yet?. ISS 2022. https://doi.org/10.1145/3567724",
   "basedOn": [
    {
     "location": "§4.4, p. 18",
     "url": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.html#at-s4-4-p-18"
    },
    {
     "location": "§6, p. 23",
     "url": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.html#at-s6-p-23"
    }
   ]
  },
  {
   "question": "Can AI identify remote meeting participants shown on a screen in recorded video?",
   "answer": "To some extent. Using one camera pointed towards the screen showing the remote participants, 61.47% of all participants, remote or in the room, were identified correctly as long as their faces were visible. The authors see this as a good starting point.",
   "source": "Thomas Neumayr, Mirjam Augstein, Johannes Schönböck, Sean Rintel, Helmut Leeb, and Thomas Teichmeister. 2022. Semi-Automated Analysis of Collaborative Interaction: Are We There Yet?. ISS 2022. https://doi.org/10.1145/3567724",
   "basedOn": [
    {
     "location": "§4.3, p. 16",
     "url": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.html#at-s4-3-p-16"
    }
   ]
  }
 ],
 "cites": [
  {
   "url": "https://seanrintel.com/publications/whats-going-on-here-the-pedagogy-of-a-data-analysis-session.html",
   "title": "What’s going on here? The pedagogy of a data analysis session",
   "location": "p. 26"
  }
 ],
 "quantitative": [
  {
   "measure": "Frames in which the occluded participant was recognised",
   "value": "93 of the 211 frames (44.08%)",
   "location": "§4.3, p. 16"
  },
  {
   "measure": "In-room and remote participants identified when faces visible",
   "value": "61.47%",
   "location": "§4.3, p. 16"
  },
  {
   "measure": "Table recognised in a 120-frame sample",
   "value": "81 of the frames (67.5%)",
   "location": "§4.3, p. 17"
  },
  {
   "measure": "Correct gaze target: participant often facing the camera; participant mostly facing away",
   "value": "56 of the 93 frames (60.2%); 71 frames (33.7%)",
   "location": "§4.4, p. 19"
  }
 ],
 "implications": [
  {
   "summary": "Do not replace human analysts. Interweave human and machine effort, because automated systems carry biases and rare cases are hard for data-driven AI.",
   "quote": "Thus, we argue for interweaving human as well as machine efforts in the research process related to analysis of collaborative interaction.",
   "location": "§6, p. 23"
  },
  {
   "summary": "Use several cameras per room, with synchronisation based on robust confidence ratings, and a depth camera with high-resolution, front-facing imagery for gaze.",
   "quote": "we think a combination of highresolution 2D imagery, depth information, and a front-facing camera angle is best for reliable gaze estimation",
   "location": "§5.1, p. 21"
  },
  {
   "summary": "Controlled lab studies are a better first use than field studies with crowds.",
   "quote": "controlled lab studies are a more favorable first candidate for real-world application of such a system as opposed to field studies in classrooms or museums",
   "location": "§6, p. 23"
  },
  {
   "summary": "Protect privacy by never using real names for profile images, informing participants and seeking consent, and limiting use to scientific studies."
  }
 ],
 "limitations": [
  {
   "summary": "The evaluation was small-scale, with five participants and judgements on short segments. Results depend heavily on the hardware and services used."
  },
  {
   "summary": "The tool cannot yet reliably detect fine-grained collaboration elements such as coupling styles.",
   "quote": "at the moment we cannot reliably detect and distinguish between fine-grained elements of a collaboration such as coupling styles",
   "location": "§6, p. 24"
  },
  {
   "summary": "Hardware calibration matters, because depth and colour images did not always cover the same area."
  },
  {
   "summary": "Pointing, lip and speech detection were not mature enough to evaluate."
  }
 ],
 "figures": [
  {
   "src": "https://seanrintel.com/publications/figures/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet/fig-4.webp",
   "caption": "Fig. 4. High level depiction of ACACIA's research pipeline. (Pointing currently is in an experimental state and was not evaluated, yet).",
   "alt": "Pipeline diagram: video streams or MP4 files go through preprocessing (people, objects, gaze, body tracking) into a database with an editor, then postprocessing (tracking, time analysis) and a viewer that exports statistics and images.",
   "location": "Fig. 4, p. 8",
   "credit": "Figure 4 from Thomas Neumayr, Mirjam Augstein, Johannes Schönböck, Sean Rintel, Helmut Leeb, and Thomas Teichmeister. 2022. Semi-Automated Analysis of Collaborative Interaction: Are We There Yet?. ISS 2022. https://doi.org/10.1145/3567724"
  },
  {
   "src": "https://seanrintel.com/publications/figures/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet/fig-7.webp",
   "caption": "Fig. 7. Overview of external services used in our approach. Please note that Lip and Speech detection have not been systematically evaluated, yet, and should be seen as a suggestion for future work.",
   "alt": "Diagram of services: image frames go to Microsoft Cognitive Services Face (faces and names), Computer Vision (objects) and Project Tokyo (gaze), Azure Kinect depth gives tracked bodies, and greyed-out lip and speech services are future work.",
   "location": "Fig. 7, p. 11",
   "credit": "Figure 7 from Thomas Neumayr, Mirjam Augstein, Johannes Schönböck, Sean Rintel, Helmut Leeb, and Thomas Teichmeister. 2022. Semi-Automated Analysis of Collaborative Interaction: Are We There Yet?. ISS 2022. https://doi.org/10.1145/3567724"
  },
  {
   "src": "https://seanrintel.com/publications/figures/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet/fig-5.webp",
   "caption": "Fig. 5. Left: main area of ACACIA's viewer. Right: call-out of frame control bar (top) and visualization options (bottom). (a) marks the visualization pane showing the selected visualizations on the current frame, (b) the frame control bar allows navigating the video frames, and (c) shows a list of visualization options to choose from. Please note that some options (e.g., Speaking or Workgroups) are experimental and have not been evaluated, yet.",
   "alt": "ACACIA viewer screenshot: a video frame of a meeting with detected people outlined, a frame control bar, and a panel of checkboxes for visualisation options, shown enlarged.",
   "location": "Figure 5, p. 10",
   "credit": "Figure 5 from Thomas Neumayr, Mirjam Augstein, Johannes Schönböck, Sean Rintel, Helmut Leeb, and Thomas Teichmeister. 2022. Semi-Automated Analysis of Collaborative Interaction: Are We There Yet?. ISS 2022. https://doi.org/10.1145/3567724"
  }
 ],
 "pageNumbersReferTo": "https://seanrintel.com/publications/semi-automated-analysis-of-collaborative-interaction-are-we-there-yet.pdf",
 "verified": "2026-10-25",
 "studyFigure": {
  "about": "The opening figure on this paper's page, made for the site with AI assistance from this verified brief. It is not the paper's own figure. Evidence entries index this file's quantitative, findings and limitations arrays.",
  "study": {
   "kind": "single",
   "summary": "System design and a small evaluation of ACACIA, an AI tool that takes a first pass at coding video of people collaborating. Five university staff solved logic puzzles across two linked rooms.",
   "steps": [
    {
     "label": "Five people, two rooms",
     "detail": "Two participants in one room and three in another solved logic puzzles over linked Surface Hub touchscreens, with one Azure Kinect depth camera per room."
    },
    {
     "label": "60 minutes of video",
     "detail": "The tool processed around 60 minutes of recording, 30 minutes from each room."
    },
    {
     "label": "Frame-by-frame checks",
     "detail": "Researchers compared the tool's output with human coders' judgements on a 3.5 minute segment and a 2-minute sample."
    },
    {
     "label": "Team discussion",
     "detail": "The research team discussed ambiguous cases of who was who, which object was which and where people looked."
    }
   ]
  },
  "tool": {
   "steps": [
    {
     "title": "Load and register",
     "text": "The researcher loads a study recording and registers each participant with a name and a profile image."
    },
    {
     "title": "AI first pass",
     "text": "The system identifies people in every frame, detects objects and estimates where each person is looking.",
     "fields": [
      "people",
      "objects",
      "gaze"
     ],
     "part": true
    },
    {
     "title": "Track across frames",
     "text": "The system follows each person from frame to frame by matching faces to bodies.",
     "part": true
    },
    {
     "title": "Researcher corrects",
     "text": "In an editor, the researcher adds, moves or deletes people and objects and draws regions of interest by hand.",
     "part": true
    },
    {
     "title": "View and export",
     "text": "A viewer overlays names, objects and gaze paths on each frame and exports tables of who looked at or interacted with whom or what."
    }
   ],
   "note": "The AI takes the first pass and researchers correct it, which the paper calls machine-in-the-loop."
  },
  "findings": {
   "legend": null,
   "stages": [
    {
     "title": "Finding people",
     "sub": "how well the tool recognised and identified participants",
     "items": [
      {
       "dir": "none",
       "text": "In these recordings, the tool identified people reliably whenever the camera saw them clearly.",
       "detail": "The tool identified people even when people faced away, by tracking which face belongs to which body.",
       "evidence": [
        "findings[0]"
       ]
      },
      {
       "dir": "none",
       "text": "One participant hidden behind another was recognised in 93 of 211 frames.",
       "detail": "That is 44.08% of the evaluated segment. Occlusion, meaning one person blocking the camera's view of another, was the main failure.",
       "evidence": [
        "findings[1]",
        "quantitative[0]"
       ]
      },
      {
       "dir": "none",
       "text": "With faces visible, 61.47% of participants in the room or on screen were identified correctly.",
       "detail": "Each room had a single camera, pointed towards the large screen showing the remote participants.",
       "evidence": [
        "findings[2]",
        "quantitative[1]"
       ]
      }
     ]
    },
    {
     "title": "Objects and gaze",
     "sub": "where the tool needed human help",
     "items": [
      {
       "dir": "none",
       "text": "In these recordings, automatic object detection was too broad and unstable to use.",
       "detail": "Researchers instead marked the regions that mattered, such as a worksheet and parts of the screen, which took only minutes. A table was recognised in 81 of 120 sampled frames.",
       "evidence": [
        "findings[3]",
        "quantitative[2]"
       ]
      },
      {
       "dir": "none",
       "text": "Gaze estimates found the right target in 60.2% of frames for a person often facing the camera.",
       "detail": "For a person mostly facing away, the figure was 33.7%. Gaze estimation means judging where each person was looking.",
       "evidence": [
        "findings[4]",
        "quantitative[3]"
       ]
      }
     ]
    },
    {
     "title": "Overall",
     "sub": "the authors' judgement of readiness",
     "items": [
      {
       "dir": "none",
       "text": "The authors estimate that automated analysis of collaboration video is half-way there.",
       "detail": "Automated coding cannot yet replace human coders, and better camera placement is the main step still needed. The evaluation was small, with five participants.",
       "evidence": [
        "findings[5]",
        "limitations[0]"
       ]
      }
     ]
    }
   ]
  },
  "keyTerm": "machine-in-the-loop",
  "takeaway": "In machine-in-the-loop coding of collaboration video, AI can take a useful first pass at who is present and where, while researchers correct and extend it. Better camera views may close much of the remaining gap."
 }
}