<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "journalpublishing.dtd"><article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" dtd-version="2.0" xml:lang="en" article-type="research-article"><front><journal-meta><journal-id journal-id-type="nlm-ta">JMIR Med Inform</journal-id><journal-id journal-id-type="publisher-id">medinform</journal-id><journal-id journal-id-type="index">7</journal-id><journal-title>JMIR Medical Informatics</journal-title><abbrev-journal-title>JMIR Med Inform</abbrev-journal-title><issn pub-type="epub">2291-9694</issn><publisher><publisher-name>JMIR Publications</publisher-name><publisher-loc>Toronto, Canada</publisher-loc></publisher></journal-meta><article-meta><article-id pub-id-type="publisher-id">v13i1e67845</article-id><article-id pub-id-type="doi">10.2196/67845</article-id><article-categories><subj-group subj-group-type="heading"><subject>Original Paper</subject></subj-group></article-categories><title-group><article-title>Simulation of Clinical Visits as a Novel Approach to Evaluate Digital Health in Multiple Sclerosis: Simulation Study</article-title></title-group><contrib-group><contrib contrib-type="author"><name name-style="western"><surname>Bove</surname><given-names>Riley</given-names></name><degrees>MD</degrees><xref ref-type="aff" rid="aff1">1</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Capezzuto</surname><given-names>Luca</given-names></name><degrees>PharmD</degrees><xref ref-type="aff" rid="aff2">2</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>West</surname><given-names>Imogen</given-names></name><degrees>MMath</degrees><xref ref-type="aff" rid="aff3">3</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Dryden</surname><given-names>Simon</given-names></name><degrees>PhD</degrees><xref ref-type="aff" rid="aff4">4</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Ghafur</surname><given-names>Saira</given-names></name><degrees>MD</degrees><xref ref-type="aff" rid="aff4">4</xref></contrib><contrib contrib-type="author" corresp="yes"><name name-style="western"><surname>Halligan</surname><given-names>Jack</given-names></name><degrees>MBA, MD</degrees><xref ref-type="aff" rid="aff4">4</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Hubeaux</surname><given-names>Stanislas</given-names></name><degrees>MSc</degrees><xref ref-type="aff" rid="aff2">2</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Kazlauskaite</surname><given-names>Agne</given-names></name><degrees>PhD</degrees><xref ref-type="aff" rid="aff2">2</xref></contrib></contrib-group><aff id="aff1"><institution>Department of Neurology, UCSF Weill Institute for Neurosciences, University of California, San Francisco</institution><addr-line>San Francisco</addr-line><addr-line>CA</addr-line><country>United States</country></aff><aff id="aff2"><institution>F. Hoffmann-La Roche Ltd</institution><addr-line>Basel</addr-line><country>Switzerland</country></aff><aff id="aff3"><institution>Roche Products Limited</institution><addr-line>London</addr-line><country>United Kingdom</country></aff><aff id="aff4"><institution>Prova Health</institution><addr-line>Bermondsey Street</addr-line><addr-line>London</addr-line><country>United Kingdom</country></aff><contrib-group><contrib contrib-type="editor"><name name-style="western"><surname>Chen</surname><given-names>Qingyu</given-names></name></contrib></contrib-group><contrib-group><contrib contrib-type="reviewer"><name name-style="western"><surname>Schependom</surname><given-names>Jeroen Van</given-names></name></contrib><contrib contrib-type="reviewer"><name name-style="western"><surname>Adegoke</surname><given-names>Kola</given-names></name></contrib></contrib-group><author-notes><corresp>Correspondence to Jack Halligan, MBA, MD, Prova Health, Bermondsey Street, London, SE1 3XF, United Kingdom, +44 7742161980; <email>jack.halligan@provahealth.com</email></corresp></author-notes><pub-date pub-type="collection"><year>2025</year></pub-date><pub-date pub-type="epub"><day>11</day><month>12</month><year>2025</year></pub-date><volume>13</volume><elocation-id>e67845</elocation-id><history><date date-type="received"><day>22</day><month>11</month><year>2024</year></date><date date-type="rev-recd"><day>05</day><month>08</month><year>2025</year></date><date date-type="accepted"><day>27</day><month>10</month><year>2025</year></date></history><copyright-statement>&#x00A9; Riley Bove, Luca Capezzuto, Imogen West, Simon Dryden, Saira Ghafur, Jack Halligan, Stanislas Hubeaux, Agne Kazlauskaite. Originally published in JMIR Medical Informatics (<ext-link ext-link-type="uri" xlink:href="https://medinform.jmir.org">https://medinform.jmir.org</ext-link>), 11.12.2025. </copyright-statement><copyright-year>2025</copyright-year><license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/"><p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (<ext-link ext-link-type="uri" xlink:href="https://creativecommons.org/licenses/by/4.0/">https://creativecommons.org/licenses/by/4.0/</ext-link>), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Medical Informatics, is properly cited. The complete bibliographic information, a link to the original publication on <ext-link ext-link-type="uri" xlink:href="https://medinform.jmir.org/">https://medinform.jmir.org/</ext-link>, as well as this copyright and license information must be included.</p></license><self-uri xlink:type="simple" xlink:href="https://medinform.jmir.org/2025/1/e67845"/><abstract><sec><title>Background</title><p>Efforts are being made to integrate digital health technologies into clinical care for multiple sclerosis (MS) to improve patient monitoring. Efficiently probing how they might impact clinical care could streamline digital tool development. The Floodlight digital tool, comprising 5 smartphone sensor&#x2013;based tests, was used to generate health-related data on patient function and symptoms in a clinical simulation.</p></sec><sec><title>Objective</title><p>The study had 3 objectives: (1) assess the utility of simulated clinical encounters as a research methodology for exploring the introduction of digital health technologies into clinical practice in MS, (2) confirm the fidelity of the simulated environment and patient cases developed and understand what metrics (eg, workflow, comprehensive evaluation) could be generated, and (3) generate insights into the utility of digitally collected data, including usability, clinical decision contribution, and impact on workflows, in clinical practice.</p></sec><sec sec-type="methods"><title>Methods</title><p>A total of 2 patient cases consisting of clinical, radiological, and digital health data were developed with clinician input. US-based neurologists prepared for and conducted 2 simulated teleconsultations each, with an actor briefed on case profiles. Floodlight data were available, via the Floodlight MS&#x2122; Health Care Professional Portal, for 1 of the 2 consultations. Participant neurologists completed interviews and surveys assessing the fidelity of the cases presented, user experience and workflow metrics, patient concerns identified, care decisions made, and confidence in making decisions.</p></sec><sec sec-type="results"><title>Results</title><p>All 10 neurologists indicated that the simulations were high-fidelity representations of real consultations. Using the Floodlight technology for the first time, median time taken to prepare for and conduct the consultation was ~1.7&#x2010;2 minutes longer, with slightly greater mental effort reported by participants, compared with not using the tool. The Floodlight MS Health Care Professional Portal scored an &#x201C;above average&#x201D; 79 on the System Usability Scale and an &#x201C;acceptable&#x201D; Net Promoter Score of 10. In total, 6 of the 10 neurologists &#x201C;strongly agreed&#x201D; that it was easier and quicker to identify patient concerns when they had access to the patient-generated Floodlight data to prepare for their encounters than when they did not. Overall, more care and management decisions were taken when the digital tool was used (37 vs 29). Of those 37 decisions, Floodlight data were reported as a trigger for 20 decisions, always in combination with other elements including patient history (20/20) and clinical exam findings (9/20).</p></sec><sec sec-type="conclusions"><title>Conclusions</title><p>These findings advance our understanding of clinical simulation as a method for evaluating digital tools and other innovative technologies for MS care. High-fidelity patient cases could be provided for the mock teleconsultations, and the simulated clinical environment was useful for evaluating usability and utility of a new digital tool&#x2014;yielding preliminary evidence on how digital data could be accessed and utilized by neurologists to support routine MS care.</p></sec></abstract><kwd-group><kwd>clinical simulation</kwd><kwd>digital health</kwd><kwd>digital health technology</kwd><kwd>multiple sclerosis</kwd><kwd>smartphone</kwd><kwd>teleconsultations</kwd><kwd>patient remote assessment</kwd><kwd>impact on clinical workflows</kwd><kwd>clinical utility</kwd></kwd-group></article-meta></front><body><sec id="s1" sec-type="intro"><title>Introduction</title><sec id="s1-1"><title>Background</title><p>Multiple sclerosis (MS) is a central nervous system disorder marked by significant heterogeneity in its clinical presentation and characterized by disease progression from onset [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref2">2</xref>]. MS, often commencing with subtle signs and advancing to substantial disability, manifests through a wide spectrum of symptoms, ranging from sensory disturbances and motor deficits to cognitive impairments and emotional challenges, profoundly impacting patients, their families, and society [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref4">4</xref>].</p><p>To date, monitoring in MS has been reliant upon brief, infrequent clinical assessments (typically occurring once or twice annually, or at the time of a relapse), without sufficiently sensitive and objective (ie, free from rater-dependent variability) assessments of progression available to health care professionals (HCPs) [<xref ref-type="bibr" rid="ref5">5</xref>]. Moreover, due to time constraints, standard assessments of disability in MS, such as the Expanded Disability Status Scale (EDSS) [<xref ref-type="bibr" rid="ref6">6</xref>], which requires ~30 minutes to conduct [<xref ref-type="bibr" rid="ref7">7</xref>], are not always performed in clinical appointments. Remote consultations (teleconsultations) are a recognized alternative to clinic visits [<xref ref-type="bibr" rid="ref8">8</xref>] and have been adopted by the MS community, particularly during the COVID-19 pandemic [<xref ref-type="bibr" rid="ref9">9</xref>,<xref ref-type="bibr" rid="ref10">10</xref>].</p><p>Digital health technologies offer greater access to care in MS and opportunities to improve frequent monitoring [<xref ref-type="bibr" rid="ref11">11</xref>], enabling the capture of ongoing subtle or subclinical progression of MS [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref12">12</xref>-<xref ref-type="bibr" rid="ref14">14</xref>] in specific domains such as cognition (which can often be missed). These technologies can be remotely administered and are patient-centered and operated, allowing longitudinal assessment of multiple functional domains in a naturalistic environment [<xref ref-type="bibr" rid="ref15">15</xref>,<xref ref-type="bibr" rid="ref16">16</xref>]. Furthermore, they offer the opportunity to improve the involvement of patients in their disease or treatment course [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref18">18</xref>] and to enhance conversations between HCPs and people with MS [<xref ref-type="bibr" rid="ref19">19</xref>]. By providing objective data and bridging the gap of information in between clinical visits, sensor-based digital assessments may ultimately allow HCPs to make more informed decisions, evaluate treatment effects, and act earlier on indications of disease progression [<xref ref-type="bibr" rid="ref11">11</xref>] or symptomatic exacerbation prior to scheduled clinical visits.</p><p>Despite the potential benefits, evaluating, testing, and optimizing the integration of new and innovative solutions into health care settings can itself be a time- and resource-consuming process. Hence, there is growing demand for agile methods to rapidly learn how digital solutions can play a role in clinical practice [<xref ref-type="bibr" rid="ref20">20</xref>]. Clinical simulation, a low-cost and scalable methodology whereby data are collected from users as they perform tasks in realistic clinical scenarios, can rapidly generate new insights. For example, they allow the exploration of the possible impact of novel methodologies without the risks inherent in deploying them in real clinical settings (eg, of misdiagnoses [<xref ref-type="bibr" rid="ref21">21</xref>,<xref ref-type="bibr" rid="ref22">22</xref>]) or of the expansion of technology to new user groups or in new clinical contexts [<xref ref-type="bibr" rid="ref22">22</xref>]. The preliminary insights generated from simulation models can support the refinement of technologies before rollout or scaling up [<xref ref-type="bibr" rid="ref23">23</xref>]. To confirm that the insights gained from these clinical simulations are meaningful, they must be of high fidelity and account for a number of variables (including behavioral ones) and the differences inherent among real-world practices [<xref ref-type="bibr" rid="ref21">21</xref>,<xref ref-type="bibr" rid="ref22">22</xref>].</p></sec><sec id="s1-2"><title>Aims and Objectives</title><p>This feasibility study was designed to evaluate clinical simulation as a method for assessing the preparation and conduct of teleconsultations. This included evaluating the fidelity of the simulated environment (compared with real practice) and the patient cases developed. An additional aim was to assess the feasibility of using data derived from smartphone-based remote assessment technology in combination with MS teleconsultations, including their usability and impacts on clinical workflows (eg, preparation time and time to identifying primary patient concerns), care decisions, and disease management. The Floodlight technology [<xref ref-type="bibr" rid="ref24">24</xref>] was utilized as the digital tool for evaluation. Accordingly, the aim of this study was to evaluate the potential impact of integrating Floodlight MS into clinical practice, generating insights that may also be relevant to similar digital health tools. As it is expected that any new test or technology will initially impose a higher mental load on clinicians, the study also aimed to quantify the extent of this increase.</p><p>Floodlight MS comprised 5 smartphone sensor&#x2013;based tests (hereafter referred to as &#x201C;smartphone-based tests&#x201D;) presented on an app, measuring information processing speed (Cognitive Test [<xref ref-type="bibr" rid="ref25">25</xref>]), hand function (Pinching Test [<xref ref-type="bibr" rid="ref24">24</xref>] and Draw a Shape Test [<xref ref-type="bibr" rid="ref26">26</xref>]), and gait (U-Turn Test [<xref ref-type="bibr" rid="ref27">27</xref>] and 2-Minute Walk Test [<xref ref-type="bibr" rid="ref28">28</xref>,<xref ref-type="bibr" rid="ref29">29</xref>]), as well as a Patient Journal including 2 patient-reported outcome (PRO) measures (Daily Status Questions and a Symptom Tracker [<xref ref-type="bibr" rid="ref30">30</xref>]). The data were consolidated in a dashboard for HCPs (Floodlight MS HCP Portal), which was accessible between clinical visits [<xref ref-type="bibr" rid="ref5">5</xref>], and constituted the focus of the technological assessment in this research.</p></sec></sec><sec id="s2" sec-type="methods"><title>Methods</title><sec id="s2-1"><title>Recruitment of Neurologists</title><p>US-based neurologists were recruited from Prova Health&#x2019;s existing network, professional communities (eg, LinkedIn), and academic publications, and through snowball sampling (where research participants recruit future participants), and they were sent email invitations to take part in the study. Neurologists were required to have managed a minimum of 40 people with MS per month and to have had no previous experience using the Floodlight technology. Convenience sampling was utilized to recruit primarily academic neurologists, as they were expected to give more granular feedback on the technology. There was no defined sample size due to the exploratory nature of this feasibility study.</p></sec><sec id="s2-2"><title>Patient Case Overview</title><p>A total of 2 patient cases were presented to each participant neurologist. Radiology reports and clinical letters were drafted by qualified radiologists and medical doctors to describe the cases and reflect the real magnetic resonance imaging and clinical data in a realistic narrative. For additional authenticity, synthetic social and family histories consistent with typical MS presentations were included. All cases were validated by 2 expert neurologists.</p><p>The goal was to achieve high-fidelity patient cases that were representative of real-world cases while minimizing the variation inherent among people with MS. Patient cases included (experimental) or did not include (control) Floodlight data, thus simulating real-world scenarios in which neurologists manage people with MS with or without access to the Floodlight technology (see the <italic>Patient Case Development</italic> section for more details).</p></sec><sec id="s2-3"><title>Patient Actor</title><p>A professional actor was recruited to play the patients&#x2019; roles during the teleconsultations. They were recruited from a company with 25 years of experience in providing trained actors to simulate patients with medical, surgical, and psychiatric conditions. The patient roles were based on generated behavioral profiles, which were used to inform the biographies for the 2 patient cases. The actor was provided with detailed scripts developed by a medical doctor and validated by neurologists to ensure [<xref ref-type="bibr" rid="ref1">1</xref>] the consistency and correctness of answers.</p></sec><sec id="s2-4"><title>Conduct of Clinical Simulations</title><p>Each clinical simulation session lasted approximately 2.5 hours and was conducted via Zoom. Medically qualified team members from Prova Health served as moderators, facilitating introductions (~5 min), reviewing the simulation session guide with the neurologists (~5 min), and providing training on the Floodlight technology (~30 min). After the simulated teleconsultations had begun, the moderators took notes with their cameras and microphones switched off. Teleconsultations were designed to replicate those that the participant neurologists would perform in their clinical practices. The individual simulated teleconsultation sessions for each of the 2 presented patient cases consisted of preparation (~5 min) and conduct (~30 min) phases, both led by the participant neurologist. Participant neurologists were instructed to evaluate the relevant information and recommend an appropriate management plan. In the preparation phase, clinical information provided with or without Floodlight data was reviewed for each patient case. The &#x201C;patient&#x201D; (played by the professional actor) then joined the call. Following the conclusion of each simulated teleconsultation, participant neurologists completed a short survey (~5 min) to collect feedback on the examined case. At the end of both sessions, the moderator shared a link to the postsimulation survey (~7 min) and conducted a brief interview (~5 min) with each participant neurologist.</p></sec><sec id="s2-5"><title>Patient Case Development</title><p>The 2 patient cases presented to each participant neurologist were developed using deidentified clinical and Floodlight data (see the <italic>Patient Cases Selection Criteria</italic> section). For both cases, the participant neurologists reviewed data from 3 recent clinical visits, including patient demographics, symptoms, diagnosis, treatment and medical history, and radiology reports based on original magnetic resonance imaging metrics.</p><p>The clinical data presented in the letters included EDSS scores, Symbol Digit Modalities Test (SDMT) scores, 9-Hole Peg Test times, and Timed 25-Foot Walk times. The main clinical traits emerging from clinical case narratives are presented in <xref ref-type="table" rid="table1">Table 1</xref>. It should be noted that although both patient cases presented gait impairment, Patient Case B was designed to show significant worsening of gait impairment in the last year.</p><p>Floodlight data visualized in the Floodlight MS HCP Portal were presented for 1 of the 2 cases reviewed by each neurologist. The orders of the patient cases and of the interventions were randomized before the participant neurologists started the teleconsultations (<xref ref-type="fig" rid="figure1">Figure 1</xref>).</p><table-wrap id="t1" position="float"><label>Table 1.</label><caption><p>Clinical traits, per patient case, presented during teleconsultation simulation.</p></caption><table id="table1" frame="hsides" rules="groups"><thead><tr><td align="left" valign="bottom">Simulation patient case</td><td align="left" valign="bottom">Main clinical traits emerging from clinical case narratives and actor&#x2019;s interpretation</td></tr></thead><tbody><tr><td align="left" valign="top">Patient Case A</td><td align="left" valign="top"><list list-type="bullet"><list-item><p>Cognitive function decline: decrease in SDMT<sup><xref ref-type="table-fn" rid="table1fn1">a</xref></sup> score of &#x2265;20% in the previous 12 months</p></list-item><list-item><p>Hand motor function decline: increase in 9HPT<sup><xref ref-type="table-fn" rid="table1fn2">b</xref></sup> time of &#x2265;20% in the previous 6&#x2010;12 months</p></list-item><list-item><p>History of gait impairment, as per T25FW<sup><xref ref-type="table-fn" rid="table1fn3">c</xref></sup> test data</p></list-item><list-item><p>Bowel and bladder issues reported in the previous 12 months</p></list-item><list-item><p>Vision changes in previous history</p></list-item><list-item><p>Fatigue</p></list-item></list></td></tr><tr><td align="left" valign="top">Patient Case B</td><td align="left" valign="top"><list list-type="bullet"><list-item><p>Gait capacity decline: increase in T25FW time of &#x2265;20% in the previous 6&#x2010;12 months</p></list-item><list-item><p>Reported bladder issues in the previous 12 months</p></list-item><list-item><p>Reported mood issues in the previous 12 months</p></list-item><list-item><p>Spasticity and pain in the previous 12 months</p></list-item></list></td></tr></tbody></table><table-wrap-foot><fn id="table1fn1"><p><sup>a</sup>SDMT: Symbol Digit Modalities Test.</p></fn><fn id="table1fn2"><p><sup>b</sup>9HPT: 9-Hole Peg Test.</p></fn><fn id="table1fn3"><p><sup>c</sup>T25FW: Timed 25-Foot Walk.</p></fn></table-wrap-foot></table-wrap><fig position="float" id="figure1"><label>Figure 1.</label><caption><p>Randomization of patient cases before the initiation of teleconsultation. The order of cases and experimental conditions was randomized between participant neurologists, and consultations were conducted with 2 patient cases (A and B). Half (n=5) of the participant neurologists were randomized to have access to the Floodlight MS HCP Portal for the first case, while the other half had access to the solution for the second case. Due to a randomization error, 6 out of the 10 participant neurologists were assigned to the Floodlight MS HCP Portal for Patient Case A, instead of 5 out of 10. HCP: health care professional; MS: multiple sclerosis.</p></caption><graphic alt-version="no" mimetype="image" position="float" xlink:type="simple" xlink:href="medinform_v13i1e67845_fig01.png"/></fig></sec><sec id="s2-6"><title>Data Sources</title><p>Clinical data from the CONSONANCE study (NCT03523858) were used in conjunction with Floodlight performance and symptom tracker data collected over 12 consecutive months. CONSONANCE is an ongoing, phase 3b, 4-year, open-label study assessing the effectiveness and safety of ocrelizumab in adults with progressive multiple sclerosis.</p><p>Participants enrolled in the CONSONANCE study performed smartphone-based tests using a precursor to Floodlight MS for clinical research purposes, providing a battery of digital assessments and digital PROs. Data from the Pinching Test, the Draw a Shape Test, the U-Turn Test, and the 2-Minute Walk Test (all performed daily) were utilized, alongside data from a version of the Cognitive Test (performed weekly; involving matching a sequence of symbols to digits within 90 s).</p><p>Data from 2 PRO measures collected via the Floodlight Patient Journal were also used. These were Daily Status Questions (&#x201C;How are you feeling physically now?&#x201D; and &#x201C;How is your mood now?&#x201D;) and a biweekly Symptom Tracker to track symptoms related to cognition (attention deficit, memory loss, brain &#x201C;fog&#x201D; sensation, mental fatigue), upper extremity function (sensory disturbance, muscle weakness, clumsiness, muscular spasm, pain), lower extremity function (sensory disturbance, muscle weakness, clumsiness, muscular spasm, pain), gait (trouble walking, lack of balance, walking fatigue), and bowel or bladder dysfunction (incontinence, bladder or urinary problems, constipation), as well as the time that any new or worsening symptoms manifested and whether or not this was perceived to be a relapse. All Floodlight data collected in the CONSONANCE trial and used in this study were imported into the Floodlight MS HCP Portal (version 1.5 at the time of the research).</p></sec><sec id="s2-7"><title>Patient Cases Selection Criteria</title><p>Patient profiles were selected from the CONSONANCE dataset based on selected criteria that indicated a clinical performance decline observed in the previous 6&#x2010;12 months, based on standard clinical assessments of cognitive function, hand motor function, or gait capacity.</p><p>For Patient Case A, the key selection criteria included cognitive function decline (a decrease in SDMT score of &#x2265;20% in the previous 12 mo) and hand motor function decline (increase in 9-Hole Peg Test time of &#x2265;20% in the previous 6&#x2010;12 mo). For Patient Case B, the criteria included gait capacity decline (increase in Timed 25-Foot Walk time of &#x2265;20% in the previous 6&#x2010;12 mo) as well as 2 consecutive reports of fatigue and bladder issues (score of 3&#x2010;6; some of the most impactful &#x201C;invisible&#x201D; symptoms of MS sourced via the SymptoMScreen). Some of the symptoms experienced by people living with MS may not be externally evident or easy to report and quantify, such as fatigue, mood disorders, cognitive impairments, pain, bladder or bowel dysfunction, sexual dysfunction, and vision changes. These symptoms can be difficult for the patient to report and articulate during a clinical encounter (eg, due to subjectivity bias, poor recall, or stigma), and for physicians to capture (limitations in available measurement tools, ample reliance on patient recall on the visit day) [<xref ref-type="bibr" rid="ref31">31</xref>]. The clinical performance decline, together with reported signs of &#x201C;invisible&#x201D; symptoms, provided impetus for a care decision to be taken during the clinical simulation.</p></sec><sec id="s2-8"><title>Data Extraction</title><p>Clinical data from 3 recent in-person clinical visits were extracted to present to the participant neurologists (<xref ref-type="fig" rid="figure2">Figure 2</xref>). No clinical data from the final in-person visit (the time point corresponding to the day of the teleconsultation in the simulation) were provided. Of note, cognitive decline had been confirmed at this visit for the patient in Patient Case A (as measured by the SDMT); these data were not provided to the participant neurologists in order to mimic a common clinical scenario in MS, where evident patient deterioration may precede formal confirmation through standard clinical assessments.</p><fig position="float" id="figure2"><label>Figure 2.</label><caption><p>Clinical data extraction for teleconsultation simulation. Clinical data on ideation, data retrieval, case development, and validation of patient cases were extracted for teleconsultation simulation. Parameters such as multiple sclerosis (MS) phenotype, clinical factors, symptom reports, and synthetic data such as age and sex were included to provide transparency in demographic data. Floodlight data from the most recent 12-month period for each clinical profile were extracted for one of the cases presented to each participant neurologist. HCP: health care professional; MRI: magnetic resonance imaging.</p></caption><graphic alt-version="no" mimetype="image" position="float" xlink:type="simple" xlink:href="medinform_v13i1e67845_fig02.png"/></fig><p>To build the patient cases, MS phenotype and date of diagnosis; clinical parameters, including recent disease history (eg, relapses, EDSS evolution); symptom reports; and current and prior treatments (eg, disease-modifying therapies) were also extracted. Additional &#x201C;synthetic&#x201D; data were used to construct demographic (age, sex) and radiology reports.</p><p>Floodlight data from the prior 12-month period for each clinical profile were also extracted to be provided for 1 of the 2 cases presented to each participant neurologist.</p></sec><sec id="s2-9"><title>Study Outcomes and Data Collection</title><p>Study outcomes were either moderator-reported or participant neurologist&#x2013;reported (via participant surveys and interviews; <xref ref-type="table" rid="table2">Table 2</xref>). Measures included questions regarding the fidelity of the simulated cases developed; use of the digital tool (Floodlight MS), including user experience metrics for data visualization, overall experience (Net Promoter Score), and usability (System Usability Scale [<xref ref-type="bibr" rid="ref32">32</xref>]); and workflow metrics, including time for case preparation, teleconsultation time, and mental effort required (Paas scale [<xref ref-type="bibr" rid="ref33">33</xref>]). Confidence in approaching clinical consultations was also measured. Additionally, moderators recorded the time taken for the identification of patient concerns, the recognition of &#x201C;invisible&#x201D; MS symptoms, and the care decisions prompted and taken (including change in treatment management) with versus without having Floodlight data available.</p><p>A medically qualified moderator from the research team observed each simulation session and recorded outcome measures such as time spent on each case. The participants were asked to self-report on the fidelity of simulation, user experience metrics, workflow metrics, clinical consultations, patient concerns, and care decisions. This was done using postcase surveys, postcase conduct surveys, postsimulation surveys, or a postsimulation interview. Each of these was measured using various questions.</p><table-wrap id="t2" position="float"><label>Table 2.</label><caption><p>Methods used to report on the study outcomes.</p></caption><table id="table2" frame="hsides" rules="groups"><thead><tr><td align="left" valign="bottom" rowspan="2">Measure</td><td align="left" valign="bottom" rowspan="2">Moderator measured</td><td align="left" valign="bottom" colspan="4">Participant neurologist reported</td></tr><tr><td align="left" valign="bottom">Survey after case preparation</td><td align="left" valign="bottom">Survey after case conduct</td><td align="left" valign="bottom">Survey after simulation</td><td align="left" valign="bottom">Interview after simulation</td></tr></thead><tbody><tr><td align="left" valign="top" colspan="6">Fidelity of simulation</td></tr><tr><td align="left" valign="top">&#x2003;Similarity of patient cases to those seen in day-to-day practice</td><td align="left" valign="top"/><td align="left" valign="top"/><td align="left" valign="top"/><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"/></tr><tr><td align="left" valign="top">&#x2003;Similarity of simulation to day-to-day practice consultations</td><td align="left" valign="top"/><td align="left" valign="top"/><td align="left" valign="top"/><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"/></tr><tr><td align="left" valign="top">&#x2003;Using clinical simulation to evaluate the Floodlight MS<sup><xref ref-type="table-fn" rid="table2fn1">a</xref></sup> HCP<sup><xref ref-type="table-fn" rid="table2fn2">b</xref></sup> Portal</td><td align="left" valign="top"/><td align="left" valign="top"/><td align="left" valign="top"/><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"><bold>X</bold></td></tr><tr><td align="left" valign="top" colspan="6">User experience metrics</td></tr><tr><td align="left" valign="top">&#x2003;User feedback on the Floodlight MS HCP Portal</td><td align="left" valign="top"/><td align="left" valign="top"/><td align="left" valign="top"/><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"><bold>X</bold></td></tr><tr><td align="left" valign="top">&#x2003;Usability of the Floodlight MS HCP Portal</td><td align="left" valign="top"/><td align="left" valign="top"/><td align="left" valign="top"/><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"/></tr><tr><td align="left" valign="top" colspan="6">Workflow metrics</td></tr><tr><td align="left" valign="top">&#x2003;Time taken to prepare for teleconsultation</td><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"/><td align="left" valign="top"/><td align="left" valign="top"/><td align="left" valign="top"/></tr><tr><td align="left" valign="top">&#x2003;Time taken to conduct teleconsultation</td><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"/><td align="left" valign="top"/><td align="left" valign="top"/><td align="left" valign="top"/></tr><tr><td align="left" valign="top">&#x2003;Reported mental effort associated with information acquisition and application</td><td align="left" valign="top"/><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"/><td align="left" valign="top"/></tr><tr><td align="left" valign="top" colspan="6">Clinical consultations, patient concerns, and care decisions</td></tr><tr><td align="left" valign="top">&#x2003;Reported confidence level when preparing for and making care decisions during a teleconsultation</td><td align="left" valign="top"/><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"/><td align="left" valign="top"/></tr><tr><td align="left" valign="top">&#x2003;Identification of the patient&#x2019;s primary concern</td><td align="left" valign="top"/><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"/><td align="left" valign="top"><bold>X</bold></td></tr><tr><td align="left" valign="top">&#x2003;Time taken to identify the patient&#x2019;s primary concern during teleconsultation (independently verified by a second clinician using screen recordings)</td><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"/><td align="left" valign="top"/><td align="left" valign="top"/><td align="left" valign="top"/></tr><tr><td align="left" valign="top">&#x2003;Recognition of &#x201C;invisible symptoms of MS&#x201D;<sup><xref ref-type="table-fn" rid="table2fn3">c</xref></sup> and their proactive discussion with patients</td><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"/><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"/><td align="left" valign="top"/></tr><tr><td align="left" valign="top">&#x2003;Care decisions taken and reported prompts for these decisions</td><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"/><td align="left" valign="top"><bold>X</bold></td><td align="left" valign="top"/><td align="left" valign="top"><bold>X</bold></td></tr></tbody></table><table-wrap-foot><fn id="table2fn1"><p><sup>a</sup>MS: multiple sclerosis.</p></fn><fn id="table2fn2"><p><sup>b</sup>HCP: health care professional.</p></fn><fn id="table2fn3"><p><sup>c</sup>Invisible symptoms, including fatigue, mood disorders, cognitive impairments, pain, bladder or bowel dysfunction, sexual dysfunction, and vision changes can be experienced by people living with MS. Some of these symptoms may not be externally evident or easy to report and quantify and can be difficult for the patient to report and articulate during a clinical encounter (eg, due to subjectivity bias, poor recall, or stigma) or for physicians to capture (eg, limitations in available measurement tools, ample reliance on patient recall on the visit day).</p></fn></table-wrap-foot></table-wrap></sec><sec id="s2-10"><title>Data Analysis</title><p>For continuous, time-dependent, moderator-assessed measures, the median times were calculated and presented as box and whisker diagrams. Additionally, these time-dependent measures were collated into groups, and all categorical results were presented as absolute numbers within each category and presented as bar charts. An a priori decision was made not to perform hypothesis-testing statistics on data from this feasibility study.</p></sec><sec id="s2-11"><title>Ethical Considerations</title><p>CONSONANCE patient data anonymity was retained throughout the preparation and conduct of this study. The protocol (22-PROH-102: Evaluation of Floodlight MS using clinical simulation) was reviewed by Pearl Institutional Review Board on September 20, 2022, and determined to be exempt from ethics approval according to U.S. Food and Drug Administration 21 Code of Federal Regulations (CFR) 56.104 and 45CFR46.104(b)(3): (3) Benign Behavioral Interventions - Adults. The study was conducted in accordance with the most recent version of the Declaration of Helsinki. Signed consent forms were obtained from the participant neurologists before the simulated teleconsultations were conducted. Participant neurologists were compensated for their time according to local Fair Market Value rates.</p></sec></sec><sec id="s3" sec-type="results"><title>Results</title><sec id="s3-1"><title>Participant Neurologist Demographics</title><p>A total of 10 US-based neurologists were recruited as participants (7 from large academic centers and 3 from specialist neurology centers). A total of 6 participant neurologists had &#x003E;10 years of experience in performing MS consultations, 1 had 5&#x2010;10 years of experience, and 3 had 3&#x2010;5 years of experience. The number of consultations performed by participant neurologists in their current practice in a typical week was &#x003E;50 (n=2), 40&#x2010;50 (n=1), 30&#x2010;40 (n=1), 20&#x2010;30 (n=3), or 10&#x2010;20 (n=3). Further, participant neurologists performed 25%&#x2010;50% (n=3) or &#x003C;25% (n=7) of all consultations as teleconsultations in their current practice.</p></sec><sec id="s3-2"><title>Patient Case and Simulation Fidelity</title><p>Participant neurologists felt that the simulations were high-fidelity representations of real consultations. All reported that the patient cases and scenarios were &#x201C;very&#x201D; or &#x201C;somewhat&#x201D; similar to typical practice (<xref ref-type="table" rid="table3">Table 3</xref>). Feedback from participants on the performance of the professional patient actor was mostly very positive:</p><disp-quote><p>Shockingly. He was very good. I say that. I haven&#x2019;t done this in a while, but very often a mock patient doesn&#x2019;t get the real essence of MS. This guy did great. I even wondered, did he have MS, how he got so good at it? ... They would have had to practice and discuss things with many patients and doctors to get this good.</p></disp-quote><disp-quote><p>The actor did a great job. Intuitively knew how to pick up on what I was asking.</p></disp-quote><table-wrap id="t3" position="float"><label>Table 3.</label><caption><p>Participant neurologists&#x2019; experiences of patient case and simulation fidelity.<sup><xref ref-type="table-fn" rid="table3fn1">a</xref></sup></p></caption><table id="table3" frame="hsides" rules="groups"><thead><tr><td align="left" valign="top">Participant neurologists&#x2019; experiences</td><td align="center" valign="top" colspan="5">Number of participant neurologists</td></tr><tr><td align="left" valign="top"/><td align="center" valign="top">Very different</td><td align="left" valign="top">Somewhat different</td><td align="left" valign="top">Neutral</td><td align="left" valign="top">Somewhat similar</td><td align="left" valign="top">Very similar</td></tr></thead><tbody><tr><td align="left" valign="top">How similar were these patient cases to those you see in your day-to-day practice?</td><td align="left" valign="top">0</td><td align="left" valign="top">0</td><td align="left" valign="top">0</td><td align="left" valign="top">4</td><td align="left" valign="top">6</td></tr><tr><td align="left" valign="top">How similar were these consultations to the consultations you complete during a typical week?</td><td align="left" valign="top">0</td><td align="left" valign="top">0</td><td align="left" valign="top">0</td><td align="left" valign="top">5</td><td align="left" valign="top">5</td></tr></tbody></table><table-wrap-foot><fn id="table3fn1"><p><sup>a</sup>Participant neurologists reported similarity of patient cases to those seen in day-to-day practice and similarity of simulation to day-to-day practice consultations via surveys and questionnaires.</p></fn></table-wrap-foot></table-wrap></sec><sec id="s3-4"><title>User Experience Metrics</title><p>The Floodlight MS HCP Portal scored an &#x201C;above average&#x201D; score of 79 on the System Usability Scale (<xref ref-type="fig" rid="figure3">Figure 3A</xref>). In terms of overall experience of using the Floodlight technology, the Floodlight MS HCP Portal scored an &#x201C;acceptable&#x201D; Net Promoter Score of 10, based on the participant neurologists&#x2019; reports of how likely they would be to recommend it to a colleague (<xref ref-type="fig" rid="figure3">Figure 3B</xref>).</p><p>Participant neurologists agreed that data visualizations in the Floodlight MS HCP Portal were &#x201C;very&#x201D; or &#x201C;somewhat&#x201D; useful for upper extremity function (10/10), cognition (10/10), gait (9/10), and mood status (obtained via the Floodlight Daily Status Questions; 9/10; <xref ref-type="fig" rid="figure4">Figure 4</xref>). The &#x201C;mood status&#x201D; aspect of the Patient Journal was rated as the most useful of the data visualizations presented, with 8/10 ratings of &#x201C;very useful&#x201D;.</p><fig position="float" id="figure3"><label>Figure 3.</label><caption><p>User experience metrics for the Floodlight technology, including (A) SUS and (B) NPS [<xref ref-type="bibr" rid="ref34">34</xref>]. The bars represent responses on the user experience of Floodlight MS (SUS) and user experience for data visualization and the overall experience (NPS); these were reported via participant surveys and interviews. Items alternate between positive and negative phrasing, with higher scores indicating more favorable usability perceptions. HCP: health care professional; MS: multiple sclerosis; NPS: net promoter score; SUS: System Usability Scale.</p></caption><graphic alt-version="no" mimetype="image" position="float" xlink:type="simple" xlink:href="medinform_v13i1e67845_fig03.png"/></fig><fig position="float" id="figure4"><label>Figure 4.</label><caption><p>User experience metrics for the usefulness of data visualizations in the Floodlight MS HCP Portal. Participant neurologists provided feedback on whether data visualizations in the Floodlight MS HCP Portal were useful for factors including upper extremity function, gait, and cognition. HCP: health care professional; MS: multiple sclerosis.</p></caption><graphic alt-version="no" mimetype="image" position="float" xlink:type="simple" xlink:href="medinform_v13i1e67845_fig04.png"/></fig></sec><sec id="s3-5"><title>Workflow Metrics</title><p>In terms of the time taken for patient case preparation, cases took a median of 7 minutes and 36 seconds with Floodlight and 5 minutes and 57 seconds without Floodlight (<xref ref-type="fig" rid="figure5">Figure 5A</xref>). The median time taken to conduct the teleconsultations was 22 minutes with Floodlight and 20 minutes without Floodlight (<xref ref-type="fig" rid="figure5">Figure 5B</xref>). When using the Floodlight technology, participant neurologists also reported investing a slightly higher level of mental effort to prepare for (<xref ref-type="fig" rid="figure5">Figure 5C</xref>; very high mental effort to rather high mental effort: n=3 vs n=2) and conduct patient cases (<xref ref-type="fig" rid="figure5">Figure 5D</xref>; very, very high mental effort to rather high mental effort: n=6 vs n=5).</p><fig position="float" id="figure5"><label>Figure 5.</label><caption><p>Workflow metrics. (A) Preparation time for patient cases with and without Floodlight data. (B) Teleconsultation time with and without Floodlight data. (C) Mental effort required preparing for patient cases with and without Floodlight data. (D) Mental effort required when conducting patient cases with and without Floodlight data. The classification key uses shades of blue for data obtained &#x201C;with Floodlight&#x201D; and shades of red for data obtained &#x201C;without Floodlight&#x201D;; shade of gray uses the same classification in both &#x201C;with Floodlight&#x201D; and &#x201C;without Floodlight&#x201D;.</p></caption><graphic alt-version="no" mimetype="image" position="float" xlink:type="simple" xlink:href="medinform_v13i1e67845_fig05.png"/></fig></sec><sec id="s3-6"><title>Clinical Consultations and Patient Concerns</title><p>About 6 out of the 10 neurologists &#x201C;strongly agreed&#x201D; that it was easier to identify patient concerns within the relevant functional domains with access to Floodlight data during the consultation preparation phase versus without access (where 2/10 &#x201C;strongly agreed&#x201D;; <xref ref-type="fig" rid="figure6">Figure 6A</xref>). The median time taken for participant neurologists to start discussing the primary domain of concern was 2 minutes, 14 seconds when using the Floodlight technology and 2 minutes, and 17 seconds without the Floodlight technology (<xref ref-type="fig" rid="figure6">Figure 6B</xref>).</p><fig position="float" id="figure6"><label>Figure 6.</label><caption><p>Patients&#x2019; primary domains of concern: (A) identification and (B) time taken. The identification of patient concern and the median time taken for participant neurologists to initiate discussions on the primary domain of concern were assessed during the consultation preparation phase with and without access to Floodlight data.</p></caption><graphic alt-version="no" mimetype="image" position="float" xlink:type="simple" xlink:href="medinform_v13i1e67845_fig06.png"/></fig></sec><sec id="s3-7"><title>Recognition of &#x201C;Invisible&#x201D; Symptoms of MS</title><p>The median time taken to discuss each &#x201C;invisible&#x201D; symptom was between 1 and 1.5 minutes (<xref ref-type="table" rid="table4">Table 4</xref>). In Patient Case A, which was structured to show a prominent cognitive decline, cognitive issues, and mood change, discussions exceeded 2 minutes. In Patient Case B, bladder dysfunction (which was referred to in the clinical letters and without the support of Floodlight information) was also discussed for over 2 minutes. While differences can be gleaned at the individual symptom level, there was no clear pattern to the length of the discussions. Being aware of the symptom by means of the clinical letters did not necessarily seem to be a clear determinant in the participant neurologist spending more time on discussions with the patient actor. There were also no consistent differences in the median time spent discussing symptoms across domains with and without the use of the Floodlight technology.</p><table-wrap id="t4" position="float"><label>Table 4.</label><caption><p>Median time spent discussing each &#x201C;invisible&#x201D; symptom<sup><xref ref-type="table-fn" rid="table4fn1">a</xref></sup> with patients (such as fatigue and bladder dysfunction that are difficult to evaluate by health care professionals [HCPs]) across all teleconsultations with and without the Floodlight technology.</p></caption><table id="table4" frame="hsides" rules="groups"><thead><tr><td align="left" valign="bottom">Symptom</td><td align="center" valign="bottom" colspan="2">Patient Case A<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td><td align="center" valign="bottom" colspan="2">Patient Case B<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td></tr><tr><td align="left" valign="bottom"/><td align="center" valign="bottom">With Floodlight (min)</td><td align="center" valign="bottom">Without Floodlight (min)</td><td align="center" valign="bottom">With Floodlight (min)</td><td align="center" valign="bottom">Without Floodlight (min)</td></tr></thead><tbody><tr><td align="left" valign="top">Vision changes</td><td align="left" valign="top">01:32<italic><sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></italic></td><td align="left" valign="top">01:18<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td><td align="left" valign="top">00:30</td><td align="left" valign="top">00:10</td></tr><tr><td align="left" valign="top">Sexual dysfunction</td><td align="left" valign="top">00:05</td><td align="left" valign="top">01:15</td><td align="left" valign="top">00:17<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td><td align="left" valign="top">00:55<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td></tr><tr><td align="left" valign="top">Mood changes</td><td align="left" valign="top">02:18</td><td align="left" valign="top">01:26</td><td align="left" valign="top">01:32<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td><td align="left" valign="top">01:50<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td></tr><tr><td align="left" valign="top">Physical and emotional pain</td><td align="left" valign="top">01:55</td><td align="left" valign="top">01:15</td><td align="left" valign="top"><italic>0</italic>1:54<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td><td align="left" valign="top">01:12<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td></tr><tr><td align="left" valign="top">Bowel and bladder dysfunction</td><td align="left" valign="top">00:30<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td><td align="left" valign="top">01:10<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td><td align="left" valign="top">01:14<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td><td align="left" valign="top">02:47<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td></tr><tr><td align="left" valign="top">Cognitive changes</td><td align="left" valign="top">02:16<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td><td align="left" valign="top">02:06<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td><td align="left" valign="top">00:27</td><td align="left" valign="top">00:53</td></tr><tr><td align="left" valign="top">Fatigue</td><td align="left" valign="top">00:40<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td><td align="left" valign="top">01:01<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup></td><td align="left" valign="top">01:08</td><td align="left" valign="top">01:39</td></tr><tr><td align="left" valign="top">Median across all domains</td><td align="left" valign="top">01:25</td><td align="left" valign="top">01:22</td><td align="left" valign="top">01:07</td><td align="left" valign="top">01:34</td></tr></tbody></table><table-wrap-foot><fn id="table4fn1"><p><sup>a</sup>Invisible symptoms, including fatigue, mood disorders, cognitive impairments, pain, bladder or bowel dysfunction, sexual dysfunction, and vision changes can be experienced by people living with MS. Some of these symptoms may not be externally evident or easy to report and quantify and can be difficult for the patient to report and articulate during a clinical encounter (eg, due to subjectivity bias, poor recall, or stigma) or for physicians to capture (eg, limitations in available measurement tools, ample reliance on patient recall on the visit day).</p></fn><fn id="table4fn2"><p><sup>b</sup>These are the symptoms that were previously reported, per the clinical documentation provided, for Patient Case A and Patient Case B.</p></fn></table-wrap-foot></table-wrap></sec><sec id="s3-8"><title>Care Decisions and Change in Disease Management Decisions</title><p>Floodlight data were reported as a trigger for 20 of the 37 care and change in disease management decisions (including regarding disease-modifying therapies, additional medications, referrals, or investigations), always in combination with other triggers such as patient history (20/20) or clinical exam findings (9/20; <xref ref-type="fig" rid="figure7">Figure 7</xref>). Overall, a greater number of care decisions were taken when using the Floodlight technology (37) versus without (29), including making a referral to another HCP (9 vs 6), or adding, changing, or discontinuing a nonpharmacologic therapy (6 vs 5) or disease-modifying therapy (4 vs 2).</p><fig position="float" id="figure7"><label>Figure 7.</label><caption><p>Care decisions, including change in disease management, prompted by Floodlight data and taken when using the Floodlight technology, versus without. Floodlight data were reported as a trigger for care and change in disease management decisions, including regarding disease-modifying therapies, additional medications, referrals, or investigations. In all instances, Floodlight data were used in conjunction with other triggers such as patient history or clinical exam findings. HCP: health care professional.</p></caption><graphic alt-version="no" mimetype="image" position="float" xlink:type="simple" xlink:href="medinform_v13i1e67845_fig07.png"/></fig></sec><sec id="s3-9"><title>Overall Feedback</title><p>Overall, participant neurologists valued access to longitudinal data, particularly regarding hand function and cognition. Many participant neurologists found the aspects of the user interface challenging when using the Floodlight MS HCP Portal. They felt that user interface improvements were the most important changes to be made, and they noted the following could help improve the interface: &#x201C;Separating different data points and putting them into separate charts,&#x201D; &#x201C;...an overall curve, visualizing the trend over time,&#x201D; and &#x201C;&#x2026;being able to change the y axis to separate data points for easier review.&#x201D;</p></sec><sec id="s3-10"><title>Data Availability</title><p>For up-to-date details on Roche&#x2019;s global policy on the sharing of clinical information and how to request access to related clinical study documents, please visit [<xref ref-type="bibr" rid="ref35">35</xref>].</p></sec></sec><sec id="s4" sec-type="discussion"><title>Discussion</title><sec id="s4-1"><title>Principal Findings</title><p>The clinical simulations of MS teleconsultations described in this study demonstrate the ability of this research methodology to deliver high-fidelity patient cases. They also enabled direct comparison of teleconsultations conducted with and without the use of digital health technologies.</p><p>Clinical simulation offered several advantages, including the ability to present relevant, representative, paradigmatic MS cases and to efficiently generate insights on the utility and usability of new digital technology in clinical visits. This provided early real-world insights from neurologists who were na&#x00EF;ve to the Floodlight technology and who received only brief introductory training on its use. Specifically, it provided an opportunity to move beyond the expected impact of the tool and, instead, explore the approach used by participants na&#x00EF;ve to the digital technology (after a brief orientation) in actual clinical decision-making.</p><p>Using real patient data from the CONSONANCE study enabled access to extensive existing data without additional patient consent or data acquisition, allowing case selection based on in-depth clinical information. The richness of the CONSONANCE dataset minimized the need for synthetic data, which was limited but where used, derived from valid sources such as narrative radiology reports and clinical letters. While this approach ensured high-fidelity and clinically relevant patient scenarios, it introduced a limitation in the generalizability of the simulated cases as the patient population in the CONSONANCE study was composed entirely of people with progressive multiple sclerosis. This highlights the capability of clinical simulation to leverage rich, real-world datasets while also underscoring the importance of including a broader spectrum of patient profiles and validated synthetic cases to enhance external validity, scalability, and applicability across diverse MS phenotypes. Additional simulated patient cases can be easily generated from other clinical trials or by generating synthetic (but realistic) patient case data. Further simulation research should expand the number and diversity of patient cases, including data from varied trial populations and validated synthetic cases to better capture the heterogeneity of MS care. It is important to ensure that these cases are representative of typical patients seen in clinical practice and developed with appropriate ethical considerations to avoid the introduction of unwanted bias. Additionally, it is also essential to ensure that clinical trial data used for patient cases do not allow patient identities to be inferred and that blinding procedures are preserved.</p><p>This study explored complex MS cases, with multiple patient concerns and functional domains affected, to capture the complexity of real-world MS clinical visits and explore the impact of utilizing a digital tool (Floodlight technology). As with &#x201C;real&#x201D; clinical encounters with people living with MS, distilling prior history and functional testing into condensed clinical letters for use in the consultations was challenging, due to the heterogeneity of symptoms, scales, and other assessments used. For example, bladder symptoms reported by patients in the SymptoMScreen were consistently underscored in the clinicians&#x2019; EDSS Bowel/Bladder Functional System [<xref ref-type="bibr" rid="ref36">36</xref>-<xref ref-type="bibr" rid="ref38">38</xref>].</p><p>Research applications of simulation can include training methodologies for educational purposes [<xref ref-type="bibr" rid="ref22">22</xref>], such as in training for medical students or less experienced HCPs, and to enhance clinical competence [<xref ref-type="bibr" rid="ref21">21</xref>]. Additionally, simulation can be used to improve primary care office&#x2013;based readiness to respond to medical emergencies [<xref ref-type="bibr" rid="ref39">39</xref>] or to enhance cultural sensitivity, communication, and empathy among HCPs [<xref ref-type="bibr" rid="ref40">40</xref>]. Simulation can also be used as an investigative methodology to answer research questions that otherwise could not be answered feasibly, safely, ethically, or in a timely fashion in clinical settings [<xref ref-type="bibr" rid="ref22">22</xref>], as pertains to our study.</p><p>The simulation methodology provided key insights into integrating a digital health tool into routine MS care. Initial findings pertained to how neurologists na&#x00EF;ve to this technology began to access and interpret these data. Indeed, there is a known socialization process required for the adoption of new technology [<xref ref-type="bibr" rid="ref41">41</xref>], and this must be accounted for in the plans that developers have for the implementation of new tools. Neurologists scored the clinician-facing HCP Portal for accessing Floodlight MS data &#x201C;above average&#x201D; on the System Usability Scale, indicating a high level of usability. Feedback from postsimulation surveys and interviews provides a foundation for iterative improvements to the user interface. Among the available data on the portal, the &#x201C;mood status&#x201D; aspect of the Patient Journal (Daily Status Questions) was rated as the most useful of the data visualizations presented, which is potentially indicative of a medical need (eg, more systematic depression monitoring). Finally, as expected, incorporating Floodlight data into teleconsultation preparation required slightly more time and mental effort than consultation without such technology. Interestingly, the median time was only 1.7 minutes longer for patient case preparation, which seems relatively modest compared with the potential gains from having Floodlight data available. This also indicates that the brief presimulation training was effective and supports the tool&#x2019;s scalability. With increased familiarity, time required to use the Floodlight technology during the visits would likely decrease. However, even a short amount of additional time requirement could nonetheless present a potential barrier to use. For example, when evaluating the initial launch of the patient codesigned, high-usability-scoring, point-of-care clinical dashboard MS NeuroShare, clinicians were less likely to use it on busy clinic days, despite the reported improvements in the quality of communication and patient understanding of management care plans. New solutions to alleviate clinician workloads must be evaluated against the time required to implement them, such as educating patients on the clinical applications [<xref ref-type="bibr" rid="ref41">41</xref>].</p><p>Digital health tools can capture objective data between clinical visits. This can avoid reliance on patient recall of symptoms, helping to provide clinicians with a more complete picture of symptoms and enabling discussions between HCPs and patients. The second set of exploratory insights, therefore, concerned the impact that ready access to Floodlight data might have on clinical discussion and management. Overall, while participant neurologists started discussing the primary domain of concern after 2 minutes both with or without the Floodlight technology, most participants said that it was easier to identify patients&#x2019; concerns when they had access to the Floodlight MS HCP Portal (<xref ref-type="fig" rid="figure6">Figure 6</xref>A). Further, more care decisions were taken when using Floodlight data versus without, for example, making a referral to another HCP or amending a disease-modifying therapy.</p></sec><sec id="s4-2"><title>Limitations</title><p>While simulations can represent and provide insights on the dynamics of MS in real-life clinical practice, the method is limited by the ability to account for operational and behavioral variables that occur in clinical practices, and it may, therefore, not be a fully accurate representation. While this methodology is inherently scalable, its primary limitation lies in the effort needed to recruit and train actors. By comparison, on-site studies involving real patients are even more constrained, as they can only include individuals residing near the clinical study site(s). Although the actors performed convincingly, simulations cannot fully substitute for real patient interactions. Real patients may experience factors such as anxiety about their clinical outcomes, fatigue from poor sleep, or depression, all of which can influence the dynamics of clinical encounters; however, it must be noted that this is not one of the aims of this study. The limited numbers of participants and patient cases in this study, important for this initial testing of the research methodology, naturally limit generalizability to real-life clinical consultations. Virtual simulations could enrich the recruitment of diverse neurologists by offering more convenient participation as well as the potential to recruit neurologists from a wider geographic area. This will be important in studies that seek to quantify differences in clinical decision-making. Furthermore, a potential source of bias relevant to all simulations involving observation of tasks is the Hawthorne effect, where participants modify their behavior when they know they are being observed [<xref ref-type="bibr" rid="ref42">42</xref>,<xref ref-type="bibr" rid="ref43">43</xref>]. Mitigation strategies to overcome this included informing participating neurologists in advance about the study objectives, emphasizing that the research focus was not primarily on diagnostic accuracy, and assuring them that their identities would remain confidential. To further minimize the Hawthorne effect, the observer&#x2019;s camera and microphone were switched off during sessions; however, the observer remained present on the call to ensure study fidelity and provide technical support if needed. It should be acknowledged that despite these measures, such biases cannot be entirely eliminated in simulation-based research, although increasing the number of participants might reduce their impact.</p><p>Further, exploring cognitive impairment as a primary domain of concern highlighted some limitations of using simulated encounters because some participant neurologists described typically engaging patients&#x2019; family members or care partners when assessing the extent and impact of a patient&#x2019;s cognitive challenges, and family members were not included in the simulated scenarios. An alternative approach in clinical simulations exploring more focused digital tools would be to curate simpler patient cases with only 1 domain of concern and a few specific invisible symptoms and to focus the investigation on more specific aspects.</p></sec><sec id="s4-3"><title>Conclusions</title><p>Clinical simulation of MS teleconsultations allowed evidence generation on the potential utility of a digital health technology tool in MS care. The simulated patient cases and environment were found to be of high fidelity and enabled the direct comparison of teleconsultations conducted with and without the use of the Floodlight technology. A number of domains could be explored, including tool usability, influence of Floodlight data on clinical decisions, and impact on clinical workflows. Employing this methodology provided preliminary evidence that the Floodlight technology could support routine MS care, as it was found to be usable and useful, with minimal additional time and mental effort required on the part of neurologists previously na&#x00EF;ve to the technology.</p><p>The findings of this study support expanded research into and development of clinical simulation methodologies for neurological research. This could expedite integrating innovative methods or interventions in health care settings, as well as in medical training. In the digital health space specifically, simulations could help to anticipate what socialization of technologies may require, to steer guidance on usage, explain their value to clinicians, and help health systems plan implementation.</p></sec></sec></body><back><ack><p>We would like to thank all study participants and their families. We would also like to thank Active Image Roleplay for providing the professional actors, and Philipp D&#x2019;Alfonso, Joan Pi&#x0148;ol Fustero, and Harold Gonzalez Hernandez, who helped with the technical setup of the Floodlight MS HCP Portal simulated environment. Writing and editorial assistance for this manuscript was provided by Cynthia J Henley-Smith, PhD, and Terri Desmonds, PhD, of Nucleus Global, an Inizio company in the United Kingdom, and funded by F. Hoffmann-La Roche Ltd.</p></ack><notes><sec><title>Funding</title><p>Funding for this study was provided by F. Hoffmann-La Roche Ltd.</p></sec></notes><fn-group><fn fn-type="con"><p>Conceptualization: AK, JH, LC, RB, SG</p><p>Data curation: SD</p><p>Formal analysis: SD, SH</p><p>Funding acquisition: LC</p><p>Investigation: JH, RB, SD</p><p>Methodology: JH, SD, SG</p><p>Project administration: JH, LC</p><p>Resources: JH, LC</p><p>Supervision: JH, LC</p><p>Visualization: SD, SH</p><p>Writing &#x2013; review &#x0026; editing: AK, IW, JH, LC, RB, SD, SG, SH</p></fn><fn fn-type="conflict"><p>RB has received consultancy fees from Alexion, EMD Serono, Horizon, Janssen, and TG Therapeutics; and is funded by the NMSS Harry Weaver Award, NIH, DOD, and NSF, as well as Biogen, Eli Lilly, F. Hoffmann-La Roche Ltd, and Genentech. LC is an employee of F. Hoffmann-La Roche Ltd. IC was an employee of F. Hoffmann-La Roche Ltd when this study was conducted and is currently an employee of CS Genetics. SD was a subcontractor to Prova Health when this study was conducted and is currently an employee of Imperial College London. SG is an employee of and shareholder in Prova Health and an employee of Imperial College London. JH is an employee of and shareholder in Prova Health. SH is an employee of and shareholder in F. Hoffmann-La Roche Ltd. AK is an employee of and shareholder in F. Hoffmann-La Roche Ltd.</p></fn></fn-group><glossary><title>Abbreviations</title><def-list><def-item><term id="abb1">EDSS</term><def><p>Expanded Disability Status Scale</p></def></def-item><def-item><term id="abb2">HCP</term><def><p>health care professional</p></def></def-item><def-item><term id="abb3">MS</term><def><p>multiple sclerosis</p></def></def-item><def-item><term id="abb4">PRO</term><def><p>patient-reported outcome</p></def></def-item><def-item><term id="abb5">SDMT</term><def><p>Symbol Digit Modalities Test</p></def></def-item></def-list></glossary><ref-list><title>References</title><ref id="ref1"><label>1</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Kappos</surname><given-names>L</given-names> </name><name name-style="western"><surname>Wolinsky</surname><given-names>JS</given-names> </name><name name-style="western"><surname>Giovannoni</surname><given-names>G</given-names> </name><etal/></person-group><article-title>Contribution of relapse-independent progression vs relapse-associated worsening to overall confirmed disability accumulation in typical relapsing multiple sclerosis in a pooled analysis of 2 randomized clinical trials</article-title><source>JAMA Neurol</source><year>2020</year><month>09</month><day>1</day><volume>77</volume><issue>9</issue><fpage>1132</fpage><lpage>1140</lpage><pub-id pub-id-type="doi">10.1001/jamaneurol.2020.1568</pub-id><pub-id pub-id-type="medline">32511687</pub-id></nlm-citation></ref><ref id="ref2"><label>2</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Moccia</surname><given-names>M</given-names> </name><name name-style="western"><surname>Lanzillo</surname><given-names>R</given-names> </name><name name-style="western"><surname>Palladino</surname><given-names>R</given-names> </name><etal/></person-group><article-title>Cognitive impairment at diagnosis predicts 10-year multiple sclerosis progression</article-title><source>Mult Scler</source><year>2016</year><month>04</month><volume>22</volume><issue>5</issue><fpage>659</fpage><lpage>667</lpage><pub-id pub-id-type="doi">10.1177/1352458515599075</pub-id><pub-id pub-id-type="medline">26362896</pub-id></nlm-citation></ref><ref id="ref3"><label>3</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Filippi</surname><given-names>M</given-names> </name><name name-style="western"><surname>Bar-Or</surname><given-names>A</given-names> </name><name name-style="western"><surname>Piehl</surname><given-names>F</given-names> </name><etal/></person-group><article-title>Multiple sclerosis</article-title><source>Nat Rev Dis Primers</source><year>2018</year><month>11</month><day>8</day><volume>4</volume><issue>1</issue><fpage>43</fpage><pub-id pub-id-type="doi">10.1038/s41572-018-0041-4</pub-id><pub-id pub-id-type="medline">30410033</pub-id></nlm-citation></ref><ref id="ref4"><label>4</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Thompson</surname><given-names>AJ</given-names> </name><name name-style="western"><surname>Baranzini</surname><given-names>SE</given-names> </name><name name-style="western"><surname>Geurts</surname><given-names>J</given-names> </name><name name-style="western"><surname>Hemmer</surname><given-names>B</given-names> </name><name name-style="western"><surname>Ciccarelli</surname><given-names>O</given-names> </name></person-group><article-title>Multiple sclerosis</article-title><source>Lancet</source><year>2018</year><month>04</month><day>21</day><volume>391</volume><issue>10130</issue><fpage>1622</fpage><lpage>1636</lpage><pub-id pub-id-type="doi">10.1016/S0140-6736(18)30481-1</pub-id><pub-id pub-id-type="medline">29576504</pub-id></nlm-citation></ref><ref id="ref5"><label>5</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>van der Walt</surname><given-names>A</given-names> </name><name name-style="western"><surname>Butzkueven</surname><given-names>H</given-names> </name><name name-style="western"><surname>Shin</surname><given-names>RK</given-names> </name><etal/></person-group><article-title>Developing a digital solution for remote assessment in multiple sclerosis: from concept to software as a medical device</article-title><source>Brain Sci</source><year>2021</year><month>09</month><day>21</day><volume>11</volume><issue>9</issue><fpage>1247</fpage><pub-id pub-id-type="doi">10.3390/brainsci11091247</pub-id><pub-id pub-id-type="medline">34573267</pub-id></nlm-citation></ref><ref id="ref6"><label>6</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Kurtzke</surname><given-names>JF</given-names> </name></person-group><article-title>Rating neurologic impairment in multiple sclerosis: an expanded disability status scale (EDSS)</article-title><source>Neurology (ECronicon)</source><year>1983</year><month>11</month><volume>33</volume><issue>11</issue><fpage>1444</fpage><lpage>1452</lpage><pub-id pub-id-type="doi">10.1212/wnl.33.11.1444</pub-id><pub-id pub-id-type="medline">6685237</pub-id></nlm-citation></ref><ref id="ref7"><label>7</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Fouad</surname><given-names>AM</given-names> </name><name name-style="western"><surname>Abdel Naseer</surname><given-names>M</given-names> </name><name name-style="western"><surname>Farghaly</surname><given-names>M</given-names> </name><name name-style="western"><surname>Hegazy</surname><given-names>MI</given-names> </name></person-group><article-title>New algorithmic approach for easier and faster extended disability status scale calculation</article-title><source>Mult Scler J Exp Transl Clin</source><year>2023</year><volume>9</volume><issue>1</issue><fpage>20552173231155055</fpage><pub-id pub-id-type="doi">10.1177/20552173231155055</pub-id><pub-id pub-id-type="medline">36816811</pub-id></nlm-citation></ref><ref id="ref8"><label>8</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Hatcher-Martin</surname><given-names>JM</given-names> </name><name name-style="western"><surname>Busis</surname><given-names>NA</given-names> </name><name name-style="western"><surname>Cohen</surname><given-names>BH</given-names> </name><etal/></person-group><article-title>American Academy of Neurology telehealth position statement</article-title><source>Neurology (ECronicon)</source><year>2021</year><month>08</month><day>17</day><volume>97</volume><issue>7</issue><fpage>334</fpage><lpage>339</lpage><pub-id pub-id-type="doi">10.1212/WNL.0000000000012185</pub-id><pub-id pub-id-type="medline">33986141</pub-id></nlm-citation></ref><ref id="ref9"><label>9</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Bove</surname><given-names>R</given-names> </name><name name-style="western"><surname>Poole</surname><given-names>S</given-names> </name><name name-style="western"><surname>Cuneo</surname><given-names>R</given-names> </name><etal/></person-group><article-title>Remote observational research for multiple sclerosis: a natural experiment</article-title><source>Neurol Neuroimmunol Neuroinflamm</source><year>2023</year><month>03</month><volume>10</volume><issue>2</issue><fpage>e200070</fpage><pub-id pub-id-type="doi">10.1212/NXI.0000000000200070</pub-id><pub-id pub-id-type="medline">36585249</pub-id></nlm-citation></ref><ref id="ref10"><label>10</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>McGinley</surname><given-names>M</given-names> </name><name name-style="western"><surname>Carlson</surname><given-names>JJ</given-names> </name><name name-style="western"><surname>Reihm</surname><given-names>J</given-names> </name><etal/></person-group><article-title>VIRTUAL versus usual in-office care for multiple sclerosis: the VIRTUAL-MS multi-site randomized clinical trial study protocol</article-title><source>Contemp Clin Trials</source><year>2024</year><month>07</month><volume>142</volume><fpage>107544</fpage><pub-id pub-id-type="doi">10.1016/j.cct.2024.107544</pub-id><pub-id pub-id-type="medline">38657731</pub-id></nlm-citation></ref><ref id="ref11"><label>11</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Marziniak</surname><given-names>M</given-names> </name><name name-style="western"><surname>Brichetto</surname><given-names>G</given-names> </name><name name-style="western"><surname>Feys</surname><given-names>P</given-names> </name><name name-style="western"><surname>Meyding-Lamad&#x00E9;</surname><given-names>U</given-names> </name><name name-style="western"><surname>Vernon</surname><given-names>K</given-names> </name><name name-style="western"><surname>Meuth</surname><given-names>SG</given-names> </name></person-group><article-title>The use of digital and remote communication technologies as a tool for multiple sclerosis management: narrative review</article-title><source>JMIR Rehabil Assist Technol</source><year>2018</year><month>04</month><day>24</day><volume>5</volume><issue>1</issue><fpage>e5</fpage><pub-id pub-id-type="doi">10.2196/rehab.7805</pub-id><pub-id pub-id-type="medline">29691208</pub-id></nlm-citation></ref><ref id="ref12"><label>12</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Giovannoni</surname><given-names>G</given-names> </name><name name-style="western"><surname>Butzkueven</surname><given-names>H</given-names> </name><name name-style="western"><surname>Dhib-Jalbut</surname><given-names>S</given-names> </name><etal/></person-group><article-title>Brain health: time matters in multiple sclerosis</article-title><source>Mult Scler Relat Disord</source><year>2016</year><month>09</month><volume>9 Suppl 1</volume><fpage>S5</fpage><lpage>S48</lpage><pub-id pub-id-type="doi">10.1016/j.msard.2016.07.003</pub-id><pub-id pub-id-type="medline">27640924</pub-id></nlm-citation></ref><ref id="ref13"><label>13</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Giovannoni</surname><given-names>G</given-names> </name></person-group><article-title>Disease-modifying treatments for early and advanced multiple sclerosis: a new treatment paradigm</article-title><source>Curr Opin Neurol</source><year>2018</year><month>06</month><volume>31</volume><issue>3</issue><fpage>233</fpage><lpage>243</lpage><pub-id pub-id-type="doi">10.1097/WCO.0000000000000561</pub-id><pub-id pub-id-type="medline">29634596</pub-id></nlm-citation></ref><ref id="ref14"><label>14</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><collab>University of California, San Francisco MS-EPIC Team</collab><name name-style="western"><surname>Cree</surname><given-names>BAC</given-names> </name><name name-style="western"><surname>Hollenbach</surname><given-names>JA</given-names> </name><etal/></person-group><article-title>Silent progression in disease activity-free relapsing multiple sclerosis</article-title><source>Ann Neurol</source><year>2019</year><month>05</month><volume>85</volume><issue>5</issue><fpage>653</fpage><lpage>666</lpage><pub-id pub-id-type="doi">10.1002/ana.25463</pub-id><pub-id pub-id-type="medline">30851128</pub-id></nlm-citation></ref><ref id="ref15"><label>15</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Bove</surname><given-names>R</given-names> </name><name name-style="western"><surname>White</surname><given-names>CC</given-names> </name><name name-style="western"><surname>Giovannoni</surname><given-names>G</given-names> </name><etal/></person-group><article-title>Evaluating more naturalistic outcome measures: a 1-year smartphone study in multiple sclerosis</article-title><source>Neurol Neuroimmunol Neuroinflamm</source><year>2015</year><month>12</month><volume>2</volume><issue>6</issue><fpage>e162</fpage><pub-id pub-id-type="doi">10.1212/NXI.0000000000000162</pub-id><pub-id pub-id-type="medline">26516627</pub-id></nlm-citation></ref><ref id="ref16"><label>16</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Boukhvalova</surname><given-names>AK</given-names> </name><name name-style="western"><surname>Kowalczyk</surname><given-names>E</given-names> </name><name name-style="western"><surname>Harris</surname><given-names>T</given-names> </name><etal/></person-group><article-title>Identifying and quantifying neurological disability via smartphone</article-title><source>Front Neurol</source><year>2018</year><volume>9</volume><fpage>740</fpage><pub-id pub-id-type="doi">10.3389/fneur.2018.00740</pub-id><pub-id pub-id-type="medline">30233487</pub-id></nlm-citation></ref><ref id="ref17"><label>17</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Quinn</surname><given-names>CC</given-names> </name><name name-style="western"><surname>Clough</surname><given-names>SS</given-names> </name><name name-style="western"><surname>Minor</surname><given-names>JM</given-names> </name><name name-style="western"><surname>Lender</surname><given-names>D</given-names> </name><name name-style="western"><surname>Okafor</surname><given-names>MC</given-names> </name><name name-style="western"><surname>Gruber-Baldini</surname><given-names>A</given-names> </name></person-group><article-title>WellDoc mobile diabetes management randomized controlled trial: change in clinical and behavioral outcomes and patient and physician satisfaction</article-title><source>Diabetes Technol Ther</source><year>2008</year><month>06</month><volume>10</volume><issue>3</issue><fpage>160</fpage><lpage>168</lpage><pub-id pub-id-type="doi">10.1089/dia.2008.0283</pub-id><pub-id pub-id-type="medline">18473689</pub-id></nlm-citation></ref><ref id="ref18"><label>18</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Bauer</surname><given-names>AM</given-names> </name><name name-style="western"><surname>Rue</surname><given-names>T</given-names> </name><name name-style="western"><surname>Munson</surname><given-names>SA</given-names> </name><etal/></person-group><article-title>Patient-oriented health technologies: patients&#x2019; perspectives and use</article-title><source>J Mob Technol Med</source><year>2017</year><month>08</month><volume>6</volume><issue>2</issue><fpage>1</fpage><lpage>10</lpage><pub-id pub-id-type="doi">10.7309/jmtm.6.2.1</pub-id><pub-id pub-id-type="medline">28936236</pub-id></nlm-citation></ref><ref id="ref19"><label>19</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Colligan</surname><given-names>E</given-names> </name><name name-style="western"><surname>Metzler</surname><given-names>A</given-names> </name><name name-style="western"><surname>Tiryaki</surname><given-names>E</given-names> </name></person-group><article-title>Shared decision-making in multiple sclerosis</article-title><source>Mult Scler</source><year>2017</year><month>02</month><volume>23</volume><issue>2</issue><fpage>185</fpage><lpage>190</lpage><pub-id pub-id-type="doi">10.1177/1352458516671204</pub-id><pub-id pub-id-type="medline">27663871</pub-id></nlm-citation></ref><ref id="ref20"><label>20</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Guo</surname><given-names>C</given-names> </name><name name-style="western"><surname>Ashrafian</surname><given-names>H</given-names> </name><name name-style="western"><surname>Ghafur</surname><given-names>S</given-names> </name><name name-style="western"><surname>Fontana</surname><given-names>G</given-names> </name><name name-style="western"><surname>Gardner</surname><given-names>C</given-names> </name><name name-style="western"><surname>Prime</surname><given-names>M</given-names> </name></person-group><article-title>Challenges for the evaluation of digital health solutions&#x2014;a call for innovative evidence generation approaches</article-title><source>NPJ Digit Med</source><year>2020</year><volume>3</volume><fpage>110</fpage><pub-id pub-id-type="doi">10.1038/s41746-020-00314-2</pub-id><pub-id pub-id-type="medline">32904379</pub-id></nlm-citation></ref><ref id="ref21"><label>21</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Al-Elq</surname><given-names>AH</given-names> </name></person-group><article-title>Simulation-based medical teaching and learning</article-title><source>J Family Community Med</source><year>2010</year><month>01</month><volume>17</volume><issue>1</issue><fpage>35</fpage><lpage>40</lpage><pub-id pub-id-type="doi">10.4103/1319-1683.68787</pub-id><pub-id pub-id-type="medline">22022669</pub-id></nlm-citation></ref><ref id="ref22"><label>22</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Cheng</surname><given-names>A</given-names> </name><name name-style="western"><surname>Auerbach</surname><given-names>M</given-names> </name><name name-style="western"><surname>Hunt</surname><given-names>EA</given-names> </name><etal/></person-group><article-title>Designing and conducting simulation-based research</article-title><source>Pediatrics</source><year>2014</year><month>06</month><volume>133</volume><issue>6</issue><fpage>1091</fpage><lpage>1101</lpage><pub-id pub-id-type="doi">10.1542/peds.2013-3267</pub-id><pub-id pub-id-type="medline">24819576</pub-id></nlm-citation></ref><ref id="ref23"><label>23</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Bove</surname><given-names>R</given-names> </name><name name-style="western"><surname>Bevan</surname><given-names>C</given-names> </name><name name-style="western"><surname>Crabtree</surname><given-names>E</given-names> </name><etal/></person-group><article-title>Toward a low-cost, in-home, telemedicine-enabled assessment of disability in multiple sclerosis</article-title><source>Mult Scler</source><year>2019</year><month>10</month><volume>25</volume><issue>11</issue><fpage>1526</fpage><lpage>1534</lpage><pub-id pub-id-type="doi">10.1177/1352458518793527</pub-id><pub-id pub-id-type="medline">30141729</pub-id></nlm-citation></ref><ref id="ref24"><label>24</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Graves</surname><given-names>JS</given-names> </name><name name-style="western"><surname>Elantkowski</surname><given-names>M</given-names> </name><name name-style="western"><surname>Zhang</surname><given-names>YP</given-names> </name><etal/></person-group><article-title>Assessment of upper extremity function in multiple sclerosis: feasibility of a digital pinching test</article-title><source>JMIR Form Res</source><year>2023</year><month>10</month><day>2</day><volume>7</volume><fpage>e46521</fpage><pub-id pub-id-type="doi">10.2196/46521</pub-id><pub-id pub-id-type="medline">37782540</pub-id></nlm-citation></ref><ref id="ref25"><label>25</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Dondelinger</surname><given-names>F</given-names> </name><name name-style="western"><surname>Thomann</surname><given-names>AE</given-names> </name><name name-style="western"><surname>Volkova-Volkmar</surname><given-names>E</given-names> </name><name name-style="western"><surname>Lipsmeier</surname><given-names>F</given-names> </name><name name-style="western"><surname>Sprengel</surname><given-names>J</given-names> </name><name name-style="western"><surname>McGinley</surname><given-names>M</given-names> </name><etal/></person-group><article-title>A digital remote assessment for measuring impairment in information processing speed in people with MS</article-title><source>Mult Scler</source><year>2021</year><volume>27</volume><issue>Suppl</issue><fpage>P303</fpage><pub-id pub-id-type="doi">10.1177/13524585211044667</pub-id></nlm-citation></ref><ref id="ref26"><label>26</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Creagh</surname><given-names>AP</given-names> </name><name name-style="western"><surname>Simillion</surname><given-names>C</given-names> </name><name name-style="western"><surname>Scotland</surname><given-names>A</given-names> </name><etal/></person-group><article-title>Smartphone-based remote assessment of upper extremity function for multiple sclerosis using the Draw a Shape Test</article-title><source>Physiol Meas</source><year>2020</year><month>06</month><day>19</day><volume>41</volume><issue>5</issue><fpage>054002</fpage><pub-id pub-id-type="doi">10.1088/1361-6579/ab8771</pub-id><pub-id pub-id-type="medline">32259798</pub-id></nlm-citation></ref><ref id="ref27"><label>27</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Cheng</surname><given-names>WY</given-names> </name><name name-style="western"><surname>Bourke</surname><given-names>AK</given-names> </name><name name-style="western"><surname>Lipsmeier</surname><given-names>F</given-names> </name><etal/></person-group><article-title>U-turn speed is a valid and reliable smartphone-based measure of multiple sclerosis-related gait and balance impairment</article-title><source>Gait Posture</source><year>2021</year><month>02</month><volume>84</volume><fpage>120</fpage><lpage>126</lpage><pub-id pub-id-type="doi">10.1016/j.gaitpost.2020.11.025</pub-id><pub-id pub-id-type="medline">33310432</pub-id></nlm-citation></ref><ref id="ref28"><label>28</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Bourke</surname><given-names>AK</given-names> </name><name name-style="western"><surname>Scotland</surname><given-names>A</given-names> </name><name name-style="western"><surname>Lipsmeier</surname><given-names>F</given-names> </name><name name-style="western"><surname>Gossens</surname><given-names>C</given-names> </name><name name-style="western"><surname>Lindemann</surname><given-names>M</given-names> </name></person-group><article-title>Gait characteristics harvested during a smartphone-based self-administered 2-minute walk test in people with multiple sclerosis: test-retest reliability and minimum detectable change</article-title><source>Sensors (Basel)</source><year>2020</year><month>10</month><day>19</day><volume>20</volume><issue>20</issue><fpage>5906</fpage><pub-id pub-id-type="doi">10.3390/s20205906</pub-id><pub-id pub-id-type="medline">33086734</pub-id></nlm-citation></ref><ref id="ref29"><label>29</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Creagh</surname><given-names>AP</given-names> </name><name name-style="western"><surname>Simillion</surname><given-names>C</given-names> </name><name name-style="western"><surname>Bourke</surname><given-names>AK</given-names> </name><etal/></person-group><article-title>Smartphone- and smartwatch-based remote characterisation of ambulation in multiple sclerosis during the two-minute walk test</article-title><source>IEEE J Biomed Health Inform</source><year>2021</year><month>03</month><volume>25</volume><issue>3</issue><fpage>838</fpage><lpage>849</lpage><pub-id pub-id-type="doi">10.1109/JBHI.2020.2998187</pub-id><pub-id pub-id-type="medline">32750915</pub-id></nlm-citation></ref><ref id="ref30"><label>30</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Midaglia</surname><given-names>L</given-names> </name><name name-style="western"><surname>Mulero</surname><given-names>P</given-names> </name><name name-style="western"><surname>Montalban</surname><given-names>X</given-names> </name><etal/></person-group><article-title>Adherence and satisfaction of smartphone- and smartwatch-based remote active testing and passive monitoring in people with multiple sclerosis: nonrandomized interventional feasibility study</article-title><source>J Med Internet Res</source><year>2019</year><month>08</month><day>30</day><volume>21</volume><issue>8</issue><fpage>e14863</fpage><pub-id pub-id-type="doi">10.2196/14863</pub-id><pub-id pub-id-type="medline">31471961</pub-id></nlm-citation></ref><ref id="ref31"><label>31</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Lakin</surname><given-names>L</given-names> </name><name name-style="western"><surname>Davis</surname><given-names>BE</given-names> </name><name name-style="western"><surname>Binns</surname><given-names>CC</given-names> </name><name name-style="western"><surname>Currie</surname><given-names>KM</given-names> </name><name name-style="western"><surname>Rensel</surname><given-names>MR</given-names> </name></person-group><article-title>Comprehensive approach to management of multiple sclerosis: addressing invisible symptoms&#x2014;a narrative review</article-title><source>Neurol Ther</source><year>2021</year><month>06</month><volume>10</volume><issue>1</issue><fpage>75</fpage><lpage>98</lpage><pub-id pub-id-type="doi">10.1007/s40120-021-00239-2</pub-id><pub-id pub-id-type="medline">33877583</pub-id></nlm-citation></ref><ref id="ref32"><label>32</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Hyzy</surname><given-names>M</given-names> </name><name name-style="western"><surname>Bond</surname><given-names>R</given-names> </name><name name-style="western"><surname>Mulvenna</surname><given-names>M</given-names> </name><etal/></person-group><article-title>System usability scale benchmarking for digital health apps: meta-analysis</article-title><source>JMIR Mhealth Uhealth</source><year>2022</year><month>08</month><day>18</day><volume>10</volume><issue>8</issue><fpage>e37290</fpage><pub-id pub-id-type="doi">10.2196/37290</pub-id><pub-id pub-id-type="medline">35980732</pub-id></nlm-citation></ref><ref id="ref33"><label>33</label><nlm-citation citation-type="book"><person-group person-group-type="author"><name name-style="western"><surname>Paas</surname><given-names>F</given-names> </name><name name-style="western"><surname>Ayres</surname><given-names>P</given-names> </name><name name-style="western"><surname>Pachman</surname><given-names>M</given-names> </name></person-group><person-group person-group-type="editor"><name name-style="western"><surname>Robinson</surname><given-names>DH</given-names> </name><name name-style="western"><surname>Schraw</surname><given-names>G</given-names> </name></person-group><article-title>Assessment of cognitive load in multimedia learning: theory, methods and applications</article-title><source>Recent Innovations in Educational Technology That Facilitate Student Learning</source><year>2008</year><access-date>2025-12-02</access-date><publisher-name>Information Age Publishing Inc</publisher-name><fpage>11</fpage><lpage>36</lpage><comment><ext-link ext-link-type="uri" xlink:href="https://researchers.mq.edu.au/en/publications/assessment-of-cognitive-load-in-multimedia-learning-theory-method/">https://researchers.mq.edu.au/en/publications/assessment-of-cognitive-load-in-multimedia-learning-theory-method/</ext-link></comment><pub-id pub-id-type="other">9781593116521</pub-id></nlm-citation></ref><ref id="ref34"><label>34</label><nlm-citation citation-type="web"><person-group person-group-type="author"><name name-style="western"><surname>Luck</surname><given-names>I</given-names> </name></person-group><source>What is a Good Net Promoter Score (NPS)? 4 Approaches</source><year>2023</year><access-date>2024-07-23</access-date><publisher-name>Customer Gauge</publisher-name><comment><ext-link ext-link-type="uri" xlink:href="https://customergauge.com/what-is-a-good-net-promoter-score">https://customergauge.com/what-is-a-good-net-promoter-score</ext-link></comment></nlm-citation></ref><ref id="ref35"><label>35</label><nlm-citation citation-type="web"><article-title>Our commitment to transparency of clinical study information</article-title><source>Roche</source><access-date>2025-11-19</access-date><comment><ext-link ext-link-type="uri" xlink:href="https://go.roche.com/data_sharing">https://go.roche.com/data_sharing</ext-link></comment></nlm-citation></ref><ref id="ref36"><label>36</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Meyer-Moock</surname><given-names>S</given-names> </name><name name-style="western"><surname>Feng</surname><given-names>YS</given-names> </name><name name-style="western"><surname>Maeurer</surname><given-names>M</given-names> </name><name name-style="western"><surname>Dippel</surname><given-names>FW</given-names> </name><name name-style="western"><surname>Kohlmann</surname><given-names>T</given-names> </name></person-group><article-title>Systematic literature review and validity evaluation of the Expanded Disability Status Scale (EDSS) and the Multiple Sclerosis Functional Composite (MSFC) in patients with multiple sclerosis</article-title><source>BMC Neurol</source><year>2014</year><month>03</month><day>25</day><volume>14</volume><fpage>58</fpage><pub-id pub-id-type="doi">10.1186/1471-2377-14-58</pub-id><pub-id pub-id-type="medline">24666846</pub-id></nlm-citation></ref><ref id="ref37"><label>37</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Strober</surname><given-names>L</given-names> </name><name name-style="western"><surname>DeLuca</surname><given-names>J</given-names> </name><name name-style="western"><surname>Benedict</surname><given-names>RH</given-names> </name><etal/></person-group><article-title>Symbol Digit Modalities Test: a valid clinical trial endpoint for measuring cognition in multiple sclerosis</article-title><source>Mult Scler</source><year>2019</year><month>11</month><volume>25</volume><issue>13</issue><fpage>1781</fpage><lpage>1790</lpage><pub-id pub-id-type="doi">10.1177/1352458518808204</pub-id><pub-id pub-id-type="medline">30334474</pub-id></nlm-citation></ref><ref id="ref38"><label>38</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Kaplan</surname><given-names>TB</given-names> </name><name name-style="western"><surname>Gopal</surname><given-names>A</given-names> </name><name name-style="western"><surname>Block</surname><given-names>VJ</given-names> </name><etal/></person-group><article-title>Challenges to longitudinal characterization of lower urinary tract dysfunction in multiple sclerosis</article-title><source>Mult Scler Relat Disord</source><year>2022</year><month>06</month><volume>62</volume><fpage>103793</fpage><pub-id pub-id-type="doi">10.1016/j.msard.2022.103793</pub-id><pub-id pub-id-type="medline">35461057</pub-id></nlm-citation></ref><ref id="ref39"><label>39</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Monachino</surname><given-names>A</given-names> </name><name name-style="western"><surname>Caraher</surname><given-names>C</given-names> </name><name name-style="western"><surname>Ginsberg</surname><given-names>J</given-names> </name><name name-style="western"><surname>Bailey</surname><given-names>C</given-names> </name><name name-style="western"><surname>White</surname><given-names>E</given-names> </name></person-group><article-title>Medical emergencies in the primary care setting: an evidence based practice approach using simulation to improve readiness</article-title><source>J Pediatr Nurs</source><year>2019</year><volume>49</volume><fpage>72</fpage><lpage>78</lpage><pub-id pub-id-type="doi">10.1016/j.pedn.2019.09.017</pub-id><pub-id pub-id-type="medline">31670140</pub-id></nlm-citation></ref><ref id="ref40"><label>40</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Turkelson</surname><given-names>C</given-names> </name><name name-style="western"><surname>Cooper</surname><given-names>D</given-names> </name><name name-style="western"><surname>Creech</surname><given-names>C</given-names> </name><name name-style="western"><surname>Schellenberg</surname><given-names>K</given-names> </name><name name-style="western"><surname>Behnke</surname><given-names>L</given-names> </name></person-group><article-title>Using simulation to enhance nurse practitioner students cultural sensitivity, communication, and empathy with vulnerable populations</article-title><source>Clin Simul Nurs</source><year>2021</year><month>07</month><volume>56</volume><fpage>108</fpage><lpage>116</lpage><pub-id pub-id-type="doi">10.1016/j.ecns.2021.04.011</pub-id></nlm-citation></ref><ref id="ref41"><label>41</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Bove</surname><given-names>R</given-names> </name><name name-style="western"><surname>Bruce</surname><given-names>CA</given-names> </name><name name-style="western"><surname>Lunders</surname><given-names>CK</given-names> </name><etal/></person-group><article-title>Electronic health record technology designed for the clinical encounter: MS NeuroShare</article-title><source>Neurol Clin Pract</source><year>2021</year><month>08</month><volume>11</volume><issue>4</issue><fpage>318</fpage><lpage>326</lpage><pub-id pub-id-type="doi">10.1212/CPJ.0000000000000986</pub-id><pub-id pub-id-type="medline">34484932</pub-id></nlm-citation></ref><ref id="ref42"><label>42</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Gould</surname><given-names>D</given-names> </name><name name-style="western"><surname>Lindstr&#x00F6;m</surname><given-names>H</given-names> </name><name name-style="western"><surname>Purssell</surname><given-names>E</given-names> </name><name name-style="western"><surname>Wigglesworth</surname><given-names>N</given-names> </name></person-group><article-title>Electronic hand hygiene monitoring: accuracy, impact on the Hawthorne effect and efficiency</article-title><source>J Infect Prev</source><year>2020</year><month>07</month><volume>21</volume><issue>4</issue><fpage>136</fpage><lpage>143</lpage><pub-id pub-id-type="doi">10.1177/1757177420907999</pub-id><pub-id pub-id-type="medline">32655694</pub-id></nlm-citation></ref><ref id="ref43"><label>43</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Sujatha</surname><given-names>BK</given-names> </name><name name-style="western"><surname>Reddy</surname><given-names>MT</given-names> </name><name name-style="western"><surname>Pathak</surname><given-names>P</given-names> </name></person-group><article-title>Camouflage in research&#x2014;the Hawthorne effect</article-title><source>Int J Dev Res</source><year>2019</year><access-date>2025-12-02</access-date><volume>9</volume><fpage>15710</fpage><comment><ext-link ext-link-type="uri" xlink:href="https://journalijdr.com/camouflage-research-%E2%80%93-hawthorne-effect">https://journalijdr.com/camouflage-research-%E2%80%93-hawthorne-effect</ext-link></comment></nlm-citation></ref></ref-list></back></article>