<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMI</journal-id>
      <journal-id journal-id-type="nlm-ta">JMIR Med Inform</journal-id>
      <journal-title>JMIR Medical Informatics</journal-title>
      <issn pub-type="epub">2291-9694</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v9i8e23219</article-id>
      <article-id pub-id-type="pmid">34459741</article-id>
      <article-id pub-id-type="doi">10.2196/23219</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Viewpoint</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Viewpoint</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>A System to Support Diverse Social Program Management</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Lovis</surname>
            <given-names>Christian</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Chiou</surname>
            <given-names>Erin</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Mircheva</surname>
            <given-names>Iskra</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" corresp="yes">
          <name name-style="western">
            <surname>McKillop</surname>
            <given-names>Mollie</given-names>
          </name>
          <degrees>MPH, PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <address>
            <institution>IBM Watson Health</institution>
            <addr-line>75 Binney Street</addr-line>
            <addr-line>Cambridge, MA, 02142</addr-line>
            <country>United States</country>
            <phone>1 3322073519</phone>
            <email>mollie.mckillop@ibm.com</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-4805-5989</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author">
          <name name-style="western">
            <surname>Snowdon</surname>
            <given-names>Jane</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-7064-9305</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author">
          <name name-style="western">
            <surname>Willis</surname>
            <given-names>Van C</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-5577-0545</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author">
          <name name-style="western">
            <surname>Alevy</surname>
            <given-names>Shira</given-names>
          </name>
          <degrees>EdM</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-7006-4128</ext-link>
        </contrib>
        <contrib id="contrib5" contrib-type="author">
          <name name-style="western">
            <surname>Rizvi</surname>
            <given-names>Rubina</given-names>
          </name>
          <degrees>MD, PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-9432-3086</ext-link>
        </contrib>
        <contrib id="contrib6" contrib-type="author">
          <name name-style="western">
            <surname>Rewalt</surname>
            <given-names>Karen</given-names>
          </name>
          <degrees>MBA</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-0798-6824</ext-link>
        </contrib>
        <contrib id="contrib7" contrib-type="author">
          <name name-style="western">
            <surname>Lefebvre-Paillé</surname>
            <given-names>Charlyne</given-names>
          </name>
          <degrees>MA</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-8642-6269</ext-link>
        </contrib>
        <contrib id="contrib8" contrib-type="author">
          <name name-style="western">
            <surname>Kassler</surname>
            <given-names>William</given-names>
          </name>
          <degrees>MPH, MD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-0238-0898</ext-link>
        </contrib>
        <contrib id="contrib9" contrib-type="author">
          <name name-style="western">
            <surname>Purcell Jackson</surname>
            <given-names>Gretchen</given-names>
          </name>
          <degrees>MD, PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-3242-8058</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>IBM Watson Health</institution>
        <addr-line>Cambridge, MA</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Vanderbilt University Medical Center</institution>
        <addr-line>Nashville, TN</addr-line>
        <country>United States</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Mollie McKillop <email>mollie.mckillop@ibm.com</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <month>8</month>
        <year>2021</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>30</day>
        <month>8</month>
        <year>2021</year>
      </pub-date>
      <volume>9</volume>
      <issue>8</issue>
      <elocation-id>e23219</elocation-id>
      <history>
        <date date-type="received">
          <day>5</day>
          <month>8</month>
          <year>2020</year>
        </date>
        <date date-type="rev-request">
          <day>26</day>
          <month>10</month>
          <year>2020</year>
        </date>
        <date date-type="rev-recd">
          <day>8</day>
          <month>12</month>
          <year>2020</year>
        </date>
        <date date-type="accepted">
          <day>5</day>
          <month>6</month>
          <year>2021</year>
        </date>
      </history>
      <copyright-statement>©Mollie McKillop, Jane Snowdon, Van C Willis, Shira Alevy, Rubina Rizvi, Karen Rewalt, Charlyne Lefebvre-Paillé, William Kassler, Gretchen Purcell Jackson. Originally published in JMIR Medical Informatics (https://medinform.jmir.org), 30.08.2021.</copyright-statement>
      <copyright-year>2021</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Medical Informatics, is properly cited. The complete bibliographic information, a link to the original publication on https://medinform.jmir.org/, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://medinform.jmir.org/2021/8/e23219" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>Social programs are services provided by governments, nonprofits, and other organizations to help improve the health and well-being of individuals, families, and communities. Social programs aim to deliver services effectively and efficiently, but they are challenged by information silos, limited resources, and the need to deliver frequently changing mandated benefits.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>We aim to explore how an information system designed for social programs helps deliver services effectively and efficiently across diverse programs.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>This viewpoint describes the configurable and modular architecture of Social Program Management (SPM), a system to support efficient and effective delivery of services through a wide range of social programs and lessons learned from implementing SPM across diverse settings. We explored usage data to inform the engagement and impact of SPM on the efficient and effective delivery of services.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>The features and functionalities of SPM seem to support the goals of social programs. We found that SPM provides fundamental management processes and configurable program-specific components to support social program administration; has been used by more than 280,000 caseworkers serving more than 30 million people in 13 countries; contains features designed to meet specific user requirements; supports secure information sharing and collaboration through data standardization and aggregation; and offers configurability and flexibility, which are important for digital transformation and organizational change.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>SPM is a user-centered, configurable, and flexible system for managing social program workflows.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>other clinical informatics applications</kwd>
        <kwd>process management tools</kwd>
        <kwd>requirements analysis and design</kwd>
        <kwd>consumer health informatics</kwd>
        <kwd>public health</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <p>Government and community-based organizations are responsible for delivering social services to clients through social programs provided at the national, state, county, and city levels. Social programs are critical to the health and welfare of many citizens, as they provide a wide range of benefits [<xref ref-type="bibr" rid="ref1">1</xref>], such as (1) health and human services for health insurance, prevention services, child welfare, and nutrition assistance; (2) workforce services for unemployment insurance programs and job training; and (3) social security programs offering income support and benefits [<xref ref-type="bibr" rid="ref2">2</xref>].</p>
      <p>In contrast to commercial entities, government agencies administering social programs face unique challenges regarding service delivery and their operational processes, including (1) information silos that limit decision-making abilities, (2) requirements to balance privacy with data sharing and transparent use of public funds, (3) reduced financial resources but growing demand for services, (4) legislative and organizational influences on eligibility and entitlement, and (5) the need to document the delivery of mandated services for beneficiaries across multiple categorical programs [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref4">4</xref>]. These challenges are exacerbated by societal shifts, such as increasing income inequality, aging populations, unemployment, ongoing changes in government policies, and resource constraints [<xref ref-type="bibr" rid="ref5">5</xref>-<xref ref-type="bibr" rid="ref7">7</xref>].</p>
      <p>Opportunities exist for information technology to address these challenges by improving the efficiency and transparency of social program workflows and enabling collaboration among stakeholders. These goals can be achieved by centralizing critical data, streamlining eligibility determination and case management, and improving communication within and across organizations [<xref ref-type="bibr" rid="ref8">8</xref>]. Previous preliminary research indicates that the use of information systems to support high-quality and efficient service delivery is promising but limited in scope and does not meet the unique needs of social programs [<xref ref-type="bibr" rid="ref8">8</xref>-<xref ref-type="bibr" rid="ref10">10</xref>]. Comprehensive systems specifically designed for social programs have not been previously described in the literature.</p>
      <p>This viewpoint describes the design, functionality, and selected applications of a software solution for social services, which combines domain-specific business processes with a flexible open architecture to allow for needed configurability while standardizing data elements. Specific design principles that aim to address the unique challenges encountered by social programs, along with examples of implementation and usage, are described.</p>
      <p>Our system was designed and developed by subject matter experts in social programs, including industry specialists, service professionals experienced in social programs, and social program product developers. The system was developed for social service and human service agencies to advance digital transformation, intending to support a wide range of constituents across the health and human services enterprise. Potential users include, but are not limited to, Medicaid program managers, directors, analysts, caseworkers and care managers, clients, and beneficiaries. The system aims to prioritize the needs of users and beneficiaries to unify multidisciplinary teams. User-centered research methods, including user shadowing, interviews, surveys, and scenario testing, were used to identify user needs, and human-centered design leveraging iterative co-development and prototyping were used to develop the system.</p>
    </sec>
    <sec>
      <title>System Scope</title>
      <p>Social Program Management (SPM) supports two basic types of social programs: (1) programs in which eligibility is determined primarily based on need and (2) programs where eligibility for benefits and services is determined based on previous contributions [<xref ref-type="bibr" rid="ref11">11</xref>,<xref ref-type="bibr" rid="ref12">12</xref>]. SPM also supports care and protection programs, such as child welfare programs, where eligibility is based on practice models and assessments; <xref rid="figure1" ref-type="fig">Figure 1</xref> describes the full scope and scale of service organizations that SPM supports.</p>
      <fig id="figure1" position="float">
        <label>Figure 1</label>
        <caption>
          <p>Scope of services supported by Social Program Management. Social Program Management serves health and social service organizations at all levels of government and nonprofit organizations.</p>
        </caption>
        <graphic xlink:href="medinform_v9i8e23219_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
      </fig>
      <p>These diverse social programs share similar goals and challenges with the aim of improving service delivery to and outcomes for beneficiaries. To meet these goals, social programs require systems that (1) support complex eligibility and entitlement, (2) provide beneficiaries with easy access to services, (3) enable efficient management of high case volumes, (4) provide decision support and knowledge management tools, (5) allow flexibility for changes in policies and processes, and (6) reduce the potential for fraud and abuse. These needs must be met throughout triage, initial contract and registration, determination of eligibility for benefits, service planning and delivery, and outcome evaluation. Delivering services across these program stages relies on secure data gathering, documentation, retrieval, validation, auditing, and analysis. SPM has features and capabilities that address each of these needs (<xref ref-type="table" rid="table1">Table 1</xref>) [<xref ref-type="bibr" rid="ref13">13</xref>].</p>
      <table-wrap position="float" id="table1">
        <label>Table 1</label>
        <caption>
          <p>Features and capabilities to address needs.</p>
        </caption>
        <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
          <col width="30"/>
          <col width="380"/>
          <col width="0"/>
          <col width="590"/>
          <thead>
            <tr valign="top">
              <td colspan="3">Challenge and need</td>
              <td>Features and functionalities</td>
            </tr>
          </thead>
          <tbody>
            <tr valign="top">
              <td colspan="4">
                <bold>Information and programs are managed in silos</bold>
              </td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Integrate service delivery</td>
              <td colspan="2">
                <list list-type="bullet">
                  <list-item>
                    <p>Centralized repository for the integrated management of cases maintained in the system and external systems</p>
                  </list-item>
                  <list-item>
                    <p>Multidisciplinary team portal provides a means for cross-agency and cross-program teams to collaborate on cases</p>
                  </list-item>
                  <list-item>
                    <p>Indexing on cases and participants provides master data management, which allows for identifying individuals across systems, especially web services</p>
                  </list-item>
                  <list-item>
                    <p>Citizen portal and multichannel access allows clients to check on status, submit applications, and manage benefits on agency websites</p>
                  </list-item>
                </list>
              </td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Consolidate infrastructure</td>
              <td colspan="2">
                <list list-type="bullet">
                  <list-item>
                    <p>The system data model supports needs-based and contribution-based programs</p>
                  </list-item>
                  <list-item>
                    <p>Program-specific modules for social security, health and human services, and workforce services based on a common data model</p>
                  </list-item>
                </list>
              </td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Provide standards-based integration</td>
              <td colspan="2">
                <list list-type="bullet">
                  <list-item>
                    <p>The system data model can be deployed as web services</p>
                  </list-item>
                  <list-item>
                    <p>Preconfigured adapters and enterprise application integration connectors to facilitate integration with existing systems</p>
                  </list-item>
                </list>
              </td>
            </tr>
            <tr valign="top">
              <td colspan="4">
                <bold>Data protection and privacy</bold>
              </td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Prevent unauthorized access to sensitive data</td>
              <td colspan="2">
                <list list-type="bullet">
                  <list-item>
                    <p>Role-based and data field level security for personal and case data</p>
                  </list-item>
                </list>
              </td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Protect the integrity and privacy of personal data while sharing data responsibly</td>
              <td colspan="2">
                <list list-type="bullet">
                  <list-item>
                    <p>Configurable security levels to prevent unauthorized access to data while allowing multiple stakeholders to view a client’s data with the appropriate level of access</p>
                  </list-item>
                  <list-item>
                    <p>Auditing and tracking of transactions involving sensitive data</p>
                  </list-item>
                  <list-item>
                    <p>Auditing and traceability for logging time, date, and the user responsible for any read, update, and delete actions for any specified participant or case data elements</p>
                  </list-item>
                </list>
              </td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Reduce inaccurate or duplicate data</td>
              <td colspan="2">
                <list list-type="bullet">
                  <list-item>
                    <p>Integrated case management module supports centralized or distributed maintenance and sharing of case and participant data across programs</p>
                  </list-item>
                </list>
              </td>
            </tr>
            <tr valign="top">
              <td colspan="4">
                <bold>Shrinking budgets</bold>
              </td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Provide standard programs and eligibility and entitlement rules with opportunities for customization</td>
              <td colspan="2">
                <list list-type="bullet">
                  <list-item>
                    <p>Eligibility and entitlement rules for income support and assessments for child welfare programs</p>
                  </list-item>
                  <list-item>
                    <p>Configurable, packaged connectors and adapters for integration to existing and service-oriented applications</p>
                  </list-item>
                </list>
              </td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Leverage and consolidate cost-effective, existing infrastructure to deploy new program solutions</td>
              <td colspan="2">
                <list list-type="bullet">
                  <list-item>
                    <p>Support for open standards and de facto standards to ensure deployment on the widest possible range of operating systems, hardware platforms, and middleware</p>
                  </list-item>
                </list>
              </td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Provide tools to allow incremental approaches to implementation</td>
              <td colspan="2">
                <list list-type="bullet">
                  <list-item>
                    <p>Web services and configurable business processes allow for maximum flexibility in deployment and implementation options</p>
                  </list-item>
                </list>
              </td>
            </tr>
            <tr valign="top">
              <td colspan="4">
                <bold>Increasing demand for services</bold>
              </td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Deliver high performance and scalability</td>
              <td colspan="2">
                <list list-type="bullet">
                  <list-item>
                    <p>Eligibility and entitlement engine optimized for processing and calculating high volumes of rules-based assessments and complex reassessments where information may need to be retroactively changed for large populations of clients</p>
                  </list-item>
                  <list-item>
                    <p>Supervisor workspace for real-time analysis and dynamic allocation of workloads across a department or agency</p>
                  </list-item>
                </list>
              </td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Offer broad access and reliability</td>
              <td colspan="2">
                <list list-type="bullet">
                  <list-item>
                    <p>Multichannel access through a device-independent web-based user interface</p>
                  </list-item>
                  <list-item>
                    <p>Configurable citizen portal for access to cross-program screening and eligibility</p>
                  </list-item>
                </list>
              </td>
            </tr>
            <tr valign="top">
              <td colspan="4">
                <bold>Legislative and organizational change</bold>
              </td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Supply configurable systems that can adapt to legislation without reprogramming and support different organizational structures</td>
              <td colspan="2">
                <list list-type="bullet">
                  <list-item>
                    <p>Configurable eligibility and entitlement engine supporting complex reassessments</p>
                  </list-item>
                  <list-item>
                    <p>Configurable regional administration segregates duties by location or organization</p>
                  </list-item>
                  <list-item>
                    <p>Concurrent execution of reassessment batch jobs with ongoing web-based transactions</p>
                  </list-item>
                </list>
              </td>
            </tr>
            <tr valign="top">
              <td colspan="4">
                <bold>Balancing accuracy, consistency, and outcome focus</bold>
              </td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Comply with legislation in the delivery of benefits across large populations while also individualizing services for clients and families</td>
              <td colspan="2">
                <list list-type="bullet">
                  <list-item>
                    <p>Integrated service-planning templates based on best practices, such as structured decision-making assessments, which are developed by the National Council on Crime and Delinquency</p>
                  </list-item>
                  <list-item>
                    <p>Intelligent evidence gathering module scripts and templates for consistent, structured capture of information for caseworkers and beneficiaries</p>
                  </list-item>
                  <list-item>
                    <p>Decision assist module, a configurable rules-based matrix designed to assure consistency and accuracy in rendering decisions</p>
                  </list-item>
                  <list-item>
                    <p>The social enterprise collaboration module provides a common platform and set of tools for multidisciplinary collaboration in social program organizations</p>
                  </list-item>
                </list>
              </td>
            </tr>
          </tbody>
        </table>
      </table-wrap>
    </sec>
    <sec>
      <title>System Architecture</title>
      <sec>
        <title>Overview</title>
        <p>The features and functionalities described in <xref ref-type="table" rid="table1">Table 1</xref> are organized within a single user-centered system comprising modules, the SPM data model, administration application, and business and technical services. The architecture for SPM version 7.0.9, the latest version, is outlined in <xref rid="figure2" ref-type="fig">Figure 2</xref>, and the modules and applications of the SPM are described in the following sections (see <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref> for technical aspects of SPM).</p>
        <fig id="figure2" position="float">
          <label>Figure 2</label>
          <caption>
            <p>Social Program Management design. Social Program Management provides core processing and infrastructure components for social program management through its platform. Business services provide support for management needs common across all types of organizations. Modules complement the system platform and provide additional functionality and system configurability.</p>
          </caption>
          <graphic xlink:href="medinform_v9i8e23219_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>SPM Modules</title>
        <p>The SPM modules presented in <xref ref-type="table" rid="table2">Table 2</xref> support repeatable processes that are common across programs. These processes include managing client information and data regarding benefits, automatic assessment of eligibility and entitlement, management of tasks, communication, and scheduling. Each module is supported by the SPM platform, including the data model and administrative, business, and technical services. The SPM modularization supports the incremental modernization of systems used by social programs. As the data model is application-agnostic, the integration of future modules or functionalities is supported. These modules support a comprehensive range of functionalities required for service delivery in social programs. Modules are divided by those that exist for all implementations of SPM (enterprise-wide applications) and those specific to a particular implementation and user agreement (implementation-specific modules).</p>
        <table-wrap position="float" id="table2">
          <label>Table 2</label>
          <caption>
            <p>Social Program Management modules and applications.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="200"/>
            <col width="190"/>
            <col width="610"/>
            <thead>
              <tr valign="top">
                <td>Name</td>
                <td>Type</td>
                <td>Description</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Intake</td>
                <td>Enterprise-wide</td>
                <td>The intake module facilitates integration with external systems that manage the client. The intake module notifies an external system when an intake is approved. The external system can then retrieve the details of the intake for further processing.</td>
              </tr>
              <tr valign="top">
                <td>Participant management</td>
                <td>Enterprise-wide</td>
                <td>A participant is the system term for any individual or organization about which a social enterprise wants to record information. Participant management provides for the creation and maintenance of all relevant basic information such as contact details, addresses, communications, demographic information, and alternate names and identifiers.</td>
              </tr>
              <tr valign="top">
                <td>Integrated case management</td>
                <td>Enterprise-wide</td>
                <td>Integrated case management provides functionality to facilitate the creation, management, and tracking of cases in support of social program service delivery. Interactions between participants and the agency and any associated program or service delivery are recorded. These interactions include assessment, eligibility determination, case approval, program delivery, outcome evaluation, and closure.</td>
              </tr>
              <tr valign="top">
                <td>Evidence management</td>
                <td>Enterprise-wide</td>
                <td>Evidence is any data collected in support of a case. In general, such information is program-specific, although various types of evidence may be shared through a number of programs. Typically, the primary use of evidence is for the determination of program eligibility and entitlement. Evidence management provides capabilities to standardize and simplify the process of defining, creating, and maintaining such program-specific, temporal data.</td>
              </tr>
              <tr valign="top">
                <td>Eligibility and entitlement</td>
                <td>Enterprise-wide</td>
                <td>The rules for determining both eligibility and entitlement are typically dictated by a combination of legislation, policy, and operating procedure. For many programs, such rules are determined by federal, state, or local governments. These rules often change. The system provides two mechanisms to help social enterprises deal with the problem of changing evidence:<break/><list list-type="bullet"><list-item><p>The ability to detect the change and the ability to initiate a reassessment where required</p></list-item><list-item><p>Overpayment and underpayment processing where the system compares the old and new situations automatically detects any overpayments or underpayments and initiates appropriate action</p></list-item></list></td>
              </tr>
              <tr valign="top">
                <td>Financial management</td>
                <td>Enterprise-wide</td>
                <td>Financial management manages and tracks the financial transactions associated with program delivery, including benefit payments and liability recovery. Financial management generates, manages, and tracks the financial transactions associated with cases and participants. It also supports the issue of payments and the creation of liabilities as determined by assessment and entitlement processing.</td>
              </tr>
              <tr valign="top">
                <td>Funded program management</td>
                <td>Enterprise-wide</td>
                <td>Funded program management is used to manage funds that can be obligated to clients in need of assistance or to provide payment to providers for services.</td>
              </tr>
              <tr valign="top">
                <td>Supervisor workspace</td>
                <td>Enterprise-wide</td>
                <td>Supervisor workspace provides dashboard-style views of a social program’s workload. It allows managers to monitor workloads through supervisor dashboard views of staff assignments, real-time display and access to information, centralized management of cases and tasks, prioritization, and allocation of workloads.</td>
              </tr>
              <tr valign="top">
                <td>Income support</td>
                <td>Implementation-specific modules</td>
                <td>Income support delivers health and social program components, business processes, toolsets, and interfaces on a dynamically configurable architecture that allows an administrator to change rules without writing code. Income support is designed for programs that provide food, cash, and medical assistance.</td>
              </tr>
              <tr valign="top">
                <td>Child welfare</td>
                <td>Implementation-specific modules</td>
                <td>Child welfare provides case management tools that support agencies that work to safeguard children, promote well-being, and support child permanency. Child services facilitate intake, ongoing case management, child abuse investigations, removal of children from unsafe situations, and the adoption of children.</td>
              </tr>
              <tr valign="top">
                <td>Life event management</td>
                <td>Implementation-specific modules</td>
                <td>Life event management helps the caseworker to collect evidence and provide guidance that is based on a client’s life event, such as the birth of a child, marriage, divorce, or change in employment.</td>
              </tr>
              <tr valign="top">
                <td>Verification</td>
                <td>Implementation-specific modules</td>
                <td>The verification engine streamlines the process of verifying evidence that is used in determining eligibility and entitlement as part of program delivery. It provides the functions that are needed for efficient management of verifications where policy or legislation mandates that evidence is verified as a prerequisite for eligibility.</td>
              </tr>
              <tr valign="top">
                <td>Business intelligence and analytics</td>
                <td>Implementation-specific modules</td>
                <td>Business intelligence and analytics is a decision support solution that helps social program organizations analyze the effectiveness of their programs and gain insight into the efficiency of their operations. It is scalable from the program to enterprise level. It consists of embedded analytics, domain-specific dashboards, extract, transform, and load functions, and tool-independent, predefined, domain-specific (only for social program management) data marts.</td>
              </tr>
              <tr valign="top">
                <td>Provider management</td>
                <td>Implementation-specific modules</td>
                <td>Provider management manages the interactions between the agency and its outside providers, such as foster families, housing facilities, and other vendors.</td>
              </tr>
              <tr valign="top">
                <td>Identity intelligence</td>
                <td>Implementation-specific modules</td>
                <td>Identity intelligence aims to give caseworkers the confidence that an applicant is who they say they are and are not duplicated within the system, which might result in duplicate benefits. The product solves this problem by applying analytics to client data and verifies information with limited worker involvement.</td>
              </tr>
              <tr valign="top">
                <td>Income support for medical assistance</td>
                <td>Implementation-specific modules</td>
                <td>Income support for medical assistance is specifically built to provide business tools and processes for the management of traditional medical assistance programs, plus the Affordable Care Act and modified adjusted gross income-based Medicaid programs.</td>
              </tr>
              <tr valign="top">
                <td>Universal access</td>
                <td>Implementation-specific modules</td>
                <td>Universal access is a fully configurable web-based citizen-facing application that enables agencies to offer a web self-service solution to their clients. Universal access can provide a greater number of clients with access to programs and services by allowing clients to complete key tasks on the web without the assistance of a worker.</td>
              </tr>
              <tr valign="top">
                <td>Appeals</td>
                <td>Implementation-specific modules</td>
                <td>Appeals is an automated solution that provides support for the appeals and fair hearings process. Appeals automates the intake, hearings, and decision processes and manages participants in the appeals process. Appeals supports multilevel appeals in which multiple issues for an appellant and respondent can be viewed at a single appeal hearing.</td>
              </tr>
              <tr valign="top">
                <td>Evidence broker</td>
                <td>Implementation-specific modules</td>
                <td>Evidence broker facilitates flexible data sharing between different case types and between agencies.</td>
              </tr>
              <tr valign="top">
                <td>Outcome management</td>
                <td>Implementation-specific modules</td>
                <td>Outcome management provides organizations that deliver social programs with a framework and automated tools to create and manage outcome plans for clients and their families. Outcome management is designed to help organizations assess needs, establish goals, plan for goal attainment, and track progress.</td>
              </tr>
              <tr valign="top">
                <td>Social enterprise collaboration</td>
                <td>Implementation-specific modules</td>
                <td>Social enterprise collaboration is a common platform and set of tools for multidisciplinary collaboration in social programs. Multidisciplinary teams are involved in supporting the needs of clients and families, including other agencies, local providers, and interested community partners.</td>
              </tr>
              <tr valign="top">
                <td>Archiving</td>
                <td>Implementation-specific modules</td>
                <td>As database size grows, performance can degrade rapidly. A large percentage of data in a social program database is unlikely to be accessed daily. Performance can be greatly improved if infrequently accessed data are removed from the production environment. Archiving stores and maintains inactive data in a repository so that it can be retrieved when necessary.</td>
              </tr>
            </tbody>
          </table>
        </table-wrap>
      </sec>
    </sec>
    <sec>
      <title>System Usage and Case Reports</title>
      <p>The latest version of SPM has been used by more than 50 programs in 13 countries (<xref rid="figure3" ref-type="fig">Figure 3</xref>) and 13 languages. In total, more than 280,000 caseworkers worldwide have used SPM with 30 million beneficiaries.</p>
      <fig id="figure3" position="float">
        <label>Figure 3</label>
        <caption>
          <p>Worldwide usage of Social Program Management.</p>
        </caption>
        <graphic xlink:href="medinform_v9i8e23219_fig3.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
      </fig>
      <sec>
        <title>Case Report 1: User-Centered Design for Social Program Efficiency</title>
        <p>Eligibility and entitlement determination are often complex, time-consuming, and one of the most frustrating aspects of social program service delivery for administrators and clients. A large US city department of human resources used SPM’s universal access module to redesign their Supplemental Nutrition Assistance Program (SNAP) [<xref ref-type="bibr" rid="ref14">14</xref>] to digitize services and reduce wait times compared with manual processing of applications and limit the need for in-person visits.</p>
        <p>In April 2018, the city implemented a citizen-facing portal, available in 7 languages, to deliver the SNAP and other benefits via clients’ own desktop and mobile phones. The design of the portal was informed by 30 shadowing sessions, 10 interviews, and 20 scenario testing sessions. Iterative prototyping and co-design workshops were also conducted to refine the portal design.</p>
        <p>A retrospective pre-post comparison of SPM metrics (ie, web and mobile log-ins, web-based applications and recertifications, and calls to update profile information) was conducted using monthly data collected in April 2018 and April 2019 (<xref ref-type="table" rid="table3">Table 3</xref>). The median age of the SNAP recipients was 25 to 44 years, and most were women (870,000/1,523,502, 57.11%). With regard to ethnic background, 27.83% (424,050/1,523,502) were Black only, 24.43% (372,245/1,523,502) were Hispanic and White only, 10.35% (157,723/1,523,502) were Black and Hispanic only, 11.23% (171,157/1,523,502) were multiethnic, 14.6% (222,495/1,523,502) were White, 10.29% (156,813/1,523,502) were Asian, and 1.25% (19,019/1,523,502) were of other ethnicities.</p>
        <table-wrap position="float" id="table3">
          <label>Table 3</label>
          <caption>
            <p>Social Program Management universal access pre- and postimplementation system metrics.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="360"/>
            <col width="220"/>
            <col width="230"/>
            <col width="190"/>
            <thead>
              <tr valign="top">
                <td>Metric</td>
                <td>Predeployment (April 2018)</td>
                <td>Postdeployment (April 2019)</td>
                <td>Percentage change (%)</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>System log-ins (web and mobile)</td>
                <td>915,532</td>
                <td>1,684,248</td>
                <td>+83.96</td>
              </tr>
              <tr valign="top">
                <td>Applications and recertifications received</td>
                <td>33,421</td>
                <td>40,198</td>
                <td>+20.28</td>
              </tr>
              <tr valign="top">
                <td>Profile update calls<sup>a</sup></td>
                <td>9696</td>
                <td>17,547</td>
                <td>+80.97</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table3fn1">
              <p><sup>a</sup>Profile update calls to center staff are required to update profile information.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <p>In the assessed periods, application rejections because of failure to provide documentation were reduced by 20% from 2674 in April 2018 to 2139 in April 2019 and center visits were reduced by 37% from 71,116 in April 2018 to 44,803 in April 2019. Of the 30,000 SNAP applications submitted in August 2019, 80% were submitted on the web through client-mobile devices via SPM’s universal access module. Client experience satisfaction was also measured with a web-based 5-star rating survey emailed to participants after they completed the application process. Responses (27,128) were collected with an average rating of 4.31 out of 5 (5 being the highest; 1 being the lowest) for the SNAP application and 4.44 out of 5 for the SNAP recertification.</p>
      </sec>
      <sec>
        <title>Case Report 2: Flexibility and Standardization in Digital Transformation</title>
        <p>A US state’s health and human services department designed a program to improve the way state county departments provided services to families and allow caseworkers to spend less time on administrative tasks and more time helping individuals and families. Specifically, the state wanted real-time data sharing and aggregation across different health and human services divisions. At the time, most families were served through multiple categorical programs. Concurrently, the state wanted to limit the amount of system customization for each county yet be flexible enough to allow each to use legacy systems in a consolidated system managing all benefits and services until legacy systems could be sunset.</p>
        <p>SPM was implemented in 2012, and information technology systems were modernized in more than 100 counties for 8 years. Deployments of SPM were designed to be interoperable with legacy systems so that incremental rollout could occur [<xref ref-type="bibr" rid="ref15">15</xref>]. SPM has replaced or is in the process of replacing approximately 20 legacy systems with a single one. SPM’s common data model provides data sharing; therefore, caseworkers no longer need to enter data into multiple systems, spending less time on administrative tasks and more time assisting families. Participants’ administrative data can be viewed and shared in real time to support a holistic view of the client, their needs, and the analysis of progress toward goals and programmatic outcomes. Currently, at least 3.5 million individuals across the state have been provided benefits through SPM [<xref ref-type="bibr" rid="ref16">16</xref>].</p>
      </sec>
      <sec>
        <title>Case Report 3: Data Aggregation, Sharing, and Collaboration</title>
        <p>In a large city in Germany, 7 local districts and 40 regional agencies are responsible for protecting children from abuse. Two special government organization units support these districts and agencies in defining and monitoring policies and providing financial and technical resources. In an effort to provide better outcomes and serve clients more efficiently, the government sought to use SPM to improve processes for client intake, case management, and communication between agencies. Specific needs included improving upon reports of abuse, traditionally done through telephone and fax machines, and better data sharing and collaboration. The government leveraged the SPM platform to address these needs, including applications for verification of evidence and provider management for managing interactions between the government and local agencies, such as foster care. Implementation-specific modules included the social enterprise collaboration for supporting multidisciplinary teams and the child welfare module for managing child abuse cases. The solution was implemented in stages where standards for social services relevant to child abuse cases, such as foster care, were defined; an interface among SPM, legacy systems, and systems of local agencies was then deployed. In the second stage, modules for managing clients were implemented. These modules provided local agencies and the government with case management capabilities for documenting and sharing information related to interactions with children and families, services, contacts, worker visits, and judicial processes.</p>
        <p>SPM allows caseworkers to view and manage a wide range of information in one place, from the initial receipt of an allegation through the final case outcome. This information enables bidirectional information sharing among local agencies and can then be used to document outcomes digitally. For example, when an allegation is made, supporting information such as police reports is automatically imported into SPM. As of June 2019, SPM in this city had 1300 users, 370,000 clients, and more than 200,000 cases and processed approximately 50,000 transactions per month since its implementation in 2014. This represents an average increase of 60% in the number of cases processed per year compared with the legacy system.</p>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Overview</title>
        <p>We presented a configurable and modular system that delivers integrated cross-program and cross-agency solutions for needs-based and contribution-based social service programs [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref17">17</xref>-<xref ref-type="bibr" rid="ref21">21</xref>]. Although social programs provide a wide variety of benefits across distinct and heterogeneous populations, fundamental needs are shared across programs that deliver services. SPM provides a set of functionalities that are generalizable and support government agencies and their beneficiaries across diverse program types and locations. These functionalities are configurable and adaptable to address the unique context of each social program.</p>
        <p>The flexibility of SPM provides advantages in implementation and change management for the digital transformation of social programs. Most legacy social program systems have automated key program processes. Usually developed ad hoc, these older systems are complex, heterogeneous, and high maintenance. At the same time, these organizations tend to be risk-averse and often dependent on specific products or platforms [<xref ref-type="bibr" rid="ref22">22</xref>]. Rather than imposing a need to redevelop the entire technology infrastructure, SPM can provide an interface between new solutions and legacy systems through modularity and open standards.</p>
        <p>Finally, data aggregation and sharing are important for improving outcomes and tracking program success by reducing information gaps and providing a holistic view of clients. Previous research has demonstrated that integrated case management with a multidisciplinary approach may improve positive outcomes for clients [<xref ref-type="bibr" rid="ref23">23</xref>]. SPM supports comprehensive data on service delivery for accountability and caseworker decision-making through a common data model. At the same time, a combination of business and infrastructure security mechanisms keeps client data protected and secure during program or agency collaboration, supporting trust in social programs.</p>
        <p>This system description is limited in that the implementations described are on-premise solutions currently. Data quality and silos can limit the extent of insights and analytics, and how these data are presented influences decision-making. Improving program performance and achieving better health outcomes requires bringing together and presenting data visually to enhance decision-making abilities. These are common challenges for any enterprise-wide solution. We have made the platform more flexible and portable by moving more core processes of SPM to the cloud. We have released SPM to work on an open-source container application platform so that users can secure and use their data across multiple environments, including public and private clouds. We are also providing analytic capabilities across social programs and in a visual format at the point of decision-making to better assess the impact of social programs on health outcomes.</p>
        <p>This viewpoint highlights the features of SPM with use cases selected to illustrate its generalizable features, such as benefits management, health and human services administration, and case coordination. These case studies were limited by organizations that were willing to share data and participate in the research. We were not able to design metric collection a priori, so the data for each use case are based on what could be provided by participating organizations.</p>
      </sec>
      <sec>
        <title>Conclusions</title>
        <p>SPM is a user-centered, configurable, and flexible system designed to manage social program workflows. Its features and functionalities support the goals of social programs through improved service delivery to beneficiaries with functionalities and features for complex eligibility and entitlement, convenient access to services, complex case management, organizational and policy change management, and program transparency. More than 50 government organizations, 280,000 caseworkers, and 30 million beneficiaries are served through SPM, demonstrating the flexibility and scalability across social program types and settings in designing administrative systems that support a streamlined workflow.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group>
      <supplementary-material id="app1">
        <label>Multimedia Appendix 1</label>
        <p>Description of Social Program Management architecture.</p>
        <media xlink:href="medinform_v9i8e23219_app1.docx" xlink:title="DOCX File , 14 KB"/>
      </supplementary-material>
    </app-group>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">SNAP</term>
          <def>
            <p>Supplemental Nutrition Assistance Program</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">SPM</term>
          <def>
            <p>Social Program Management</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>The authors would like to thank the teams at Diona, Accenture, and KPMG with whom they closely collaborated on the design, development, and execution of various aspects of case report one. The authors would also like to acknowledge the following people for their contributions to the project: Graham Harper, Haeun Jin, David Way, Ciara Layden, Ann Murphy, Pragya Singh, Shane McFadden, Alan Kenny, Shira Alevy, David Brotman, Brett South, Hannah Helmy, and Joanne Hastings.</p>
    </ack>
    <fn-group>
      <fn fn-type="conflict">
        <p>The authors of this manuscript are employed by IBM.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kerlin</surname>
              <given-names>JA</given-names>
            </name>
          </person-group>
          <source>Social Enterprise: A Global Comparison</source>
          <year>2009</year>
          <publisher-loc>Lebanon, New Hampshire</publisher-loc>
          <publisher-name>University Press of New England</publisher-name>
          <fpage>1</fpage>
          <lpage>240</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="web">
          <article-title>Government benefits</article-title>
          <source>USA.gov</source>
          <access-date>2019-12-10</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.usa.gov/benefits">https://www.usa.gov/benefits</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Parton</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>Changes in the form of knowledge in social work: From the 'social' to the 'informational'?</article-title>
          <source>Br J Soc Work</source>
          <year>2006</year>
          <month>11</month>
          <day>08</day>
          <volume>38</volume>
          <issue>2</issue>
          <fpage>253</fpage>
          <lpage>69</lpage>
          <pub-id pub-id-type="doi">10.1093/bjsw/bcl337</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gillingham</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Graham</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>Designing electronic information systems for the future: Social workers and the challenge of New Public Management</article-title>
          <source>Crit Soc Policy</source>
          <year>2015</year>
          <month>12</month>
          <day>10</day>
          <volume>36</volume>
          <issue>2</issue>
          <fpage>187</fpage>
          <lpage>204</lpage>
          <pub-id pub-id-type="doi">10.1177/0261018315620867</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kulik</surname>
              <given-names>CT</given-names>
            </name>
            <name name-style="western">
              <surname>Ryan</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Harper</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>George</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Aging populations and management</article-title>
          <source>Acad Manage J</source>
          <year>2014</year>
          <month>08</month>
          <volume>57</volume>
          <issue>4</issue>
          <fpage>929</fpage>
          <lpage>35</lpage>
          <pub-id pub-id-type="doi">10.5465/amj.2014.4004</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Saez</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Income and wealth inequality: evidence and policy implications</article-title>
          <source>Contemp Econ Policy</source>
          <year>2016</year>
          <month>10</month>
          <day>14</day>
          <volume>35</volume>
          <issue>1</issue>
          <fpage>7</fpage>
          <lpage>25</lpage>
          <pub-id pub-id-type="doi">10.1111/coep.12210</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Norström</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Grönqvist</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>The great recession, unemployment and suicide</article-title>
          <source>J Epidemiol Community Health</source>
          <year>2015</year>
          <month>02</month>
          <day>22</day>
          <volume>69</volume>
          <issue>2</issue>
          <fpage>110</fpage>
          <lpage>6</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://jech.bmj.com/lookup/pmidlookup?view=long&#38;pmid=25339416"/>
          </comment>
          <pub-id pub-id-type="doi">10.1136/jech-2014-204602</pub-id>
          <pub-id pub-id-type="medline">25339416</pub-id>
          <pub-id pub-id-type="pii">jech-2014-204602</pub-id>
          <pub-id pub-id-type="pmcid">PMC4316842</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Carrilio</surname>
              <given-names>TE</given-names>
            </name>
          </person-group>
          <article-title>Accountability, evidence, and the use of information systems in social service programs</article-title>
          <source>J Soc Work</source>
          <year>2008</year>
          <month>04</month>
          <day>01</day>
          <volume>8</volume>
          <issue>2</issue>
          <fpage>135</fpage>
          <lpage>48</lpage>
          <pub-id pub-id-type="doi">10.1177/1468017307088495</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Scurlock-Evans</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Upton</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>The role and nature of evidence: a systematic review of social workers' evidence-based practice orientation, attitudes, and implementation</article-title>
          <source>J Evid Inf Soc Work</source>
          <year>2015</year>
          <month>03</month>
          <day>06</day>
          <volume>12</volume>
          <issue>4</issue>
          <fpage>369</fpage>
          <lpage>99</lpage>
          <pub-id pub-id-type="doi">10.1080/15433714.2013.853014</pub-id>
          <pub-id pub-id-type="medline">25747891</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Grundy</surname>
              <given-names>Judith</given-names>
            </name>
            <name name-style="western">
              <surname>Grundy</surname>
              <given-names>John</given-names>
            </name>
          </person-group>
          <article-title>A survey of Australian human services agency software usage</article-title>
          <source>J Technol Hum Serv</source>
          <year>2013</year>
          <month>01</month>
          <volume>31</volume>
          <issue>1</issue>
          <fpage>84</fpage>
          <lpage>94</lpage>
          <pub-id pub-id-type="doi">10.1080/15228835.2012.751297</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wallace</surname>
              <given-names>LS</given-names>
            </name>
          </person-group>
          <article-title>A view of health care around the world</article-title>
          <source>Ann Fam Med</source>
          <year>2013</year>
          <month>01</month>
          <day>14</day>
          <volume>11</volume>
          <issue>1</issue>
          <fpage>84</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.annfammed.org/cgi/pmidlookup?view=long&#38;pmid=23319511"/>
          </comment>
          <pub-id pub-id-type="doi">10.1370/afm.1484</pub-id>
          <pub-id pub-id-type="medline">23319511</pub-id>
          <pub-id pub-id-type="pii">11/1/84</pub-id>
          <pub-id pub-id-type="pmcid">PMC3596027</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sigerist</surname>
              <given-names>HE</given-names>
            </name>
          </person-group>
          <article-title>From Bismarck to Beveridge: Developments and trends in social security legislation</article-title>
          <source>J Public Health Policy</source>
          <year>1999</year>
          <volume>20</volume>
          <issue>4</issue>
          <fpage>474</fpage>
          <pub-id pub-id-type="doi">10.2307/3343133</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="web">
          <article-title>Cúram business application suite on IBM System Z</article-title>
          <source>IBM Redbooks</source>
          <year>2009</year>
          <access-date>2019-12-10</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.redbooks.ibm.com/abstracts/sg247715.html?Open">http://www.redbooks.ibm.com/abstracts/sg247715.html?Open</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="web">
          <article-title>Supplemental Nutrition Assistance Program (SNAP)</article-title>
          <source>USDA-FNS</source>
          <access-date>2019-11-03</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.fns.usda.gov/snap/supplemental-nutrition-assistance-program">https://www.fns.usda.gov/snap/supplemental-nutrition-assistance-program</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="web">
          <article-title>Joint Legislative Oversight Committee, on Health and Human Services</article-title>
          <source>Department of Health and Human Services NC FAST</source>
          <year>2016</year>
          <access-date>2021-07-23</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.ncleg.gov/documentsites/committees/JLOCHHS/Handouts%20and%20Minutes%20by%20Interim/2016-17%20Interim%20JLOC-HHS%20Handouts/September%2027,%202016/Item%20V-DHHS-ITUpdates-JLOCSept27.pdf">https://www.ncleg.gov/documentsites/committees/JLOCHHS/Handouts%20and%20Minutes%20by%20Interim/2016-17%20Interim%20JLOC-HHS%20Handouts/September%2027,%202016/Item%20V-DHHS-ITUpdates-JLOCSept27.pdf</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gibbs</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Perry-Manning</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Department of Health and Human Services NC FAST update</article-title>
          <source>Joint Legislative Oversight Committee on Information Technology</source>
          <year>2018</year>
          <access-date>2021-07-23</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.ncleg.gov/DocumentSites/committees/JLOCIT//03-08-2018/JLOC-IT_NC%20FASTUpdate_20180308.pdf">https://www.ncleg.gov/DocumentSites/committees/JLOCIT//03-08-2018/JLOC-IT_NC%20FASTUpdate_20180308.pdf</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Colvin</surname>
              <given-names>AD</given-names>
            </name>
            <name name-style="western">
              <surname>Bullock</surname>
              <given-names>AN</given-names>
            </name>
          </person-group>
          <article-title>Technology acceptance in social work education: implications for the field practicum</article-title>
          <source>J Teach Soc Work</source>
          <year>2014</year>
          <month>10</month>
          <day>14</day>
          <volume>34</volume>
          <issue>5</issue>
          <fpage>496</fpage>
          <lpage>513</lpage>
          <pub-id pub-id-type="doi">10.1080/08841233.2014.952869</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lagsten</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Andersson</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Use of information systems in social work – challenges and an agenda for future research</article-title>
          <source>Euro J Soc Work</source>
          <year>2018</year>
          <month>01</month>
          <day>19</day>
          <volume>21</volume>
          <issue>6</issue>
          <fpage>850</fpage>
          <lpage>62</lpage>
          <pub-id pub-id-type="doi">10.1080/13691457.2018.1423554</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Devlieghere</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Roose</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>Electronic Information Systems: In search of responsive social work</article-title>
          <source>J Soc Work</source>
          <year>2018</year>
          <month>02</month>
          <day>08</day>
          <volume>18</volume>
          <issue>6</issue>
          <fpage>650</fpage>
          <lpage>65</lpage>
          <pub-id pub-id-type="doi">10.1177/1468017318757296</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sarwar</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Harris</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Children’s services in the age of information technology: what matters most to frontline professionals</article-title>
          <source>J Soc Work</source>
          <year>2018</year>
          <month>07</month>
          <day>13</day>
          <volume>19</volume>
          <issue>6</issue>
          <fpage>699</fpage>
          <lpage>718</lpage>
          <pub-id pub-id-type="doi">10.1177/1468017318788194</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Carrilio</surname>
              <given-names>TE</given-names>
            </name>
            <name name-style="western">
              <surname>Packard</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Clapp</surname>
              <given-names>JD</given-names>
            </name>
          </person-group>
          <article-title>Nothing in—nothing out</article-title>
          <source>Admin Soc Work</source>
          <year>2004</year>
          <month>02</month>
          <day>04</day>
          <volume>27</volume>
          <issue>4</issue>
          <fpage>61</fpage>
          <lpage>75</lpage>
          <pub-id pub-id-type="doi">10.1300/j147v27n04_05</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Flemig</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Osborne</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Kinder</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>Risky business—reconceptualizing risk and innovation in public services</article-title>
          <source>Public Money  Manage</source>
          <year>2016</year>
          <month>07</month>
          <day>11</day>
          <volume>36</volume>
          <issue>6</issue>
          <fpage>425</fpage>
          <lpage>32</lpage>
          <pub-id pub-id-type="doi">10.1080/09540962.2016.1206751</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Busse</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Stahl</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Integrated care experiences and outcomes in Germany, the Netherlands, and England</article-title>
          <source>Health Aff (Millwood)</source>
          <year>2014</year>
          <month>09</month>
          <volume>33</volume>
          <issue>9</issue>
          <fpage>1549</fpage>
          <lpage>58</lpage>
          <pub-id pub-id-type="doi">10.1377/hlthaff.2014.0419</pub-id>
          <pub-id pub-id-type="medline">25201659</pub-id>
          <pub-id pub-id-type="pii">33/9/1549</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
