<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMI</journal-id>
      <journal-id journal-id-type="nlm-ta">JMIR Med Inform</journal-id>
      <journal-title>JMIR Medical Informatics</journal-title>
      <issn pub-type="epub">2291-9694</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v11i1e38861</article-id>
      <article-id pub-id-type="pmid">36662569</article-id>
      <article-id pub-id-type="doi">10.2196/38861</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Original Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Original Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>A Linked Open Data–Based Terminology to Describe Libre/Free and Open-source Software: Incremental Development Study</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Lovis</surname>
            <given-names>Christian</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Schulz</surname>
            <given-names>Stefan</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Lehocki</surname>
            <given-names>Fedor</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" corresp="yes">
          <name name-style="western">
            <surname>Jahn</surname>
            <given-names>Franziska</given-names>
          </name>
          <degrees>Dr rer med</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <address>
            <institution>Institute of Medical Informatics, Statistics and Epidemiology</institution>
            <institution>Faculty of Medicine</institution>
            <institution>Leipzig University</institution>
            <addr-line>Härtelstr. 16-18</addr-line>
            <addr-line>Leipzig, 04107</addr-line>
            <country>Germany</country>
            <phone>49 341 97 16194</phone>
            <email>franziska.jahn@imise.uni-leipzig.de</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-7687-8544</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author">
          <name name-style="western">
            <surname>Ammenwerth</surname>
            <given-names>Elske</given-names>
          </name>
          <degrees>MET, Dr sc hum</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-3244-6918</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author">
          <name name-style="western">
            <surname>Dornauer</surname>
            <given-names>Verena</given-names>
          </name>
          <degrees>MSc, Dr med</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-5506-8411</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author">
          <name name-style="western">
            <surname>Höffner</surname>
            <given-names>Konrad</given-names>
          </name>
          <degrees>Dr rer nat</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-7358-3217</ext-link>
        </contrib>
        <contrib id="contrib5" contrib-type="author">
          <name name-style="western">
            <surname>Bindel</surname>
            <given-names>Michelle</given-names>
          </name>
          <degrees>MSc</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-9383-9678</ext-link>
        </contrib>
        <contrib id="contrib6" contrib-type="author">
          <name name-style="western">
            <surname>Karopka</surname>
            <given-names>Thomas</given-names>
          </name>
          <degrees>Dipl-Ing</degrees>
          <xref rid="aff3" ref-type="aff">3</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-4448-6853</ext-link>
        </contrib>
        <contrib id="contrib7" contrib-type="author">
          <name name-style="western">
            <surname>Winter</surname>
            <given-names>Alfred</given-names>
          </name>
          <degrees>Dr sc hum</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-0179-954X</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Institute of Medical Informatics, Statistics and Epidemiology</institution>
        <institution>Faculty of Medicine</institution>
        <institution>Leipzig University</institution>
        <addr-line>Leipzig</addr-line>
        <country>Germany</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Institute of Medical Informatics</institution>
        <institution>University for Health Sciences, Medical Informatics and Technology</institution>
        <addr-line>Hall in Tirol</addr-line>
        <country>Austria</country>
      </aff>
      <aff id="aff3">
        <label>3</label>
        <institution>GNU Solidario</institution>
        <addr-line>Las Palmas de Gran Canaria</addr-line>
        <country>Spain</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Franziska Jahn <email>franziska.jahn@imise.uni-leipzig.de</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <year>2023</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>20</day>
        <month>1</month>
        <year>2023</year>
      </pub-date>
      <volume>11</volume>
      <elocation-id>e38861</elocation-id>
      <history>
        <date date-type="received">
          <day>19</day>
          <month>4</month>
          <year>2022</year>
        </date>
        <date date-type="rev-request">
          <day>7</day>
          <month>9</month>
          <year>2022</year>
        </date>
        <date date-type="rev-recd">
          <day>28</day>
          <month>10</month>
          <year>2022</year>
        </date>
        <date date-type="accepted">
          <day>9</day>
          <month>11</month>
          <year>2022</year>
        </date>
      </history>
      <copyright-statement>©Franziska Jahn, Elske Ammenwerth, Verena Dornauer, Konrad Höffner, Michelle Bindel, Thomas Karopka, Alfred Winter. Originally published in JMIR Medical Informatics (https://medinform.jmir.org), 20.01.2023.</copyright-statement>
      <copyright-year>2023</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Medical Informatics, is properly cited. The complete bibliographic information, a link to the original publication on https://medinform.jmir.org/, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://medinform.jmir.org/2023/1/e38861" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>There is a variety of libre/free and open-source software (LIFOSS) products for medicine and health care. To support health care and IT professionals select an appropriate software product for given tasks, several comparison studies and web platforms, such as Medfloss.org, are available. However, due to the lack of a uniform terminology for health informatics, ambiguous or imprecise terms are used to describe the functionalities of LIFOSS. This makes comparisons of LIFOSS difficult and may lead to inappropriate software selection decisions. Using Linked Open Data (LOD) promises to address these challenges.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>We describe LIFOSS systematically with the help of the underlying Health Information Technology Ontology (HITO). We publish HITO and HITO-based software product descriptions using LOD to obtain the following benefits: (1) linking and reusing existing terminologies and (2) using Semantic Web tools for viewing and querying the LIFOSS data on the World Wide Web.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>HITO was incrementally developed and implemented. First, classes for the description of software products in health IT evaluation studies were identified. Second, requirements for describing LIFOSS were elicited by interviewing domain experts. Third, to describe domain-specific functionalities of software products, existing catalogues of features and enterprise functions were analyzed and integrated into the HITO knowledge base. As a proof of concept, HITO was used to describe 25 LIFOSS products.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>HITO provides a defined set of classes and their relationships to describe LIFOSS in medicine and health care. With the help of linked or integrated catalogues for languages, programming languages, licenses, features, and enterprise functions, the functionalities of LIFOSS can be precisely described and compared. We publish HITO and the LIFOSS descriptions as LOD; they can be queried and viewed using different Semantic Web tools, such as Resource Description Framework (RDF) browsers, SPARQL Protocol and RDF Query Language (SPARQL) queries, and faceted searches. The advantages of providing HITO as LOD are demonstrated by practical examples.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>HITO is a building block to achieving unambiguous communication among health IT professionals and researchers. Providing LIFOSS product information as LOD enables barrier-free and easy access to data that are often hidden in user manuals of software products or are not available at all. Efforts to establish a unique terminology of medical and health informatics should be further supported and continued.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>health informatics</kwd>
        <kwd>ontology</kwd>
        <kwd>free/libre open-source software</kwd>
        <kwd>software applications</kwd>
        <kwd>health IT</kwd>
        <kwd>terminology</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <sec>
        <title>Background</title>
        <p>Libre/free and open-source software (LIFOSS) products are increasingly used to support various tasks in health care. LIFOSS generally refers to software products with openly available source code that users and developers can view, analyze, modify, and redistribute.</p>
        <p>For example, there are free and open-source software products to implement radiology information systems, picture archiving and communication systems (PACS), patient administration systems, and electronic health record (EHR) systems. Especially in low-resource settings, using LIFOSS can help establish computer-based health information systems [<xref ref-type="bibr" rid="ref1">1</xref>-<xref ref-type="bibr" rid="ref3">3</xref>]. Along with their use in hospital and medical practice settings, LIFOSS products are available for mobile health, telemedicine, and public health (eg, [<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref5">5</xref>]). Moreover, the COVID-19 pandemic has led to the development of numerous mobile applications for contact tracing, risk assessment, or appointment scheduling, which are often based on LIFOSS and used both in low-resource settings and industrial countries [<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref7">7</xref>]. Since 2010, the Medfloss.org database has provided descriptions of LIFOSS used in health care and medicine [<xref ref-type="bibr" rid="ref8">8</xref>]. As of October 2022, it lists 385 software products and describes them by characteristics like “license,” “application type,” “enterprise function,” “language,” “platform,” and “home page.” Due to the iterative and sometimes uncontrolled growth of the self-developed nomenclature over the last few years, there are several inconsistencies in the software descriptions on Medfloss.org. First, there are misleading assignments of descriptors to characteristics. For example, “laboratory,” “cellular networks,” and “virtual reality” are listed as enterprise functions supported by a software product. However, they describe the setting where the software might be used or special features of the software. Second, the lack of uniform terminology in health informatics has led to the use of synonyms or overlapping terms. For example, the borders between “electronic health systems,” “electronic medical record systems,” and “hospital management systems” are not clearly defined, sometimes leading to ambiguous descriptions of software products. The lack of uniform terminology for describing medical and health care software products and LIFOSS is also apparent when analyzing comparisons of LIFOSS for EHR systems. In several studies published during the last 15 years [<xref ref-type="bibr" rid="ref1">1</xref>-<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref9">9</xref>-<xref ref-type="bibr" rid="ref11">11</xref>], each research group selected different criteria and descriptors for comparing the technical and functional characteristics of EHR systems.</p>
        <p>Indeed, the lack of uniform terminology is not restricted to LIFOSS. There are already several terminologies in health care and health informatics that could potentially be used to describe software. For example, in addition to allowing for precise descriptions of disease patterns, SNOMED (Systematized Nomenclature of Medicine) lists health occupations and environments that may describe the health facilities or settings in which LIFOSS is used. The World Health Organization (WHO) classifies digital health interventions by hierarchies or lists of clients, health care providers, health system managers, data service features, and application system categories [<xref ref-type="bibr" rid="ref12">12</xref>]. The Health Level 7 (HL7) EHR System Functional Model is a comprehensive catalogue of EHR systems’ features [<xref ref-type="bibr" rid="ref13">13</xref>]. Finally, some health informatics textbooks provide systematic collections of application systems in health care and their typical features (eg, [<xref ref-type="bibr" rid="ref14">14</xref>]). However, at first sight, these different terminologies are not easily comparable with each other due to the use of synonyms or homonyms. Linking these different terminologies requires semantic analyses based on a uniform set of concepts, along with easy-to-use methods and tools to link these data.</p>
        <p>Linked Open Data (LOD) are regarded as the state-of-the-art principle for linking and structuring concepts from different terminologies. LOD are identifiable by a URI and provided using the Resource Description Framework (RDF) standard [<xref ref-type="bibr" rid="ref15">15</xref>].</p>
      </sec>
      <sec>
        <title>Benefits of Unified LIFOSS Terminology Using LOD</title>
        <p>A unified LIFOSS terminology using LOD has several advantages. First, the use of predefined and open terminologies supports the search for and comparison of software products. Second, further knowledge, such as results of assessment and evaluation studies, can be linked easily to the software descriptions and thus support evidence-based health informatics. For example, linking systematic descriptions of software products with descriptors from other projects can support ontology-based approaches to software requirements engineering [<xref ref-type="bibr" rid="ref16">16</xref>].</p>
        <p>The Austrian-German research project “Health Information Technology Ontology (HITO)” aims to systematically describe software products and their installations in health care. It uses an underlying ontology named HITO, LOD methods and tools, and freely available catalogues to describe software characteristics. HITO is developed based on different use cases in which precise software descriptions are needed, such as when selecting LIFOSS or commercial software products, searching for evidence about the installation of software products, and communicating about software products among stakeholders in health care. In this paper, we focus on LIFOSS products and describe them with the help of openly available catalogues. Especially for LIFOSS products, information about software characteristics is freely available, and LIFOSS developers are likely to recognize the potential advantage of spreading knowledge about their products with the help of LOD.</p>
      </sec>
      <sec>
        <title>Objectives</title>
        <p>This study aims to (1) describe LIFOSS systematically with the help of precise descriptors that are captured in HITO and (2) publish HITO and HITO-based software product descriptions as LOD using Semantic Web tools for viewing and querying LIFOSS data on the World Wide Web.</p>
      </sec>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Requirements Elicitation</title>
        <sec>
          <title>Initial Steps</title>
          <p>As the first step toward precise descriptions of software products, the classes and relationships that are useful to describe software products had to be identified. HITO was developed and refined iteratively by collecting requirements of several use cases. Each use case focused on a situation in which a clear terminology for software products was considered essential. These use cases dealt with the description of LIFOSS or commercial software products for potential and current users in health care settings or the description of software product installations in evaluation studies on health IT interventions. We selected diverse use cases to identify the most relevant characteristics for these diverse situations. Based on these use cases, we incrementally built the ontology that contains a general pattern for describing software products in health care.</p>
        </sec>
        <sec>
          <title>Use Case 1: Evaluation of Digital Health Interventions</title>
          <p>The first HITO use case dealt with evaluation studies in health informatics. In evaluation studies, it is crucial to carefully describe the intervention in a reproducible and clear manner to allow generalizability of the findings and their aggregation, for example, in the form of systematic reviews. We used an inductive approach and extracted software descriptions from 24 randomly selected published health IT evaluation studies [<xref ref-type="bibr" rid="ref17">17</xref>]. We found that software product installations were mainly described by their features, application system type, organizational units where they are used, and user groups. These characteristics were specified as classes and added to HITO (<xref ref-type="table" rid="table1">Table 1</xref>). Altogether, the software descriptions found in evaluation studies are sparse and concentrate on selected evaluated features. Evaluation studies also often use inconsistent terms, which motivated us to develop a clear terminology for software products.</p>
          <table-wrap position="float" id="table1">
            <label>Table 1</label>
            <caption>
              <p>HITO<sup>a</sup> classes to describe software products in health IT evaluation studies (HITO use case 1).</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="230"/>
              <col width="770"/>
              <thead>
                <tr valign="top">
                  <td>HITO class (characteristic of software products)</td>
                  <td>Description and examples</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td>Software product</td>
                  <td>Piece of software that is sold as a commercial product or distributed under an open-source license</td>
                </tr>
                <tr valign="top">
                  <td>Feature</td>
                  <td>Functionalities offered by a software product that directly contribute to the fulfillment of 1 or more enterprise functions (eg, email notification of new results, user directory to control any access)</td>
                </tr>
                <tr valign="top">
                  <td>Application system type</td>
                  <td>Commonly used names for categories of software product installations in health care (eg, radiology information system, CPOE<sup>b</sup> system)</td>
                </tr>
                <tr valign="top">
                  <td>Organizational unit</td>
                  <td>Health care setting in which the software product is used and an evaluation study was conducted (eg, laboratory, department of pediatrics)</td>
                </tr>
                <tr valign="top">
                  <td>User group</td>
                  <td>Health care staff who uses the software product installation (eg, nurse, radiologist)</td>
                </tr>
              </tbody>
            </table>
            <table-wrap-foot>
              <fn id="table1fn1">
                <p><sup>a</sup>HITO: Health Information Technology Ontology.</p>
              </fn>
              <fn id="table1fn2">
                <p><sup>b</sup>CPOE: computerized physician order entry.</p>
              </fn>
            </table-wrap-foot>
          </table-wrap>
        </sec>
        <sec>
          <title>Use Case 2: Description of LIFOSS With Medfloss.org Project Database</title>
          <p>In the LIFOSS use case, we extended the list of HITO classes by carefully analyzing the Medfloss.org project database. Medfloss.org aims to offer an overview of LIFOSS projects related to medical informatics and health care [<xref ref-type="bibr" rid="ref8">8</xref>]. Although Medfloss.org is not maintained anymore, it is still provided in cooperation with 3 LIFOSS-related working groups of the International Medical Informatics Association, the European Federation for Medical Informatics Association, and the International Society for Telemedicine and eHealth. Within Medfloss.org, the LIFOSS products are described by using a predefined set of categories.</p>
          <p>We started this use case by surveying 2 operators of the Medfloss.org database. They were asked independently to answer a survey with 11 open questions and 1 closed question. The survey asked about the users of Medfloss.org, the relevance of the categories used to describe software products, and the positive and negative experiences with the categories used to describe LIFOSS.</p>
          <p>The results of this survey show that Medfloss.org is intended to be used by physicians or other health care staff, IT administrators, information managers, and software developers to select appropriate LIFOSS for certain health care tasks. Each LIFOSS product is described by 11 categories, such as “enterprise function,” “application type,” and “license,” on the platform. For each category, the list of descriptors has grown over the years as new LIFOSS descriptions were added.</p>
          <p>The answers of the platform operators dealing with the assessment of the current terminology and its representation on the platform were arranged according to a SWOT (Strengths, Weaknesses, Opportunities, and Threats) analysis (<xref ref-type="boxed-text" rid="box1">Textbox 1</xref>).</p>
          <p>Both operators rated the “enterprise function,” “application type,” “status,” “license,” “standard,” “language,” “client type,” and “platform” categories of Medfloss.org terminology as “important” or “very important” for the description of LIFOSS. The categories “popularity,” “database,” and “programming language/toolkit” were rated less important by 1 of the website operators.</p>
          <p>The strengths of the Medfloss.org terminology used include the rough categorization of software products by application system types and the faceted search on the website that is based on categories for describing the software. However, the survey showed that the categories currently lack the possibility to describe the functionalities and application types with enough precision. Therefore, an opportunity to support Medfloss.org terminology may be to integrate existing terminologies such as WHO’s classification of digital health interventions [<xref ref-type="bibr" rid="ref12">12</xref>] or the HL7 EHR System Functional Model [<xref ref-type="bibr" rid="ref13">13</xref>]. Furthermore, the folksonomy (ie, the collection of users’ tags for certain objects) of the platform users is not handled by the search functions on Medfloss.org [<xref ref-type="bibr" rid="ref18">18</xref>].</p>
          <p>Based on the SWOT analysis, we selected 9 (out of 10) of the Medfloss.org categories and added these to HITO (<xref ref-type="table" rid="table2">Table 2</xref>). Some classes used on Medfloss.org were renamed, such as “standard,” which was changed to “interoperability standard” to sum up interoperability standards, as well as frameworks describing how to use interoperability standards, such as Integrating the Healthcare Enterprise (IHE). A more fine-grained classification according to interoperability levels was examined for common interoperability standards but proved to be impractical. Many interoperability standards such as HL7 Fast Healthcare Interoperability Services (FHIR) can be assigned to multiple interoperability levels. The Medfloss.org category “programming language/toolkit” was split into 2 classes to distinguish between these different concepts.</p>
          <boxed-text id="box1" position="float">
            <title>SWOT (Strengths, Weaknesses, Opportunities, and Threats) analysis for Medfloss.org.</title>
            <p>
              <bold>Strengths</bold>
            </p>
            <list list-type="bullet">
              <list-item>
                <p>Current set of categories to describe libre/free and open-source software</p>
              </list-item>
              <list-item>
                <p>Rough categorization by application system types</p>
              </list-item>
              <list-item>
                <p>Usefulness of the categories to provide a faceted search on the platform</p>
              </list-item>
            </list>
            <p>
              <bold>Weaknesses</bold>
            </p>
            <list list-type="bullet">
              <list-item>
                <p>Conceptual overlaps in categories (eg, electronic health record and electronic medical record)</p>
              </list-item>
              <list-item>
                <p>Missing hierarchies for enterprise functions</p>
              </list-item>
              <list-item>
                <p>Missing detailed functional descriptions</p>
              </list-item>
            </list>
            <p>
              <bold>Opportunities</bold>
            </p>
            <list list-type="bullet">
              <list-item>
                <p>Enhancements of categories by existing terminologies seems possible</p>
              </list-item>
              <list-item>
                <p>Modeling of user group–dependent categories may increase usefulness</p>
              </list-item>
            </list>
            <p>
              <bold>Threats</bold>
            </p>
            <list list-type="bullet">
              <list-item>
                <p>No handling of synonyms within users’ search terms</p>
              </list-item>
            </list>
          </boxed-text>
          <table-wrap position="float" id="table2">
            <label>Table 2</label>
            <caption>
              <p>HITO<sup>a</sup> to describe LIFOSS<sup>b</sup> products (HITO use case 2).</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="190"/>
              <col width="240"/>
              <col width="570"/>
              <thead>
                <tr valign="top">
                  <td>Medfloss.org class name</td>
                  <td>HITO class</td>
                  <td>Description and examples</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td>Client type</td>
                  <td>Client</td>
                  <td>The client type on which a software product can be run (mobile, native, or web).</td>
                </tr>
                <tr valign="top">
                  <td>Database</td>
                  <td>Database management system</td>
                  <td>Some examples are PostgreSQL<sup>c</sup> or MySQL.</td>
                </tr>
                <tr valign="top">
                  <td>Enterprise function</td>
                  <td>Enterprise function</td>
                  <td>Describes what action humans or machines must carry out in a certain enterprise to contribute to its mission or goals (eg, patient admission, order entry).</td>
                </tr>
                <tr valign="top">
                  <td>Home page</td>
                  <td>Home page</td>
                  <td>Home page of the software product or its development project.</td>
                </tr>
                <tr valign="top">
                  <td>Standard</td>
                  <td>Interoperability standard</td>
                  <td>Ability of 2 or more components to exchange information and to use the information that has been exchanged. Under this class name, interoperability standards (eg, HL7<sup>d</sup> FHIR<sup>e</sup> or DICOM<sup>f</sup>) or frameworks describing how to use standards (eg, IHE<sup>g</sup>) are summed up.</td>
                </tr>
                <tr valign="top">
                  <td>Language</td>
                  <td>Language</td>
                  <td>Languages in which the software product is available (eg, English, French, and German).</td>
                </tr>
                <tr valign="top">
                  <td>License</td>
                  <td>License</td>
                  <td>The license under which a software product is distributed.</td>
                </tr>
                <tr valign="top">
                  <td>Platform</td>
                  <td>Operating system</td>
                  <td>The operating system a software uses (eg, Windows). A software product might be able to run on a variety of operating systems.</td>
                </tr>
                <tr valign="top">
                  <td>Programming language/toolkit</td>
                  <td>Programming language</td>
                  <td>The programming language used to develop a software product (eg, Java or Python).</td>
                </tr>
                <tr valign="top">
                  <td>Programming language/toolkit</td>
                  <td>Programming library or toolkit</td>
                  <td>Programming toolkits are utility programs that are used to develop and maintain software. Programming libraries are a collection of prewritten functions that are ready to be used in coding. Both help programmers develop software in a fast and safe manner.</td>
                </tr>
              </tbody>
            </table>
            <table-wrap-foot>
              <fn id="table2fn1">
                <p><sup>a</sup>HITO: Health Information Technology Ontology.</p>
              </fn>
              <fn id="table2fn2">
                <p><sup>b</sup>LIFOSS: libre/free and open-source software.</p>
              </fn>
              <fn id="table2fn3">
                <p><sup>c</sup>SQL: Structured Query Language.</p>
              </fn>
              <fn id="table2fn4">
                <p><sup>d</sup>HL7: Health Level 7.</p>
              </fn>
              <fn id="table2fn5">
                <p><sup>e</sup>FHIR: Fast Healthcare Interoperability Resources.</p>
              </fn>
              <fn id="table2fn6">
                <p><sup>f</sup>DICOM: Digital Imaging and Communications in Medicine.</p>
              </fn>
              <fn id="table2fn7">
                <p><sup>g</sup>IHE: Integrating the Healthcare Enterprise.</p>
              </fn>
            </table-wrap-foot>
          </table-wrap>
        </sec>
        <sec>
          <title>Further HITO Use Cases</title>
          <p>We summarized 3 further use cases that have not elicited new HITO classes related to software product characteristics.</p>
          <p>The third use case deals with the description of commercial software products used in health care. This use case confirmed the set of HITO classes that we already identified by describing LIFOSS products. However, for commercial software products, it is quite challenging to describe them based on these classes because meaningful descriptions of commercial software products are rarely publicly available.</p>
          <p>In the fourth use case, the existing HITO classes were linked with competency levels for IT staff in health care organizations, which might be useful for creating job advertisements.</p>
          <p>In the fifth use case, findings from the HITO project were discussed with practitioners like hospital chief information officers and industry representatives to discuss the applicability and broader use of the HITO project’s findings in practice.</p>
        </sec>
      </sec>
      <sec>
        <title>Software Product Descriptions as LOD</title>
        <p>After building HITO from the described use cases, we published HITO and HITO-based software product descriptions using LOD.</p>
        <p>LOD are web data with an open license. They allow the use of Semantic Web tools for viewing and querying LIFOSS data on the World Wide Web. LOD also allow for linking and reusing existing terminologies. To be considered “5-star LOD,” the data need to be machine-readable, presented in a nonproprietary format, use open standards of the World Wide Web Consortium (W3C), and be linked to other data [<xref ref-type="bibr" rid="ref15">15</xref>]. We strived to achieve 5-star linked open HITO data.</p>
        <p>Accordingly, we used RDF Schema (RDFS) and the Web Ontology Language (OWL). RDFS and OWL are W3C standards used to define the types of elements of discourse as classes. The <italic>class</italic> “software product,” for example, represents the set of all individual software products. <italic>Properties</italic> represent possible binary-typed relationships between individuals of certain classes, for example, between software products and features. Using RDF, facts (relationships between individuals) are expressed as subject-predicate-object triples, whose elements may be defined and stored in different places. This allows for reusing existing vocabularies and interlinking with existing knowledge bases, forming the LOD cloud. Each RDF resource (a class, an individual, or a relationship) has a URL where it is published both in human-readable (ie, HTML) format and in machine-processable RDF serialization format. To browse RDF data comfortably, tools like RickView [<xref ref-type="bibr" rid="ref19">19</xref>] can be used and modified. SPARQL Protocol and RDF Query Language (SPARQL) end points allow free access to structured read-only queries for humans and as application programming interfaces (APIs) for several tools.</p>
      </sec>
      <sec>
        <title>Integration of Software Product and Health-Related Terminologies</title>
        <p>One advantage of LOD is their easy integration of existing data sources that are already available in RDF format. Therefore, for the HITO classes (<xref ref-type="table" rid="table1">Tables 1</xref> and <xref ref-type="table" rid="table2">2</xref>) that characterize software products, we searched for “catalogues” (ie, lists of terms that can be used as instances for the respective class). For the selection of suitable catalogues, we defined the following criteria:</p>
        <p>[The catalogue should be authored by an established scientific or standardization organization.] OR [The catalogue must be scientifically plausible with regard to reproducibility, having undergone peer-review or having been developed by more than 5 persons.] OR [The catalogue must be openly available and developed by a large community.]</p>
        <p>We used a broad literature and web search and our knowledge of the field to identify related taxonomies and catalogues that can be useful for describing instances of HITO classes (<xref ref-type="table" rid="table2">Table 2</xref>). In the following paragraphs, we provide a brief overview of the catalogues we investigated.</p>
        <p>We started with DBpedia, a popular and large knowledge base comprising billions of triples extracted from Wikipedia, texts, and other sources [<xref ref-type="bibr" rid="ref20">20</xref>,<xref ref-type="bibr" rid="ref21">21</xref>]. We analyzed DBpedia to identify possible instances or subclasses for the HITO classes. For the classes “language,” “operating system,” and “programming language,” we found DBpedia classes with suitable instances that we replaced the associated HITO classes with to increase interoperability. Other instances of DBpedia classes, such as the “license” class, were not suitable for integration because DBpedia does not semantically differentiate between licenses for software products and licenses for other purposes, like drivers’ licenses. For software product licenses, we integrated subclasses of the class “open-source software license” derived from the Software Ontology for biomedical software [<xref ref-type="bibr" rid="ref22">22</xref>].</p>
        <p>The most challenging task was the integration of catalogues for the classes “application system type,” “enterprise function,” “feature,” “organizational unit,” and “user group,” for which we needed instances or subclasses related to health care. We checked the following sources for the integration of catalogues into HITO:</p>
        <list list-type="bullet">
          <list-item>
            <p>HL7 EHR System Functional Model [<xref ref-type="bibr" rid="ref13">13</xref>]: Using the examples of 2 installations of commercial software products for EHR systems, we assessed whether the features of the software product could be described by this model. We found that the whole list of conformance criteria defined in this model would be too detailed for a HITO catalogue. However, the section labels that are provided by this model, such as “manage allergy, intolerance, and adverse reaction list,” provide an appropriate level of detail for feature descriptions in HITO.</p>
          </list-item>
          <list-item>
            <p>The textbook by Winter et al [<xref ref-type="bibr" rid="ref14">14</xref>] describes a set of application system types and a set of enterprise functions in hospitals. An analysis of Medfloss.org revealed that the sets of enterprise functions or application system types could be used to tag 71% and 42%, respectively, of 356 Medfloss.org software products analyzed in use case 2 [<xref ref-type="bibr" rid="ref23">23</xref>]. Due to the textbook’s focus on hospitals, appropriate terms for software products used in other health institutions or for public health were missing.</p>
          </list-item>
          <list-item>
            <p>Taxonomy for health IT by Varshney et al [<xref ref-type="bibr" rid="ref24">24</xref>]: The application system types identified by Varshney and colleagues proved to be too coarse-grained for classification. Only 105 (29%) of 356 Medfloss.org software product entries could be classified by the taxonomy [<xref ref-type="bibr" rid="ref23">23</xref>].</p>
          </list-item>
          <list-item>
            <p>WHO classification of digital health interventions [<xref ref-type="bibr" rid="ref12">12</xref>]: This multiaxial classification lists system categories and interventions for clients, health care providers, health system managers, and data services. The 25 system categories correspond to application system types in HITO, and their strength lies in their focus on application systems for health care networks rather than for single institutions. For the hierarchically grouped interventions such as “2.10 laboratory and diagnostics imaging management” and “2.10.1 transmit diagnostic result to healthcare provider,” a more differentiated assignment to functions and features in the context of HITO is needed. Therefore, “2.10 laboratory and diagnostics imaging management” is listed as an enterprise function in HITO, whereas “2.10.1 transmit diagnostic result to healthcare provider” is listed as a feature in HITO.</p>
          </list-item>
          <list-item>
            <p>SNOMED Clinical Terms (CT) [<xref ref-type="bibr" rid="ref25">25</xref>]: For the 24 evaluation studies analyzed in use case 1, we found that the software products’ user groups or the organizational units where the software products are used can sufficiently be described by subclasses of SNOMED CT’s “occupation” or “environment” classes, respectively.</p>
          </list-item>
          <list-item>
            <p>Features for PACS selection [<xref ref-type="bibr" rid="ref26">26</xref>]: The PACS features described on 3 hierarchy levels were transformed into a flat list of 38 features useful for describing examples of PACS software products.</p>
          </list-item>
          <list-item>
            <p>The LIS (laboratory information system) Functionality Assessment Toolkit of the Association for Pathology Informatics [<xref ref-type="bibr" rid="ref27">27</xref>]: This toolkit lists 850 functionality statements and describes a methodology for selecting a LIS. In comparison to the other feature catalogues we described, the functionality statements are very detailed. For integration into HITO, a clustering of the functionality statements would be necessary to obtain a manageable LIS feature list.</p>
          </list-item>
        </list>
      </sec>
      <sec>
        <title>Describing Software Products Using HITO</title>
        <p>With the help of HITO and the selected catalogues we described, 25 LIFOSS products were described. These were selected to represent different application system types and due to their comprehensive, openly available documentation. For the description of supported enterprise functions and features, we extracted terms from the software product documentation and linked as many as possible to catalogue entries of HITO. Information about the LIFOSS products was extracted by 1 team member (author MB) and checked by another team member (author FJ).</p>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <p>HITO was modeled and published using Semantic Web technologies. The Unified Modeling Language class diagram in <xref rid="figure1" ref-type="fig">Figure 1</xref> describes the structure of HITO. The classes shown to the left of “software product” describe the general characteristics of software products. For classes with domain-specific instances (ie, application system type, feature, enterprise function, organizational unit, and user group), we applied a scheme of 3 interrelated classes named <italic>&#60;classname&#62;</italic>Catalogue, <italic>&#60;classname&#62;</italic>Classified, and <italic>&#60;classname&#62;</italic>Citation. A <italic>&#60;classname&#62;</italic>Catalogue is a health IT–related collection, such as the HL7 EHR Functional Model or the WHO classifications for digital interventions. A <italic>&#60;classname&#62;</italic>Classified is a category that belongs to exactly 1 catalogue. A <italic>&#60;classname&#62;</italic>Citation is a textual label extracted from available software manuals, descriptions, and studies and thus represents a part of the folksonomy contained in HITO [<xref ref-type="bibr" rid="ref28">28</xref>].</p>
      <p>To illustrate how this scheme applies, we describe the domain-specific characteristics of the Orthanc software [<xref ref-type="bibr" rid="ref29">29</xref>,<xref ref-type="bibr" rid="ref30">30</xref>] in <xref ref-type="table" rid="table3">Table 3</xref>. The developers of Orthanc refer to it as “mini-PACS,” “DICOM (Digital Imaging and Communications in Medicine) server,” “VNA (vendor-neutral archive),” and “viewer of medical images.” These terms are assigned to the software product Orthanc as “application system type citations.” In turn, “DICOM server” and “mini-PACS” have links to the classified application system type “PACS” from the application system type catalogue in Winter et al [<xref ref-type="bibr" rid="ref14">14</xref>]. The supported enterprise function citations extracted from the Orthanc website, such as “image archiving,” “image management,” and “research about the automated analysis of medical images,” were linked to the more general terms “laboratory and diagnostics imaging management” and “research and education” from enterprise function catalogues. For the 16 feature citations extracted from the Orthanc online documentation, we identified linkable classified features of the WHO classification of digital health interventions and the PACS feature list. Some classified features or enterprise function terms have direct links to the software product. These assignments had no match with citations and were done by domain experts.</p>
      <p>Overall, with the help of HITO, we described 25 LIFOSS products in similar detail as Orthanc. All software product descriptions are available as LOD. We described HITO classes and relationships of the ontology and individual software products using RDF. The HITO SPARQL end point [<xref ref-type="bibr" rid="ref31">31</xref>] allows queries using SPARQL (<xref rid="figure2" ref-type="fig">Figure 2</xref>).</p>
      <p>The RickView application allows for browsing through the ontology and knowledge base [<xref ref-type="bibr" rid="ref32">32</xref>]. Another suitable way to query is with a faceted search [<xref ref-type="bibr" rid="ref33">33</xref>], whereby integrated terminologies can be used to find software products of a certain application system type or supporting combinations of features and functions (<xref rid="figure3" ref-type="fig">Figure 3</xref>).</p>
      <p>The ontology is also publicly available under version control in a GitHub repository [<xref ref-type="bibr" rid="ref34">34</xref>]. It can be downloaded in RDF Turtle format to be viewed in ontology editors like Protégé. HITO is dedicated to the public domain and uses a Creative Commons Zero v1.0 Universal license. There are a few exceptions for integrated terms from SNOMED [<xref ref-type="bibr" rid="ref25">25</xref>] and the WHO classification of digital health interventions [<xref ref-type="bibr" rid="ref12">12</xref>].</p>
      <fig id="figure1" position="float">
        <label>Figure 1</label>
        <caption>
          <p>The Health Information Technology Ontology (HITO), version 22.05, specifies the classes and relationships that are used to describe software products. The complete class diagram is available on the HITO website [<xref ref-type="bibr" rid="ref28">28</xref>].</p>
        </caption>
        <graphic xlink:href="medinform_v11i1e38861_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
      </fig>
      <table-wrap position="float" id="table3">
        <label>Table 3</label>
        <caption>
          <p>Application system types, supported functions, features, user groups and organizational units of the Orthanc software.</p>
        </caption>
        <table border="1" rules="groups" cellpadding="5" frame="hsides" width="1000" cellspacing="0">
          <col width="30"/>
          <col width="350"/>
          <col width="620"/>
          <thead>
            <tr valign="top">
              <td colspan="2">&#60;className&#62; and &#60;classname&#62;Citation from the software documentation</td>
              <td>&#60;classname&#62;Classified in HITO<sup>a</sup> (&#60;classname&#62;Catalogue)</td>
            </tr>
          </thead>
          <tbody>
            <tr valign="top">
              <td colspan="3">
                <bold>Application system type</bold>
              </td>
            </tr>
            <tr valign="top">
              <td/>
              <td>DICOM<sup>b</sup> server</td>
              <td>PACS<sup>c</sup> (application systems in [<xref ref-type="bibr" rid="ref14">14</xref>])</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>mini-PACS</td>
              <td>PACS (application systems in [<xref ref-type="bibr" rid="ref14">14</xref>])</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>VNA<sup>d</sup></td>
              <td>N/A<sup>e</sup></td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Viewer of medical images</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Web viewer</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td colspan="3">
                <bold>Enterprise function</bold>
              </td>
            </tr>
            <tr valign="top">
              <td/>
              <td>N/A</td>
              <td>Execution of radiological examinations (enterprise functions from [<xref ref-type="bibr" rid="ref14">14</xref>])</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Image archiving</td>
              <td>Laboratory and diagnostics imaging management (enterprise functions from [<xref ref-type="bibr" rid="ref12">12</xref>])</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Image communication</td>
              <td>Laboratory and diagnostics imaging management (enterprise functions from [<xref ref-type="bibr" rid="ref12">12</xref>])</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Image distribution</td>
              <td>Laboratory and diagnostics imaging management (enterprise functions from [<xref ref-type="bibr" rid="ref12">12</xref>])</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Image management</td>
              <td>Laboratory and diagnostics imaging management (enterprise functions from [<xref ref-type="bibr" rid="ref12">12</xref>])</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Research about the automated analysis of medical images</td>
              <td>Research and education (enterprise functions from [<xref ref-type="bibr" rid="ref14">14</xref>])</td>
            </tr>
            <tr valign="top">
              <td colspan="3">
                <bold>Feature</bold>
              </td>
            </tr>
            <tr valign="top">
              <td/>
              <td>N/A</td>
              <td>Capture diagnostic results from digital devices (features from [<xref ref-type="bibr" rid="ref12">12</xref>])</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Data management for clinical routine and medical research</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>DICOM identifiers</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>DICOM network protocol</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Evaluations Report</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Injury surveillance system registration report</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Listing available servers</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Plugin mechanism to add new modules</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Retrieve images</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Retrieving DICOM resources from WADO-RS<sup>f</sup> server</td>
              <td>Compatibility and integration with other systems and products (PACS feature list [<xref ref-type="bibr" rid="ref26">26</xref>])</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Search the content</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Send images</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Sending DICOM resources to a STOW-RS<sup>g</sup> server</td>
              <td>Compatibility and integration with other systems and products (PACS feature list [<xref ref-type="bibr" rid="ref26">26</xref>])</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Test the connection</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Top diseases report</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td colspan="3">
                <bold>User group</bold>
              </td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Radiologist</td>
              <td>Radiologist occupation [<xref ref-type="bibr" rid="ref25">25</xref>]</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Researcher</td>
              <td>Researcher occupation [<xref ref-type="bibr" rid="ref25">25</xref>]</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Software/hardware integrators in the medical field</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Network engineer</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Physicist</td>
              <td>Physicist occupation [<xref ref-type="bibr" rid="ref25">25</xref>]</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>System engineer</td>
              <td>N/A</td>
            </tr>
            <tr valign="top">
              <td colspan="3">
                <bold>Organizational unit</bold>
              </td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Health centers</td>
              <td>Health center environment [<xref ref-type="bibr" rid="ref25">25</xref>]</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Hospital environment</td>
              <td>Hospital environment [<xref ref-type="bibr" rid="ref25">25</xref>]</td>
            </tr>
            <tr valign="top">
              <td/>
              <td>Radiology department</td>
              <td>Radiology department environment [<xref ref-type="bibr" rid="ref25">25</xref>]</td>
            </tr>
          </tbody>
        </table>
        <table-wrap-foot>
          <fn id="table3fn1">
            <p><sup>a</sup>HITO: Health Information Technology Ontology.</p>
          </fn>
          <fn id="table3fn2">
            <p><sup>b</sup>DICOM: Digital Imaging and Communications in Medicine.</p>
          </fn>
          <fn id="table3fn3">
            <p><sup>c</sup>PACS: picture archiving and communication system.</p>
          </fn>
          <fn id="table3fn4">
            <p><sup>d</sup>VNA: vendor-neutral archive.</p>
          </fn>
          <fn id="table3fn5">
            <p><sup>e</sup>N/A: not applicable.</p>
          </fn>
          <fn id="table3fn6">
            <p><sup>f</sup>WADO-RS: Web Access to DICOM Objects by RESTful (representational state transfer) Services.</p>
          </fn>
          <fn id="table3fn7">
            <p><sup>g</sup>STOW-RS: Store Over the Web by RESTful Services.</p>
          </fn>
        </table-wrap-foot>
      </table-wrap>
      <fig id="figure2" position="float">
        <label>Figure 2</label>
        <caption>
          <p>A SPARQL Protocol and RDF Query Language (SPARQL) query (on the left) and its results (on the right). RDF: Resource Description Framework.</p>
        </caption>
        <graphic xlink:href="medinform_v11i1e38861_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
      </fig>
      <fig id="figure3" position="float">
        <label>Figure 3</label>
        <caption>
          <p>Faceted search for software products supporting “Laboratory and Diagnostics Imaging Management”. HITO: Health Information Technology Ontology.</p>
        </caption>
        <graphic xlink:href="medinform_v11i1e38861_fig3.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
      </fig>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Principal Results</title>
        <p>In this project, we identified characteristics useful for describing software products and systematically captured them as classes in HITO. Accordingly, we exploited the properties of ontologies that enable semantic description and linking of data.</p>
        <p>For a thorough functional description of software products in health care, we described the enterprise functions supported and features offered by the software products. For health-specific characteristics of software products, we analyzed and integrated existing terminologies for enterprise functions, features, application system types, organizational units, and user groups as catalogues. Because we expected the relevant sources for such catalogues to come not only from science but also from practice, a systematic review of the scientific literature for health IT–related terminologies would not have led to sufficient results. Accordingly, we based the selection of catalogues on statements by domain experts among the project team and project partners, supplemented by targeted PubMed and Google searches on specific application system types.</p>
        <p>Thus, we used a case-based and agile approach to identify classes, relationships, and catalogues best suited for describing software products in health care. In the use cases considered so far, ontological reasoning had no relevance. Therefore, to date, only few axioms are used in HITO. As the next step toward more interoperability with other formal ontologies, HITO could use an upper-level ontology such as Basic Formal Ontology [<xref ref-type="bibr" rid="ref35">35</xref>], General Formal Ontology [<xref ref-type="bibr" rid="ref36">36</xref>], or gist [<xref ref-type="bibr" rid="ref37">37</xref>]. A first feasibility check of these ontologies showed that the gist ontology, which defines typical upper-level business concepts, may be the most appropriate for the scope of HITO.</p>
        <p>With the help of HITO, we described 25 LIFOSS products in detail that, together with less detailed descriptions of single commercial software products and software products extracted from evaluation studies, form our knowledge base. The descriptions of these software products could be regarded as a proof of concept. However, we noticed the interpretative degrees of freedom in assigning correct enterprise functions and features to software products. To ensure the validity of further software descriptions, it would be helpful to calculate interrater reliability among 2 independent experts. For this, further software product entries of the Medfloss.org database that have not yet been considered in HITO’s knowledge base could be used.</p>
        <p>As postulated by Berners-Lee [<xref ref-type="bibr" rid="ref15">15</xref>], HITO’s availability as LOD facilitates its barrier-free access and use. In particular, the integrated catalogues for enterprise functions, features, and application system types provide HITO users with rich terminology for functionalities of software products. However, since there is more than 1 catalogue for each of these characteristics, new terminological problems arise. The catalogue entries of different catalogues must be mapped to each other to achieve comparability of software products described with the help of different catalogues. Linking these catalogues is part of ongoing research. Together with the folksonomy terms that are already connected to catalogue entries, HITO users will be able to retrieve the most suitable software using a broad range of search terms. The catalogues currently integrated into HITO focus more on health care rather than on medical research tasks (ie, software products like research databases may not be sufficiently described by HITO). However, integrating further catalogues describing research-related enterprise functions or features could be possible.</p>
        <p>Nevertheless, publishing HITO and its knowledge base as LOD implies that the contents of HITO are available under an open license. Thus, for the broadly accepted nomenclature SNOMED CT, we could only check the principal suitability of the SNOMED “environment” and “occupation” concepts based on a small set of examples that we included in HITO with permission from SNOMED International. Due to license requirements, SNOMED CT terms cannot be made available as LOD.</p>
        <p>In summary, HITO provides an openly available framework for the description of health care–related software that can be used by researchers who publish studies on digital health interventions or by developers and users who need to describe software.</p>
      </sec>
      <sec>
        <title>Conclusions</title>
        <p>We recognize that health informatics continues to face a terminology problem. Establishing a uniform terminology for software products used in health care is currently unachievable due to several coexisting terminologies from both research and practice. Linking the terms from different terminologies by similarity relationships is the first step toward more transparency. This will also help identify misunderstandings that may be caused by synonyms, homonyms, or conceptual overlaps. Simply knowing that the term “EHR system” can stand for an institutional or cross-institutional application system or a collection of digital documents related to a person's health prevents problems related to misunderstanding. A researcher authoring a study on a digital health intervention by an EHR system knows that the term “EHR system” must be further specified, for example, by enterprise functions and features as listed in HITO.</p>
        <p>Nevertheless, we should further strive for a consented, uniform terminology of health informatics. Taking different coexisting terminologies as a basis, methods of qualitative content analyses such as inductive category formation [<xref ref-type="bibr" rid="ref38">38</xref>], supported by (semi)automatic text extraction, may lead the way toward an established language for health informatics.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group/>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">API</term>
          <def>
            <p>application programming interface</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">DICOM</term>
          <def>
            <p>Digital Imaging and Communications in Medicine</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">EHR</term>
          <def>
            <p>electronic health record</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb4">HITO</term>
          <def>
            <p>Health Information Technology Ontology</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb5">HL7</term>
          <def>
            <p>Health Level 7</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb6">LIFOSS</term>
          <def>
            <p>libre/free and open-source software</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb7">LIS</term>
          <def>
            <p>laboratory information system</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb8">LOD</term>
          <def>
            <p>Linked Open Data</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb9">OWL</term>
          <def>
            <p>Web Ontology Language</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb10">PACS</term>
          <def>
            <p>picture archiving and communication system</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb11">RDF</term>
          <def>
            <p>Resource Description Framework</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb12">RDFS</term>
          <def>
            <p>Resource Description Framework Schema</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb13">SNOMED</term>
          <def>
            <p>Systematized Nomenclature of Medicine</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb14">SNOMED CT</term>
          <def>
            <p>Systematized Nomenclature of Medicine Clinical Terms</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb15">SPARQL</term>
          <def>
            <p>SPARQL Protocol and RDF Query Language</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb16">SWOT</term>
          <def>
            <p>Strengths, Weaknesses, Opportunities, and Threats</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb17">VNA</term>
          <def>
            <p>vendor-neutral archive</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb18">W3C</term>
          <def>
            <p>World Wide Web Consortium</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb19">WHO</term>
          <def>
            <p>World Health Organization</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>The work presented here is part of the project “HITO: a Health IT Ontology,” funded by the Deutsche Forschungsgemeinschaft (DFG, German Research Foundation) - WI 1605/11-1 and the Austrian Science Fund (I 3726-N31). The publication of this article was partly funded by the Open Access Publishing Fund of Leipzig University, supported by the German Research Foundation for the Open Access Publication Funding program.</p>
    </ack>
    <fn-group>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Aminpour</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Sadoughi</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Ahamdi</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Utilization of open source electronic health record around the world: A systematic review</article-title>
          <source>J Res Med Sci</source>
          <year>2014</year>
          <month>01</month>
          <volume>19</volume>
          <issue>1</issue>
          <fpage>57</fpage>
          <lpage>64</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.jmsjournal.net/article.asp?issn=1735-1995;year=2014;volume=19;issue=1;spage=57;epage=64;aulast=Aminpour"/>
          </comment>
          <pub-id pub-id-type="medline">24672566</pub-id>
          <pub-id pub-id-type="pmcid">PMC3963324</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Syzdykova</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Malta</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Zolfo</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Diro</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Oliveira</surname>
              <given-names>JL</given-names>
            </name>
          </person-group>
          <article-title>Open-source electronic health record systems for low-resource settings: systematic review</article-title>
          <source>JMIR Med Inform</source>
          <year>2017</year>
          <month>11</month>
          <day>13</day>
          <volume>5</volume>
          <issue>4</issue>
          <fpage>e44</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://medinform.jmir.org/2017/4/e44/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/medinform.8131</pub-id>
          <pub-id pub-id-type="medline">29133283</pub-id>
          <pub-id pub-id-type="pii">v5i4e44</pub-id>
          <pub-id pub-id-type="pmcid">PMC5703976</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Millard</surname>
              <given-names>PS</given-names>
            </name>
            <name name-style="western">
              <surname>Bru</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Berger</surname>
              <given-names>CA</given-names>
            </name>
          </person-group>
          <article-title>Open-source point-of-care electronic medical records for use in resource-limited settings: systematic review and questionnaire surveys</article-title>
          <source>BMJ Open</source>
          <year>2012</year>
          <volume>2</volume>
          <issue>4</issue>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmjopen.bmj.com/lookup/pmidlookup?view=long&#38;pmid=22763661"/>
          </comment>
          <pub-id pub-id-type="doi">10.1136/bmjopen-2011-000690</pub-id>
          <pub-id pub-id-type="medline">22763661</pub-id>
          <pub-id pub-id-type="pii">bmjopen-2011-000690</pub-id>
          <pub-id pub-id-type="pmcid">PMC3391372</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Haddad</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Selsky</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Hoffman</surname>
              <given-names>JE</given-names>
            </name>
            <name name-style="western">
              <surname>Kravitz</surname>
              <given-names>RL</given-names>
            </name>
            <name name-style="western">
              <surname>Estrin</surname>
              <given-names>DE</given-names>
            </name>
            <name name-style="western">
              <surname>Sim</surname>
              <given-names>I</given-names>
            </name>
          </person-group>
          <article-title>Making sense of mobile health data: an open architecture to improve individual- and population-level health</article-title>
          <source>J Med Internet Res</source>
          <year>2012</year>
          <volume>14</volume>
          <issue>4</issue>
          <fpage>e112</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.jmir.org/2012/4/e112/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.2152</pub-id>
          <pub-id pub-id-type="medline">22875563</pub-id>
          <pub-id pub-id-type="pii">v14i4e112</pub-id>
          <pub-id pub-id-type="pmcid">PMC3510692</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ngabo</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Nguimfack</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Nwaigwe</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Mugeni</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Muhoza</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Wilson</surname>
              <given-names>DR</given-names>
            </name>
            <name name-style="western">
              <surname>Kalach</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Gakuba</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Karema</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Binagwaho</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Designing and implementing an innovative SMS-based alert system (RapidSMS-MCH) to monitor pregnancy and reduce maternal and child deaths in Rwanda</article-title>
          <source>Pan Afr Med J</source>
          <year>2012</year>
          <volume>13</volume>
          <fpage>31</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/23330022"/>
          </comment>
          <pub-id pub-id-type="medline">23330022</pub-id>
          <pub-id pub-id-type="pmcid">PMC3542808</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <collab>Rawat S</collab>
          </person-group>
          <article-title>Impact of open source during COVID-19 pandemic</article-title>
          <source>OpenSense Labs</source>
          <year>2021</year>
          <month>01</month>
          <day>28</day>
          <access-date>2022-12-20</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://opensenselabs.com/blog/articles/open-source-covid-19-pandemic">https://opensenselabs.com/blog/articles/open-source-covid-19-pandemic</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Alanzi</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>A review of mobile applications available in the app and Google Play stores used during the COVID-19 outbreak</article-title>
          <source>J Multidiscip Healthc</source>
          <year>2021</year>
          <volume>14</volume>
          <fpage>45</fpage>
          <lpage>57</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://dx.doi.org/10.2147/JMDH.S285014"/>
          </comment>
          <pub-id pub-id-type="doi">10.2147/JMDH.S285014</pub-id>
          <pub-id pub-id-type="medline">33469298</pub-id>
          <pub-id pub-id-type="pii">285014</pub-id>
          <pub-id pub-id-type="pmcid">PMC7812813</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Streidl</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Demski</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Karopka</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>Medical free/libre and open source software</article-title>
          <source>Medfloss.org</source>
          <access-date>2022-01-13</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://medfloss.org/">https://medfloss.org/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zaidan</surname>
              <given-names>AA</given-names>
            </name>
            <name name-style="western">
              <surname>Zaidan</surname>
              <given-names>BB</given-names>
            </name>
            <name name-style="western">
              <surname>Al-Haiqi</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Kiah</surname>
              <given-names>MLM</given-names>
            </name>
            <name name-style="western">
              <surname>Hussain</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Abdulnabi</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Evaluation and selection of open-source EMR software packages based on integrated AHP and TOPSIS</article-title>
          <source>J Biomed Inform</source>
          <year>2015</year>
          <month>03</month>
          <volume>53</volume>
          <fpage>390</fpage>
          <lpage>404</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S1532-0464(14)00262-7"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.jbi.2014.11.012</pub-id>
          <pub-id pub-id-type="medline">25483886</pub-id>
          <pub-id pub-id-type="pii">S1532-0464(14)00262-7</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Purkayastha</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Allam</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Maity</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Gichoya</surname>
              <given-names>JW</given-names>
            </name>
          </person-group>
          <article-title>Comparison of open-source electronic health record systems based on functional and user performance criteria</article-title>
          <source>Healthc Inform Res</source>
          <year>2019</year>
          <month>04</month>
          <volume>25</volume>
          <issue>2</issue>
          <fpage>89</fpage>
          <lpage>98</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.e-hir.org/DOIx.php?id=10.4258/hir.2019.25.2.89"/>
          </comment>
          <pub-id pub-id-type="doi">10.4258/hir.2019.25.2.89</pub-id>
          <pub-id pub-id-type="medline">31131143</pub-id>
          <pub-id pub-id-type="pmcid">PMC6517630</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Flores</surname>
              <given-names>ZAE</given-names>
            </name>
            <name name-style="western">
              <surname>Win</surname>
              <given-names>KT</given-names>
            </name>
            <name name-style="western">
              <surname>Susilo</surname>
              <given-names>W</given-names>
            </name>
          </person-group>
          <article-title>Functionalities of free and open electronic health record systems</article-title>
          <source>Int J Technol Assess Health Care</source>
          <year>2010</year>
          <month>10</month>
          <volume>26</volume>
          <issue>4</issue>
          <fpage>382</fpage>
          <lpage>9</lpage>
          <pub-id pub-id-type="doi">10.1017/S0266462310001121</pub-id>
          <pub-id pub-id-type="medline">20974022</pub-id>
          <pub-id pub-id-type="pii">S0266462310001121</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <collab>WHO</collab>
          </person-group>
          <article-title>Classification of Digital Health Interventions v 1.0: a shared language to describe the uses of digital technology for health</article-title>
          <source>World Health Organization</source>
          <year>2018</year>
          <access-date>2022-12-22</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://apps.who.int/iris/bitstream/handle/10665/260480/WHO-RHR-18.06-eng.pdf">https://apps.who.int/iris/bitstream/handle/10665/260480/WHO-RHR-18.06-eng.pdf</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="web">
          <article-title>HL7 Electronic Health Record System Functional Model, Release 2.1</article-title>
          <source>Health Level 7 International</source>
          <year>2020</year>
          <access-date>2022-12-21</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.hl7.org/implement/standards/product_brief.cfm?product_id=528">http://www.hl7.org/implement/standards/product_brief.cfm?product_id=528</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Winter</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Haux</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Ammenwerth</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Brigl</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Hellrung</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Jahn</surname>
              <given-names>F</given-names>
            </name>
          </person-group>
          <source>Health Information Systems - Architectures and Strategies</source>
          <year>2011</year>
          <publisher-loc>London, UK</publisher-loc>
          <publisher-name>Springer</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Berners-Lee</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>Linked data: design issues</article-title>
          <source>W3C</source>
          <year>2009</year>
          <access-date>2019-11-06</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.w3.org/DesignIssues/LinkedData.html">https://www.w3.org/DesignIssues/LinkedData.html</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Dermeval</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Vilela</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Bittencourt</surname>
              <given-names>II</given-names>
            </name>
            <name name-style="western">
              <surname>Castro</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Isotani</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Brito</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Silva</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Applications of ontologies in requirements engineering: a systematic review of the literature</article-title>
          <source>Requirements Eng</source>
          <year>2015</year>
          <month>2</month>
          <day>14</day>
          <volume>21</volume>
          <issue>4</issue>
          <fpage>405</fpage>
          <lpage>437</lpage>
          <pub-id pub-id-type="doi">10.1007/s00766-015-0222-6</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Dornauer</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Jahn</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Hoeffner</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Winter</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Ammenwerth</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Use of natural language processing for precise retrieval of key elements of health IT evaluation studies</article-title>
          <source>Stud Health Technol Inform</source>
          <year>2020</year>
          <month>06</month>
          <day>26</day>
          <volume>272</volume>
          <fpage>95</fpage>
          <lpage>98</lpage>
          <pub-id pub-id-type="doi">10.3233/SHTI200502</pub-id>
          <pub-id pub-id-type="medline">32604609</pub-id>
          <pub-id pub-id-type="pii">SHTI200502</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Trant</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Studying social tagging and folksonomy: a review and framework</article-title>
          <source>Digital Libraries and User-Generated Content</source>
          <year>2009</year>
          <volume>10</volume>
          <issue>1</issue>
          <fpage>1</fpage>
          <lpage>44</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Höffner</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>RickView: A fast RDF viewer (linked data browser)</article-title>
          <source>Crates.io</source>
          <year>2022</year>
          <access-date>2022-10-18</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://crates.io/crates/rickview">https://crates.io/crates/rickview</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lehmann</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Isele</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Jakob</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Jentzsch</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Kontokostas</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Mendes</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Hellmann</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Morsey</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>van</surname>
              <given-names>KP</given-names>
            </name>
            <name name-style="western">
              <surname>Auer</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Bizer</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>DBpedia - A large-scale, multilingual knowledge base extracted from Wikipedia</article-title>
          <source>SWJ</source>
          <year>2015</year>
          <volume>6</volume>
          <issue>2</issue>
          <fpage>167</fpage>
          <lpage>195</lpage>
          <pub-id pub-id-type="doi">10.3233/sw-140134</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hofer</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Hellmann</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Dojchinovski</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Frey</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>The new DBpedia release cycle: increasing agility and efficiency in knowledge extraction workflows</article-title>
          <year>2020</year>
          <conf-name>16th International Conference on Semantic Systems</conf-name>
          <conf-date>September 7–10</conf-date>
          <conf-loc>Amsterdam, the Netherlands</conf-loc>
          <fpage>1</fpage>
          <lpage>18</lpage>
          <pub-id pub-id-type="doi">10.1007/978-3-030-59833-4</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Malone</surname>
              <given-names>James</given-names>
            </name>
            <name name-style="western">
              <surname>Brown</surname>
              <given-names>Andy</given-names>
            </name>
            <name name-style="western">
              <surname>Lister</surname>
              <given-names>Allyson L</given-names>
            </name>
            <name name-style="western">
              <surname>Ison</surname>
              <given-names>Jon</given-names>
            </name>
            <name name-style="western">
              <surname>Hull</surname>
              <given-names>Duncan</given-names>
            </name>
            <name name-style="western">
              <surname>Parkinson</surname>
              <given-names>Helen</given-names>
            </name>
            <name name-style="western">
              <surname>Stevens</surname>
              <given-names>Robert</given-names>
            </name>
          </person-group>
          <article-title>The Software Ontology (SWO): a resource for reproducibility in biomedical data analysis, curation and digital preservation</article-title>
          <source>J Biomed Semantics</source>
          <year>2014</year>
          <volume>5</volume>
          <fpage>25</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://jbiomedsem.biomedcentral.com/articles/10.1186/2041-1480-5-25"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/2041-1480-5-25</pub-id>
          <pub-id pub-id-type="medline">25068035</pub-id>
          <pub-id pub-id-type="pii">2041-1480-5-25</pub-id>
          <pub-id pub-id-type="pmcid">PMC4098953</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Jahn</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Bindel</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Höffner</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Ghalandari</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Schneider</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Stäubert</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Dornauer</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Karopka</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Ammenwerth</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Winter</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Towards precise descriptions of medical free/libre and open source software</article-title>
          <source>Stud Health Technol Inform</source>
          <year>2020</year>
          <month>06</month>
          <day>16</day>
          <volume>270</volume>
          <fpage>463</fpage>
          <lpage>468</lpage>
          <pub-id pub-id-type="doi">10.3233/SHTI200203</pub-id>
          <pub-id pub-id-type="medline">32570427</pub-id>
          <pub-id pub-id-type="pii">SHTI200203</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Varshney</surname>
              <given-names>U</given-names>
            </name>
            <name name-style="western">
              <surname>Nickerson</surname>
              <given-names>RC</given-names>
            </name>
            <name name-style="western">
              <surname>Muntermann</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Taxonomy development in health-IT</article-title>
          <source>Proceedings of the 19th Americas Conference on Information Systems</source>
          <year>2013</year>
          <conf-name>AMCIS'13</conf-name>
          <conf-date>August 15-17</conf-date>
          <conf-loc>Chicago, Illinois</conf-loc>
          <fpage>1</fpage>
          <lpage>10</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="web">
          <article-title>SNOMED CT browser</article-title>
          <source>SNOMED International</source>
          <access-date>2022-01-19</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://browser.ihtsdotools.org/?">https://browser.ihtsdotools.org/?</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Joshi</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Narra</surname>
              <given-names>VR</given-names>
            </name>
            <name name-style="western">
              <surname>Joshi</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Melson</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>PACS administrators' and radiologists' perspective on the importance of features for PACS selection</article-title>
          <source>J Digit Imaging</source>
          <year>2014</year>
          <month>08</month>
          <volume>27</volume>
          <issue>4</issue>
          <fpage>486</fpage>
          <lpage>95</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/24744278"/>
          </comment>
          <pub-id pub-id-type="doi">10.1007/s10278-014-9682-3</pub-id>
          <pub-id pub-id-type="medline">24744278</pub-id>
          <pub-id pub-id-type="pmcid">PMC4090402</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="web">
          <article-title>LIS Functionality Assessment Toolkit</article-title>
          <source>Association for Pathology Informatics</source>
          <access-date>2022-01-19</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.pathologyinformatics.org/lis-toolkit">https://www.pathologyinformatics.org/lis-toolkit</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="web">
          <article-title>HITO: A Health IT ontology for systematically describing application systems and software products in health it</article-title>
          <source>HITO</source>
          <year>2022</year>
          <access-date>2022-04-12</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://hitontology.eu">https://hitontology.eu</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Jordogne</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Osimis</surname>
              <given-names>SE</given-names>
            </name>
          </person-group>
          <article-title>Orthanc: Open-source, lightweight DICOM server</article-title>
          <source>Orthanc</source>
          <year>2022</year>
          <access-date>2022-02-07</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.orthanc-server.com/index.php">https://www.orthanc-server.com/index.php</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Jodogne</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>The Orthanc ecosystem for medical imaging</article-title>
          <source>J Digit Imaging</source>
          <year>2018</year>
          <month>06</month>
          <volume>31</volume>
          <issue>3</issue>
          <fpage>341</fpage>
          <lpage>352</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/29725964"/>
          </comment>
          <pub-id pub-id-type="doi">10.1007/s10278-018-0082-y</pub-id>
          <pub-id pub-id-type="medline">29725964</pub-id>
          <pub-id pub-id-type="pii">10.1007/s10278-018-0082-y</pub-id>
          <pub-id pub-id-type="pmcid">PMC5959835</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="web">
          <article-title>SPARQL Query Editor</article-title>
          <source>HITO</source>
          <access-date>2022-03-24</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://hitontology.eu/sparql">http://hitontology.eu/sparql</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="web">
          <article-title>HITO Health IT Ontology</article-title>
          <source>HITO</source>
          <access-date>2022-03-22</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://hitontology.eu/ontology/">https://hitontology.eu/ontology/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="web">
          <article-title>HITO software products search</article-title>
          <source>HITO</source>
          <access-date>2022-03-24</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://hitontology.eu/search/softwareproduct.html">https://hitontology.eu/search/softwareproduct.html</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="web">
          <article-title>The Health IT Ontology</article-title>
          <source>GitHub Repository</source>
          <access-date>2022-10-13</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://github.com/hitontology/ontology">https://github.com/hitontology/ontology</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="web">
          <article-title>21838-2:2021 Information technology — Top-level ontologies (TLO) — Part 2: Basic Formal Ontology (BFO)</article-title>
          <source>International Organization for Standardization</source>
          <year>2021</year>
          <access-date>2022-10-26</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.iso.org/standard/74572.html">https://www.iso.org/standard/74572.html</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref36">
        <label>36</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Loebe</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Burek</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Herre</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>GFO: the General Formal Ontology</article-title>
          <source>Appl Ontol</source>
          <year>2022</year>
          <month>03</month>
          <day>15</day>
          <volume>17</volume>
          <issue>1</issue>
          <fpage>71</fpage>
          <lpage>106</lpage>
          <pub-id pub-id-type="doi">10.3233/ao-220264</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref37">
        <label>37</label>
        <nlm-citation citation-type="web">
          <article-title>Semantic arts: gist</article-title>
          <source>GitHub Repository</source>
          <access-date>2022-10-25</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://github.com/semanticarts/gist">https://github.com/semanticarts/gist</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref38">
        <label>38</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mayring</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <person-group person-group-type="editor">
            <name name-style="western">
              <surname>Bikner-Ahsbahs</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Knipping</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Presmeg</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>Qualitative content analysis: theoretical background and procedures</article-title>
          <source>Approaches to Qualitative Research in Mathematics Education</source>
          <year>2015</year>
          <publisher-loc>Dordrecht, the Netherlands</publisher-loc>
          <publisher-name>Springer</publisher-name>
          <fpage>365</fpage>
          <lpage>380</lpage>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
