<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMI</journal-id>
      <journal-id journal-id-type="nlm-ta">JMIR Med Inform</journal-id>
      <journal-title>JMIR Medical Informatics</journal-title>
      <issn pub-type="epub">2291-9694</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v8i9e18846</article-id>
      <article-id pub-id-type="pmid">32955457</article-id>
      <article-id pub-id-type="doi">10.2196/18846</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Original Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Original Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Chronological Age Assessment in Young Individuals Using Bone Age Assessment Staging and Nonradiological Aspects: Machine Learning Multifactorial Approach</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Eysenbach</surname>
            <given-names>Gunther</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Sokolova</surname>
            <given-names>Marina</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Carvalho</surname>
            <given-names>Darlinton</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Kocbek</surname>
            <given-names>Primoz</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Brumen</surname>
            <given-names>Bostjan</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author">
          <name name-style="western">
            <surname>Dallora</surname>
            <given-names>Ana Luiza</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-6752-017X</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author">
          <name name-style="western">
            <surname>Kvist</surname>
            <given-names>Ola</given-names>
          </name>
          <degrees>MD</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-8730-5901</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author">
          <name name-style="western">
            <surname>Berglund</surname>
            <given-names>Johan Sanmartin</given-names>
          </name>
          <degrees>PhD, MD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-4312-2246</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author">
          <name name-style="western">
            <surname>Ruiz</surname>
            <given-names>Sandra Diaz</given-names>
          </name>
          <degrees>PhD, MD</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-2016-4195</ext-link>
        </contrib>
        <contrib id="contrib5" contrib-type="author">
          <name name-style="western">
            <surname>Boldt</surname>
            <given-names>Martin</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff3" ref-type="aff">3</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-9316-4842</ext-link>
        </contrib>
        <contrib id="contrib6" contrib-type="author">
          <name name-style="western">
            <surname>Flodmark</surname>
            <given-names>Carl-Erik</given-names>
          </name>
          <degrees>PhD, MD</degrees>
          <xref rid="aff4" ref-type="aff">4</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-9705-4766</ext-link>
        </contrib>
        <contrib id="contrib7" contrib-type="author" corresp="yes">
          <name name-style="western">
            <surname>Anderberg</surname>
            <given-names>Peter</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <address>
            <institution>Department of Health</institution>
            <institution>Blekinge Institute of Technology</institution>
            <addr-line>Valhallavägen 1</addr-line>
            <addr-line>Karlskrona, 371 41</addr-line>
            <country>Sweden</country>
            <phone>46 0734223736</phone>
            <email>peter.anderberg@bth.se</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-9870-8477</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Department of Health</institution>
        <institution>Blekinge Institute of Technology</institution>
        <addr-line>Karlskrona</addr-line>
        <country>Sweden</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Department of Pediatric Radiology</institution>
        <institution>Karolinska University Hospital</institution>
        <addr-line>Stockholm</addr-line>
        <country>Sweden</country>
      </aff>
      <aff id="aff3">
        <label>3</label>
        <institution>Department of Computer Science</institution>
        <institution>Blekinge Institute of Technology</institution>
        <addr-line>Karlskrona</addr-line>
        <country>Sweden</country>
      </aff>
      <aff id="aff4">
        <label>4</label>
        <institution>Department of Clinical Sciences</institution>
        <institution>Lund University</institution>
        <addr-line>Lund</addr-line>
        <country>Sweden</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Peter Anderberg <email>peter.anderberg@bth.se</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <month>9</month>
        <year>2020</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>21</day>
        <month>9</month>
        <year>2020</year>
      </pub-date>
      <volume>8</volume>
      <issue>9</issue>
      <elocation-id>e18846</elocation-id>
      <history>
        <date date-type="received">
          <day>23</day>
          <month>3</month>
          <year>2020</year>
        </date>
        <date date-type="rev-request">
          <day>12</day>
          <month>6</month>
          <year>2020</year>
        </date>
        <date date-type="rev-recd">
          <day>6</day>
          <month>8</month>
          <year>2020</year>
        </date>
        <date date-type="accepted">
          <day>13</day>
          <month>8</month>
          <year>2020</year>
        </date>
      </history>
      <copyright-statement>©Ana Luiza Dallora, Ola Kvist, Johan Sanmartin Berglund, Sandra Diaz Ruiz, Martin Boldt, Carl-Erik Flodmark, Peter Anderberg. Originally published in JMIR Medical Informatics (http://medinform.jmir.org), 21.09.2020.</copyright-statement>
      <copyright-year>2020</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Medical Informatics, is properly cited. The complete bibliographic information, a link to the original publication on http://medinform.jmir.org/, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="http://medinform.jmir.org/2020/9/e18846/" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>Bone age assessment (BAA) is used in numerous pediatric clinical settings as well as in legal settings when entities need an estimate of chronological age (CA) when valid documents are lacking. The latter case presents itself as critical as the law is harsher for adults and granted rights along with imputability changes drastically if the individual is a minor. Traditional BAA methods have drawbacks such as exposure of minors to radiation, they do not consider factors that might affect the bone age, and they mostly focus on a single region. Given the critical scenarios in which BAA can affect the lives of young individuals, it is important to focus on the drawbacks of the traditional methods and investigate the potential of estimating CA through BAA.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>This study aims to investigate CA estimation through BAA in young individuals aged 14-21 years with machine learning methods, addressing the drawbacks of research using magnetic resonance imaging (MRI), assessment of multiple regions of interest, and other factors that may affect the bone age.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>MRI examinations of the radius, distal tibia, proximal tibia, distal femur, and calcaneus were performed on 465 men and 473 women (aged 14-21 years). Measures of weight and height were taken from the subjects, and a questionnaire was given for additional information (self-assessed Tanner Scale, physical activity level, parents' origin, and type of residence during upbringing). Two pediatric radiologists independently assessed the MRI images to evaluate their stage of bone development (blinded to age, gender, and each other). All the gathered information was used in training machine learning models for CA estimation and minor versus adult classification (threshold of 18 years). Different machine learning methods were investigated.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>The minor versus adult classification produced accuracies of 0.90 and 0.84 for male and female subjects, respectively, with high recalls for the classification of minors. The CA estimation for the 8 age groups (aged 14-21 years) achieved mean absolute errors of 0.95 years and 1.24 years for male and female subjects, respectively. However, for the latter, a lower error occurred only for the ages of 14 and 15 years.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>This study investigates CA estimation through BAA using machine learning methods in 2 ways: minor versus adult classification and CA estimation in 8 age groups (aged 14-21 years), while addressing the drawbacks in the research on BAA. The first achieved good results; however, for the second case, the BAA was not precise enough for the classification.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>chronological age assessment</kwd>
        <kwd>bone age</kwd>
        <kwd>skeletal maturity</kwd>
        <kwd>machine learning</kwd>
        <kwd>magnetic resonance imaging</kwd>
        <kwd>radius</kwd>
        <kwd>distal tibia</kwd>
        <kwd>proximal tibia</kwd>
        <kwd>distal femur</kwd>
        <kwd>calcaneus</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <sec>
        <title>Background</title>
        <p>Skeletal maturity is a radiological concept that refers to the stage of bone development in an individual [<xref ref-type="bibr" rid="ref1">1</xref>]. This maturation process occurs gradually in the growth plates and is measured by the degree of mineralization of the bone along with its size and shape [<xref ref-type="bibr" rid="ref1">1</xref>]. Bone age (BA) is a closely related concept in which age is estimated based on the degree of skeletal maturity of an individual [<xref ref-type="bibr" rid="ref2">2</xref>].</p>
        <p>The estimation of the BA of an individual, or bone age assessment (BAA), is performed in numerous clinical settings involving diagnosis and time of treatment of orthopedics, orthodontics, endocrinology, growth disorders, and estimations of final height [<xref ref-type="bibr" rid="ref3">3</xref>]. In these cases, the BA of an individual is assessed by medical professionals and compared with their chronological age (CA). If they are found to be relatively advanced or retarded, appropriate actions are taken by the medical professionals.</p>
        <p>BAA is also performed outside the clinical setting when legal entities need an estimation of the CA of an individual for judicial decisions when valid documents are lacking. This refers to cases regarding adoption, criminal proceedings, and pedopornography judicial issues as well as in determining age fraud in youth sports competitions [<xref ref-type="bibr" rid="ref4">4</xref>-<xref ref-type="bibr" rid="ref7">7</xref>]. Furthermore, with the upsurge of immigration due to the rise of worldwide conflicts, another critical scenario in which BAA is applied concerns the determination of an individual being minor in the absence of valid or trustworthy documents. This is the case of numerous young asylum seekers who are given special rights granted by the United Nations Convention on the Rights of the Child, regarding reception, health care, and education [<xref ref-type="bibr" rid="ref8">8</xref>,<xref ref-type="bibr" rid="ref9">9</xref>].</p>
        <p>From these examples, it is possible to assume that, especially regarding legal standpoints, BAA is a crucial tool for making high stake decisions that have the potential to greatly affect individuals’ lives.</p>
      </sec>
      <sec>
        <title>Traditional BAA</title>
        <p>The traditional methods for BAA are based on the appearance of growth plates through the analysis of diaphysis (primary ossification centers) and epiphysis (secondary ossification centers), where cartilage tissue gradually turns into bone tissue during the process of bone development. A process that ceases when the diaphysis and epiphysis are fused, indicating that the growth plate is ossified [<xref ref-type="bibr" rid="ref1">1</xref>].</p>
        <p>The most common procedures for BAA are the Greulich-Pyle (GP) and Tanner-Whitehouse (TW) methods. Both of these methods assess radiographic images of the hand and wrist areas as these are regions of interest (ROIs) with a large number of ossification centers aggregated in a small area that can easily have images taken from.</p>
        <p>The GP method [<xref ref-type="bibr" rid="ref10">10</xref>] attributes BA by comparing the radiograph image of the individual being assessed to the nearest reference image in a hand and wrist atlas in terms of bone development. The TW method [<xref ref-type="bibr" rid="ref11">11</xref>] is a scoring system that evaluates the ulna, radius, carpals, and 13 short bones of the hand. Scores are attributed to these regions based on the stage of bone development, which ranges from A to I. The scores are then aggregated in a total score that is converted into the BA.</p>
        <p>Having been developed in the 30s and 50s, the GP and TW methods, respectively, conveyed groundbreaking developments in numerous clinical settings and are still heavily employed for BAA purposes to this day.</p>
      </sec>
      <sec>
        <title>Other Proposed BAA Methods</title>
        <p>The field of BAA evolved as the GP and TW methods were proposed, exploring new ROIs with different ossification timings. This section summarizes the proposed studies regarding BAA in various ROIs.</p>
        <p>Newer hand and wrist studies on BAA include the Gilsanz and Ratib [<xref ref-type="bibr" rid="ref1">1</xref>] digital hand atlas and the Fels method [<xref ref-type="bibr" rid="ref12">12</xref>]. The first is composed of artificially created reference images that represent the average development of 29 classes of subjects aged from 0 to 18 years. The Fels method [<xref ref-type="bibr" rid="ref12">12</xref>] is a statistical method that provides a relative measure of the BA and standard error that takes into consideration the distribution of chronological ages in the study’s sample with BA similar to the individual being assessed. It is based on 98 indicators of bone maturity (ossification, radiopaque densities, bony projection, shape changes, and ossification of epiphysis).</p>
        <p>Clavicle staging systems observe one or both sides of the medial clavicular epiphysis. The method proposed by Kreitner et al [<xref ref-type="bibr" rid="ref13">13</xref>] presents 4 stages of ossification of the medial clavicular epiphysis, in which the last stage may have an epiphyseal scar visible. Schmeling et al [<xref ref-type="bibr" rid="ref9">9</xref>] proposed 5 stages of ossification, but the last stage was only achieved when the epiphyseal scar was not apparent. Kellinghaus et al [<xref ref-type="bibr" rid="ref14">14</xref>] built on the Schmeling et al [<xref ref-type="bibr" rid="ref9">9</xref>] staging by applying subclassifications for the second and third stages. These studies report complete ossification of this growth plate around the ages of 26 to 27 years.</p>
        <p>Knee studies proposed staging systems that also vary on subscales on specific stages and the appearance of the epiphyseal scar in the last stage. O’Connor et al [<xref ref-type="bibr" rid="ref15">15</xref>] proposed 5 stages of ossification of the distal femur, proximal tibia, and proximal fibula epiphysis (the epiphyseal scar may be visible in the last stage). Dedouit et al [<xref ref-type="bibr" rid="ref16">16</xref>] proposed 5 stages of ossification of the distal femur and proximal tibia epiphysis, assessing the appearance of cartilage signal intensity with magnetic resonance imaging (MRI). Krammer et al [<xref ref-type="bibr" rid="ref17">17</xref>] proposed 5 stages of ossification of the distal femur epiphysis, with subclassifications on the second and third stages, with the last stage achieved only when the epiphyseal scar is no longer visible. This method also makes use of MRI images. Knee studies usually argue that a subject is younger or older than the age of 18 years.</p>
        <p>Studies on foot ROIs are usually concerned with younger ages. Ekizoglu et al [<xref ref-type="bibr" rid="ref18">18</xref>] proposed a staging system for the foot ROI that shows complete ossification in the ages between 12 and 16 years.</p>
        <p>Not very much is explored in the literature, the arm ROI was studied in the proximal humerus epiphysis by Ekizoglu et al [<xref ref-type="bibr" rid="ref19">19</xref>] employing a scoring system based on Schmeling et al [<xref ref-type="bibr" rid="ref9">9</xref>] and Kellinghaus et al [<xref ref-type="bibr" rid="ref14">14</xref>] on MRI images. This study points out the earliest ages for the last stage of ossification at 17 and 18 years.</p>
      </sec>
      <sec>
        <title>Drawbacks in Assessing Chronological Age Using BAA Methods</title>
        <p>In the lack of valid or trustworthy documents, BAA is currently employed as a valuable tool for legal entities to evaluate CA with regard to important legal ages. Nevertheless, it is possible to identify several drawbacks of the largely employed GP and TW methods as well as recently proposed methods, regarding the use of BAA for CA determination:</p>
        <list list-type="bullet">
          <list-item>
            <p>They almost exclusively employ medical imaging techniques that expose the individual to ionizing radiation, such as radiographs, which raises grave ethical issues especially with regard to exposing minors to radiation for nontherapeutic purposes.</p>
          </list-item>
          <list-item>
            <p>They only focus on the physical appearance of the growth plates, not including other information that might possibly affect bone development [<xref ref-type="bibr" rid="ref20">20</xref>].</p>
          </list-item>
          <list-item>
            <p>They mostly focus on a single ROI, which in the vast majority of cases is the hand area [<xref ref-type="bibr" rid="ref20">20</xref>].</p>
          </list-item>
        </list>
        <p>The first drawback can be addressed by the employment of MRI technology, which is already present in some of the mentioned knee and arm studies. Besides being a radiation-free modality of medical imaging, it also allows the manipulation of contrast to highlight different tissue types [<xref ref-type="bibr" rid="ref21">21</xref>]. The epiphyseal plate consists of cartilage tissue, which is mainly composed of collagen fiber protein. Collagen has a 3D structure of fibers that, in MRI images, is shown as zones of different intensities, giving it a multilaminar appearance. It is known that the structure of cartilage changes in terms of the number of laminae and thickness in the course of bone development [<xref ref-type="bibr" rid="ref22">22</xref>]. Hence, contrary to radiographs that highlight the bone, the MRI technology might have the potential to offer better visualization of growth plates, thus being an interesting radiation-free modality of medical imaging for BAA.</p>
        <p>To address the second drawback, the methods for assessing BA should investigate factors that may play a role in the process of bone development and ossification of growth plates, that is, BMI [<xref ref-type="bibr" rid="ref20">20</xref>,<xref ref-type="bibr" rid="ref23">23</xref>], pubertal growth [<xref ref-type="bibr" rid="ref24">24</xref>], physical activity [<xref ref-type="bibr" rid="ref25">25</xref>], ethnicity [<xref ref-type="bibr" rid="ref8">8</xref>,<xref ref-type="bibr" rid="ref20">20</xref>,<xref ref-type="bibr" rid="ref26">26</xref>], and socioeconomic factors [<xref ref-type="bibr" rid="ref8">8</xref>], which are often overlooked [<xref ref-type="bibr" rid="ref20">20</xref>].</p>
        <p>Addressing the third drawback could be done by employing multiple ROIs. When it comes to estimations of CA, most of the BAA studies, especially methods that propose stages of maturity for set ROIs, follow an approach of identifying the minimum age in which the ossification of the growth plate is completed for a particular ROI. These studies usually focus on a single age of legal importance, which varies significantly between countries, with ages ranging from 14 to 21 years [<xref ref-type="bibr" rid="ref13">13</xref>]. Using multiple ROIs may provide more information about more ages.</p>
        <p>An additional drawback that is specific to the GP and TW methods is that they are based on data collected from subjects of average and upper socioeconomic classes in the 30s and 50s, respectively. Hence, these methods may not reflect secular trends that nowadays point to higher height and earlier puberty [<xref ref-type="bibr" rid="ref27">27</xref>], which could affect the accuracy of the methods. For the TW method, an update released in 2001 (TW3) revised the calculation of the BA from the attributed scores to address this problem [<xref ref-type="bibr" rid="ref28">28</xref>].</p>
      </sec>
      <sec>
        <title>Machine Learning for BAA</title>
        <p>From the presented drawbacks, it is noticeable that the BAA research could benefit from methods that are able to aggregate multiple pieces of information (ie, multiple ROIs and factors) in a systematic way. A technology that is able to work in this setting is machine learning (ML), which is already widely employed in diverse medical fields, such as diabetes, cancer, cardiology, mental health, and the analysis of clinical text data [<xref ref-type="bibr" rid="ref29">29</xref>,<xref ref-type="bibr" rid="ref30">30</xref>]. ML consists of various types of algorithms that are able to learn how to perform a task from a set of examples while improving its performance based on its experience in carrying out a particular task. It builds a model that encapsulates the knowledge to perform the task; then, in light of new data, the model is able to correctly perform the learned task within an acceptable measure of performance [<xref ref-type="bibr" rid="ref31">31</xref>].</p>
        <p>ML algorithms have already been employed in various models for assessing the BA of an individual. A recent systematic literature review on BAA with ML methods [<xref ref-type="bibr" rid="ref20">20</xref>] showed that the research is heavily focused on models that make use of a single ROI, the hand in most cases, having radiographs as the choice of imaging technology and do not usually consider other factors that could play a role in bone development [<xref ref-type="bibr" rid="ref20">20</xref>]. The most notable, commercially available ML BAA system is the BoneXpert [<xref ref-type="bibr" rid="ref32">32</xref>], which performs an automatic radiograph analysis based on the GP and TW methods. However, it covers the age range of 2 to 17 years and leaves out important legal ages.</p>
      </sec>
      <sec>
        <title>Objectives of the Study</title>
        <p>Given the importance of the assessment of CA through BAA in numerous scenarios and its potential ways of affecting the lives of young individuals, it is important to focus on the drawbacks of the methods currently in use and investigate the potential of BAA in estimating CA. Thus, the objectives of this study are as follows:</p>
        <list list-type="bullet">
          <list-item>
            <p>To investigate the extent to which ML models can aid in CA estimation through BAA in young individuals aged 14 to 21 years.</p>
          </list-item>
          <list-item>
            <p>To investigate whether ML models can aid in the determination of minors through BAA, considering the threshold of 18 years, in young individuals aged 14 to 21 years.</p>
          </list-item>
          <list-item>
            <p>To address the drawbacks in the research on CA estimation from BAA, with regard to using radiation-free medical imaging technology, the assessment of multiple ROIs and other factors that may play a role in bone development.</p>
          </list-item>
        </list>
      </sec>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Overview</title>
        <p>To train the CA estimation ML models proposed in this paper, MRI images of the wrist, knee, and foot were taken from volunteer subjects and assessed by radiologists to evaluate their stage of bone development. The 5 growth zones considered in this study were calcaneus, distal tibia, proximal tibia, distal femur, and radius. Each growth zone was assessed separately and blinded to gender and age.</p>
        <p>Before the examination, the subjects had their height and weight measured for the BMI calculation and were asked to answer a questionnaire to gather information on their physical activity level, parents’ origin, type of residence during upbringing, and a self-assessed Tanner Scale of pubertal growth [<xref ref-type="bibr" rid="ref33">33</xref>,<xref ref-type="bibr" rid="ref34">34</xref>].</p>
        <p>All radiological and nonradiological data gathered were used to train binary and multiclass classifiers. For the binary classifier, the individuals in the sample were divided into minors or adults, with a threshold of 18 years, and the classification followed into discriminating individuals into 1 of the 2 classes. The multiclass classifier aims to classify an individual into 1 of the 8 classes defined by age groups ranging from 14 to 21 years.</p>
        <p>The remainder of this section details the population, data used in the experiments, statistical analysis, and procedures for model building in the experiments.</p>
      </sec>
      <sec>
        <title>Recruitment</title>
        <p>This study prospectively conducted MRI examinations of 938 healthy subjects (465 males and 473 females) aged between 14 and 21 years (inclusive), during 2017 and 2018. The participants of the study had images taken from the knee, foot, and wrist in the same examination session. Additionally, the weight and height of each participant were also collected to calculate the BMI.</p>
        <p>The following criteria were used to determine participation in the study:</p>
        <list list-type="bullet">
          <list-item>
            <p>Inclusion criteria: the participants should have been born in Sweden, where the study was conducted, and have a birth certificate verified by the Swedish national authorities.</p>
          </list-item>
          <list-item>
            <p>Exclusion criteria: a history of bilateral fractures or trauma near the regions of assessment, a history of chronic disease or the use of long-term medications, noncompliance during the examination, having resided outside Sweden for more than six consecutive months, or past or current pregnancy (all female subjects were tested).</p>
          </list-item>
        </list>
      </sec>
      <sec>
        <title>Data Privacy and Study Ethics</title>
        <p>The study was conducted in accordance with the Declaration of Helsinki and was approved by the Central Ethical Review Board in Stockholm (diary numbers: 2017/4-31/4, 2017/1184-32, 2017/1773-32). Written informed consent was obtained from all subjects and legal guardians (in the case of subjects aged younger than 18 years). All data were anonymized and stratified by age and gender.</p>
      </sec>
      <sec>
        <title>Population</title>
        <p>A total of 455 male and 467 female subjects constituted the final sample (<xref ref-type="table" rid="table1">Table 1</xref>). After the MRI examinations and assessment of images by radiologists, 10 male and 6 female subjects were removed from the study’s sample because they had the assessment of one or more ROI missing. The missing values for the assessment by the radiologists could be due to one of the following reasons: movement artifact, error in the sequence that made the image nongradable, likely trauma in the region of assessment, and missing MRI examination in one or more ROIs.</p>
        <table-wrap position="float" id="table1">
          <label>Table 1</label>
          <caption>
            <p>Demographics of the final sample.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="350"/>
            <col width="70"/>
            <col width="70"/>
            <col width="70"/>
            <col width="70"/>
            <col width="70"/>
            <col width="70"/>
            <col width="70"/>
            <col width="70"/>
            <col width="0"/>
            <col width="90"/>
            <thead>
              <tr valign="top">
                <td>Demographics</td>
                <td colspan="8">Age group</td>
                <td colspan="2">Total</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>14</td>
                <td>15</td>
                <td>16</td>
                <td>17</td>
                <td>18</td>
                <td>19</td>
                <td>20</td>
                <td colspan="2">21</td>
                <td>
                  <break/>
                </td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Number of female subjects</td>
                <td>59</td>
                <td>58</td>
                <td>57</td>
                <td>60</td>
                <td>59</td>
                <td>57</td>
                <td>57</td>
                <td colspan="2">60</td>
                <td>467</td>
              </tr>
              <tr valign="top">
                <td>Number of male subjects</td>
                <td>58</td>
                <td>56</td>
                <td>60</td>
                <td>58</td>
                <td>53</td>
                <td>58</td>
                <td>53</td>
                <td colspan="2">59</td>
                <td>455</td>
              </tr>
            </tbody>
          </table>
        </table-wrap>
      </sec>
      <sec>
        <title>Data and Data Collection Procedures</title>
        <p>The data used to train the classifiers were the radiologists’ assessment of the calcaneus, distal tibia, proximal tibia, distal femur, and radius growth zones; the additional information gathered before the examination was physical activity level, parents’ origin, type of residence during upbringing, and a self-assessed Tanner Scale of pubertal growth and BMI. The following section details the data and procedures for collection.</p>
        <sec>
          <title>MRI Examinations</title>
          <p>MRI examinations were performed to capture images of the calcaneus, distal tibia, proximal tibia, distal femur, and radius growth plates of the subjects participating in the study. All MRI examinations were conducted within 6 months of the subjects’ birthday date on 1.5-T whole-body MRI scanners with dedicated hand, knee, and ankle coils. The examinations were performed on the nondominant side of the knee, hand, and foot, save when past fracture or trauma had taken place near the region. In these cases, the dominant side was imaged. The images of all ROIs were taken in the same examination session.</p>
          <p>The examinations were carried out at 2 sites. Site 1 used Magnetom Avanto Fit (Siemens Healthcare GmbH) and Achieva (Philips Healthcare) whole-body scanners, and Site 2 used a Signa (GE Healthcare) whole-body scanner. All examinations followed the same protocol, which included a T2 sequence with cartilage dedicated exposure. The settings were 256×256 pixel resolution and 160×160 mm field of view.</p>
        </sec>
        <sec>
          <title>Assessment of Magnetic Resonance Images</title>
          <p>The assessment of the MRI images was performed independently by 2 radiologists with 3 and 30 years of experience in pediatric radiology, who were blinded to the age and gender of the participants. A third radiologist with 13 years of experience in pediatric radiology assessed the images when the first 2 radiologists could not reach a final agreement about the stage.</p>
          <p>The staging system used to assess MRI images is a version of the staging methods proposed by Dedouit et al [<xref ref-type="bibr" rid="ref16">16</xref>] and Kellinghaus et al [<xref ref-type="bibr" rid="ref14">14</xref>] with minor modifications. This staging is defined as follows:</p>
          <list list-type="bullet">
            <list-item>
              <p>Stage 1: Continuous, stripe-like, cartilage signal intensity is present between the metaphysis and epiphysis with a thickness greater than 1.5 mm with a multilaminar appearance.</p>
            </list-item>
            <list-item>
              <p>Stage 2: Continuous cartilage signal intensity is present between the metaphysis and epiphysis with a thickness greater than 1.5 mm with increased signal intensity but without a multilaminar appearance.</p>
            </list-item>
            <list-item>
              <p>Stage 3: Continuous cartilage signal intensity is present between the metaphysis and epiphysis with a thickness of less than 1.5 mm with increased signal intensity.</p>
            </list-item>
            <list-item>
              <p>Stage 4a: Noncontinuous cartilage signal intensity. A hazy area involving one-third or less of the growth plate is present between the metaphysis and epiphysis, representing the epiphyseal-metaphyseal fusion.</p>
            </list-item>
            <list-item>
              <p>Stage 4b: Noncontinuous cartilage signal intensity. A hazy area involving between one-third and two-third of the growth plate is present between the metaphysis and epiphysis, representing epiphyseal-metaphyseal fusion.</p>
            </list-item>
            <list-item>
              <p>Stage 4c: Noncontinuous cartilage signal intensity A hazy area involving more than two-thirds of the growth plate is present between the metaphysis and epiphysis, representing epiphyseal-metaphyseal fusion.</p>
            </list-item>
            <list-item>
              <p>Stage 5: The epiphyseal cartilage fused completely with or without an epiphyseal scar in all MRI slices.</p>
            </list-item>
          </list>
        </sec>
        <sec>
          <title>Body Mass Index</title>
          <p>The BMI was calculated using the measures of the participants’ weight <italic>w</italic> and height <italic>h</italic>, as in the following equation 1 [<xref ref-type="bibr" rid="ref35">35</xref>]:</p>
          <graphic xlink:href="medinform_v8i9e18846_fig3.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          <p>Data characteristics regarding the calculated BMI for the subjects are shown in the <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>.</p>
        </sec>
        <sec>
          <title>Questionnaire Information</title>
          <p>Additional information from the participants was gathered by a questionnaire given to them at the examination session. The information gathered by the questionnaire refers to the variables “Residence,” “Physical Activity,” “Parent Origin,” and “Tanner Scale,” shown in <xref ref-type="table" rid="table2">Table 2</xref>, which summarizes all input and output variables considered for building the models. Data characteristics regarding the data collected by the questionnaire are shown in the <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>.</p>
          <table-wrap position="float" id="table2">
            <label>Table 2</label>
            <caption>
              <p>Summary of the input and output variables considered in the model building.</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="30"/>
              <col width="120"/>
              <col width="550"/>
              <col width="300"/>
              <thead>
                <tr valign="top">
                  <td colspan="2">Variable</td>
                  <td>Description</td>
                  <td>Values</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td colspan="3">
                    <bold>Input variables</bold>
                  </td>
                  <td>
                    <break/>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Radius</td>
                  <td>Radiologists’ assessments of the Radius growth zone</td>
                  <td>Stage 1; Stage 2; Stage 3; Stage 4a; Stage 4b; Stage 4c; Stage 5</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Distal femur</td>
                  <td>Radiologists’ assessments of the distal femur growth zone</td>
                  <td>Stage 1; Stage 2; Stage 3; Stage 4a; Stage 4b; Stage 4c; Stage 5</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Proximal tibia</td>
                  <td>Radiologists’ assessments of the proximal tibia growth zone</td>
                  <td>Stage 1; Stage 2; Stage 3; Stage 4a; Stage 4b; Stage 4c; Stage 5</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Distal tibia</td>
                  <td>Radiologists’ assessments of the distal tibia growth zone</td>
                  <td>Stage 1; Stage 2; Stage 3; Stage 4a; Stage 4b; Stage 4c; Stage 5</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Calcaneus</td>
                  <td>Radiologists’ assessments of the calcaneus growth zone</td>
                  <td>Stage 1; Stage 2; Stage 3; Stage 4a; Stage 4b; Stage 4c; Stage 5</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>BMI</td>
                  <td>Body mass index of the participant, calculated as in the equation (1)</td>
                  <td>Numeric</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Residence</td>
                  <td>Type of residence the participant lives in (or lived during upbringing)</td>
                  <td>Rented; owned</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Physical activity</td>
                  <td>The participants’ daily level of activity</td>
                  <td>Highly inactive; inactive; little active; active; highly active</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Parent origin</td>
                  <td>Origin of the participants’ parents, regarding if they were born outside Sweden or not</td>
                  <td>No foreign-born parents; one foreign-born parent; both foreign-born parents</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Tanner scale</td>
                  <td>Self-assessed Tanner Scale for pubertal growth [<xref ref-type="bibr" rid="ref33">33</xref>,<xref ref-type="bibr" rid="ref34">34</xref>]</td>
                  <td>Stage 1; Stage 2; Stage 3; Stage 4; Stage 5</td>
                </tr>
                <tr valign="top">
                  <td colspan="4">
                    <bold>Output variables</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Minor</td>
                  <td>Characterizes the participant as being a minor or not, regarding the threshold of 18 years. This is the output variable for the binary classification models</td>
                  <td>Yes; no</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Age</td>
                  <td>Regards the age group which the participant belongs to. This is the output variable for the multi-class classification models</td>
                  <td>14; 15; 16; 17; 18; 19; 20; 21</td>
                </tr>
              </tbody>
            </table>
          </table-wrap>
        </sec>
      </sec>
      <sec>
        <title>Statistical Analyses</title>
        <p>The Cohen kappa coefficient [<xref ref-type="bibr" rid="ref36">36</xref>] and percent of agreement [<xref ref-type="bibr" rid="ref37">37</xref>] were calculated to measure the interobserver agreement between the pediatric radiologists in all investigated ROIs. Statistical analyses were performed using SPSS Statistics (version 24; IBM Corp).</p>
      </sec>
      <sec>
        <title>Model Building</title>
        <sec>
          <title>CA Estimation Models</title>
          <p>In this study, various ML algorithms were investigated to build classifiers to discriminate subjects into minor (positive class) or adults (negative class) and classifiers to classify subjects into 1 of 8 age groups (14 to 21 years). Models for male and female subjects were built separately.</p>
        </sec>
        <sec>
          <title>Data Preprocessing</title>
          <p>The data used to build the models consisted of the radiologists’ assessment of the 5 growth zones, following the aforementioned stages, the questionnaire information, and the calculated BMI. These data presented missing values that were handled by the K-nearest neighbor (KNN) multiple imputations. This technique finds K complete entries that are the closest to an incomplete entry (ie, contains missing data) and fills its missing values with the mean (in the case of numeric variables) or the most frequent one (in the case of categorical variables) [<xref ref-type="bibr" rid="ref38">38</xref>]. In this study, the number of nearest neighbors K for the KNN imputation was set to 1. The motivation for this choice is based on literature findings that advise limiting K as a way to preserve the original variability of the data, reducing the risk of entries having few neighbors that are too distant from each other [<xref ref-type="bibr" rid="ref39">39</xref>]. There is also a risk of increasing the influence of noise in the data with a small K, but as in the data set of this study, the highest rate of imputed instances was 1.9%; this influence was considered to be not very relevant. The distance used by the KNN multiple imputation technique was the Gower distance [<xref ref-type="bibr" rid="ref40">40</xref>]. The number of imputed instances for each variable in both male and female subsets is shown in <xref ref-type="table" rid="table3">Table 3</xref>.</p>
          <table-wrap position="float" id="table3">
            <label>Table 3</label>
            <caption>
              <p>Number of imputed instances and percentage over the male and female data sets.</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="580"/>
              <col width="210"/>
              <col width="210"/>
              <thead>
                <tr valign="top">
                  <td>Variable</td>
                  <td>Male data set, n (%)</td>
                  <td>Female data set, n (%)</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td>Radiologists' assessments of the radius, distal femur, proximal tibia, distal tibia, calcaneus</td>
                  <td>0 (0)</td>
                  <td>0 (0)</td>
                </tr>
                <tr valign="top">
                  <td>Residence</td>
                  <td>1 (0.2)</td>
                  <td>3 (0.6)</td>
                </tr>
                <tr valign="top">
                  <td>Physical activity</td>
                  <td>9 (1.9)</td>
                  <td>6 (1.2)</td>
                </tr>
                <tr valign="top">
                  <td>Tanner Scale</td>
                  <td>3 (0.6)</td>
                  <td>1 (0.2)</td>
                </tr>
                <tr valign="top">
                  <td>BMI</td>
                  <td>0 (0)</td>
                  <td>0 (0)</td>
                </tr>
                <tr valign="top">
                  <td>Parents origin</td>
                  <td>0 (0)</td>
                  <td>3 (0.6)</td>
                </tr>
              </tbody>
            </table>
          </table-wrap>
        </sec>
        <sec>
          <title>ML Algorithms</title>
          <p>The choice of the ML algorithms explored in this study was based on the summary of the evidence of a recently published systematic literature review (SLR) on the application of ML for BAA [<xref ref-type="bibr" rid="ref20">20</xref>]. This SLR points out that the studies proposing BAA classifiers employ algorithms of the following categories: artificial neural networks, support vector machines, Bayesian networks, decision trees, and K-nearest neighbors. An additional search was conducted in the literature (Scopus, PubMed, and Web of Science), after the search date of the mentioned SLR [<xref ref-type="bibr" rid="ref20">20</xref>] (February 2019) to look for additional algorithms, but no new categories were found to be added to the list.</p>
          <p>Another motivation for this choice of ML algorithms is that it also guarantees a diversified list of classifiers that make use of different types of learning techniques, such as rule-based, instance-based, Bayesian inference, kernel, and perceptron learners. We referred to the following book by Kuhn and Johnson [<xref ref-type="bibr" rid="ref41">41</xref>] for the specific algorithms and implementations used in this study.</p>
          <p>Therefore, the choice of ML algorithms for the experiments of this study includes decision tree, random forest, multilayer perceptron, support vector machines, naïve Bayes, and K-nearest neighbors.</p>
        </sec>
        <sec>
          <title>Experimental Setup</title>
          <p>All experiments were performed using a stratified, nested cross-validation [<xref ref-type="bibr" rid="ref42">42</xref>]. In this approach, in each iteration, one fold of the outer cross-validation is used for testing and the remaining 4 are used in an inner cross-validation to tune the algorithm’s hyperparameters. This was done to obtain a more reliable estimate of the error as the test fold in each outer iteration is not used to execute performance optimization [<xref ref-type="bibr" rid="ref43">43</xref>]. It is also worth noting that the data splits were performed in a stratified manner, which means that the classes’ proportions in each split are kept the same as in the total sample. In the experiments of this study, a five-fold outer, three-fold inner stratified nested cross-validation was performed. The reduced number of folds in the inner cross-validation was employed to avoid having a low number of subjects to represent each class in the folders, due to the high number of classes in the multiclass classification problem. Additionally, before each inner cross-validation, a grid search was performed to find suitable hyperparameters for each of the selected ML algorithms. The hyperparameters for each selected algorithm are listed in <xref ref-type="table" rid="table4">Table 4</xref>. The ML experiments were conducted in the R framework with the <italic>caret</italic> package. The default versions of the algorithms were used.</p>
          <table-wrap position="float" id="table4">
            <label>Table 4</label>
            <caption>
              <p>Configuration of the R algorithms included in the experiment.</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="390"/>
              <col width="280"/>
              <col width="330"/>
              <thead>
                <tr valign="top">
                  <td>ML<sup>a</sup> algorithm</td>
                  <td>R implementation</td>
                  <td>Tuning parameters</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td>Decision tree</td>
                  <td>
                    <italic>rpart</italic>
                  </td>
                  <td>
                    <italic>cp</italic>
                  </td>
                </tr>
                <tr valign="top">
                  <td>Random forest</td>
                  <td>
                    <italic>rf</italic>
                  </td>
                  <td>
                    <italic>mtry</italic>
                  </td>
                </tr>
                <tr valign="top">
                  <td>Multi-layer perceptron</td>
                  <td>
                    <italic>mlp</italic>
                  </td>
                  <td>
                    <italic>size</italic>
                  </td>
                </tr>
                <tr valign="top">
                  <td>Support vector machines</td>
                  <td>
                    <italic>svmRadial</italic>
                  </td>
                  <td>
                    <italic>Sigma, C</italic>
                  </td>
                </tr>
                <tr valign="top">
                  <td>Naïve Bayes</td>
                  <td>
                    <italic>nb</italic>
                  </td>
                  <td>
                    <italic>fL, usekernel, adjust</italic>
                  </td>
                </tr>
                <tr valign="top">
                  <td>K-nearest neighbors</td>
                  <td>
                    <italic>knn</italic>
                  </td>
                  <td>
                    <italic>k</italic>
                  </td>
                </tr>
              </tbody>
            </table>
            <table-wrap-foot>
              <fn id="table4fn1">
                <p><sup>a</sup>ML: machine learning.</p>
              </fn>
            </table-wrap-foot>
          </table-wrap>
        </sec>
      </sec>
      <sec>
        <title>Model Evaluation Metrics</title>
        <p>The performance metrics used to evaluate the models were as follows: mean absolute error (MAE), root mean squared error (RMSE), accuracy, precision, recall, and area under the curve (AUC), as in Gaudette and Japkowicz [<xref ref-type="bibr" rid="ref44">44</xref>] and Sokolova and Lapalme [<xref ref-type="bibr" rid="ref45">45</xref>] guidelines for ordinal multiclass classification. For the binary classification models, all but MAE and RMSE are used. The SDs for each metric are also reported.</p>
        <p>The MAE represents the mean of the absolute difference between the estimated age output of the classifier and the correct CA of the subject, over all examples. The RMSE gives more weight to larger errors compared with MAE, which tends to prefer fewer errors overall. The MAE and RMSE are calculated as follows in equations 2 and 3, respectively:</p>
        <graphic xlink:href="medinform_v8i9e18846_fig4.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        <graphic xlink:href="medinform_v8i9e18846_fig5.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        <p>Where n is the number of samples, <inline-graphic xlink:href="medinform_v8i9e18846_fig6.png" xlink:type="simple" mimetype="image"/> is the estimated age, and y is the CA of the subject.</p>
        <p>For the remaining evaluation metrics, considering l the number of classes, we define the following:</p>
        <list list-type="bullet">
          <list-item>
            <p>True-positives (TP): Entries predicted to be in class C<sub>l</sub>  actually in class C<sub>l</sub>.</p>
          </list-item>
          <list-item>
            <p>False-positives (FP): Entries predicted to be in C<sub>l</sub> but are not actually in class C<sub>l</sub>.</p>
          </list-item>
          <list-item>
            <p>True-negatives (TN): Entries not predicted to be in C<sub>l</sub> and are not actually in class C<sub>l</sub>.</p>
          </list-item>
          <list-item>
            <p>False-negatives (FN): Entries not predicted to be in C<sub>l</sub>, but are actually in class C<sub>l</sub>.</p>
          </list-item>
        </list>
        <p>The accuracy, precision, recall, and AUC for binary classification are calculated as follows:</p>
        <graphic xlink:href="medinform_v8i9e18846_fig7.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        <graphic xlink:href="medinform_v8i9e18846_fig8.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        <graphic xlink:href="medinform_v8i9e18846_fig9.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        <graphic xlink:href="medinform_v8i9e18846_fig10.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        <p>In the case of the multiclass classification, these are calculated as the average of the metrics calculated for each class C<sub>l</sub> (macro averaging) [<xref ref-type="bibr" rid="ref45">45</xref>]. The AUC metric is calculated by averaging pairwise comparisons, as proposed by Hand and Till [<xref ref-type="bibr" rid="ref46">46</xref>].</p>
        <p>General results are given for the ML algorithms in terms of the mean and SDs of each of the performance metrics for the outer cross-validation test sets. In-depth results are given to the best performing models.</p>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <sec>
        <title>Interobserver Agreement</title>
        <p>The kappa Cohen coefficient was calculated to evaluate the agreement between the 2 observers’ assessments of the MRI images. The results indicated substantial agreement according to the general guidelines [<xref ref-type="bibr" rid="ref47">47</xref>] for all of the assessed ROIs: 0.77 for the calcaneus, 0.65 for the distal femur, 0.72 for the distal tibia, 0.73 for the proximal tibia, and 0.67 for the radius.</p>
        <p>The percent agreement for the assessed ROI was as follows: 94.2% for the calcaneus, 80.8% for the distal femur, 90.6% for the distal tibia, 86.8% for the proximal tibia, and 79.4% for the radius. These results show that the radiologists agreed on a stage in the vast majority of cases.</p>
      </sec>
      <sec>
        <title>Results of the Growth Plate Assessments</title>
        <p>The results of the assessments of the calcaneus, distal tibia, proximal tibia, distal femur, and radius for male and female subjects are shown in detail in <xref ref-type="supplementary-material" rid="app2">Multimedia Appendices 2</xref> and <xref ref-type="supplementary-material" rid="app3">3</xref>, respectively.</p>
        <p>In all of the assessed growth plates, for both sexes, stages 1 and 2 were not evidenced. Few instances of stage 3 were observed on male subjects on the calcaneus and radius growth plates, accounting for 2 and 15 cases, respectively. In female subjects, stage 3 was evidenced in only 2 cases for the radius growth plate.</p>
        <p>The female subjects’ results show that for all assessed growth plates, nearly all or most of the sample was already in the last stage of ossification (stage 5): 94.6% of the calcaneus, 90.8% of the distal tibia, 81.6% of the proximal tibia, 74.5% of the distal femur, and 65.5% of the radius cases. These numbers moderately change for male subjects, accounting for 80.4% of the calcaneus, 70.1% of the distal tibia, 57.6% of the proximal tibia, 54.9% of the distal femur, and 47.4% of the radius cases.</p>
        <p><xref ref-type="table" rid="table5">Table 5</xref> shows the proportion within each age group of subjects who had all of the growth plates considered in this study already in stage 5. This table shows that female subjects had all growth plates fused 2 years before the male subjects. For female subjects, from the age of 19 years, all subjects of the sample already have all of the growth plates fused, although for male subjects, the same happens from the age of 21 years.</p>
        <table-wrap position="float" id="table5">
          <label>Table 5</label>
          <caption>
            <p>Numbers and percentages (over each age group) of subjects with all of the growth plates in stage 5, for male and female subjects.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="90"/>
            <col width="460"/>
            <col width="420"/>
            <thead>
              <tr valign="top">
                <td colspan="2">Characteristic</td>
                <td>Female subjects, n (%)</td>
                <td>Male subjects, n (%)</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="4">
                  <bold>Age group (years)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>14</td>
                <td>2 (3.3)</td>
                <td>0 (0)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>15</td>
                <td>8 (13.7)</td>
                <td>0 (0)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>16</td>
                <td>23 (40.3)</td>
                <td>3 (5)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>17</td>
                <td>44 (73.3)</td>
                <td>13 (22.4)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>18</td>
                <td>53 (89.8)</td>
                <td>31 (58.4)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>19</td>
                <td>57 (100)</td>
                <td>50 (86.2)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>20</td>
                <td>57 (100)</td>
                <td>50 (94.3)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>21</td>
                <td>60 (100)</td>
                <td>59 (100)</td>
              </tr>
              <tr valign="top">
                <td colspan="2">Total</td>
                <td>304 (65.1)</td>
                <td>206 (45.2)</td>
              </tr>
            </tbody>
          </table>
        </table-wrap>
      </sec>
      <sec>
        <title>Results for the Classification of Minors Versus Adults</title>
        <p>A threshold of 18 years was used to determine adulthood in the classification of minors versus adults, which is the case in many European countries. MAE and RMSE were not used as performance metrics in this case because for classifications they only make sense in the context of an ordinal classification. The results for the male subjects’ binary classifiers in terms of the mean and SD of the performance metrics on the outer cross-validation test sets are shown in <xref ref-type="table" rid="table6">Table 6</xref>.</p>
        <table-wrap position="float" id="table6">
          <label>Table 6</label>
          <caption>
            <p>Mean performance metrics and respective SDs (in years) for the classification of minor versus adults for the male subjects.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="200"/>
            <col width="200"/>
            <col width="210"/>
            <col width="200"/>
            <col width="190"/>
            <thead>
              <tr valign="top">
                <td>Types</td>
                <td>Accuracy, mean (SD)</td>
                <td>AUC<sup>a</sup>, mean (SD)</td>
                <td>Precision, mean (SD)</td>
                <td>Recall, mean (SD)</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Decision tree</td>
                <td>0.90 (0.02)</td>
                <td>0.90 (0.02)</td>
                <td>0.86 (0.04)</td>
                <td>0.96 (0.03)</td>
              </tr>
              <tr valign="top">
                <td>Random forest</td>
                <td>0.90 (0.01)</td>
                <td>0.90 (0.01)</td>
                <td>0.87 (0.03)</td>
                <td>0.94 (0.04)</td>
              </tr>
              <tr valign="top">
                <td>Support vector machines</td>
                <td>0.90 (0.02)</td>
                <td>0.90 (0.02)</td>
                <td>0.87 (0.04)</td>
                <td>0.93 (0.07)</td>
              </tr>
              <tr valign="top">
                <td>Multi-layer perceptron</td>
                <td>0.82 (0.17)</td>
                <td>0.82 (0.16)</td>
                <td>0.79 (0.16)</td>
                <td>0.95 (0.04)</td>
              </tr>
              <tr valign="top">
                <td>K-nearest neighbors</td>
                <td>0.87 (0.02)</td>
                <td>0.87 (0.02)</td>
                <td>0.84 (0.03)</td>
                <td>0.92 (0.03)</td>
              </tr>
              <tr valign="top">
                <td>Naïve bayes</td>
                <td>0.73 (0.04)</td>
                <td>0.74 (0.04)</td>
                <td>0.65 (0.03)</td>
                <td>1.00 (0.00)</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table6fn1">
              <p><sup>a</sup>AUC: area under the curve.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <p>The decision tree, random forest, and support vector machine algorithms had very similar results in general, presenting no significant difference between them. The random forest algorithm was chosen in terms of the best combination of precision and recall, but in practical settings, there are no differences between these algorithms. <xref ref-type="table" rid="table7">Table 7</xref> shows the random forest results for each of the outer cross-validation test sets. The average model was chosen in terms of median accuracy, which was 0.90. Between Models 1 and 4, Model 1 was chosen for better recall in classifying minors. The optimized hyperparameter given by the grid search for Model 1 was <italic>mtry=2</italic> (number of candidate variables at each tree split)<italic>.</italic></p>
        <table-wrap position="float" id="table7">
          <label>Table 7</label>
          <caption>
            <p>Performance results for the Random Forest algorithm on each of the outer cross-validation test sets, for the male sample.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="150"/>
            <col width="170"/>
            <col width="170"/>
            <col width="170"/>
            <col width="170"/>
            <col width="170"/>
            <thead>
              <tr valign="top">
                <td>Model</td>
                <td>Accuracy</td>
                <td>AUC<sup>a</sup></td>
                <td>Precision</td>
                <td>Recall—minors</td>
                <td>Recall—adults</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>1 (median)</td>
                <td>0.90</td>
                <td>0.90</td>
                <td>0.83</td>
                <td>1.00</td>
                <td>0.80</td>
              </tr>
              <tr valign="top">
                <td>2</td>
                <td>0.89</td>
                <td>0.89</td>
                <td>0.87</td>
                <td>0.91</td>
                <td>0.87</td>
              </tr>
              <tr valign="top">
                <td>3</td>
                <td>0.89</td>
                <td>0.89</td>
                <td>0.87</td>
                <td>0.96</td>
                <td>0.83</td>
              </tr>
              <tr valign="top">
                <td>4</td>
                <td>0.90</td>
                <td>0.90</td>
                <td>0.90</td>
                <td>0.89</td>
                <td>0.91</td>
              </tr>
              <tr valign="top">
                <td>5</td>
                <td>0.92</td>
                <td>0.92</td>
                <td>0.89</td>
                <td>0.96</td>
                <td>0.89</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table7fn1">
              <p><sup>a</sup>AUC: area under the curve.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <p><xref rid="figure1" ref-type="fig">Figure 1</xref> presents the results achieved by Model 1 per age group. It is important to note that even with low accuracy results for the age of 17 years (41.7%), the model still minimizes the error of classifying minors as adults, achieving a recall of 100% for this classification.</p>
        <fig id="figure1" position="float">
          <label>Figure 1</label>
          <caption>
            <p>Accuracy per age group for the minor versus adults classification model, for male subjects.</p>
          </caption>
          <graphic xlink:href="medinform_v8i9e18846_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <p>In the case of the female subjects, the decision tree, random forest, and multi-layer perceptron algorithms presented very similar results (<xref ref-type="table" rid="table8">Table 8</xref>), which do not present a relevant significant difference between them. The chosen algorithm for the female subject case was the random forest algorithm for the best combination of performance measures. <xref ref-type="table" rid="table9">Table 9</xref> shows the results for the random forest algorithm in each of the outer cross-validation test sets. Except for Model 1, there was essentially no relevant variation between models, and in practical settings, they can be considered equal. Thus, Model 2 was chosen as the average model. The optimized hyperparameter given by the grid search for Model 1 was mtry=6.</p>
        <table-wrap position="float" id="table8">
          <label>Table 8</label>
          <caption>
            <p>Mean performance metrics and respective SDs (in years) for the classification of minor versus adults for the female subjects.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="240"/>
            <col width="190"/>
            <col width="190"/>
            <col width="190"/>
            <col width="190"/>
            <thead>
              <tr valign="top">
                <td>Types</td>
                <td>Accuracy, mean (SD)</td>
                <td>AUC<sup>a</sup>, mean (SD)</td>
                <td>Precision, mean (SD)</td>
                <td>Recall, mean (SD)</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Decision tree</td>
                <td>0.82 (0.02)</td>
                <td>0.82 (0.02)</td>
                <td>0.74 (0.02)</td>
                <td>0.97 (0.01)</td>
              </tr>
              <tr valign="top">
                <td>Random forest</td>
                <td>0.83 (0.02)</td>
                <td>0.83 (0.01)</td>
                <td>0.76 (0.02)</td>
                <td>0.97 (0.01)</td>
              </tr>
              <tr valign="top">
                <td>Support vector machines</td>
                <td>0.81 (0.04)</td>
                <td>0.81 (0.04)</td>
                <td>0.75 (0.04)</td>
                <td>0.92 (0.05)</td>
              </tr>
              <tr valign="top">
                <td>Multi-layer perceptron</td>
                <td>0.82 (0.02)</td>
                <td>0.82 (0.02)</td>
                <td>0.75 (0.02)</td>
                <td>0.95 (0.04)</td>
              </tr>
              <tr valign="top">
                <td>K-nearest neighbors</td>
                <td>0.78 (0.06)</td>
                <td>0.78 (0.06)</td>
                <td>0.73 (0.06)</td>
                <td>0.87 (0.08)</td>
              </tr>
              <tr valign="top">
                <td>Naïve bayes</td>
                <td>0.67 (0.03)</td>
                <td>0.67 (0.02)</td>
                <td>0.60 (0.02)</td>
                <td>1.00 (0.00)</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table8fn1">
              <p><sup>a</sup>AUC: area under the curve.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <table-wrap position="float" id="table9">
          <label>Table 9</label>
          <caption>
            <p>Performance results for the random forest algorithm on each of the outer cross-validation test sets, for the female sample.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="150"/>
            <col width="170"/>
            <col width="170"/>
            <col width="170"/>
            <col width="170"/>
            <col width="170"/>
            <thead>
              <tr valign="top">
                <td>Model</td>
                <td>Accuracy</td>
                <td>AUC<sup>a</sup></td>
                <td>Precision</td>
                <td>Recall—minors</td>
                <td>Recall—adults</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>1</td>
                <td>0.81</td>
                <td>0.81</td>
                <td>0.73</td>
                <td>0.96</td>
                <td>0.66</td>
              </tr>
              <tr valign="top">
                <td>2 (median)</td>
                <td>0.84</td>
                <td>0.84</td>
                <td>0.77</td>
                <td>0.96</td>
                <td>0.72</td>
              </tr>
              <tr valign="top">
                <td>3</td>
                <td>0.84</td>
                <td>0.84</td>
                <td>0.77</td>
                <td>0.98</td>
                <td>0.70</td>
              </tr>
              <tr valign="top">
                <td>4</td>
                <td>0.84</td>
                <td>0.84</td>
                <td>0.78</td>
                <td>0.96</td>
                <td>0.72</td>
              </tr>
              <tr valign="top">
                <td>5</td>
                <td>0.84</td>
                <td>0.84</td>
                <td>0.77</td>
                <td>0.98</td>
                <td>0.70</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table9fn1">
              <p><sup>a</sup>AUC: area under the curve.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <p>The accuracies per age group are shown in the graph in <xref rid="figure2" ref-type="fig">Figure 2</xref>. The model achieved lower accuracies for the ages of 16 and 17 years (50.0% and 58.3%, respectively), but as in the case of the male subjects, the model minimizes the worst type of error, which is the misclassification of minors, achieving a high recall of 96%.</p>
        <fig id="figure2" position="float">
          <label>Figure 2</label>
          <caption>
            <p>Accuracy per age group for the minor versus adults classification model, for female subjects.</p>
          </caption>
          <graphic xlink:href="medinform_v8i9e18846_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>Results for the CA Estimation Models</title>
        <p>The CA estimation models are multiclass classifiers that aim to classify subjects in 1 of the 8 age groups (from 14 to 21 years). <xref ref-type="table" rid="table10">Table 10</xref> shows the results for the male subjects’ models in terms of the mean and SDs of the performances on the outer cross-validation test sets. The best performing algorithm in the male case was the multilayer perceptron (MLP), which achieved the best MAE (0.98 years), mean RMSE (1.32 years), and mean precision (0.65 years) values, in addition to having the second-best values of mean accuracy and mean AUC.</p>
        <table-wrap position="float" id="table10">
          <label>Table 10</label>
          <caption>
            <p>Mean (SD) of the performance metrics for the male subjects’ classification models.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="150"/>
            <col width="140"/>
            <col width="140"/>
            <col width="150"/>
            <col width="140"/>
            <col width="140"/>
            <col width="140"/>
            <thead>
              <tr valign="top">
                <td>Algorithms</td>
                <td>MAE<sup>a</sup> (years), mean (SD)</td>
                <td>Accuracy, mean (SD)</td>
                <td>RMSE<sup>b</sup> (years), mean (SD)</td>
                <td>AUC<sup>c</sup>, mean (SD)</td>
                <td>Precision, mean (SD)</td>
                <td>Recall, mean (SD)</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Decision tree</td>
                <td>1.28 (0.13)</td>
                <td>0.32 (0.03)</td>
                <td>1.78 (0.17)</td>
                <td>0.81 (0.02)</td>
                <td>0.49 (0.06)</td>
                <td>0.81 (0.11)</td>
              </tr>
              <tr valign="top">
                <td>Random forest</td>
                <td>1.04 (0.07)</td>
                <td>0.34 (0.03)</td>
                <td>1.44 (0.13)</td>
                <td>0.85 (0.01)</td>
                <td>0.57 (0.09)</td>
                <td>0.73 (0.14)</td>
              </tr>
              <tr valign="top">
                <td>Support vector machine</td>
                <td>1.03 (0.09)</td>
                <td>0.34 (0.03)</td>
                <td>1.43 (0.08)</td>
                <td>0.85 (0.01)</td>
                <td>0.52 (0.09)</td>
                <td>0.67 (0.12)</td>
              </tr>
              <tr valign="top">
                <td>Multi-layer perceptron</td>
                <td>0.98 (0.08)</td>
                <td>0.33 (0.02)</td>
                <td>1.32 (0.13)</td>
                <td>0.84 (0.01)</td>
                <td>0.65 (0.27)</td>
                <td>0.61 (0.31)</td>
              </tr>
              <tr valign="top">
                <td>K-nearest neighbor</td>
                <td>1.16 (0.11)</td>
                <td>0.30 (0.04)</td>
                <td>1.57 (0.15)</td>
                <td>0.82 (0.03)</td>
                <td>0.59 (0.10)</td>
                <td>0.59 (0.10)</td>
              </tr>
              <tr valign="top">
                <td>Naïve bayes</td>
                <td>1.07 (0.10)</td>
                <td>0.29 (0.02)</td>
                <td>1.39 (0.19)</td>
                <td>0.81 (0.01)</td>
                <td>0.57 (0.06)</td>
                <td>0.58 (0.21)</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table10fn1">
              <p><sup>a</sup>MAE: mean absolute error.</p>
            </fn>
            <fn id="table10fn2">
              <p><sup>b</sup>RMSE: root mean squared error.</p>
            </fn>
            <fn id="table10fn3">
              <p><sup>c</sup>AUC: area under the curve.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <p>The performances of the MLP algorithm on each of the outer cross-validation test sets are shown in <xref ref-type="table" rid="table11">Table 11</xref>. The average model was chosen in terms of the median MAE, which corresponds to Model 1, with a value of 0.95 years. The optimized hyperparameter given by the grid search for the average MLP model was <italic>size=27</italic> (number of units in the hidden layer). The average model was chosen to select an algorithm that would not be overly optimistic in its estimation.</p>
        <table-wrap position="float" id="table11">
          <label>Table 11</label>
          <caption>
            <p>Performance results for the multi-layer perceptron algorithm on each of the outer cross-validation test sets, for the male sample.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="160"/>
            <col width="140"/>
            <col width="140"/>
            <col width="140"/>
            <col width="140"/>
            <col width="140"/>
            <col width="140"/>
            <thead>
              <tr valign="top">
                <td>Model</td>
                <td>MAE<sup>a</sup> (years)</td>
                <td>Accuracy</td>
                <td>AUC<sup>b</sup></td>
                <td>RMSE<sup>c</sup> (years)</td>
                <td>Recall</td>
                <td>Precision</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>1 (median)</td>
                <td>0.95</td>
                <td>0.33</td>
                <td>0.83</td>
                <td>1.29</td>
                <td>0.91</td>
                <td>0.48</td>
              </tr>
              <tr valign="top">
                <td>2</td>
                <td>1.08</td>
                <td>0.30</td>
                <td>0.85</td>
                <td>1.40</td>
                <td>073</td>
                <td>0.35</td>
              </tr>
              <tr valign="top">
                <td>3</td>
                <td>0.89</td>
                <td>0.32</td>
                <td>0.84</td>
                <td>1.17</td>
                <td>0.17</td>
                <td>1.00</td>
              </tr>
              <tr valign="top">
                <td>4</td>
                <td>0.91</td>
                <td>0.33</td>
                <td>0.83</td>
                <td>1.23</td>
                <td>0.83</td>
                <td>0.59</td>
              </tr>
              <tr valign="top">
                <td>5</td>
                <td>1.05</td>
                <td>0.35</td>
                <td>0.84</td>
                <td>1.49</td>
                <td>0.42</td>
                <td>0.83</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table11fn1">
              <p><sup>a</sup>MAE: mean absolute error.</p>
            </fn>
            <fn id="table11fn2">
              <p><sup>b</sup>AUC: area under the curve.</p>
            </fn>
            <fn id="table11fn3">
              <p><sup>c</sup>RMSE: root mean squared error.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <p>The results for the chosen model, discriminated by age groups, are shown in <xref ref-type="table" rid="table12">Table 12</xref>. The model shows lower errors for the younger and older ages of the age spam considered in the study. In addition, the model has a clear trend of overestimating the ages of the male subjects in general. Thus, even with an MAE of 0.95 years, the model is limited to its capacity to classify individuals from the age of 16 years. From the age of 19 years, the model tends to classify all subjects as 20 years old as nearly all subjects of these ages have all growth plates on stage 5.</p>
        <table-wrap position="float" id="table12">
          <label>Table 12</label>
          <caption>
            <p>Mean absolute error and SD for the average male model.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="110"/>
            <col width="110"/>
            <col width="120"/>
            <col width="110"/>
            <col width="110"/>
            <col width="110"/>
            <col width="110"/>
            <col width="110"/>
            <col width="110"/>
            <thead>
              <tr valign="top">
                <td>Measure</td>
                <td colspan="8">Age group</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>14</td>
                <td>15</td>
                <td>16</td>
                <td>17</td>
                <td>18</td>
                <td>19</td>
                <td>20</td>
                <td>21</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>MAE<sup>a</sup> (years)</td>
                <td>0.18 (SD 0.60)</td>
                <td>0.82 (SD 0.90)</td>
                <td>1.25 (SD 1.44)</td>
                <td>1.91 (SD 1.56)</td>
                <td>1.50 (SD 1.45)</td>
                <td>1.00 (SD 0.60)</td>
                <td>0.00 (SD 0.00)</td>
                <td>0.92 (SD 0.29)</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table12fn1">
              <p><sup>a</sup>MAE: mean absolute error.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <p><xref ref-type="table" rid="table13">Table 13</xref> shows the results for the CA estimation models for female subjects in terms of the mean and standard deviations of the performances on the outer cross-validation test sets. In the case of the female subjects, the best performing algorithm was the support vector machine (SVM), which achieved the best MAE (1.21 years), mean accuracy (0.32), mean RMSE (1.68 years), and mean AUC (0.80).</p>
        <table-wrap position="float" id="table13">
          <label>Table 13</label>
          <caption>
            <p>Mean (SD) of the performance metrics, for the female subjects’ classification models.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="150"/>
            <col width="140"/>
            <col width="140"/>
            <col width="150"/>
            <col width="140"/>
            <col width="140"/>
            <col width="140"/>
            <thead>
              <tr valign="top">
                <td>Algorithms</td>
                <td>MAE<sup>a</sup> (years), mean (SD)</td>
                <td>Accuracy, mean (SD)</td>
                <td>RMSE<sup>b</sup> (years), mean (SD)</td>
                <td>AUC<sup>c</sup>, mean (SD)</td>
                <td>Precision, mean (SD)</td>
                <td>Recall, mean (SD)</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Decision tree</td>
                <td>1.31 (0.09)</td>
                <td>0.28 (0.02)</td>
                <td>1.78 (0.13)</td>
                <td>0.80 (0.02)</td>
                <td>0.56 (0.05)</td>
                <td>0.82 (0.09)</td>
              </tr>
              <tr valign="top">
                <td>Random forest</td>
                <td>1.29 (0.10)</td>
                <td>0.30 (0.03)</td>
                <td>1.77 (0.10)</td>
                <td>0.79 (0.02)</td>
                <td>0.59 (0.13)</td>
                <td>0.74 (0.17)</td>
              </tr>
              <tr valign="top">
                <td>Support vector machine</td>
                <td>1.21 (0.06)</td>
                <td>0.32 (0.04)</td>
                <td>1.68 (0.06)</td>
                <td>0.80 (0.01)</td>
                <td>0.55 (0.07)</td>
                <td>0.71 (0.11)</td>
              </tr>
              <tr valign="top">
                <td>Multi-layer perceptron</td>
                <td>1.36 (0.24)</td>
                <td>0.30 (0.02)</td>
                <td>1.85 (0.37)</td>
                <td>0.77 (0.02)</td>
                <td>0.60 (0.11)</td>
                <td>0.63 (0.22)</td>
              </tr>
              <tr valign="top">
                <td>K-nearest neighbors</td>
                <td>1.41 (0.12)</td>
                <td>0.30 (0.02)</td>
                <td>1.96 (0.12)</td>
                <td>0.76 (0.03)</td>
                <td>0.55 (0.07)</td>
                <td>0.61 (0.18)</td>
              </tr>
              <tr valign="top">
                <td>Naïve bayes</td>
                <td>1.74 (0.23)</td>
                <td>0.22 (0.02)</td>
                <td>2.23 (0.27)</td>
                <td>0.65 (0.03)</td>
                <td>0.58 (0.06)</td>
                <td>0.82 (0.09)</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table13fn1">
              <p><sup>a</sup>MAE: mean absolute error.</p>
            </fn>
            <fn id="table13fn2">
              <p><sup>b</sup>RMSE: root mean squared error.</p>
            </fn>
            <fn id="table13fn3">
              <p><sup>c</sup>AUC: area under the curve.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <p><xref ref-type="table" rid="table14">Table 14</xref> shows the performance results for each of the outer cross-validation test sets for the SVM algorithm. For the case of the female subjects, the median resulted in an MAE of 1.24 years, which pertained to Models 1 and 2. Model 1 was chosen as the average model for presenting the best accuracy between the two. The optimized parameter given by the grid search for the average SVM model was <italic>sigma=0.0421</italic> (kernel parameter) <italic>and C=4</italic> (penalty parameter).</p>
        <p>The MAE results per age group are shown in <xref ref-type="table" rid="table15">Table 15</xref>. As in the male subjects’ case, the female model also overestimates the ages of female subjects in general, but with higher MAE and standard deviations.</p>
        <table-wrap position="float" id="table14">
          <label>Table 14</label>
          <caption>
            <p>Performance results for the support vector machine algorithm on each of the outer cross-validation test sets, for the female sample.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="140"/>
            <col width="140"/>
            <col width="150"/>
            <col width="140"/>
            <col width="140"/>
            <col width="140"/>
            <col width="150"/>
            <thead>
              <tr valign="top">
                <td>Model</td>
                <td>MAE<sup>a</sup> (years)</td>
                <td>Accuracy</td>
                <td>AUC<sup>b</sup></td>
                <td>RMSE<sup>c</sup> (years)</td>
                <td>Recall</td>
                <td>Precision</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>1 (median)</td>
                <td>1.24</td>
                <td>0.37</td>
                <td>0.79</td>
                <td>1.75</td>
                <td>0.75</td>
                <td>0.56</td>
              </tr>
              <tr valign="top">
                <td>2</td>
                <td>1.24</td>
                <td>0.27</td>
                <td>0.80</td>
                <td>1.67</td>
                <td>0.55</td>
                <td>0.67</td>
              </tr>
              <tr valign="top">
                <td>3</td>
                <td>1.25</td>
                <td>0.33</td>
                <td>0.78</td>
                <td>1.70</td>
                <td>0.75</td>
                <td>0.50</td>
              </tr>
              <tr valign="top">
                <td>4</td>
                <td>1.11</td>
                <td>0.32</td>
                <td>0.81</td>
                <td>1.58</td>
                <td>0.67</td>
                <td>0.53</td>
              </tr>
              <tr valign="top">
                <td>5</td>
                <td>1.20</td>
                <td>0.32</td>
                <td>0.81</td>
                <td>1.72</td>
                <td>0.83</td>
                <td>0.83</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table14fn1">
              <p><sup>a</sup>MAE: mean absolute error.</p>
            </fn>
            <fn id="table14fn2">
              <p><sup>b</sup>AUC: area under the curve.</p>
            </fn>
            <fn id="table14fn3">
              <p><sup>c</sup>RMSE: root mean squared error.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <table-wrap position="float" id="table15">
          <label>Table 15</label>
          <caption>
            <p>Mean absolute error and standard deviation for the male median model</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="120"/>
            <col width="110"/>
            <col width="110"/>
            <col width="110"/>
            <col width="110"/>
            <col width="110"/>
            <col width="110"/>
            <col width="110"/>
            <col width="110"/>
            <thead>
              <tr valign="top">
                <td>Measure</td>
                <td colspan="8">Age group</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>14</td>
                <td>15</td>
                <td>16</td>
                <td>17</td>
                <td>18</td>
                <td>19</td>
                <td>20</td>
                <td>21</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>MAE<sup>a</sup> (years), mean (SD)</td>
                <td>0.42 (0.79)</td>
                <td>1.42 (1.93)</td>
                <td>1.17 (1.79)</td>
                <td>2.17 (2.49)</td>
                <td>1.75 (1.88)</td>
                <td>1.09 (1.43)</td>
                <td>0.90 (1.27)</td>
                <td>1.00 (1.34)</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table15fn1">
              <p><sup>a</sup>MAE: mean absolute error.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Principal Findings</title>
        <p>This paper presents experiments with the estimation of CA and classification of minors versus adults (on the threshold of 18 years) of male and female subjects using ML algorithms. To build the models, 2 radiologists assessed the stage of bone development of the calcaneus, distal tibia, proximal tibia, distal femur, and radius growth plates of 455 male and 467 female volunteer subjects (922 subjects in total) from MRI images. Additional variables were also used to build the models: BMI, physical activity level, parents’ origin, type of residence during upbringing, and self-assessed Tanner Scale of pubertal growth. The methodology adopted in the study aimed at addressing the drawbacks of the BAA methods that are employed in CA estimation for legal scenarios.</p>
        <p>From the stage assessments of the MRI images, we could infer that female subjects mature earlier than male subjects regarding the bone development of the knee, wrist, and foot, which is in line with prior studies [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref18">18</xref>,<xref ref-type="bibr" rid="ref48">48</xref>,<xref ref-type="bibr" rid="ref49">49</xref>]. In this study, the first age in which the whole sample had all fused growth plates (stage 5) was 19 years for female (467/467, 100%) and 21 years for male (455/455, 100%) subjects.</p>
        <p>Another important point to be discussed with regard to the stage assessments is that the female sample had cases that had all of the considered growth plates already fused since the age of 14 years, spamming throughout all ages considered in the study (14 to 21 years). Since the assessment of stage 5, unlike the other stages, requires that all of the slices from the MRI examination to present a fused growth plate, even if there is a degree of misassessment, it would still mean that these cases would display a well-advanced level of maturation in all of these ages, implying a high degree of biological variation in the female sample with regard to BA. Additionally, in total, 65.5% (304/467) of the female sample consisted of cases in which the subjects presented all growth plates already in stage 5, which means that for classification purposes, for more than half of the sample, the estimation of CA would depend only on the additional factors (self-assessed Tanner Scale, BMI, residence type, physical activity, and parents origin), which were not enough to discriminate between age groups. This hindered the performance of classifiers, especially the CA estimation models. The same phenomenon occurred for the male sample, which also negatively affected the performance of the classifiers, but to a lesser degree, as 45.2% (206/455) of the sample had all growth plates of stage 5, from the age of 16 to 21 years.</p>
        <p>The minors versus adults classification achieved good accuracy results for both male (90%) and female subjects (84%). These models portrayed a drop in the performance for the ages of 16 and 17; however, the recalls regarding the correct classification of minors were very high in both male and female cases (100% and 96%, respectively). This is important because the problem of minors versus adults classification is asymmetric as the misclassification of minors for adults in a judicial scenario is much more problematic than the inverse. In most cases, the application of the law is harsher for adults, and imputability, along with granted rights, can drastically change between these groups.</p>
        <p>The CA estimation models achieved MAEs of 0.95 years and 1.24 years for male and female subjects, respectively. However, a look at a depth of the models showed that for both male and female models, only the ages of 14 and 15 years achieved acceptable MAE values. It could be argued that for the ages of 16 to 21 years, the estimation of a precise CA based on stages of bone development of the calcaneus, distal tibia, proximal tibia, distal femur, and radius growth plates would be somewhat unfit for male individuals and very unfit for female individuals. Furthermore, we could argue that staging may not offer a precise enough measure for the estimation of the CA of individuals of the ages considered in this study.</p>
        <p>Compared with dental age, height, and age at menarche, BA is still the most reliable biological indicator for assessing maturation in young individuals [<xref ref-type="bibr" rid="ref50">50</xref>], but it may not be a strong predictor of CA. BAA was conceived to be used in conjunction with CA to evaluate the maturation of an individual that can be delayed or advanced due to various factors that may include hormonal disorders, and chronic illnesses [<xref ref-type="bibr" rid="ref8">8</xref>].</p>
        <p>Regarding the agreement of the radiologists on the assessment of the growth plates' stage of development, substantial agreement was achieved, which is a satisfactory result as there is a lack of guidelines for BAA using MRI in the research. In addition, the individuals employed in the assessment of the MRI images were specialized pediatric radiologists with experience in BAA.</p>
        <p>From a methodological point of view, this study employed a nested cross-validation approach that aims to avoid reporting overly optimistic results that could be derived from a <italic>lucky</italic> test set.</p>
      </sec>
      <sec>
        <title>Comparison With Prior Work</title>
        <p>Most of the studies in the area of BAA that employ ML algorithms aim to build automatic approaches for estimating BA and evaluating BA given by radiologists [<xref ref-type="bibr" rid="ref20">20</xref>]. The biggest initiative for proposing automated approaches in this direction was the Radiological Society of North America (RSNA) 2018 Bone Age Challenge [<xref ref-type="bibr" rid="ref51">51</xref>]. This challenge provided a database of circa 12,000 radiographs of subjects from 0 to 19 years, labeled with the BA given by radiologists, following the GP method. Although the first places achieved MAEs of 4.26, 4.35, and 4.38 months, these results are not comparable with our results because the aim of our study was to estimate CA, and the RSNA challenge goal was to propose models for predicting the BA given by radiologists [<xref ref-type="bibr" rid="ref51">51</xref>]. In addition, it is worth mentioning that however large the sample provided for the challenge, only 0.74% (94/12,612) of the sample consisted of 18- and 19-year-old subjects, which are important legal ages.</p>
        <p>For studies that employ BA concepts to predict the CA of subjects, there are studies by Dallora et al [<xref ref-type="bibr" rid="ref52">52</xref>] and Stern et al [<xref ref-type="bibr" rid="ref53">53</xref>]. Both employ MRI as the medical imaging of choice, and most importantly, they are not based on traditional BAA to make their predictions of CA. They employ deep learning technology, which is able to learn the important features in the images and then perform regression or classification [<xref ref-type="bibr" rid="ref54">54</xref>]. The reasoning behind using deep learning to interpret images and learn features is that it is difficult for humans to translate image features into descriptive means, and it is easy to lose information on the process. On the other hand, this problem has a reduced risk of occurring with algorithms able to analyze images pixel by pixel [<xref ref-type="bibr" rid="ref55">55</xref>]. Dallora et al [<xref ref-type="bibr" rid="ref52">52</xref>] used knee MRI images and achieved an MAE of 0.793 years for male subjects in the range of 14 to 20 years, and 0.988 years for female subjects in the range of 14 to 19 years. Stern et al [<xref ref-type="bibr" rid="ref53">53</xref>] used MRI images of the hand and achieved an MAE of 0.82 years for male subjects in the range of 13 to 19 years. A previous study by Stern et al [<xref ref-type="bibr" rid="ref56">56</xref>] proposed a deep learning multifactorial approach that used MRI volumes of the hand, clavicle, and teeth to estimate the CA of male subjects aged 13 to 25 years, achieving an MAE of 1.01 years. The study by Tang et al [<xref ref-type="bibr" rid="ref57">57</xref>] used MRI for CA estimation in adolescents from 12 to 17 years, which leaves out the legal age of 18, using artificial neural networks. This was also a multifactorial approach that considered the subjects' height, weight, and bone marrow composition intensity quantified by MRI and TW3 assessment, achieving a mean disparity (comparison between the mean CA for all subjects and the mean estimated age for all subjects) of 0.1 years. This study also demonstrated that the BA given by the TW3 method was consistently lower than that of the subjects' CA. The study by Hillewig et al [<xref ref-type="bibr" rid="ref58">58</xref>] investigated a multiple ROI approach that considered primarily the radiologists' assessment of MRI images of the clavicle, but also the assessment of x-rays of the hand and wrist area, with the aim of determining whether an individual is younger or older than 18 years, considering a sample of subjects from 16 to 26 years. It was evidenced that the clavicle assessment in stage IV (according to the Schmeling et al [<xref ref-type="bibr" rid="ref9">9</xref>] and Kreitner et al [<xref ref-type="bibr" rid="ref13">13</xref>] staging systems) was particularly important for age determination; however, in cases where staging is challenging for radiologists, the assessment of the hand and wrist area is essential.</p>
      </sec>
      <sec>
        <title>Limitations</title>
        <p>Regarding limitations of the study, it could be argued that due to the high number of classes in the multiclass classification, the sample size in each class would not be large enough to build a generalizable model. However, to address this issue, we employed methods to ensure that the model would not overfit and for not choosing the most overly optimistic choice given by the nested cross-validation. In addition, during data collection, we ensured a uniform number of subjects in each class to guarantee a balanced data set.</p>
        <p>The selected ROI for this work took into consideration the stress levels for the minors and young adult subjects with regard to the MRI examination. Hence, the clavicle and arm were not considered because it would require the subjects to go head in the MRI machine, which could cause discomfort and stress to the young subjects due to loud noises and small enclosed spaces. In addition, the clavicle has a high risk of producing moving artifacts due to breathing movements. On the practical side, the examination time was on average 15 min, and the inclusion of these 2 regions would take approximately double the time.</p>
      </sec>
      <sec>
        <title>Conclusions</title>
        <p>This paper presented models for CA estimation and minors versus adults classification (on a threshold of 18 years) using ML algorithms. The models were trained with radiologists assessment of the calcaneus, distal tibia, proximal tibia, distal femur, and radius; and additional information regarding physical activity level, parents' origin, type of residence during upbringing, and a self-assessed Tanner Scale of pubertal growth. The models proposed for the classification of minor versus adults produced accuracies of 90% and 84% for male and female subjects, respectively, with very high recalls for the classification of minors. However, for the chronological age estimation for the 8 age groups, ranging from to 14 to 21, the variables in the model did not turn out to be precise enough for estimating the exact CA, only showing acceptable values of MAE for the ages of 14 and 15 years.</p>
        <p>Future research should focus on applying deep learning technology for the estimation of CA using multiple ROIs.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group>
      <supplementary-material id="app1">
        <label>Multimedia Appendix 1</label>
        <p>Data characteristics of the variables collected through the questionnaire and BMI.</p>
        <media xlink:href="medinform_v8i9e18846_app1.pdf" xlink:title="PDF File  (Adobe PDF File), 96 KB"/>
      </supplementary-material>
      <supplementary-material id="app2">
        <label>Multimedia Appendix 2</label>
        <p>Results from the assessment of magnetic resonance images of male subjects.</p>
        <media xlink:href="medinform_v8i9e18846_app2.pdf" xlink:title="PDF File  (Adobe PDF File), 25 KB"/>
      </supplementary-material>
      <supplementary-material id="app3">
        <label>Multimedia Appendix 3</label>
        <p>Results from the assessment of magnetic resonance images of female subjects.</p>
        <media xlink:href="medinform_v8i9e18846_app3.pdf" xlink:title="PDF File  (Adobe PDF File), 24 KB"/>
      </supplementary-material>
    </app-group>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">AUC</term>
          <def>
            <p>area under the curve</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">BA</term>
          <def>
            <p>bone age</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">BAA</term>
          <def>
            <p>bone age assessment</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb4">CA</term>
          <def>
            <p>chronological age</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb5">GP</term>
          <def>
            <p>Greulich-Pyle</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb6">KNN</term>
          <def>
            <p>K-nearest neighbors</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb7">MAE</term>
          <def>
            <p>mean absolute error</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb8">MLP</term>
          <def>
            <p>multilayer perceptron</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb9">MRI</term>
          <def>
            <p>magnetic resonance imaging</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb10">RMSE</term>
          <def>
            <p>root mean squared error</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb11">ROI</term>
          <def>
            <p>region of interest</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb12">RSNA</term>
          <def>
            <p>Radiological Society of North America</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb13">SLR</term>
          <def>
            <p>systematic literature review</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb14">SVM</term>
          <def>
            <p>support vector machines</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb15">TW</term>
          <def>
            <p>Tanner-Whitehouse</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>The authors would like to express their greatest appreciation to the participants and staff who participated in our study. This work was supported by the National Board of Health and Welfare of Sweden (Socialstyrelsen). The funding source had no involvement in the study design, data collection, analysis, interpretation, or reporting of this work.</p>
    </ack>
    <fn-group>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gilsanz</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Ratib</surname>
              <given-names>O</given-names>
            </name>
          </person-group>
          <source>-</source>
          <year>2005</year>
          <publisher-loc>Hand Bone Age</publisher-loc>
          <publisher-name>A Digital Atlas Of Skeletal Maturity. Springer Science &#38; Business Media</publisher-name>
          <fpage>A</fpage>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mansourvar</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Ismail</surname>
              <given-names>MA</given-names>
            </name>
            <name name-style="western">
              <surname>Herawan</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Raj</surname>
              <given-names>RG</given-names>
            </name>
            <name name-style="western">
              <surname>Kareem</surname>
              <given-names>SA</given-names>
            </name>
            <name name-style="western">
              <surname>Nasaruddin</surname>
              <given-names>FH</given-names>
            </name>
          </person-group>
          <article-title>Automated bone age assessment: motivation, taxonomies, and challenges</article-title>
          <source>Comput Math Methods Med</source>
          <year>2013</year>
          <volume>2013</volume>
          <fpage>391626</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1155/2013/391626"/>
          </comment>
          <pub-id pub-id-type="doi">10.1155/2013/391626</pub-id>
          <pub-id pub-id-type="medline">24454534</pub-id>
          <pub-id pub-id-type="pmcid">PMC3876824</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Satoh</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Bone age: assessment methods and clinical applications</article-title>
          <source>Clin Pediatr Endocrinol</source>
          <year>2015</year>
          <month>10</month>
          <volume>24</volume>
          <issue>4</issue>
          <fpage>143</fpage>
          <lpage>52</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/26568655"/>
          </comment>
          <pub-id pub-id-type="doi">10.1297/cpe.24.143</pub-id>
          <pub-id pub-id-type="medline">26568655</pub-id>
          <pub-id pub-id-type="pii">2015-0011</pub-id>
          <pub-id pub-id-type="pmcid">PMC4628949</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Dvorak</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>George</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Junge</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Hodler</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Application of MRI of the wrist for age determination in international U-17 soccer competitions</article-title>
          <source>Br J Sports Med</source>
          <year>2007</year>
          <month>08</month>
          <volume>41</volume>
          <issue>8</issue>
          <fpage>497</fpage>
          <lpage>500</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/17347314"/>
          </comment>
          <pub-id pub-id-type="doi">10.1136/bjsm.2006.033431</pub-id>
          <pub-id pub-id-type="medline">17347314</pub-id>
          <pub-id pub-id-type="pii">bjsm.2006.033431</pub-id>
          <pub-id pub-id-type="pmcid">PMC2465442</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Dvorak</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>George</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Junge</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Hodler</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Age determination by magnetic resonance imaging of the wrist in adolescent male football players</article-title>
          <source>Br J Sports Med</source>
          <year>2007</year>
          <month>01</month>
          <volume>41</volume>
          <issue>1</issue>
          <fpage>45</fpage>
          <lpage>52</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/17021001"/>
          </comment>
          <pub-id pub-id-type="doi">10.1136/bjsm.2006.031021</pub-id>
          <pub-id pub-id-type="medline">17021001</pub-id>
          <pub-id pub-id-type="pii">bjsm.2006.031021</pub-id>
          <pub-id pub-id-type="pmcid">PMC2465138</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Schmidt</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Vieth</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Timme</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Dvorak</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Schmeling</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Examination of ossification of the distal radial epiphysis using magnetic resonance imaging. New insights for age estimation in young footballers in FIFA tournaments</article-title>
          <source>Sci Justice</source>
          <year>2015</year>
          <month>03</month>
          <volume>55</volume>
          <issue>2</issue>
          <fpage>139</fpage>
          <lpage>44</lpage>
          <pub-id pub-id-type="doi">10.1016/j.scijus.2014.12.003</pub-id>
          <pub-id pub-id-type="medline">25754000</pub-id>
          <pub-id pub-id-type="pii">S1355-0306(14)00168-3</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cunha</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Baccino</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Martrille</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Ramsthaler</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Prieto</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Schuliar</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Lynnerup</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Cattaneo</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>The problem of aging human remains and living individuals: a review</article-title>
          <source>Forensic Sci Int</source>
          <year>2009</year>
          <month>12</month>
          <day>15</day>
          <volume>193</volume>
          <issue>1-3</issue>
          <fpage>1</fpage>
          <lpage>13</lpage>
          <pub-id pub-id-type="doi">10.1016/j.forsciint.2009.09.008</pub-id>
          <pub-id pub-id-type="medline">19879075</pub-id>
          <pub-id pub-id-type="pii">S0379-0738(09)00370-3</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hjern</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Brendler-Lindqvist</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Norredam</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Age assessment of young asylum seekers</article-title>
          <source>Acta Paediatr</source>
          <year>2012</year>
          <month>01</month>
          <volume>101</volume>
          <issue>1</issue>
          <fpage>4</fpage>
          <lpage>7</lpage>
          <pub-id pub-id-type="doi">10.1111/j.1651-2227.2011.02476.x</pub-id>
          <pub-id pub-id-type="medline">21950617</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Schmeling</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Schulz</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Reisinger</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Mühler</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Wernecke</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Geserick</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Studies on the time frame for ossification of the medial clavicular epiphyseal cartilage in conventional radiography</article-title>
          <source>Int J Legal Med</source>
          <year>2004</year>
          <month>02</month>
          <volume>118</volume>
          <issue>1</issue>
          <fpage>5</fpage>
          <lpage>8</lpage>
          <pub-id pub-id-type="doi">10.1007/s00414-003-0404-5</pub-id>
          <pub-id pub-id-type="medline">14534796</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Greulich</surname>
              <given-names>WW</given-names>
            </name>
            <name name-style="western">
              <surname>Pyle</surname>
              <given-names>SI</given-names>
            </name>
          </person-group>
          <article-title>Radiographic atlas of skeletal development of the hand and wrist</article-title>
          <source>Am J Med Sci</source>
          <year>1959</year>
          <volume>238</volume>
          <issue>3</issue>
          <fpage>393</fpage>
          <pub-id pub-id-type="doi">10.1097/00000441-195909000-00030</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ehrenberg</surname>
              <given-names>AS</given-names>
            </name>
            <name name-style="western">
              <surname>Tanner</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Whitehouse</surname>
              <given-names>RH</given-names>
            </name>
            <name name-style="western">
              <surname>Marshall</surname>
              <given-names>WA</given-names>
            </name>
            <name name-style="western">
              <surname>Healy</surname>
              <given-names>MJ</given-names>
            </name>
            <name name-style="western">
              <surname>Goldstein</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Assessment of skeletal maturity and prediction of adult height (TW2 method)</article-title>
          <source>Appl Stat</source>
          <year>1977</year>
          <volume>26</volume>
          <issue>1</issue>
          <fpage>80</fpage>
          <pub-id pub-id-type="doi">10.2307/2346874</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Nahhas</surname>
              <given-names>RW</given-names>
            </name>
            <name name-style="western">
              <surname>Sherwood</surname>
              <given-names>RJ</given-names>
            </name>
            <name name-style="western">
              <surname>Chumlea</surname>
              <given-names>WC</given-names>
            </name>
            <name name-style="western">
              <surname>Duren</surname>
              <given-names>DL</given-names>
            </name>
          </person-group>
          <article-title>An update of the statistical methods underlying the FELS method of skeletal maturity assessment</article-title>
          <source>Ann Hum Biol</source>
          <year>2013</year>
          <volume>40</volume>
          <issue>6</issue>
          <fpage>505</fpage>
          <lpage>14</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/23992229"/>
          </comment>
          <pub-id pub-id-type="doi">10.3109/03014460.2013.806591</pub-id>
          <pub-id pub-id-type="medline">23992229</pub-id>
          <pub-id pub-id-type="pmcid">PMC4013782</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kreitner</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Schweden</surname>
              <given-names>FJ</given-names>
            </name>
            <name name-style="western">
              <surname>Riepert</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Nafe</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Thelen</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Bone age determination based on the study of the medial extremity of the clavicle</article-title>
          <source>Eur Radiol</source>
          <year>1998</year>
          <volume>8</volume>
          <issue>7</issue>
          <fpage>1116</fpage>
          <lpage>22</lpage>
          <pub-id pub-id-type="doi">10.1007/s003300050518</pub-id>
          <pub-id pub-id-type="medline">9724422</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kellinghaus</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Schulz</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Vieth</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Schmidt</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Pfeiffer</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Schmeling</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Enhanced possibilities to make statements on the ossification status of the medial clavicular epiphysis using an amplified staging scheme in evaluating thin-slice CT scans</article-title>
          <source>Int J Legal Med</source>
          <year>2010</year>
          <month>07</month>
          <volume>124</volume>
          <issue>4</issue>
          <fpage>321</fpage>
          <lpage>5</lpage>
          <pub-id pub-id-type="doi">10.1007/s00414-010-0448-2</pub-id>
          <pub-id pub-id-type="medline">20354711</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>O'Connor</surname>
              <given-names>JE</given-names>
            </name>
            <name name-style="western">
              <surname>Bogue</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Spence</surname>
              <given-names>LD</given-names>
            </name>
            <name name-style="western">
              <surname>Last</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>A method to establish the relationship between chronological age and stage of union from radiographic assessment of epiphyseal fusion at the knee: an Irish population study</article-title>
          <source>J Anat</source>
          <year>2008</year>
          <month>02</month>
          <volume>212</volume>
          <issue>2</issue>
          <fpage>198</fpage>
          <lpage>209</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1111/j.1469-7580.2007.00847.x"/>
          </comment>
          <pub-id pub-id-type="doi">10.1111/j.1469-7580.2007.00847.x</pub-id>
          <pub-id pub-id-type="medline">18179475</pub-id>
          <pub-id pub-id-type="pii">JOA847</pub-id>
          <pub-id pub-id-type="pmcid">PMC2408977</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Dedouit</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Auriol</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Rousseau</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Rougé</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Crubézy</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Telmon</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>Age assessment by magnetic resonance imaging of the knee: a preliminary study</article-title>
          <source>Forensic Sci Int</source>
          <year>2012</year>
          <month>04</month>
          <day>10</day>
          <volume>217</volume>
          <issue>1-3</issue>
          <fpage>232.e1</fpage>
          <lpage>7</lpage>
          <pub-id pub-id-type="doi">10.1016/j.forsciint.2011.11.013</pub-id>
          <pub-id pub-id-type="medline">22153621</pub-id>
          <pub-id pub-id-type="pii">S0379-0738(11)00548-2</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Krämer</surname>
              <given-names>JA</given-names>
            </name>
            <name name-style="western">
              <surname>Schmidt</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Jürgens</surname>
              <given-names>KU</given-names>
            </name>
            <name name-style="western">
              <surname>Lentschig</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Schmeling</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Vieth</surname>
              <given-names>V</given-names>
            </name>
          </person-group>
          <article-title>Forensic age estimation in living individuals using 3.0 T MRI of the distal femur</article-title>
          <source>Int J Legal Med</source>
          <year>2014</year>
          <month>05</month>
          <volume>128</volume>
          <issue>3</issue>
          <fpage>509</fpage>
          <lpage>14</lpage>
          <pub-id pub-id-type="doi">10.1007/s00414-014-0967-3</pub-id>
          <pub-id pub-id-type="medline">24504560</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ekizoglu</surname>
              <given-names>O</given-names>
            </name>
            <name name-style="western">
              <surname>Hocaoglu</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Can</surname>
              <given-names>IO</given-names>
            </name>
            <name name-style="western">
              <surname>Inci</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Aksoy</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Bilgili</surname>
              <given-names>MG</given-names>
            </name>
          </person-group>
          <article-title>Magnetic resonance imaging of distal tibia and calcaneus for forensic age estimation in living individuals</article-title>
          <source>Int J Legal Med</source>
          <year>2015</year>
          <month>07</month>
          <volume>129</volume>
          <issue>4</issue>
          <fpage>825</fpage>
          <lpage>31</lpage>
          <pub-id pub-id-type="doi">10.1007/s00414-015-1187-1</pub-id>
          <pub-id pub-id-type="medline">25904076</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ekizoglu</surname>
              <given-names>O</given-names>
            </name>
            <name name-style="western">
              <surname>Inci</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Ors</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Kacmaz</surname>
              <given-names>IE</given-names>
            </name>
            <name name-style="western">
              <surname>Basa</surname>
              <given-names>CD</given-names>
            </name>
            <name name-style="western">
              <surname>Can</surname>
              <given-names>IO</given-names>
            </name>
            <name name-style="western">
              <surname>Kranioti</surname>
              <given-names>EF</given-names>
            </name>
          </person-group>
          <article-title>Applicability of T1-weighted MRI in the assessment of forensic age based on the epiphyseal closure of the humeral head</article-title>
          <source>Int J Legal Med</source>
          <year>2019</year>
          <month>01</month>
          <volume>133</volume>
          <issue>1</issue>
          <fpage>241</fpage>
          <lpage>8</lpage>
          <pub-id pub-id-type="doi">10.1007/s00414-018-1868-7</pub-id>
          <pub-id pub-id-type="medline">29804276</pub-id>
          <pub-id pub-id-type="pii">10.1007/s00414-018-1868-7</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Dallora</surname>
              <given-names>AL</given-names>
            </name>
            <name name-style="western">
              <surname>Anderberg</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Kvist</surname>
              <given-names>O</given-names>
            </name>
            <name name-style="western">
              <surname>Mendes</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Diaz Ruiz</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Sanmartin Berglund</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Bone age assessment with various machine learning techniques: a systematic literature review and meta-analysis</article-title>
          <source>PLoS One</source>
          <year>2019</year>
          <volume>14</volume>
          <issue>7</issue>
          <fpage>e0220242</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://dx.plos.org/10.1371/journal.pone.0220242"/>
          </comment>
          <pub-id pub-id-type="doi">10.1371/journal.pone.0220242</pub-id>
          <pub-id pub-id-type="medline">31344143</pub-id>
          <pub-id pub-id-type="pii">PONE-D-19-09344</pub-id>
          <pub-id pub-id-type="pmcid">PMC6657881</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Crema</surname>
              <given-names>MD</given-names>
            </name>
            <name name-style="western">
              <surname>Roemer</surname>
              <given-names>FW</given-names>
            </name>
            <name name-style="western">
              <surname>Marra</surname>
              <given-names>MD</given-names>
            </name>
            <name name-style="western">
              <surname>Burstein</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Gold</surname>
              <given-names>GE</given-names>
            </name>
            <name name-style="western">
              <surname>Eckstein</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Baum</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Mosher</surname>
              <given-names>TJ</given-names>
            </name>
            <name name-style="western">
              <surname>Carrino</surname>
              <given-names>JA</given-names>
            </name>
            <name name-style="western">
              <surname>Guermazi</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Articular cartilage in the knee: current MR imaging techniques and applications in clinical practice and research</article-title>
          <source>Radiographics</source>
          <year>2011</year>
          <volume>31</volume>
          <issue>1</issue>
          <fpage>37</fpage>
          <lpage>61</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/21257932"/>
          </comment>
          <pub-id pub-id-type="doi">10.1148/rg.311105084</pub-id>
          <pub-id pub-id-type="medline">21257932</pub-id>
          <pub-id pub-id-type="pii">31/1/37</pub-id>
          <pub-id pub-id-type="pmcid">PMC6939857</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gründer</surname>
              <given-names>W</given-names>
            </name>
          </person-group>
          <article-title>MRI assessment of cartilage ultrastructure</article-title>
          <source>NMR Biomed</source>
          <year>2006</year>
          <month>11</month>
          <volume>19</volume>
          <issue>7</issue>
          <fpage>855</fpage>
          <lpage>76</lpage>
          <pub-id pub-id-type="doi">10.1002/nbm.1092</pub-id>
          <pub-id pub-id-type="medline">17075962</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>de Simone</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Farello</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Palumbo</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Gentile</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Ciuffreda</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Olioso</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Cinque</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>De Matteis</surname>
              <given-names>F</given-names>
            </name>
          </person-group>
          <article-title>Growth charts, growth velocity and bone development in childhood obesity</article-title>
          <source>Int J Obes Relat Metab Disord</source>
          <year>1995</year>
          <month>12</month>
          <volume>19</volume>
          <issue>12</issue>
          <fpage>851</fpage>
          <lpage>7</lpage>
          <pub-id pub-id-type="medline">8963351</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cutler</surname>
              <given-names>GB</given-names>
            </name>
          </person-group>
          <article-title>The role of estrogen in bone growth and maturation during childhood and adolescence</article-title>
          <source>J Steroid Biochem Mol Biol</source>
          <year>1997</year>
          <month>04</month>
          <volume>61</volume>
          <issue>3-6</issue>
          <fpage>141</fpage>
          <lpage>4</lpage>
          <pub-id pub-id-type="medline">9365183</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mirtz</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Chandler</surname>
              <given-names>JP</given-names>
            </name>
            <name name-style="western">
              <surname>Eyers</surname>
              <given-names>CM</given-names>
            </name>
          </person-group>
          <article-title>The effects of physical activity on the epiphyseal growth plates: a review of the literature on normal physiology and clinical implications</article-title>
          <source>J Clin Med Res</source>
          <year>2011</year>
          <month>02</month>
          <day>12</day>
          <volume>3</volume>
          <issue>1</issue>
          <fpage>1</fpage>
          <lpage>7</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/22043265"/>
          </comment>
          <pub-id pub-id-type="doi">10.4021/jocmr477w</pub-id>
          <pub-id pub-id-type="medline">22043265</pub-id>
          <pub-id pub-id-type="pmcid">PMC3194019</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ontell</surname>
              <given-names>FK</given-names>
            </name>
            <name name-style="western">
              <surname>Ivanovic</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Ablin</surname>
              <given-names>DS</given-names>
            </name>
            <name name-style="western">
              <surname>Barlow</surname>
              <given-names>TW</given-names>
            </name>
          </person-group>
          <article-title>Bone age in children of diverse ethnicity</article-title>
          <source>AJR Am J Roentgenol</source>
          <year>1996</year>
          <month>12</month>
          <volume>167</volume>
          <issue>6</issue>
          <fpage>1395</fpage>
          <lpage>8</lpage>
          <pub-id pub-id-type="doi">10.2214/ajr.167.6.8956565</pub-id>
          <pub-id pub-id-type="medline">8956565</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Karlberg</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Secular trends in pubertal development</article-title>
          <source>Horm Res</source>
          <year>2002</year>
          <volume>57</volume>
          <issue>Suppl 2</issue>
          <fpage>19</fpage>
          <lpage>30</lpage>
          <pub-id pub-id-type="doi">10.1159/000058096</pub-id>
          <pub-id pub-id-type="medline">12065922</pub-id>
          <pub-id pub-id-type="pii">58096</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Tanner</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Whitehouse</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Cameron</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <source>Assessment of skeletal maturity and prediction of adult height (TW3 Method). 3rd edn</source>
          <year>2001</year>
          <publisher-loc>WB Saunders</publisher-loc>
          <publisher-name>London</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Shen</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>CJ</given-names>
            </name>
            <name name-style="western">
              <surname>Jiang</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Song</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>He</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Wong</surname>
              <given-names>SY</given-names>
            </name>
            <name name-style="western">
              <surname>Fang</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Ming</surname>
              <given-names>W</given-names>
            </name>
          </person-group>
          <article-title>Artificial intelligence versus clinicians in disease diagnosis: systematic review</article-title>
          <source>JMIR Med Inform</source>
          <year>2019</year>
          <month>08</month>
          <day>16</day>
          <volume>7</volume>
          <issue>3</issue>
          <fpage>e10010</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://medinform.jmir.org/2019/3/e10010/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/10010</pub-id>
          <pub-id pub-id-type="medline">31420959</pub-id>
          <pub-id pub-id-type="pii">v7i3e10010</pub-id>
          <pub-id pub-id-type="pmcid">PMC6716335</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Triantafyllidis</surname>
              <given-names>AK</given-names>
            </name>
            <name name-style="western">
              <surname>Tsanas</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Applications of machine learning in real-life digital health interventions: review of the literature</article-title>
          <source>J Med Internet Res</source>
          <year>2019</year>
          <month>04</month>
          <day>5</day>
          <volume>21</volume>
          <issue>4</issue>
          <fpage>e12286</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2019/4/e12286/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/12286</pub-id>
          <pub-id pub-id-type="medline">30950797</pub-id>
          <pub-id pub-id-type="pii">v21i4e12286</pub-id>
          <pub-id pub-id-type="pmcid">PMC6473205</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mitchell</surname>
              <given-names>TM</given-names>
            </name>
          </person-group>
          <article-title>Machine learning and data mining</article-title>
          <source>Commun ACM</source>
          <year>1999</year>
          <month>11</month>
          <volume>42</volume>
          <issue>11</issue>
          <fpage>30</fpage>
          <lpage>6</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.ri.cmu.edu/pub_files/pub1/mitchell_tom_1999_1/mitchell_tom_1999_1.pdf"/>
          </comment>
          <pub-id pub-id-type="doi">10.1145/319382.319388</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Thodberg</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Kreiborg</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Juul</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Pedersen</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>The BoneXpert method for automated determination of skeletal maturity</article-title>
          <source>IEEE Trans Med Imaging</source>
          <year>2009</year>
          <month>01</month>
          <volume>28</volume>
          <issue>1</issue>
          <fpage>52</fpage>
          <lpage>66</lpage>
          <pub-id pub-id-type="doi">10.1109/TMI.2008.926067</pub-id>
          <pub-id pub-id-type="medline">19116188</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Marshall</surname>
              <given-names>WA</given-names>
            </name>
            <name name-style="western">
              <surname>Tanner</surname>
              <given-names>JM</given-names>
            </name>
          </person-group>
          <article-title>Variations in pattern of pubertal changes in girls</article-title>
          <source>Arch Dis Child</source>
          <year>1969</year>
          <month>06</month>
          <volume>44</volume>
          <issue>235</issue>
          <fpage>291</fpage>
          <lpage>303</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/5785179"/>
          </comment>
          <pub-id pub-id-type="doi">10.1136/adc.44.235.291</pub-id>
          <pub-id pub-id-type="medline">5785179</pub-id>
          <pub-id pub-id-type="pmcid">PMC2020314</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Marshall</surname>
              <given-names>WA</given-names>
            </name>
            <name name-style="western">
              <surname>Tanner</surname>
              <given-names>JM</given-names>
            </name>
          </person-group>
          <article-title>Variations in the pattern of pubertal changes in boys</article-title>
          <source>Arch Dis Child</source>
          <year>1970</year>
          <month>02</month>
          <volume>45</volume>
          <issue>239</issue>
          <fpage>13</fpage>
          <lpage>23</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/5440182"/>
          </comment>
          <pub-id pub-id-type="doi">10.1136/adc.45.239.13</pub-id>
          <pub-id pub-id-type="medline">5440182</pub-id>
          <pub-id pub-id-type="pmcid">PMC2020414</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Keys</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Fidanza</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Karvonen</surname>
              <given-names>MJ</given-names>
            </name>
            <name name-style="western">
              <surname>Kimura</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Taylor</surname>
              <given-names>HL</given-names>
            </name>
          </person-group>
          <article-title>Indices of relative weight and obesity</article-title>
          <source>Int J Epidemiol</source>
          <year>2014</year>
          <month>06</month>
          <volume>43</volume>
          <issue>3</issue>
          <fpage>655</fpage>
          <lpage>65</lpage>
          <pub-id pub-id-type="doi">10.1093/ije/dyu058</pub-id>
          <pub-id pub-id-type="medline">24691951</pub-id>
          <pub-id pub-id-type="pii">dyu058</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref36">
        <label>36</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cohen</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>A coefficient of agreement for nominal scales</article-title>
          <source>Educ Psychol Meas</source>
          <year>2016</year>
          <month>07</month>
          <day>2</day>
          <volume>20</volume>
          <issue>1</issue>
          <fpage>37</fpage>
          <lpage>46</lpage>
          <pub-id pub-id-type="doi">10.1177/001316446002000104</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref37">
        <label>37</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bobicev</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Sokolova</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Inter-Annotator Agreement in Sentiment Analysis: Machine Learning Perspective</article-title>
          <source>RANLP</source>
          <year>2017</year>
          <volume>97</volume>
          <fpage>-</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.acl-bg.org/proceedings/2017/RANLP%202017/pdf/RANLP015.pdf"/>
          </comment>
          <pub-id pub-id-type="doi">10.26615/978-954-452-049-6_015</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref38">
        <label>38</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Nearest neighbor selection for iteratively kNN imputation</article-title>
          <source>J Syst Softw</source>
          <year>2012</year>
          <month>11</month>
          <volume>85</volume>
          <issue>11</issue>
          <fpage>2541</fpage>
          <lpage>52</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://paperpile.com/b/JmyQqQ/8LYXs"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.jss.2012.05.073</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref39">
        <label>39</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Beretta</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Santaniello</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Nearest neighbor imputation algorithms: a critical evaluation</article-title>
          <source>BMC Med Inform Decis Mak</source>
          <year>2016</year>
          <month>07</month>
          <day>25</day>
          <volume>16</volume>
          <issue>Suppl 3</issue>
          <fpage>74</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmcmedinformdecismak.biomedcentral.com/articles/10.1186/s12911-016-0318-z"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s12911-016-0318-z</pub-id>
          <pub-id pub-id-type="medline">27454392</pub-id>
          <pub-id pub-id-type="pii">10.1186/s12911-016-0318-z</pub-id>
          <pub-id pub-id-type="pmcid">PMC4959387</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref40">
        <label>40</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gower</surname>
              <given-names>JC</given-names>
            </name>
          </person-group>
          <article-title>A general coefficient of similarity and some of its properties</article-title>
          <source>Biometrics</source>
          <year>1971</year>
          <month>12</month>
          <volume>27</volume>
          <issue>4</issue>
          <fpage>857</fpage>
          <pub-id pub-id-type="doi">10.2307/2528823</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref41">
        <label>41</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kuhn</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Johnson</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Applied Predictive Modeling</article-title>
          <source>Springer, New York, NY</source>
          <year>2013</year>
          <fpage>-</fpage>
          <pub-id pub-id-type="doi">10.1007/978-1-4614-6849-3</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref42">
        <label>42</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Krstajic</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Buturovic</surname>
              <given-names>LJ</given-names>
            </name>
            <name name-style="western">
              <surname>Leahy</surname>
              <given-names>DE</given-names>
            </name>
            <name name-style="western">
              <surname>Thomas</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Cross-validation pitfalls when selecting and assessing regression and classification models</article-title>
          <source>J Cheminform</source>
          <year>2014</year>
          <month>03</month>
          <day>29</day>
          <volume>6</volume>
          <issue>1</issue>
          <fpage>10</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://dx.doi.org/10.1186/1758-2946-6-10"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/1758-2946-6-10</pub-id>
          <pub-id pub-id-type="medline">24678909</pub-id>
          <pub-id pub-id-type="pii">10.1186/1758-2946-6-10</pub-id>
          <pub-id pub-id-type="pmcid">PMC3994246</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref43">
        <label>43</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wainer</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Cawley</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Nested cross-validation when selecting classifiers is overzealous for most practical applications. arXiv cs</article-title>
          <source>LG</source>
          <year>2018</year>
          <volume>Available</volume>
          <fpage>-</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://arxiv.org/abs/1809.09446"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref44">
        <label>44</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gaudette</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Japkowicz</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>Evaluation Methods for Ordinal Classification</article-title>
          <source>Advances in Artificial Intelligence. Springer Berlin Heidelberg. pp. 207?</source>
          <year>2009</year>
          <fpage>210</fpage>
          <pub-id pub-id-type="doi">10.1007/978-3-642-01818-3_25</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref45">
        <label>45</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sokolova</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Lapalme</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>A systematic analysis of performance measures for classification tasks</article-title>
          <source>Inf Process Manag</source>
          <year>2009</year>
          <month>07</month>
          <volume>45</volume>
          <issue>4</issue>
          <fpage>427</fpage>
          <lpage>37</lpage>
          <pub-id pub-id-type="doi">10.1016/j.ipm.2009.03.002</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref46">
        <label>46</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hand</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Till</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>A simple generalisation of the area under the ROC curve for multiple class classification problems</article-title>
          <source>Mach Learn</source>
          <year>2001</year>
          <volume>45</volume>
          <fpage>186</fpage>
          <pub-id pub-id-type="doi">10.1023/A:1010920819831</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref47">
        <label>47</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Landis</surname>
              <given-names>JR</given-names>
            </name>
            <name name-style="western">
              <surname>Koch</surname>
              <given-names>GG</given-names>
            </name>
          </person-group>
          <article-title>The measurement of observer agreement for categorical data</article-title>
          <source>Biometrics</source>
          <year>1977</year>
          <month>03</month>
          <volume>33</volume>
          <issue>1</issue>
          <fpage>159</fpage>
          <lpage>74</lpage>
          <pub-id pub-id-type="medline">843571</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref48">
        <label>48</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Margalit</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Cottrill</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Nhan</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Yu</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Tang</surname>
              <given-names>X</given-names>
            </name>
            <name name-style="western">
              <surname>Fritz</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>RJ</given-names>
            </name>
          </person-group>
          <article-title>The spatial order of physeal maturation in the normal human knee using magnetic resonance imaging</article-title>
          <source>J Pediatr Orthop</source>
          <year>2019</year>
          <month>04</month>
          <volume>39</volume>
          <issue>4</issue>
          <fpage>e318</fpage>
          <lpage>22</lpage>
          <pub-id pub-id-type="doi">10.1097/BPO.0000000000001298</pub-id>
          <pub-id pub-id-type="medline">30451813</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref49">
        <label>49</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>O'Connor</surname>
              <given-names>JE</given-names>
            </name>
            <name name-style="western">
              <surname>Coyle</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Bogue</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Spence</surname>
              <given-names>LD</given-names>
            </name>
            <name name-style="western">
              <surname>Last</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Age prediction formulae from radiographic assessment of skeletal maturation at the knee in an Irish population</article-title>
          <source>Forensic Sci Int</source>
          <year>2014</year>
          <month>01</month>
          <volume>234</volume>
          <fpage>188.e1</fpage>
          <lpage>8</lpage>
          <pub-id pub-id-type="doi">10.1016/j.forsciint.2013.10.032</pub-id>
          <pub-id pub-id-type="medline">24262807</pub-id>
          <pub-id pub-id-type="pii">S0379-0738(13)00477-5</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref50">
        <label>50</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cox</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>The biology of bone maturation and ageing</article-title>
          <source>Acta Paediatr Suppl</source>
          <year>1997</year>
          <month>11</month>
          <volume>423</volume>
          <fpage>107</fpage>
          <lpage>8</lpage>
          <pub-id pub-id-type="doi">10.1111/j.1651-2227.1997.tb18386.x</pub-id>
          <pub-id pub-id-type="medline">9401555</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref51">
        <label>51</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Halabi</surname>
              <given-names>SS</given-names>
            </name>
            <name name-style="western">
              <surname>Prevedello</surname>
              <given-names>LM</given-names>
            </name>
            <name name-style="western">
              <surname>Kalpathy-Cramer</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Mamonov</surname>
              <given-names>AB</given-names>
            </name>
            <name name-style="western">
              <surname>Bilbily</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Cicero</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Pan</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Pereira</surname>
              <given-names>LA</given-names>
            </name>
            <name name-style="western">
              <surname>Sousa</surname>
              <given-names>RT</given-names>
            </name>
            <name name-style="western">
              <surname>Abdala</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Kitamura</surname>
              <given-names>FC</given-names>
            </name>
            <name name-style="western">
              <surname>Thodberg</surname>
              <given-names>HH</given-names>
            </name>
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Shih</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Andriole</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Kohli</surname>
              <given-names>MD</given-names>
            </name>
            <name name-style="western">
              <surname>Erickson</surname>
              <given-names>BJ</given-names>
            </name>
            <name name-style="western">
              <surname>Flanders</surname>
              <given-names>AE</given-names>
            </name>
          </person-group>
          <article-title>The RSNA pediatric bone age machine learning challenge</article-title>
          <source>Radiology</source>
          <year>2019</year>
          <month>02</month>
          <volume>290</volume>
          <issue>2</issue>
          <fpage>498</fpage>
          <lpage>503</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/30480490"/>
          </comment>
          <pub-id pub-id-type="doi">10.1148/radiol.2018180736</pub-id>
          <pub-id pub-id-type="medline">30480490</pub-id>
          <pub-id pub-id-type="pmcid">PMC6358027</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref52">
        <label>52</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Dallora</surname>
              <given-names>AL</given-names>
            </name>
            <name name-style="western">
              <surname>Berglund</surname>
              <given-names>JS</given-names>
            </name>
            <name name-style="western">
              <surname>Brogren</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Kvist</surname>
              <given-names>O</given-names>
            </name>
            <name name-style="western">
              <surname>Diaz Ruiz</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Dübbel</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Anderberg</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>Age assessment of youth and young adults using magnetic resonance imaging of the knee: a deep learning approach</article-title>
          <source>JMIR Med Inform</source>
          <year>2019</year>
          <month>12</month>
          <day>5</day>
          <volume>7</volume>
          <issue>4</issue>
          <fpage>e16291</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://medinform.jmir.org/2019/4/e16291/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/16291</pub-id>
          <pub-id pub-id-type="medline">31804183</pub-id>
          <pub-id pub-id-type="pii">v7i4e16291</pub-id>
          <pub-id pub-id-type="pmcid">PMC6923761</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref53">
        <label>53</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Štern</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Payer</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Urschler</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Automated age estimation from MRI volumes of the hand</article-title>
          <source>Med Image Anal</source>
          <year>2019</year>
          <month>12</month>
          <volume>58</volume>
          <fpage>101538</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S1361-8415(18)30479-1"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.media.2019.101538</pub-id>
          <pub-id pub-id-type="medline">31400620</pub-id>
          <pub-id pub-id-type="pii">S1361-8415(18)30479-1</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref54">
        <label>54</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ker</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Rao</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Lim</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>Deep learning applications in medical image analysis</article-title>
          <source>IEEE Access</source>
          <year>2018</year>
          <volume>6</volume>
          <fpage>9375</fpage>
          <lpage>89</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://paperpile.com/b/JmyQqQ/YVRQr"/>
          </comment>
          <pub-id pub-id-type="doi">10.1109/ACCESS.2017.2788044</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref55">
        <label>55</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ravi</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Wong</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Deligianni</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Berthelot</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Andreu-Perez</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Lo</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Yang</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Deep learning for health informatics</article-title>
          <source>IEEE J Biomed Health Inform</source>
          <year>2017</year>
          <month>01</month>
          <volume>21</volume>
          <issue>1</issue>
          <fpage>4</fpage>
          <lpage>21</lpage>
          <pub-id pub-id-type="doi">10.1109/JBHI.2016.2636665</pub-id>
          <pub-id pub-id-type="medline">28055930</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref56">
        <label>56</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Stern</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Payer</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Giuliani</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Urschler</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Automatic age estimation and majority age classification from multi-factorial MRI data</article-title>
          <source>IEEE J Biomed Health Inform</source>
          <year>2019</year>
          <month>07</month>
          <volume>23</volume>
          <issue>4</issue>
          <fpage>1392</fpage>
          <lpage>403</lpage>
          <pub-id pub-id-type="doi">10.1109/JBHI.2018.2869606</pub-id>
          <pub-id pub-id-type="medline">31059459</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref57">
        <label>57</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Tang</surname>
              <given-names>FH</given-names>
            </name>
            <name name-style="western">
              <surname>Chan</surname>
              <given-names>JL</given-names>
            </name>
            <name name-style="western">
              <surname>Chan</surname>
              <given-names>BK</given-names>
            </name>
          </person-group>
          <article-title>Accurate age determination for adolescents using magnetic resonance imaging of the hand and wrist with an artificial neural network-based approach</article-title>
          <source>J Digit Imaging</source>
          <year>2019</year>
          <month>04</month>
          <volume>32</volume>
          <issue>2</issue>
          <fpage>283</fpage>
          <lpage>9</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/30324428"/>
          </comment>
          <pub-id pub-id-type="doi">10.1007/s10278-018-0135-2</pub-id>
          <pub-id pub-id-type="medline">30324428</pub-id>
          <pub-id pub-id-type="pii">10.1007/s10278-018-0135-2</pub-id>
          <pub-id pub-id-type="pmcid">PMC6456634</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref58">
        <label>58</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hillewig</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Degroote</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>van der Paelt</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Visscher</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Vandemaele</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Lutin</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>D'Hooghe</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Vandriessche</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Piette</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Verstraete</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Magnetic resonance imaging of the sternal extremity of the clavicle in forensic age estimation: towards more sound age estimates</article-title>
          <source>Int J Legal Med</source>
          <year>2013</year>
          <month>05</month>
          <volume>127</volume>
          <issue>3</issue>
          <fpage>677</fpage>
          <lpage>89</lpage>
          <pub-id pub-id-type="doi">10.1007/s00414-012-0798-z</pub-id>
          <pub-id pub-id-type="medline">23224029</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
