@Article{info:doi/10.2196/68863,
author="Ohno, Yukiko
and Aomori, Tohru
and Nishiyama, Tomohiro
and Kato, Riri
and Fujiki, Reina
and Ishikawa, Haruki
and Kiyomiya, Keisuke
and Isawa, Minae
and Mochizuki, Mayumi
and Aramaki, Eiji
and Ohtani, Hisakazu",
title="Performance Improvement of a Natural Language Processing Tool for Extracting Patient Narratives Related to Medical States From Japanese Pharmaceutical Care Records by Increasing the Amount of Training Data: Natural Language Processing Analysis and Validation Study",
journal="JMIR Med Inform",
year="2025",
month="Mar",
day="4",
volume="13",
pages="e68863",
keywords="natural language processing; NLP; named entity recognition; NER; deep learning; pharmaceutical care record; electronic medical record; EMR; Japanese",
abstract="Background: Patients' oral expressions serve as valuable sources of clinical information to improve pharmacotherapy. Natural language processing (NLP) is a useful approach for analyzing unstructured text data, such as patient narratives. However, few studies have focused on using NLP for narratives in the Japanese language. Objective: We aimed to develop a high-performance NLP system for extracting clinical information from patient narratives by examining the performance progression with a gradual increase in the amount of training data. Methods: We used subjective texts from the pharmaceutical care records of Keio University Hospital from April 1, 2018, to March 31, 2019, comprising 12,004 records from 6559 cases. After preprocessing, we annotated diseases and symptoms within the texts. We then trained and evaluated a deep learning model (bidirectional encoder representations from transformers combined with a conditional random field [BERT-CRF]) through 10-fold cross-validation. The annotated data were divided into 10 subsets, and the amount of training data was progressively increased over 10 steps. We also analyzed the causes of errors. Finally, we applied the developed system to the analysis of case report texts to evaluate its usability for texts from other sources. Results: The F1-score of the system improved from 0.67 to 0.82 as the amount of training data increased from 1200 to 12,004 records. The F1-score reached 0.78 with 3600 records and was largely similar thereafter. As performance improved, errors from incorrect extractions decreased significantly, which resulted in an increase in precision. For case reports, the F1-score also increased from 0.34 to 0.41 as the training dataset expanded from 1200 to 12,004 records. Performance was lower for extracting symptoms from case report texts compared with pharmaceutical care records, suggesting that this system is more specialized for analyzing subjective data from pharmaceutical care records. Conclusions: We successfully developed a high-performance system specialized in analyzing subjective data from pharmaceutical care records by training a large dataset, with near-complete saturation of system performance with about 3600 training records. This system will be useful for monitoring symptoms, offering benefits for both clinical practice and research. ",
issn="2291-9694",
doi="10.2196/68863",
url="https://medinform.jmir.org/2025/1/e68863",
url="https://doi.org/10.2196/68863",
url="http://www.ncbi.nlm.nih.gov/pubmed/40053805"
}