diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index f087a35..9cc19d6 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -4,7 +4,7 @@ on: push: branches: [main, master] pull_request: - + branches: [main, master, development] name: R-CMD-check.yaml permissions: read-all diff --git a/NAMESPACE b/NAMESPACE index 74d8a16..df8a6fb 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -23,11 +23,13 @@ importFrom(DatabaseConnector,createConnectionDetails) importFrom(DatabaseConnector,dbExecute) importFrom(DatabaseConnector,dbWriteTable) importFrom(DatabaseConnector,disconnect) +importFrom(DatabaseConnector,executeSql) importFrom(DatabaseConnector,renderTranslateExecuteSql) importFrom(R6,R6Class) importFrom(SqlRender,readSql) importFrom(SqlRender,render) importFrom(SqlRender,translate) +importFrom(checkmate,assertCharacter) importFrom(checkmate,assertClass) importFrom(checkmate,assertDirectory) importFrom(checkmate,assertDirectoryExists) diff --git a/R/ConceptRelationshipToAncestorTables.R b/R/ConceptRelationshipToAncestorTables.R index 0ea5d61..7c4635a 100644 --- a/R/ConceptRelationshipToAncestorTables.R +++ b/R/ConceptRelationshipToAncestorTables.R @@ -1,27 +1,38 @@ #' Convert CONCEPT_RELATIONSHIP to CONCEPT_ANCESTOR #' #' @description -#' Creates and populates the CONCEPT_ANCESTOR table +#' Creates and populates the CONCEPT_ANCESTOR table #' from CONCEPT_RELATIONSHIP table data. #' #' @param connection A DatabaseConnector connection object #' @param vocabularyDatabaseSchema Schema containing the vocabulary and STCM tables -#' @param sourceToConceptMapTable Name of the SOURCE_TO_CONCEPT_MAP_EXTENDED table +#' @param vocabularyList Vector of vocabulary_ids to include (default: c("ICD10")) #' #' @return #' Invisible TRUE if successful #' +#' @importFrom checkmate assertClass assertString assertCharacter +#' @importFrom SqlRender readSql render translate +#' @importFrom DatabaseConnector executeSql +#' #' @export -conceptRelationshipToAncestorTables <- function(connection, - vocabularyDatabaseSchema, - sourceToConceptMapTable - ) { - +conceptRelationshipToAncestorTables <- function( + connection, + vocabularyDatabaseSchema, + vocabularyList) { # Input validation connection |> checkmate::assertClass("DBIConnection") vocabularyDatabaseSchema |> checkmate::assertString() - sourceToConceptMapTable |> checkmate::assertString() + vocabularyList |> checkmate::assertCharacter() + + vocabulariesInTheDatabase <- DatabaseConnector::dbGetQuery( + connection, + "SELECT vocabulary_id FROM vocabulary" + ) |> dplyr::pull(vocabulary_id) + + vocabularyList |> checkmate::assertSubset(vocabulariesInTheDatabase) + # Read and render SQL # Get SQL script path sqlPath <- system.file( "sql", "sql_server", @@ -33,13 +44,13 @@ conceptRelationshipToAncestorTables <- function(connection, sql <- SqlRender::readSql(sqlPath) sql <- SqlRender::render(sql, vocabularyDatabaseSchema = vocabularyDatabaseSchema, - sourceToConceptMapTable = sourceToConceptMapTable) + vocabularyList = paste(paste0("'", vocabularyList, "'"), collapse = ",")) # Translate to target dialect sql <- SqlRender::translate(sql, targetDialect = connection@dbms) - + # Execute SQL DatabaseConnector::executeSql(connection, sql) - + return(invisible(TRUE)) } diff --git a/R/RunAll.R b/R/RunAll.R index f4d8efa..99e5a51 100644 --- a/R/RunAll.R +++ b/R/RunAll.R @@ -1,7 +1,7 @@ #' Run All Validation and Upload Steps #' #' @description -#' Runs the complete workflow of validating vocabulary files and uploading them to CDM tables. +#' Runs the complete workflow of validating vocabulary files and uploading them to CDM tables. #' It performs the following steps: #' 1. Validate the vocabulary folder #' 2. If sourceToConceptMapTable is not NULL, create the SourceToConceptMap table @@ -111,6 +111,52 @@ runAll <- function( return(validationLogTibble) } + # create the ancestor tables + message("Creating the ancestor tables") + errorMessage <- "" + tryCatch( + { + # get all the vocabulary ids used as ancestors + vocabularyIds <- c() + vocabulariesTibble <- readr::read_csv(pathToVocabularyFolder |> file.path("vocabularies.csv"), show_col_types = FALSE) + for (i in 1:nrow(vocabulariesTibble)) { + vocabularyIds <- c(vocabularyIds, vocabulariesTibble$source_vocabulary_id[i]) + pathToUsagiFile <- file.path(pathToVocabularyFolder, vocabulariesTibble$path_to_usagi_file[i]) + usagiTibble <- readUsagiFile(pathToUsagiFile) + if ("ADD_INFO:sourceParentVocabulary" %in% names(usagiTibble)) { + vocabularyId <- usagiTibble |> + dplyr::distinct(`ADD_INFO:sourceParentVocabulary`) |> + dplyr::pull(`ADD_INFO:sourceParentVocabulary`) |> + stringr::str_split("\\|") |> + unlist() + vocabularyIds <- c(vocabularyIds, vocabularyId) + } + } + vocabularyList <- vocabularyIds |> + unique() |> + na.omit() |> + setdiff("") + + conceptRelationshipToAncestorTables( + connection = connection, + vocabularyDatabaseSchema = vocabularyDatabaseSchema, + vocabularyList = vocabularyList + ) + }, + error = function(e) { + errorMessage <<- e$message + } + ) + + if (errorMessage != "") { + validationLogTibble <- dplyr::bind_rows(validationLogTibble, dplyr::tibble( + context = "conceptRelationshipToAncestorTables", + type = "ERROR", + step = "creating the ancestor tables", + message = errorMessage + )) + } + # close the connection DatabaseConnector::disconnect(connection) diff --git a/R/validateVocabularyFolder.R b/R/validateVocabularyFolder.R index 86fee69..63e8d47 100644 --- a/R/validateVocabularyFolder.R +++ b/R/validateVocabularyFolder.R @@ -124,5 +124,5 @@ validateVocabularyFolder <- function(pathToVocabularyFolder, connection, vocabul validationsLogTibble <- validationsLogTibble |> dplyr::bind_rows(validationLogTibble) } - return(validationsLogTibble) + return(validationsLogTibble |> dplyr::select(context, type, step, message)) } diff --git a/inst/sql/sql_server/CONCEPT_RELATIONSHIPToANCESTOR.sql b/inst/sql/sql_server/CONCEPT_RELATIONSHIPToANCESTOR.sql new file mode 100644 index 0000000..739e6af --- /dev/null +++ b/inst/sql/sql_server/CONCEPT_RELATIONSHIPToANCESTOR.sql @@ -0,0 +1,57 @@ +-- DESCRIPTION: +-- Adds non-standard FinOMOP and specified vocabularies concept ancestry to the concetp_ancestor table +-- +-- PARAMETERS: +-- +-- - vocabularyDatabaseSchema: schema containing the vocabulary and STCM tables +-- - vocabularyList: comma-separated list of vocabulary_ids to include + +-- 1- Create a temporary table with concept relationships that have +-- 1-1 Any concept > 2 billion or from specified vocabularies +-- 1-2 Only get that have relationship `Subsumes` present + +DROP TABLE IF EXISTS #relationships; + +SELECT cr.concept_id_1, cr.concept_id_2 +INTO #relationships +FROM @vocabularyDatabaseSchema.concept c +INNER JOIN @vocabularyDatabaseSchema.concept_relationship cr + ON cr.concept_id_1 = c.concept_id +WHERE c.vocabulary_id IN (@vocabularyList) + AND cr.relationship_id = 'Subsumes' +ORDER BY cr.concept_id_1, cr.concept_id_2; + +-- 2- Insert the created non-standard concept ancestries to concept_ancestor table in omop vocab +INSERT INTO @vocabularyDatabaseSchema.concept_ancestor +( + ancestor_concept_id, + descendant_concept_id, + min_levels_of_separation, + max_levels_of_separation +) +-- 3- Create a min and max distance as 1 for every concept relation +-- 3-1 Recursively check descendant_concept_id until there are no descendants +WITH RECURSIVE ancestor_cte AS ( +-- Base case: direct relationships + SELECT concept_id_1 AS ancestor_concept_id, + concept_id_2 AS descendant_concept_id, + 1 AS min_levels_of_separation, + 1 AS max_levels_of_separation + FROM #relationships + + UNION ALL + +-- Recursive case: find descendant relationships + SELECT r.concept_id_1 AS ancestor_concept_id, + c.descendant_concept_id AS descendant_concept_id, + c.min_levels_of_separation + 1 AS min_levels_of_separation, + c.max_levels_of_separation + 1 AS max_levels_of_separation + FROM #relationships r + JOIN ancestor_cte c + ON r.concept_id_2 = c.ancestor_concept_id +) +SELECT * +FROM ancestor_cte; + +-- 4- Remove the temporary table +DROP TABLE #relationships; diff --git a/inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb b/inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb index f86557b..b69b5f7 100644 Binary files a/inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb and b/inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb differ diff --git a/inst/testdata/createTestData.R b/inst/testdata/createTestData.R index a885aa9..2d3aed2 100644 --- a/inst/testdata/createTestData.R +++ b/inst/testdata/createTestData.R @@ -60,7 +60,7 @@ dplyr::filter( # ) conceptRelationship_new <- conceptRelationship |> - dplyr::filter(relationship_id %in% c("Maps to", "Concept replaced by", "Concept same_as to", "Concept poss_eq to")) |> + dplyr::filter(relationship_id %in% c("Maps to", "Concept replaced by", "Concept same_as to", "Concept poss_eq to", "Subsumes")) |> dplyr::semi_join( concept_codes |> dplyr::filter(is.na(standard_concept)), by = c("concept_id_1" = "concept_id") diff --git a/man/conceptRelationshipToAncestorTables.Rd b/man/conceptRelationshipToAncestorTables.Rd index b23c79c..8919a9d 100644 --- a/man/conceptRelationshipToAncestorTables.Rd +++ b/man/conceptRelationshipToAncestorTables.Rd @@ -7,7 +7,7 @@ conceptRelationshipToAncestorTables( connection, vocabularyDatabaseSchema, - sourceToConceptMapTable + vocabularyList ) } \arguments{ @@ -15,7 +15,7 @@ conceptRelationshipToAncestorTables( \item{vocabularyDatabaseSchema}{Schema containing the vocabulary and STCM tables} -\item{sourceToConceptMapTable}{Name of the SOURCE_TO_CONCEPT_MAP_EXTENDED table} +\item{vocabularyList}{Vector of vocabulary_ids to include (default: c("ICD10"))} } \value{ Invisible TRUE if successful diff --git a/tests/testthat/test-conceptRelationshipToAncestorTables.R b/tests/testthat/test-conceptRelationshipToAncestorTables.R new file mode 100644 index 0000000..fb1f047 --- /dev/null +++ b/tests/testthat/test-conceptRelationshipToAncestorTables.R @@ -0,0 +1,45 @@ +test_that("conceptRelationshipToAncestorTables creates CONCEPT_ANCESTOR table from CONCEPT_RELATIONSHIP table from ICD10", { + # Setup test paths and parameters + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + vocabularyDatabaseSchema <- "main" + + # Create connection to test database + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + # conceptRelationshipToAncestorTables + conceptRelationshipToAncestorTables( + connection = connection, + vocabularyDatabaseSchema = vocabularyDatabaseSchema, + vocabularyList = c("ICD10") + ) + + # Check if CONCEPT_ANCESTOR table exists + ancestor <- dplyr::tbl(connection, "CONCEPT_ANCESTOR") |> + dplyr::collect() + + ancestor |> nrow() |> expect_gt(0) + + # check icd10 for asthma 45596282 + asthmaChildren <- ancestor |> + dplyr::filter(ancestor_concept_id == 45596282) |> + dplyr::collect() + + asthmaChildren |> nrow() |> expect_equal(4) + asthmaChildren |> dplyr::pull(descendant_concept_id) |> expect_setequal(c(45548118, 45557624, 45557625, 45562456)) + + # check descendant Chronic lower respiratory diseases 40475107 + crdDescendant <- ancestor |> + dplyr::filter(ancestor_concept_id == 40475107) |> + dplyr::collect() + + crdDescendant |> nrow() |> expect_equal(24) + crdDescendant |> dplyr::count(min_levels_of_separation, max_levels_of_separation) |> + dplyr::pull(n) |> + expect_equal(c(8,16)) + +}) diff --git a/tests/testthat/test-runAll.R b/tests/testthat/test-runAll.R index 32c453b..74da666 100644 --- a/tests/testthat/test-runAll.R +++ b/tests/testthat/test-runAll.R @@ -1,38 +1,36 @@ -# test_that("runAll works", { -# # Set up test data -# pathToVocabularyFolder <- system.file("testdata/VOCABULARIES", package = "ROMOPMappingTools") -# pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() -# withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) -# vocabularyDatabaseSchema <- "main" -# validationResultsFolder <- file.path(tempdir(), "validationResults") -# dir.create(validationResultsFolder, showWarnings = FALSE, recursive = TRUE) -# withr::defer(unlink(validationResultsFolder, recursive = TRUE)) +test_that("runAll works", { + # Set up test data + pathToVocabularyFolder <- system.file("testdata/VOCABULARIES", package = "ROMOPMappingTools") + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + vocabularyDatabaseSchema <- "main" + validationResultsFolder <- file.path(tempdir(), "validationResults") + dir.create(validationResultsFolder, showWarnings = FALSE, recursive = TRUE) + withr::defer(unlink(validationResultsFolder, recursive = TRUE)) -# # Create connection details for test database -# connectionDetails <- DatabaseConnector::createConnectionDetails( -# dbms = "duckdb", -# server = pathToOMOPVocabularyDuckDBfile -# ) + # Create connection details for test database + connectionDetails <- DatabaseConnector::createConnectionDetails( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) -# # Run function -# validationLogTibble <- runAll( -# pathToVocabularyFolder = pathToVocabularyFolder, -# connectionDetails = connectionDetails, -# vocabularyDatabaseSchema = vocabularyDatabaseSchema, -# validationResultsFolder = validationResultsFolder -# ) + # Run function + validationLogTibble <- runAll( + pathToVocabularyFolder = pathToVocabularyFolder, + connectionDetails = connectionDetails, + vocabularyDatabaseSchema = vocabularyDatabaseSchema, + validationResultsFolder = validationResultsFolder + ) -# # Check results -# validationLogTibble |> dplyr::filter(type == "ERROR") -# validationLogTibble |> expect_s3_class("tbl_df") -# validationLogTibble |> dplyr::filter(type == "ERROR") |> nrow() |> expect_equal(0) + # Check results + validationLogTibble |> dplyr::filter(type == "ERROR") |> nrow() |> expect_equal(0) -# # check the validation results folder -# expect_true(file.exists(file.path(validationResultsFolder, "validationLogTibble.csv"))) -# expect_true(file.exists(file.path(validationResultsFolder, "resultsDQD.json"))) + # check the validation results folder + expect_true(file.exists(file.path(validationResultsFolder, "validationLogTibble.csv"))) + expect_true(file.exists(file.path(validationResultsFolder, "resultsDQD.json"))) -# resultsDQD <- jsonlite::read_json(file.path(validationResultsFolder, "resultsDQD.json"), simplifyVector = TRUE) -# resultsDQD$CheckResults |> dplyr::as_tibble() |> dplyr::filter(failed==1) -# }) + resultsDQD <- jsonlite::read_json(file.path(validationResultsFolder, "resultsDQD.json"), simplifyVector = TRUE) + resultsDQD$CheckResults |> dplyr::as_tibble() |> dplyr::filter(failed==1) +}) diff --git a/vignettes/filesFormat.Rmd b/vignettes/filesFormat.Rmd index 681f67c..e76c219 100644 --- a/vignettes/filesFormat.Rmd +++ b/vignettes/filesFormat.Rmd @@ -57,6 +57,7 @@ The ADD_INFO:validationMessages column is added by validateUsagiFile and contain | ADD_INFO:sourceParents | character | Parent codes in source vocabulary | not empty, if more that one parent, separated by, combination of sourceParents and sourceParentVocabulary must exits in the CDM or in the usagi file | | ADD_INFO:sourceParentVocabulary | character | Vocabularies of parent codes | if empty, the vocabulary is itself, if more that one parent, separated by | | ADD_INFO:validationMessages | character | Column added by validateUsagiFile | Optional | +| ADD_INFO:autoUpdatingInfo | character | Column added by updateUsagiFile | Optional | # vocabularies.csv file format @@ -72,3 +73,27 @@ It is a csv file with the following columns: | path_to_usagi_file | character | The path to the vocabulary's Usagi file | not empty, file must exist | | path_to_news_file | character | The path to the vocabulary's news file | not empty, file must exist | | ignore | boolean | Indicates if the vocabulary should be ignored in processing | not empty | + + +# SOURCE_TO_CONCEPT_MAP_EXTENDED table format + +The SOURCE_TO_CONCEPT_MAP_EXTENDED is an extension of the SOURCE_TO_CONCEPT_MAP table, see [CDM](https://ohdsi.github.io/CommonDataModel/cdm54.html#source_to_concept_map). +It is used to store the source to concept map extended information. + +The SOURCE_TO_CONCEPT_MAP_EXTENDED table has the following columns: + +| Name | Type | Description | Rules | +| ------------------------ | --------- | ---------------------------------------------------------- | -------- | +| source_code | character | Source code for the concept | not empty | +| source_vocabulary_id | character | Source vocabulary the concept was mapped from | not empty, must exist in VOCABULARY table | +| source_code_description | character | Description of source code | not empty | +| target_concept_id | integer | Concept ID of the target concept | not empty, must exist in CONCEPT table | +| target_vocabulary_id | character | Target vocabulary the concept was mapped to | not empty, must exist in VOCABULARY table | +| valid_start_date | date | Date when mapping became valid | not empty, must be before valid_end_date | +| valid_end_date | date | Date when mapping became invalid | not empty, must be after valid_start_date | +| invalid_reason | character | Reason why mapping was invalidated | empty if valid_end_date is 2099-12-31 | +| source_concept_id | integer | Source concept ID | not empty | +| source_concept_class | character | Concept class in source vocabulary | not empty, less than 20 characters | +| source_domain | character | Domain in source vocabulary | not empty, must exist in DOMAIN table | +| source_parents_concept_ids | character | Parent concept IDs in source vocabulary | optional, comma-separated list | + diff --git a/vignettes/indivudualFunctionsExample.Rmd b/vignettes/indivudualFunctionsExample.Rmd deleted file mode 100644 index 629a5d0..0000000 --- a/vignettes/indivudualFunctionsExample.Rmd +++ /dev/null @@ -1,379 +0,0 @@ ---- -title: "Step by step example using individual functions" -output: rmarkdown::html_vignette -vignette: > - %\VignetteIndexEntry{run} - %\VignetteEncoding{UTF-8} - %\VignetteEngine{knitr::rmarkdown} -editor_options: - chunk_output_type: console ---- - - - -```{r, include = FALSE} -knitr::opts_chunk$set( - collapse = TRUE, - comment = "#>" -) -``` - -```{r setup} -library(ROMOPMappingTools) -``` - - -# Intro - -This is a step by step example of how to use the individual functions of the ROMOPMappingTools package. -Example files are included in the package. In the `inst/testdata` folder you can find the files used in this example. - -# Populating the STCM table - -The SourceToConceptMap (STCM) table is a table in the OMOP vocabulary database that contains the mappings between source codes and concept ids. -The STCM table is used to store the mappings for the vocabularies. - -We can populate the STCM table from a Usagi file or from a folder with Usagi files. - -## Singel Usagi file to STCM table - -### Reading the Usagi file - -For reading the Usagi file, we can use the `readUsagiFile` function, which returns a tibble with the correct columns formated. -It can read a standard Usagi file or an extended Usagi file. -In example we will read a extended Usagi file, from the test data. -This file contains the mappings for the ICD10fi vocabulary. - -```{r} -pathToUsagiFile <- system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools") - -usagiTibble <- readUsagiFile(pathToUsagiFile) - -usagiTibble |> dplyr::glimpse() -``` - -### Validating the Usagi file - -For validating the Usagi file, we can use the `validateUsagiFile` function. -This function needs a connection to the OMOP vocabulary database and schema to where the vocabulary tables are stored in order to make some of the validations. -The function also needs a path to a file where, if errors are found, a new Usagi file with the errors will be created. -The function also need the number used to offset the source concept ids in the Usagi file. -The function returns a tibble with a summary of the validations conducted. - -For this example, we will use the test database in DuckDB format. -This test database contains only the ICD10 vocabulary with all the keys in other tables (see `inst/testdata/createTestData.R` for more details). - -```{r} -pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() - -connectionDetails <- DatabaseConnector::createConnectionDetails( - dbms = "duckdb", - server = pathToOMOPVocabularyDuckDBfile -) - -connection <- DatabaseConnector::connect(connectionDetails) -vocabularyDatabaseSchema <- "main" - -pathToValidatedUsagiFile <- tempfile(fileext = "usagi_validated.csv") -``` - -```{r} -validationsSummary <- validateUsagiFile( - pathToUsagiFile, - connection, - vocabularyDatabaseSchema, - pathToValidatedUsagiFile = pathToValidatedUsagiFile, - sourceConceptIdOffset = 2000500000 -) -``` - -```{r} -knitr::kable(validationsSummary) -``` - -In this case the Usagi file is valid and no errors are found. Hence, the new validated Usagi file remains unchanged. - -However, we can see what happens with a usagi file with errors. - -```{r} -pathToUsagiFileWithErrors <- system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi_with_errors.usagi.csv", package = "ROMOPMappingTools") - -usagiTibbleWithErrors <- readUsagiFile(pathToUsagiFileWithErrors) -``` - -```{r} -validationsSummaryWithErrors <- validateUsagiFile( - pathToUsagiFileWithErrors, - connection, - vocabularyDatabaseSchema, - pathToValidatedUsagiFile = pathToValidatedUsagiFile, - sourceConceptIdOffset = 2000500000 -) -``` - -```{r} -knitr::kable(validationsSummaryWithErrors) -``` - -In this case, if we open the new validate Usagi with the Usagi software these mapping with errors will appear as FLAGED -Additionaly, the `ADD_INFO:validationMessages` column will indicate the exact error or errors found. - -![Usagi with errors](./images/Usagi_with_errors.png) - -### Updating the Usagi file - -If the vocabulary has been updated since the Usagi file was created, it may happen that some of the mappings are outdated. -This will be detected by the `validateUsagiFile` and show as a "ConceptIds outdated" error. - -```{r} -pathToOutdatedUsagiFile <- system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi_outdated.usagi.csv", package = "ROMOPMappingTools") - -validationsSummaryWithErrors <- validateUsagiFile( - pathToOutdatedUsagiFile, - connection, - vocabularyDatabaseSchema, - pathToValidatedUsagiFile = pathToValidatedUsagiFile, - sourceConceptIdOffset = 2000500000 -) - -knitr::kable(validationsSummaryWithErrors) -``` - -In this case, we can update the Usagi file using the `updateUsagiFile` function - -```{r} -pathToUpdatedUsagiFile <- tempfile(fileext = "usagi_updated.csv") - -updateSummary <- updateUsagiFile( - pathToOutdatedUsagiFile, - connection, - vocabularyDatabaseSchema, - pathToUpdatedUsagiFile, - skipValidation = TRUE - ) - -knitr::kable(updateSummary) -``` - -This fuction updates changes in `domain_id`, `concept_name` and if the mapped `concept_id` point to a non-standard concept it will try to find a new mapping for it -(This is done by looking at the relationship table for relationships of the old concept_id by "Maps to", "Concept replaced by", "Concept same_as to" and "Concept poss_eq to" in that order). - -Some times the new concept_id is not found, in this case the function will return a warning and not update the concept_id. - -The new updates Usagi file can be validated again with the `validateUsagiFile` function to check if there are any errors. - -```{r} -validationsSummaryWithErrors <- validateUsagiFile( - pathToUpdatedUsagiFile, - connection, - vocabularyDatabaseSchema, - pathToValidatedUsagiFile = pathToValidatedUsagiFile, - sourceConceptIdOffset = 2000500000 -) - -knitr::kable(validationsSummaryWithErrors) -``` - -Unfortunatelly, sometimes the updateUsagiFile is introducing new errors, in this case updates in the vocabulary have introduced invalid domain combinations. -Moreover, some of the mappings could not be updated because the new concept_id was not found. -This need to be fixed by the user by reviewing the Usagi file. - -### Uploading the Usagi file into the SourceToConceptMap table in a database - -We will continue with the validated Usagi file. - -For uploading the validated Usagi file into the SourceToConceptMap table in a database, we can use the `appendUsagiFileToSTCMtable` function. -This function needs a connection to the database and schema where the vocabulary is stored and the name of the SourceToConceptMap table. -Most CDM database have a standard SourceToConceptMap table named `source_to_concept_map`. -This table can be used if we wish to process the Usagi file as a standard Usagi file. - -However, if we wish to process the Usagi file as an extended Usagi file, we need to create an extended SourceToConceptMap table. -This can be done using the `createSourceToConceptMapExtended` function. - -```{r} -sourceToConceptMapTable <- "source_to_concept_map_extended" -createSourceToConceptMapExtended(connection, vocabularyDatabaseSchema, sourceToConceptMapTable) -``` - -```{r} -appendUsagiFileToSTCMtable( - vocabularyId = "ICD10", - pathToUsagiFile, - connection, - vocabularyDatabaseSchema, - sourceToConceptMapTable -) -``` - -We can see that the Usagi file has been uploaded into the SourceToConceptMap table. - -```{r} -dplyr::tbl(connection, "source_to_concept_map_extended") |> - dplyr::filter(source_vocabulary_id == "ICD10") |> - dplyr::collect() -``` - -Notice that only the approved mappings (mappingStatus == "APPROVED") are uploaded to the STCM table. - -Close the connection to the database. -```{r} -DatabaseConnector::disconnect(connection) -``` - -## Multiple Usagi files to STCM table - -More often than not, we will have multiple Usagi files to upload to the STCM table. -In this case, we can use the `validateVocabularyFolder` and `vocabularyFolderToSTCMAndVocabularyTables` functions. -The `validateVocabularyFolder` function will validate all the Usagi files in the vocabulary folder and return a tibble with the validations. -The `vocabularyFolderToSTCMAndVocabularyTables` function will upload all the Usagi files to the STCM table and add the vocabularies to the VOCABULARY table. - -In the `inst/testdata` folder we have a folder with multiple Usagi files one for ICD10fi and one for UNITSfi. `vocabularies.csv` file is a file that contains the vocabulary information for the vocabulary folder. - -``` -inst/testdata/VOCABULARIES/ -├── vocabularies.csv -├── ICD10fi/ -│ ├── ICD10fi.usagi.csv -│ └── NEWS.md -└── UNITfi/ - ├── UNITfi.usagi.csv - └── NEWS.md -``` - -Create a new database : - -```{r} -pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() - -connectionDetails <- DatabaseConnector::createConnectionDetails( - dbms = "duckdb", - server = pathToOMOPVocabularyDuckDBfile -) - -connection <- DatabaseConnector::connect(connectionDetails) -vocabularyDatabaseSchema <- "main" -``` - -### Validate all the Usagi files in the vocabulary folder - -This function nees as an input a path to a folder with a `vocabularies.csv` file, a connection to the database, the schema with the vocabulary tables and a folder where to store the validated Usagi files if they are not valid. -The format of the `vocabularies.csv` is described in the [Tables description and rules](usagiFileFormat.html) vignette. - - -```{r} -pathToVocabularyFolder <- system.file("testdata/VOCABULARIES", package = "ROMOPMappingTools") -pathToValidatedUsagiFolder <- tempdir() - -validationsLogTibble <- validateVocabularyFolder(pathToVocabularyFolder, connection, vocabularyDatabaseSchema, pathToValidatedUsagiFolder) -``` - -The function will return a tibble with the validations on the `vocabularies.csv` file and all the Usagi files. - -```{r} -knitr::kable(validationsLogTibble) -``` - - -### Upload all the Usagi files to the STCM table - -Similary we can use the `vocabularyFolderToSTCMAndVocabularyTables` function to upload all the Usagi files to the STCM table. -This function will also upload the vocabularies.csv file to the VOCABULARY table. - -```{r} -createSourceToConceptMapExtended(connection, vocabularyDatabaseSchema, sourceToConceptMapTable) -``` - -```{r} -vocabularyFolderToSTCMVocabularyConcepClassTables(pathToVocabularyFolder, connection, vocabularyDatabaseSchema, sourceToConceptMapTable) -``` - -```{r} -dplyr::tbl(connection, "VOCABULARY") |> - dplyr::collect() -``` - -```{r} -dplyr::tbl(connection, "source_to_concept_map_extended") |> - dplyr::collect() -``` - - -# Copying the STCM table to the CDM tables - -The STCM table can be copied to the CDM tables using the `STCMToCDM` function. -This function needs a connection to the database and schema where the vocabulary tables are stored, the schema with the vocabulary tables and the name of the SourceToConceptMap table. - -This function solely call to the SQL code in the `inst/sql/STCMToCDM.sql` file. -This SQL code can be translated to any other database supported by DatabaseConnector, and be applied directly. - - -```{r} -STCMToCDMTables(connection, vocabularyDatabaseSchema, sourceToConceptMapTable) -``` - -This populates the CONCEPT table: -```{r} -dplyr::tbl(connection, "CONCEPT") |> - dplyr::filter(vocabulary_id == "ICD10fi") |> - dplyr::collect() -``` - -The CONCEPT_RELATIONSHIP table with - -- the 'Maps to' relationships: -```{r} -dplyr::tbl(connection, "CONCEPT_RELATIONSHIP") |> - dplyr::filter(relationship_id == "Maps to") |> - dplyr::filter(concept_id_1 > 2000500101) |> - dplyr::collect() -``` - -- the 'Maps from' relationships: -```{r} -dplyr::tbl(connection, "CONCEPT_RELATIONSHIP") |> - dplyr::filter(relationship_id == "Mapped from") |> - dplyr::filter(concept_id_2 > 2000500101) |> - dplyr::collect() -``` - -And if the columns `sourceConceptCode` and `sourceConceptVocabularyId` are present in the STCM table, they will be used to populate the CONCEPT_RELATIONSHIP table with - -- the 'Is a' relationships: -```{r} -dplyr::tbl(connection, "CONCEPT_RELATIONSHIP") |> - dplyr::filter(relationship_id == "Is a") |> - dplyr::filter(concept_id_1 > 2000500101) |> - dplyr::collect() -``` - -- the `Subsumes` relationships: -```{r} -dplyr::tbl(connection, "CONCEPT_RELATIONSHIP") |> - dplyr::filter(relationship_id == "Subsumes") |> - dplyr::filter(concept_id_1 > 2000500101) |> - dplyr::collect() -``` - -# Populating the CONCEPT_ANCESTOR table - - - -Close the connection to the database. -```{r} -DatabaseConnector::disconnect(connection) -``` - - -# Validate the CDM tables with DataQualityDashboard - -```{r} -# Create connectionDetails from the existing connection -validationResultsFolder <- tempdir() - -validationLogTibble <- validateCDMtablesWithDQD(connectionDetails, vocabularyDatabaseSchema, validationResultsFolder) -``` - -```{r} -knitr::kable(validationLogTibble) -``` - diff --git a/vignettes/runAll.Rmd b/vignettes/runAll.Rmd deleted file mode 100644 index 46fa735..0000000 --- a/vignettes/runAll.Rmd +++ /dev/null @@ -1,181 +0,0 @@ ---- -title: "Running the complete mapping process" -output: rmarkdown::html_vignette -vignette: > - %\VignetteIndexEntry{Running the complete mapping process} - %\VignetteEncoding{UTF-8} - %\VignetteEngine{knitr::rmarkdown} -editor_options: - chunk_output_type: console ---- - -```{r, include = FALSE} -knitr::opts_chunk$set( - collapse = TRUE, - comment = "#>" -) -``` - -```{r setup} -library(ROMOPMappingTools) -``` - -# Introduction - -The `runAll` function provides a streamlined way to execute the complete mapping process: -1. Validates all Usagi files in a vocabulary folder -2. Creates and populates the SOURCE_TO_CONCEPT_MAP_EXTENDED table -3. Creates CDM vocabulary tables from the mappings -4. Validates the resulting CDM tables using DataQualityDashboard - -# Required Setup - -## File Structure - -Your vocabulary folder should contain: -- A `vocabularies.csv` file describing the files (see the [File Format vignette](fileFormat.html)) -- One subfolder for each vocabulary containing: - - A Usagi mapping file (.csv) - - A NEWS.md file with the release notes for the vocabulary - -Example structure: -``` -vocabularies/ -├── vocabularies.csv -├── ICD10fi/ -│ ├── NEWS.md -│ └── ICD10fi.usagi.csv -└── UNITfi/ - ├── NEWS.md - └── UNITfi.usagi.csv -``` - -For testing, you can use the folder structure that comes in this package. -```{r} -pathToVocabularyFolder <- system.file("testdata/VOCABULARIES", package = "ROMOPMappingTools") -``` - -## Database Connection - -You'll need connection details to a database where: -- You have permissions to create and modify tables -- The OMOP Vocabulary tables are already loaded - -For testing, you can use the DuckDB database that is included in the package. -```{r} -pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() - -connectionDetails <- DatabaseConnector::createConnectionDetails( - dbms = "duckdb", - server = pathToOMOPVocabularyDuckDBfile -) -``` - -# Running the Process - -The next code block demonstrates using the `runAll()` function with parameters we set up above. -The `pathToVocabularyFolder` references our test data folder created earlier. -The `vocabularyDatabaseSchema` "main" matches our DuckDB test database. -We leave `sourceToConceptMapTable` as `NULL` with will create the SourceToConceptMap table with the name "source_to_concept_map_extended". -The `validationResultsFolder` is set to a temporary directory to store the validation and DQD results. -These parameters match the connection and file structure we established in the setup section. - - -```{r eval=FALSE} -# Define paths and parameters -pathToVocabularyFolder <- pathToVocabularyFolder -vocabularyDatabaseSchema <- "main" -sourceToConceptMapTable <- NULL # if NULL, the SourceToConceptMap table will be created with the name "source_to_concept_map_extended" -validationResultsFolder <- file.path(tempdir(), "validationResults") -dir.create(validationResultsFolder, showWarnings = FALSE, recursive = TRUE) - -# Run the complete process -validationLogTibble <- runAll( - connectionDetails = connectionDetails, - pathToVocabularyFolder = pathToVocabularyFolder, - vocabularyDatabaseSchema = vocabularyDatabaseSchema, - sourceToConceptMapTable = sourceToConceptMapTable, - validationResultsFolder = validationResultsFolder -) -``` - -# Understanding the Results - -The function returns a tibble containing a log of the different steps in the process. - -```{r} -#knitr::kable(validationLogTibble) -``` - -If any there is any error in the column `type` the process may have not finished correctly. - -In that case you can find more details in the `message` column and the files ourputed in the `validationResultsFolder`. - -## Output Files - -The process creates several output files: - -1. Validation Log: - - `validationLogTibble.csv` - Contains the validation log - -2. Validated vocabularies.csv (if errors were found): - - `vocabularies.csv` - Contains the validated vocabularies - -3. Validated Usagi Files (if errors were found): - - `vocabularies/` - Directory containing validated vocabulary files - - Each vocabulary gets its own subdirectory with validated Usagi files - - Only created if validation errors were found - -4. Data Quality Dashboard Results: - - `resultsDQD.json` - Contains the full DQD results in JSON format - -example of the folder structure: - -``` -validationResultsFolder/ -├── validationLogTibble.csv -├── vocabularies.csv (if errors were found) -├── vocabularies/ -│ ├── ICD10fi/ -│ │ └── ICD10fi.usagi.csv (if errors were found) -│ └── UNITfi/ -│ └── UNITfi.usagi.csv (if errors were found) -├── resultsDQD.json -``` - -You can view the DQD results in an interactive dashboard: -```{r eval=FALSE} -resultsDQDjson <- file.path(validationResultsFolder, "resultsDQD.json") -DataQualityDashboard::viewDqDashboard(resultsDQDjson) -``` - -## Database Changes - -The process also creates or modifies several tables in your database: - -1. SOURCE_TO_CONCEPT_MAP_EXTENDED: - - Contains all validated mappings from Usagi files - - Includes extended information like source domains and parent concepts - -2. CDM Vocabulary Tables: - - CONCEPT - New source concepts from your mappings - - CONCEPT_RELATIONSHIP - Mapping relationships between source and standard concepts - - CONCEPT_CLASS - New concept classes from your source vocabularies - - VOCABULARY - New vocabulary entries for your source vocabularies - -These tables are created in the schema specified by vocabularyDatabaseSchema. - -## Troubleshooting - -If you encounter errors (type = "ERROR" in the validation log): - -1. Check the validation messages in the log tibble -2. Examine the validated Usagi files in the validationResultsFolder -3. Review the DQD results for specific quality issues -4. Fix any identified issues in your source Usagi files -5. Run the process again - -For detailed information about specific errors and how to fix them, refer to the [File Format vignette](fileFormat.html). - - - diff --git a/vignettes/validateUsagiWithErrors.png b/vignettes/validateUsagiWithErrors.png deleted file mode 100644 index 008fd90..0000000 Binary files a/vignettes/validateUsagiWithErrors.png and /dev/null differ diff --git a/vignettes/workAsGithubRepo.Rmd b/vignettes/workAsGithubRepo.Rmd new file mode 100644 index 0000000..b4739f0 --- /dev/null +++ b/vignettes/workAsGithubRepo.Rmd @@ -0,0 +1,29 @@ +--- +title: "Running the complete mapping process" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Running the complete mapping process} + %\VignetteEncoding{UTF-8} + %\VignetteEngine{knitr::rmarkdown} +editor_options: + chunk_output_type: console +--- + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) +``` + +```{r setup} +library(ROMOPMappingTools) +``` + +# Introduction + +The `runAll` function provides a streamlined way to execute the complete mapping process: +1. Validates all Usagi files in a vocabulary folder +2. Creates and populates the SOURCE_TO_CONCEPT_MAP_EXTENDED table +3. Creates CDM vocabulary tables from the mappings +4. Validates the resulting CDM tables using DataQualityDashboard diff --git a/vignettes/workWithMultipleMappingFiles.Rmd b/vignettes/workWithMultipleMappingFiles.Rmd new file mode 100644 index 0000000..48c0df2 --- /dev/null +++ b/vignettes/workWithMultipleMappingFiles.Rmd @@ -0,0 +1,241 @@ +--- +title: "Step by step example using individual functions" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{run} + %\VignetteEncoding{UTF-8} + %\VignetteEngine{knitr::rmarkdown} +editor_options: + chunk_output_type: console +--- + + + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) +``` + +```{r setup} +library(ROMOPMappingTools) +``` + + +# Intro + +This vignette shows how to use some of the functions of the ROMOPMappingTools package to work with a multiple Usagi mapping files. +Validating their format, updating them after a vocabulary update, uploading them to the STCM table, and transforming the STCM table to the CDM tables. + +For automating all the process in a github repository, please refer to the [Work as a github repository](workAsGithubRepo.html) vignette. +For working with a single Usagi file, please refer to the [Work with individual mapping files](workWithOneMappingFile.html) vignette. + +Example files are included in the package. In the `inst/testdata` folder you can find the files used in this example. + +## Setting up the forders structure + +To work with multiple Usagi files, we need to have a folder with the Usagi files and a `vocabularies.csv` file. + +The `vocabularies.csv` file is a file that contains the vocabulary information and the path to the Usagi and NEWS.md files asociated with each vocabulary, +see the [Tables description and rules](usagiFileFormat.html) vignette for more details. + +We recommend to have a folder structure like the one use in this example: + +``` +inst/testdata/VOCABULARIES/ +├── vocabularies.csv +├── ICD10fi/ +│ ├── ICD10fi.usagi.csv +│ └── NEWS.md +└── UNITfi/ + ├── UNITfi.usagi.csv + └── NEWS.md +``` + +We use a root folder containing the `vocabularies.csv` and subdirectories named after the vocabulary id. +Each subdirectory contains the Usagi and NEWS.md files. In this case we have two vocabularies: ICD10fi and UNITfi. + + +## Target database + +For this example we will use the test database in DuckDB format included in the package. +Create a new database by making a copy of the `inst/testdata/OMOPVocabulary.duckdb` file. + +```{r} +pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + +connectionDetails <- DatabaseConnector::createConnectionDetails( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile +) + +connection <- DatabaseConnector::connect(connectionDetails) +vocabularyDatabaseSchema <- "main" +``` + +## Validate all the Usagi files in the vocabulary folder + +This function nees as an input a path to a folder with a `vocabularies.csv` file, a connection to the database, +the schema with the vocabulary tables and a folder where to store the validated Usagi files if they are not valid. +This function will validate the format of the `vocabularies.csv` file and all the Usagi files. +Details on the Usagi file validations are described in the [Work with individual mapping files](workWithOneMappingFile.html) vignette. + +```{r} +pathToVocabularyFolder <- system.file("testdata/VOCABULARIES", package = "ROMOPMappingTools") +pathToValidatedUsagiFolder <- tempdir() + +validationsLogTibble <- validateVocabularyFolder( + pathToVocabularyFolder, + connection, + vocabularyDatabaseSchema, + pathToValidatedUsagiFolder +) +``` + +The function will return a tibble with the validations on the `vocabularies.csv` file and all the Usagi files. + +```{r} +knitr::kable(validationsLogTibble) +``` + + +## Upload all the Usagi files to the STCM table + +If all the validations pass, we can use the `vocabularyFolderToSTCMAndVocabularyTables` function to upload all the Usagi files to the STCM table. +If some of the validations fail, we recoment fix the errors following the [Work with individual mapping files](workWithOneMappingFile.html) vignette. + +If we are using the Usagi-extended format, we also need to create the source_to_concept_map_extended table, see the [Files format](filesFormat.html) vignette for more details. + +```{r} +sourceToConceptMapTable <- "source_to_concept_map_extended" +createSourceToConceptMapExtended(connection, vocabularyDatabaseSchema, sourceToConceptMapTable) +``` + +`vocabularyFolderToSTCMVocabularyConcepClassTables` needs the path to the vocabulary folder, a connection to the database, the schema with the vocabulary tables and the name of the SourceToConceptMap table. + + +```{r} +vocabularyFolderToSTCMVocabularyConcepClassTables( + pathToVocabularyFolder, + connection, + vocabularyDatabaseSchema, + sourceToConceptMapTable +) +``` + +This function will populate the VOCABULARY table with the `vocabularies.csv` file and the source_to_concept_map_extended table with the Usagi-extended files. + +```{r} +dplyr::tbl(connection, "VOCABULARY") |> + dplyr::collect() +``` + +We can see on the botton that the ICD10fi and UNITfi vocabularies have been added to the VOCABULARY table. + +```{r} +dplyr::tbl(connection, "source_to_concept_map_extended") |> + dplyr::collect() +``` + +We can see the source_to_concept_map_extended table has been populated with the Usagi-extended files. + +## Copying the STCM table to the CDM tables + +The STCM table can be copied to the CDM tables using the `STCMToCDM` function. +This function needs a connection to the database and schema where the vocabulary tables are stored, the schema with the vocabulary tables and the name of the SourceToConceptMap table. + +This function solely call to the SQL code in the `inst/sql/STCMToCDM.sql` file, hence it be also used outside the package. +This SQL code can be translated to any other database supported by DatabaseConnector, and be applied directly. + +```{r} +STCMToCDMTables(connection, vocabularyDatabaseSchema, sourceToConceptMapTable) +``` + +This populates the CONCEPT table: +```{r} +dplyr::tbl(connection, "CONCEPT") |> + dplyr::filter(vocabulary_id == "ICD10fi") |> + dplyr::collect() +``` + +The CONCEPT_RELATIONSHIP table with + +- the 'Maps to' relationships: +```{r} +dplyr::tbl(connection, "CONCEPT_RELATIONSHIP") |> + dplyr::filter(relationship_id == "Maps to") |> + dplyr::filter(concept_id_1 > 2000500101) |> + dplyr::collect() +``` + +- the 'Maps from' relationships: +```{r} +dplyr::tbl(connection, "CONCEPT_RELATIONSHIP") |> + dplyr::filter(relationship_id == "Mapped from") |> + dplyr::filter(concept_id_2 > 2000500101) |> + dplyr::collect() +``` + +And if the columns `sourceConceptCode` and `sourceConceptVocabularyId` are present in the STCM table, they will be used to populate the CONCEPT_RELATIONSHIP table with + +- the 'Is a' relationships: +```{r} +dplyr::tbl(connection, "CONCEPT_RELATIONSHIP") |> + dplyr::filter(relationship_id == "Is a") |> + dplyr::filter(concept_id_1 > 2000500101) |> + dplyr::collect() +``` + +- the `Subsumes` relationships: +```{r} +dplyr::tbl(connection, "CONCEPT_RELATIONSHIP") |> + dplyr::filter(relationship_id == "Subsumes") |> + dplyr::filter(concept_id_1 > 2000500101) |> + dplyr::collect() +``` + +## Populating the CONCEPT_ANCESTOR table + +Since we have added the "Is a" and "Subsumes" relationships to the CONCEPT_RELATIONSHIP table, we can use this information to populate the CONCEPT_ANCESTOR table. +This is done with the `conceptRelationshipToAncestorTables` function. + +This can be applied to any non-standard vocabulary, not only the ones included in the `vocabularies.csv` file. + +```{r} +conceptRelationshipToAncestorTables( + connection, + vocabularyDatabaseSchema, + vocabularyList = c("ICD10", "ICD10fi", "UNITfi") +) +``` + +Close the connection to the database. +```{r} +DatabaseConnector::disconnect(connection) +``` + + +## Validate the new CDM tables with DataQualityDashboard + +Since we have introduced changes in the OMOP CDM table, we can use the DataQualityDashboard package to validate that we havent introduced errors. +We include the function `validateCDMtablesWithDQD` in the package to facilitate this task. + +```{r} +# Create connectionDetails from the existing connection +validationResultsFolder <- tempdir() + +validationLogTibble <- validateCDMtablesWithDQD(connectionDetails, vocabularyDatabaseSchema, validationResultsFolder) +``` + +```{r} +knitr::kable(validationLogTibble) +``` + +We can see that there are no errors. +```{r} +validationLogTibble |> + dplyr::filter(type == "ERROR") |> + knitr::kable() +``` + diff --git a/vignettes/workWithOneMappingFile.Rmd b/vignettes/workWithOneMappingFile.Rmd new file mode 100644 index 0000000..2db6681 --- /dev/null +++ b/vignettes/workWithOneMappingFile.Rmd @@ -0,0 +1,178 @@ +--- +title: "Working with individual mapping files" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{run} + %\VignetteEncoding{UTF-8} + %\VignetteEngine{knitr::rmarkdown} +editor_options: + chunk_output_type: console +--- + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) +``` + +```{r setup} +library(ROMOPMappingTools) +``` + +# Intro + +This vignette shows how to use some of the functions of the ROMOPMappingTools package to work with a single Usagi mapping file. +Reading and Usagi file, validating its format, or updating it after a vocabulary update. +For trasforming a single usagi file into C&CR tables of the OMOP vocabulary, we recommend follow the same steps as in the [Work with multiple mapping files](workWithMultipleMappingFiles.html) vignette. +This is because the process need some other information that is not included in the Usagi file, but n the 'vocabularies.csv' file. +For automating all the process in a github repository, please refer to the [Work as a github repository](workAsGithubRepo.html) vignette. + +Example files are included in the package. In the `inst/testdata` folder you can find the files used in this example. + +## Reading a Usagi file + +For reading the Usagi file, we can use the `readUsagiFile` function, which returns a tibble with the correct columns formated. +It can read a standard Usagi file or an extended Usagi file, see the [Usagi file format](usagiFileFormat.html) vignette for more details. +In this example we will read a extended Usagi file, from the test data. +This file contains the mappings for the ICD10fi vocabulary. + +```{r} +pathToUsagiFile <- system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools") + +usagiTibble <- readUsagiFile(pathToUsagiFile) + +usagiTibble |> dplyr::glimpse() +``` + +## Validating a Usagi file + +To validate if all the information in the Usagi file is correct, we can use the `validateUsagiFile` function. +This function takes an Usagi or Usagi-extended file and and performs a series of validations, see the function help for more details `?validateUsagiFile`. +The function also needs a connection to the OMOP vocabulary database and schema to where the vocabulary tables are stored in order to make some of the validations. +The function also need the number used to offset the source concept ids in the Usagi file. +The function returns a tibble with a summary of the validations conducted and if error are found, a new Usagi file with the errors will be created in the specified path. + +For this example, we will use the test database in DuckDB format included in the package. +This test database contains only the ICD10 vocabulary with all the keys in other tables (see `inst/testdata/createTestData.R` for more details). + +```{r} +pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + +connectionDetails <- DatabaseConnector::createConnectionDetails( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile +) + +connection <- DatabaseConnector::connect(connectionDetails) +vocabularyDatabaseSchema <- "main" + +pathToValidatedUsagiFile <- tempfile(fileext = "usagi_validated.csv") +``` + +```{r} +validationsSummary <- validateUsagiFile( + pathToUsagiFile, + connection, + vocabularyDatabaseSchema, + pathToValidatedUsagiFile = pathToValidatedUsagiFile, + sourceConceptIdOffset = 2000500000 +) +``` + +```{r} +knitr::kable(validationsSummary) +``` + +In this case the Usagi file is valid and no errors are found. Hence, the new validated Usagi file remains unchanged. + +However, we can see what happens with a usagi file with errors. +In this case we use an other Usagi file with all type of errors, wich is included in the package for unit testing purposes. + +```{r} +pathToUsagiFileWithErrors <- system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi_with_errors.usagi.csv", package = "ROMOPMappingTools") + +usagiTibbleWithErrors <- readUsagiFile(pathToUsagiFileWithErrors) +``` + +```{r} +validationsSummaryWithErrors <- validateUsagiFile( + pathToUsagiFileWithErrors, + connection, + vocabularyDatabaseSchema, + pathToValidatedUsagiFile = pathToValidatedUsagiFile, + sourceConceptIdOffset = 2000500000 +) +``` + +```{r} +knitr::kable(validationsSummaryWithErrors) +``` + +In this case, if we open the new validate Usagi with the Usagi software these mapping with errors will appear as FLAGGED. +Additionally, the `ADD_INFO:validationMessages` column will indicate the exact error or errors found. + +![Usagi with errors](./img/Usagi_with_errors.png) + +## Updating a Usagi file + +If the vocabulary has been updated since the Usagi file was created, it may happen that some of the mappings are outdated. +This will be detected by the `validateUsagiFile` and show as a "ConceptIds outdated" error. + +In this case we will use an other Usagi file with outdated concept ids, which is included in the package for unit testing purposes. + +```{r} +pathToOutdatedUsagiFile <- system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi_outdated.usagi.csv", package = "ROMOPMappingTools") + +validationsSummaryWithErrors <- validateUsagiFile( + pathToOutdatedUsagiFile, + connection, + vocabularyDatabaseSchema, + pathToValidatedUsagiFile = pathToValidatedUsagiFile, + sourceConceptIdOffset = 2000500000 +) + +knitr::kable(validationsSummaryWithErrors) +``` + +If outdated error are detected, we can attempt to update the Usagi file automatically using the `updateUsagiFile` function. +This function takes an Usagi or Usagi-extended file, a connection to the database, the schema with the vocabulary tables and a path to a file where to store the updated Usagi file. + +```{r} +pathToUpdatedUsagiFile <- tempfile(fileext = "usagi_updated.csv") + +updateSummary <- updateUsagiFile( + pathToOutdatedUsagiFile, + connection, + vocabularyDatabaseSchema, + pathToUpdatedUsagiFile, + skipValidation = TRUE + ) + +knitr::kable(updateSummary) +``` + +This fuction updates changes in `domain_id`, `concept_name` and if the mapped `concept_id` point to a non-standard concept it will try to find a new mapping for it +(This is done by looking at the relationship table for relationships of the old concept_id by "Maps to", "Concept replaced by", "Concept same_as to" and "Concept poss_eq to" in that order). +A new column `ADD_INFO:autoUpdatingInfo` is added to the updated Usagi file to show the specific changes made to the file. + +Some times, like in this case, a new concept_id can not be found, this is shown as a warning. + +The new updates Usagi file can be validated again with the `validateUsagiFile` function to check if there are any errors. + +```{r} +validationsSummaryWithErrors <- validateUsagiFile( + pathToUpdatedUsagiFile, + connection, + vocabularyDatabaseSchema, + pathToValidatedUsagiFile = pathToValidatedUsagiFile, + sourceConceptIdOffset = 2000500000 +) + +knitr::kable(validationsSummaryWithErrors) +``` + +Unfortunatelly, sometimes the updateUsagiFile is introducing new errors, in this case updates in the vocabulary have introduced invalid domain combinations. +Moreover, some of the mappings could not be updated because the new concept_id was not found. +This need to be fixed by the user by reviewing the Usagi file. + \ No newline at end of file