Merge branch 'release3' of https://github.com/bio2rdf/bio2rdf-scripts into release3

Author: micheldumontier

clinicaltrials/clinicaltrials.php

@@ -174,8 +174,12 @@ function parse_dir(){
 			if($i % 10000 == 0) {parent::clear();}
 			$trial_id = basename($file,'.xml');
 			if(parent::getParameterValue('id_list') == '' || in_array($trial_id, $ids)) {
-				echo "Processing $trial_id".PHP_EOL;
-				$this->process_file($file);
+				if(filesize($file)!=0) {
+					echo "Processing $trial_id".PHP_EOL;
+					$this->process_file($file);
+				} else{
+					echo "Processing $trial_id -> Empty!".PHP_EOL;
+				}
 			}
 		}
 		echo "Finished.".PHP_EOL;

linkedSPLs/LinkedSPLs-activeMoiety/README

@@ -7,17 +7,11 @@ This folder will hold a graph that maps the active moiety components of an SPL t
 Inputs are original mappings listed as belows:
 
 (1) PT to UNII
-
 (2) UNII to RxNORM
-
 (3) PT to Drugbank
-
 (4) PT to ChEBI
-
 (5) RxNORM to OMOP
-
 (6) RxNORM to DrOn
-
 (7) RxNORM to NDFRT (EPC)
 
 Outputs is a RDF/XML graph that represents all active moiety with linked resouces.
@@ -27,28 +21,10 @@ Outputs is a RDF/XML graph that represents all active moiety with linked resouce
 Procedures to get active moieties RDF graph
 ################################################################################
 
-<STEP 1>: prepare mappings in folder mappings/
-
-----------------------------------------------
-mappings of PT to UNII (FDA)
-----------------------------------------------
+<1>: query to get omopid mapping and put in mapping folder
 
-FROM "../../LinkedSPLs-update/data/FDA/FDAPreferredSubstanceToUNII.txt"
+mappings/active-ingredient-omopid-rxcui.dsv
 
-----------------------------------------------
-mappings of UNII to RXCUI (UMLS)
-----------------------------------------------
-
-FROM "../../LinkedSPLs-update/data/UMLS/UNIIs-Rxcuis-from-UMLS.txt"
-
-----------------------------------------------
-mappings/dron-to-chebi-and-rxnorm.txt
-----------------------------------------------
-FROM "../../LinkedSPLs-update/mappings/DrOn-to-RxNorm/dron-to-chebi-and-rxnorm.txt"
-
-----------------------------------------------
-mappings/active-ingredient-omopid-rxcui-09042015.dsv
-----------------------------------------------
 
 query OMOP CDM V5 (GeriOMOP) by SQL query below:
 SELECT cpt.CONCEPT_ID as omopid, cpt.CONCEPT_CODE as rxcui FROM
@@ -60,21 +36,6 @@ cpt.CONCEPT_CLASS = 'Ingredient';
 
 right click result table and export to csv ('|' delimited) 
 
-----------------------------------------------
-mappings/UNIIToChEBI-<DATE>.txt
-----------------------------------------------
-FROM "../../LinkedSPLs-update/mappings/PT-UNII-ChEBI-mapping/UNIIToChEBI-<DATE>.txt"
-
-----------------------------------------------
-mappings/pt_drugbank-<DATE>.txt
-----------------------------------------------
-FROM "../../LinkedSPLs-update/mappings/ChEBI-DrugBank-bio2rdf-mapping/fda-substance-preferred-name-to-drugbank-<DATE>.txt"
-
-----------------------------------------------
-mappings/EPC_extraction_most_recent_<DATE>.txt"
-----------------------------------------------
-FROM "../../LinkedSPLs-update/mappings/pharmacologic_class_indexing/EPC_extraction_most_recent_<DATE>.txt"
-
 
 <STEP 2>: run python script to merge those mappings together
 
@@ -92,7 +53,7 @@ outputs: activeMoietySub-in-rdf.xml
 
 
 ################################################################################
-PRE#REQUISITES:
+PRE-REQUISITES:
 ################################################################################
 
 python libraries:

linkedSPLs/LinkedSPLs-activeMoiety/mergeToActiveMoiety.py

@@ -20,12 +20,12 @@
 PT_UNII = "../LinkedSPLs-update/data/FDA/FDAPreferredSubstanceToUNII.txt"
 UNII_RXCUI = "../LinkedSPLs-update/data/UMLS/UNIIs-Rxcuis-from-UMLS.txt"
 
-PT_CHEBI = "mappings/UNIIToChEBI-06102015.txt"
-PT_DRUGBANK = "mappings/fda-substance-preferred-name-to-drugbank-06102015.txt"
+PT_CHEBI = "../LinkedSPLs-update/mappings/PT-UNII-ChEBI-mapping/UNIIToChEBI.txt"
+PT_DRUGBANK = "../LinkedSPLs-update/mappings/ChEBI-DrugBank-bio2rdf-mapping/fda-substance-preferred-name-to-drugbank.txt"
 
-UNII_NUI_PREFERRED_NAME_ROLE = "mappings/EPC_extraction_most_recent_06102015.txt"
-DRON_CHEBI_RXCUI = "mappings/cleaned-dron-chebi-rxcui-ingredient-06222015.txt"
-OMOP_RXCUI = "mappings/active-ingredient-omopid-rxcui-09042015.dsv"
+UNII_NUI_PREFERRED_NAME_ROLE = "../LinkedSPLs-update/mappings/pharmacologic_class_indexing/EPC_extraction_most_recent.txt"
+DRON_CHEBI_RXCUI = "../LinkedSPLs-update/mappings/DrOn-to-RxNorm/cleaned-dron-chebi-rxcui-ingredient.txt"
+OMOP_RXCUI = "mappings/active-ingredient-omopid-rxcui.dsv"
 
 ## Get UNII - PT - RXCUI
 unii_pt_cols = ['unii','pt']

linkedSPLs/LinkedSPLs-activeMoiety/mergedActiveMoiety.csv

@@ -37,14 +37,20 @@ converted_rxnorm_mappings_<DATE>.txt
 
 Dailymed product label indexing mapping from "linkedSPLs/LinkedSPLs-update/mappings/RxNORM-mapping/converted_rxnorm_mappings_<DATE>"
 
-$ cat converted_rxnorm_mappings_06262015.txt | cut -f1,2 -d\| | sort | uniq > setid_rxcui.txt
+$ cd mappings
+
+$ cp ../../LinkedSPLs-update/mappings/RxNORM-mapping/converted_rxnorm_mappings_10132015.txt converted_rxnorm_mappings.txt
+
+$ cat converted_rxnorm_mappings.txt | cut -f1,2 -d\| | sort | uniq > setid_rxcui.txt
 
 ----------------------------------------------------------
 mappings of FDA preferred term and setId
 ----------------------------------------------------------
 
 $ mysql -u <username> -p
 
+$ use linkedSPLs;
+
 SELECT setId, fullName FROM linkedSPLs.structuredProductLabelMetadata INTO OUTFILE '/tmp/setid_fullname.txt' FIELDS TERMINATED BY ',' ENCLOSED BY '"' LINES TERMINATED BY '\n';
 
 $ cp /tmp/setid_fullname.txt ../linkedSPLs/LinkedSPLs-clinicalDrug/mappings/

linkedSPLs/LinkedSPLs-clinicalDrug/README

@@ -17,11 +17,12 @@
 RXNORM_BASE_URI = "http://purl.bioontology.org/ontology/RXNORM/"
 
 ## Define data inputs
-DRON_RXCUI = "mappings/cleaned-dron-to-rxcui-drug-06222015.txt"
-OMOP_RXCUI = "mappings/imeds_drugids_to_rxcuis.csv"
+DRON_RXCUI = "../LinkedSPLs-update/mappings/DrOn-to-RxNorm/cleaned-dron-to-rxcui-drug.txt"
 SETID_RXCUI = "mappings/setid_rxcui.txt"
 FULLNAME_SETID = "mappings/setid_fullname.txt"
-#OMOP_RXCUI = "mappings/clinical-drug-omopid-rxcui-09042015.dsv"
+
+#OMOP_RXCUI = "mappings/imeds_drugids_to_rxcuis.csv"
+OMOP_RXCUI = "mappings/clinical-drug-omopid-rxcui-09042015.dsv"
 
 
 
@@ -71,6 +72,6 @@
 output_DF = fullname_rxcui_setid_DF.merge(dron_omop_rxcui_DF, on=['rxcui'], how='left')
 
 print output_DF.info()
-print output_DF
+#print output_DF
 
 output_DF.to_csv('mergedClinicalDrug.tsv', sep='\t', index=False)

linkedSPLs/LinkedSPLs-clinicalDrug/mergeToClinicalDrug.py

@@ -1,6 +1,11 @@
 CODE TO GENERATE THE SQL AND LINKED-DATA VERSION OF LINKEDSPLS
 Authors: Richard Boyce, Greg Gardner, Yifan Ning
-Date: 09/25/2015
+
+Last updated Date: 09/25/2015
+
+problematic labels:
+spls/e8ae0b66-25de-41b1-8013-7f414bbb7568.xml
+
 
 ################################################################################
 OVERVIEW
@@ -37,34 +42,49 @@ Update all linkedSPLs mappings by command below
 $ ant linkedSPLs-update
 
 Update piece by piece (recommended)
+create schema linkedSPLs
+modify db connection information in db-connection.properties (if creating linkedSPLs)
+$ ant unzip-spls (if creating linkedSPLs)
+$ ant createTableSchema (if creating linkedSPLs)
+$ ant load-loincSection (if creating linkedSPLs)
+$ ant loadDailymedSPLsToRDB (ensure tables are truncated)
 
 $ ant load-FDAPreferredSubstanceToUNII          
 $ ant load-FDA_UNII_to_ChEBI         
+$ ant load-FDAPreferredSubstanceToRxNORM
+$ ant load-FDA_SUBSTANCE_TO_DRUGBANK_BIO2RDF   
+$ ant load-SPLSetIDToRxNORM    
+$ ant load-RXNORM_NDFRT_INGRED_Table         
+$ ant load-FDA_EPC_Table                           
 $ ant load-ChEBI_DRUGBANK_BIO2RDF               
-$ ant loadDailymedSPLsToRDB                     
-$ ant load-DrOn_RXCUI_DRUG                      
-$ ant load-DrOn_RXCUI_INGREDIENT                
-$ ant load-FDA_EPC_Table                        
+                
 $ ant load-FDAPharmgxTable                      
-$ ant load-FDAPharmgxTableToOntologyMap         
-$ ant load-FDAPreferredSubstanceToRxNORM        
-$ ant load-FDAPreferredSubstanceToRxNORM-restAPI
-$ ant load-FDA_SUBSTANCE_TO_DRUGBANK_BIO2RDF    
-$ ant load-loincSection                         
+$ ant load-FDAPharmgxTableToOntologyMap
+         
+$ ant load-DrOn_RXCUI_DRUG                      
+$ ant load-DrOn_RXCUI_INGREDIENT        
+
+-- deprecated --
+$ ant load-FDAPreferredSubstanceToRxNORM-restAPI                         
 $ ant load-OMOPId-RXCUIs-from-OHDSI             
-$ ant load-RXNORM_NDFRT_INGRED_Table            
-$ ant load-SPLSetIDToRxNORM   
 
 
 
 ################################################################################
 PRE-REQUISITES (Download all source data before run any ant command)
 ################################################################################
 
+Install python libs if deploy linkedSPLs on new environment
+
+apt-get install libxml2-dev libxslt1-dev python-dev
+apt-get install python-lxml
+apt-get install python-feedparser
+
+
 Download and organize all source data files in data folder
 
 --------------------------------------------------------
-product label sections and mappings from Dailymed:
+Dailymed (product label sections, indexing and mappings) 
 --------------------------------------------------------
 
 (1) dailymed-labels:
@@ -91,7 +111,7 @@ unzip XMLs to folder "pharmacologic_class_indexing_spl_files"
 $ cd pharmacologic_class_indexing_spl_files; unzip \*.zip; rm \*.zip
 
 --------------------------------------------------------
-FDA Preferred terms, UNIIs from FDA:
+FDA (Preferred terms, UNIIs):
 --------------------------------------------------------
 
 Download from http://fdasis.nlm.nih.gov/srs/jsp/srs/uniiListDownload.jsp
@@ -109,47 +129,51 @@ Keep in directory LinkedSPLs-update/data/FDA
 Edit LinkedSPLs-update/data-source.properties to reset FDA_UNII_NAMES and FDA_UNII_RECORDS 
 
 --------------------------------------------------------
-Drug bank Id from Drugbank: 
+Drugbank (Drug bank Id) : 
 --------------------------------------------------------
 
 Download from http://www.drugbank.ca/downloads
 
 download drugbank.xml as drugbankX.X and keep in directory LinkedSPLs-update/data/DrugBank
 
 --------------------------------------------------------
-UMLS:
+UMLS (rxcui):
 --------------------------------------------------------
 
 Download RXNORM mappings (full rxnorm) from UMLS at "http://www.nlm.nih.gov/research/umls/rxnorm/docs/rxnormfiles.html"
 
 keep in directory: "LinkedSPLs-update/data/UMLS"
 
---------------------------------------------------------
-umlsdbmi:
---------------------------------------------------------
-Repository: https://bitbucket.org/uamsdbmi/dron
+------------------------------------------------------------------------
+PharmagxTable && FDAPharmgxTableToOntologyMap
+-----------------------------------------------------------------------
 
-DrOn to RxNorm and ChEBI
+Get CSVs from solomon
+biomarker-to-ontology-mapping.csv
+genetic-biomarker-table-raw-import.csv
 
-Dron mapping file (dron-rxnorm.owl for drug and dron-ingredient.owl for ingredients) download from:
-https://bitbucket.org/uamsdbmi/dron/src
+put at "LinkedSPLs-update/mappings/FDA-pharmacogenetic-info-mapping/"
 
-Install readland:
-$ sudo apt-get install redland-utils
+Edit data-source.properties
+ex.
+BIOMARKER = mappings/FDA-pharmacogenetic-info-mapping/biomarker-to-ontology-mapping-07242015.xlsx
+GENETIC= genetic-biomarker-table-update-07242015.csv
 
-Load dron mapping for ingredient into a triple store by:
-rdfproc -n dron parse dron-ingredient.owl
+--------------------------------------------------------
+umlsdbmi (DronId for drug and ingredient):
+--------------------------------------------------------
+Repository: https://bitbucket.org/uamsdbmi/dron
+Code: https://bitbucket.org/uamsdbmi/dron/src
 
-rdfproc -c dron-ingredient query sparql - '       
-PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> PREFIX owl: <http://www.w3.org/2002/07/owl#> PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> PREFIX dron: <http://purl.obolibrary.org/obo/dron#>  SELECT * WHERE {   ?dron dron:DRON_00010000 ?rxcui. }' > dron-chebi-rxcui-ingredient.txt
+Download repository from "https://bitbucket.org/uamsdbmi/dron/downloads"
 
+unzip repository and then:
+copy dron-rxnorm.owl at data/umasdbmi/dron-rxnorm.owl
+copy dron-ingredient.owl at data/umasdbmi/dron-ingredient.owl
 
-Load dron mapping for drug product into triple store by:
-rdfproc -n dron-drug parse dron-rxnorm.owl
+dron-rxnorm.owl for drug product
+dron-ingredient.owl for active ingredients
 
-Mappings pulled using:
-rdfproc -c dron-drug query sparql - '       
-PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> PREFIX owl: <http://www.w3.org/2002/07/owl#> PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> PREFIX dron: <http://purl.obolibrary.org/obo/dron#>  SELECT * WHERE {   ?dron dron:DRON_00010000 ?rxcui. }' > dron-rxcui-drug.txt
 
 ------------------------------------------------------------------------
 OMOP concept Id from OHDSI or query OMOP CDM V5 (GeriOMOP) by SQL query