Update_LOINC_Part1.gradle

/*
*************************************
* LOINC Vocabulary update. Part 1   *
* Version 0.0.7                     *
*************************************
*/

group 'com.sci.vocabulary'
version '1.0-SNAPSHOT'

apply plugin: 'java'

sourceCompatibility = 1.8

/*repositories {
    mavenLocal()
    mavenCentral()
}*/

/*dependencies {
    compile 'org.apache.poi:poi:3.9'
    compile 'org.apache.poi:poi-ooxml:3.9'
}*/

buildscript {
    repositories {
        mavenLocal()
        mavenCentral()
    }
    dependencies {
        classpath 'org.apache.poi:poi:3.9'
        classpath 'org.apache.poi:poi-ooxml:3.9'
        classpath 'javax.mail:javax.mail-api:1.5.5'
        classpath 'org.apache.pdfbox:pdfbox:2.0.5'
        classpath group: 'commons-io', name: 'commons-io', version: '2.5'

        /*classpath 'javax.mail:mail:1.4.7'
        classpath 'org.apache.ant:ant-javamail:1.10.1'*/

        /*classpath 'ant:ant-javamail:1.6.5'
        classpath 'javax.activation:activation:1.1.1'
        classpath 'javax.mail:mail:1.4.7'*/
    }
}

repositories {
    mavenLocal()
    mavenCentral()
}

configurations {
    mail
}

dependencies {
    mail 'org.apache.ant:ant-javamail:1.8.1'
}

import org.apache.commons.io.FileUtils

/*
configurations { antMail }
dependencies {
    antMail 'ant:ant-javamail:1.6.5'
    antMail 'javax.activation:activation:1.1.1'
    antMail 'javax.mail:mail:1.4.7'
}
ant.taskdef(
        name: 'mail2',
        classname: 'org.apache.tools.ant.taskdefs.email.EmailTask',
        classpath: configurations.antMail.asPath
)
*/
/**
 * Auxiliary classes. TODO: Further should be used java-defined and compiled only
 */
public class StringBuilderWrapper implements java.lang.Appendable {
    private static final String IDENT_CHAR = " " *4
    private StringBuilder wrapped

    public StringBuilderWrapper() {
        wrapped = new StringBuilder()
    }
    public StringBuilderWrapper(String value) {
        this()
        wrapped.append(IDENT_CHAR).append(value)
    }

    public static StringBuilderWrapper getBuilder() {
        return new StringBuilderWrapper()
    }

    public static String readFromFile(File file) {
        if (file == null)
            return ''
        def builder = getBuilder()
        def br = new BufferedReader(new FileReader(file))
        def strLine
        while ((strLine = br.readLine()) != null) {
            builder.append('\n').append(strLine)
        }
        return builder.toString()
    }

    @Override
    Appendable append(CharSequence csq) throws IOException {
        wrapped.append(IDENT_CHAR).append(csq)
        return this
    }
    @Override
    Appendable append(CharSequence csq, int start, int end) throws IOException {
        wrapped.append(IDENT_CHAR).append(csq, start, end)
        return this
    }
    @Override
    Appendable append(char c) throws IOException {
        wrapped.append(IDENT_CHAR).append(c)
        return this
    }
    @Override
    String toString() {
        return wrapped.toString()
    }
}

// Load LOINC properties
Properties props = new Properties()
props.load(new FileInputStream("${project.rootDir}/Update_LOINC.properties"))
props.each {prop ->
    project.ext.set(prop.key, prop.value)
}
props.load(new FileInputStream("${project.getBuildFile().toString().replace('.gradle', '.properties')}"))
props.each {prop ->
    project.ext.set(prop.key, prop.value)
}

def tempDir = File.createTempDir()
tempDir.deleteOnExit()

/**
 * Just renew user schema and prepare it for further vocabulary update
 */
task fastRecreateSchema(description: project.properties['fastRecreate.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*       '${description}' action started...     *"
    println '*                                             *'
    println '***********************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    def sqlFile = "${project.properties['basePath']}/${project.properties['fastRecreate.sqlFileName']}"
    def logFile = "${project.properties['logPath']}/${project.properties['fastRecreate.logFileName']}"
    def streamLogFile = "${project.properties['logPath']}/${project.properties['fastRecreate.streamLogFileName']}"
    def streamErrorFile = "${project.properties['logPath']}/${project.properties['fastRecreate.streamErrFileName']}"

    def db = [un     : project.properties['db.userName'],
              pw     :  project.properties['db.password'],
              tns    : project.properties['db.tnsName'],
              sqlfile: sqlFile,
              logfile: logFile]
    def sout = new StringBuilderWrapper(), serr = new StringBuilderWrapper()
    def sqlplus = """sqlplus -L ${db.un}/${db.pw}@${db.tns} @${db.sqlfile} ${db.logfile}""".execute()
    sqlplus.consumeProcessOutput(sout, serr)
    sqlplus.waitFor()

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    // Write to console
    System.out.println(sout.toString())

    // Write to logfile
    File f = new File(streamLogFile)
    f.write(sout.toString())

    // Write to error file
    f = new File(streamErrorFile)
    f.write(serr.toString())

    /* Check for errors */
    if (project.properties['stop.on.error'].toBoolean() && (sqlplus.exitValue() != 0 || (serr != null && serr.toString().length() > 0))) {
        throw new GradleException(String.format(project.properties['error.message.with.code'], sqlplus.exitValue(), name))
    }

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

/**
 * This task will prepare user schema for vocabulary update routine.
 * It will create auxiliary table - the "ICD10CM_TABLE" in our case, for example.
 */
task createSourceTable(dependsOn: fastRecreateSchema, description: project.properties['createSourceTable.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*   '${description}' action started...   *"
    println '*                                             *'
    println '***********************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    def sqlFile = "${project.properties['basePath']}/${project.properties['createSourceTable.sqlFileName']}"
    def logFile = "${project.properties['logPath']}/${project.properties['createSourceTable.logFileName']}"
    def streamLogFile = "${project.properties['logPath']}/${project.properties['createSourceTable.streamLogFileName']}"
    def streamErrorFile = "${project.properties['logPath']}/${project.properties['createSourceTable.streamErrFileName']}"

    def db = [un     : project.properties['db.userName'],
              pw     : project.properties['db.password'],
              tns    : project.properties['db.tnsName'],
              sqlfile: sqlFile,
              logfile: logFile]
    def sout = new StringBuilderWrapper(), serr = new StringBuilderWrapper()
    def sqlplus = """sqlplus ${db.un}/${db.pw}@${db.tns} @${db.sqlfile} ${db.logfile}""".execute()
    sqlplus.consumeProcessOutput(sout, serr)
    sqlplus.waitFor()

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    // Write to console
    try {
        // Write compact SQLPlus's spool log
        System.out.println(StringBuilderWrapper.readFromFile(new File(logFile)))
    } catch (IOException ioe) {
        // Write whole SQLPlus's log
    System.out.println(sout.toString())
    }

    // Write to logfile
    File f = new File(streamLogFile)
    f.write(sout.toString())

    // Write to error file
    f = new File(streamErrorFile)
    f.write(serr.toString())

    /* Check for errors */
    if (project.properties['stop.on.error'].toBoolean() && (sqlplus.exitValue() != 0 || (serr != null && serr.toString().length() > 0))) {
        throw new GradleException(String.format(project.properties['error.message.with.code'], sqlplus.exitValue(), name))
    }

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

task mockDownload(dependsOn: createSourceTable, description: project.properties['downloadUpdatePack.description']) {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*       '${description}' action started...          *"
    println '*                                             *'
    println '***********************************************'

    System.out.println('temp: ' + tempDir)

    /* Downloading file */
    // Start action checkpoint
    def timeStart = System.currentTimeMillis()
    println new StringBuilderWrapper('downloading file...')
    println new StringBuilderWrapper("stored to: ${tempDir.getPath()}")

    def srcFile = "${projectDir}/${project.properties['basePath']}/dowbnload/'${project.properties['prepareTable.manualWorkResultPath']}"
    def dest = "${tempDir.getPath()}"
    copy {
        from srcFile
        into dest
    }

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"

}

/**
 * Task which will download vocabulary update package from source host.
 * This one - 'ftp://ftp.cdc.gov/pub/Health_Statistics/NCHS/Publications/ICD10CM' is our case.
 */
task downloadUpdatePack(/*dependsOn: createSourceTable, */ description: project.properties['downloadUpdatePack.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*       '${description}' action started...          *"
    println '*                                             *'
    println '***********************************************'

    System.out.println('temp: ' + tempDir)

    /* Downloading file */
    // Start action checkpoint
    def timeStart = System.currentTimeMillis()
    println new StringBuilderWrapper('downloading file...')
    println new StringBuilderWrapper("stored to: ${tempDir.getPath()}")
    // Download 'full set' package
    ant.get(src: project.properties['downloadUpdatePack.fullSetUrl'],
            dest: tempDir,
            verbose: true,
            user: project.properties['downloadUpdatePack.username'],
            password: project.properties['downloadUpdatePack.password'])
    // Download 'multiaxial hierarchy' package
    ant.get(src: project.properties['downloadUpdatePack.multiaxialHierarchyUrl'],
            dest: tempDir,
            verbose: true,
            user: project.properties['downloadUpdatePack.username'],
            password: project.properties['downloadUpdatePack.password'])
    // Download 'forms' package
    ant.get(src: project.properties['downloadUpdatePack.formsUrl'],
            dest: tempDir,
            verbose: true,
            user: project.properties['downloadUpdatePack.username'],
            password: project.properties['downloadUpdatePack.password'])
    // Download 'association' package
    ant.get(src: project.properties['downloadUpdatePack.associationUrl'],
            dest: tempDir,
            verbose: true,
            user: project.properties['downloadUpdatePack.username'],
            password: project.properties['downloadUpdatePack.password'])
    // Download 'mappings' package
    ant.get(src: project.properties['downloadUpdatePack.mappingsUrl'],
            dest: tempDir,
            verbose: true,
            user: project.properties['downloadUpdatePack.username'],
            password: project.properties['downloadUpdatePack.password'])
    println new StringBuilderWrapper('download completed')

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}
/**
 * Task will rename the file 'icd10cm_order_20xx.txt' to the 'icd10cm.txt'.
 */
task renameFile(dependsOn: mockDownload, description: project.properties['renameFile.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*       '${description}' action started...       *"
    println '*                                             *'
    println '***********************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    def oldFilePath = "${tempDir.getPath()}/${project.properties['renameFile.srcFileName']}"
    def newFilePath = "${tempDir.getPath()}/${project.properties['renameFile.destFileName']}"

    File fOld = new File(oldFilePath)
    File fNew = new File(newFilePath)

    if (fNew.exists()) {
        throw new IOException(String.format(project.properties['renameFile.error.already.exists'], newFilePath))
    }

    boolean isSuccess = fOld.renameTo(fNew);
    if (!isSuccess) {
        println project.properties['renameFile.error.other']
        throw new GradleException(String.format(project.properties['error.message'], name))
    }
    println new StringBuilderWrapper('renamed successfully...')

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."
    
    println "*** '${description}' action done ***"
}

/**
 * Task performed import CSV data to database
 */
task importData(dependsOn: renameFile, description: project.properties['import.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*        '${description}' action started...           *"
    println '*                                             *'
    println '***********************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    def dataFile = "${tempDir.getPath()}/${project.properties['importData.dataFileName']}"
    def logFile = "${project.properties['logPath']}/${project.properties['importData.logFileName']}"
    def discardFile = "${project.properties['logPath']}/${project.properties['importData.discardFileName']}"
    def badFile = "${project.properties['logPath']}/${project.properties['importData.badFileName']}"
    def streamLogFile = "${project.properties['logPath']}/${project.properties['importData.streamLogFileName']}"
    def streamErrorFile = "${project.properties['logPath']}/${project.properties['importData.streamErrFileName']}"

    def bosOut = new ByteArrayOutputStream(), bosErr = new ByteArrayOutputStream()
    def sout = new StringBuilderWrapper(), serr = new StringBuilderWrapper()
    exec {
        executable "sqlldr"
        args "USERID=${project.properties['db.userName']}/${project.properties['db.password']}@${project.properties['db.tnsName']} " +
                "CONTROL=${project.properties['basePath']}/${project.properties['import.controlFileName']} DATA=$dataFile LOG=$logFile DISCARD=$discardFile BAD=$badFile"
        standardOutput = bosOut
        errorOutput = bosErr
    }
    def bisOut = new ByteArrayInputStream(bosOut.toByteArray())
    bisOut.withReader { out->
        out.eachLine {
            sout.append(it).append('\n')
        }
    }
    def bisErr = new ByteArrayInputStream(bosErr.toByteArray())
    bisErr.withReader { out ->
        out.eachLine {
            serr.append(it).append('\n')
        }
    }

    // Write to console
    /*
     *  When import prefer to write stream log to console (it's shorter)
     * */
    System.out.println(sout.toString())

    // Write to logfile
    File f = new File(streamLogFile)
    f.write(sout.toString())

    // Write to error file
    f = new File(streamErrorFile)
    f.write(serr.toString())

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

/**
 * This task will run 'Load stage' script.
 * Seems it's is specific for each vocabulary.
 */
task loadStage(dependsOn: importData, description: project.properties['loadStage.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*     '${description}' action started...   *"
    println '*                                             *'
    println '***********************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    def sqlFile = "${project.properties['basePath']}/${project.properties['loadStage.sqlFileName']}"
    def logFile = "${project.properties['logPath']}/${project.properties['loadStage.logFileName']}"
    def streamLogFile = "${project.properties['logPath']}/${project.properties['loadStage.streamLogFileName']}"
    def streamErrorFile = "${project.properties['logPath']}/${project.properties['loadStage.streamErrFileName']}"

    def db = [un     : project.properties['db.userName'],
              pw     : project.properties['db.password'],
              tns    : project.properties['db.tnsName'],
              sqlfile: sqlFile,
              logfile: logFile]
    def sout = new StringBuilderWrapper(), serr = new StringBuilderWrapper()
    def sqlplus = """sqlplus ${db.un}/${db.pw}@${db.tns} @${db.sqlfile} ${db.logfile}""".execute()
    sqlplus.consumeProcessOutput(sout, serr)
    sqlplus.waitFor()

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    // Write to console
    try {
        System.out.println(StringBuilderWrapper.readFromFile(new File(logFile)))
        System.out.println(StringBuilderWrapper.readFromFile(new File(logFileProcedureDrug)))
    } catch (IOException ioe) {
        // Write whole SQLPlus's log
    System.out.println(sout.toString())
    }

    // Write to logfile
    File f = new File(streamLogFile)
    f.write(sout.toString())

    // Write to error file
    f = new File(streamErrorFile)
    f.write(serr.toString())

    /* Check for errors */
    if (project.properties['stop.on.error'].toBoolean() && (sqlplus.exitValue() != 0 || (serr != null && serr.toString().length() > 0))) {
        throw new GradleException(String.format(project.properties['error.message.with.code'], sqlplus.exitValue(), name))
    }

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

/**
 * This task will create 'Manual Table' to pass for manual mapping processing
 * */
task createManualTable(dependsOn: loadStage, description: project.properties['createManualTable.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*   '${description}' action started...   *"
    println '*                                             *'
    println '***********************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    def sqlFile = "${project.properties['basePath']}/${project.properties['createManualTable.sqlFileName']}"
    def logFile = "${project.properties['logPath']}/${project.properties['createManualTable.logFileName']}"
    def streamLogFile = "${project.properties['logPath']}/${project.properties['createManualTable.streamLogFileName']}"
    def streamErrorFile = "${project.properties['logPath']}/${project.properties['createManualTable.streamErrFileName']}"

    def db = [un       : project.properties['db.userName'],
              pw       : project.properties['db.password'],
              tns      : project.properties['db.tnsName'],
              sqlfile  : sqlFile,
              tableName: "${project.properties['manualTableName']}",
              logfile  : logFile]
    def sout = new StringBuilderWrapper(), serr = new StringBuilderWrapper()
    def sqlplus = """sqlplus ${db.un}/${db.pw}@${db.tns} @${db.sqlfile} ${db.logfile} ${db.tableName}""".execute()

    sqlplus.consumeProcessOutput(sout, serr)
    sqlplus.waitFor()

    // Write to console
    try {
        // Write compact SQLPlus's spool log
        System.out.println(StringBuilderWrapper.readFromFile(new File(logFile)))
    } catch (IOException ioe) {
        // Write whole SQLPlus's log
    System.out.println(sout.toString())
    }

    // Write to logfile
    File f = new File(streamLogFile)
    f.write(sout.toString())

    // Write to error file
    f = new File(streamErrorFile)
    f.write(serr.toString())

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

task extractManualTable(/*dependsOn: [createManualTable, loadStage]*/dependsOn: createManualTable, description: project.properties['extractManualTable.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "* '${description}' action started... *"
    println '*                                             *'
    println '***********************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    // TODO: Need to deeply check why piped output does not works here (the > operator below)
    def sqlFile = "${project.properties['basePath']}/${project.properties['extractManualTable.sqlFileName']}"
    def streamLogFile = "${project.properties['logPath']}/${project.properties['extractManualTable.streamLogFileName']}"
    def streamErrorFile = "${project.properties['logPath']}/${project.properties['extractManualTable.streamErrFileName']}"

    def db = [un        : project.properties['db.userName'],
              pw        : project.properties['db.password'],
              tns       : project.properties['db.tnsName'],
              expPath   : "${project.properties['basePath']}",
              sqlfile   : sqlFile,
              expcsvfile: "${project.properties['manualTableFileName']}",
              tableName : "${project.properties['manualTableName']}"]
    def sout = new StringBuilderWrapper(), serr = new StringBuilderWrapper()
    def sqlplus = """sqlplus ${db.un}/${db.pw}@${db.tns} @${db.sqlfile} ${db.expPath} ${db.expcsvfile} ${db.tableName}""".execute()
    
    println new StringBuilderWrapper('extracting table...')

    sqlplus.consumeProcessOutput(sout, serr)
    sqlplus.waitFor()

    // Write to logfile
    /*
   Skip writing log file because it size could be so big.
   Only error stream should be written.

    File f = new File(streamLogFile)
    f.write(sout.toString())
   */

    // Write to error file
    File f = new File(streamErrorFile)
    f.write(serr.toString())

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

/**
 * Send e-mail notification
 */
task sendMail(dependsOn: extractManualTable, description: project.properties['sendMail.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "* '${description}' action started...  *"
    println '*                                             *'
    println '***********************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    def mailParams = [
            mailhost       : project.properties['sendMail.mailHost'],
            subject        : project.properties['sendMail.Subject'],
            messagemimetype: "text/plain",
            user           : project.properties['sendMail.user'],
            password       : project.properties['sendMail.password']
    ]

    println new StringBuilderWrapper('sending mail...')

    configurations.mail.each {
        org.apache.tools.ant.Project.class.classLoader.addURL(it.toURI().toURL())
    }

    ant.mail( mailParams ) {
        from(address: project.properties['sendMail.sentFrom'])
        to(address: project.properties['sendMail.sentTo'])
        message(String.format(project.properties['sendMail.Message'], "${project.properties['basePath']}/${project.properties['manualTableFileName']}"))
    }

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000

    println "Time elapsed: ${timeElapsed} seconds."
    println "*** '${description}' action done ***"
}

/**
 * Bootstrap action
 *
 * TODO: Needs to find solution to measure whole task chain execution.
 * TODO: This one via doFirst/doLast actions isn't working.
 */
task startUpdate(dependsOn: sendMail, description: project.properties['startUpdate.description']) {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*      '${description}' action started...       *"
    println '*                                             *'
    println '***********************************************'

    println "Updating of '${project.properties['vocabulary.display.name']}' vocabulary on the '${project.properties['db.userName']}@${project.properties['db.tnsName']}' schema...\n"

    project.ext['logPath'] = "${project.properties['basePath']}/${project.properties['baseLogPath']}"
    def folder = new File(project.properties['logPath'])
    if (!folder.exists()) {
        folder.mkdirs()
        println "Log directory is created: ${folder.getPath()}"
    } else {
        FileUtils.cleanDirectory(folder)
        println "Log directory is cleaned: ${folder.getPath()}"
    }

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    doLast {
        // Finish action checkpoint
        def timeFinish = System.currentTimeMillis()

        // Measure time execution
        def timeElapsed = (timeFinish - timeStart) / 1000
        println "\n\nTotal time elapsed: ${timeElapsed} seconds."
        println "Updating of '${project.properties['vocabulary.display.name']}' vocabulary on the '${project.properties['db.userName']}@${project.properties['db.tnsName']}' schema is completed successfully..."
        println '***********************************************'
        println '*                                             *'
        println "*      '${description}' action is done...       *"
        println '*                                             *'
        println '***********************************************\n'
    }
}

task writeToPdf(description: project.properties['writeToPdf.description']) << {
	System.out.println('stop.on.error: ' + project.properties['stop.on.error'] + ', project dir: ' + projectDir)
}