build.gradle

//#############################
//# Copyright 2016 Otmar Ertl #
//#############################


project.ext {
    paperDir = 'paper'
    pythonDir = 'python'
    cppDir = 'c++'
    dataDir = 'data'
}

//#########################
// random data generation
//#########################

task buildCardinalitiesGenerator(type: Exec) {
    workingDir project.cppDir
    inputs.files "$project.cppDir/definecardinalities.cpp"
    outputs.files "$project.cppDir/definecardinalities"
    standardOutput = new ByteArrayOutputStream()
    commandLine 'g++','-std=c++11','-Wall','definecardinalities.cpp','-o','definecardinalities'
}

task buildHyperLogLogDataGenerator(type: Exec) {
    workingDir project.cppDir
    inputs.files "$project.cppDir/hyperloglog_data_generator.cpp", "$project.cppDir/hyperloglog.hpp"
    outputs.files "$project.cppDir/hyperloglog_data_generator"
    standardOutput = new ByteArrayOutputStream()
    commandLine 'g++','-std=c++11','-Wall','hyperloglog_data_generator.cpp','-Ofast','-march=native','-fopenmp','-o','hyperloglog_data_generator'
}

task generateSimulatedCardinalities(type: Exec) {
    workingDir project.cppDir
    inputs.files "$project.cppDir/definecardinalities"
    outputs.files "$project.dataDir/cardinalities.dat"
    doFirst {
        standardOutput = new FileOutputStream("$project.dataDir/cardinalities.dat")
    }
    commandLine './definecardinalities'
    dependsOn 'buildCardinalitiesGenerator'
}

task generateRandomHyperLogLogData(type: Exec) {
    workingDir project.cppDir
    inputs.files "$project.cppDir/hyperloglog_data_generator","$project.dataDir/cardinalities.dat"
    standardOutput = new ByteArrayOutputStream()
    commandLine './hyperloglog_data_generator'
    dependsOn 'buildHyperLogLogDataGenerator','generateSimulatedCardinalities'
}

task buildJointHyperLogLogDataGenerator(type: Exec) {
    workingDir project.cppDir
    inputs.files "$project.cppDir/joint_hyperloglog_data_generator.cpp", "$project.cppDir/hyperloglog.hpp", "$project.cppDir/two_hyperloglog_statistic.hpp"
    outputs.files "$project.cppDir/joint_hyperloglog_data_generator"
    standardOutput = new ByteArrayOutputStream()
    commandLine 'g++','-std=c++11','-Wall','joint_hyperloglog_data_generator.cpp','-Ofast','-march=native','-fopenmp','-o','joint_hyperloglog_data_generator'
}

task generateJointHyperLogLogData(type: Exec) {
    workingDir project.cppDir
    inputs.files "$project.cppDir/joint_hyperloglog_data_generator","$project.dataDir/cardinalities.dat"
    standardOutput = new ByteArrayOutputStream()
    commandLine './joint_hyperloglog_data_generator'
    dependsOn 'buildJointHyperLogLogDataGenerator','generateSimulatedCardinalities'
}


//##################################
// benchmark
//##################################

task buildBenchmark(type: Exec) {
    workingDir project.cppDir
    inputs.files "$project.cppDir/benchmark.cpp", "$project.cppDir/cardinality_estimation.hpp", "$project.cppDir/hyperloglog.hpp"
    outputs.files "$project.cppDir/benchmark"
    standardOutput = new ByteArrayOutputStream()
    commandLine 'g++','-Wall','-std=c++11','benchmark.cpp','-Ofast','-march=native','-o','benchmark'
}

task runBenchmark(type: Exec) {
    workingDir project.cppDir
    inputs.files "$project.cppDir/benchmark"
    standardOutput = new ByteArrayOutputStream()
    commandLine './benchmark'
    dependsOn 'buildBenchmark','updateCpuInfo'
}

task updateCpuInfo(type: Exec) {
    workingDir project.cppDir
    commandLine 'lshw','-C','cpu'
    outputs.files "$project.dataDir/cpu_info.dat"
    doFirst {
        standardOutput = new FileOutputStream("$project.dataDir/cpu_info.dat")
    }
}

//#########################
// figures
//#########################

def figureNames = [
    'original_estimate',
    'raw_estimate',
    'small_range_estimate',
    'max_likelihood_estimate_12_20',
    'max_likelihood_estimate_12_14',
    'max_likelihood_estimate_8_24',
    'max_likelihood_estimate_16_16',
    'max_likelihood_estimate_12_52',
    'max_likelihood_estimate_12_0',
    'max_likelihood_estimate_22_10',
    'raw_corrected_estimate_12_20',
    'raw_corrected_estimate_12_14',
    'raw_corrected_estimate_8_24',
    'raw_corrected_estimate_16_16',
    'raw_corrected_estimate_12_52',
    'raw_corrected_estimate_12_0',
    'raw_corrected_estimate_22_10',
    'equal-register-probability',
    'helper',
    'max_likelihood_avg_exec_time',
    'corrected_raw_avg_exec_time',
    'stdev_comparison',
    'power-series-function-minus-1']

task buildCardinalityCalculation(type: Exec) {
    workingDir project.cppDir
    inputs.files "$project.cppDir/cardinality_calculator.cpp", "$project.cppDir/cardinality_estimation.hpp", "$project.cppDir/hyperloglog.hpp"
    outputs.files "$project.cppDir/cardinality_calculator"
    standardOutput = new ByteArrayOutputStream()
    commandLine 'g++','-std=c++11','-Wall','cardinality_calculator.cpp','-Ofast','-march=native','-fopenmp','-o','cardinality_calculator'
}

task doCardinalityCalculation(type: Exec) {
    workingDir project.cppDir
    inputs.files "$project.cppDir/cardinality_calculator", "$project.cppDir/param.txt", "$project.dataDir/cardinalities.dat"
    outputs.files "$project.dataDir/12_20_flajolet_estimates.dat",
        "$project.dataDir/12_20_flajolet_small_range_estimates.dat",
        "$project.dataDir/12_20_flajolet_mid_range_estimates.dat",
        "$project.dataDir/12_20_max_likelihood_estimates.dat",
        "$project.dataDir/12_14_max_likelihood_estimates.dat",
        "$project.dataDir/8_24_max_likelihood_estimates.dat",
        "$project.dataDir/16_16_max_likelihood_estimates.dat",
        "$project.dataDir/12_52_max_likelihood_estimates.dat",
        "$project.dataDir/12_0_max_likelihood_estimates.dat",
        "$project.dataDir/22_10_max_likelihood_estimates.dat",
        "$project.dataDir/12_20_corrected_raw_estimates.dat",
        "$project.dataDir/12_14_corrected_raw_estimates.dat",
        "$project.dataDir/8_24_corrected_raw_estimates.dat",
        "$project.dataDir/16_16_corrected_raw_estimates.dat",
        "$project.dataDir/12_52_corrected_raw_estimates.dat",
        "$project.dataDir/12_0_corrected_raw_estimates.dat",
        "$project.dataDir/22_10_corrected_raw_estimates.dat",
        "$project.dataDir/cardinalities.dat"
    standardOutput = new FileOutputStream("$project.cppDir/cardinality_calculator.out")
    commandLine './cardinality_calculator'
    dependsOn 'buildCardinalityCalculation','generateSimulatedCardinalities'
}

task helperFunctionFig(type: Exec) {
    workingDir project.pythonDir
    inputs.files "$project.pythonDir/helper-function.py"
    outputs.files "$project.paperDir/helper.svg"
    standardOutput = new ByteArrayOutputStream()
    commandLine 'python3', 'helper-function.py'
}

task powerSeriesFunctionFig(type: Exec) {
    workingDir project.pythonDir
    inputs.files "$project.pythonDir/power-series-function.py"
    outputs.files "$project.paperDir/power-series-function-minus-1.svg"
    standardOutput = new ByteArrayOutputStream()
    commandLine 'python3', 'power-series-function.py'
}

task equalRegisterProbabilityFig(type: Exec) {
    workingDir project.pythonDir
    inputs.files "$project.pythonDir/equal-register-probability.py"
    outputs.files "$project.paperDir/equal-register-probability.svg"
    standardOutput = new ByteArrayOutputStream()
    commandLine 'python3', 'equal-register-probability.py'
}


task createBenchmarkCharts(type: Exec) {
    workingDir project.pythonDir
    inputs.files "$project.pythonDir/benchmark-charts.py",
        "$project.dataDir/benchmark_results_12_20.dat",
        "$project.dataDir/benchmark_results_12_52.dat"
    outputs.files "$project.paperDir/avg_exec_time.svg",
        "$project.paperDir/max_likelihood_avg_outer_loop_iterations.svg",
        "$project.paperDir/max_likelihood_avg_inner_loop_1_iterations.svg",
        "$project.paperDir/max_likelihood_avg_inner_loop_2_iterations.svg",
        "$project.paperDir/max_likelihood_avg_number_log_evaluations.svg",
        "$project.paperDir/corrected_raw_avg_exec_time.svg"
    standardOutput = new ByteArrayOutputStream()
    commandLine 'python3', 'benchmark-charts.py'
}

task createDistributionCharts(type: Exec) {
    workingDir project.pythonDir
    inputs.files "$project.pythonDir/distribution_charts.py",
        "$project.dataDir/12_20_flajolet_estimates.dat",
        "$project.dataDir/12_20_flajolet_small_range_estimates.dat",
        "$project.dataDir/12_20_flajolet_mid_range_estimates.dat",
        "$project.dataDir/12_20_max_likelihood_estimates.dat",
        "$project.dataDir/12_14_max_likelihood_estimates.dat",
        "$project.dataDir/8_24_max_likelihood_estimates.dat",
        "$project.dataDir/16_16_max_likelihood_estimates.dat",
        "$project.dataDir/12_52_max_likelihood_estimates.dat",
        "$project.dataDir/12_0_max_likelihood_estimates.dat",
        "$project.dataDir/22_10_max_likelihood_estimates.dat",
        "$project.dataDir/12_20_corrected_raw_estimates.dat",
        "$project.dataDir/12_14_corrected_raw_estimates.dat",
        "$project.dataDir/8_24_corrected_raw_estimates.dat",
        "$project.dataDir/16_16_corrected_raw_estimates.dat",
        "$project.dataDir/12_52_corrected_raw_estimates.dat",
        "$project.dataDir/12_0_corrected_raw_estimates.dat",
        "$project.dataDir/22_10_corrected_raw_estimates.dat",
        "$project.dataDir/cardinalities.dat"
    outputs.files "$project.paperDir/original_estimate.svg",
        "$project.paperDir/raw_estimate.svg",
        "$project.paperDir/small_range_estimate.svg",
        "$project.paperDir/max_likelihood_estimate_12_20.svg",
        "$project.paperDir/max_likelihood_estimate_12_14.svg",
        "$project.paperDir/max_likelihood_estimate_8_24.svg",
        "$project.paperDir/max_likelihood_estimate_16_16.svg",
        "$project.paperDir/max_likelihood_estimate_12_52.svg",
        "$project.paperDir/max_likelihood_estimate_12_0.svg",
        "$project.paperDir/max_likelihood_estimate_22_10.svg",
        "$project.paperDir/raw_corrected_estimate_12_20.svg",
        "$project.paperDir/raw_corrected_estimate_12_14.svg",
        "$project.paperDir/raw_corrected_estimate_8_24.svg",
        "$project.paperDir/raw_corrected_estimate_16_16.svg",
        "$project.paperDir/raw_corrected_estimate_12_52.svg",
        "$project.paperDir/raw_corrected_estimate_12_0.svg",
        "$project.paperDir/raw_corrected_estimate_22_10.svg"
    standardOutput = new ByteArrayOutputStream()
    commandLine 'python3', 'distribution_charts.py'
    dependsOn 'generateSimulatedCardinalities','doCardinalityCalculation'
}

task createComparisonCharts(type: Exec) {
    workingDir project.pythonDir
    inputs.files "$project.pythonDir/comparison.py",
        "$project.dataDir/12_20_flajolet_small_range_estimates.dat",
        "$project.dataDir/12_20_flajolet_mid_range_estimates.dat",
        "$project.dataDir/12_20_max_likelihood_estimates.dat",
        "$project.dataDir/12_20_corrected_raw_estimates.dat",
        "$project.dataDir/cardinalities.dat"
    outputs.files "$project.paperDir/mean_comparison.svg",
        "$project.paperDir/stdev_comparison.svg"
    standardOutput = new ByteArrayOutputStream()
    commandLine 'python3', 'comparison.py'
    dependsOn 'generateSimulatedCardinalities','doCardinalityCalculation'
}

task svgFigures {
    dependsOn 'helperFunctionFig', 'createDistributionCharts', 'createBenchmarkCharts','powerSeriesFunctionFig','createComparisonCharts','equalRegisterProbabilityFig'
    doLast {
    }
}

task pdfFigures {

    def svgFigureFiles = figureNames.collect {"$project.paperDir/" + it + ".svg"}
    def pdfFigureFiles = figureNames.collect {"$project.paperDir/" + it + ".pdf"}

    inputs.files svgFigureFiles
    outputs.files pdfFigureFiles

    dependsOn 'svgFigures'

    doLast {
        figureNames.each { figureName ->
            exec {
                commandLine 'inkscape', "${project.paperDir}/${figureName}.svg", "--export-pdf=${project.paperDir}/${figureName}.pdf"
            }
        }
    }
}

task figures {
    dependsOn 'svgFigures', 'pdfFigures'
    doLast {
    }
}

//#################################
// joint cardinality calculation
//#################################

task buildJointCardinalityCalculation(type: Exec) {
    workingDir project.cppDir
    inputs.files "$project.cppDir/joint_cardinality_calculation.cpp", "$project.cppDir/cardinality_estimation.hpp", "$project.cppDir/hyperloglog.hpp", "$project.cppDir/two_hyperloglog_statistic.hpp"
    outputs.files "$project.cppDir/joint_cardinality_calculation"
    standardOutput = new ByteArrayOutputStream()
    commandLine 'g++','-std=c++11','-Wall','joint_cardinality_calculation.cpp','-Ofast','-march=native','-o','joint_cardinality_calculation'
    // commandLine 'g++','-std=c++11','-I/usr/include/eigen3','-Wall','joint_cardinality_calculation.cpp','-Ofast','-march=native','-lceres','-lglog','-fopenmp','-lblas','-llapack','-lcholmod', '-o','joint_cardinality_calculation'
}

task doJointCardinalityCalculation(type: Exec) {
    workingDir project.cppDir
    inputs.files "$project.cppDir/joint_cardinality_calculation", "$project.cppDir/param.txt", "$project.dataDir/hll_joint/joint_cardinalities.dat"
    outputs.files "$project.paperDir/joint_cardinality_calculation.csv"
    standardOutput = new FileOutputStream("$project.cppDir/joint_cardinality_calculation.out")
    commandLine './joint_cardinality_calculation'
    dependsOn 'buildJointCardinalityCalculation','generateSimulatedCardinalities'
}

//#########################
// paper
//#########################

task pdflatex1(type: Exec) {
    workingDir project.paperDir
    inputs.files "$project.paperDir/paper.tex"
    outputs.files "$project.paperDir/paper.aux"
    outputs.upToDateWhen { false }
    standardOutput = new ByteArrayOutputStream()
    commandLine 'pdflatex','paper'
}

task bibtex(type: Exec) {
    workingDir project.paperDir
    inputs.files "$project.paperDir/paper.aux","$project.paperDir/bibliography.bib"
    outputs.files "$project.paperDir/paper.bbl", "$project.paperDir/paper.blg"
    outputs.upToDateWhen { false }
    standardOutput = new ByteArrayOutputStream()
    commandLine 'bibtex','paper'
    dependsOn 'pdflatex1'
}

task pdflatex2(type: Exec) {
    workingDir project.paperDir

    def pdfFigureFiles = figureNames.collect {"$project.paperDir/" + it + ".pdf"}

    inputs.files "$project.paperDir/paper.bbl",
        "$project.paperDir/paper.blg",
        "$project.paperDir/paper.tex",
        "$project.paperDir/joint_cardinality_calculation.csv",
        pdfFigureFiles
    outputs.files "$project.paperDir/paper.aux"
    outputs.upToDateWhen { false }
    standardOutput = new ByteArrayOutputStream()
    commandLine 'pdflatex','paper'
    dependsOn 'bibtex'
}

task pdflatex3(type: Exec) {

    def pdfFigureFiles = figureNames.collect {"$project.paperDir/" + it + ".pdf"}

    workingDir project.paperDir
    inputs.files "$project.paperDir/paper.bbl",
        "$project.paperDir/paper.blg",
        "$project.paperDir/paper.tex",
        "$project.paperDir/intersection.csv",
        pdfFigureFiles,
        "$project.paperDir/max_errror_propagation_factor1.txt",
        "$project.paperDir/max_errror_propagation_factor2.txt",
        "$project.paperDir/max_errror_propagation_factor3.txt",
        "$project.paperDir/max_errror_propagation_factor4.txt"
    outputs.files "$project.paperDir/paper.aux", "$project.paperDir/paper.pdf"
    outputs.upToDateWhen { false }
    standardOutput = new ByteArrayOutputStream()
    commandLine 'pdflatex','paper'
    dependsOn 'pdflatex2', 'calculateMaxErrorPropagationFactor'
}

task calculateMaxErrorPropagationFactor(type: Exec) {
    workingDir project.pythonDir
    inputs.files "$project.pythonDir/recursion-stability.py"
    outputs.files "$project.paperDir/max_errror_propagation_factor1.txt",
        "$project.paperDir/max_errror_propagation_factor2.txt",
        "$project.paperDir/max_errror_propagation_factor3.txt",
        "$project.paperDir/max_errror_propagation_factor4.txt"
    standardOutput = new ByteArrayOutputStream()
    commandLine 'python3', 'recursion-stability.py'
}

task paper {
    dependsOn 'pdflatex3'
    doLast {
    }
}

//#########################
// publication
//#########################

task arxiv(type: Zip) {
   def pdfFigureFiles = figureNames.collect {"$project.paperDir/" + it + ".pdf"}
   pdfFigureFiles.each {
        from it
   }
   from "$project.paperDir/paper.bbl"
   from "$project.paperDir/paper.tex"

   from "$project.paperDir/joint_cardinality_calculation.csv"
   from "$project.paperDir/max_errror_propagation_factor1.txt"
   from "$project.paperDir/max_errror_propagation_factor2.txt"
   from "$project.paperDir/max_errror_propagation_factor3.txt"
   from "$project.paperDir/max_errror_propagation_factor4.txt"

   archiveName 'arxiv.zip'
   destinationDir file(project.paperDir)
}