Feature/dart (#12)

Author: zhangwei217245

.docker/dev_base.Dockerfile

@@ -17,6 +17,8 @@ RUN apt-get update && apt-get install -y \
     libmpich-dev \
     libhdf5-dev \
     libhdf5-mpich-dev \
+    libtiff5 \
+    libtiff5-dev \
     uuid \
     uuid-dev \
     autoconf \

examples/llsm/.gitignore

@@ -0,0 +1 @@
+build

examples/llsm/CMakeLists.txt

@@ -0,0 +1,96 @@
+cmake_minimum_required (VERSION 2.8.12)
+
+# Setup cmake policies.
+foreach(p
+    CMP0012
+    CMP0013
+    CMP0014
+    CMP0022 # CMake 2.8.12
+    CMP0025 # CMake 3.0
+    CMP0053 # CMake 3.1
+    CMP0054 # CMake 3.1
+    CMP0074 # CMake 3.12
+    CMP0075 # CMake 3.12
+    CMP0083 # CMake 3.14
+    CMP0093 # CMake 3.15
+  )
+  if(POLICY ${p})
+    cmake_policy(SET ${p} NEW)
+  endif()
+endforeach()
+
+project(PDC_LLSM_EXAM C)
+
+set(LLSM_EXT_INCLUDE_DIRS "")
+set(LLSM_EXT_LIBRARIES "")
+
+
+
+set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Choose the type of build." FORCE)
+
+find_package(PDC REQUIRED)
+if(PDC_FOUND)
+    #message(STATUS "PDC include directory: ${PDC_INCLUDE_DIR}")
+  set(LLSM_EXT_INCLUDE_DIRS ${PDC_INCLUDE_DIR}
+    ${LLSM_EXT_INCLUDE_DIRS}
+    )
+  set(LLSM_EXT_LIBRARIES pdc ${LLSM_EXT_LIBRARIES})
+endif()
+
+option(USE_SYSTEM_MPI "Use system-installed OpenMP." ON)
+if(USE_SYSTEM_MPI)
+  find_package(MPI)
+  if(MPI_FOUND)
+    add_definitions(-DLLSM_ENABLE_MPI=1)
+    SET(CMAKE_C_COMPILER ${MPI_C_COMPILER})
+    SET(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER})
+    set(LLSM_EXT_INCLUDE_DIRS ${MPI_C_INCLUDE_PATH}
+      ${LLSM_EXT_INCLUDE_DIRS}
+      )
+    set(LLSM_EXT_LIBRARIES ${MPI_C_LIBRARIES} ${LLSM_EXT_LIBRARIES})
+  endif()
+endif()
+
+option(USE_SYSTEM_OPENMP "Use system-installed OpenMP." ON)
+if(USE_SYSTEM_OPENMP)
+  find_package(OpenMP REQUIRED)
+  if(OPENMP_FOUND)
+      add_definitions(-DENABLE_OPENMP=1)
+      set(ENABLE_OPENMP 1)
+      set(OPENMP_LIBRARIES "${OpenMP_C_LIBRARIES}")
+      set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fopenmp")
+  else()
+      message(FATAL_ERROR "OpenMP not found")
+  endif()
+endif()
+
+include_directories(
+  ${LLSM_EXT_INCLUDE_DIRS}
+)
+
+
+# Find LibTIFF
+option(USE_LIB_TIFF "Enable LibTiff." ON)
+if(USE_LIB_TIFF)
+  find_package(TIFF REQUIRED)
+  if(TIFF_FOUND)
+    set(LLSM_LIB_SOURCE 
+      llsm_aux/parallelReadTiff.c
+      llsm_aux/csvReader.c
+      llsm_aux/pdc_list.c
+    )
+    # Add the LibTIFF include directory to the include path
+    include_directories(${TIFF_INCLUDE_DIRS})
+    add_library(llsm_tiff ${LLSM_LIB_SOURCE})
+    target_compile_options(llsm_tiff PRIVATE ${OpenMP_C_FLAGS})
+    target_link_libraries(llsm_tiff PUBLIC ${OpenMP_C_LIBRARIES})
+    target_link_libraries(llsm_tiff PUBLIC ${TIFF_LIBRARIES})
+    target_include_directories(llsm_tiff PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/llsm)
+
+    add_executable(llsm_importer llsm_importer.c)
+    target_link_libraries(llsm_importer ${PDC_EXT_LIB_DEPENDENCIES} pdc ${TIFF_LIBRARIES} llsm_tiff ${LLSM_EXT_LIBRARIES})
+    target_include_directories(llsm_importer PUBLIC ${LLSM_EXT_INCLUDE_DIRS})
+  else()
+    message(WARNING "LibTiff not found, ignore building the executables which requires LibTiff support.")
+  endif()
+endif()

tools/LLSM_IMPORTER.md examples/llsm/LLSM_IMPORTER.md

@@ -1,7 +1,8 @@
 #include "parallelReadTiff.h"
 #include "tiffio.h"
+#include "inttypes.h"
 
-#define ENABLE_OPENMP
+// #define ENABLE_OPENMP
 
 #ifdef ENABLE_OPENMP
 #include "omp.h"
@@ -50,7 +51,7 @@ readTiffParallelBak(uint64_t x, uint64_t y, uint64_t z, const char *fileName, vo
 
             int counter = 0;
             while (!TIFFSetDirectory(tif, (uint64_t)dir) && counter < 3) {
-                printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n", w, fileName, dir,
+                printf("Thread %d: File \"%s\" Directory \"%"PRId64"\" failed to open. Try %d\n", w, fileName, dir,
                        counter + 1);
                 counter++;
             }
@@ -344,7 +345,7 @@ readTiffParallel2DBak(uint64_t x, uint64_t y, uint64_t z, const char *fileName,
 
             int counter = 0;
             while (!TIFFSetDirectory(tif, (uint64_t)0) && counter < 3) {
-                printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n", w, fileName, dir,
+                printf("Thread %d: File \"%s\" Directory \"%"PRId64"\" failed to open. Try %d\n", w, fileName, dir,
                        counter + 1);
                 counter++;
             }

tools/llsm/csvReader.c examples/llsm/llsm_aux/csvReader.c

@@ -4,22 +4,17 @@
 #include <getopt.h>
 #include <time.h>
 
-#ifndef ENABLE_MPI
-#define ENABLE_MPI
-#endif
-
-#ifdef ENABLE_MPI
+#ifdef LLSM_ENABLE_MPI
 #include "mpi.h"
-// #undef ENABLE_MPI
 #endif
 
 #include "pdc.h"
 // #include "pdc_client_server_common.h"
 // #include "pdc_client_connect.h"
 
-#include "llsm/parallelReadTiff.h"
-#include "llsm/pdc_list.h"
-#include "llsm/csvReader.h"
+#include "llsm_aux/parallelReadTiff.h"
+#include "llsm_aux/pdc_list.h"
+#include "llsm_aux/csvReader.h"
 #include <libgen.h>
 
 typedef struct llsm_importer_args_t {
@@ -126,7 +121,7 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell)
 
     duration = getDoubleTimestamp() - start; // end timing the operation and calculate duration in nanoseconds
 
-    printf("[Rank %4d] Region_Transfer %s_[%d_Bytes] Done! Time taken: %.4f seconds\n", rank,
+    printf("[Rank %4d] Region_Transfer %s_[%ld_Bytes] Done! Time taken: %.4f seconds\n", rank,
            fileName_cell->field_value, image_info->tiff_size, duration);
 
     // add metadata tags based on the csv row
@@ -372,7 +367,7 @@ main(int argc, char *argv[])
 #endif
 
     if (rank == 0) {
-        printf("[Completion Time] LLSM IMPORTER FINISHES! Time taken: %.4f seconds\n", rank, duration);
+        printf("[Completion Time] LLSM IMPORTER FINISHES! Time taken: %.4f seconds\n", duration);
     }
     // free memory for csv table
     csv_free_table(csv_table);

tools/llsm/csvReader.h examples/llsm/llsm_aux/csvReader.h

@@ -1,5 +1,75 @@
-project(${PDC_SOURCE_DIR})
-cmake_minimum_required (VERSION 3.0)
+set(TOOLS_EXT_INCLUDE "")
+set(TOOLS_EXT_LIB "")
+
+# *************************************************
+# *  MERCURY
+# *************************************************
+find_package(MERCURY REQUIRED)
+if(MERCURY_FOUND)
+  set(TOOLS_EXT_INCLUDE ${MERCURY_INCLUDE_DIRS}
+    ${TOOLS_EXT_INCLUDE}
+    )
+  set(TOOLS_EXT_LIB ${MERCURY_LIBRARIES} ${TOOLS_EXT_LIB})
+endif()
+
+#HDF5
+find_package(HDF5 MODULE)
+  if(NOT HDF5_FOUND)
+    message(STATUS "Could not find HDF5, fallback to NO_MODULE mode.")
+    find_package(HDF5 NO_MODULE NAMES hdf5 COMPONENTS C shared)
+    if(NOT HDF5_FOUND)
+      message(FATAL_ERROR "Could not find HDF5, please check HDF5_DIR or make sure that HDF5 has ben compiled with shared libraries enabled.")
+    else()
+      set(HDF5_LIBRARIES ${HDF5_LIBRARIES} hdf5-shared)
+      set(HDF5_INCLUDE_DIRS ${HDF5_INCLUDE_DIRS} ${HDF5_INCLUDE_DIR})
+    endif()
+  endif()
+
+if (HDF5_FOUND)
+  set(TOOLS_EXT_INCLUDE
+    ${TOOLS_EXT_INCLUDE}
+    ${HDF5_INCLUDE_DIRS}
+  )
+  set(TOOLS_EXT_LIB
+    ${TOOLS_EXT_LIB}
+    ${HDF5_LIBRARIES}
+  )
+endif()
+
+# option(USE_SYSTEM_HDF5 "Use system-installed HDF5." ON)
+# if(USE_SYSTEM_HDF5)
+#   find_package(HDF5 NO_MODULE NAMES hdf5 COMPONENTS C shared)
+#   if(HDF5_FOUND)
+#     set(HDF5_C_SHARED_LIBRARY hdf5-shared)
+#   #    if(NOT TARGET ${HDF5_C_SHARED_LIBRARY})
+#   #      message(FATAL_ERROR "Could not find hdf5 shared target, please make "
+#   #"sure that HDF5 has ben compiled with shared libraries enabled.")
+#   #    endif()
+#     set(TOOLS_EXT_INCLUDE
+#         ${TOOLS_EXT_INCLUDE}
+#         ${HDF5_INCLUDE_DIR}
+#       )
+#     set(TOOLS_EXT_LIB
+#       ${TOOLS_EXT_LIB}
+#       ${HDF5_C_SHARED_LIBRARY}
+#     )
+#   endif()
+# else()
+#   # Allow for HDF5 autotools builds
+#   find_package(HDF5 MODULE REQUIRED)
+#   if(HDF5_FOUND)
+#     set(TOOLS_EXT_INCLUDE
+#       ${TOOLS_EXT_INCLUDE}
+#       ${HDF5_INCLUDE_DIRS}
+#     )
+#     set(TOOLS_EXT_LIB
+#       ${TOOLS_EXT_LIB}
+#       ${HDF5_LIBRARIES}
+#     )
+#   else()
+#     message(FATAL_ERROR "Could not find HDF5, please check HDF5_DIR.")
+#   endif()
+# endif()
 
 include_directories(
   ${CMAKE_CURRENT_SOURCE_DIR}
@@ -10,8 +80,10 @@ include_directories(
   ${PDC_SOURCE_DIR}/src/client_api/include
   $ENV{HOME}/Sandbox/c-blosc/blosc
   $ENV{HOME}/include
+  ${TOOLS_EXT_INCLUDE}
 )
 
+
 set(PROGRAMS
   pdc_import
   pdc_export
@@ -22,12 +94,5 @@ add_library(cjson cjson/cJSON.c)
 
 foreach(program ${PROGRAMS})
   add_executable(${program} ${program}.c)
-  target_link_libraries(${program} pdc)
-  target_link_libraries(${program} cjson)
-endforeach(program)
-
-
-# *******************************************
-# Add the HDF5 library for pdc-neon
-# *******************************************
-FIND_LIBRARY(HDF5_LIBRARY NAMES hdf5_debug PATHS $ENV{HOME}/lib)
+  target_link_libraries(${program} pdc cjson ${TOOLS_EXT_LIB})
+endforeach(program)