Reverted changes to pass EMAIL, LICENSE options to zingg

Added a file to configure env variables that are needed for spark

Author: navinrathore

client/src/main/java/zingg/client/ClientOptions.java

@@ -49,7 +49,7 @@ public class ClientOptions {
 	static {	//This is the canonical list of Zingg options.
 		optionMaster.put(CONF, new Option(CONF, true, "JSON configuration with data input output locations and field definitions", false, true));
 		optionMaster.put(PHASE, new Option(PHASE, true, Util.join(ZinggOptions.getAllZinggOptions(), "|"), false, true, ZinggOptions.getAllZinggOptions()));
-		optionMaster.put(LICENSE, new Option(LICENSE, true, "location of license file", false, false));
+		optionMaster.put(LICENSE, new Option(LICENSE, true, "location of license file", false, true));
 		optionMaster.put(JOBID, new Option(JOBID, true, "database job id for logging", false, false));
 		optionMaster.put(EMAIL, new Option(EMAIL, true, "notification email id. Can be an alias", false, false));
 		optionMaster.put(FORMAT, new Option(FORMAT, true, "format of the data", false, false));

config/zingg-env.sh

@@ -0,0 +1,21 @@
+#### ZINGG_EXTRA_JARS #############################################
+# The ZINGG_EXTRA_JARS variable is set to pass additional dependencies to ZINGG. This env variable must be set while working with BigQuery, Snowflake, Mysql etc.
+# multiple jars should be separated by comma
+# e.g. ZINGG_EXTRA_JARS=one.jar,two.jar,three.jar
+#ZINGG_EXTRA_JARS=
+
+
+#### ZINGG_EXTRA_SPARK_CONF #######################################
+# The ZINGG_EXTRA_SPARK_CONF variable must be set to pass additional parameters to pass to spark. For example, when running Zingg with BigQuery.
+# the variable must be in below format. Multiple 'conf' entries can be combined and passed in this variable.
+# e.g. ZINGG_EXTRA_SPARK_CONF="--conf spark.hadoop.fs.gs.impl=com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem"
+#ZINGG_EXTRA_SPARK_CONF=
+
+
+#### SPARK_EXECUTOR_MEMORY ########################################
+# The SPARK_EXECUTOR_MEMORY variable updates spark.executor.memory. It may be modified based on memory available in the system. Default is 8GB
+SPARK_EXECUTOR_MEMORY=8g
+
+#### SPARK_DRIVER_MEMORY ##########################################
+# The SPARK_DRIVER_MEMORY variable updates spark.driver.memory. It may be modified based on memory available in the system. Default is 8GB.
+SPARK_DRIVER_MEMORY=8g

scripts/load-zingg-env.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+ZINGG_ENV_SH="zingg-env.sh"
+export ZINGG_CONF_DIR="$(dirname "$0")"/../config
+
+ZINGG_ENV_SH="${ZINGG_CONF_DIR}/${ZINGG_ENV_SH}"
+if [[ -f "${ZINGG_ENV_SH}" ]]; then
+  # Promote all variable declarations to environment (exported) variables
+  set -a
+  . ${ZINGG_ENV_SH}
+  set +a
+fi

scripts/zingg.sh

@@ -4,6 +4,12 @@ ZINGG_JARS=$ZINGG_HOME/zingg-0.3.3-SNAPSHOT.jar
 [email protected]
 LICENSE="test"
 
+# Set the ZINGG environment variables
+ZINGG_ENV="$(dirname "$0")"/load-zingg-env.sh
+if [[ -f "${ZINGG_ENV}" ]]; then
+  source ${ZINGG_ENV}
+fi
+
 if [[ -z "${ZINGG_EXTRA_JARS}" ]]; then
   OPTION_JARS=""
 else
@@ -16,4 +22,16 @@ else
   OPTION_SPARK_CONF="${ZINGG_EXTRA_SPARK_CONF}"
 fi
 
-$SPARK_HOME/bin/spark-submit --master $SPARK_MASTER $OPTION_JARS $OPTION_SPARK_CONF --conf spark.serializer=org.apache.spark.serializer.KryoSerializer --conf spark.default.parallelism="8" --conf spark.executor.extraJavaOptions="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+HeapDumpOnOutOfMemoryError -Xloggc:/tmp/memLog.txt -XX:+UseCompressedOops" --conf spark.executor.memory=10g --conf spark.debug.maxToStringFields=200 --driver-class-path $ZINGG_JARS --class zingg.client.Client $ZINGG_JARS $@ --email $EMAIL --license $LICENSE 
+if [[ -z "${SPARK_EXECUTOR_MEMORY}" ]]; then
+  SPARK_EXECUTOR_MEMORY=8g
+fi  
+OPTION_EXECUTOR_MEMORY="--conf spark.executor.memory=${SPARK_EXECUTOR_MEMORY}"
+
+if [[ -z "${SPARK_DRIVER_MEMORY}" ]]; then
+  SPARK_DRIVER_MEMORY=8g
+fi
+OPTION_DRIVER_MEMORY="--conf spark.driver.memory=${SPARK_DRIVER_MEMORY}"
+
+# All the additional options must be added here
+ALL_OPTIONS=" ${OPTION_DRIVER_MEMORY} ${OPTION_EXECUTOR_MEMORY} ${OPTION_JARS} ${OPTION_SPARK_CONF} "
+$SPARK_HOME/bin/spark-submit --master $SPARK_MASTER ${ALL_OPTIONS} --conf spark.serializer=org.apache.spark.serializer.KryoSerializer --conf spark.default.parallelism="8" --conf spark.executor.extraJavaOptions="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+HeapDumpOnOutOfMemoryError -Xloggc:/tmp/memLog.txt -XX:+UseCompressedOops" --conf spark.debug.maxToStringFields=200 --driver-class-path $ZINGG_JARS --class zingg.client.Client $ZINGG_JARS $@ --email $EMAIL --license $LICENSE