METRON-2038 Enrichment Loader Fails When Run as MR Job (nickwallen) closes apache#1358

Author: nickwallen

dependencies_with_url.csv

@@ -194,6 +194,7 @@ com.google.guava:guava:jar:18.0:compile,ASLv2,
 com.google.inject.extensions:guice-servlet:jar:3.0:compile,ASLv2,
 com.google.inject:guice:jar:3.0:compile,ASLv2,
 com.googlecode.disruptor:disruptor:jar:2.10.4:compile,The Apache Software License, Version 2.0,http://code.google.com/p/disruptor/
+com.lmax:disruptor:jar:3.3.0:compile,The Apache Software License, Version 2.0,https://github.com/LMAX-Exchange/disruptor/
 com.lmax:disruptor:jar:3.3.2:compile,The Apache Software License, Version 2.0,https://github.com/LMAX-Exchange/disruptor/
 com.googlecode.json-simple:json-simple:jar:1.1:compile,The Apache Software License, Version 2.0,http://code.google.com/p/json-simple/
 com.googlecode.json-simple:json-simple:jar:1.1.1:compile,The Apache Software License, Version 2.0,http://code.google.com/p/json-simple/
@@ -301,6 +302,10 @@ org.jboss.resteasy:resteasy-jaxrs:jar:3.0.1.Final:compile,ASLv2,
 org.joda:joda-convert:jar:1.2:compile,Apache 2,http://joda-convert.sourceforge.net
 org.mortbay.jetty:jetty-util:jar:6.1.26:compile,ASLv2,
 org.mortbay.jetty:jetty:jar:6.1.26:compile,ASLv2,
+org.mortbay.jetty:jetty-sslengine:jar:6.1.26:compile,ASLv2,
+org.mortbay.jetty:jsp-2.1:jar:6.1.14:compile,ASLv2,
+org.mortbay.jetty:jsp-api-2.1:jar:6.1.14:compile,ASLv2,
+org.mortbay.jetty:servlet-api-2.5:jar:6.1.14:compile,ASLv2,
 org.noggit:noggit:jar:0.6:compile,Apache License, Version 2.0,http://github.com/yonik/noggit
 org.scannotation:scannotation:jar:1.0.3:compile,Apache License V2.0,http://scannotation.sf.net
 org.slf4j:log4j-over-slf4j:jar:1.6.6:compile,Apache Software Licenses,http://www.slf4j.org
@@ -498,4 +503,4 @@ joda-time:joda-time:jar:2.10:compile
 org.elasticsearch:securesm:jar:1.2:compile
 com.github.stephenc.jcip:jcip-annotations:jar:1.0-1:compile,ASLv2,http://stephenc.github.io/jcip-annotations/
 com.nimbusds:nimbus-jose-jwt:jar:4.41.2:compile,ASLv2,https://bitbucket.org/connect2id/nimbus-jose-jwt/wiki/Home
-
+tomcat:jasper-compiler:jar:5.5.23:compile,ASLv2,https://tomcat.apache.org/

metron-platform/metron-data-management/pom.xml

@@ -186,7 +186,6 @@
                 <artifactId>hadoop-common</artifactId>
               </exclusion>
             </exclusions>
-            <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.hbase</groupId>
@@ -210,7 +209,6 @@
                     <artifactId>hadoop-common</artifactId>
                 </exclusion>
             </exclusions>
-            <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>com.opencsv</groupId>
@@ -344,7 +342,6 @@
             <scope>test</scope>
         </dependency>
     </dependencies>
-
     <build>
         <plugins>
             <plugin>
@@ -380,7 +377,6 @@
                                 </relocation>
                              </relocations>
                              <transformers>
-                                 <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/>
                                 <transformer
                                   implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
                                      <resources>
@@ -395,6 +391,11 @@
                                     <addHeader>false</addHeader>
                                     <projectName>${project.name}</projectName>
                                 </transformer-->
+                                <!--
+                                  ClassIndexTransformer needs to go LAST. For some reason it will clobber other
+                                  transformers from operating when it is put first.
+                                -->
+                                <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/>
                             </transformers>
                             <artifactSet>
                                 <excludes>

metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh

@@ -1,21 +1,21 @@
 #!/bin/bash
-# 
+#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# 
+#
 
 BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
 [ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
@@ -31,21 +31,14 @@ export METRON_VERSION=${project.version}
 export METRON_HOME=/usr/metron/$METRON_VERSION
 export CLASSNAME="org.apache.metron.dataloads.nonbulk.flatfile.SimpleEnrichmentFlatFileLoader"
 export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
-export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
+export HBASE_CONF=${HBASE_CONF:-/etc/hbase/conf}
 export HADOOP_OPTS="$HADOOP_OPTS $METRON_JVMFLAGS"
 if [ $(which hadoop) ]
 then
-  HADOOP_CLASSPATH=${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath`
-  for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
-    if [ -f $jar ];then
-      LIBJARS="$jar,$LIBJARS"
-    fi
-  done
-  export HADOOP_CLASSPATH
-  hadoop jar $METRON_HOME/lib/$DM_JAR $CLASSNAME -libjars ${LIBJARS} "$@"
+  export HADOOP_CLASSPATH="$HBASE_CONF"
+  hadoop jar $METRON_HOME/lib/$DM_JAR $CLASSNAME "$@"
 else
   echo "Warning: Metron cannot find the hadoop client on this node.  This means that loading via Map Reduce will NOT function."
-  CP=$METRON_HOME/lib/$DM_JAR:/usr/metron/${METRON_VERSION}/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath`
+  CP=$METRON_HOME/lib/$DM_JAR:$HBASE_CONF
   java $METRON_JVMFLAGS -cp $CP $CLASSNAME "$@"
 fi
-