Updated Code for #758

tools/docker/Dockerfile → Dockerfile

@@ -2,7 +2,7 @@ FROM java:openjdk-8-alpine
 
 MAINTAINER [email protected]
 
-ENV       VERSION 2.3.0-RC1
+ENV       VERSION 2.5.0-SNAPSHOT
 ENV       WORKDIR /usr/share/opentsdb
 ENV       LOGDIR  /var/log/opentsdb
 ENV       DATADIR /data/opentsdb
@@ -32,10 +32,10 @@ ENV       TSDB_PORT  4244
 
 WORKDIR   $WORKDIR
 
-ADD       libs $WORKDIR/libs
-ADD       logback.xml $WORKDIR
+ADD       third_party/*/*.jar $WORKDIR/libs/
+ADD       src/logback.xml $WORKDIR
 ADD       tsdb-$VERSION.jar $WORKDIR
-ADD       opentsdb.conf $ETCDIR/opentsdb.conf
+ADD       src/opentsdb.conf $ETCDIR/opentsdb.conf
 
 VOLUME    ["/etc/openstsdb"]
 VOLUME    ["/data/opentsdb"]

Makefile.am

@@ -32,6 +32,8 @@ dist_noinst_DATA = pom.xml.in build-aux/rpm/opentsdb.conf \
   build-aux/rpm/logback.xml build-aux/rpm/init.d/opentsdb \
   build-aux/rpm/systemd/[email protected]
 tsdb_SRC := \
+	src/core/AbstractSpanGroup.java \
+	src/core/AbstractQuery.java \
 	src/core/AggregationIterator.java	\
 	src/core/Aggregator.java	\
 	src/core/Aggregators.java	\
@@ -48,6 +50,7 @@ tsdb_SRC := \
 	src/core/DownsamplingSpecification.java \
 	src/core/FillingDownsampler.java \
 	src/core/FillPolicy.java \
+	src/core/GroupCallback.java \
 	src/core/Histogram.java	\
 	src/core/HistogramAggregation.java	\
 	src/core/HistogramAggregationIterator.java	\
@@ -78,14 +81,18 @@ tsdb_SRC := \
 	src/core/RequestBuilder.java	\
 	src/core/RowKey.java	\
 	src/core/RowSeq.java	\
+	src/core/RpcResponder.java	\
 	src/core/iRowSeq.java	\
 	src/core/SaltScanner.java	\
 	src/core/SeekableView.java	\
+	src/core/SeekableViewChain.java	\
 	src/core/SimpleHistogram.java	\
 	src/core/SimpleHistogramDataPointAdapter.java	\
 	src/core/SimpleHistogramDecoder.java	\
 	src/core/Span.java	\
 	src/core/SpanGroup.java	\
+	src/core/SplitRollupQuery.java  \
+	src/core/SplitRollupSpanGroup.java  \
 	src/core/TSDB.java	\
 	src/core/Tags.java	\
 	src/core/TsdbQuery.java	\
@@ -120,6 +127,7 @@ tsdb_SRC := \
 	src/query/expression/ExpressionReader.java	\
 	src/query/expression/Expressions.java	\
 	src/query/expression/ExpressionTree.java	\
+	src/query/expression/FirstDifference.java	\
 	src/query/expression/HighestCurrent.java	\
 	src/query/expression/HighestMax.java	\
 	src/query/expression/IntersectionIterator.java	\
@@ -316,12 +324,17 @@ test_SRC := \
 	test/core/TestRateSpan.java	\
 	test/core/TestRowKey.java	\
 	test/core/TestRowSeq.java	\
+	test/core/TestRpcResponsder.java	\
 	test/core/TestSaltScanner.java	\
+	test/core/TestSeekableViewChain.java	\
 	test/core/TestSpan.java	\
 	test/core/TestSpanGroup.java	\
+	test/core/TestSplitRollupQuery.java	\
+	test/core/TestSplitRollupSpanGroup.java	\
 	test/core/TestTags.java	\
 	test/core/TestTSDB.java	\
 	test/core/TestTSDBAddPoint.java	\
+	test/core/TestTSDBTableAvailability.java	\
 	test/core/TestTsdbQueryDownsample.java	\
 	test/core/TestTsdbQueryDownsampleSalted.java	\
 	test/core/TestTsdbQuery.java	\
@@ -376,6 +389,7 @@ test_SRC := \
 	test/query/pojo/TestTimeSpan.java	\
 	test/rollup/TestRollupConfig.java	\
 	test/rollup/TestRollupInterval.java	\
+	test/rollup/TestRollupQuery.java	\
 	test/rollup/TestRollupSeq.java	\
 	test/rollup/TestRollupUtils.java	\
 	test/search/TestSearchPlugin.java	\
@@ -414,6 +428,7 @@ test_SRC := \
 	test/tsd/TestRTPublisher.java	\
 	test/tsd/TestSearchRpc.java	\
 	test/tsd/TestStatsRpc.java \
+	test/tsd/TestStatusRpc.java \
 	test/tsd/TestSuggestRpc.java	\
 	test/tsd/TestTreeRpc.java	\
 	test/tsd/TestUniqueIdRpc.java	\

build-aux/deb/logback.xml

@@ -64,6 +64,9 @@
   <logger name="org.apache.zookeeper" level="INFO"/>
   <logger name="org.hbase.async" level="INFO"/>
   <logger name="com.stumbleupon.async" level="INFO"/>
+  <logger name="net.opentsdb.tools" level="INFO">
+    <appender-ref ref="STDOUT"/>
+  </logger>
 
   <!-- Fallthrough root logger and router -->
   <root level="INFO">

build-aux/rpm/logback.xml

@@ -64,6 +64,9 @@
   <logger name="org.apache.zookeeper" level="INFO"/>
   <logger name="org.hbase.async" level="INFO"/>
   <logger name="com.stumbleupon.async" level="INFO"/>
+  <logger name="net.opentsdb.tools" level="INFO">
+    <appender-ref ref="STDOUT"/>
+  </logger>
 
   <!-- Fallthrough root logger and router -->
   <root level="INFO">

configure.ac

@@ -14,7 +14,7 @@
 # along with this library.  If not, see <http://www.gnu.org/licenses/>.
 
 # Semantic Versioning (see http://semver.org/).
-AC_INIT([opentsdb], [2.4.0RC2], [[email protected]])
+AC_INIT([opentsdb], [2.5.0-SNAPSHOT], [[email protected]])
 AC_CONFIG_AUX_DIR([build-aux])
 AM_INIT_AUTOMAKE([foreign])
 

screwdriver.yaml

@@ -1,7 +1,11 @@
 shared:
-  image: maven
+  image: maven:3-adoptopenjdk-8
 
 jobs:
+  pr:
+    steps:
+      - run_arbitrary_script: apt-get update && apt-get install autoconf make python -y && ./build.sh pom.xml && mvn clean test --quiet
   main:
+    requires: [~pr, ~commit]
     steps:
-      - run_arbitrary_script: apt-get update && apt-get install autoconf make -y && ./build.sh pom.xml && mvn clean test --quiet
+      - run_arbitrary_script: apt-get update && apt-get install autoconf make python -y && ./build.sh pom.xml && mvn clean test --quiet

src/core/AbstractQuery.java

@@ -0,0 +1,66 @@
+// This file is part of OpenTSDB.
+// Copyright (C) 2010-2012  The OpenTSDB Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 2.1 of the License, or (at your
+// option) any later version.  This program is distributed in the hope that it
+// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
+// General Public License for more details.  You should have received a copy
+// of the GNU Lesser General Public License along with this program.  If not,
+// see <http://www.gnu.org/licenses/>.
+package net.opentsdb.core;
+
+import org.hbase.async.HBaseException;
+
+public abstract class AbstractQuery implements Query {
+    /**
+     * Runs this query.
+     *
+     * @return The data points matched by this query.
+     * <p>
+     * Each element in the non-{@code null} but possibly empty array returned
+     * corresponds to one time series for which some data points have been
+     * matched by the query.
+     * @throws HBaseException if there was a problem communicating with HBase to
+     *                        perform the search.
+     */
+    @Override
+    public DataPoints[] run() throws HBaseException {
+        try {
+            return runAsync().joinUninterruptibly();
+        } catch (RuntimeException e) {
+            throw e;
+        } catch (Exception e) {
+            throw new RuntimeException("Should never be here", e);
+        }
+    }
+
+    /**
+     * Runs this query.
+     *
+     * @return The data points matched by this query and applied with percentile calculation
+     * <p>
+     * Each element in the non-{@code null} but possibly empty array returned
+     * corresponds to one time series for which some data points have been
+     * matched by the query.
+     * @throws HBaseException        if there was a problem communicating with HBase to
+     *                               perform the search.
+     * @throws IllegalStateException if the query is not a histogram query
+     */
+    @Override
+    public DataPoints[] runHistogram() throws HBaseException {
+        if (!isHistogramQuery()) {
+            throw new RuntimeException("Should never be here");
+        }
+
+        try {
+            return runHistogramAsync().joinUninterruptibly();
+        } catch (RuntimeException e) {
+            throw e;
+        } catch (Exception e) {
+            throw new RuntimeException("Should never be here", e);
+        }
+    }
+}

src/core/AbstractSpanGroup.java

@@ -0,0 +1,70 @@
+// This file is part of OpenTSDB.
+// Copyright (C) 2010-2012  The OpenTSDB Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 2.1 of the License, or (at your
+// option) any later version.  This program is distributed in the hope that it
+// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
+// General Public License for more details.  You should have received a copy
+// of the GNU Lesser General Public License along with this program.  If not,
+// see <http://www.gnu.org/licenses/>.
+package net.opentsdb.core;
+
+import java.util.*;
+
+import org.hbase.async.Bytes;
+import org.hbase.async.Bytes.ByteMap;
+
+import com.stumbleupon.async.Callback;
+import com.stumbleupon.async.Deferred;
+
+import net.opentsdb.meta.Annotation;
+import net.opentsdb.rollup.RollupQuery;
+
+/**
+ * Groups multiple spans together and offers a dynamic "view" on them.
+ * <p>
+ * This is used for queries to the TSDB, where we might group multiple
+ * {@link Span}s that are for the same time series but different tags
+ * together.  We need to "hide" data points that are outside of the
+ * time period of the query and do on-the-fly aggregation of the data
+ * points coming from the different Spans, using an {@link Aggregator}.
+ * Since not all the Spans will have their data points at exactly the
+ * same time, we also do on-the-fly linear interpolation.  If needed,
+ * this view can also return the rate of change instead of the actual
+ * data points.
+ * <p>
+ * This is one of the rare (if not the only) implementations of
+ * {@link DataPoints} for which {@link #getTags} can potentially return
+ * an empty map.
+ * <p>
+ * The implementation can also dynamically downsample the data when a
+ * sampling interval a downsampling function (in the form of an
+ * {@link Aggregator}) are given.  This is done by using a special
+ * iterator when using the {@link Span.DownsamplingIterator}.
+ */
+abstract class AbstractSpanGroup implements DataPoints {
+    /**
+     * Finds the {@code i}th data point of this group in {@code O(n)}.
+     * Where {@code n} is the number of data points in this group.
+     */
+    protected DataPoint getDataPoint(int i) {
+        if (i < 0) {
+            throw new IndexOutOfBoundsException("negative index: " + i);
+        }
+        final int saved_i = i;
+        final SeekableView it = iterator();
+        DataPoint dp = null;
+        while (it.hasNext() && i >= 0) {
+            dp = it.next();
+            i--;
+        }
+        if (i != -1 || dp == null) {
+            throw new IndexOutOfBoundsException("index " + saved_i
+                    + " too large (it's >= " + size() + ") for " + this);
+        }
+        return dp;
+    }
+}

src/core/AggregationIterator.java

@@ -783,7 +783,7 @@ public double nextDoubleValue() {
         r = Double.MAX_VALUE;
         break;
       case MIN:
-        r = Double.MIN_VALUE;
+        r = -Double.MAX_VALUE;
         break;
       case PREV:
         r = y0;

src/core/ByteBufferList.java

@@ -58,6 +58,20 @@ public void add(final byte[] buf, final int offset, final int len) {
     total_length += len;
   }
 
+  /**
+   * Removes the last added segment from the segments array and returns it to the caller
+   *
+   * @return byte array representing the most recently added segment or null if no segments exist
+   */
+  public BufferSegment removeLastSegment() {
+	  if (segments.isEmpty()) {
+		  return null;
+	  }
+	  BufferSegment seg = segments.remove(segments.size() - 1);
+	  total_length -= seg.len;
+	  return seg;
+  }
+
   /**
    * Get the most recently added segment.
    * 
@@ -73,7 +87,7 @@ public byte[] getLastSegment() {
 
   /**
    * Get the number of segments that have added to this buffer list.
-   * 
+   *
    * @return the segment count
    */
   public int segmentCount() {
@@ -82,7 +96,7 @@ public int segmentCount() {
 
   /**
    * Get the accumulated bytes as a single byte array (may be a zero-byte array if empty).
-   * 
+   *
    * @param padding the number of additional bytes to include at the end
    * @return the accumulated bytes
    */

src/core/ColumnDatapointIterator.java

@@ -120,6 +120,19 @@ public void writeToBuffers(ByteBufferList compQualifier, ByteBufferList compValu
     compValue.add(value, value_offset, current_val_length);
   }
 
+  /**
+   * Write a new qualifier and its associated value to the compacted qualifier buffer and compacted values buffer respectively.
+   *
+   * @param newQualifier - the new qualifier
+   * @param newVal - the new value
+   * @param compactedQual - the qualifiers buffer
+   * @param compactedVal - the values buffer
+   */
+  public void writeToBuffers(byte[] newQualifier, byte[] newVal, ByteBufferList compactedQual, ByteBufferList compactedVal) {
+	  compactedQual.add(newQualifier, 0, newQualifier.length);
+	  compactedVal.add(newVal, 0, newVal.length);
+  }
+
   public void writeToBuffersFromOffset(ByteBufferList compQualifier, ByteBufferList compValue, Pair<Integer, Integer> offsets, Pair<Integer, Integer> offsetLengths) {
     compQualifier.add(qualifier, offsets.getKey(), offsetLengths.getKey());
 	compValue.add(value, offsets.getValue(), offsetLengths.getValue());
@@ -186,6 +199,20 @@ private boolean update() {
     return true;
   }
 
+  /**
+   * @return the flags mask from the qualifier
+   */
+  public short getFlagsFromQualifier() {
+	  return Internal.getFlagsFromQualifier(qualifier, qualifier_offset);
+  }
+
+  /**
+   * @return the column timestamp
+   */
+  public long getColumnTimestamp() {
+	  return column_timestamp;
+  }
+
   // order in ascending order by timestamp, descending order by row timestamp (so we find the
   // entry we are going to keep first, and don't have to copy over it)
   @Override