Skip to content

Commit 08729e9

Browse files
ryannedolanryannedolan
authored
Update release artifacts (#75)
* Add hoptimator-jdbc-driver and hoptimator-api release artifacts * Fix non-determinism in integration tests * Move Source, Sink, Job, Pipeline to API * Upgrade gradle
1 parent 1b0a57f commit 08729e9

File tree

30 files changed

+266
-140
lines changed

30 files changed

+266
-140
lines changed

‎Makefile

+6-3
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,10 @@ build:
1010
bounce: build undeploy deploy deploy-samples deploy-config deploy-demo
1111

1212
# Integration tests expect K8s and Kafka to be running
13-
integration-tests:
13+
integration-tests: deploy-dev-environment deploy-samples
14+
kubectl wait kafka.kafka.strimzi.io/one --for=condition=Ready --timeout=10m -n kafka
15+
kubectl wait kafkatopic.kafka.strimzi.io/existing-topic-1 --for=condition=Ready --timeout=10m -n kafka
16+
kubectl wait kafkatopic.kafka.strimzi.io/existing-topic-2 --for=condition=Ready --timeout=10m -n kafka
1417
kubectl port-forward -n kafka svc/one-kafka-external-0 9092 & echo $$! > port-forward.pid
1518
./gradlew intTest || kill `cat port-forward.pid`
1619
kill `cat port-forward.pid`
@@ -50,8 +53,8 @@ deploy-dev-environment: deploy-config
5053
kubectl apply -f ./deploy/samples/kafkadb.yaml
5154

5255
undeploy-dev-environment:
53-
kubectl delete $(shell kubectl get kafkatopic.kafka.strimzi.io -o name -n kafka) -n kafka || echo "skipping"
54-
kubectl delete $(shell kubectl get strimzi -o name -n kafka) -n kafka || echo "skipping"
56+
kubectl delete kafkatopic.kafka.strimzi.io -n kafka --all || echo "skipping"
57+
kubectl delete strimzi -n kafka --all || echo "skipping"
5558
kubectl delete pvc -l strimzi.io/name=one-kafka -n kafka || echo "skipping"
5659
kubectl delete -f "https://strimzi.io/install/latest?namespace=kafka" -n kafka || echo "skipping"
5760
kubectl delete -f ./deploy/samples/kafkadb.yaml || echo "skipping"

‎gradle/wrapper/gradle-wrapper.jar

-17.6 KB
Binary file not shown.

‎gradle/wrapper/gradle-wrapper.properties

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
distributionBase=GRADLE_USER_HOME
22
distributionPath=wrapper/dists
3-
distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.2-bin.zip
3+
distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip
44
networkTimeout=10000
5+
validateDistributionUrl=true
56
zipStoreBase=GRADLE_USER_HOME
67
zipStorePath=wrapper/dists

‎gradlew

+21-13
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
# See the License for the specific language governing permissions and
1616
# limitations under the License.
1717
#
18+
# SPDX-License-Identifier: Apache-2.0
19+
#
1820

1921
##############################################################################
2022
#
@@ -55,7 +57,7 @@
5557
# Darwin, MinGW, and NonStop.
5658
#
5759
# (3) This script is generated from the Groovy template
58-
# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
60+
# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
5961
# within the Gradle project.
6062
#
6163
# You can find Gradle at https://github.com/gradle/gradle/.
@@ -83,10 +85,9 @@ done
8385
# This is normally unused
8486
# shellcheck disable=SC2034
8587
APP_BASE_NAME=${0##*/}
86-
APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
87-
88-
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
89-
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
88+
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
89+
APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
90+
' "$PWD" ) || exit
9091

9192
# Use the maximum available, or set MAX_FD != -1 to use that value.
9293
MAX_FD=maximum
@@ -133,26 +134,29 @@ location of your Java installation."
133134
fi
134135
else
135136
JAVACMD=java
136-
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
137+
if ! command -v java >/dev/null 2>&1
138+
then
139+
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
137140
138141
Please set the JAVA_HOME variable in your environment to match the
139142
location of your Java installation."
143+
fi
140144
fi
141145

142146
# Increase the maximum file descriptors if we can.
143147
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
144148
case $MAX_FD in #(
145149
max*)
146150
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
147-
# shellcheck disable=SC3045
151+
# shellcheck disable=SC2039,SC3045
148152
MAX_FD=$( ulimit -H -n ) ||
149153
warn "Could not query maximum file descriptor limit"
150154
esac
151155
case $MAX_FD in #(
152156
'' | soft) :;; #(
153157
*)
154158
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
155-
# shellcheck disable=SC3045
159+
# shellcheck disable=SC2039,SC3045
156160
ulimit -n "$MAX_FD" ||
157161
warn "Could not set maximum file descriptor limit to $MAX_FD"
158162
esac
@@ -197,11 +201,15 @@ if "$cygwin" || "$msys" ; then
197201
done
198202
fi
199203

200-
# Collect all arguments for the java command;
201-
# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
202-
# shell script including quotes and variable substitutions, so put them in
203-
# double quotes to make sure that they get re-expanded; and
204-
# * put everything else in single quotes, so that it's not re-expanded.
204+
205+
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
206+
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
207+
208+
# Collect all arguments for the java command:
209+
# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
210+
# and any embedded shellness will be escaped.
211+
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
212+
# treated as '${Hostname}' itself on the command line.
205213

206214
set -- \
207215
"-Dorg.gradle.appname=$APP_BASE_NAME" \

‎gradlew.bat

+12-10
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
@rem See the License for the specific language governing permissions and
1414
@rem limitations under the License.
1515
@rem
16+
@rem SPDX-License-Identifier: Apache-2.0
17+
@rem
1618

1719
@if "%DEBUG%"=="" @echo off
1820
@rem ##########################################################################
@@ -43,11 +45,11 @@ set JAVA_EXE=java.exe
4345
%JAVA_EXE% -version >NUL 2>&1
4446
if %ERRORLEVEL% equ 0 goto execute
4547

46-
echo.
47-
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
48-
echo.
49-
echo Please set the JAVA_HOME variable in your environment to match the
50-
echo location of your Java installation.
48+
echo. 1>&2
49+
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
50+
echo. 1>&2
51+
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
52+
echo location of your Java installation. 1>&2
5153

5254
goto fail
5355

@@ -57,11 +59,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe
5759

5860
if exist "%JAVA_EXE%" goto execute
5961

60-
echo.
61-
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
62-
echo.
63-
echo Please set the JAVA_HOME variable in your environment to match the
64-
echo location of your Java installation.
62+
echo. 1>&2
63+
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
64+
echo. 1>&2
65+
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
66+
echo location of your Java installation. 1>&2
6567

6668
goto fail
6769

‎hoptimator-api/build.gradle

+46
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,53 @@
11
plugins {
22
id 'java'
3+
id 'maven-publish'
34
}
45

56
dependencies {
67
// plz keep it this way
78
}
9+
10+
publishing {
11+
repositories {
12+
maven {
13+
name 'GitHubPackages'
14+
url = 'https://maven.pkg.github.com/linkedin/Hoptimator'
15+
credentials {
16+
username = System.getenv('GITHUB_ACTOR')
17+
password = System.getenv('GITHUB_TOKEN')
18+
}
19+
}
20+
maven {
21+
name 'LinkedInJFrog'
22+
url 'https://linkedin.jfrog.io/artifactory/hoptimator'
23+
credentials {
24+
username = System.getenv('JFROG_USERNAME')
25+
password = System.getenv('JFROG_API_KEY')
26+
}
27+
}
28+
}
29+
publications {
30+
maven(MavenPublication) {
31+
groupId = 'com.linkedin.hoptimator'
32+
artifactId = 'hoptimator-api'
33+
version = System.getenv('VERSION')
34+
from components.java
35+
pom {
36+
name = 'hoptimator-api'
37+
description = 'API for extending Hoptimator'
38+
url = 'https://github.com/linkedin/Hoptimator'
39+
licenses {
40+
license {
41+
name = 'BSD 2-Clause'
42+
url = 'https://raw.githubusercontent.com/linkedin/Hoptimator/main/LICENSE'
43+
}
44+
}
45+
scm {
46+
connection = 'scm:git:git://github.com:linkedin/Hoptimator.git'
47+
developerConnection = 'scm:git:ssh://github.com:linkedin/Hoptimator.git'
48+
url = 'https://github.com/linkedin/Hoptimator'
49+
}
50+
}
51+
}
52+
}
53+
}

‎hoptimator-util/src/main/java/com/linkedin/hoptimator/util/Job.java renamed to ‎hoptimator-api/src/main/java/com/linkedin/hoptimator/Job.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
1-
package com.linkedin.hoptimator.util;
1+
package com.linkedin.hoptimator;
22

33
import java.util.function.Function;
44

5-
import org.apache.calcite.sql.SqlDialect;
6-
75

86
public class Job {
97

‎hoptimator-util/src/main/java/com/linkedin/hoptimator/util/MaterializedView.java renamed to ‎hoptimator-api/src/main/java/com/linkedin/hoptimator/MaterializedView.java

+1-5
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,9 @@
1-
package com.linkedin.hoptimator.util;
1+
package com.linkedin.hoptimator;
22

33
import java.util.List;
44
import java.util.function.Function;
55
import java.util.stream.Collectors;
66

7-
import org.apache.calcite.sql.SqlDialect;
8-
9-
import com.linkedin.hoptimator.util.planner.Pipeline;
10-
117

128
public class MaterializedView {
139

‎hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/Pipeline.java renamed to ‎hoptimator-api/src/main/java/com/linkedin/hoptimator/Pipeline.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
1-
package com.linkedin.hoptimator.util.planner;
1+
package com.linkedin.hoptimator;
22

33
import java.sql.SQLException;
44
import java.util.ArrayList;
55
import java.util.List;
66

7-
import com.linkedin.hoptimator.Deployable;
8-
97

108
/**
119
* A set of Deployable objects that work together to deliver data.

‎hoptimator-api/src/main/java/com/linkedin/hoptimator/Sink.java

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package com.linkedin.hoptimator;
2+
3+
import java.util.List;
4+
import java.util.Map;
5+
6+
7+
public class Sink extends Source {
8+
9+
public Sink(String database, List<String> path, Map<String, String> options) {
10+
super(database, path, options);
11+
}
12+
13+
@Override
14+
public String toString() {
15+
return "Sink[" + pathString() + "]";
16+
}
17+
}

‎hoptimator-util/src/main/java/com/linkedin/hoptimator/util/Source.java renamed to ‎hoptimator-api/src/main/java/com/linkedin/hoptimator/Source.java

+2-10
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,19 @@
1-
package com.linkedin.hoptimator.util;
1+
package com.linkedin.hoptimator;
22

33
import java.util.List;
44
import java.util.Map;
55
import java.util.stream.Collectors;
66

7-
import org.apache.calcite.rel.type.RelDataType;
8-
97

108
public class Source {
119

1210
private final String database;
1311
private final List<String> path;
14-
private final RelDataType rowType;
1512
private final Map<String, String> options;
1613

17-
public Source(String database, List<String> path, RelDataType rowType, Map<String, String> options) {
14+
public Source(String database, List<String> path, Map<String, String> options) {
1815
this.database = database;
1916
this.path = path;
20-
this.rowType = rowType;
2117
this.options = options;
2218
}
2319

@@ -42,10 +38,6 @@ public List<String> path() {
4238
return path;
4339
}
4440

45-
public RelDataType rowType() {
46-
return rowType;
47-
}
48-
4941
protected String pathString() {
5042
return path.stream().collect(Collectors.joining("."));
5143
}

‎hoptimator-api/src/main/java/com/linkedin/hoptimator/SqlDialect.java

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package com.linkedin.hoptimator;
2+
3+
4+
public enum SqlDialect {
5+
ANSI,
6+
FLINK
7+
}

‎hoptimator-avro/build.gradle

-2
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@ plugins {
22
id 'java'
33
}
44

5-
6-
75
dependencies {
86
implementation project(':hoptimator-api')
97
implementation libs.avro

‎hoptimator-cli/src/main/java/sqlline/HoptimatorAppConfig.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@
99

1010
import org.apache.calcite.jdbc.CalciteConnection;
1111
import org.apache.calcite.rel.RelNode;
12-
import org.apache.calcite.sql.dialect.AnsiSqlDialect;
1312
import org.jline.reader.Completer;
1413

14+
import com.linkedin.hoptimator.SqlDialect;
1515
import com.linkedin.hoptimator.jdbc.HoptimatorDriver;
1616
import com.linkedin.hoptimator.util.DeploymentService;
1717
import com.linkedin.hoptimator.util.planner.PipelineRel;
@@ -89,7 +89,7 @@ public void execute(String line, DispatchCallback dispatchCallback) {
8989
try {
9090
RelNode rel = HoptimatorDriver.convert(conn.createPrepareContext(), sql).root.rel;
9191
PipelineRel.Implementor plan = DeploymentService.plan(rel);
92-
sqlline.output(plan.sql().apply(AnsiSqlDialect.DEFAULT));
92+
sqlline.output(plan.sql().apply(SqlDialect.ANSI));
9393
} catch (SQLException e) {
9494
sqlline.error(e);
9595
dispatchCallback.setToFailure();

0 commit comments

Comments
 (0)