Skip to content

Commit d9c02a2

Browse files
mdesmethashhar
mdesmet
authored and
hashhar
committed
Add development server
1 parent 3e10fd7 commit d9c02a2

15 files changed

+268
-138
lines changed

‎etc/catalog/jmx.properties

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
connector.name=jmx

‎etc/catalog/memory.properties

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
connector.name=memory

‎etc/catalog/tpcds.properties

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
connector.name=tpcds

‎etc/catalog/tpch.properties

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
connector.name=tpch
2+
tpch.splits-per-node=4

‎etc/config-pre-466.properties

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
node.id=coordinator
2+
node.environment=test
3+
4+
coordinator=true
5+
node-scheduler.include-coordinator=true
6+
http-server.http.port=8080
7+
query.max-memory=1GB
8+
discovery.uri=http://localhost:8080
9+
10+
# Disable http request log
11+
http-server.log.enabled=false

‎etc/config.properties

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
node.id=coordinator
2+
node.environment=test
3+
4+
coordinator=true
5+
experimental.concurrent-startup=true
6+
node-scheduler.include-coordinator=true
7+
http-server.http.port=8080
8+
query.max-memory=1GB
9+
discovery.uri=http://localhost:8080
10+
11+
# spooling protocol settings
12+
protocol.spooling.enabled=true
13+
protocol.spooling.shared-secret-key=jxTKysfCBuMZtFqUf8UJDQ1w9ez8rynEJsJqgJf66u0=
14+
protocol.spooling.retrieval-mode=coordinator_proxy
15+
16+
# Disable http request log
17+
http-server.log.enabled=false

‎etc/jvm-pre-466.config

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
-server
2+
-Xmx2G
3+
-XX:G1HeapRegionSize=32M
4+
-XX:+ExplicitGCInvokesConcurrent
5+
-XX:+ExitOnOutOfMemoryError
6+
-XX:+HeapDumpOnOutOfMemoryError
7+
-XX:-OmitStackTraceInFastThrow
8+
-XX:ReservedCodeCacheSize=150M
9+
-XX:PerMethodRecompilationCutoff=10000
10+
-XX:PerBytecodeRecompilationCutoff=10000
11+
-Djdk.attach.allowAttachSelf=true
12+
# jdk.nio.maxCachedBufferSize controls what buffers can be allocated in per-thread "temporary buffer cache" (sun.nio.ch.Util). Value of 0 disables the cache.
13+
-Djdk.nio.maxCachedBufferSize=0
14+
# Allow loading dynamic agent used by JOL
15+
-XX:+EnableDynamicAgentLoading
16+
-XX:+UnlockDiagnosticVMOptions

‎etc/jvm.config

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
-server
2+
-Xmx2G
3+
-XX:G1HeapRegionSize=32M
4+
-XX:+ExplicitGCInvokesConcurrent
5+
-XX:+ExitOnOutOfMemoryError
6+
-XX:+HeapDumpOnOutOfMemoryError
7+
-XX:-OmitStackTraceInFastThrow
8+
-XX:ReservedCodeCacheSize=150M
9+
-XX:PerMethodRecompilationCutoff=10000
10+
-XX:PerBytecodeRecompilationCutoff=10000
11+
-Djdk.attach.allowAttachSelf=true
12+
# jdk.nio.maxCachedBufferSize controls what buffers can be allocated in per-thread "temporary buffer cache" (sun.nio.ch.Util). Value of 0 disables the cache.
13+
-Djdk.nio.maxCachedBufferSize=0
14+
# Allow loading dynamic agent used by JOL
15+
-XX:+EnableDynamicAgentLoading
16+
-XX:+UnlockDiagnosticVMOptions
17+
--enable-native-access=ALL-UNNAMED

‎etc/spooling-manager.properties

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
spooling-manager.name=filesystem
2+
fs.s3.enabled=true
3+
fs.location=s3://spooling/
4+
s3.endpoint=http://localstack:4566/
5+
s3.region=us-east-1
6+
s3.aws-access-key=test
7+
s3.aws-secret-key=test
8+
s3.path-style-access=true

‎setup.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,9 @@
4646
"pre-commit",
4747
"black",
4848
"isort",
49-
"keyring"
49+
"keyring",
50+
"testcontainers",
51+
"boto3"
5052
]
5153

5254
setup(

‎tests/development_server.py

+139
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
import os
2+
import time
3+
from contextlib import contextmanager
4+
from pathlib import Path
5+
6+
from testcontainers.core.container import DockerContainer
7+
from testcontainers.core.network import Network
8+
from testcontainers.core.waiting_utils import wait_for_logs
9+
from testcontainers.localstack import LocalStackContainer
10+
11+
from trino.constants import DEFAULT_PORT
12+
13+
MINIO_ROOT_USER = "minio-access-key"
14+
MINIO_ROOT_PASSWORD = "minio-secret-key"
15+
16+
TRINO_VERSION = os.environ.get("TRINO_VERSION") or "latest"
17+
TRINO_HOST = "localhost"
18+
19+
20+
def create_bucket(s3_client):
21+
bucket_name = "spooling"
22+
try:
23+
print("Checking for bucket existence...")
24+
response = s3_client.list_buckets()
25+
buckets = [bucket["Name"] for bucket in response["Buckets"]]
26+
if bucket_name in buckets:
27+
print("Bucket exists!")
28+
return
29+
except s3_client.exceptions.ClientError as e:
30+
if not e.response['Error']['Code'] == '404':
31+
print("An error occurred:", e)
32+
return
33+
34+
try:
35+
print("Creating bucket...")
36+
s3_client.create_bucket(
37+
Bucket=bucket_name,
38+
)
39+
print("Bucket created!")
40+
except s3_client.exceptions.ClientError as e:
41+
print("An error occurred:", e)
42+
43+
44+
@contextmanager
45+
def start_development_server(port=None, trino_version=TRINO_VERSION):
46+
network = None
47+
localstack = None
48+
trino = None
49+
50+
try:
51+
network = Network().create()
52+
supports_spooling_protocol = TRINO_VERSION == "latest" or int(TRINO_VERSION) >= 466
53+
if supports_spooling_protocol:
54+
localstack = LocalStackContainer(image="localstack/localstack:latest", region_name="us-east-1") \
55+
.with_name("localstack") \
56+
.with_network(network) \
57+
.with_bind_ports(4566, 4566) \
58+
.with_bind_ports(4571, 4571) \
59+
.with_env("SERVICES", "s3")
60+
61+
# Start the container
62+
print("Starting LocalStack container...")
63+
localstack.start()
64+
65+
# Wait for logs indicating MinIO has started
66+
wait_for_logs(localstack, "Ready.", timeout=30)
67+
68+
# create spooling bucket
69+
create_bucket(localstack.get_client("s3"))
70+
71+
trino = DockerContainer(f"trinodb/trino:{trino_version}") \
72+
.with_name("trino") \
73+
.with_network(network) \
74+
.with_env("TRINO_CONFIG_DIR", "/etc/trino") \
75+
.with_bind_ports(DEFAULT_PORT, port)
76+
77+
root = Path(__file__).parent.parent
78+
79+
trino = trino \
80+
.with_volume_mapping(str(root / "etc/catalog"), "/etc/trino/catalog")
81+
82+
# Enable spooling config
83+
if supports_spooling_protocol:
84+
trino \
85+
.with_volume_mapping(
86+
str(root / "etc/spooling-manager.properties"),
87+
"/etc/trino/spooling-manager.properties", "rw") \
88+
.with_volume_mapping(str(root / "etc/jvm.config"), "/etc/trino/jvm.config") \
89+
.with_volume_mapping(str(root / "etc/config.properties"), "/etc/trino/config.properties")
90+
else:
91+
trino \
92+
.with_volume_mapping(str(root / "etc/jvm-pre-466.config"), "/etc/trino/jvm.config") \
93+
.with_volume_mapping(str(root / "etc/config-pre-466.properties"), "/etc/trino/config.properties")
94+
95+
print("Starting Trino container...")
96+
trino.start()
97+
98+
# Wait for logs indicating the service has started
99+
wait_for_logs(trino, "SERVER STARTED", timeout=60)
100+
101+
# Otherwise some tests fail with No nodes available
102+
time.sleep(2)
103+
104+
yield localstack, trino, network
105+
finally:
106+
# Stop containers when exiting the context
107+
if trino:
108+
try:
109+
print("Stopping Trino container...")
110+
trino.stop()
111+
except Exception as e:
112+
print(f"Error stopping Trino container: {e}")
113+
114+
if localstack:
115+
try:
116+
print("Stopping LocalStack container...")
117+
localstack.stop()
118+
except Exception as e:
119+
print(f"Error stopping LocalStack container: {e}")
120+
121+
if network:
122+
try:
123+
print("Removing network...")
124+
network.remove()
125+
except Exception as e:
126+
print(f"Error removing network: {e}")
127+
128+
129+
def main():
130+
"""Run Trino setup independently from pytest."""
131+
with start_development_server(port=DEFAULT_PORT):
132+
print(f"Trino started at {TRINO_HOST}:{DEFAULT_PORT}")
133+
134+
# Keep the process running so that the containers stay up
135+
input("Press Enter to stop containers...")
136+
137+
138+
if __name__ == "__main__":
139+
main()

0 commit comments

Comments
 (0)