Skip to content

Commit ad771f9

Browse files
zqhZYzqhZY
committed
create dockerfiles
1 parent 3d4a83b commit ad771f9

11 files changed

+554
-0
lines changed

‎docker-flink/.flink-conf.yaml.swp

20 KB
Binary file not shown.

‎docker-flink/Dockerfile

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
################################################################################
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
################################################################################
18+
19+
FROM java:8-jre-alpine
20+
21+
# Install requirements
22+
RUN apk add --no-cache bash snappy
23+
24+
# Configure Flink version
25+
ARG FLINK_VERSION=1.1.1
26+
ARG HADOOP_VERSION=27
27+
ARG SCALA_VERSION=2.11
28+
29+
# Flink environment variables
30+
ARG FLINK_INSTALL_PATH=/opt
31+
ENV FLINK_HOME $FLINK_INSTALL_PATH/flink
32+
ENV PATH $PATH:$FLINK_HOME/bin
33+
34+
# Install build dependencies and flink
35+
RUN set -x && \
36+
mkdir -p $FLINK_INSTALL_PATH && \
37+
apk --update add --virtual build-dependencies curl && \
38+
curl -s $(curl -s https://www.apache.org/dyn/closer.cgi\?preferred\=true)flink/flink-${FLINK_VERSION}/flink-${FLINK_VERSION}-bin-hadoop${HADOOP_VERSION}-scala_${SCALA_VERSION}.tgz | \
39+
tar xvz -C $FLINK_INSTALL_PATH && \
40+
ln -s $FLINK_INSTALL_PATH/flink-$FLINK_VERSION $FLINK_HOME && \
41+
addgroup -S flink && adduser -D -S -H -G flink -h $FLINK_HOME flink && \
42+
chown -R flink:flink $FLINK_INSTALL_PATH/flink-$FLINK_VERSION && \
43+
chown -h flink:flink $FLINK_HOME && \
44+
sed -i -e "s/echo \$mypid >> \$pid/echo \$mypid >> \$pid \&\& wait/g" $FLINK_HOME/bin/flink-daemon.sh && \
45+
apk del build-dependencies && \
46+
rm -rf /var/cache/apk/*
47+
48+
# Configure container
49+
ADD flink-conf.yaml $FLINK_HOME/conf/flink-conf.yaml
50+
#USER flink
51+
ADD docker-entrypoint.sh $FLINK_HOME/bin/
52+
ENTRYPOINT ["docker-entrypoint.sh"]
53+
CMD ["sh", "-c"]

‎docker-flink/README.md

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
Apache Flink cluster deployment on docker using docker-compose
2+
3+
# Installation
4+
5+
Install the most recent stable version of docker
6+
https://docs.docker.com/installation/
7+
8+
Install the most recent stable version of docker-compose
9+
https://docs.docker.com/compose/install/
10+
11+
## Bluemix PaaS
12+
13+
If you want to build the image on Bluemix redirect the docker cli to the remote endpoints. There is plenty of documentation how
14+
to obtain a Bluemix account, so I'm not specifying details. Once you have it and log in using the cloud foundry CLI 'cf' you also
15+
need the container specific plugin 'ic'. With that you can get the URL to the remote docker host as well as the path to
16+
the certificates. If you search for "Logging in to the IBM Containers CLI plug-in" you get the details. <br>Here an example:<br>
17+
export DOCKER_HOST=tcp://containers-api.eu-gb.bluemix.net:8443<br>
18+
export DOCKER_CERT_PATH=/home/markus/.ice/certs/containers-api.eu-gb.bluemix.net/3c63cb44-86d8-4e89-9a40-f8f3f894a09f<br>
19+
export DOCKER_TLS_VERIFY=1<br>
20+
21+
Now when proceeding to the next paragraph 'build' the docker commands build and run the image (provided you use ./docker-compose.sh instead of the native docker-compose command.
22+
Do not forget to allocate and bind a public IP address with the 'cf ic ip' set of commands afterwards.
23+
24+
# Build
25+
26+
Images are based on the official Java Alpine (OpenJDK 8) image. If you want to
27+
build the flink image run:
28+
29+
sh build.sh
30+
31+
or
32+
33+
docker build -t flink .
34+
35+
If you want to build the container for a specific version of flink/hadoop/scala
36+
you can configure it in the respective args:
37+
38+
docker build --build-arg FLINK_VERSION=1.0.3 --build-arg HADOOP_VERSION=26 --build-arg SCALA_VERSION=2.10 -t "flink:1.0.3-hadoop2.6-scala_2.10" flink
39+
40+
# Deploy
41+
42+
- Deploy cluster and see config/setup log output (best run in a screen session)
43+
44+
docker-compose up
45+
46+
- Deploy as a daemon (and return)
47+
48+
docker-compose up -d
49+
50+
- Scale the cluster up or down to *N* TaskManagers
51+
52+
docker-compose scale taskmanager=<N>
53+
54+
- Access the Job Manager container
55+
56+
docker exec -it $(docker ps --filter name=flink_jobmanager --format={{.ID}}) /bin/sh
57+
58+
- Kill the cluster
59+
60+
docker-compose kill
61+
62+
- Upload jar to the cluster
63+
64+
docker cp <your_jar> $(docker ps --filter name=flink_jobmanager --format={{.ID}}):/<your_path>
65+
66+
- Copy file to all the nodes in the cluster
67+
68+
for i in $(docker ps --filter name=flink --format={{.ID}}); do
69+
docker cp <your_file> $i:/<your_path>
70+
done
71+
72+
- Run a topology
73+
74+
From the jobmanager:
75+
76+
docker exec -it $(docker ps --filter name=flink_jobmanager --format={{.ID}}) flink run -m <jobmanager:port> -c <your_class> <your_jar> <your_params>
77+
78+
If you have a local flink installation:
79+
80+
$FLINK_HOME/bin/flink run -m <jobmanager:port> <your_jar>
81+
82+
or
83+
84+
$FLINK_HOME/bin/flink run -m <jobmanager:port> -c <your_class> <your_jar> <your_params>
85+
86+
### Ports
87+
88+
- The Web Client is on port `48081`
89+
- JobManager RPC port `6123` (default, not exposed to host)
90+
- TaskManagers RPC port `6122` (default, not exposed to host)
91+
- TaskManagers Data port `6121` (default, not exposed to host)
92+
93+
Edit the `docker-compose.yml` file to edit port settings.

‎docker-flink/build.sh

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/sh
2+
3+
################################################################################
4+
# Licensed to the Apache Software Foundation (ASF) under one
5+
# or more contributor license agreements. See the NOTICE file
6+
# distributed with this work for additional information
7+
# regarding copyright ownership. The ASF licenses this file
8+
# to you under the Apache License, Version 2.0 (the
9+
# "License"); you may not use this file except in compliance
10+
# with the License. You may obtain a copy of the License at
11+
#
12+
# http://www.apache.org/licenses/LICENSE-2.0
13+
#
14+
# Unless required by applicable law or agreed to in writing, software
15+
# distributed under the License is distributed on an "AS IS" BASIS,
16+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
# See the License for the specific language governing permissions and
18+
# limitations under the License.
19+
################################################################################
20+
21+
docker build -t "flink" .

‎docker-flink/docker-compose.yml

+85
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
################################################################################
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
################################################################################
18+
19+
version: "2"
20+
services:
21+
jobmanager:
22+
image: flink
23+
ports:
24+
- "48085:8081"
25+
expose:
26+
- "6123"
27+
command: jobmanager
28+
depends_on:
29+
- zookeeper
30+
- hadoop
31+
links:
32+
- "zookeeper"
33+
- "hadoop"
34+
jobmanager2:
35+
image: flink
36+
ports:
37+
- "48086:8081"
38+
expose:
39+
- "6123"
40+
command: jobmanager
41+
depends_on:
42+
- zookeeper
43+
- hadoop
44+
links:
45+
- "zookeeper"
46+
- "hadoop"
47+
taskmanager:
48+
image: flink
49+
expose:
50+
- "6121"
51+
- "6122"
52+
depends_on:
53+
- jobmanager
54+
- jobmanager2
55+
- zookeeper
56+
- hadoop
57+
command: taskmanager
58+
links:
59+
- "jobmanager"
60+
- "zookeeper"
61+
zookeeper:
62+
image: zookeeper
63+
ports:
64+
- "2181:2181"
65+
expose:
66+
- "2181"
67+
hadoop:
68+
image: sequenceiq/hadoop-docker:2.7.1
69+
ports:
70+
- "50070:50070"
71+
- "8088:8088"
72+
- "9000:9000"
73+
- "8020:8020"
74+
expose:
75+
- "8020"
76+
- "8042"
77+
- "8088"
78+
- "9000"
79+
- "10020"
80+
- "19888"
81+
- "50010"
82+
- "50020"
83+
- "50070"
84+
- "50075"
85+
- "50090"

‎docker-flink/docker-entrypoint.sh

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/bin/sh
2+
3+
################################################################################
4+
# Licensed to the Apache Software Foundation (ASF) under one
5+
# or more contributor license agreements. See the NOTICE file
6+
# distributed with this work for additional information
7+
# regarding copyright ownership. The ASF licenses this file
8+
# to you under the Apache License, Version 2.0 (the
9+
# "License"); you may not use this file except in compliance
10+
# with the License. You may obtain a copy of the License at
11+
#
12+
# http://www.apache.org/licenses/LICENSE-2.0
13+
#
14+
# Unless required by applicable law or agreed to in writing, software
15+
# distributed under the License is distributed on an "AS IS" BASIS,
16+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
# See the License for the specific language governing permissions and
18+
# limitations under the License.
19+
################################################################################
20+
21+
if [ "$1" = "jobmanager" ]; then
22+
echo "Starting Job Manager"
23+
#sed -i -e "s/jobmanager.rpc.address: localhost/jobmanager.rpc.address: `hostname -f`/g" $FLINK_HOME/conf/flink-conf.yaml
24+
25+
# make use of container linking and exploit the jobmanager entry in /etc/hosts
26+
sed -i -e "s/jobmanager.rpc.address: localhost/jobmanager.rpc.address: jobmanager/g" $FLINK_HOME/conf/flink-conf.yaml
27+
28+
echo "config file: " && grep '^[^\n#]' $FLINK_HOME/conf/flink-conf.yaml
29+
$FLINK_HOME/bin/jobmanager.sh start cluster
30+
elif [ "$1" = "taskmanager" ]; then
31+
32+
# make use of container linking and exploit the jobmanager entry in /etc/hosts
33+
sed -i -e "s/jobmanager.rpc.address: localhost/jobmanager.rpc.address: jobmanager/g" $FLINK_HOME/conf/flink-conf.yaml
34+
35+
sed -i -e "s/taskmanager.numberOfTaskSlots: 1/taskmanager.numberOfTaskSlots: `grep -c ^processor /proc/cpuinfo`/g" $FLINK_HOME/conf/flink-conf.yaml
36+
37+
echo "Starting Task Manager"
38+
echo "config file: " && grep '^[^\n#]' $FLINK_HOME/conf/flink-conf.yaml
39+
$FLINK_HOME/bin/taskmanager.sh start
40+
else
41+
$@
42+
fi

0 commit comments

Comments
 (0)