Skip to content

Commit 15a5e01

Browse files
anthonyrousselanthonyroussel
authored
Upgrade Thrift to 0.14.1 and Hive to v3.1.3 (#70)
* Upgrade Thrift to 0.14.1 and Hive to v3.1.3 * Upgrade dependencies of Dockerfile build
1 parent bf1c180 commit 15a5e01

File tree

9 files changed

+11831
-5555
lines changed

9 files changed

+11831
-5555
lines changed

‎docker/Dockerfile

+25-24
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
1-
FROM ubuntu:16.04
2-
1+
FROM ubuntu:22.04
2+
33
RUN apt-get update && apt-get install -y --no-install-recommends \
4-
curl wget \
5-
python3.5 \
4+
curl wget procps \
5+
python3 \
66
python3-pip \
77
git \
88
vim \
9-
openjdk-8-jdk \
10-
net-tools
9+
openjdk-8-jdk-headless \
10+
net-tools \
11+
&& rm -rf /var/lib/apt/lists/*
1112

1213
# Install Hadoop.
1314
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
14-
ENV HADOOP_VERSION 3.2.0
15+
ENV HADOOP_VERSION 3.3.3
1516
ENV HADOOP_URL https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz
1617
RUN set -x \
1718
&& curl -fsSL "$HADOOP_URL" -o /tmp/hadoop.tar.gz \
@@ -71,10 +72,10 @@ RUN chmod a+x /entrypoint.sh
7172
# install JPam for PAM authentication
7273
RUN wget https://jaist.dl.sourceforge.net/project/jpam/jpam/jpam-1.1/JPam-Linux_amd64-1.1.tgz && \
7374
tar zxf JPam-Linux_amd64-1.1.tgz && \
74-
cp JPam-1.1/libjpam.so /opt/hadoop-3.2.0/lib/native && \
75+
cp JPam-1.1/libjpam.so /opt/hadoop-3.3.3/lib/native && \
7576
rm -rf JPam-Linux_amd64-1.1.tgz
7677
RUN useradd sqlflow && echo 'sqlflow:sqlflow' | chpasswd
77-
78+
7879
ENTRYPOINT ["/entrypoint.sh"]
7980

8081
RUN mkdir /cmd
@@ -111,7 +112,7 @@ RUN chmod a+x /cmd/start_nodemanager.sh
111112

112113
# For Hive
113114
ARG HIVE_VERSION
114-
ENV HIVE_VERSION=${HIVE_VERSION:-2.3.2}
115+
ENV HIVE_VERSION=${HIVE_VERSION:-3.1.3}
115116
ENV HIVE_HOME /opt/hive
116117
ENV PATH $HIVE_HOME/bin:$PATH
117118
ENV HADOOP_HOME /opt/hadoop-$HADOOP_VERSION
@@ -142,21 +143,21 @@ RUN apt-get update && apt-get install -y --no-install-recommends mysql-server
142143
VOLUME /var/lib/mysql
143144

144145
WORKDIR /tmp
145-
RUN wget --quiet https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.47.tar.gz
146-
RUN tar -xzf mysql-connector-java-5.1.47.tar.gz
147-
RUN cp mysql-connector-java-5.1.47/mysql-connector-java-5.1.47.jar /opt/hive/lib/
148-
RUN rm -rf /tmp/mysql-connector-java-5.1.47.tar.gz
149-
RUN rm -rf /tmp/mysql-connector-java-5.1.47
146+
RUN wget --quiet https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.29.tar.gz
147+
RUN tar -xzf mysql-connector-java-8.0.29.tar.gz
148+
RUN cp mysql-connector-java-8.0.29/mysql-connector-java-8.0.29.jar /opt/hive/lib/
149+
RUN rm -rf /tmp/mysql-connector-java-8.0.29.tar.gz
150+
RUN rm -rf /tmp/mysql-connector-java-8.0.29
150151

151152
RUN mkdir /dataset
152153
COPY dataset/popularize_churn.sql /dataset/popularize_churn.sql
153154
COPY dataset/popularize_iris.sql /dataset/popularize_iris.sql
154155
COPY dataset/create_model_db.sql /dataset/create_model_db.sql
155156

156157
# Install the Go compiler.
157-
RUN wget --quiet https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz
158-
RUN tar -C /usr/local -xzf go1.11.5.linux-amd64.tar.gz
159-
RUN rm go1.11.5.linux-amd64.tar.gz
158+
RUN wget --quiet https://dl.google.com/go/go1.18.4.linux-amd64.tar.gz
159+
RUN tar -C /usr/local -xzf go1.18.4.linux-amd64.tar.gz
160+
RUN rm go1.18.4.linux-amd64.tar.gz
160161
RUN apt-get install -y --no-install-recommends build-essential
161162
ENV PATH $PATH:/usr/local/go/bin
162163

@@ -172,20 +173,20 @@ RUN cd / && curl -sL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux
172173
rm mconda-install.sh
173174
ENV PATH="/miniconda/bin:$PATH"
174175

175-
RUN ls /miniconda/bin && /miniconda/bin/conda create -y -q -n sqlflow-dev python=3.6 && \
176+
RUN ls /miniconda/bin && /miniconda/bin/conda create -y -q -n sqlflow-dev python=3.10 && \
176177
echo ". /miniconda/etc/profile.d/conda.sh" >> ~/.bashrc && \
177178
echo "source activate sqlflow-dev" >> ~/.bashrc && \
178179
bash -c "source activate sqlflow-dev && python -m pip install \
179-
tensorflow==2.0.0-alpha0 \
180+
tensorflow==2.9.1 \
180181
mysql-connector-python \
181182
impyla \
182183
jupyter"
183184
# Install protobuf
184-
RUN wget --quiet https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip && \
185+
RUN wget --quiet https://github.com/protocolbuffers/protobuf/releases/download/v21.3/protoc-21.3-linux-x86_64.zip && \
185186
apt-get install -y unzip && \
186-
unzip -qq protoc-3.6.1-linux-x86_64.zip -d /usr/local && \
187-
rm protoc-3.6.1-linux-x86_64.zip && \
188-
go get github.com/golang/protobuf/protoc-gen-go && \
187+
unzip -qq protoc-21.3-linux-x86_64.zip -d /usr/local && \
188+
rm protoc-21.3-linux-x86_64.zip && \
189+
go install google.golang.org/protobuf/cmd/protoc-gen-go@latest && \
189190
mv /go/bin/protoc-gen-go /usr/local/bin/
190191

191192
RUN echo "go get -t sqlflow.org/gohive && go test -v sqlflow.org/gohive" > /build_and_test.bash

‎go.mod

+5-4
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,17 @@ module sqlflow.org/gohive
33
go 1.18
44

55
require (
6-
github.com/apache/thrift v0.12.0
7-
github.com/beltran/gohive v1.3.0
6+
github.com/apache/thrift v0.14.1
7+
github.com/beltran/gohive v1.5.3
88
github.com/stretchr/testify v1.7.1
99
)
1010

1111
require (
12-
github.com/beltran/gosasl v0.0.0-20200715011608-d5475aebb293 // indirect
12+
github.com/beltran/gosasl v0.0.0-20220331024818-65cde6a69a35 // indirect
1313
github.com/beltran/gssapi v0.0.0-20200324152954-d86554db4bab // indirect
1414
github.com/davecgh/go-spew v1.1.0 // indirect
15-
github.com/go-zookeeper/zk v1.0.1 // indirect
15+
github.com/go-zookeeper/zk v1.0.3 // indirect
16+
github.com/pkg/errors v0.9.1 // indirect
1617
github.com/pmezard/go-difflib v1.0.0 // indirect
1718
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
1819
)

‎go.sum

+10-6
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,19 @@
1-
github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs=
2-
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
3-
github.com/beltran/gohive v1.3.0 h1:7e1TGJ/F/mMpoZ1JLevHkoqc0iQnxwEN9y8gn9uKoqU=
4-
github.com/beltran/gohive v1.3.0/go.mod h1:TcPLlZLQbom57zWZpiG25iG9DFbyJgl/W4NbYlqMD5E=
5-
github.com/beltran/gosasl v0.0.0-20200715011608-d5475aebb293 h1:1wRvU44e78w7zJoynLqrXLKSI+VEFEaWmzyq5JdUx7I=
1+
github.com/apache/thrift v0.14.1 h1:Yh8v0hpCj63p5edXOLaqTJW0IJ1p+eMW6+YSOqw1d6s=
2+
github.com/apache/thrift v0.14.1/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
3+
github.com/beltran/gohive v1.5.3 h1:fGvLt4IoVbXMbg4B14igkwCd435ODug95C9tQcbKCmg=
4+
github.com/beltran/gohive v1.5.3/go.mod h1:BJbXAhof7gWk5+kl0y6Ox8TFDJ1xv6gwDksF7l15LFI=
65
github.com/beltran/gosasl v0.0.0-20200715011608-d5475aebb293/go.mod h1:Qx8cW6jkI8riyzmklj80kAIkv+iezFUTBiGU0qHhHes=
6+
github.com/beltran/gosasl v0.0.0-20220331024818-65cde6a69a35 h1:iQwgRNFGmoKDj0bCxzMOxaT7EHFVFArnm20WqHHvuKE=
7+
github.com/beltran/gosasl v0.0.0-20220331024818-65cde6a69a35/go.mod h1:Qx8cW6jkI8riyzmklj80kAIkv+iezFUTBiGU0qHhHes=
78
github.com/beltran/gssapi v0.0.0-20200324152954-d86554db4bab h1:ayfcn60tXOSYy5zUN1AMSTQo4nJCf7hrdzAVchpPst4=
89
github.com/beltran/gssapi v0.0.0-20200324152954-d86554db4bab/go.mod h1:GLe4UoSyvJ3cVG+DVtKen5eAiaD8mAJFuV5PT3Eeg9Q=
910
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
1011
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
11-
github.com/go-zookeeper/zk v1.0.1 h1:LmXNmSnkNsNKai+aDu6sHRr8ZJzIrHJo8z8Z4sm8cT8=
1212
github.com/go-zookeeper/zk v1.0.1/go.mod h1:gpJdHazfkmlg4V0rt0vYeHYJHSL8hHFwV0qOd+HRTJE=
13+
github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg=
14+
github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
15+
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
16+
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
1317
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
1418
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
1519
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

‎hiveserver2/README.md

+9-3
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,18 @@
22

33
For the convenience to access Hive from clients in various languages, the Hive developers created Hive Server, which is a Thrift service. The currently well-used version is known as Hive Server 2.
44

5-
To write a Hive Server 2 client in Go, we need to use the `thrift` command to compile the Thrift service definition file [`TCLIService.thrift`](https://github.com/apache/hive/blob/master/service-rpc/if/TCLIService.thrift) from Hive Server 2 codebase, into Go source code.
5+
To write a Hive Server 2 client in Go, we need to use the `thrift` command to compile the Thrift service definition file [`TCLIService.thrift`](https://github.com/apache/hive/blob/master/service-rpc/if/TCLIService.thrift) from Hive Server 2 codebase, into Go source code:
66

7-
According to their [blog post](https://cwiki.apache.org/confluence/display/Hive/HowToContribute), the Hive developers for some reasons locks the Thrift version to 0.9.3, which is pretty old that you might not want to install it on your computer. Thanks to the Thrift team, who releases Thrift in Docker images and we can use the 0.9.3 version of Docker image for the compilation:
7+
```bash
8+
curl -sS https://raw.githubusercontent.com/apache/hive/rel/release-3.1.3/service-rpc/if/TCLIService.thrift > TCLIService.thrift
9+
```
10+
11+
According to their [blog post](https://cwiki.apache.org/confluence/display/Hive/HowToContribute#HowToContribute-GeneratingThriftCode), the Hive developers recommends to use Thrift v0.14.1 to generate the Hive's auto-generated Thrift code:
812

913
```bash
10-
docker run --rm -it -v $PWD:/work -w /work thrift:0.9.3 thrift -r --gen go TCLIService.thrift
14+
docker run --rm -it -v $PWD:/work -w /work \
15+
anthonyroussel/thrift:0.14.1 \
16+
thrift -r --gen go TCLIService.thrift
1117
```
1218

1319
The above command generates Go source code in the subdirectory `./gen-go/tcliservice`.

0 commit comments

Comments
 (0)