From f8ecad319e44a07b90440055bce7c7538dbc239b Mon Sep 17 00:00:00 2001 From: Cyber-SiKu Date: Tue, 27 Sep 2022 16:52:09 +0800 Subject: [PATCH] pfs: build in centos7 Signed-off-by: Cyber-SiKu --- docker/curve/1.2/centos7/Dockerfile | 321 ++++++++++++++++++++++++---- docker/curve/1.2/centos7/Makefile | 8 +- 2 files changed, 282 insertions(+), 47 deletions(-) diff --git a/docker/curve/1.2/centos7/Dockerfile b/docker/curve/1.2/centos7/Dockerfile index a938d23..95e814d 100644 --- a/docker/curve/1.2/centos7/Dockerfile +++ b/docker/curve/1.2/centos7/Dockerfile @@ -1,48 +1,281 @@ FROM centos:7 as fiu LABEL maintainer="Cyber-SiKu " -# ADD https://mirrors.163.com/.help/CentOS7-Base-163.repo /etc/yum.repos.d/CentOS-Base.repo -RUN yum clean all && yum makecache \ - && yum install -y wget \ - gcc \ - python3 \ - make \ - && wget https://github.com/albertito/libfiu/archive/refs/tags/1.00.tar.gz \ - && tar -xzvf 1.00.tar.gz \ - && cd libfiu-1.00 \ - && make && make install +RUN yum install -y \ + centos-release-scl \ + centos-release-scl-rh&& \ + yum update -y && \ + yum clean all +ADD https://curve-build.nos-eastchina1.126.net/libfiu-1.00.tar.gz / +RUN yum install -y \ + devtoolset-9-gcc \ + devtoolset-9-gcc-c++ \ + devtoolset-9-libstdc++-devel \ + devtoolset-9-make \ + python3 && \ + echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc && \ + source /etc/bashrc && \ + tar -xzvf libfiu-1.00.tar.gz && rm libfiu-1.00.tar.gz && \ + cd libfiu-1.00 && \ + make && make install -FROM centos:7 as builder - -COPY --from=fiu /usr/local/include/fiu*.h /usr/local/include/ -COPY --from=fiu /usr/local/lib/ /usr/local/lib -RUN yum clean all && yum makecache \ - && yum install -y wget \ - make \ - patch \ - gcc \ - gcc-c++ \ - clang \ - zlib-devel \ - openssl-devel \ - libuuid-devel \ - libnl3-devel \ - libcurl-devel \ - python-devel \ +FROM centos:7 AS curve-builder +LABEL maintainer="Cyber-SiKu " +COPY --from=fiu /usr/local/include/fiu*.h /usr/include/ +COPY --from=fiu /usr/local/lib/ /usr/lib +ARG TARGETARCH +RUN yum install -y \ + wget \ + patch \ + zlib-devel \ + openssl-devel \ + libuuid-devel \ + libnl3-devel \ + libcurl-devel \ + python-devel \ + libunwind-devel \ + fuse3-devel \ + sudo \ + lz4-devel \ + snappy-devel \ + centos-release-scl \ + centos-release-scl-rh \ + which && \ + if [ "$TARGETARCH" = "amd64" ]; then \ + BAZEL_ARCH=x86_64; \ + elif [ "$TARGETARCH" = "arm64" ]; then \ + BAZEL_ARCH=arm64; \ + else \ + BAZEL_ARCH=x86_64; \ + fi && \ + wget https://curve-build.nos-eastchina1.126.net/bazelbuild/bazelisk/bazelisk-linux-${BAZEL_ARCH} && \ + mv bazelisk-linux-${BAZEL_ARCH} /usr/bin/bazelisk && \ + ln -s /usr/bin/bazelisk /usr/bin/bazel && \ + wget https://curve-build.nos-eastchina1.126.net/bazelbuild/4.2.2/bazel-4.2.2-linux-${BAZEL_ARCH} && \ + mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-${BAZEL_ARCH}/bin && \ + mv bazel-4.2.2-linux-${BAZEL_ARCH} /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-${BAZEL_ARCH}/bin/bazel && \ + wget https://curve-build.nos-eastchina1.126.net/bazelbuild/0.17.2/bazel-0.17.2-linux-${BAZEL_ARCH} && \ + mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-${BAZEL_ARCH}/bin && \ + mv bazel-0.17.2-linux-${BAZEL_ARCH} /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-${BAZEL_ARCH}/bin/bazel && \ + chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-${BAZEL_ARCH}/bin/bazel /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-${BAZEL_ARCH}/bin/bazel /usr/bin/bazel /usr/bin/bazelisk && \ + if [ "$TARGETARCH" = "arm64" ]; then \ + yum install -y epel-release && \ + yum install -y \ + rh-git227-git \ + python2-pip \ python-setuptools \ - python3-devel \ - python3-setuptools \ - python3-wheel \ - libunwind-devel \ - sudo \ - && wget https://curve-build.nos-eastchina1.126.net/bazelbuild/bazelisk/bazelisk-linux-amd64 \ - && mv bazelisk-linux-amd64 /usr/bin/bazelisk \ - && ln -s /usr/bin/bazelisk /usr/bin/bazel \ - && wget https://curve-build.nos-eastchina1.126.net/bazelbuild/4.2.2/bazel-4.2.2-linux-x86_64 \ - && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin \ - && mv bazel-4.2.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel \ - && wget https://curve-build.nos-eastchina1.126.net/bazelbuild/0.17.2/bazel-0.17.2-linux-x86_64\ - && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin \ - && mv bazel-0.17.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin/bazel \ - && chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin/bazel /usr/bin/bazel /usr/bin/bazelisk \ - && yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm \ - && yum install -y git + python2-wheel \ + llvm-toolset-7.0-cmake \ + java-1.8.0-openjdk \ + java-1.8.0-openjdk-devel && \ + echo "export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.352.b08-2.el7_9.aarch64" >> /etc/bashrc && \ + echo "source /opt/rh/llvm-toolset-7.0/enable" >> /etc/bashrc && \ + echo "source /opt/rh/rh-git227/enable" >> /etc/bashrc && \ + echo "source /opt/rh/httpd24/enable" >> /etc/bashrc; \ + else \ + yum install -y \ + sclo-git212 \ + python27-python-pip \ + python27-python-setuptools \ + llvm-toolset-7-cmake \ + python27-python-wheel && \ + echo "source /opt/rh/sclo-git212/enable" >> /etc/bashrc && \ + echo "source /opt/rh/llvm-toolset-7/enable" >> /etc/bashrc && \ + echo "source /opt/rh/python27/enable" >> /etc/bashrc; \ + fi && \ + yum install -y \ + devtoolset-9-gcc \ + devtoolset-9-gcc-c++ \ + devtoolset-9-libstdc++-devel \ + devtoolset-9-make \ + llvm-toolset-7.0-cmake && \ + echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc && \ + source /etc/bashrc && \ + pip install pyinstaller==3.2 -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com + +FROM curve-builder AS curve-1.2.5 +LABEL maintainer="Cyber-SiKu " +# ENV GITHUB_PROXY=https://ghproxy.com/ +ENV GITHUB_PROXY= +ARG TARGETARCH +RUN source /etc/bashrc && \ + git clone ${GITHUB_PROXY}https://github.com/opencurve/curve --branch v1.2.5 /curve-1.2.5 && \ + cd /curve-1.2.5/ && \ + # git init && \ + if [ "$TARGETARCH" = "arm64" ]; then \ + git config --global core.autocrlf input; \ + curl https://curve-build.nos-eastchina1.126.net/arm64.patch | git apply; \ + sed -i "s;0.17.2;0.17.2-;g" mk-tar.sh; \ + else \ + sed -i "s;/usr/bin/python2;/opt/rh/python27/root/usr/bin/python2;g" mk-tar.sh; \ + fi && \ + sed -i "s;tag_version=9.9.9;tag_version=1.2.5;g" mk-tar.sh && \ + sed -i "s;commit_id=`git show --abbrev-commit HEAD|head -n 1|awk '{print \$2}'`;commit_id=\"86e8e1a9\";g" mk-tar.sh && \ + bash replace-curve-repo.sh && bash mk-tar.sh release && mv curve_1.2.5+86e8e1a9.tar.gz / + +FROM centos:7 AS polardb-builder +LABEL maintainer="Cyber-SiKu " +CMD bash +# avoid missing locale problem +RUN sed -i 's/override_install_langs/# &/' /etc/yum.conf +# add EPEL and scl source +RUN rpmkeys --import file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 && \ + yum install -y epel-release centos-release-scl && \ + rpmkeys --import file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7 && \ + rpmkeys --import file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-SCLo && \ + yum update -y && \ + yum clean all +# GCC and LLVM +RUN yum install -y \ + devtoolset-9-gcc \ + devtoolset-9-gcc-c++ \ + devtoolset-9-gdb \ + devtoolset-9-libstdc++-devel \ + devtoolset-9-make \ + llvm-toolset-7.0-llvm-devel \ + llvm-toolset-7.0-clang-devel \ + llvm-toolset-7.0-cmake && \ + yum clean all +# dependencies +RUN yum install -y \ + libicu-devel \ + pam-devel \ + readline-devel \ + libxml2-devel \ + libxslt-devel \ + openldap-devel \ + openldap-clients \ + openldap-servers \ + libuuid-devel \ + xerces-c-devel \ + bison \ + flex \ + gettext \ + tcl-devel \ + python-devel \ + perl-IPC-Run \ + perl-Expect \ + perl-Test-Simple \ + perl-DBD-Pg \ + perl-ExtUtils-Embed \ + perl-ExtUtils-MakeMaker \ + zlib-devel \ + krb5-devel \ + krb5-workstation \ + krb5-server \ + protobuf-devel \ + libaio-devel \ + fuse-devel \ + gtest-devel && \ + ln /usr/lib64/perl5/CORE/libperl.so /usr/lib64/libperl.so && \ + yum clean all +# install basic tools +RUN yum install -y \ + git lcov psmisc sudo vim \ + less \ + net-tools \ + python2-psycopg2 \ + python2-requests \ + tar \ + shadow-utils \ + which \ + binutils\ + libtool \ + perf \ + make sudo \ + wget \ + util-linux libunwind && \ + yum clean all + +# copy curve-sdk +COPY --from=curve-1.2.5 /curve_1.2.5+86e8e1a9.tar.gz / +RUN tar -zxvf curve_1.2.5+86e8e1a9.tar.gz && rm curve_1.2.5+86e8e1a9.tar.gz && \ + cd /curve/curve-sdk && \ + cp -f lib/* /usr/lib64 && \ + cp -f bin/* /usr/bin && \ + mkdir -p /usr/curvefs && \ + cp -f curvefs/* /usr/curvefs && \ + cp -f include/* /usr/include && \ + rm -rf /curve && \ + mkdir -p /etc/curve +COPY --from=curve-1.2.5 /curve-1.2.5/conf/client.conf /etc/curve/client.conf + +# set to empty if GitHub is not barriered +# ENV GITHUB_PROXY=https://ghproxy.com/ +ENV GITHUB_PROXY= +ENV ZLOG_VERSION=1.2.14 +ENV OPENSSL_VERSION=OpenSSL_1_1_1k +ENV PFSD_VERSION=curvebs_sdk_devio +# install dependencies from GitHub mirror +RUN cd /usr/local && \ + # zlog for PFSD + wget --no-verbose --no-check-certificate "${GITHUB_PROXY}https://github.com/HardySimpson/zlog/archive/refs/tags/${ZLOG_VERSION}.tar.gz" && \ + # PFSD + wget --no-verbose --no-check-certificate "${GITHUB_PROXY}https://github.com/opencurve/PolarDB-FileSystem/archive/refs/heads/${PFSD_VERSION}.tar.gz" && \ + # OpenSSL 1.1.1 + wget --no-verbose --no-check-certificate "${GITHUB_PROXY}https://github.com/openssl/openssl/archive/refs/tags/${OPENSSL_VERSION}.tar.gz" && \ + # enable build tools + echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc && \ + echo "source /opt/rh/llvm-toolset-7.0/enable" >> /etc/bashrc && \ + source /etc/bashrc && \ + # unzip and install zlog + gzip -d $ZLOG_VERSION.tar.gz && \ + tar xpf $ZLOG_VERSION.tar && \ + cd zlog-$ZLOG_VERSION && \ + make && make install && \ + echo '/usr/local/lib' >> /etc/ld.so.conf && ldconfig && \ + cd .. && \ + rm -rf $ZLOG_VERSION* && \ + rm -rf zlog-$ZLOG_VERSION && \ + # unzip and install OpenSSL 1.1.1 + gzip -d $OPENSSL_VERSION.tar.gz && \ + tar xpf $OPENSSL_VERSION.tar && \ + cd openssl-$OPENSSL_VERSION && \ + ./config --prefix=/usr/local/openssl && make -j8 && make install && \ + cp /usr/local/openssl/lib/lib* /usr/local/lib/ && \ + cp -r /usr/local/openssl/include/openssl /usr/local/include/ && \ + cd .. && \ + rm -rf $OPENSSL_VERSION* && \ + rm -rf openssl-$OPENSSL_VERSION && \ + ldconfig && \ + # unzip and install PFSD + gzip -d $PFSD_VERSION.tar.gz && \ + tar xpf $PFSD_VERSION.tar && \ + cd PolarDB-FileSystem-$PFSD_VERSION && \ + sed -i 's/-march=native //' CMakeLists.txt && \ + ./autobuild.sh && ./install.sh && \ + cd .. && \ + rm -rf $PFSD_VERSION* && \ + rm -rf PolarDB-FileSystem-$PFSD_VERSION* +# add software repository of Node.js LTS +# @see https://github.com/nodesource/distributions +# use yarn for documentation development +RUN curl -fsSL https://rpm.nodesource.com/setup_16.x | bash - && \ + curl -sL https://dl.yarnpkg.com/rpm/yarn.repo | tee /etc/yum.repos.d/yarn.repo && \ + yum install -y yarn && \ + yum clean all +# create default user +ENV USER_NAME=postgres +RUN echo "create default user" && \ + groupadd -r $USER_NAME && \ + useradd -g $USER_NAME $USER_NAME -p '' && \ + usermod -aG wheel $USER_NAME +WORKDIR /home/$USER_NAME +# modify conf +RUN echo "modify conf" && \ + mkdir -p /var/log/pfs && chown $USER_NAME /var/log/pfs && \ + mkdir -p /var/run/pfs && chown $USER_NAME /var/run/pfs && \ + mkdir -p /var/run/pfsd && chown $USER_NAME /var/run/pfsd && \ + mkdir -p /dev/shm/pfsd && chown $USER_NAME /dev/shm/pfsd && \ + mkdir -p /data/log/curve/ && chown $USER_NAME /data/log/curve/ && \ + sed -i "s;# __CURVEADM_TEMPLATE__ /curvebs/client/logs __CURVEADM_TEMPLATE__;;g" /etc/curve/client.conf && \ + touch /var/run/pfsd/.pfsd && \ + chkconfig --add pfsd_env && \ + echo "ulimit -c unlimited" >> /home/postgres/.bashrc && \ + echo "export PATH=/home/postgres/tmp_basedir_polardb_pg_1100_bld/bin:\$PATH" >> /home/postgres/.bashrc && \ + echo "export PGHOST=127.0.0.1" >> /home/postgres/.bashrc && \ + echo "alias pg='psql -h /home/postgres/tmp_master_dir_polardb_pg_1100_bld/'" >> /home/postgres/.bashrc && \ + rm /etc/localtime && \ + cp /usr/share/zoneinfo/UTC /etc/localtime && \ + sed -i 's/4096/unlimited/g' /etc/security/limits.d/20-nproc.conf && \ + sed -i 's/vim/vi/g' /root/.bashrc +USER $USER_NAME diff --git a/docker/curve/1.2/centos7/Makefile b/docker/curve/1.2/centos7/Makefile index 66e8457..e20329c 100644 --- a/docker/curve/1.2/centos7/Makefile +++ b/docker/curve/1.2/centos7/Makefile @@ -1,4 +1,6 @@ -.PHONY: build +.PHONY: curve-builder, polardb-builder -build: - docker build --tag opencurvedocker/curve-build:latest-centos7 $(PWD) +curve-builder: + docker build --target curve-builder --tag opencurvedocker/curve-build:latest-centos7 $(PWD) +polardb-builder: + docker build --target polardb-builder --tag opencurvedocker/polardb-build:latest-centos7 $(PWD) \ No newline at end of file