Merge pull request #292 from mozilla-services/gcp_pubsub

Add grpc Pub/Sub support

Author: Mike Trinkala

CMakeLists.txt

@@ -74,6 +74,7 @@ cjson
 compat
 cuckoo_filter
 elasticsearch
+gcp
 # geoip # deprecated use maxminddb
 hindsight
 heka

Dockerfile

@@ -27,15 +27,22 @@ RUN yum makecache && \
     rpm -i parquet-cpp-1.3.1-1.x86_64.rpm && \
     cat /etc/yum.conf | grep -v override_install_langs > /etc/yum.conf.lang && \
     cp /etc/yum.conf.lang /etc/yum.conf && \
-    yum reinstall -y glibc-common
+    yum reinstall -y glibc-common && \
+    yum install -y stow && \
+    curl -OL https://hsadmin.trink.com/packages/centos7/external/grpc_stow.tgz && \
+    if [[ `sha256sum grpc_stow.tgz | awk '{print $1}'` != \
+        "65dba4a11ccc09ced4dad64ef196cab6299736a5f5e0df83fef6f1046213797b" ]]; then exit 1; fi && \
+    tar -C / -zxf grpc_stow.tgz && \
+    stow -d /usr/local/stow protobuf-3 grpc googleapis
 
 # Use devtoolset-6
 ENV PERL5LIB='PERL5LIB=/opt/rh/devtoolset-6/root//usr/lib64/perl5/vendor_perl:/opt/rh/devtoolset-6/root/usr/lib/perl5:/opt/rh/devtoolset-6/root//usr/share/perl5/vendor_perl' \
     X_SCLS=devtoolset-6 \
     PCP_DIR=/opt/rh/devtoolset-6/root \
     LD_LIBRARY_PATH=/opt/rh/devtoolset-6/root/usr/lib64:/opt/rh/devtoolset-6/root/usr/lib \
     PATH=/opt/rh/devtoolset-6/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
-    PYTHONPATH=/opt/rh/devtoolset-6/root/usr/lib64/python2.7/site-packages:/opt/rh/devtoolset-6/root/usr/lib/python2.7/site-packages
+    PYTHONPATH=/opt/rh/devtoolset-6/root/usr/lib64/python2.7/site-packages:/opt/rh/devtoolset-6/root/usr/lib/python2.7/site-packages \
+    PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
 
 # Compile and install lua_sandbox and hindsight using master branch
 RUN git clone https://github.com/mozilla-services/lua_sandbox && \

aws/CMakeLists.txt

@@ -14,5 +14,6 @@ find_package(aws-cpp-sdk-monitoring 1.1.46 REQUIRED CONFIG)
 
 set(MODULE_SRCS kinesis.cpp aws.def)
 set(INSTALL_MODULE_PATH ${INSTALL_IOMODULE_PATH})
+set(TEST_CONFIGURATION "aws")
 include(sandbox_module)
 target_link_libraries(aws aws-cpp-sdk-kinesis aws-cpp-sdk-monitoring)

cmake/sandbox_module.cmake

@@ -11,7 +11,7 @@ else()
     set(CPACK_GENERATOR         "TGZ")
     set(CMAKE_C_FLAGS           "-std=c99 -pedantic -Werror -Wall -Wextra -fPIC")
     set(CMAKE_C_FLAGS_RELEASE   "-O2")
-    set(CMAKE_CXX_FLAGS "-std=c++0x -pedantic -Werror -Wall -Wextra -fPIC -isystem /usr/local/include -isystem /opt/local/include")
+    set(CMAKE_CXX_FLAGS "-std=c++14 -pedantic -Werror -Wall -Wextra -fPIC -isystem /usr/local/include -isystem /opt/local/include")
     set(CMAKE_CXX_FLAGS_RELEASE ${CMAKE_C_FLAGS_RELEASE})
     set(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
 endif()

gcp/CMakeLists.txt

@@ -0,0 +1,34 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+cmake_minimum_required(VERSION 3.6)
+project(gcp VERSION 0.0.1 LANGUAGES C CXX)
+set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "GCP Lua Modules")
+set(CPACK_DEBIAN_PACKAGE_DEPENDS "luasandbox-lpeg (>= 1.0.9)")
+string(REGEX REPLACE "[()]" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_DEBIAN_PACKAGE_DEPENDS})
+
+find_package(PkgConfig)
+pkg_check_modules(GRPC REQUIRED IMPORTED_TARGET protobuf grpc++ grpc)
+
+set(GOOGLE_APIS_DIR "/usr/local/include")
+include_directories(${GOOGLE_APIS_DIR})
+set(PUBSUB_PB ${GOOGLE_APIS_DIR}/google/pubsub/v1/pubsub.pb.cc)
+set(HTTP_PB ${GOOGLE_APIS_DIR}/google/api/http.pb.cc)
+
+set(MODULE_SRCS
+  pubsub.cpp
+  ${GOOGLE_APIS_DIR}/google/api/annotations.pb.cc
+  ${HTTP_PB}
+  ${PUBSUB_PB}
+  ${GOOGLE_APIS_DIR}/google/pubsub/v1/pubsub.grpc.pb.cc
+  gcp.def)
+set(INSTALL_MODULE_PATH ${INSTALL_IOMODULE_PATH})
+set(TEST_CONFIGURATION "gcp")
+include(sandbox_module)
+
+set_source_files_properties(${PUBSUB_PB} PROPERTIES COMPILE_FLAGS -Wno-unused-parameter)
+set_source_files_properties(${HTTP_PB} PROPERTIES COMPILE_FLAGS -Wno-unused-parameter)
+target_link_libraries(gcp ${GRPC_STATIC_LIBRARIES})
+set(GRPC_SHARE_DIR ${CMAKE_INSTALL_DATAROOTDIR}/luasandbox/grpc)
+install(FILES /usr/local/share/grpc/roots.pem DESTINATION ${GRPC_SHARE_DIR})

gcp/gcp.def

@@ -0,0 +1,2 @@
+EXPORTS
+luaopen_gcp_pubsub

gcp/index.md

@@ -0,0 +1,8 @@
+# GCP Lua Module
+
+## Overview
+Lua wrapper for the GCP gRPC APIs.
+
+## Modules
+
+* [Pub/Sub](pubsub.md)