Orchestration in VDMS with Kubernetes

.github/coverage/cpp.develop.coverage_report.txt

@@ -14,7 +14,7 @@ src/CommunicationManager.cc                   46       0     0%   42-43,46-47,49
 src/DescriptorsCommand.cc                    725     116    16%   56,63-68,73,75-79,81-85,87,89,92-93,96-98,101,103-104,107-113,115,118,121,168-170,174,188-192,244-255,259-260,264-269,279,287-290,304-306,310,325-334,336,356-359,364,370-374,391,399,401-404,407,410-411,413-417,420,423,425-426,429-431,433-434,436-437,440-441,443-445,451,456-457,459,461-462,465,467-470,473-478,484-485,488,490-491,493,495-496,499,503-504,506,508,511,513-518,520-524,526,529,532-533,536-537,548,551,556,562-563,568-569,571-574,577,583-584,589-590,593-599,602,604-605,607,609-610,612-613,617,619-624,627-632,635,638-641,644-645,647,649,652-653,656-657,660,663,670-671,673-675,677,680-681,683,686,689,691-693,701,705,707,710,712-715,718-723,725-729,731,734,736,738,741,744-745,747,749,751-756,759,761-763,766-767,769,771,773-774,776,778,780-782,784,786-791,796-797,800,802,804,811-812,815,819,821,824,826,828-831,834-838,840-844,846,848-851,853-855,858,862-871,876-877,880,886,888,891-892,895,899,904,906,909,912-913,917,923-927,929,934,937,940-941,943-946,950,954-955,957,959-961,963,965-966,968,970-972,974-976,981-983,986-989,992,995-997,1000-1002,1004,1006,1008,1012-1016,1020,1023,1030-1033,1035-1036,1038,1040-1041,1043-1045,1047,1052,1055-1056,1059-1062,1064-1065,1069,1071-1074,1076-1079,1084,1087-1089,1091,1093,1095,1097,1099-1102,1104-1108,1111-1112,1115,1117,1119,1121-1125,1128-1129,1132-1135,1137,1139-1146,1151,1153,1155-1156,1159-1162,1164,1166,1168-1175,1179-1184,1190,1192,1194-1197,1200,1202,1205-1208,1210-1214,1220,1222,1224-1227,1232,1234,1236-1237,1240-1242,1244-1247,1250-1251,1253-1256,1260-1262,1264,1266,1268-1271,1274-1275,1277-1278,1280,1282-1283,1285-1291,1295-1296,1300-1301,1303-1304,1312-1313,1316-1317,1319,1321-1328,1332,1336-1337,1341-1345,1350-1351,1353
 src/DescriptorsManager.cc                     24      19    79%   49-50,57-58,73
 src/ExceptionsCommand.cc                       6       0     0%   35-40
-src/ImageCommand.cc                          322     157    48%   55,59,63,65,67-69,71,73-76,78,81,86,88-89,97,99,106,109,111-112,114-115,117-118,120-121,124,151,162-163,174-175,177,182-185,195-196,198,203-206,221-229,231-233,246-247,257-267,269-270,272-273,278,286,297,304,308,311,313,315,337,339-340,343-348,350,352,374-376,379-381,385-388,394,396,403-406,420,427,433-436,440-441,452-455,458-463,468-470,481-484,489-493,498-499,501-502,504-508,511,513-517,520-523,526-527,530,532,537
+src/ImageCommand.cc                          323     157    48%   68,72,76,78,80-82,84,86-89,104-105,110,115,117-118,126,128,135,138,140-141,143-144,146-147,149-150,153,180,191-192,203-204,206,211-214,224-225,227,232-235,250-258,260-262,275-276,286-296,298-299,301-302,307,315,326,333,337,340,342,344,366,368-369,372-377,379,381,403-405,408-410,414-417,423,425,432-435,449,456,462-465,469-470,481-484,487-492,497-499,510-513,518-522,527-528,530-531,533-537,540,542-546,549-552,555-556,559,561,566
 src/ImageLoop.cc                             253     232    91%   63,130,182-185,215,221,265,285,288,297-298,300,307-308,322-323,330,334,338
 src/Neo4jBaseCommands.cc                      38       0     0%   7-8,12,14-15,17,21,23-24,26,30,32-33,35,39,41-42,44,48,50,53,57-59,62-70,72,74,76-77,80
 src/Neo4JHandlerCommands.cc                  109       0     0%   50,54-55,57-58,61,65-69,72,74,76-80,82,84,86,92,95-96,98,100,102,106-107,110-111,114-119,123-124,126,128-129,132-133,137-139,141,144,149-151,155,157,160,162,165-166,169,172,176-178,180-186,188-191,196,198-201,203,206,209,211-213,217-220,222-225,228-231,233-237,239-244,249
@@ -47,7 +47,7 @@ src/vcl/TDBSparseDescriptorSet.cc            245     230    93%   163,190-191,23
 src/vcl/utils.cc                              71      63    88%   54-55,65,71,79,91,93,121
 src/vcl/Video.cc                             746     565    75%   66,127,133,138,160,166-167,189,191-195,198-201,218-223,231,233-239,241-245,248-250,254-255,260-261,263-264,266,268,271-272,274-275,278-279,296,314-327,344,346,348-350,379,412-414,456,465,488,494,517,637,639,650,656,660-661,670,689,692,694-695,698-699,728-729,750-752,755-756,758-759,762-763,782-785,813-815,818,821-823,838-841,849,851,853-858,870-873,907,929,944,967,971,1000-1001,1020-1021,1042,1079,1087,1104,1119,1123,1130,1134,1151,1154,1159-1160,1163-1164,1166,1177,1181,1186,1204-1207,1260,1264,1266-1267,1269-1270,1272,1274,1277-1278,1281-1282,1288-1292
 src/vdms.cc                                  108       0     0%   39,41-42,44-47,49-50,52-55,57-58,61-63,65-66,68,70,73-75,78,82,84-87,89,91-97,99-102,104-105,107-110,112-114,116-119,121-122,124-127,129-130,132-135,137-138,140-141,145-147,150-153,156,158-160,163,165,168,171-177,183,185,188-189,193,196,202-203,206,212-215,218,220
-src/VDMSConfig.cc                            221     208    94%   129-131,206,208,211-212,218-219,223-224,335-336
+src/VDMSConfig.cc                            226     213    94%   131-133,209,211,214-215,221-222,226-227,338-339
 src/VideoCommand.cc                          474     117    24%   50,53-54,56-58,60,62,65-66,68-69,72,74-76,78-80,82,84-87,89-90,92-93,95,97-99,102,109,111,116,121-124,130,132,158-161,167-168,170,181,184,201,213,217-220,227-229,231-233,239,241-247,249-250,253-255,257-259,261-262,264,266-278,280-282,284-285,296,300,325,329,331,333,335,337,340-341,343,346,350,352,357-358,380-381,383-384,387-392,394,396,398-399,405,407,429-431,436,442-445,449-454,456-463,467-473,475,480-485,488,490-491,494-496,504,509,527-532,535-539,555,558,560-562,565-567,569-570,572-576,579-580,583-585,587,589-591,594-597,601-606,611-612,614-615,617-621,624-626,628,630-632,634-637,640-641,644,646,651,664,666-673,677,680,683,688-689,691-695,698-699,701,703,705,708,712,714,716-719,721-723,726,728,730,732-733,735-736,740,745,748-749,751-753,755,757,759-761,763-764,767-769,773-776,780-786,790-794,798,801,803,805,807,809-813,817-821,824-825,827-830,833-836,841-842,846-851,855-856,859-860
 src/VideoLoop.cc                             249     200    80%   33,81,98-101,103-109,180,188,197,201,207,211,217,220,290,312,315,324-325,327,331-332,334-335,339-342,344,346-349,351-354,356-357,359,361,370
 utils/src/comm/ConnClient.cc                  69      57    82%   49,55,59-60,98,103,108,114,120,127,130,149
@@ -57,5 +57,5 @@ utils/src/comm/Exception.cc                    6       0     0%   35-40
 utils/src/stats/SystemStats.cc               250     249    99%   453
 utils/src/timers/TimerMap.cc                  82      75    91%   126,151,153,155-158
 ------------------------------------------------------------------------------
-TOTAL                                      10303    6623    64%
+TOTAL                                      10309    6628    64%
 ------------------------------------------------------------------------------

.github/coverage/cpp.develop.coverage_value.txt

@@ -1 +1 @@
-64.2822
+64.2933

.github/scripts/Dockerfile.checkin

@@ -15,6 +15,7 @@ ARG NEO4J_PASS=""
 ARG NEO4J_ENDPOINT=""
 ARG AWS_API_PORT=9000
 ARG AWS_CONSOLE_PORT=9001
+ARG USE_K8S="OFF"
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV DEBCONF_NOWARNINGS="yes"
@@ -42,12 +43,14 @@ RUN apt-get update -y && apt-get upgrade -y && \
     apt-get install -o 'Acquire::Retries=3' -y --no-install-suggests \
         --no-install-recommends --fix-broken --fix-missing \
         apt-transport-https automake bison build-essential bzip2 ca-certificates \
-        cppzmq-dev curl ed flex g++ gcc git gnupg-agent javacc libarchive-tools libatlas-base-dev \
+        cppzmq-dev curl ed flex g++ bazel-bootstrap gcc git gnupg-agent javacc libarchive-tools libatlas-base-dev \
         libavcodec-dev libavformat-dev libavutil-dev libboost-all-dev libbz2-dev libc-ares-dev \
         libcurl4-openssl-dev libdc1394-dev libgflags-dev libgoogle-glog-dev \
         libgtk-3-dev libgtk2.0-dev libhdf5-dev libjpeg-dev libjpeg62-turbo-dev libjsoncpp-dev \
         libleveldb-dev liblmdb-dev liblz4-dev libncurses5-dev libopenblas-dev libopenmpi-dev \
         libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev libtbbmalloc2 \
+        libwebsockets-dev uncrustify \
+        libbison-dev \
         libtiff-dev libtiff5-dev libtool linux-libc-dev mpich openjdk-17-jdk-headless \
         pkg-config procps software-properties-common swig unzip uuid-dev && \
     apt-get --purge remove -y python3.11 && apt-get autoremove -y && \
@@ -163,6 +166,29 @@ RUN curl -L -O https://github.com/gpakosz/peg/releases/download/${PEG_VERSION}/p
         ./configure --disable-werror --prefix=/opt/dist/usr && \
         make clean check && make install -w --debug
 
+RUN if [ "$USE_K8S" = "ON" ]; then \
+        git clone --depth 1 https://github.com/yaml/libyaml.git /dependencies/libyaml && \
+        cd /dependencies/libyaml && \
+        mkdir build && \
+        cd build && \
+        cmake -DCMAKE_INSTALL_PREFIX=/usr/local/ -DBUILD_TESTING=OFF  -DBUILD_SHARED_LIBS=ON .. && \
+        make && \
+        make install; \
+    fi
+
+#Kubernetes
+RUN if [ "$USE_K8S" = "ON" ]; then \
+        git clone https://github.com/kubernetes-client/c.git /dependencies/k8s && \
+        ls /dependencies/k8s && \
+        CLIENT_REPO_ROOT=/dependencies/k8s && \
+        cd ${CLIENT_REPO_ROOT}/kubernetes && \
+        mkdir build && \
+        cd build && \
+        cmake .. && \
+        make && \
+        make install; \
+    fi
+
 # CLEANUP
 RUN rm -rf /dependencies /usr/local/share/doc /usr/local/share/man && \
     mkdir -p /opt/dist/usr/include/x86_64-linux-gnu && \
@@ -189,18 +215,24 @@ COPY user_defined_operations /vdms/user_defined_operations
 COPY utils /vdms/utils
 COPY CMakeLists.txt /vdms/CMakeLists.txt
 COPY config-vdms.json /vdms/config-vdms.json
+COPY kubeConfig.json /vdms/kubeConfig.json
 COPY docker/override_default_config.py /vdms/override_default_config.py
 COPY --from=build /opt/dist /
 COPY --from=build /usr/local/bin/python${PYTHON_BASE} /usr/local/bin/python${PYTHON_BASE}
 COPY --from=build /usr/local/lib/python${PYTHON_BASE} /usr/local/lib/python${PYTHON_BASE}
 COPY --from=build ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+COPY --from=build /usr/local/include/kubernete[s] /usr/local/include/kubernetes
+COPY --from=build /usr/include/libwebsocket[s] /usr/include/libwebsockets
+COPY --from=build /usr/local/lib/libkubernete[s.so] /usr/local/lib/libkubernetes.so
+COPY --from=build /usr/local/lib/libyam[l.so] /usr/local/lib/libyaml.so
+COPY --from=build /usr/lib/x86_64-linux-gnu/libwebsocket[s.so] /usr/lib/x86_64-linux-gnu/libwebsockets.so
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
 # hadolint ignore=DL3008,SC2086
 RUN apt-get update -y && apt-get upgrade -y && \
     apt-get install -o 'Acquire::Retries=3' -y --no-install-suggests \
         --no-install-recommends --fix-broken --fix-missing \
-        build-essential bzip2 cppzmq-dev curl g++ gcc git javacc libarchive-tools libavcodec-dev \
+        build-essential bzip2 cppzmq-dev curl g++ gcc git javacc libarchive-tools libavcodec-dev bazel-bootstrap \
         libavformat-dev libcurl4-openssl-dev libdc1394-dev libgoogle-glog-dev libgtk-3-dev \
         libhdf5-dev libjpeg62-turbo-dev libjsoncpp-dev libopenblas-dev libpng-dev librdkafka-dev \
         libssl-dev libswscale-dev libtbb-dev libtbbmalloc2 libtiff5-dev libzip-dev openjdk-17-jdk-headless \

.gitignore

@@ -5,6 +5,8 @@ db/
 fr_client
 tmp
 *.onnx
+tests/k8s_test_files/
+tests/run_k8s_tests.sh
 
 # VS Code Specific
 .devcontainer

CMakeLists.txt

@@ -41,6 +41,7 @@ protobuf_generate(
   PROTOC_OUT_DIR "${PROTO_BINARY_DIR}"
 )
 
+option(USE_K8S "Enable kubernetes client libraries" OFF)
 option(CLIENT "Built client library." OFF)
 if (CLIENT)
   add_definitions("-D CLIENT")
@@ -59,7 +60,14 @@ else()
   add_subdirectory(distributed)
 
   link_directories(/usr/local/lib /usr/lib/x86_64-linux-gnu/)
-  include_directories(/usr/include/jsoncpp utils/include/ src/pmgd/include src/pmgd/util include/ src/vcl /usr/include ${CMAKE_CURRENT_BINARY_DIR}/utils/src/protobuf)
+  message("USE_K8S:" ${USE_K8S})
+  if(USE_K8S)
+    include_directories(/usr/include/jsoncpp utils/include/ src/pmgd/include src/pmgd/util include/ src/vcl /usr/include ${CMAKE_CURRENT_BINARY_DIR}/utils/src/protobuf /usr/local/include/kubernetes /usr/include/libwebsockets)
+    add_definitions("-D HAS_KUBERNETES_CLIENT")
+  else()
+    include_directories(/usr/include/jsoncpp utils/include/ src/pmgd/include src/pmgd/util include/ src/vcl /usr/include ${CMAKE_CURRENT_BINARY_DIR}/utils/src/protobuf)
+  endif()
+
   add_library(dms SHARED
     src/BackendNeo4j.cc
     src/BoundingBoxCommand.cc
@@ -87,8 +95,7 @@ else()
     src/VideoCommand.cc
     src/AutoDeleteNode.cc
     src/ImageLoop.cc
-    src/VideoLoop.cc
-  )
+    src/VideoLoop.cc)
   target_link_libraries(dms vcl pmgd pmgd-util protobuf tbb tiledb vdms-utils pthread -lcurl -lzmq -lzip ${AWSSDK_LINK_LIBRARIES} neo4j-client)
   add_executable(vdms src/vdms.cc)
   target_link_libraries(vdms dms vdms_protobuf vcl tiledb faiss flinng jsoncpp ${OpenCV_LIBS} ${AWSSDK_LINK_LIBRARIES})

INSTALL.md

@@ -275,8 +275,29 @@ cd $VDMS_DEP_DIR/libomni
 make clean check
 sudo make install -w --debug
 ```
+
 <br>
 
+#### **Kubernetes Client**
+Installation required only if you plan to use the kubernetes environment
+Follow [[Kubernetes README](kubernetes/README.md)] for how to set up the environment.
+```bash
+git clone --depth 1 https://github.com/yaml/libyaml.git /dependencies/libyaml
+cd $VDMS_DEP_DIR/libyaml
+mkdir build && cd build
+cmake -DCMAKE_INSTALL_PREFIX=/usr/local/ -DBUILD_TESTING=OFF  -DBUILD_SHARED_LIBS=ON ..
+make
+sudo make install
+
+CLIENT_REPO_ROOT=$VDMS_DEP_DIR/k8s
+git clone https://github.com/kubernetes-client/c.git ${CLIENT_REPO_ROOT}
+cd ${CLIENT_REPO_ROOT}/kubernetes
+mkdir build && cd build
+cmake ..
+make
+sudo make install
+```
+
 ## Install VDMS
 This version of VDMS treats PMGD as a submodule so both libraries are compiled at one time. After entering the vdms directory, the command `git submodule update --init --recursive` will pull pmgd into the appropriate directory. Furthermore, Cmake is used to compile all directories.
 ```bash
@@ -297,7 +318,7 @@ sed -i "s|#include <libavcodec/avcodec.h>||" include/vcl/KeyFrame.h
 sed -i "s|#include <libavcodec/bsf.h>||" include/vcl/KeyFrame.h
 ```
 
-When compiling on a target without Optane persistent memory, use the following:
+When compiling on a target without Optane persistent memory and without Kubernetes, use the following:
 ```bash
 mkdir build && cd build
 cmake ..
@@ -313,6 +334,14 @@ make ${BUILD_THREADS}
 cp ../config-vdms.json .
 ```
 
+If you plan on setting up the Kubernetes environment with VDMS and remote operations (experimental), use the following:
+```bash
+mkdir build && cd build
+cmake -DUSE_K8S=ON ..
+make ${BUILD_THREADS}
+cp ../config-vdms.json .
+```
+
 ***NOTE:*** If error similar to `cannot open shared object file: No such file or directory` obtained during loading shared libraries, such as `libpmgd.so` or `libvcl.so`, add the correct directories to `LD_LIBRARY_PATH`. This may occur for non-root users. To find the correct directory, run `find` command for missing object file. An example solution for missing `libpmgd.so` and `libvcl.so` is:
 ```bash
 find / -name "libpmgd*so*" # <Path_to_VDMS_directory>/build/src/pmgd/src

config-vdms.json

@@ -13,5 +13,6 @@
     // use_endpoint: [true|false] in case of "storage_type" is equals to "aws", this key is used to specify whether it is going to use a "mocked" AWS connection
     "use_endpoint": false,
     "bucket_name": "minio-bucket",
+    "k8s_container": false,
     "more-info": "github.com/IntelLabs/vdms"
 }

custom_bash.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+cd /vdms/user_defined_operations/
+python3 -m pip install -r requirements.txt
+python3 -m pip install sk-video
+python3 -m pip install imutils
+python3 -m pip install zmq

docker/README.md

@@ -12,6 +12,10 @@ If you are under a proxy, use:
     cd base
     Dockerfile | docker build -t vdms:latest --build-arg=http_proxy --build-arg=https_proxy .
 
+If you are building with Kubernetes client support, use:
+    cd base
+    Dockerfile | docker build -t vdms:latest --build-arg USE_K8S="ON" .
+
 To run the docker image as a container, include the --net flag.
 This flag is needed as the server will be accepting connections on the default VDMS port (55555).