refine the transcode service

Author: Pengfei Qu

CMakeLists.txt

@@ -14,6 +14,10 @@ if (NOT DEFINED NLIVES)
     set(NLIVES "1")
 endif()
 
+if (NOT DEFINED PLATFORM)
+    set(PLATFORM "Xeon")
+endif()
+
 if (NOT DEFINED SCENARIO)
     set(SCENARIO "cdn")
 endif()
@@ -32,6 +36,6 @@ endforeach()
 
 # legal message
 execute_process(COMMAND printf "\nThis script will build third party components licensed under various open source licenses into your container images. The terms under which those components may be used and distributed can be found with the license document that is provided with those components. Please familiarize yourself with those terms to ensure your distribution of those components complies with the terms of those licenses.\n\n")
-execute_process(COMMAND printf "\n-- Setting: SCENARIO=${SCENARIO}\n")
+execute_process(COMMAND printf "\n-- Setting: PLATFORM=${PLATFORM} SCENARIO=${SCENARIO}\n")
 execute_process(COMMAND printf "-- Setting: NVODS=${NVODS}, NLIVES=${NLIVES}\n")
 execute_process(COMMAND printf "-- Setting: REGISTRY=${REGISTRY}\n")

cdn-server/schedule.py

@@ -12,11 +12,12 @@
 class ScheduleHandler(web.RequestHandler):
     @gen.coroutine
     def get(self):
-        stream = self.requeist.uri.replace("/schedule/", "")
+        stream = self.request.uri.replace("/schedule/", "")
 
         # schedule producing the stream
         print("request received to process stream: "+stream, flush=True)
         producer = Producer()
+        msg={}
         msg.update({
             "name":stream.split("/")[1],
             "type":stream.split("/")[0],
@@ -26,7 +27,7 @@ def get(self):
             "target": "file",
             "platform": "software"
             })
-        producer.send(KAFKA_TOPIC_VODS, msg)
+        producer.send(KAFKA_TOPIC_VODS, json.dumps(msg))
         producer.close()
 
         # wait until file is available, return it

deployment/kubernetes/yaml/build.sh

@@ -4,11 +4,12 @@ DIR=$(dirname $(readlink -f "$0"))
 NVODS="${1:-1}"
 NLIVES="${2:-1}"
 SCENARIO="${3:-cdn}"
-REGISTRY="$4"
+PLATFORM="${4:-Xeon}"
+REGISTRY="$5"
 HOSTIP=$(ip route get 8.8.8.8 | awk '/ src /{split(substr($0,index($0," src ")),f);print f[2];exit}')
 
 . "${DIR}/../volume-info.sh"
-echo "NVODS=${NVODS} NLIVES=${NLIVES} SCENARIO=${SCENARIO}"
+echo "NVODS=${NVODS} NLIVES=${NLIVES} SCENARIO=${SCENARIO} PLATFORM=${PLATFORM}"
 for template in $(find "${DIR}" -maxdepth 1 -name "*.yaml.m4" -print); do
-    m4 -DNVODS=${NVODS} -DNLIVES=${NLIVES} -DSCENARIO=${SCENARIO} -DHOSTIP=${HOSTIP} -DREGISTRY_PREFIX=${REGISTRY} $(env | grep _VOLUME_ | sed 's/^/-D/') -I "${DIR}" "${template}" > "${template/.m4/}"
+    m4 -DNVODS=${NVODS} -DNLIVES=${NLIVES} -DSCENARIO=${SCENARIO} -DPLATFORM=${PLATFORM} -DHOSTIP=${HOSTIP} -DREGISTRY_PREFIX=${REGISTRY} $(env | grep _VOLUME_ | sed 's/^/-D/') -I "${DIR}" "${template}" > "${template/.m4/}"
 done

deployment/kubernetes/yaml/live-deploy.yaml.m4

@@ -23,7 +23,7 @@ spec:
       enableServiceLinks: false
       containers:
         - name: live-defn(`LIDX')
-          image: defn(`REGISTRY_PREFIX')ovc_software_transcode_service:latest
+          image: defn(`REGISTRY_PREFIX')`ovc_transcode_'defn(`PLATFORM_SUFFIX'):latest
           imagePullPolicy: IfNotPresent
           resources:
             limits:

deployment/kubernetes/yaml/start.sh

@@ -4,7 +4,8 @@ DIR=$(dirname $(readlink -f "$0"))
 NVODS="${1:-1}"
 NLIVES="${2:-1}"
 SCENARIO="${3:-cdn}"
-REGISTRY="$4"
+PLATFORM="${4:-Xeon}"
+REGISTRY="$5"
 
 # Set Bash color
 ECHO_PREFIX_INFO="\033[1;32;40mINFO...\033[0;0m"

deployment/kubernetes/yaml/vod-deploy.yaml.m4

@@ -20,7 +20,7 @@ spec:
       enableServiceLinks: false
       containers:
         - name: vod
-          image: defn(`REGISTRY_PREFIX')ovc_software_transcode_service:latest
+          image: defn(`REGISTRY_PREFIX')`ovc_transcode_'defn(`PLATFORM_SUFFIX'):latest
           imagePullPolicy: IfNotPresent
           resources:
             limits:

script/build.sh

@@ -5,24 +5,34 @@ if test -z "${DIR}"; then
     exit -1
 fi
 
-REGISTRY="$4"
+PLATFORM="${4:-Xeon}"
+REGISTRY="$5"
 
-# build image(s) in order (to satisfy dependencies)
-for dep in .8 .7 .6 .5 .4 .3 .2 .1 ''; do
-    if test -f "${DIR}/Dockerfile$dep"; then
-        image=$(grep -m1 '#' "$DIR/Dockerfile$dep" | cut -d' ' -f2)
-        if test -z "$dep"; then image="$IMAGE"; fi
-
-        if grep -q 'AS build' "${DIR}/Dockerfile$dep"; then
-            docker build --network=host --file="${DIR}/Dockerfile$dep" --target build -t "$image:build" "$DIR" $(env | grep -E '_(proxy|REPO|VER)=' | sed 's/^/--build-arg /') --build-arg UID=$(id -u) --build-arg GID=$(id -g)
-        fi
-
-        docker build --network=host --file="${DIR}/Dockerfile$dep" -t "$image:latest" "$DIR" $(env | grep -E '_(proxy|REPO|VER)=' | sed 's/^/--build-arg /') --build-arg UID=$(id -u) --build-arg GID=$(id -g)
+build_docker() {
+    docker_file="$1"
+    shift
+    image_name="$1"
+    shift
+    if test -f "$docker_file.m4"; then
+        m4 -I "$(dirname $docker_file)" "$docker_file.m4" > "$docker_file"
+    fi
+    (cd "$DIR"; docker build --network host --file="$docker_file" "$@" -t "$image_name" "$DIR" $(env | cut -f1 -d= | grep -E '_(proxy|REPO|VER)$' | sed 's/^/--build-arg /') --build-arg UID=$(id -u) --build-arg GID=$(id -g))
 
-        # if REGISTRY is specified, push image to the private registry
-        if [ -n "$REGISTRY" ]; then
-            docker tag "$image:latest" "$REGISTRY$image:latest"
-            docker push "$REGISTRY$image:latest"
-        fi
+    # if REGISTRY is specified, push image to the private registry
+    if [ -n "$REGISTRY" ]; then
+        docker tag "$image_name" "$REGISTRY$image_name"
+        docker push "$REGISTRY$image_name"
     fi
+}
+
+# build image(s) in order (to satisfy dependencies)
+#for dep in .8 .7 .6 .5 .4 .3 .2 .1 ''; do
+for dep in '.5.*' '.4.*' '.3.*' '.2.*' '.1.*' '.0.*' ''; do
+    dirs=("$DIR/$PLATFORM" "$DIR")
+    for dockerfile in $(find "${dirs[@]}" -maxdepth 1 -name "Dockerfile$dep" -print 2>/dev/null); do
+        echo ${dirs[@]}
+        image=$(head -n 1 "$dockerfile" | grep '# ' | cut -d' ' -f2)
+        if test -z "$image"; then image="$IMAGE"; fi
+        build_docker "$dockerfile" "$image"
+    done
 done

script/deployment.cmake

@@ -1,2 +1,2 @@
-add_custom_target(start_${service} "${CMAKE_CURRENT_SOURCE_DIR}/start.sh" "${service}" "${NVODS}" "${NLIVES}" "${SCENARIO}" "${REGISTRY}")
+add_custom_target(start_${service} "${CMAKE_CURRENT_SOURCE_DIR}/start.sh" "${service}" "${NVODS}" "${NLIVES}" "${SCENARIO}" "${PLATFORM}" "${REGISTRY}")
 add_custom_target(stop_${service} "${CMAKE_CURRENT_SOURCE_DIR}/stop.sh" "${service}")

script/service.cmake

@@ -1,3 +1,3 @@
 if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/build.sh")
-    add_custom_target(build_${service} ALL "${CMAKE_CURRENT_SOURCE_DIR}/build.sh" "${NVODS}" "${NLIVES}" "${SCENARIO}" "${REGISTRY}")
+    add_custom_target(build_${service} ALL "${CMAKE_CURRENT_SOURCE_DIR}/build.sh" "${NVODS}" "${NLIVES}" "${SCENARIO}" "${PLATFORM}" "${REGISTRY}")
 endif()

xcode-server/CMakeLists.txt

@@ -1 +1,3 @@
-include("${CMAKE_SOURCE_DIR}/script/scan-all.cmake")
+set(service "ovc_transcode_service")
+include("${CMAKE_SOURCE_DIR}/script/service.cmake")
+add_dependencies(build_${service} build_ovc_cdn_common)

xcode-server/software/Dockerfile xcode-server/Xeon/Dockerfile

@@ -1,6 +1,9 @@
+# ovc_transcode_xeon
 
 FROM openvisualcloud/xeon-ubuntu1804-media-ffmpeg:20.7
 RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y -q --no-install-recommends python3-tornado python3-kafka python3-kazoo && rm -rf /var/lib/apt/lists/*
+
+COPY --from=ovc_cdn_common /home/ /home/
 COPY *.py  /home/
 CMD    ["/home/main.py"]
 WORKDIR /home

xcode-server/hardware/Dockerfile xcode-server/Xeone3/Dockerfile

@@ -1,8 +1,10 @@
+# ovc_transcode_xeone3
 
 FROM openvisualcloud/xeone3-ubuntu1804-media-ffmpeg:20.7
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y -q --no-install-recommends python3-tornado python3-kafka python3-kazoo && rm -rf /var/lib/apt/lists/*
 
+COPY --from=ovc_cdn_common /home/ /home/
 COPY *.py  /home/
 CMD    ["/home/main.py"]
 WORKDIR /home

xcode-server/build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash -e
+
+DIR=$(dirname $(readlink -f "$0"))
+
+. "${DIR}/../script/build.sh"