Merge pull request #189 from QuPengfei/multi-device

Multi device

Author: xwu2git

common/ffmpegcmd.py

@@ -38,19 +38,22 @@
         "AVC": "libx264",
         "HEVC": "libsvt_hevc"
     },
-    "hw": {
+    "vaapi": {
         "AVC": "h264_vaapi",
         "HEVC": "hevc_vaapi"
+    },
+    "qsv": {
+        "AVC": "h264_qsv",
+        "HEVC": "hevc_qsv"
     }
 }
 class FFMpegCmd:
-    def __init__(self, in_params, out_params, streaming_type, params, loop=0, hw="false"):
+    def __init__(self, in_params, out_params, streaming_type, params, loop=0, acc_type="sw", device=None):
         self._in_file=in_params
         self._target=out_params
         self._tc_params=params if params else default_params["tc_params"]
         self._hls_dash_params=params["hls_dash_params"] if "hls_dash_params" in params.keys() else default_params["hls_dash_params"]
-        self._hw=hw
-        self._platform="hw" if hw == "true" else "sw"
+        self._acc_type=acc_type
 
         self._segment_num=self._hls_dash_params["segment_num"]
         self._duration=self._hls_dash_params["duration"]
@@ -65,8 +68,16 @@ def __init__(self, in_params, out_params, streaming_type, params, loop=0, hw="fa
         self._cmd_base=["ffmpeg", "-hide_banner", "-y"]
         if loop:
             self._cmd_base = self._cmd_base + ["-stream_loop", "-1"]
-        if self._hw == "true":
-            self._cmd_base = self._cmd_base +  ["-hwaccel", "vaapi", "-hwaccel_device", "/dev/dri/renderD128", "-hwaccel_output_format", "vaapi"]
+
+        self._device=device
+        if not device and self._acc_type != "sw":
+            self._device = "/dev/dri/renderD128"
+
+        if self._acc_type == "vaapi":
+            self._cmd_base = self._cmd_base +  ["-hwaccel", "vaapi", "-hwaccel_device", self._device, "-hwaccel_output_format", "vaapi"]
+        elif self._acc_type == "qsv":
+            self._cmd_base = self._cmd_base +  ["-hwaccel", "qsv", "-qsv_device", self._device, "-c:v", "h264_qsv"]
+
         self._cmd_base = self._cmd_base + ["-i", self._in_file]
 
         self._keyframe_interval = 0
@@ -83,7 +94,7 @@ def __init__(self, in_params, out_params, streaming_type, params, loop=0, hw="fa
         self._codec = self._get_codec()
         # hls and dash
         self._cmd_static = ["-c:v", self._codec, "-profile:v", "main", "-sc_threshold", "0", "-strict", "-2"]
-        if self._hw == "true":
+        if self._acc_type != "sw":
             self._cmd_static = ["-profile:v", "main", "-c:v", self._codec]
         self._cmd_static += ["-g", str(self._keyframe_interval)]
 
@@ -92,7 +103,7 @@ def _to_kps(self, bitrate):
         return str(int(bitrate/1000))+"k"
 
     def _get_codec(self):
-        return codec_setting[self._platform][self._codec_type]
+        return codec_setting[self._acc_type][self._codec_type]
 
     def stream_info(self, in_file):
         ffprobe_cmd = ["ffprobe", "-v", "quiet", "-print_format", "json", "-show_streams", in_file]
@@ -130,8 +141,11 @@ def _hls(self):
                 continue
 
             cmd_1 = ["-vf", "scale=w="+str(width)+":"+"h="+str(height)]
-            if self._hw == "true":
+            if self._acc_type == "vaapi":
                 cmd_1 = ["-vf", "scale_vaapi=w="+str(width)+":"+"h="+str(height)+":format=nv12"]
+            elif self._acc_type == "qsv":
+                cmd_1 = ["-vf", "scale_qsv=w="+str(width)+":"+"h="+str(height)+":format=nv12"]
+
             cmd_2 = ["-b:v", v_bitrate, "-maxrate", maxrate]
             cmd_3 = ["-f", self._streaming_type]
             cmd_4 = ["-hls_segment_filename", self._target+"/"+name+"_"+"%03d.ts", self._target+"/"+name+".m3u8"]
@@ -160,8 +174,10 @@ def _dash(self):
             if self._frame_height < height:
                 continue
             cmd_1 = ["-map", "[out"+str(count) +"]", "-b:v"+":"+str(count), v_bitrate, "-maxrate"+":"+str(count), maxrate]
-            if self._hw == "true":
+            if self._acc_type == "vaapi":
                 cmd_scale += [";", "[mid"+str(count) +"]", "scale_vaapi=w="+str(width)+":"+"h="+str(height)+":format=nv12","[out"+str(count) +"]"]
+            elif self._acc_type == "qsv":
+                cmd_scale += [";", "[mid"+str(count) +"]", "scale_qsv=w="+str(width)+":"+"h="+str(height)+":format=nv12","[out"+str(count) +"]"]
             else:
                 cmd_scale += [";", "[mid"+str(count) +"]", "scale=w="+str(width)+":"+"h="+str(height),"[out"+str(count) +"]"]
             cmd_abr += cmd_1
@@ -182,14 +198,17 @@ def _tc(self):
             maxrate = self._to_kps(item[2] * self._max_bitrate_ratio)
             name= self._target+"/"+self._codec_type+"_"+str(height)+"p."+self._streaming_type if self._streaming_type == "mp4" else self._target+"_"+self._codec_type+str(height)+"p"
 
-            if self._hw == "true":
+            if self._acc_type == "vaapi":
                 cmd_1 += ["-vf", "scale_vaapi=w="+str(width)+":"+"h="+str(height)+":format=nv12", "-c:v", self._codec]
                 cmd_1 += ["-profile:v", "main", "-b:v", v_bitrate, "-maxrate", v_bitrate, "-r", params["framerate"],"-g", params["gop_size"], "-bf", params["bframe"], "-an", "-f", self._streaming_type, name]
+            elif self._acc_type == "qsv":
+                cmd_1 += ["-vf", "scale_qsv=w="+str(width)+":"+"h="+str(height)+":format=nv12", "-c:v", self._codec]
+                cmd_1 += ["-profile:v", "main", "-b:v", v_bitrate, "-maxrate", v_bitrate, "-r", params["framerate"],"-g", params["gop_size"], "-bf", params["bframe"], "-an", "-f", self._streaming_type, name]
             else:
                 cmd_1 += ["-vf", "scale=w="+str(width)+":"+"h="+str(height),"-c:v", self._codec, "-b:v", v_bitrate]
                 cmd_1 += ["-r", params["framerate"],"-g", params["gop_size"], "-bf", params["bframe"], "-refs", params["refs"], "-preset", params["preset"], "-forced-idr", params["forced_idr"], "-an", "-f", self._streaming_type, name]
 
-        return  cmd_1 + ["-abr_pipeline"]
+        return  cmd_1
 
     def cmd(self):
         cmd = []

deployment/kubernetes/helm/cdn-transcode/templates/vod.yaml

@@ -1,20 +1,22 @@
 
+{{- range $deviceIdx := until ( int ( $.Values.hwDeviceNum ) ) }}
+
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   labels:
-    app: vod-service
-  name: vod-service
+    app: vod-service-{{ $deviceIdx }}
+  name: vod-service-{{ $deviceIdx }}
 spec:
   selector:
     matchLabels:
-      app: vod-service
-  replicas: {{ .Values.vodTranscode.replicas }}
+      app: vod-service-{{ $deviceIdx }}
+  replicas: {{ $.Values.vodTranscode.replicas }}
   template:
     metadata:
       creationTimestamp: null
       labels:
-        app: vod-service
+        app: vod-service-{{ $deviceIdx }}
     spec:
       containers:
       - args:
@@ -23,11 +25,13 @@ spec:
         - /home/main.py
         image: {{ $.Values.registryPrefix }}tc_transcode_{{ lower $.Values.platform }}:latest
         imagePullPolicy: IfNotPresent
-        name: vod-service
+        name: vod-service-{{ $deviceIdx }}
         env:
 {{- if ne $.Values.platform "Xeon" }}
-          - name: HW_ACCELERATOR
-            value: "true"
+          - name: HW_ACC_TYPE
+            value: {{ $.Values.hwAccType }}
+          - name: HW_DEVICE
+            value: /dev/dri/renderD{{ add $deviceIdx 128 }}
 {{- end }}
           - name: NO_PROXY
             value: "*"
@@ -45,6 +49,7 @@ spec:
 #            gpu.intel.com/i915: 1
         securityContext:
           privileged: true
+          runAsUser: 0
 {{- end }}
       volumes:
         - name: archive
@@ -55,6 +60,10 @@ spec:
             claimName: video-cache
       restartPolicy: Always
 
+---
+{{- end }}
+
+
 {{- if eq "transcode" $.Values.scenario }}
 ---
 
@@ -71,10 +80,6 @@ spec:
           image: {{ $.Values.registryPrefix }}tc_benchmark_service:latest
           imagePullPolicy: IfNotPresent
           env:
-{{- if ne $.Values.platform "Xeon" }}
-            - name: HW_ACCELERATOR
-              value: "true"
-{{- end }}
             - name: NO_PROXY
               value: "*"
             - name: no_proxy
@@ -85,13 +90,6 @@ spec:
               readOnly: true
             - mountPath: /var/www/video
               name: video-cache
-{{- if ne $.Values.platform "Xeon" }}
-#        resources:
-#          limits:
-#            gpu.intel.com/i915: 1
-        securityContext:
-          privileged: true
-{{- end }}
       volumes:
           - name: video-archive
             persistentVolumeClaim:

deployment/kubernetes/helm/cdn-transcode/values.yaml.m4

@@ -4,6 +4,10 @@ registryPrefix: "defn(`REGISTRY_PREFIX')"
 # platform specifies the target platform: Xeon or XeonE3.
 platform: "defn(`PLATFORM')"
 
+# transcoding with HW QSV or VAAPI: qsv or vaapi.
+hwAccType: "qsv"
+hwDeviceNum: ifelse(defn(`PLATFORM'),`SG1',4,1)
+
 # scenario specifies the mode: cdn or batch.
 scenario: "defn(`SCENARIO')"
 

deployment/kubernetes/yaml/configure.m4

@@ -1,4 +1,7 @@
 
+define(`HW_ACC_PLUGIN_TYPE',`qsv')
+define(`HW_DEVICE_NUM',ifelse(defn(`PLATFORM'),`SG1',4,1))
+
 define(`CDN_CPU',2)
 define(`CDN_MEMORY',2000)
 

deployment/kubernetes/yaml/platform.m4

@@ -1,6 +1,6 @@
 define(`PLATFORM_SUFFIX',translit(defn(`PLATFORM'),`A-Z',`a-z'))dnl
 define(`PLATFORM_RESOURCES',dnl
-ifelse(defn(`PLATFORM'),`XeonE3',dnl
+ifelse(defn(`PLATFORM'),`Xeon',,dnl
 #          resources:
 #            limits:
 #              gpu.intel.com/i915: 1

deployment/kubernetes/yaml/vod.yaml.m4

@@ -1,25 +1,28 @@
-include(platform.m4)
+include(../../../script/loop.m4)
 include(configure.m4)
+include(platform.m4)
+
+loop(DEVICEIDX,0,eval(defn(`HW_DEVICE_NUM')-1),`
 
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: vod
+  name: vod-defn(`DEVICEIDX')
   labels:
-    app: vod
+    app: vod-defn(`DEVICEIDX')
 spec:
   replicas: defn(`NVODS')
   selector:
     matchLabels:
-      app: vod
+      app: vod-defn(`DEVICEIDX')
   template:
     metadata:
       labels:
-        app: vod
+        app: vod-defn(`DEVICEIDX')
     spec:
       enableServiceLinks: false
       containers:
-        - name: vod
+        - name: vod-defn(`DEVICEIDX')
           image: defn(`REGISTRY_PREFIX')`tc_transcode_'defn(`PLATFORM_SUFFIX'):latest
           imagePullPolicy: IfNotPresent
 ifelse(defn(`SCENARIO'),`transcode',`dnl
@@ -32,8 +35,12 @@ ifelse(defn(`SCENARIO'),`transcode',`dnl
               memory: eval(defn(`VOD_MEMORY')*2)Mi
 ')dnl
           env:
-            - name: HW_ACCELERATOR
-              value: ifelse(defn(`PLATFORM'),`Xeon',"false","true")
+            - name: HW_ACC_TYPE
+              value: ifelse(defn(`PLATFORM'),`Xeon',"sw","defn(`HW_ACC_PLUGIN_TYPE')")
+ifelse(defn(`PLATFORM'),`Xeon',,`dnl
+            - name: HW_DEVICE
+              value: "`/dev/dri/renderD'eval(defn(`DEVICEIDX')+128)"
+')dnl
             - name: NO_PROXY
               value: "*"
             - name: no_proxy
@@ -54,6 +61,9 @@ defn(`PLATFORM_RESOURCES')dnl
                claimName: video-archive
 PLATFORM_NODE_SELECTOR(`Xeon')dnl
 
+---
+')
+
 ifelse(defn(`SCENARIO'),`transcode',`
 ---
 
@@ -70,8 +80,6 @@ spec:
           image: defn(`REGISTRY_PREFIX')tc_benchmark_service:latest
           imagePullPolicy: IfNotPresent
           env:
-            - name: HW_ACCELERATOR
-              value: ifelse(defn(`PLATFORM'),`Xeon',"false","true")
             - name: NO_PROXY
               value: "*"
             - name: no_proxy
@@ -82,7 +90,6 @@ spec:
               readOnly: true
             - mountPath: /var/www/video
               name: video-cache
-defn(`PLATFORM_RESOURCES')dnl
       volumes:
           - name: video-archive
             persistentVolumeClaim:

xcode-server/main.py

@@ -26,7 +26,8 @@
 HLS_ROOT = "/var/www/video/hls"
 MP4_ROOT = "/var/www/video/mp4"
 
-hw=os.environ["HW_ACCELERATOR"]
+HW_ACC_TYPE=os.getenv("HW_ACC_TYPE","sw")
+HW_DEVICE=os.getenv("HW_DEVICE",None)
 
 fps_regex = re.compile(
             r"\s*frame=\s*(?P<frame_count>\d+)\s*fps=\s*(?P<fps>\d+\.?\d*).*"
@@ -109,7 +110,7 @@ def process_stream_vods(msg):
 
     if zk.process_start():
         try:
-            cmd = FFMpegCmd(ARCHIVE_ROOT+"/"+stream_name, target_root+"/"+stream_name, stream_type, params=stream_parameters, hw=hw, loop=loop).cmd()
+            cmd = FFMpegCmd(ARCHIVE_ROOT+"/"+stream_name, target_root+"/"+stream_name, stream_type, params=stream_parameters, acc_type=HW_ACC_TYPE, loop=loop, device=HW_DEVICE).cmd()
             if cmd:
                 print(cmd, flush=True)
                 r = execute(idx, stream_name, cmd)
@@ -155,7 +156,7 @@ def process_stream_lives(msg):
 
     if zk.process_start():
         try:
-            cmd = FFMpegCmd(ARCHIVE_ROOT+"/"+stream_name, target_name, stream_type, params=stream_parameters, hw=hw, loop=loop).cmd()
+            cmd = FFMpegCmd(ARCHIVE_ROOT+"/"+stream_name, target_name, stream_type, params=stream_parameters, acc_type=HW_ACC_TYPE, loop=loop, device=HW_DEVICE).cmd()
 
             if cmd:
                 print(cmd, flush=True)