update 133-135(Colorization, ENet, Style-transfer

Author: JimmyHHua

README.md

@@ -170,4 +170,15 @@ code_128 | [DNN- Call the Detection Model of Tensorflow](python/code_128)   | 
 code_129 | [DNN- Call the Openpose Implementation Attitude Assessment](python/code_129)   | ✔️
 code_130 | [DNN- Call YOLO Object Detection Network](python/code_130)   | ✔️
 code_131 | [DNN- YOLOv3-tiny Real-time Object Detection](python/code_131)   | ✔️
-code_132 | [DNN- Single and Multiple Image Detection](python/code_132)   | ✔️
+code_132 | [DNN- Single and Multiple Image Detection](python/code_132)   | ✔️
+code_133 | [DNN- Colorful Image Colorization ](python/code_133)   | ✔️
+code_134 | [DNN- ENet Image Segmentation](python/code_134)   | ✔️
+code_135 | [DNN- Real-time Fast Image Style Transfer](python/code_135)   | ✔️
+
+---
+
+### Appendix
+
+⛳️ The weight can be download from Google Driver：
+
+🌱 [Weight for DNN](https://drive.google.com/drive/folders/1mg6VXpkvEmyL1scaelX5FnW8uw1gk9iq?usp=sharing)

README_CN.md

@@ -170,3 +170,14 @@ code_129 | [DNN- 调用openpose模型实现姿态评估](python/code_129)   | 
 code_130 | [DNN- 调用YOLO对象检测网络](python/code_130)   | ✔️
 code_131 | [DNN- YOLOv3-tiny版本实时对象检测](python/code_131)   | ✔️
 code_132 | [DNN- 单张与多张图像的推断](python/code_132)   | ✔️
+code_133 | [DNN- 图像颜色化模型使用 ](python/code_133)   | ✔️
+code_134 | [DNN- ENet实现图像分割](python/code_134)   | ✔️
+code_135 | [DNN- 实时快速的图像风格迁移](python/code_135)   | ✔️
+
+---
+
+### 附录
+
+⛳️ DNN模块的一些模型下载可以从下面的谷歌云中获取：
+
+🌱 [Weight for DNN](https://drive.google.com/drive/folders/1mg6VXpkvEmyL1scaelX5FnW8uw1gk9iq?usp=sharing)

python/code_133/2.jpg

@@ -0,0 +1,26 @@
+## OpenCV DNN 图像颜色化模型使用 
+
+OpenCV DNN在4.0还支持灰度图像的彩色化模型，是根据2016年ECCV的论文而来，基于卷积神经网络模型，通过对Lab色彩空间进行量化分割，映射到最终的CNN输出结果，最后转换为RGB彩色图像。
+
+相关论文详见：
+
+Arxiv: [https://arxiv.org/pdf/1603.08511.pdf](https://arxiv.org/pdf/1603.08511.pdf)
+
+OpenCV DNN使用该模型时候，除了正常的Caffe模型与配置文件之外，还需要一个Lab的量化表。
+
+原图：
+
+<img src='2.jpg'>
+
+Colorization：
+
+<img src='2color.jpg'>
+
+
+原图：
+
+<img src='4.jpg'>
+
+Colorization：
+
+<img src='4color.jpg'>

python/code_133/2color.jpg

@@ -0,0 +1,53 @@
+import numpy as np
+import cv2 as cv
+
+W_in = 224
+H_in = 224
+modelTxt = "../model/color/colorization_deploy_v2.prototxt";
+modelBin = "../model/color/colorization_release_v2.caffemodel";
+pts_txt = "../model/color/pts_in_hull.npy";
+
+# Select desired model
+net = cv.dnn.readNetFromCaffe(modelTxt, modelBin)
+pts_in_hull = np.load(pts_txt) # load cluster centers
+
+# populate cluster centers as 1x1 convolution kernel
+pts_in_hull = pts_in_hull.transpose().reshape(2, 313, 1, 1)
+net.getLayer(net.getLayerId('class8_ab')).blobs = [pts_in_hull.astype(np.float32)]
+net.getLayer(net.getLayerId('conv8_313_rh')).blobs = [np.full([1, 313], 2.606, np.float32)]
+
+frame = cv.imread("4.jpg")
+h, w = frame.shape[:2]
+img_rgb = (frame[:,:,[2, 1, 0]] * 1.0 / 255).astype(np.float32)
+
+img_lab = cv.cvtColor(img_rgb, cv.COLOR_RGB2Lab)
+img_l = img_lab[:,:,0] # pull out L channel
+(H_orig,W_orig) = img_rgb.shape[:2] # original image size
+
+# resize image to network input size
+img_rs = cv.resize(img_rgb, (W_in, H_in))
+img_lab_rs = cv.cvtColor(img_rs, cv.COLOR_RGB2Lab)
+img_l_rs = img_lab_rs[:,:,0]
+img_l_rs -= 50 # subtract 50 for mean-centering
+
+# run network
+net.setInput(cv.dnn.blobFromImage(img_l_rs))
+ab_dec = net.forward()[0,:,:,:].transpose((1,2,0))
+
+(H_out,W_out) = ab_dec.shape[:2]
+ab_dec_us = cv.resize(ab_dec, (W_orig, H_orig))
+img_lab_out = np.concatenate((img_l[:,:,np.newaxis],ab_dec_us),axis=2)
+
+img_bgr_out = np.clip(cv.cvtColor(img_lab_out, cv.COLOR_Lab2BGR), 0, 1)
+print(img_bgr_out.shape)
+frame = cv.resize(frame, (w, h))
+cv.imshow('origin', frame)
+gray = cv.cvtColor(frame, cv.COLOR_RGB2GRAY)
+cv.imshow('gray', gray)
+#cv.imwrite('gray.jpg', gray)
+# fix 4.0 imshow issue
+cv.normalize(img_bgr_out, img_bgr_out, 0, 255, cv.NORM_MINMAX)
+cv.imshow('colorized', cv.resize(np.uint8(img_bgr_out), (w, h)))
+cv.imwrite('4color.jpg', cv.resize(np.uint8(img_bgr_out), (w, h)))
+cv.waitKey(0)
+cv.destroyAllWindows()

python/code_133/4.jpg

@@ -0,0 +1,11 @@
+## OpenCV DNN ENet实现图像分割
+
+OpenCV DNN支持ENet网络模型的图像分割，这里采用的预先训练的ENet网络：
+
+[GitHub - e-lab/ENet-training](https://github.com/e-lab/ENet-training)
+
+基于Cityscapes数据集的训练预测结果
+ 
+<img src=street.jpg>
+
+<img src=enet_result.png>

python/code_133/4color.jpg

@@ -0,0 +1,33 @@
+import cv2 as cv
+import numpy as np
+
+# load CNN model
+bin_model = "../model/enet/model-best.net";
+net = cv.dnn.readNetFromTorch(bin_model)
+# read input data
+frame = cv.imread("street.jpg")
+frame = cv.resize(frame,(0,0), fx=0.5, fy=0.5)
+blob = cv.dnn.blobFromImage(frame, 0.00392, (512, 256), (0, 0, 0), True, False);
+cv.imshow("input", frame)
+h, w, c = frame.shape
+
+# Run a model
+net.setInput(blob)
+score = net.forward()
+# Put efficiency information.
+t, _ = net.getPerfProfile()
+label = 'Inference time: %.2f ms' % (t * 1000.0 / cv.getTickFrequency())
+score = np.squeeze(score)
+score = score.transpose((1, 2, 0))
+score = np.argmax(score, 2)
+mask = np.uint8(score)
+mask = cv.cvtColor(mask, cv.COLOR_GRAY2BGR)
+cv.normalize(mask, mask, 0, 255, cv.NORM_MINMAX)
+cmask = cv.applyColorMap(mask, cv.COLORMAP_JET)
+cmask = cv.resize(cmask, (w, h))
+dst = cv.addWeighted(frame, 0.7, cmask, 0.3, 0)
+cv.putText(dst, label, (50, 50), cv.FONT_HERSHEY_SIMPLEX, 0.75, (0, 0, 255), 2)
+cv.imshow("dst", dst)
+cv.imwrite("enet_result.png", dst)
+cv.waitKey(0)
+cv.destroyAllWindows()

python/code_133/README.md

@@ -0,0 +1,27 @@
+## OpenCV DNN 实时快速的图像风格迁移
+
+OpenCV DNN模块现在还支持图像风格迁移网络模型的加载与使用，支持的模型是基于李飞飞等人在论文《Perceptual Losses for Real-Time Style Transfer and Super-Resolution》中提到的快速图像风格迁移网络，基于感知损失来提取特征，生成图像特征与高分辨率图像。
+
+整个网络模型是基于DCGAN + 5个残差层构成，是一个典型的全卷积网络。
+
+模型下载地址:
+
+[GitHub - jcjohnson/fast-neural-style](https://github.com/jcjohnson/fast-neural-style)
+
+这个网络可以支持任意尺寸的图像输入，作者提供了很多种预训练的风格迁移模型提供使用，我下载了下面的预训练模型。：
+- composition_vii.t7
+- starry_night.t7
+- la_muse.t7
+- the_wave.t7
+- mosaic.t7
+- the_scream.t7
+- feathers.t7
+- candy.t7
+- udnie.t7
+
+这些模型都是torch框架支持的二进制权重文件，加载模型之后，就可以调用forward得到结果，通过对输出结果反向加上均值，rescale到0~255的RGB色彩空间，即可显示。
+
+Tyep | Image
+---|---
+Feathers.t7 | <img src=result_6.png>
+Candy.t7 | <img src=result_7.png>

python/code_133/opencv_133.py

@@ -0,0 +1,54 @@
+import cv2 as cv
+import numpy as np
+
+base_dir = "../model/fast_style/"
+styles = ["composition_vii.t7", "starry_night.t7", "la_muse.t7", "the_wave.t7",
+          "mosaic.t7", "the_scream.t7", "feathers.t7", "candy.t7", "udnie.t7"]
+index = 6
+net = cv.dnn.readNetFromTorch(base_dir + styles[index])
+net.setPreferableBackend(cv.dnn.DNN_BACKEND_OPENCV);
+cap = cv.VideoCapture(0)
+while cv.waitKey(1) < 0:
+    hasFrame, frame = cap.read()
+    if not hasFrame:
+        cv.waitKey()
+        break
+    cv.imshow("frame", frame)
+    inWidth = 256
+    inHeight = 256
+    h, w = frame.shape[:2]
+    inp = cv.dnn.blobFromImage(frame, 1.0, (inWidth, inHeight),
+                              (103.939, 116.779, 123.68), swapRB=False, crop=False)
+
+    # 执行风格迁移
+    net.setInput(inp)
+    out = net.forward()
+    print(out.shape)
+    t, _ = net.getPerfProfile()
+    freq = cv.getTickFrequency() / 1000
+    label = "FPS : %.2f" % (1000 / (t / freq))
+
+    # 解析输出
+    out = out.reshape(3, out.shape[2], out.shape[3])
+    print("ddddddddd", out.shape)
+    out[0] += 103.939
+    out[1] += 116.779
+    out[2] += 123.68
+    out /= 255.0
+    out = out.transpose(1, 2, 0)
+    print("new shape", out.shape)
+    #out = np.random.random((256,256,3))
+    out = np.clip(out, 0, 1)
+    out = (255 * out).astype('uint8')
+    # rescale与中值模糊，消除极值点噪声
+    #cv.normalize(out, out, 0, 255, cv.NORM_MINMAX)
+
+    out = cv.medianBlur(out, 5)
+
+    # resize and show
+    result = np.uint8(cv.resize(out, (w, h)))
+    cv.putText(result, label, (5, 25), cv.FONT_HERSHEY_SIMPLEX, 0.75, (0, 0, 255), 2)
+    cv.imshow('Fast Style Demo', result)
+    cv.imwrite("result_%d.png"%index, result)
+
+

python/code_134/README.md

@@ -0,0 +1,20 @@
+Unlabeled
+Road
+Sidewalk
+Building
+Wall
+Fence
+Pole
+TrafficLight
+TrafficSign
+Vegetation
+Terrain
+Sky
+Person
+Rider
+Car
+Truck
+Bus
+Train
+Motorcycle
+Bicycle