added augmented reality using aruco markers

Author: sunitanyk

AugmentedRealityWithArucoMarkers/README.md

@@ -0,0 +1,44 @@
+### Augmented Reality using AruCo Markers in OpenCV
+
+We show how to use the AruCo markers in OpenCV using an augmented reality application to replace the image inside a picture frame on a wall to display new images or videos in the frame.
+
+### Compilation in C++
+
+```
+g++ -std=c++11 augmented_reality_with_aruco.cpp -o augmented_reality_with_aruco.out `pkg-config --cflags --libs opencv4`
+```
+
+### How to run the code
+
+Command line usage for running the code
+
+* Python
+
+  * A single image:
+    	
+    ```
+    python3 augmented_reality_with_aruco.py --image=test.jpg
+    ```
+    
+  * A video file:
+
+    ```
+    python3 augmented_reality_with_aruco.py --video=test.mp4
+    ```       
+
+* C++:
+
+  * A single image:
+        
+    ```
+    ./augmented_reality_with_aruco.out --image=test.jpg
+    ```
+
+  * A video file:
+
+    ```
+     ./augmented_reality_with_aruco.out --video=test.mp4
+    ```
+
+### Results of YOLOv3
+<img src = "./augmented-reality-example.jpg" width = 1000 height = 282/>

AugmentedRealityWithArucoMarkers/augmented-reality-example.jpg

@@ -0,0 +1,194 @@
+// This code is written by Sunita Nayak at BigVision LLC. It is based on the OpenCV project. It is subject to the license terms in the LICENSE file found in this distribution and at http://opencv.org/license.html
+
+// Usage example:   ./augmented_reality_with_aruco.out --image=test.jpg
+//                  ./augmented_reality_with_aruco.out --video=test.mp4
+#include <fstream>
+#include <sstream>
+#include <iostream>
+
+#include <opencv2/aruco.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/highgui.hpp>
+#include <opencv2/calib3d.hpp>
+
+const char* keys =
+"{help h usage ? | | Usage examples: \n\t\t./augmented_reality_with_aruco.out --image=test.jpg \n\t\t./augmented_reality_with_aruco.out --video=test.mp4}"
+"{image i        |<none>| input image   }"
+"{video v       |<none>| input video   }"
+;
+using namespace cv;
+using namespace aruco;
+using namespace std;
+
+int main(int argc, char** argv)
+{
+    CommandLineParser parser(argc, argv, keys);
+    parser.about("Use this script to do Augmented Reality using Aruco markers in OpenCV.");
+    if (parser.has("help"))
+    {
+        parser.printMessage();
+        return 0;
+    }
+    // Open a video file or an image file or a camera stream.
+    string str, outputFile;
+    VideoCapture cap;
+    VideoWriter video;
+    Mat frame, blob;
+    
+    Mat im_src = imread("new_scenery.jpg");
+
+    try {
+        
+        outputFile = "ar_out_cpp.avi";
+        if (parser.has("image"))
+        {
+            // Open the image file
+            str = parser.get<String>("image");
+            ifstream ifile(str);
+            if (!ifile) throw("error");
+            cap.open(str);
+            str.replace(str.end()-4, str.end(), "_ar_out_cpp.jpg");
+            outputFile = str;
+        }
+        else if (parser.has("video"))
+        {
+            // Open the video file
+            str = parser.get<String>("video");
+            ifstream ifile(str);
+            if (!ifile) throw("error");
+            cap.open(str);
+            str.replace(str.end()-4, str.end(), "_ar_out_cpp.avi");
+            outputFile = str;
+        }
+        // Open the webcaom
+        else cap.open(parser.get<int>("device"));
+        
+    }
+    catch(...) {
+        cout << "Could not open the input image/video stream" << endl;
+        return 0;
+    }
+    
+    // Get the video writer initialized to save the output video
+    if (!parser.has("image")) {
+        video.open(outputFile, VideoWriter::fourcc('M','J','P','G'), 28, Size(2*cap.get(CAP_PROP_FRAME_WIDTH), cap.get(CAP_PROP_FRAME_HEIGHT)));
+    }
+    
+    // Create a window
+    static const string kWinName = "Augmented Reality using Aruco markers in OpenCV";
+    namedWindow(kWinName, WINDOW_NORMAL);
+
+    // Process frames.
+    while (waitKey(1) < 0)
+    {
+        // get frame from the video
+        cap >> frame;
+        
+        try {
+            // Stop the program if reached end of video
+            if (frame.empty()) {
+                cout << "Done processing !!!" << endl;
+                cout << "Output file is stored as " << outputFile << endl;
+                waitKey(3000);
+                break;
+            }
+
+            vector<int> markerIds;
+            
+            // Load the dictionary that was used to generate the markers.
+            Ptr<Dictionary> dictionary = getPredefinedDictionary(DICT_6X6_250);
+
+            // Declare the vectors that would contain the detected marker corners and the rejected marker candidates
+            vector<vector<Point2f>> markerCorners, rejectedCandidates;
+
+            // Initialize the detector parameters using default values
+            Ptr<DetectorParameters> parameters = DetectorParameters::create();
+
+            // Detect the markers in the image
+            detectMarkers(frame, dictionary, markerCorners, markerIds, parameters, rejectedCandidates);
+
+            // Using the detected markers, locate the quadrilateral on the target frame where the new scene is going to be displayed.
+            vector<Point> pts_dst;
+            float scalingFac = 0.02;//0.015;
+
+            Point refPt1, refPt2, refPt3, refPt4;
+
+            // finding top left corner point of the target quadrilateral
+            std::vector<int>::iterator it = std::find(markerIds.begin(), markerIds.end(), 25);
+            int index = std::distance(markerIds.begin(), it);
+            refPt1 = markerCorners.at(index).at(1);
+
+            // finding top right corner point of the target quadrilateral
+            it = std::find(markerIds.begin(), markerIds.end(), 33);
+            index = std::distance(markerIds.begin(), it);
+            refPt2 = markerCorners.at(index).at(2);
+            
+            float distance = norm(refPt1-refPt2);
+            pts_dst.push_back(Point(refPt1.x - round(scalingFac*distance), refPt1.y - round(scalingFac*distance)));
+            
+            pts_dst.push_back(Point(refPt2.x + round(scalingFac*distance), refPt2.y - round(scalingFac*distance)));
+
+            // finding bottom right corner point of the target quadrilateral
+            it = std::find(markerIds.begin(), markerIds.end(), 30);
+            index = std::distance(markerIds.begin(), it);
+            refPt3 = markerCorners.at(index).at(0);
+            pts_dst.push_back(Point(refPt3.x + round(scalingFac*distance), refPt3.y + round(scalingFac*distance)));
+
+            // finding bottom left corner point of the target quadrilateral
+            it = std::find(markerIds.begin(), markerIds.end(), 23);
+            index = std::distance(markerIds.begin(), it);
+            refPt4 = markerCorners.at(index).at(0);
+            pts_dst.push_back(Point(refPt4.x - round(scalingFac*distance), refPt4.y + round(scalingFac*distance)));
+
+            // Get the corner points of the new scene image.
+            vector<Point> pts_src;
+            pts_src.push_back(Point(0,0));
+            pts_src.push_back(Point(im_src.cols, 0));
+            pts_src.push_back(Point(im_src.cols, im_src.rows));
+            pts_src.push_back(Point(0, im_src.rows));
+
+            // Compute homography from source and destination points
+            Mat h = cv::findHomography(pts_src, pts_dst);
+
+            // Warped image
+            Mat warpedImage;
+            
+            // Warp source image to destination based on homography
+            warpPerspective(im_src, warpedImage, h, frame.size(), INTER_CUBIC);
+        
+            // Prepare a mask representing region to copy from the warped image into the original frame.
+            Mat mask = Mat::zeros(frame.rows, frame.cols, CV_8UC1);
+            fillConvexPoly(mask, pts_dst, Scalar(255, 255, 255), LINE_AA);
+            
+            // Erode the mask to not copy the boundary effects from the warping
+            Mat element = getStructuringElement( MORPH_RECT, Size(5,5));
+//            Mat element = getStructuringElement( MORPH_RECT, Size(3,3));
+            erode(mask, mask, element);
+
+            // Copy the warped image into the original frame in the mask region.
+            Mat imOut = frame.clone();
+            warpedImage.copyTo(imOut, mask);
+            
+            // Showing the original image and the new output image side by side
+            Mat concatenatedOutput;
+            hconcat(frame, imOut, concatenatedOutput);
+
+            if (parser.has("image")) imwrite(outputFile, concatenatedOutput);
+            else video.write(concatenatedOutput);
+
+            imshow(kWinName, concatenatedOutput);
+            
+        }
+        catch(const std::exception& e) {
+            cout << endl << " e : " << e.what() << endl;
+            cout << "Could not do homography !! " << endl;
+    //        return 0;
+        }
+
+    }
+
+    cap.release();
+    if (!parser.has("image")) video.release();
+
+    return 0;
+}

AugmentedRealityWithArucoMarkers/augmented_reality_with_aruco.cpp

@@ -0,0 +1,134 @@
+# This code is written by Sunita Nayak at BigVision LLC. It is based on the OpenCV project. It is subject to the license terms in the LICENSE file found in this distribution and at http://opencv.org/license.html
+
+# Usage example:   python3 augmented_reality_with_aruco.py --image=test.jpg
+#                  python3 augmented_reality_with_aruco.py --video=test.mp4
+
+import cv2 as cv
+#from cv2 import aruco
+import argparse
+import sys
+import os.path
+import numpy as np
+
+parser = argparse.ArgumentParser(description='Augmented Reality using Aruco markers in OpenCV')
+parser.add_argument('--image', help='Path to image file.')
+parser.add_argument('--video', help='Path to video file.')
+args = parser.parse_args()
+
+im_src = cv.imread("new_scenery.jpg");
+
+outputFile = "ar_out_py.avi"
+if (args.image):
+    # Open the image file
+    if not os.path.isfile(args.image):
+        print("Input image file ", args.image, " doesn't exist")
+        sys.exit(1)
+    cap = cv.VideoCapture(args.image)
+    outputFile = args.image[:-4]+'_ar_out_py.jpg'
+elif (args.video):
+    # Open the video file
+    if not os.path.isfile(args.video):
+        print("Input video file ", args.video, " doesn't exist")
+        sys.exit(1)
+    cap = cv.VideoCapture(args.video)
+    outputFile = args.video[:-4]+'_ar_out_py.avi'
+    print("Storing it as :", outputFile)
+else:
+    # Webcam input
+    cap = cv.VideoCapture(0)
+
+# Get the video writer initialized to save the output video
+if (not args.image):
+    vid_writer = cv.VideoWriter(outputFile, cv.VideoWriter_fourcc('M','J','P','G'), 28, (round(2*cap.get(cv.CAP_PROP_FRAME_WIDTH)),round(cap.get(cv.CAP_PROP_FRAME_HEIGHT))))
+
+winName = "Augmented Reality using Aruco markers in OpenCV"
+
+while cv.waitKey(1) < 0:
+    try:
+        # get frame from the video
+        hasFrame, frame = cap.read()
+        
+        # Stop the program if reached end of video
+        if not hasFrame:
+            print("Done processing !!!")
+            print("Output file is stored as ", outputFile)
+            cv.waitKey(3000)
+            break
+
+        #Load the dictionary that was used to generate the markers.
+        dictionary = cv.aruco.Dictionary_get(cv.aruco.DICT_6X6_250)
+        
+        # Initialize the detector parameters using default values
+        parameters =  cv.aruco.DetectorParameters_create()
+        
+        # Detect the markers in the image
+        markerCorners, markerIds, rejectedCandidates = cv.aruco.detectMarkers(frame, dictionary, parameters=parameters)
+
+        index = np.squeeze(np.where(markerIds==25));
+        refPt1 = np.squeeze(markerCorners[index[0]])[1];
+        
+        index = np.squeeze(np.where(markerIds==33));
+        refPt2 = np.squeeze(markerCorners[index[0]])[2];
+
+        distance = np.linalg.norm(refPt1-refPt2);
+        
+        scalingFac = 0.02;
+        pts_dst = [[refPt1[0] - round(scalingFac*distance), refPt1[1] - round(scalingFac*distance)]];
+        pts_dst = pts_dst + [[refPt2[0] + round(scalingFac*distance), refPt2[1] - round(scalingFac*distance)]];
+        
+        index = np.squeeze(np.where(markerIds==30));
+        refPt3 = np.squeeze(markerCorners[index[0]])[0];
+        pts_dst = pts_dst + [[refPt3[0] + round(scalingFac*distance), refPt3[1] + round(scalingFac*distance)]];
+
+        index = np.squeeze(np.where(markerIds==23));
+        refPt4 = np.squeeze(markerCorners[index[0]])[0];
+        pts_dst = pts_dst + [[refPt4[0] - round(scalingFac*distance), refPt4[1] + round(scalingFac*distance)]];
+
+        pts_src = [[0,0], [im_src.shape[1], 0], [im_src.shape[1], im_src.shape[0]], [0, im_src.shape[0]]];
+        
+        pts_src_m = np.asarray(pts_src)
+        pts_dst_m = np.asarray(pts_dst)
+
+        # Calculate Homography
+        h, status = cv.findHomography(pts_src_m, pts_dst_m)
+        
+        # Warp source image to destination based on homography
+        warped_image = cv.warpPerspective(im_src, h, (frame.shape[1],frame.shape[0]))
+        
+        # Prepare a mask representing region to copy from the warped image into the original frame.
+        mask = np.zeros([frame.shape[0], frame.shape[1]], dtype=np.uint8);
+        cv.fillConvexPoly(mask, np.int32([pts_dst_m]), (255, 255, 255), cv.LINE_AA);
+
+        # Erode the mask to not copy the boundary effects from the warping
+        element = cv.getStructuringElement(cv.MORPH_RECT, (3,3));
+        mask = cv.erode(mask, element, iterations=3);
+
+        # Copy the mask into 3 channels.
+        warped_image = warped_image.astype(float)
+        mask3 = np.zeros_like(warped_image)
+        for i in range(0, 3):
+            mask3[:,:,i] = mask/255
+
+        # Copy the warped image into the original frame in the mask region.
+        warped_image_masked = cv.multiply(warped_image, mask3)
+        frame_masked = cv.multiply(frame.astype(float), 1-mask3)
+        im_out = cv.add(warped_image_masked, frame_masked)
+        
+        # Showing the original image and the new output image side by side
+        concatenatedOutput = cv.hconcat([frame.astype(float), im_out]);
+        cv.imshow("AR using Aruco markers", concatenatedOutput.astype(np.uint8))
+        
+        # Write the frame with the detection boxes
+        if (args.image):
+            cv.imwrite(outputFile, concatenatedOutput.astype(np.uint8));
+        else:
+            vid_writer.write(concatenatedOutput.astype(np.uint8))
+
+
+    except Exception as inst:
+        print(inst)
+
+cv.destroyAllWindows()
+if 'vid_writer' in locals():
+    vid_writer.release()
+    print('Video writer released..')

AugmentedRealityWithArucoMarkers/augmented_reality_with_aruco.py

@@ -0,0 +1,15 @@
+#include <opencv2/highgui.hpp>
+#include <opencv2/aruco.hpp>
+
+using namespace cv;
+
+int main(int argc, char *argv[]) {
+
+    Mat markerImage;
+    Ptr<cv::aruco::Dictionary> dictionary = aruco::getPredefinedDictionary(cv::aruco::DICT_6X6_250);
+
+    aruco::drawMarker(dictionary, 33, 200, markerImage, 1);
+
+    imwrite("marker33.png", markerImage);
+
+}