Cursor Control using specific Hand Gesture (OpenCV Libraries in C++):

Overview:

This small software is a semester project for the course Data Structure and Algorithms. The software uses OpenCV libraries in C++ to detect a specific hand gesture and after that it assigns the control of the windows cursor to the movement of hand. In this software the this specific gesture is connecting the tip of index finger with the tip of thumb. In this position you got the control of the cursor. And when you show your palm (gesture vanished) the cursor will be controlled by mouse/touch-pad only. watch the video carefully for better understanding of the idea.

Video Demo:

Platform:

Windows 7 / 8 / 10

Download and Install:

1. Download the project .rar file from the this  dropbox link.

2. Open Project in Visual Studio Professional. Make sure you have Opencv 3.2 libraries installed in C:\ directory and in Environment Variables.

3. Build and Run.

Solution Workflow:

• Capturing video from webcam
• Detection of hand and finger tips.
• Detecting the contour of the largest area.
• Comparing the radius of the contour with specific value (given in the code).
• If the radius lies in the range, assign the coordinates of the center of circle to cursor.

A look at Code:

Lets us have a quick overview of the code that what it is actually doing and how it is doing so.

1.  First of all some necessary libraries and header files to include.
2.  Global Variables, camframe (frame captured from the webcam), mainframe (processing will be done on this one, it is actually a copy of the camframe).  canny_output (frame in which edges are detected). bsframe (frame after background subtraction if one wants to use bs).
3. A point which gets the value of your current screen resolution.
4. ratiox and ratioy varibles for the adjusting the ratio for the webcam image and your current screen (for the movement of cursor) which results in the matching the position of hand in image and cursor on screen.
5. Some function prototypes.
6. contour_hull function is where the main image processing comes in.
7. GetDesktopResolution() is used to find the current screen resolution.

Working of the Code:

1. Finding the screen resolution.
2. Capturing frames from webcam in BGR (blue, green, red format) in camframe.
   
3. Shift to mainframe (rest of the working on the mainframe).
4. Converting to Grayscale.
   
5. Apply Gaussian Blur (so that only major edges should remain to detect).
   
6. Applying a certain threshold (controlled by the user).
   
7. Using canny edge detector to find edges.
   
8. Finding contours.
   
9. Constructing Hulls from contours.
   
10. Drawing the Hull of largest area on camframe. 
11. Drawing the defect points (finger tips).
    
12. Drawing square and Circle around the largest Hull.
    
13. Finding the radius of the circle and checking it for a certain range.
    
14. If it lies in that range (25 < radius < 40) then assign the coordinates of the center of the circle after multiplying it with a proper factor (arise due to difference in webcam resolution and screen resolution)  to the cursor using function SetCursorPos() of windows.h.
    

Code:

//---------libraries----------

#include "opencv2/opencv.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/videoio.hpp"
#include <opencv2/highgui.hpp>
#include <opencv2/video.hpp>
#include <opencv2/core/core.hpp>
#include <stdio.h>
#include <iostream>
#include <sstream>
#include <vector>
#include <fstream>
#include <math.h>
#include <windows.h>
#include "wtypes.h"

using namespace cv;
using namespace std;

//-----------Global variables--------------

Mat camframe, mainframe, canny_output; //current frame
Mat bsframe; //if use background subtraction, foreground mask generated by MOG2 method

Ptr<BackgroundSubtractor> pMOG2; //MOG2 Background subtractor

int thresh = 110;
int max_thresh = 255;
RNG rng(12345);

Point scrres = 0;
double ratiox = 0.00;
double ratioy = 0.00;

//---------function prototypes-------------

void contour_hull(Mat &img, Mat &writimg);
void condefects(vector<Vec4i> convexityDefectsSet, vector<Point> mycontour, Mat &original);
void applybs(Mat &camframe);
void GetDesktopResolution(int& horizontal, int& vertical);

void main()
{
VideoCapture cap;
if (!cap.open(0))
return;

namedWindow("camframe", WINDOW_KEEPRATIO);
//namedWindow("mainframe", WINDOW_AUTOSIZE);
createTrackbar("Threshold", "camframe", &thresh, 255);

bool s = false;
pMOG2 = createBackgroundSubtractorMOG2(30, 80, s); //MOG2 approach

GetDesktopResolution(scrres.x, scrres.y);

while (true)
{
cap.retrieve(camframe, CV_CAP_OPENNI_BGR_IMAGE);
if (camframe.empty())
break;
flip(camframe, camframe, 1);

//hand detection without BS
cvtColor(camframe, mainframe, CV_BGR2GRAY);
blur(mainframe, mainframe, Size(3, 3));
contour_hull(mainframe, camframe);

////with background subtraction
//applybs(camframe);
//contour_hull(mainframe, camframe);

//displaying output
imshow("camframe", camframe);
//imshow("mainframe", mainframe);
                //ESC to exit
if (waitKey(1) == 27)
break;
}
}

//------function to apply background subtraction------

void applybs(Mat &camframe)
{
pMOG2->apply(camframe, bsframe);
GaussianBlur(bsframe, bsframe, Size(5, 5), 3.5, 3.5);
threshold(bsframe, bsframe, 50, 255, THRESH_BINARY);
mainframe = Scalar::all(0);
camframe.copyTo(mainframe, bsframe);
}

//------function to find and draw countours, hulls and defects------

void contour_hull(Mat &img, Mat &writimg)
{
double a, largest_area = 0;
int largest_contour_index = 0;

vector<vector<Point> > contours;
vector<Vec4i> hierarchy;

//  applying threshold
threshold(img, canny_output, thresh, 255, THRESH_BINARY);

//  Detect edges using canny
Canny(img, canny_output, thresh, thresh * 3, 3);

//  Find contours
findContours(canny_output, contours, hierarchy, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));

//   initializing Hulls
vector<vector<Point> >hull(contours.size());
vector<vector<int> >inthull(contours.size());
vector<vector<Vec4i> >defects(contours.size());

//  initializing set of points for squre and circle around the contour
vector<vector<Point> > contours_poly(contours.size());
vector<Rect> boundRect(contours.size());
vector<Point2f>center(contours.size());
vector<float>radius(contours.size());

//  constructing hull from contour points
for (int i = 0; i < contours.size(); i++)
{
convexHull(Mat(contours[i]), hull[i], false);
convexHull(Mat(contours[i]), inthull[i], false);
if (inthull[i].size()>3)
convexityDefects(contours[i], inthull[i], defects[i]);
}

//  putting circles and square around all contours
for (int i = 0; i < contours.size(); i++)
{
approxPolyDP(Mat(contours[i]), contours_poly[i], 3, true);
boundRect[i] = boundingRect(Mat(contours_poly[i]));
minEnclosingCircle((Mat)contours_poly[i], center[i], radius[i]);
}

//  Finding the largest contour 
for (int i = 0; i< contours.size(); i++) // iterate through each contour. 
{
a = contourArea(contours[i], false);  //  Find the area of contour
if (a>largest_area)
{
largest_area = a;
largest_contour_index = i;                //Store the index of largest contour
}
}

if (contours.size() > 0)
{
//  drawing the largest hull and contour
drawContours(writimg, contours, largest_contour_index, CV_RGB(0, 255, 0), 2, 8, hierarchy);
drawContours(writimg, hull, largest_contour_index, CV_RGB(0, 0, 255), 2, 8, hierarchy);

//  putting the square and the circle around the largest contour
//  circle(writimg, center[largest_contour_index], (int)radius[largest_contour_index], CV_RGB(0, 250, 154), 2, 8, 0);
rectangle(writimg, boundRect[largest_contour_index].tl(), boundRect[largest_contour_index].br(), CV_RGB(255, 20, 147), 2, 8, 0);

ratiox = (double) center[largest_contour_index].x / (img.cols);
ratioy = (double) center[largest_contour_index].y / (img.rows);
//controlling pointer with the center of the circle around the largest contour

if ((int)radius[largest_contour_index] < 40 && (int)radius[largest_contour_index] > 25)
SetCursorPos(ratiox*scrres.x, ratioy*scrres.y);

//if one wants to show all contours and hull use the code given below
/*for (int i = 0; i< contours.size(); i++)
{
drawContours(writimg, contours, i, Scalar(0,255,0), 2, 8, hierarchy, 0, Point());
drawContours(writimg, hull, (int)i, Scalar(0,0,255), 2, 8, vector<Vec4i>(), 0, Point());
circle(writimg, center[i], (int)radius[i], CV_RGB(255, 255, 0), 2, 8, 0);
rectangle(writimg, boundRect[i].tl(), boundRect[i].br(), CV_RGB(255, 255, 0), 2, 8, 0);
}*/

//drawing the contour defects'float' to 'DWORD', possible loss of data
condefects(defects[largest_contour_index], contours[largest_contour_index], writimg);
}
}

//--------contour defect finding function--------

void condefects(vector<Vec4i> convexityDefectsSet, vector<Point> mycontour, Mat &original)
{
int startIdx, endIdx;
for (int cDefIt = 0; cDefIt < convexityDefectsSet.size(); cDefIt++)
{

startIdx = convexityDefectsSet[cDefIt].val[0];
Point ptStart(mycontour[startIdx]);

endIdx = convexityDefectsSet[cDefIt].val[1];
Point ptEnd(mycontour[endIdx]);

//display start points
circle(original, ptStart, 4, CV_RGB(255, 0, 0), 2, 8);
//display all end points
circle(original, ptEnd, 4, CV_RGB(255, 0, 0), 2, 8);

}
}

//----------function to get desktop resolution---------

void GetDesktopResolution(int& horizontal, int& vertical)
{
RECT desktop;
// Get a handle to the desktop window
const HWND hDesktop = GetDesktopWindow();
// Get the size of screen to the variable desktop
GetWindowRect(hDesktop, &desktop);
// The top left corner will have coordinates (0,0)
// and the bottom right corner will have coordinates
// (horizontal, vertical)
horizontal = desktop.right;
vertical = desktop.bottom;

}