Kinect and OpenCV

After struggling several days with all stuff related to OpenKinect (libfreenect) and Microsoft Visual Studio 2008, finally I could execute the experiment on getting the Kinect RGB-Depth image wrapped with the OpenCV2.1 library functions.


Special thanks to Tisham Dhar who wrote a very nice article on his blog   :
http://whatnicklife.blogspot.com
You can access the source code from his google code page : freenectopencv.cpp

Or, the below code is taken from Tisham’s page which then combined with Canny Filter operation :
(comment out all the glview.c code and replace with the below source code)

/* freenectopencv.cpp
Copyright (C) 2010 Arne Bernin
This code is licensed to you under the terms of the GNU GPL, version 2 or version 3;
see:
http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt
http://www.gnu.org/licenses/gpl-3.0.txt
*/

/*
 * Makefile for ubuntu, assumes that libfreenect.a is in /usr/lib, and libfreenect.h is in /usr/include
 *
 * make sure you have the latest version of freenect from git!

***************************************************************************************************************************
* Makefile
***************************************************************************************************************************
CXXFLAGS = -O2 -g -Wall -fmessage-length=0 `pkg-config opencv --cflags ` -I /usr/include/libusb-1.0

OBJS = freenectopencv.o

LIBS = `pkg-config opencv --libs` -lfreenect

TARGET = kinectopencv

$(TARGET): $(OBJS)
$(CXX) -o $(TARGET) $(OBJS) $(LIBS)

all: $(TARGET)

clean:
rm -f $(OBJS) $(TARGET)

*************************************************************************************************** * End of Makefile
***************************************************************************************************
*/

#include <stdio.h>
#include <string.h>
#include <math.h>

#include <libfreenect.h>
#include <pthread.h>

#define CV_NO_BACKWARD_COMPATIBILITY

#include <cv.h>
#include <highgui.h>

#define FREENECTOPENCV_WINDOW_D "Depthimage"
#define FREENECTOPENCV_WINDOW_N "Normalimage"
#define FREENECTOPENCV_RGB_DEPTH 3
#define FREENECTOPENCV_DEPTH_DEPTH 1
#define FREENECTOPENCV_RGB_WIDTH 640
#define FREENECTOPENCV_RGB_HEIGHT 480
#define FREENECTOPENCV_DEPTH_WIDTH 640
#define FREENECTOPENCV_DEPTH_HEIGHT 480

IplImage* depthimg = 0;
IplImage* rgbimg = 0;
IplImage* tempimg = 0;
IplImage* canny_img = 0;
IplImage* canny_temp = 0;
pthread_mutex_t mutex_depth = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_t mutex_rgb = PTHREAD_MUTEX_INITIALIZER;
pthread_t cv_thread;

// callback for depthimage, called by libfreenect
void depth_cb(freenect_device *dev, void *depth, uint32_t timestamp)

{
        cv::Mat depth8;
        cv::Mat mydepth = cv::Mat( FREENECTOPENCV_DEPTH_WIDTH,FREENECTOPENCV_DEPTH_HEIGHT, CV_16UC1, depth);

        mydepth.convertTo(depth8, CV_8UC1, 1.0/4.0);
        pthread_mutex_lock( &mutex_depth );
        memcpy(depthimg->imageData, depth8.data, 640*480);
        // unlock mutex
        pthread_mutex_unlock( &mutex_depth );

}

// callback for rgbimage, called by libfreenect

void rgb_cb(freenect_device *dev, void *rgb, uint32_t timestamp)
{

        // lock mutex for opencv rgb image
        pthread_mutex_lock( &mutex_rgb );
        memcpy(rgbimg->imageData, rgb, FREENECT_VIDEO_RGB_SIZE);
        // unlock mutex
        pthread_mutex_unlock( &mutex_rgb );
}

/*
 * thread for displaying the opencv content
 */
void *cv_threadfunc (void *ptr) {
        cvNamedWindow( FREENECTOPENCV_WINDOW_D, CV_WINDOW_AUTOSIZE );
        cvNamedWindow( FREENECTOPENCV_WINDOW_N, CV_WINDOW_AUTOSIZE );
		cvNamedWindow( "Canny Image", CV_WINDOW_AUTOSIZE );
		cvNamedWindow( "Depth Canny", CV_WINDOW_AUTOSIZE );
        depthimg = cvCreateImage(cvSize(FREENECTOPENCV_DEPTH_WIDTH, FREENECTOPENCV_DEPTH_HEIGHT), IPL_DEPTH_8U, FREENECTOPENCV_DEPTH_DEPTH);
        rgbimg = cvCreateImage(cvSize(FREENECTOPENCV_RGB_WIDTH, FREENECTOPENCV_RGB_HEIGHT), IPL_DEPTH_8U, FREENECTOPENCV_RGB_DEPTH);
        tempimg = cvCreateImage(cvSize(FREENECTOPENCV_RGB_WIDTH, FREENECTOPENCV_RGB_HEIGHT), IPL_DEPTH_8U, FREENECTOPENCV_RGB_DEPTH);
		canny_img = cvCreateImage(cvSize(FREENECTOPENCV_RGB_WIDTH, FREENECTOPENCV_RGB_HEIGHT), IPL_DEPTH_8U, 1);
		canny_temp = cvCreateImage(cvSize(FREENECTOPENCV_DEPTH_WIDTH, FREENECTOPENCV_DEPTH_HEIGHT), IPL_DEPTH_8U, FREENECTOPENCV_DEPTH_DEPTH);

        // use image polling
        while (1) {
                //lock mutex for depth image
                pthread_mutex_lock( &mutex_depth );
                // show image to window
                cvCanny(depthimg, canny_temp, 50.0, 200.0, 3);
				cvCvtColor(depthimg,tempimg,CV_GRAY2BGR);
                cvCvtColor(tempimg,tempimg,CV_HSV2BGR);

                cvShowImage(FREENECTOPENCV_WINDOW_D,tempimg);
				cvShowImage("Depth Canny", canny_temp);
                //unlock mutex for depth image
                pthread_mutex_unlock( &mutex_depth );

                //lock mutex for rgb image
                pthread_mutex_lock( &mutex_rgb );
                // show image to window
                cvCvtColor(rgbimg,tempimg,CV_BGR2RGB);
				cvCvtColor(tempimg, canny_img, CV_BGR2GRAY);
                cvShowImage(FREENECTOPENCV_WINDOW_N, tempimg);

				// Canny filter
				cvCanny(canny_img, canny_img, 50.0, 200.0, 3);
				cvShowImage("Canny Image", canny_img);
                //unlock mutex
                pthread_mutex_unlock( &mutex_rgb );

                // wait for quit key
                if( cvWaitKey( 15 )==27 )
                                break;

        }
        pthread_exit(NULL);

		return NULL;

}

int main(int argc, char **argv)
{

        freenect_context *f_ctx;
        freenect_device *f_dev;

        int res = 0;
        int die = 0;
        printf("Kinect camera test\n");

        if (freenect_init(&f_ctx, NULL) < 0) {
                        printf("freenect_init() failed\n");
                        return 1;
                }

                if (freenect_open_device(f_ctx, &f_dev, 0) < 0) {
                        printf("Could not open device\n");
                        return 1;
                }

        freenect_set_depth_callback(f_dev, depth_cb);
        freenect_set_video_callback(f_dev, rgb_cb);
        freenect_set_video_format(f_dev, FREENECT_VIDEO_RGB);

        // create opencv display thread
        res = pthread_create(&cv_thread, NULL, cv_threadfunc, (void*) depthimg);
        if (res) {
                printf("pthread_create failed\n");
                return 1;
        }

        printf("init done\n");

        freenect_start_depth(f_dev);
        freenect_start_video(f_dev);

        while(!die && freenect_process_events(f_ctx) >= 0 );

}

Please notice that I am using : libfreenect for Windows + Microsoft Visual Studio 9 (2008) + OpenCV 2.1

8 responses to this post.

  1. Posted by Heinrich on July 11, 2011 at 8:52 am

    Excellent sir ^^
    I will buy it tomorrow and try it.

    Reply

  2. Posted by Heinrich on July 14, 2011 at 1:26 pm

    I’ve just bought Kinect. I spent lot of time to read document about it, it’s quite awsome. Now it’s available to down load Kinect SDK from Microsoft, and it requires VS2010.
    We should focus on Kinect SDK, i though. http://research.microsoft.com/en-us/um/redmond/projects/kinectsdk/default.aspx

    Otherwise, there are some useful support from OpenNI : http://www.studentguru.gr/blogs/vangos/archive/2011/01/20/how-to-successfully-install-kinect-windows-openni-nite.aspx

    I’m considering which one is better ^^

    Now i’m taking an exam at shool, keep your good work.
    Really nice to keep in touch with you. My yahoo ID: vanduc@ymail.com

    Reply

  3. Posted by Dhiral on September 19, 2011 at 4:58 pm

    Hi,

    I tried to use this code to connect to Kinect using libfreenect. But I get error that ‘FREENECT_VIDEO_RGB_SIZE’ : undeclared identifier
    ‘freenect_set_video_format’: identifier not found

    My libfreenect.h file does not define this macro and function. When I use different libfreenect.h, I get error that cv_threadfunc should return value.

    I am very new with libfreenect. I wish to get image stream from the kinect and use canny edge algorithm on it. Please help me if I am missing something ?

    Thanks !

    Reply

  4. Hi, I am trying to run this program on my laptop and hope it will work 🙂
    however I got a “cv.h no such a file or directory” error at line 46. Do you know how to fix that?

    Reply

  5. Posted by huangmaoyi on March 3, 2013 at 10:30 pm

    Hi I try to run this program on ubuntu11.10 and hope it will work for me. however I got a “cv.h no such file or directory” error at line 46. Do you know how to fix that? thx!

    Reply

  6. Posted by hendry on April 12, 2014 at 5:02 am

    assalamualaikum mas, i proud of you 😀
    anyway, i have no idea at all after several days try to install Freenect on SimpleCV, and made me so frustated. so, could you tell me how to do that with simplest way based on your experience on OpenCV, Please. (salam kenal dari hendri di jogja)

    Reply

    • Wa’alaikumussalam, salam kenal ya!

      I never tried to play with SimpleCV before. So i would suggest you to do some more research regarding the problem.
      But yes, the basic algorithm would be likely the same as implemented by OpenCV to communicate with Freenect (?). You may refer the algorithm by the provided link above.
      Hope you could find the solution.

      Reply

  7. Posted by anis on April 30, 2014 at 1:22 pm

    assalamualaikum,
    how can I do this with MFC application?
    what should I look?

    Reply

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s

%d bloggers like this: