Convert a fisheye image to an equirectangular image with opencv4

Question

I want to transform a single round fisheye image to an equirectangular image with a C++ algorithm and OpenCV4.

The idea is from a input image loaded on my computer like this :

I want to obtain an output image like this :

I'm using the method described on this blog : http://paulbourke.net/dome/dualfish2sphere/

The method can be described by this picture :

Unfortunately when I run my code, I obtain something like this :

I'm working on a MacOSX with Xcode and I use Terminal "ITerm2" to build and execute my code.

The code is the following :

#include <iostream>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/calib3d/calib3d.hpp>
#include <opencv2/highgui/highgui.hpp>

using namespace std;
using namespace cv;

const double PI = 3.141592653589793;
const string PATH_IMAGE = "/Users/Kenza/Desktop/Xcode_cpp_opencv/PaulBourke2/PaulBourke2/Images/img1.jpg";
const int ESC = 27;

Point2f findCorrespondingFisheyePoint(int Xe, int Ye, double He, double We, double Hf, double Wf, double FOV){
    Point2f fisheyePoint;
    double Xfn, Yfn; //Normalized Cartesian Coordinates
    double longitude, latitude, Px, Py, Pz; //Spherical Coordinates
    double r, theta; //Polar coordinates
    double Xpn, Ypn; //Normalized Polar coordinates

    //Normalize Coordinates
    Xfn = ( ( 2.0 * (double)Xe ) - We) / Wf;//Between -1 and 1
    Yfn = ( ( 2.0 * (double)Ye ) - He) / Hf;//Between -1 and 1

    //Normalize Coordinates to Spherical Coordinates
    longitude = Xfn*PI; //Between -PI and PI (2*PI interval)
    latitude = Yfn*(PI/2.0); //Between -PI/2 and PI/2 (PI interval)
    Px = cos(latitude)*cos(longitude);
    Py = cos(latitude)*sin(longitude);
    Pz = sin(latitude);

    //Spherical Coordinates to Polar Coordinates
    r =  2.0 * atan2(sqrt(pow(Px,2)+pow(Pz,2)),Py)/FOV;
    theta = atan2(Pz,-Px);
    Xpn = r * cos(theta);
    Ypn = r * sin(theta);

    //Normalize Coordinates to CartesianImage Coordinates
    fisheyePoint.x = (int)(((Xpn+1.0)*Wf)/2.0);
    fisheyePoint.y = (int)(((Ypn+1.0)*Hf)/2.0);

    return fisheyePoint;
}

int main(int argc, char** argv){

    Mat fisheyeImage, equirectangularImage;

    fisheyeImage = imread(PATH_IMAGE, CV_32FC1);
    namedWindow("Fisheye Image", WINDOW_AUTOSIZE);
    imshow("Fisheye Image", fisheyeImage);

    while(waitKey(0) != ESC) {
        //wait until the key ESC is pressed
    }

    //destroyWindow("Fisheye Image");

    int Hf, Wf; //Height, width and FOV for the input image (=fisheyeImage)
    double FOV;
    int He, We; //Height and width for the outpout image (=EquirectangularImage)

    Hf = fisheyeImage.size().height;
    Wf = fisheyeImage.size().width;
    FOV = PI; //FOV in radian

    //We keep the same ratio for the image input and the image output
    We = Wf;
    He = Hf;

    equirectangularImage.create(Hf, Wf, fisheyeImage.type()); //We create the outpout image (=EquirectangularImage)

    //For each pixels of the ouput equirectangular Image
    for (int Xe = 0; Xe <equirectangularImage.size().width; Xe++){
        for (int Ye = 0; Ye <equirectangularImage.size().height; Ye++){

            equirectangularImage.at<Vec3b>(Point(Xe,Ye)) = fisheyeImage.at<Vec3b>(findCorrespondingFisheyePoint(Xe, Ye, He, We, Hf, Wf, FOV)); //We find the corresponding point in the fisheyeImage
        }
    }

    namedWindow("Equirectangular Image", WINDOW_AUTOSIZE);
    imshow("Equirectangular Image",equirectangularImage);

    while(waitKey(0) != ESC) {
        //wait until the key ESC is pressed
    }

    destroyWindow("Fisheye Image");

    imwrite("equirectangularImage.jpg", equirectangularImage);

    return 0;

}

Answer 1

With this code, I get the result expected :

#include <iostream>
#include <opencv2/highgui/highgui.hpp>

using namespace std;
using namespace cv;

const string PATH_IMAGE = "/Users/Kenza/Desktop/Xcode_cpp_opencv/Sos/Sos/Images/img1.jpg";
const int ESC = 27;

Point2f findCorrespondingFisheyePoint(int Xe, int Ye, int We, int He, float FOV){
    Point2f fisheyePoint;
    float theta, phi, r;
    Point3f sphericalPoint;

    theta = CV_PI * (Xe / ( (float) We ) - 0.5);
    phi = CV_PI * (Ye / ( (float) He ) - 0.5);

    sphericalPoint.x = cos(phi) * sin(theta);
    sphericalPoint.y = cos(phi) * cos(theta);
    sphericalPoint.z = sin(phi);

    theta = atan2(sphericalPoint.z, sphericalPoint.x);
    phi = atan2(sqrt(pow(sphericalPoint.x,2) + pow(sphericalPoint.z,2)), sphericalPoint.y);
    r = ( (float) We ) * phi / FOV;

    fisheyePoint.x = (int) ( 0.5 * ( (float) We ) + r * cos(theta) );
    fisheyePoint.y = (int) ( 0.5 * ( (float) He ) + r * sin(theta) );

    return fisheyePoint;
}

int main(int argc, char** argv){

    Mat fisheyeImage, equirectangularImage;
    int Wf, Hf;
    float FOV;
    int We, He;

    fisheyeImage = imread(PATH_IMAGE, IMREAD_COLOR);
    namedWindow("Fisheye Image");
    imshow("fisheye Image", fisheyeImage);

    Wf = fisheyeImage.size().width;
    Hf = fisheyeImage.size().height;
    FOV = (180 * CV_PI ) / 180;

    We = Wf;
    He = Hf;

    while (waitKey(0) != ESC){

   }

    equirectangularImage.create(He, We, CV_8UC3);

    for (int Xe = 0; Xe < We; Xe++){

        for (int Ye = 0; Ye < He; Ye++){

            Point2f fisheyePoint = findCorrespondingFisheyePoint(Xe, Ye, We, He, FOV);

            if (fisheyePoint.x >= We || fisheyePoint.y >= He)

                continue;

            if (fisheyePoint.x < 0 || fisheyePoint.y < 0)
                continue;

                equirectangularImage.at<Vec3b>(Point(Xe, Ye)) = fisheyeImage.at<Vec3b>(fisheyePoint);

        }

    }

    namedWindow("Equirectangular Image");
    imshow("Equirectangular Image", equirectangularImage);

    while (waitKey(0) != ESC){

    }

    imwrite("im2.jpg", equirectangularImage);

}