How to stitch two images using homography matrix in OpenCv?

Question

I want to stitch two panoramic images using homography matrix in OpenCv. I found 3x3 homography matrix, but I can't stitch two images. I must stitch two images by hand(no build-in function). Here is my code:

import cv2
import numpy as np

MIN_MATCH_COUNT = 10

img1 = cv2.imread("pano1/cyl_image00.png")
img2 = cv2.imread("pano1/cyl_image01.png")

orb = cv2.ORB_create()

kp1, des1 = orb.detectAndCompute(img1, None)
kp2, des2 = orb.detectAndCompute(img2, None)

FLANN_INDEX_KDTREE = 0
index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
search_params = dict(checks=50)

flann = cv2.FlannBasedMatcher(index_params, search_params)

des1 = np.float32(des1)
des2 = np.float32(des2)

matches = flann.knnMatch(des1, des2, k=2)

goodMatches = []

for m, n in matches:
    if m.distance < 0.7 * n.distance:
        goodMatches.append(m)

src_pts = 0
dst_pts = 0
if len(goodMatches) > MIN_MATCH_COUNT:
    dst_pts = np.float32([kp1[m.queryIdx].pt for m in goodMatches]).reshape(-1, 2)
    src_pts = np.float32([kp2[m.trainIdx].pt for m in goodMatches]).reshape(-1, 2)


def generateRandom(src_Pts, dest_Pts, N):
    r = np.random.choice(len(src_Pts), N)
    src = [src_Pts[i] for i in r]
    dest = [dest_Pts[i] for i in r]
    return np.asarray(src, dtype=np.float32), np.asarray(dest, dtype=np.float32)


def findH(src, dest, N):
    A = []
    for i in range(N):
        x, y = src[i][0], src[i][1]
        xp, yp = dest[i][0], dest[i][1]
        A.append([x, y, 1, 0, 0, 0, -x * xp, -xp * y, -xp])
        A.append([0, 0, 0, x, y, 1, -yp * x, -yp * y, -yp])
    A = np.asarray(A)
    U, S, Vh = np.linalg.svd(A)
    L = Vh[-1, :] / Vh[-1, -1]
    H = L.reshape(3, 3)
    return H


def ransacHomography(src_Pts, dst_Pts):
    maxI = 0
    maxLSrc = []
    maxLDest = []
    for i in range(70):
        srcP, destP = generateRandom(src_Pts, dst_Pts, 4)
        H = findH(srcP, destP, 4)
        inlines = 0
        linesSrc = []
        lineDest = []
        for p1, p2 in zip(src_Pts, dst_Pts):
            p1U = (np.append(p1, 1)).reshape(3, 1)
            p2e = H.dot(p1U)
            p2e = (p2e / p2e[2])[:2].reshape(1, 2)[0]
            if cv2.norm(p2 - p2e) < 10:
                inlines += 1
                linesSrc.append(p1)
                lineDest.append(p2)
        if inlines > maxI:
            maxI = inlines
            maxLSrc = linesSrc.copy()
            maxLSrc = np.asarray(maxLSrc, dtype=np.float32)
            maxLDest = lineDest.copy()
            maxLDest = np.asarray(maxLDest, dtype=np.float32)
    Hf = findH(maxLSrc, maxLDest, maxI)
    return Hf


H = ransacHomography(src_pts, dst_pts)

So far, so good. I found homography matrix(H).

Next, I tried to stitch two panoramic images. First, I create a big array to stitch images(img3). I copied img1 to the first half of img3. I tried to find new coordinates for img2 through homography matrix and I copied new img2 coordinates to img3.

Here is my code:

height1, width1, rgb1 = img1.shape
height2, width2, rgb2 = img2.shape

img3 = np.empty((height1, width1+width2, 3))

img3[:, 0:width1] = img1/255.0


for i in range(len(img2)):
    for j in range(len(img2[0])):
        pp = H.dot(np.array([[i], [j], [1]]))
        pp = (pp / pp[2]).reshape(1, 3)[0]
        img3[int(round(pp[0])), int(round(pp[1]))] = img2[i, j]/255.0

But this part is not working. How can I solve this problem?

Answer 1

Once you have the Homography matrix you need to transform one of the images to have the same perspective as the other. This is done using the warpPerspective function in OpenCV. Once you've done the transformation, it's time to concatenate the images.

Let's say you want to transform img_1 into the perspective of img_2 and that you already have the Homography matrix H

dst = cv2.warpPerspective(img_1, H, ((img_1.shape[1] + img_2.shape[1]), img_2.shape[0])) #wraped image

# now paste them together
dst[0:img_2.shape[0], 0:img_2.shape[1]] = img_2
dst[0:img_1.shape[0], 0:img_1.shape[1]] = img_1

Also note that OpenCV already has a build in RANSAC Homography finder

H, masked = cv2.findHomography(src, dst, cv2.RANSAC, 5.0)

So it can save you a lot of code.

Check out these tutorial for more details

https://medium.com/@navekshasood/image-stitching-to-create-a-panorama-5e030ecc8f7

https://medium.com/analytics-vidhya/image-stitching-with-opencv-and-python-1ebd9e0a6d78