python cv2 incorrectly detecting square shape in multiple images

Question

I've run into a problem detecting a square with opencv.

1. Here is the reference image where I find the shape to match against used in cv2.matchShapes

2. Here are two input images used to compare against the reference image:

first image:

second image:

3. Here is the code I used (sorry for poor format):

   #!/usr/bin/env python
   # coding: utf-8
   
   import numpy as np
   import cv2
   import argparse
   import math
   
   ap = argparse.ArgumentParser()
   ap.add_argument("-i", "--image", required=True, help="path to input image")
   args = vars(ap.parse_args())
   
   img = cv2.imread(args["image"])
   # img = cv2.bitwise_not(img,img)
   # gray = cv2.imread(args["image"],0)
   gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
   
   # gray = cv2.GaussianBlur(gray, (5, 5), 0)
   cv2.imshow('gray', gray)
   cv2.waitKey(0)
   
   ret, thresh = cv2.threshold(gray, 230, 255, 1)
   cv2.imshow('thresh', thresh)
   # cv2.imwrite('./wni230.png',thresh)
   cv2.waitKey(0)
   
   square_cnts = []
   
   ##################################################
   shape = cv2.imread('./shape1.png')
   shape_gray = cv2.cvtColor(shape, cv2.COLOR_BGR2GRAY)
   ret, shape_thresh = cv2.threshold(shape_gray, 0, 255, 0)
   tmpimage, contours, h = cv2.findContours(shape_thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
   shape_cnt0 = contours[0]
   shape_approx = []
   for i in contours:
       approx = cv2.approxPolyDP(i, 0.01*cv2.arcLength(i, True), True)
       # print len(approx)
       if len(approx) == 4:
           shape_approx.append(len(approx))
   ##################################################
   # tmpimage,contours,h = cv2.findContours(thresh,1,2)
   # cv2.RETR_TREE
   tmpimage, contours, h = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
   cnt0 = contours[0]
   # from skimage import measure
   # contours = measure.find_contours(thresh, 0.8)
   
   for cnt in contours[::-1]:
       print cv2.arcLength(cnt, True)
       print cnt
       approx = cv2.approxPolyDP(cnt, 0.1*cv2.arcLength(cnt, True), True)
       print len(approx)
       if len(approx) == 5:
           print "pentagon"
           cv2.drawContours(img, [cnt], 0, 255, 2)
           cv2.imshow('tmppentagon', img)
           cv2.waitKey(0)
       elif len(approx) == 3:
           print "triangle"
           cv2.drawContours(img, [cnt], 0, (0, 255, 0), 2)
           cv2.imshow('tmptriangle', img)
           cv2.waitKey(0)
       elif len(approx) == 2:
           print 'approx:'
           print approx
           ret = cv2.matchShapes(cnt, cnt0, 1, 0.0)
           print 'match shape ret:%s' % ret
           print "two approx line"
           cv2.drawContours(img, [cnt], 0, (0, 255, 0), 2)
           cv2.imshow('twoline?', img)
           cv2.waitKey(0)
       elif len(approx) == 4:
   
           ret = cv2.matchShapes(cnt, cnt0, 1, 0.0)
           print 'match shape ret:%s' % ret
           if ret > 0.5:
               print "Parallelogram"
           elif 0.3 < ret < 0.5:
               print "Rectangle"
           elif 0 < ret < 0.3:
               print "Rhombus"
           else:
               print "square"
           print cv2.arcLength(cnt, True)
           print approx
           cv2.drawContours(img, [approx], 0, (0, 0, 255), 2)
           cv2.imshow('tmpsquare', img)
           cv2.waitKey(0)
           if int(cv2.arcLength(cnt, True)) >= 96:
   
               if math.fabs(math.sqrt((approx[0][0][0]-approx[1][0][0])**2+(approx[0][0][1]-approx[1][0][1])**2) - math.sqrt((approx[1][0][0]-approx[2][0][0])**2+(approx[1][0][1]-approx[2][0][1])**2)) <= 5:
                   x, y, w, h = cv2.boundingRect(cnt)
                   cv2.imshow('final',img[y:y+h,x:x+w])
                   cv2.waitKey(0)
               print 'target but long squere detected...'
               cv2.waitKey(0)
       elif len(approx) == 9:
           print "half-circle"
           cv2.drawContours(img,[cnt],0,(255,255,0),2)
           cv2.imshow('tmphalfcircle',img)
           cv2.waitKey(0)
       elif len(approx) > 15:
           print "circle"
           cv2.drawContours(img,[cnt],0,(0,255,255),2)
           cv2.imshow('tmpcircle',img)
           cv2.waitKey(0)
   
   cv2.imshow('img', img)
   cv2.imwrite('tmp.png', img)
   cv2.waitKey(0)
   cv2.destroyAllWindows()
   

Results for the two input images respectively:

First result:

Second result:

Env:

python 2.7.10

opencv 3.2.0

Questions:

1. Why is the length of cv2.approxPolyDP always 2 instead of 4 for both images? I would expect the result to be 4 for the square in the image.

2. Why are my cv2.matchShapes results so different? I think 0 is the ideal output, but why does matching against the second image produce such a high number?

Answer 1

Looking through the documentation on those functions, it appears approxPolyDP is returning a 2, because it can only find contours of 2 points that actually connect a polygon in the way you describe. Take a look at Ramer-Douglas-Peucker algorithm, which is what polyDP is based off of. similarly, the shapeMatch result is high if there is a lot of difference between the two matched shapes. Normally this value won't be that large unless the shapes are really different, but in this case, you seem to be matching against contours of your second image!

Look here:

tmpimage, contours, h = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnt0 = contours[0]

cnt0 is the first contour in your second image. Now further down you say:

 elif len(approx) == 2:
        print 'approx:'
        print approx
        ret = cv2.matchShapes(cnt, cnt0, 1, 0.0)

you are comparing your first in the supplied image to a contour in the same image! you got a zero difference for the first one because it just happened to be the first contour in the image, which more junk in the other test image this is less likely to happen, so you aren't going to get a match.

Further more, it appears as if you will never get a four point approxPolyDP because your contours aren't connected/ starting in the correct place. To prove this to yourself, use the following code at the top of your loop:

for cnt in contours[::-1]:
    print cv2.arcLength(cnt, True)
    approx_polygon_shape = cv2.approxPolyDP(cnt, 0.087 * cv2.arcLength(cnt, True), True)
    print "number of approx points", len(approx_polygon_shape)
    print "approx shape", approx_polygon_shape
    tempimage = input_image.copy()
    cv2.drawContours(tempimage, [approx_polygon_shape], -1, (0,255,0), 1)
    cv2.imshow("polyshape show", tempimage)
    cv2.waitKey(0)

No matter what number I put to modify the second parameter epsilon in approxPolyDP I couldn't get four points. I would either get triangles or giant amounts of points (this one happened to be six I believe). In your normal code you multply arclength by 0.1, in this situation it turns out that will typically only give you 2 points per contour!

For me this returns:

It appears the blank area in the upper right is causing the issue, you may want to blur, which looking at your code, it appears as if you already knew this might be an issue. Re adding the commented out line marked here:

    img = cv2.imread(args["image"])
    # img = cv2.bitwise_not(img,img)
    # gray = cv2.imread(args["image"],0)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    #UNCOMMENT!!!VVV
    gray = cv2.GaussianBlur(gray, (5, 5), 0)
    cv2.imshow('gray', gray)
    cv2.waitKey(0)

allows just enough blur for only four points to be chosen, even with your normal 0.1 multiplication (example output, note I don't use python2 and modified the code a bit for more reasonable output):

Trying the blurring trick with your second image also works!

If you take a look at the actual grayscale image, this works because it was blurred just enough to allow the contours to be close enough for approxPolyDP to work and "jump across" that small gap that was causing issues before. Here is what that blurred image looks like: