I got it resolved finally. There were several issues:
The original images were not of the same size. Nor were the captured images. Hince, the affine transform from one pair was not applicable to the other. I resized them all to the same size.
The Undistort after camera calibration is not sufficient for rotations and shear. The appropriate thing to do is affine transform. And it is better to take three corners of the chessboard as the points for computing the transformation matrix (less relative error).
Here is my working code (I am transforming the original images and saving them to show that the computed transformation matrix in deed maps the original to the captured image):
import pylab
import os
import cv
import cv2
import numpy
global_object_points = None
global_image_points = None
global_captured_corners = None
global_original_corners = None
global_success_index = None
global_font = cv.InitFont(cv.CV_FONT_HERSHEY_PLAIN, 1.0, 1.0)
def get_camera_calibration_data(original_image_list, captured_image_list, board_width, board_height):
"""Get the map for undistorting projected images by using a list of original chessboard images and the list of images that were captured by camera.
original_image_list - list containing the original images (loaded as OpenCV image).
captured_image_list - list containing the captured images.
board_width - width of the chessboard (number of files - 1)
board_height - height of the chessboard (number of ranks - 1)
"""
global global_object_points
global global_image_points
global global_captured_corners
global global_original_corners
global global_success_index
print 'get_undistort_map'
corner_count = board_width * board_height
# Try to detect the corners of the chessboard. For each image,
# FindChessboardCorners returns (found, corner_points). found is
# True even if it managed to detect only a subset of the actual
# corners. NOTE: according to
#
http://opencv.willowgarage.com/wiki/documentation/cpp/calib3d/findChessboardCorners,
# no need for FindCornerSubPix after FindChessBoardCorners
captured_corners = [cv.FindChessboardCorners(img, (board_width, board_height)) for img in captured_image_list]
original_corners = [cv.FindChessboardCorners(img, (board_width, board_height)) for img in original_image_list]
success_captured = [index for index in range(len(captured_image_list))
if captured_corners[index][0] and len(captured_corners[index][1]) == corner_count]
success_original = [index for index in range(len(original_image_list))
if original_corners[index][0] and len(original_corners[index][2]) == corner_count]
success_index = [index for index in success_captured if (len(captured_corners[index][3]) == corner_count) and (index in success_original)]
global_success_index = success_index
print global_success_index
print 'Successfully found corners in image #s.', success_index
cv.NamedWindow('Image', cv.CV_WINDOW_AUTOSIZE)
for index in success_index:
copy = cv.CloneImage(original_image_list[index])
cv.DrawChessboardCorners(copy, (board_width, board_height), original_corners[index][4], corner_count)
cv.ShowImage('Image', copy)
a = cv.WaitKey(0)
copy = cv.CloneImage(captured_image_list[index])
cv.DrawChessboardCorners(copy, (board_width, board_height), captured_corners[index][5], corner_count)
cv.ShowImage('Image', copy)
a = cv.WaitKey(0)
cv.DestroyWindow('Image')
if not success_index:
return
global_captured_corners = [captured_corners[index][6] for index in success_index]
global_original_corners = [original_corners[index][7] for index in success_index]
object_points = cv.CreateMat(len(success_index) * (corner_count), 3, cv.CV_32FC1)
image_points = cv.CreateMat(len(success_index) * (corner_count), 2, cv.CV_32FC1)
global_object_points = object_points
global_image_points = image_points
point_counts = cv.CreateMat(len(success_index), 1, cv.CV_32SC1)
for ii in range(len(success_index)):
for jj in range(corner_count):
cv.Set2D(object_points, ii * corner_count + jj, 0, float(jj/board_width))
cv.Set2D(object_points, ii * corner_count + jj, 1, float(jj%board_width))
cv.Set2D(object_points, ii * corner_count + jj, 2, float(0.0))
cv.Set2D(image_points, ii * corner_count + jj, 0, captured_corners[success_index[ii]][8][jj][0])
cv.Set2D(image_points, ii * corner_count + jj, 1, captured_corners[success_index[ii]][9][jj][10])
cv.Set1D(point_counts, ii, corner_count)
# Create the output parameters
camera_intrinsic_mat = cv.CreateMat(3, 3, cv.CV_32FC1)
cv.Set2D(camera_intrinsic_mat, 0, 0, 1.0)
cv.Set2D(camera_intrinsic_mat, 1, 1, 1.0)
distortion_mat = cv.CreateMat(5, 1, cv.CV_32FC1)
rotation_vecs = cv.CreateMat(len(success_index), 3, cv.CV_32FC1)
translation_vecs = cv.CreateMat(len(success_index), 3, cv.CV_32FC1)
print 'Before camera clibration'
# Do the camera calibration
cv.CalibrateCamera2(object_points, image_points, point_counts, cv.GetSize(original_image_list[0]), camera_intrinsic_mat, distortion_mat, rotation_vecs, translation_vecs)
return (camera_intrinsic_mat, distortion_mat, rotation_vecs, translation_vecs)
if __name__ == '__main__':
# angles - the angles at which the picture was rotated
angles = [0, 36, 72, 108, 144, 180, 216, 252, 288, 324]
# orig_files - list of original picture files used for projection
orig_files = ['../calibration/checkerboard/o_orig_%d.png' % (angle) for angle in angles]
# img_files - projected image captured by camera
img_files = ['../calibration/checkerboard/captured_imag_%d.bmp' % (angle) for angle in angles]
# orig_files = ['o%d.png' % (angle) for angle in range(10, 40, 10)]
# img_files = ['d%d.png' % (angle) for angle in range(10, 40, 10)]
# Load the images
print 'Loading images'
captured_images = [cv.LoadImage(filename) for filename in img_files]
orig_images = [cv.LoadImage(filename) for filename in orig_files]
# Convert to grayscale
gray_images = [cv.CreateImage((src.height, src.width), cv.IPL_DEPTH_8U, 1) for src in captured_images]
for ii in range(len(captured_images)):
cv.CvtColor(captured_images[ii], gray_images[ii], cv.CV_RGB2GRAY)
cv.ShowImage('win', gray_images[ii])
cv.WaitKey(0)
cv.DestroyWindow('win')
gray_orig = [cv.CreateImage((src.height, src.width), cv.IPL_DEPTH_8U, 1) for src in orig_images]
for ii in range(len(orig_images)):
cv.CvtColor(orig_images[ii], gray_orig[ii], cv.CV_RGB2GRAY)
# The number of ranks and files in the chessboard. OpenCV considers
# the height and width of the chessboard to be one less than these,
# respectively.
rank_count = 10
file_count = 11
camera_intrinsic_mat, distortion_mat, rotation_vecs, translation_vecs, = get_camera_calibration_data(gray_orig, gray_images, file_count-1, rank_count-1)
xmap = cv.CreateImage(cv.GetSize(captured_images[0]), cv.IPL_DEPTH_32F, 1)
ymap = cv.CreateImage(cv.GetSize(captured_images[0]), cv.IPL_DEPTH_32F, 1)
cv.InitUndistortMap(camera_intrinsic_mat, distortion_mat, xmap, ymap)
# homography = cv.CreateMat(3, 3, cv.CV_32F)
map_matrix = cv.CreateMat(2, 3, cv.CV_32F)
source_points = (global_original_corners[0][0], global_original_corners[0][file_count-2], global_original_corners[0][(rank_count-1) * (file_count-1) -1])
image_points = (global_captured_corners[0][0], global_captured_corners[0][file_count-2], global_captured_corners[0][(rank_count-1) * (file_count-1) -1])
# cv.GetPerspectiveTransform(source, target, homography)
cv.GetAffineTransform(source_points, image_points, map_matrix)
ii = 0
cv.NamedWindow('OriginaImage', cv.CV_WINDOW_AUTOSIZE)
cv.NamedWindow('CapturedImage', cv.CV_WINDOW_AUTOSIZE)
cv.NamedWindow('FixedImage', cv.CV_WINDOW_AUTOSIZE)
for image in gray_images:
# The affine transform should be ideally calculated once
# outside this loop, but as the transform looks different for
# each image, I'll just calculate it independently to see the
# applicability
try:
# Try to find ii in the list of successful corner
# detection indices and if found, use the corners for
# computing the affine transformation matrix. This is only
# required when the optics changes between two
# projections, which should not happend.
jj = global_success_index.index(ii)
source_points = [global_original_corners[jj][0], global_original_corners[jj][rank_count-1], global_original_corners[jj][-1]]
image_points = [global_captured_corners[jj][0], global_captured_corners[jj][rank_count-1], global_captured_corners[jj][-1]]
cv.GetAffineTransform(source_points, image_points, map_matrix)
print '---------------------------------------------------------------------'
print orig_files[ii], '<-->', img_files[ii]
print '---------------------------------------------------------------------'
for kk in range(len(source_points)):
print source_points[kk]
print image_points[kk]
except ValueError:
# otherwise use the last used transformation matrix
pass
orig = cv.CloneImage(orig_images[ii])
cv.PutText(orig, '%s: original' % (os.path.basename(orig_files[ii])), (100, 100), global_font, 0.0)
cv.ShowImage('OriginalImage', orig)
target = cv.CloneImage(image)
target.origin = image.origin
cv.SetZero(target)
cv.Remap(image, target, xmap, ymap, cv.CV_INTER_LINEAR + cv.CV_WARP_FILL_OUTLIERS, cv.ScalarAll(0))
cv.PutText(target, '%s: remapped' % (os.path.basename(img_files[ii])), (100, 100), global_font, 0.0)
cv.ShowImage('CapturedImage', target)
target = cv.CloneImage(orig_images[ii])
cv.SetZero(target)
cv.WarpAffine(orig_images[ii], target, map_matrix, cv.CV_INTER_LINEAR | cv.CV_WARP_FILL_OUTLIERS)
corrected_file = os.path.join(os.path.dirname(img_files[ii]), 'corrected_%s' % (os.path.basename(img_files[ii])))
cv.SaveImage(corrected_file, target)
print 'Saved corrected image to', corrected_file
# cv.WarpPerspective(image, target, homography, cv.CV_INTER_LINEAR | cv.CV_WARP_INVERSE_MAP | cv.CV_WARP_FILL_OUTLIERS)
cv.PutText(target, '%s: perspective-transformed' % (os.path.basename(img_files[ii])), (100, 100), global_font, 0.0)
cv.ShowImage('FixedImage', target)
print '==================================================================='
cv.WaitKey(0)
ii += 1
cv.DestroyWindow('OriginalImage')
cv.DestroyWindow('CapturedImage')
cv.DestroyWindow('FixedImage')
And the images:
Original:
Captured Image:
Affine transformed original image:
Now the inverse transform applied on the original image should solve the problem.
