examples/depth_warper/main.py

import argparse
import os
import sys

import cv2
import torch

import kornia as dgm


def load_depth(file_name):
    """Load the depth using the syntel SDK and converts to torch.Tensor."""
    if not os.path.isfile(file_name):
        raise AssertionError(f"Invalid file {file_name}")
    import sintel_io

    depth = sintel_io.depth_read(file_name)
    return torch.from_numpy(depth).view(1, 1, *depth.shape).float()


def load_camera_data(file_name):
    """Load the camera data using the syntel SDK and converts to torch.Tensor."""
    if not os.path.isfile(file_name):
        raise AssertionError(f"Invalid file {file_name}")
    import sintel_io

    intrinsic, extrinsic = sintel_io.cam_read(file_name)
    return intrinsic, extrinsic


def load_image(file_name):
    """Load the image with OpenCV and converts to torch.Tensor."""
    if not os.path.isfile(file_name):
        raise AssertionError(f"Invalid file {file_name}")

    # load image with OpenCV
    img = cv2.imread(file_name, cv2.IMREAD_COLOR)

    # convert image to torch tensor
    tensor = dgm.utils.image_to_tensor(img).float() / 255.0
    return tensor.view(1, *tensor.shape)  # 1xCxHxW


def load_data(root_path, sequence_name, frame_id):
    # index paths
    file_name = 'frame_%04d' % (frame_id)
    image_file = os.path.join(root_path, 'clean', sequence_name, file_name + '.png')
    depth_file = os.path.join(root_path, 'depth', sequence_name, file_name + '.dpt')
    camera_file = os.path.join(root_path, 'camdata_left', sequence_name, file_name + '.cam')
    # load the actual data
    image = load_image(image_file)
    depth = load_depth(depth_file)
    # load camera data and create pinhole
    height, width = image.shape[-2:]
    intrinsics, extrinsics = load_camera_data(camera_file)
    camera = dgm.utils.create_pinhole(intrinsics, extrinsics, height, width)
    return image, depth, camera


def DepthWarperApp():
    parser = argparse.ArgumentParser(description='Warp images by depth application.')
    # data parameters
    parser.add_argument('--input-dir', type=str, required=True, help='the path to the directory with the input data.')
    parser.add_argument('--output-dir', type=str, required=True, help='the path to output the results.')
    parser.add_argument('--sequence-name', type=str, default='alley_1', help='the name of the sequence.')
    parser.add_argument('--frame-ref-id', type=int, default=1, help='the id for the reference image in the sequence.')
    parser.add_argument('--frame-i-id', type=int, default=2, help='the id for the image i in the sequence.')
    # device parameters
    parser.add_argument('--cuda', action='store_true', default=False, help='enables CUDA training')
    parser.add_argument('--seed', type=int, default=666, metavar='S', help='random seed (default: 666)')
    args = parser.parse_args()

    torch.manual_seed(args.seed)

    # configure syntel SDK path
    root_path = os.path.abspath(args.input_dir)
    sys.path.append(os.path.join(root_path, 'sdk/python'))

    # load the data
    root_dir = os.path.join(root_path, 'training')
    img_ref, depth_ref, cam_ref = load_data(root_dir, args.sequence_name, args.frame_ref_id)
    img_i, _, cam_i = load_data(root_dir, args.sequence_name, args.frame_i_id)

    # instantiate the homography warper from `kornia`
    warper = dgm.DepthWarper(cam_i)
    warper.compute_homographies(cam_ref)

    # compute the inverse depth and warp the source image
    inv_depth_ref = 1.0 / depth_ref
    img_i_to_ref = warper(inv_depth_ref, img_i)

    # generate occlusion mask
    mask = ((img_ref - img_i_to_ref).mean(1) < 1e-1).float()

    img_vis_warped = 0.5 * img_i_to_ref + img_ref
    img_vis_warped_masked = mask * (0.5 * img_i_to_ref + img_ref)

    # save warped image to disk
    file_name = os.path.join(args.output_dir, f'warped_{args.frame_i_id}_to_{args.frame_ref_id}.png')
    cv2.imwrite(file_name, dgm.utils.tensor_to_image(255.0 * img_vis_warped))
    cv2.imwrite(file_name + 'mask.png', dgm.utils.tensor_to_image(255.0 * mask))
    cv2.imwrite(file_name + 'warpedmask.png', dgm.utils.tensor_to_image(255.0 * img_vis_warped_masked))


if __name__ == "__main__":
    DepthWarperApp()