diff --git a/camera_calibration.py b/camera_calibration.py new file mode 100644 index 0000000..4ac3949 --- /dev/null +++ b/camera_calibration.py @@ -0,0 +1,66 @@ +import numpy as np +import cv2 + +# termination criteria +criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001) +# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0) +objp = np.zeros((6*7, 3), np.float32) +objp[:, :2] = np.mgrid[0:7, 0:6].T.reshape(-1, 2) +# Arrays to store object points and image points from all the images. +objpoints = [] # 3d point in real world space +imgpoints = [] # 2d points in image plane. + +w = 1280 +h = 720 + +target_size = 720 + +s_x = (w//2-target_size//2) +e_x = (w//2+target_size//2) +s_y = 0 +e_y = target_size + +cap = cv2.VideoCapture(0) +cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1280) +cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 720) + +while cap.isOpened(): + success, img = cap.read() + if not success: + print("Ignoring empty camera frame.") + # If loading a video, use 'break' instead of 'continue'. + continue + + img = img[s_y:e_y, s_x:e_x] + img = cv2.resize(img, (1080, 1080)) + # print(img.shape) + + width = img.shape[0] + height = img.shape[1] + + gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + # Find the chess board corners + ret, corners = cv2.findChessboardCorners(gray, (7, 6), None) + # If found, add object points, image points (after refining them) + if ret == True: + objpoints.append(objp) + corners2 = cv2.cornerSubPix( + gray, corners, (11, 11), (-1, -1), criteria) + imgpoints.append(corners) + # Draw and display the corners + cv2.drawChessboardCorners(img, (7, 6), corners2, ret) + cv2.imshow('img', img) + + if cv2.waitKey(5) & 0xFF == 27: + break + +cv2.destroyAllWindows() + +ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera( + objpoints, imgpoints, gray.shape[::-1], None, None) + +print(f"width:{width}") +print(f"height:{height}") +print(f"ret:{ret}") +print(f"mtx:{mtx}") +print(f"dist:{dist}") diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..f183d42 --- /dev/null +++ b/environment.yml @@ -0,0 +1,42 @@ +name: 360_vdo_reconstruck +channels: + - defaults +dependencies: + - _libgcc_mutex=0.1 + - _openmp_mutex=4.5 + - ca-certificates=2022.4.26 + - certifi=2021.10.8 + - ld_impl_linux-64=2.35.1 + - libffi=3.3 + - libgcc-ng=9.3.0 + - libgomp=9.3.0 + - libstdcxx-ng=9.3.0 + - ncurses=6.3 + - openssl=1.1.1o + - pip=21.2.2 + - python=3.7.13 + - readline=8.1.2 + - setuptools=61.2.0 + - sqlite=3.38.3 + - tk=8.6.11 + - wheel=0.37.1 + - xz=5.2.5 + - zlib=1.2.12 + - pip: + - cached-property==1.5.2 + - chumpy==0.70 + - cycler==0.11.0 + - cython==0.29.28 + - h5py==3.6.0 + - kiwisolver==1.4.2 + - matplotlib==2.2.5 + - numpy==1.21.6 + - opencv-python==4.5.5.64 + - opendr==0.77 + - pyparsing==3.0.8 + - python-dateutil==2.8.2 + - pytz==2022.1 + - scipy==1.7.3 + - six==1.16.0 + - tqdm==4.64.0 + - typing-extensions==4.2.0 diff --git a/hdf5_read.py b/hdf5_read.py new file mode 100644 index 0000000..17bc1e9 --- /dev/null +++ b/hdf5_read.py @@ -0,0 +1,17 @@ +import h5py +import numpy as np +filename = "reconstructed_poses_noratap.hdf5" + +with h5py.File(filename, "r") as f: + # List all groups + print("Keys: %s" % f.keys()) + a_group_key = list(f.keys())[1] + + # Get the data + data = list(f[a_group_key]) + pose_data = data[0].reshape((24, 3)) + + for i in range(0, 24): + print(f"{i}:{np.degrees(pose_data[i])}") + + print(len(data[0])) diff --git a/lib/rays.py b/lib/rays.py index 0e69c85..96582a1 100644 --- a/lib/rays.py +++ b/lib/rays.py @@ -27,8 +27,14 @@ def unpose_and_select_rays(rays, Vi, smpl, rn_b, rn_m): v_ids = visible_boundary_edge_verts(rn_b, rn_m) verts = smpl.r[v_ids] + print(rn_b) + print(rn_m) + print(v_ids) + print(verts) + n, m = plucker(rays) - dist = np.linalg.norm(np.cross(verts.reshape(-1, 1, 3), n, axisa=2, axisb=1) - m, axis=2) + dist = np.linalg.norm(np.cross(verts.reshape(-1, 1, 3), + n, axisa=2, axisb=1) - m, axis=2) ray_matches = np.argmin(dist, axis=0) vert_matches = np.argmin(dist, axis=1) @@ -38,27 +44,35 @@ def unpose_and_select_rays(rays, Vi, smpl, rn_b, rn_m): M = Vi[:, :, v_ids] T = smpl.v_posevariation[v_ids].r - tmp0 = M[:, :, ray_matches] * np.hstack((rays[:, 0], np.ones((rays.shape[0], 1)))).T.reshape(1, 4, -1) - tmp1 = M[:, :, ray_matches] * np.hstack((rays[:, 1], np.ones((rays.shape[0], 1)))).T.reshape(1, 4, -1) + tmp0 = M[:, :, ray_matches] * \ + np.hstack((rays[:, 0], np.ones((rays.shape[0], 1))) + ).T.reshape(1, 4, -1) + tmp1 = M[:, :, ray_matches] * \ + np.hstack((rays[:, 1], np.ones((rays.shape[0], 1))) + ).T.reshape(1, 4, -1) rays_u_r[:, 0] = np.sum(tmp0, axis=1).T[:, :3] - T[ray_matches] rays_u_r[:, 1] = np.sum(tmp1, axis=1).T[:, :3] - T[ray_matches] rays_u_v = np.zeros_like(rays[vert_matches]) - tmp0 = M * np.hstack((rays[vert_matches, 0], np.ones((verts.shape[0], 1)))).T.reshape(1, 4, -1) - tmp1 = M * np.hstack((rays[vert_matches, 1], np.ones((verts.shape[0], 1)))).T.reshape(1, 4, -1) + tmp0 = M * np.hstack((rays[vert_matches, 0], + np.ones((verts.shape[0], 1)))).T.reshape(1, 4, -1) + tmp1 = M * np.hstack((rays[vert_matches, 1], + np.ones((verts.shape[0], 1)))).T.reshape(1, 4, -1) rays_u_v[:, 0] = np.sum(tmp0, axis=1).T[:, :3] - T rays_u_v[:, 1] = np.sum(tmp1, axis=1).T[:, :3] - T - valid_rays = dist[np.vstack((ray_matches, range(dist.shape[1]))).tolist()] < 0.12 - valid_verts = dist[np.vstack((range(dist.shape[0]), vert_matches)).tolist()] < 0.03 + valid_rays = dist[np.vstack( + (ray_matches, range(dist.shape[1]))).tolist()] < 0.12 + valid_verts = dist[np.vstack( + (range(dist.shape[0]), vert_matches)).tolist()] < 0.03 ray_matches = ray_matches[valid_rays] return np.concatenate((v_ids[ray_matches], v_ids[valid_verts])), \ - np.concatenate((rays_u_r[valid_rays], rays_u_v[valid_verts])) + np.concatenate((rays_u_r[valid_rays], rays_u_v[valid_verts])) def rays_from_points(points, camera): @@ -73,9 +87,12 @@ def rays_from_points(points, camera): def rays_from_silh(mask, camera): if cv2.__version__[0] == '2': - contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours, _ = cv2.findContours( + mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) else: - _, contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + #_, contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours, _ = cv2.findContours( + mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) silh = np.zeros_like(mask) @@ -92,7 +109,8 @@ def ray_objective(f, sigma, base_smpl, camera, vis_rn_b, vis_rn_m): base_smpl.pose[:] = f.pose camera.t[:] = f.trans - f.v_ids, f.rays_u = unpose_and_select_rays(f.rays, f.Vi, base_smpl, vis_rn_b, vis_rn_m) + f.v_ids, f.rays_u = unpose_and_select_rays( + f.rays, f.Vi, base_smpl, vis_rn_b, vis_rn_m) f.verts = base_smpl.v_shaped_personal[f.v_ids] f.dist = distance_function(f.rays_u, f.verts) diff --git a/mask_numpy2pic.py b/mask_numpy2pic.py new file mode 100644 index 0000000..a967993 --- /dev/null +++ b/mask_numpy2pic.py @@ -0,0 +1,30 @@ +import numpy as np +import cv2 +import h5py +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('input', type=str) +parser.add_argument('output', type=str) + +args = parser.parse_args() + +input_file = args.input +output_folder = args.output + + +with h5py.File(input_file, "r") as f: + # List all groups + print("Keys: %s" % f.keys()) + a_group_key = list(f.keys())[0] + + # Get the data + data = list(f[a_group_key]) + for frame, mask in enumerate(data): + print(frame) + print(mask.shape) + white_img = np.ones((1080, 1080), dtype='uint8')*255 + mask = cv2.bitwise_and(white_img, white_img, mask=mask) + print(mask.dtype) + #mask *= 255 + cv2.imwrite(f"{output_folder}/{frame}.png", mask) diff --git a/mask_person_by_bg_subtrack.py b/mask_person_by_bg_subtrack.py new file mode 100644 index 0000000..5c6c82a --- /dev/null +++ b/mask_person_by_bg_subtrack.py @@ -0,0 +1,72 @@ +import cv2 +import numpy as np + +vdo_file = "E:/CMU Med Data Analysis/3d_reconstruck/dataset/input/Noratap/noratap_360.mp4" +bg_file = "E:/CMU Med Data Analysis/3d_reconstruck/dataset/input/Noratap/bg.jpg" + +mask_out_path = "E:/CMU Med Data Analysis/3d_reconstruck/dataset/input/Noratap/mask" + +# Set Up Vdo Write +vdo_name = "E:/CMU Med Data Analysis/3d_reconstruck/dataset/input/Noratap/noratap_360_1080.avi" +print("VDO Save ", vdo_name) +vdo_out = cv2.VideoWriter(vdo_name, cv2.VideoWriter_fourcc( + *'DIVX'), 25, (1080, 1080)) + +cap = cv2.VideoCapture(vdo_file) +img_bg = cv2.imread(bg_file) + +w = img_bg.shape[0] +h = img_bg.shape[1] + +target_size = 720 + +s_x = (w//2-target_size//2) + 350 +e_x = (w//2+target_size//2) + 350 +s_y = 0 +e_y = target_size + +img_bg = img_bg[s_y:e_y, s_x:e_x] + +frame = 0 + +if (cap.isOpened() == False): + print("Error opening video stream or file") + +while(cap.isOpened()): + ret, img = cap.read() + if ret == True: + + img = img[s_y:e_y, s_x:e_x] + + cv2.imshow('Img', img) + + # bg Subtrack + img_mask = cv2.subtract(img_bg, img) + img_mask = cv2.cvtColor(img_mask, cv2.COLOR_BGR2GRAY) + + # img_mask = cv2.equalizeHist(img_mask) + + ret_th, img_mask = cv2.threshold(img_mask, 30, 255, cv2.THRESH_BINARY) + + # img_mask = cv2.dilate(img_mask, np.ones((10, 10))) + + cv2.imshow('Mask', img_mask) + + img_mask = cv2.resize(img_mask, (1080, 1080)) + img = cv2.resize(img, (1080, 1080)) + + # Save Img + cv2.imwrite(f"{mask_out_path}/{frame}.png", img_mask) + vdo_out.write(img) + + frame += 1 + + if cv2.waitKey(25) & 0xFF == ord('q'): + break + + else: + break + +vdo_out.release() +cap.release() +cv2.destroyAllWindows() diff --git a/models/bodyparts.py b/models/bodyparts.py index 65c0b7a..e5b65e4 100644 --- a/models/bodyparts.py +++ b/models/bodyparts.py @@ -1,7 +1,7 @@ #!/usr/bin/env python2 # -*- coding: utf-8 -*- -import cPickle as pkl +import pickle as pkl import numpy as np _cache = None @@ -12,7 +12,9 @@ def get_bodypart_vertex_ids(): if _cache is None: with open('assets/bodyparts.pkl', 'rb') as fp: - _cache = pkl.load(fp) + u = pkl._Unpickler(fp) + u.encoding = 'latin1' + _cache = u.load() return _cache diff --git a/models/smpl.py b/models/smpl.py index 8233217..5c36f86 100644 --- a/models/smpl.py +++ b/models/smpl.py @@ -3,7 +3,7 @@ import chumpy as ch import numpy as np -import cPickle as pkl +import pickle as pkl import scipy.sparse as sp from chumpy.ch import Ch from vendor.smpl.posemapper import posemap, Rodrigues @@ -73,8 +73,10 @@ def on_changed(self, which): def _set_up(self): self.v_shaped = self.shapedirs.dot(self.betas) + self.v_template self.v_shaped_personal = self.v_shaped + self.v_personal - self.J = ch.sum(self.J_regressor.T.reshape(-1, 1, 24) * self.v_shaped.reshape(-1, 3, 1), axis=0).T - self.v_posevariation = self.posedirs.dot(posemap(self.bs_type)(self.pose)) + self.J = ch.sum(self.J_regressor.T.reshape(-1, 1, 24) + * self.v_shaped.reshape(-1, 3, 1), axis=0).T + self.v_posevariation = self.posedirs.dot( + posemap(self.bs_type)(self.pose)) self.v_poseshaped = self.v_shaped_personal + self.v_posevariation self.A, A_global = self._global_rigid_transformation() @@ -83,24 +85,31 @@ def _set_up(self): self.V = self.A.dot(self.weights.T) - rest_shape_h = ch.hstack((self.v_poseshaped, ch.ones((self.v_poseshaped.shape[0], 1)))) - self.v_posed = ch.sum(self.V.T * rest_shape_h.reshape(-1, 4, 1), axis=1)[:, :3] + rest_shape_h = ch.hstack( + (self.v_poseshaped, ch.ones((self.v_poseshaped.shape[0], 1)))) + self.v_posed = ch.sum( + self.V.T * rest_shape_h.reshape(-1, 4, 1), axis=1)[:, :3] self.v = self.v_posed + self.trans def _global_rigid_transformation(self): results = {} pose = self.pose.reshape((-1, 3)) - parent = {i: self.kintree_table[0, i] for i in range(1, self.kintree_table.shape[1])} + parent = {i: self.kintree_table[0, i] + for i in range(1, self.kintree_table.shape[1])} - with_zeros = lambda x: ch.vstack((x, ch.array([[0.0, 0.0, 0.0, 1.0]]))) - pack = lambda x: ch.hstack([ch.zeros((4, 3)), x.reshape((4, 1))]) + def with_zeros(x): return ch.vstack( + (x, ch.array([[0.0, 0.0, 0.0, 1.0]]))) - results[0] = with_zeros(ch.hstack((Rodrigues(pose[0, :]), self.J[0, :].reshape((3, 1))))) + def pack(x): return ch.hstack([ch.zeros((4, 3)), x.reshape((4, 1))]) + + results[0] = with_zeros( + ch.hstack((Rodrigues(pose[0, :]), self.J[0, :].reshape((3, 1))))) for i in range(1, self.kintree_table.shape[1]): results[i] = results[parent[i]].dot(with_zeros(ch.hstack(( Rodrigues(pose[i, :]), # rotation around bone endpoint - (self.J[i, :] - self.J[parent[i], :]).reshape((3, 1)) # bone + (self.J[i, :] - self.J[parent[i], :] + ).reshape((3, 1)) # bone )))) results = [results[i] for i in sorted(results.keys())] @@ -162,7 +171,8 @@ def joints_coco(smpl): def model_params_in_camera_coords(trans, pose, J0, camera_t, camera_rt): - root = Rodrigues(np.matmul(Rodrigues(camera_rt).r, Rodrigues(pose[:3]).r)).r.reshape(-1) + root = Rodrigues(np.matmul(Rodrigues(camera_rt).r, + Rodrigues(pose[:3]).r)).r.reshape(-1) pose[:3] = root trans = (Rodrigues(camera_rt).dot(J0 + trans) - J0 + camera_t).r @@ -171,7 +181,8 @@ def model_params_in_camera_coords(trans, pose, J0, camera_t, camera_rt): if __name__ == '__main__': - smpl = Smpl(model='../vendor/smpl/models/basicModel_f_lbs_10_207_0_v1.0.0.pkl') + smpl = Smpl( + model='../vendor/smpl/models/basicModel_f_lbs_10_207_0_v1.0.0.pkl') smpl.pose[:] = np.random.randn(72) * .2 smpl.pose[0] = np.pi # smpl.v_personal[:] = np.random.randn(*smpl.shape) / 500. @@ -204,6 +215,6 @@ def model_params_in_camera_coords(trans, pose, J0, camera_t, camera_rt): import cv2 cv2.imshow('render_SMPL', rn.r) - print ('..Print any key while on the display window') + print('..Print any key while on the display window') cv2.waitKey(0) cv2.destroyAllWindows() diff --git a/opendr b/opendr new file mode 160000 index 0000000..bc16a6a --- /dev/null +++ b/opendr @@ -0,0 +1 @@ +Subproject commit bc16a6a51771d6e062d088ba5cede66649b7c7ec diff --git a/plk_read.py b/plk_read.py new file mode 100644 index 0000000..45b95ed --- /dev/null +++ b/plk_read.py @@ -0,0 +1,9 @@ +import pickle as pkl + +plk_file = 'camera.pkl' + +with open(plk_file, 'rb') as fp: + u = pkl._Unpickler(fp) + u.encoding = 'latin1' + data = u.load() + print(data) diff --git a/prepare_data/create_camera.py b/prepare_data/create_camera.py index 0505ded..7c4cd04 100644 --- a/prepare_data/create_camera.py +++ b/prepare_data/create_camera.py @@ -3,7 +3,7 @@ import numpy as np import argparse -import cPickle as pkl +import pickle as pkl """ @@ -18,9 +18,12 @@ parser.add_argument('out', type=str, help="Output file (.pkl)") parser.add_argument('width', type=int, help="Frame width in px") parser.add_argument('height', type=int, help="Frame height in px") -parser.add_argument('-f', type=float, nargs='*', help="Focal length in px (2,)") -parser.add_argument('-c', type=float, nargs='*', help="Principal point in px (2,)") -parser.add_argument('-k', type=float, nargs='*', help="Distortion coefficients (5,)") +parser.add_argument('-f', type=float, nargs='*', + help="Focal length in px (2,)") +parser.add_argument('-c', type=float, nargs='*', + help="Principal point in px (2,)") +parser.add_argument('-k', type=float, nargs='*', + help="Distortion coefficients (5,)") args = parser.parse_args() diff --git a/requirements.txt b/requirements.txt index 8df10d5..c94a25a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,18 @@ -opendr>=0.76 -numpy>=1.11.0 -scipy>=0.19.0 -matplotlib==2.* -h5py==2.7.* -tqdm==4.* -chumpy>=0.67 \ No newline at end of file +cached-property==1.5.2 +certifi==2021.10.8 +chumpy==0.70 +cycler==0.11.0 +Cython==0.29.28 +h5py==3.6.0 +kiwisolver==1.4.2 +matplotlib==2.2.5 +numpy==1.21.6 +opencv-python==4.5.5.64 +opendr==0.77 +pyparsing==3.0.8 +python-dateutil==2.8.2 +pytz==2022.1 +scipy==1.7.3 +six==1.16.0 +tqdm==4.64.0 +typing_extensions==4.2.0 diff --git a/resample_hdf5.py b/resample_hdf5.py new file mode 100644 index 0000000..48259fc --- /dev/null +++ b/resample_hdf5.py @@ -0,0 +1,22 @@ +import h5py +filename = "masks" + +with h5py.File(f"{filename}.hdf5", "r") as f: + # List all groups + print("Keys: %s" % f.keys()) + a_group_key = list(f.keys())[0] + + # Get the data + data = list(f[a_group_key]) + + # Resample 25 fps to 10 fps + # 5 = (5/25)*25 + resample_result = [] + for i in range(0, len(data), 5): + resample_result.append(data[i]) + + print(f"Resample Size:{len(resample_result)}") + + # Write data to HDF5 + with h5py.File(f"{filename}_resample.hdf5", "w") as data_file: + data_file.create_dataset(a_group_key, data=resample_result) diff --git a/run_step1.sh b/run_step1.sh index c1009f8..aa0af9d 100755 --- a/run_step1.sh +++ b/run_step1.sh @@ -9,7 +9,7 @@ SUBJ="$1" OUT="$2" if [[ $SUBJ = *"female"* ]]; then - MODEL='--model vendor/smpl/models/basicModel_f_lbs_10_207_0_v1.0.0.pkl' + MODEL='--model vendor/smpl/models/basicModel_m_lbs_10_207_0_v1.0.0.pkl' fi python step1_pose.py $SUBJ/keypoints.hdf5 $SUBJ/masks.hdf5 $SUBJ/camera.pkl $OUT/reconstructed_poses.hdf5 $MODEL ${@:3} \ No newline at end of file diff --git a/run_step2.sh b/run_step2.sh index 9c77cef..a574d9f 100755 --- a/run_step2.sh +++ b/run_step2.sh @@ -9,7 +9,7 @@ SUBJ="$1" OUT="$2" if [[ $SUBJ = *"female"* ]]; then - MODEL='--model vendor/smpl/models/basicModel_f_lbs_10_207_0_v1.0.0.pkl' + MODEL='--model vendor/smpl/models/basicModel_m_lbs_10_207_0_v1.0.0.pkl' fi -python step2_consensus.py $SUBJ/reconstructed_poses.hdf5 $SUBJ/masks.hdf5 $SUBJ/camera.pkl $OUT/consensus.pkl --obj_out $OUT/consensus.obj $MODEL ${@:3} +python step2_consensus.py $SUBJ/reconstructed_poses.hdf5 $SUBJ/masks.hdf5 $SUBJ/camera.pkl $OUT/consensus.pkl --obj_out $OUT/consensus.obj $MODEL ${@:3} diff --git a/run_step3.sh b/run_step3.sh index dcb8d1e..befabc5 100755 --- a/run_step3.sh +++ b/run_step3.sh @@ -9,7 +9,7 @@ SUBJ="$1" OUT="$2" if [[ $SUBJ = *"female"* ]]; then - MODEL='--model vendor/smpl/models/basicModel_f_lbs_10_207_0_v1.0.0.pkl' + MODEL='--model vendor/smpl/models/basicModel_m_lbs_10_207_0_v1.0.0.pkl' fi -python step3_texture.py $SUBJ/consensus.pkl $SUBJ/camera.pkl $SUBJ/$(basename $SUBJ).mp4 $SUBJ/reconstructed_poses.hdf5 $SUBJ/masks.hdf5 $OUT/tex-$(basename $SUBJ).jpg $MODEL ${@:3} +python step3_texture.py $SUBJ/consensus.pkl $SUBJ/camera.pkl $SUBJ/noratap_360_1080.avi $SUBJ/reconstructed_poses.hdf5 $SUBJ/masks.hdf5 $OUT/tex-$(basename $SUBJ).jpg $MODEL ${@:3} diff --git a/step1_pose.py b/step1_pose.py index c264c1d..c6b93b2 100644 --- a/step1_pose.py +++ b/step1_pose.py @@ -1,31 +1,29 @@ #!/usr/bin/env python2 # -*- coding: utf-8 -*- -import cv2 -import h5py -import argparse -import numpy as np -import chumpy as ch -import cPickle as pkl - -from opendr.camera import ProjectPoints -from opendr.lighting import LambertianPointLight -from opendr.renderer import ColoredRenderer -from opendr.filters import gaussian_pyramid - -from util import im -from util.logger import log -from lib.frame import FrameData -from models.smpl import Smpl, copy_smpl, joints_coco -from models.bodyparts import faces_no_hands - -from vendor.smplify.sphere_collisions import SphereCollisions from vendor.smplify.robustifiers import GMOf +from vendor.smplify.sphere_collisions import SphereCollisions +from models.bodyparts import faces_no_hands +from models.smpl import Smpl, copy_smpl, joints_coco +from lib.frame import FrameData +from util.logger import log +from util import im +from opendr.filters import gaussian_pyramid +from opendr.renderer import ColoredRenderer +from opendr.lighting import LambertianPointLight +from opendr.camera import ProjectPoints +import pickle as pkl +import chumpy as ch +import numpy as np +import argparse +import h5py +import cv2 def get_cb(viz_rn, f): if viz_rn is not None: - viz_rn.set(v=f.smpl, background_image=np.dstack((f.mask, f.mask, f.mask))) + viz_rn.set(v=f.smpl, background_image=np.dstack( + (f.mask, f.mask, f.mask))) viz_rn.vc.set(v=f.smpl) def cb(_): @@ -44,7 +42,8 @@ def cb(_): def collision_obj(smpl, regs): - sp = SphereCollisions(pose=smpl.pose, betas=smpl.betas, model=smpl, regs=regs) + sp = SphereCollisions( + pose=smpl.pose, betas=smpl.betas, model=smpl, regs=regs) sp.no_hands = True return sp @@ -63,7 +62,7 @@ def init(frames, body_height, b2m, viz_rn): E_height = None if body_height is not None: - E_height = height_predictor(b2m, betas) - body_height * 1000. + E_height = height_predictor(b2m, betas) - body_height * 1000. # first get a rough pose for all frames individually for i, f in enumerate(frames): @@ -112,7 +111,8 @@ def init(frames, body_height, b2m, viz_rn): for i, f in enumerate(frames): if np.sum(f.keypoints[[0, 2, 5, 8, 11], 2]) > 3.: - x0.extend([f.smpl.pose[range(21) + range(27, 30) + range(36, 60)], f.smpl.trans]) + x0.extend( + [f.smpl.pose[list(range(21, 0)) + list(range(27, 30)) + list(range(36, 60))], f.smpl.trans]) E['pose_{}'.format(i)] = f.pose_obj E['prior_{}'.format(i)] = f.pose_prior_obj * w_prior @@ -163,9 +163,11 @@ def reinit_frame(frame, null_pose, nohands, viz_rn): x0 = [frame.smpl.trans] if nohands: - x0.append(frame.smpl.pose[range(21) + range(27, 30) + range(36, 60)]) + x0.append(frame.smpl.pose[list(range(21)) + + list(range(27, 30)) + list(range(36, 60))]) else: - x0.append(frame.smpl.pose[range(21) + range(27, 30) + range(36, 72)]) + x0.append(frame.smpl.pose[list(range(21)) + + list(range(27, 30)) + list(range(36, 72))]) ch.minimize( E, @@ -190,7 +192,8 @@ def fit_pose(frame, last_smpl, frustum, nohands, viz_rn): dist_i = cv2.distanceTransform(np.uint8(frame.mask * 255), dst_type, 5) - 1 dist_i[dist_i < 0] = 0 dist_i[dist_i > 50] = 50 - dist_o = cv2.distanceTransform(255 - np.uint8(frame.mask * 255), dst_type, 5) + dist_o = cv2.distanceTransform( + 255 - np.uint8(frame.mask * 255), dst_type, 5) dist_o[dist_o > 50] = 50 rn_m = ColoredRenderer(camera=frame.camera, v=frame.smpl, f=faces, vc=np.ones_like(frame.smpl), frustum=frustum, @@ -208,9 +211,11 @@ def fit_pose(frame, last_smpl, frustum, nohands, viz_rn): E['last_trans'] = GMOf(frame.smpl.trans - last_smpl.trans, 0.05) * 50. if nohands: - x0 = [frame.smpl.pose[range(21) + range(27, 30) + range(36, 60)], frame.smpl.trans] + x0 = [frame.smpl.pose[list(range( + 21)) + list(range(27, 30)) + list(range(36, 60))], frame.smpl.trans] else: - x0 = [frame.smpl.pose[range(21) + range(27, 30) + range(36, 72)], frame.smpl.trans] + x0 = [frame.smpl.pose[list(range( + 21)) + list(range(27, 30)) + list(range(36, 72))], frame.smpl.trans] ch.minimize( E, @@ -228,19 +233,27 @@ def main(keypoint_file, masks_file, camera_file, out, model_file, prior_file, re # load data with open(model_file, 'rb') as fp: - model_data = pkl.load(fp) + u = pkl._Unpickler(fp) + u.encoding = 'latin1' + model_data = u.load() with open(camera_file, 'rb') as fp: - camera_data = pkl.load(fp) + u = pkl._Unpickler(fp) + u.encoding = 'latin1' + camera_data = u.load() with open(prior_file, 'rb') as fp: - prior_data = pkl.load(fp) + u = pkl._Unpickler(fp) + u.encoding = 'latin1' + prior_data = u.load() if 'basicModel_f' in model_file: - regs = np.load('vendor/smplify/models/regressors_locked_normalized_female.npz') + regs = np.load( + 'vendor/smplify/models/regressors_locked_normalized_female.npz') b2m = np.load('assets/b2m_f.npy') else: - regs = np.load('vendor/smplify/models/regressors_locked_normalized_male.npz') + regs = np.load( + 'vendor/smplify/models/regressors_locked_normalized_male.npz') b2m = np.load('assets/b2m_m.npy') keypoints = h5py.File(keypoint_file, 'r')['keypoints'] @@ -259,10 +272,12 @@ def main(keypoint_file, masks_file, camera_file, out, model_file, prior_file, re 'width': int(camera_data['width'] * resize), 'height': int(camera_data['height'] * resize)} if display: - debug_cam = ProjectPoints(v=base_smpl, t=camera.t, rt=camera.rt, c=camera.c, f=camera.f, k=camera.k) + debug_cam = ProjectPoints( + v=base_smpl, t=camera.t, rt=camera.rt, c=camera.c, f=camera.f, k=camera.k) debug_light = LambertianPointLight(f=base_smpl.f, v=base_smpl, num_verts=len(base_smpl), light_pos=np.zeros(3), vc=np.ones(3), light_color=np.ones(3)) - debug_rn = ColoredRenderer(camera=debug_cam, v=base_smpl, f=base_smpl.f, vc=debug_light, frustum=frustum) + debug_rn = ColoredRenderer( + camera=debug_cam, v=base_smpl, f=base_smpl.f, vc=debug_light, frustum=frustum) else: debug_rn = None @@ -271,17 +286,21 @@ def create_frame(i, smpl, copy=True): f = FrameData() f.smpl = copy_smpl(smpl, model_data) if copy else smpl - f.camera = ProjectPoints(v=f.smpl, t=camera.t, rt=camera.rt, c=camera.c, f=camera.f, k=camera.k) + f.camera = ProjectPoints( + v=f.smpl, t=camera.t, rt=camera.rt, c=camera.c, f=camera.f, k=camera.k) - f.keypoints = np.array(keypoints[i]).reshape(-1, 3) * np.array([resize, resize, 1]) + f.keypoints = np.array( + keypoints[i]).reshape(-1, 3) * np.array([resize, resize, 1]) f.J = joints_coco(f.smpl) - f.J_proj = ProjectPoints(v=f.J, t=camera.t, rt=camera.rt, c=camera.c, f=camera.f, k=camera.k) + f.J_proj = ProjectPoints( + v=f.J, t=camera.t, rt=camera.rt, c=camera.c, f=camera.f, k=camera.k) f.mask = cv2.resize(np.array(masks[i], dtype=np.float32), (0, 0), fx=resize, fy=resize, interpolation=cv2.INTER_NEAREST) f.collision_obj = collision_obj(f.smpl, regs) f.pose_prior_obj = pose_prior_obj(f.smpl, prior_data) - f.pose_obj = (f.J_proj - f.keypoints[:, :2]) * f.keypoints[:, 2].reshape(-1, 1) + f.pose_obj = ( + f.J_proj - f.keypoints[:, :2]) * f.keypoints[:, 2].reshape(-1, 1) return f @@ -291,7 +310,8 @@ def create_frame(i, smpl, copy=True): log.info('Initial fit') num_init = 5 - indices_init = np.ceil(np.arange(num_init) * num_frames * 1. / num_init).astype(np.int) + indices_init = np.ceil(np.arange(num_init) * + num_frames * 1. / num_init).astype(np.int32) init_frames = [base_frame] for i in indices_init[1:]: @@ -302,11 +322,14 @@ def create_frame(i, smpl, copy=True): # get pose frame by frame with h5py.File(out, 'w') as fp: last_smpl = None - poses_dset = fp.create_dataset("pose", (num_frames, 72), 'f', chunks=True, compression="lzf") - trans_dset = fp.create_dataset("trans", (num_frames, 3), 'f', chunks=True, compression="lzf") - betas_dset = fp.create_dataset("betas", (10,), 'f', chunks=True, compression="lzf") - - for i in xrange(num_frames): + poses_dset = fp.create_dataset( + "pose", (num_frames, 72), 'f', chunks=True, compression="lzf") + trans_dset = fp.create_dataset( + "trans", (num_frames, 3), 'f', chunks=True, compression="lzf") + betas_dset = fp.create_dataset( + "betas", (10,), 'f', chunks=True, compression="lzf") + + for i in range(num_frames-1): if i == 0: current_frame = base_frame else: diff --git a/step2_consensus.py b/step2_consensus.py index c13ef12..06f2117 100644 --- a/step2_consensus.py +++ b/step2_consensus.py @@ -5,7 +5,7 @@ import argparse import numpy as np import chumpy as ch -import cPickle as pkl +import pickle as pkl from opendr.camera import ProjectPoints from opendr.renderer import BoundaryRenderer, ColoredRenderer @@ -43,7 +43,8 @@ def fit_consensus(frames, base_smpl, camera, frustum, model_data, nohands, icp_c else: faces = base_smpl.f - vis_rn_b = BoundaryRenderer(camera=camera, frustum=frustum, f=faces, num_channels=1) + vis_rn_b = BoundaryRenderer( + camera=camera, frustum=frustum, f=faces, num_channels=1) vis_rn_m = ColoredRenderer(camera=camera, frustum=frustum, f=faces, vc=np.zeros_like(base_smpl), bgcolor=1, num_channels=1) @@ -55,8 +56,10 @@ def fit_consensus(frames, base_smpl, camera, frustum, model_data, nohands, icp_c g_symmetry = regularize_symmetry() for step, (w_laplace, w_model, w_symmetry, sigma) in enumerate(zip( - np.linspace(6.5, 4.0, icp_count) if naked else np.linspace(4.0, 2.0, icp_count), - np.linspace(0.9, 0.6, icp_count) if naked else np.linspace(0.6, 0.3, icp_count), + np.linspace(6.5, 4.0, icp_count) if naked else np.linspace( + 4.0, 2.0, icp_count), + np.linspace(0.9, 0.6, icp_count) if naked else np.linspace( + 0.6, 0.3, icp_count), np.linspace(3.6, 1.8, icp_count), np.linspace(0.06, 0.003, icp_count), )): @@ -78,7 +81,11 @@ def fit_consensus(frames, base_smpl, camera, frustum, model_data, nohands, icp_c log.info('## Matching rays with contours') for current, f in enumerate(tqdm(frames)): - E['silh_{}'.format(current)] = ray_objective(f, sigma, base_smpl, camera, vis_rn_b, vis_rn_m) + try: + E['silh_{}'.format(current)] = ray_objective( + f, sigma, base_smpl, camera, vis_rn_b, vis_rn_m) + except: + continue log.info('## Run optimization') ch.minimize( @@ -86,7 +93,8 @@ def fit_consensus(frames, base_smpl, camera, frustum, model_data, nohands, icp_c [base_smpl.v_personal, model_template.betas], method='dogleg', options={'maxiter': 15, 'e_3': 0.001}, - callback=get_cb(frames[0], base_smpl, camera, frustum) if display else None + callback=get_cb(frames[0], base_smpl, camera, + frustum) if display else None ) @@ -95,10 +103,14 @@ def main(pose_file, masks_file, camera_file, out, obj_out, num, icp_count, model # load data with open(model_file, 'rb') as fp: - model_data = pkl.load(fp) + u = pkl._Unpickler(fp) + u.encoding = 'latin1' + model_data = u.load() with open(camera_file, 'rb') as fp: - camera_data = pkl.load(fp) + u = pkl._Unpickler(fp) + u.encoding = 'latin1' + camera_data = u.load() pose_data = h5py.File(pose_file, 'r') poses = pose_data['pose'][first_frame:last_frame] @@ -106,7 +118,8 @@ def main(pose_file, masks_file, camera_file, out, obj_out, num, icp_count, model masks = h5py.File(masks_file, 'r')['masks'][first_frame:last_frame] num_frames = masks.shape[0] - indices_consensus = np.ceil(np.arange(num) * num_frames * 1. / num).astype(np.int) + indices_consensus = np.ceil( + np.arange(num) * num_frames * 1. / num).astype(np.int64) # init base_smpl = Smpl(model_data) @@ -116,7 +129,8 @@ def main(pose_file, masks_file, camera_file, out, obj_out, num, icp_count, model f=camera_data['camera_f'], k=camera_data['camera_k'], v=base_smpl) camera_t = camera_data['camera_t'] camera_rt = camera_data['camera_rt'] - frustum = {'near': 0.1, 'far': 1000., 'width': int(camera_data['width']), 'height': int(camera_data['height'])} + frustum = {'near': 0.1, 'far': 1000., 'width': int( + camera_data['width']), 'height': int(camera_data['height'])} frames = [] for i in indices_consensus: @@ -126,11 +140,13 @@ def main(pose_file, masks_file, camera_file, out, obj_out, num, icp_count, model pose_i = np.array(poses[i], dtype=np.float32) trans_i = np.array(trans[i], dtype=np.float32) - frames.append(setup_frame_rays(base_smpl, camera, camera_t, camera_rt, pose_i, trans_i, mask)) + frames.append(setup_frame_rays(base_smpl, camera, + camera_t, camera_rt, pose_i, trans_i, mask)) log.info('Set up complete.') log.info('Begin consensus fit...') - fit_consensus(frames, base_smpl, camera, frustum, model_data, nohands, icp_count, naked, display) + fit_consensus(frames, base_smpl, camera, frustum, + model_data, nohands, icp_count, naked, display) with open(out, 'wb') as fp: pkl.dump({ diff --git a/step3_texture.py b/step3_texture.py index 4042d79..2a53e41 100644 --- a/step3_texture.py +++ b/step3_texture.py @@ -5,7 +5,7 @@ import h5py import argparse import numpy as np -import cPickle as pkl +import pickle as pkl from opendr.renderer import ColoredRenderer from opendr.camera import ProjectPoints @@ -21,20 +21,27 @@ def main(consensus_file, camera_file, video_file, pose_file, masks_file, out, mo first_frame, last_frame, display): # load data with open(model_file, 'rb') as fp: - model_data = pkl.load(fp) + u = pkl._Unpickler(fp) + u.encoding = 'latin1' + model_data = u.load() with open(camera_file, 'rb') as fp: - camera_data = pkl.load(fp) + u = pkl._Unpickler(fp) + u.encoding = 'latin1' + camera_data = u.load() with open(consensus_file, 'rb') as fp: - consensus_data = pkl.load(fp) + u = pkl._Unpickler(fp) + u.encoding = 'latin1' + consensus_data = u.load() pose_data = h5py.File(pose_file, 'r') poses = pose_data['pose'][first_frame:last_frame] trans = pose_data['trans'][first_frame:last_frame] masks = h5py.File(masks_file, 'r')['masks'][first_frame:last_frame] num_frames = masks.shape[0] - indices_texture = np.ceil(np.arange(num) * num_frames * 1. / num).astype(np.int) + indices_texture = np.ceil( + np.arange(num) * num_frames * 1. / num).astype(np.int64) vt = np.load('assets/basicModel_vt.npy') ft = np.load('assets/basicModel_ft.npy') @@ -49,8 +56,10 @@ def main(consensus_file, camera_file, video_file, pose_file, masks_file, out, mo iso_vis = IsoColoredRenderer(vt, ft, base_smpl.f, resolution) camera = ProjectPoints(t=camera_data['camera_t'], rt=camera_data['camera_rt'], c=camera_data['camera_c'], f=camera_data['camera_f'], k=camera_data['camera_k'], v=base_smpl) - frustum = {'near': 0.1, 'far': 1000., 'width': int(camera_data['width']), 'height': int(camera_data['height'])} - rn_vis = ColoredRenderer(f=base_smpl.f, frustum=frustum, camera=camera, num_channels=1) + frustum = {'near': 0.1, 'far': 1000., 'width': int( + camera_data['width']), 'height': int(camera_data['height'])} + rn_vis = ColoredRenderer( + f=base_smpl.f, frustum=frustum, camera=camera, num_channels=1) cap = cv2.VideoCapture(video_file) for _ in range(first_frame): @@ -84,18 +93,25 @@ def main(consensus_file, camera_file, video_file, pose_file, masks_file, out, mo proj = camera.r in_viewport = np.logical_and( - np.logical_and(np.round(camera.r[:, 0]) >= 0, np.round(camera.r[:, 0]) < frustum['width']), - np.logical_and(np.round(camera.r[:, 1]) >= 0, np.round(camera.r[:, 1]) < frustum['height']), + np.logical_and(np.round(camera.r[:, 0]) >= 0, np.round( + camera.r[:, 0]) < frustum['width']), + np.logical_and(np.round(camera.r[:, 1]) >= 0, np.round( + camera.r[:, 1]) < frustum['height']), ) in_mask = np.zeros(camera.shape[0], dtype=np.bool) - idx = np.round(proj[in_viewport][:, [1, 0]].T).astype(np.int).tolist() + idx = np.round(proj[in_viewport][:, [1, 0]].T).astype( + np.int64).tolist() in_mask[in_viewport] = mask[idx] faces_in_mask = np.where(np.min(in_mask[base_smpl.f], axis=1))[0] visible_faces = np.intersect1d(faces_in_mask, visibility[visible]) # get the current unwrap - part_tex = iso.render(frame / 255., camera, visible_faces) + try: + part_tex = iso.render(frame / 255., camera, visible_faces) + except: + i += 1 + continue # angle under which the texels have been seen points = np.hstack((proj, np.ones((proj.shape[0], 1)))) @@ -114,9 +130,12 @@ def main(consensus_file, camera_file, video_file, pose_file, masks_file, out, mo # update best seen texels where = np.argmax(np.atleast_3d(iso_normals) - normal_agg, axis=2) - idx = np.dstack((static_indices[0], static_indices[1], where))[part_mask == 1] - tex_agg[list(idx[:, 0]), list(idx[:, 1]), list(idx[:, 2])] = part_tex[part_mask == 1] - normal_agg[list(idx[:, 0]), list(idx[:, 1]), list(idx[:, 2])] = iso_normals[part_mask == 1] + idx = np.dstack((static_indices[0], static_indices[1], where))[ + part_mask == 1] + tex_agg[list(idx[:, 0]), list(idx[:, 1]), list( + idx[:, 2])] = part_tex[part_mask == 1] + normal_agg[list(idx[:, 0]), list(idx[:, 1]), list( + idx[:, 2])] = iso_normals[part_mask == 1] if display: im.show(part_tex, id='part_tex', waittime=1) @@ -140,7 +159,8 @@ def main(consensus_file, camera_file, video_file, pose_file, masks_file, out, mo kernel = np.ones((kernel_size, kernel_size), np.uint8) inpaint_area = cv2.dilate(tex_mask, kernel) - mask_final - tex_final = cv2.inpaint(np.uint8(tex_median * 255), np.uint8(inpaint_area * 255), 3, cv2.INPAINT_TELEA) + tex_final = cv2.inpaint(np.uint8(tex_median * 255), + np.uint8(inpaint_area * 255), 3, cv2.INPAINT_TELEA) cv2.imwrite(out, tex_final) log.info('Done.')