From f4a53a04f11e86191f3a699538bd4cdafc9ada8a Mon Sep 17 00:00:00 2001 From: Patrik Huber Date: Fri, 2 Dec 2016 19:10:01 +0000 Subject: [PATCH] Replaced the 3DMM fitting with fit_shape_and_pose function --- apps/4dface.cpp | 57 ++++++------------------------------------------- 1 file changed, 6 insertions(+), 51 deletions(-) diff --git a/apps/4dface.cpp b/apps/4dface.cpp index 6bd48aa..e4abab0 100644 --- a/apps/4dface.cpp +++ b/apps/4dface.cpp @@ -203,63 +203,18 @@ int main(int argc, char *argv[]) rcr::draw_landmarks(frame, current_landmarks, { 0, 255, 0 }); // green, the new optimised landmarks } - // Fit the 3DMM. First, estimate the pose: - vector image_points; // the 2D landmark points for which a mapping to the 3D model exists. A subset of all the detected landmarks. - vector model_points; // the corresponding points in the 3D shape model - vector vertex_indices; // their vertex indices - std::tie(image_points, model_points, vertex_indices) = get_corresponding_pointset(current_landmarks, landmark_mapper, morphable_model); - auto scaled_ortho_projection = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, frame.rows); - auto rendering_params = fitting::RenderingParameters(scaled_ortho_projection, frame.cols, frame.rows); - - // Given the estimated pose, find 2D-3D contour correspondences of the front-facing face contour: - // These are the additional contour correspondences we're going to find and then use: - vector image_points_contour; - vector model_points_contour; - vector vertex_indices_contour; - // For each 2D contour landmark, get the corresponding 3D vertex point and vertex id: - auto yaw_angle = glm::degrees(glm::eulerAngles(rendering_params.get_rotation())[1]); - std::tie(image_points_contour, model_points_contour, vertex_indices_contour) = fitting::get_contour_correspondences(rcr_to_eos_landmark_collection(current_landmarks), ibug_contour, model_contour, yaw_angle, morphable_model.get_mean(), rendering_params.get_modelview(), rendering_params.get_projection(), fitting::get_opencv_viewport(frame.cols, frame.rows)); - // Add the contour correspondences to the set of landmarks that we use for the fitting: - model_points = concat(model_points, model_points_contour); - vertex_indices = concat(vertex_indices, vertex_indices_contour); - image_points = concat(image_points, image_points_contour); - - // Occluded edge fitting: Fit the occluding model contour to the detected contour landmarks: - vector occluding_contour_landmarks; - if (yaw_angle >= 0.0f) // positive yaw = subject looking to the left - { // the left contour is the occluding one we want to use ("away-facing") - auto contour_landmarks_ = rcr::filter(current_landmarks, ibug_contour.left_contour); - std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) { occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] }); }); - } - else { - auto contour_landmarks_ = rcr::filter(current_landmarks, ibug_contour.right_contour); - std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) { occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] }); }); - } - - auto mean_mesh = morphable_model.get_mean(); - auto edge_correspondences = fitting::find_occluding_edge_correspondences(mean_mesh, edge_topology, rendering_params, occluding_contour_landmarks, 200.0f); - // Add new edge correspondences: - image_points = fitting::concat(image_points, edge_correspondences.first); - vertex_indices = fitting::concat(vertex_indices, edge_correspondences.second); - for (const auto& e : edge_correspondences.second) - { - model_points.emplace_back(morphable_model.get_shape_model().get_mean_at_point(e)); - } - - // Re-estimate the pose with all landmarks, including the contour landmarks: - scaled_ortho_projection = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, frame.rows); - rendering_params = fitting::RenderingParameters(scaled_ortho_projection, frame.cols, frame.rows); - Mat affine_cam = fitting::get_3x4_affine_camera_matrix(rendering_params, frame.cols, frame.rows); - - // Fit the PCA shape model and expression blendshapes: + // Fit the 3DMM: + fitting::RenderingParameters rendering_params; vector shape_coefficients, blendshape_coefficients; - Mat shape_instance = fitting::fit_shape(affine_cam, morphable_model, blendshapes, image_points, vertex_indices, 10.0f, boost::none, shape_coefficients, blendshape_coefficients); + vector image_points; + render::Mesh mesh; + std::tie(mesh, rendering_params) = fitting::fit_shape_and_pose(morphable_model, blendshapes, rcr_to_eos_landmark_collection(current_landmarks), landmark_mapper, unmodified_frame.cols, unmodified_frame.rows, edge_topology, ibug_contour, model_contour, 3, 5, 15.0f, boost::none, shape_coefficients, blendshape_coefficients, image_points); + Mat affine_cam = fitting::get_3x4_affine_camera_matrix(rendering_params, frame.cols, frame.rows); // Draw the 3D pose of the face: draw_axes_topright(glm::eulerAngles(rendering_params.get_rotation())[0], glm::eulerAngles(rendering_params.get_rotation())[1], glm::eulerAngles(rendering_params.get_rotation())[2], frame); // Get the fitted mesh, extract the texture: - render::Mesh mesh = morphablemodel::sample_to_mesh(shape_instance, morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates()); Mat isomap = render::extract_texture(mesh, affine_cam, unmodified_frame, true, render::TextureInterpolation::NearestNeighbour, 512); // Merge the isomaps - add the current one to the already merged ones: