Add stereoscopic rendering through multiview

2025-11-08 12:40:44 +00:00 · 2021-05-07 23:19:04 +10:00
parent 600b4c9c7b
commit 15c1a76361
53 changed files with 1062 additions and 467 deletions
--- a/servers/rendering/renderer_scene_cull.cpp
+++ b/servers/rendering/renderer_scene_cull.cpp
@@ -2186,143 +2186,92 @@ bool RendererSceneCull::_light_instance_update_shadow(Instance *p_instance, cons
 	return animated_material_found;
 }

-void RendererSceneCull::render_camera(RID p_render_buffers, RID p_camera, RID p_scenario, RID p_viewport, Size2 p_viewport_size, float p_screen_lod_threshold, RID p_shadow_atlas) {
-// render to mono camera
+void RendererSceneCull::render_camera(RID p_render_buffers, RID p_camera, RID p_scenario, RID p_viewport, Size2 p_viewport_size, float p_screen_lod_threshold, RID p_shadow_atlas, Ref<XRInterface> &p_xr_interface) {
 #ifndef _3D_DISABLED

 	Camera *camera = camera_owner.getornull(p_camera);
 	ERR_FAIL_COND(!camera);

-	/* STEP 1 - SETUP CAMERA */
-	CameraMatrix camera_matrix;
-	bool ortho = false;
+	RendererSceneRender::CameraData camera_data;

-	switch (camera->type) {
-		case Camera::ORTHOGONAL: {
-			camera_matrix.set_orthogonal(
-					camera->size,
-					p_viewport_size.width / (float)p_viewport_size.height,
-					camera->znear,
-					camera->zfar,
-					camera->vaspect);
-			ortho = true;
-		} break;
-		case Camera::PERSPECTIVE: {
-			camera_matrix.set_perspective(
-					camera->fov,
-					p_viewport_size.width / (float)p_viewport_size.height,
-					camera->znear,
-					camera->zfar,
-					camera->vaspect);
-			ortho = false;
+	// Setup Camera(s)
+	if (p_xr_interface.is_null()) {
+		// Normal camera
+		Transform3D transform = camera->transform;
+		CameraMatrix projection;
+		bool vaspect = camera->vaspect;
+		bool is_ortogonal = false;

-		} break;
-		case Camera::FRUSTUM: {
-			camera_matrix.set_frustum(
-					camera->size,
-					p_viewport_size.width / (float)p_viewport_size.height,
-					camera->offset,
-					camera->znear,
-					camera->zfar,
-					camera->vaspect);
-			ortho = false;
-		} break;
+		switch (camera->type) {
+			case Camera::ORTHOGONAL: {
+				projection.set_orthogonal(
+						camera->size,
+						p_viewport_size.width / (float)p_viewport_size.height,
+						camera->znear,
+						camera->zfar,
+						camera->vaspect);
+				is_ortogonal = true;
+			} break;
+			case Camera::PERSPECTIVE: {
+				projection.set_perspective(
+						camera->fov,
+						p_viewport_size.width / (float)p_viewport_size.height,
+						camera->znear,
+						camera->zfar,
+						camera->vaspect);
+
+			} break;
+			case Camera::FRUSTUM: {
+				projection.set_frustum(
+						camera->size,
+						p_viewport_size.width / (float)p_viewport_size.height,
+						camera->offset,
+						camera->znear,
+						camera->zfar,
+						camera->vaspect);
+			} break;
+		}
+
+		camera_data.set_camera(transform, projection, is_ortogonal, vaspect);
+	} else {
+		// Setup our camera for our XR interface.
+		// We can support multiple views here each with their own camera
+		Transform3D transforms[RendererSceneRender::MAX_RENDER_VIEWS];
+		CameraMatrix projections[RendererSceneRender::MAX_RENDER_VIEWS];
+
+		uint32_t view_count = p_xr_interface->get_view_count();
+		ERR_FAIL_COND_MSG(view_count > RendererSceneRender::MAX_RENDER_VIEWS, "Requested view count is not supported");
+
+		float aspect = p_viewport_size.width / (float)p_viewport_size.height;
+
+		Transform3D world_origin = XRServer::get_singleton()->get_world_origin();
+
+		// We ignore our camera position, it will have been positioned with a slightly old tracking position.
+		// Instead we take our origin point and have our XR interface add fresh tracking data! Whoohoo!
+		for (uint32_t v = 0; v < view_count; v++) {
+			transforms[v] = p_xr_interface->get_transform_for_view(v, world_origin);
+			projections[v] = p_xr_interface->get_projection_for_view(v, aspect, camera->znear, camera->zfar);
+		}
+
+		if (view_count == 1) {
+			camera_data.set_camera(transforms[0], projections[0], false, camera->vaspect);
+		} else if (view_count == 2) {
+			camera_data.set_multiview_camera(view_count, transforms, projections, false, camera->vaspect);
+		} else {
+			// this won't be called (see fail check above) but keeping this comment to indicate we may support more then 2 views in the future...
+		}
 	}

 	RID environment = _render_get_environment(p_camera, p_scenario);

 	RENDER_TIMESTAMP("Update occlusion buffer")
-	RendererSceneOcclusionCull::get_singleton()->buffer_update(p_viewport, camera->transform, camera_matrix, ortho, RendererThreadPool::singleton->thread_work_pool);
+	// For now just cull on the first camera
+	RendererSceneOcclusionCull::get_singleton()->buffer_update(p_viewport, camera_data.main_transform, camera_data.main_projection, camera_data.is_ortogonal, RendererThreadPool::singleton->thread_work_pool);

-	_render_scene(camera->transform, camera_matrix, ortho, camera->vaspect, p_render_buffers, environment, camera->effects, camera->visible_layers, p_scenario, p_viewport, p_shadow_atlas, RID(), -1, p_screen_lod_threshold);
+	_render_scene(&camera_data, p_render_buffers, environment, camera->effects, camera->visible_layers, p_scenario, p_viewport, p_shadow_atlas, RID(), -1, p_screen_lod_threshold);
 #endif
 }

-void RendererSceneCull::render_camera(RID p_render_buffers, Ref<XRInterface> &p_interface, XRInterface::Eyes p_eye, RID p_camera, RID p_scenario, RID p_viewport, Size2 p_viewport_size, float p_screen_lod_threshold, RID p_shadow_atlas) {
-	// render for AR/VR interface
-#if 0
-	Camera *camera = camera_owner.getornull(p_camera);
-	ERR_FAIL_COND(!camera);
-
-	/* SETUP CAMERA, we are ignoring type and FOV here */
-	float aspect = p_viewport_size.width / (float)p_viewport_size.height;
-	CameraMatrix camera_matrix = p_interface->get_projection_for_eye(p_eye, aspect, camera->znear, camera->zfar);
-
-	// We also ignore our camera position, it will have been positioned with a slightly old tracking position.
-	// Instead we take our origin point and have our ar/vr interface add fresh tracking data! Whoohoo!
-	Transform3D world_origin = XRServer::get_singleton()->get_world_origin();
-	Transform3D cam_transform = p_interface->get_transform_for_eye(p_eye, world_origin);
-
-	RID environment = _render_get_environment(p_camera, p_scenario);
-
-	// For stereo render we only prepare for our left eye and then reuse the outcome for our right eye
-	if (p_eye == XRInterface::EYE_LEFT) {
-		// Center our transform, we assume basis is equal.
-		Transform3D mono_transform = cam_transform;
-		Transform3D right_transform = p_interface->get_transform_for_eye(XRInterface::EYE_RIGHT, world_origin);
-		mono_transform.origin += right_transform.origin;
-		mono_transform.origin *= 0.5;
-
-		// We need to combine our projection frustums for culling.
-		// Ideally we should use our clipping planes for this and combine them,
-		// however our shadow map logic uses our projection matrix.
-		// Note: as our left and right frustums should be mirrored, we don't need our right projection matrix.
-
-		// - get some base values we need
-		float eye_dist = (mono_transform.origin - cam_transform.origin).length();
-		float z_near = camera_matrix.get_z_near(); // get our near plane
-		float z_far = camera_matrix.get_z_far(); // get our far plane
-		float width = (2.0 * z_near) / camera_matrix.matrix[0][0];
-		float x_shift = width * camera_matrix.matrix[2][0];
-		float height = (2.0 * z_near) / camera_matrix.matrix[1][1];
-		float y_shift = height * camera_matrix.matrix[2][1];
-
-		// printf("Eye_dist = %f, Near = %f, Far = %f, Width = %f, Shift = %f\n", eye_dist, z_near, z_far, width, x_shift);
-
-		// - calculate our near plane size (horizontal only, right_near is mirrored)
-		float left_near = -eye_dist - ((width - x_shift) * 0.5);
-
-		// - calculate our far plane size (horizontal only, right_far is mirrored)
-		float left_far = -eye_dist - (z_far * (width - x_shift) * 0.5 / z_near);
-		float left_far_right_eye = eye_dist - (z_far * (width + x_shift) * 0.5 / z_near);
-		if (left_far > left_far_right_eye) {
-			// on displays smaller then double our iod, the right eye far frustrum can overtake the left eyes.
-			left_far = left_far_right_eye;
-		}
-
-		// - figure out required z-shift
-		float slope = (left_far - left_near) / (z_far - z_near);
-		float z_shift = (left_near / slope) - z_near;
-
-		// - figure out new vertical near plane size (this will be slightly oversized thanks to our z-shift)
-		float top_near = (height - y_shift) * 0.5;
-		top_near += (top_near / z_near) * z_shift;
-		float bottom_near = -(height + y_shift) * 0.5;
-		bottom_near += (bottom_near / z_near) * z_shift;
-
-		// printf("Left_near = %f, Left_far = %f, Top_near = %f, Bottom_near = %f, Z_shift = %f\n", left_near, left_far, top_near, bottom_near, z_shift);
-
-		// - generate our frustum
-		CameraMatrix combined_matrix;
-		combined_matrix.set_frustum(left_near, -left_near, bottom_near, top_near, z_near + z_shift, z_far + z_shift);
-
-		// and finally move our camera back
-		Transform3D apply_z_shift;
-		apply_z_shift.origin = Vector3(0.0, 0.0, z_shift); // z negative is forward so this moves it backwards
-		mono_transform *= apply_z_shift;
-
-		// now prepare our scene with our adjusted transform projection matrix
-		_prepare_scene(mono_transform, combined_matrix, false, false, p_render_buffers, environment, camera->visible_layers, p_scenario, p_shadow_atlas, RID(), p_screen_lod_threshold);
-	} else if (p_eye == XRInterface::EYE_MONO) {
-		// For mono render, prepare as per usual
-		_prepare_scene(cam_transform, camera_matrix, false, false, p_render_buffers, environment, camera->visible_layers, p_scenario, p_shadow_atlas, RID(), p_screen_lod_threshold);
-	}
-
-	// And render our scene...
-	_render_scene(p_render_buffers, cam_transform, camera_matrix, false, environment, camera->effects, p_scenario, p_shadow_atlas, RID(), -1, p_screen_lod_threshold);
-#endif
-};
-
 void RendererSceneCull::_frustum_cull_threaded(uint32_t p_thread, CullData *cull_data) {
 	uint32_t cull_total = cull_data->scenario->instance_data.size();
 	uint32_t total_threads = RendererThreadPool::singleton->thread_work_pool.get_thread_count();
@@ -2544,11 +2493,7 @@ void RendererSceneCull::_frustum_cull(CullData &cull_data, FrustumCullResult &cu
 	}
 }

-void RendererSceneCull::_render_scene(const Transform3D &p_cam_transform, const CameraMatrix &p_cam_projection, bool p_cam_orthogonal, bool p_cam_vaspect, RID p_render_buffers, RID p_environment, RID p_force_camera_effects, uint32_t p_visible_layers, RID p_scenario, RID p_viewport, RID p_shadow_atlas, RID p_reflection_probe, int p_reflection_probe_pass, float p_screen_lod_threshold, bool p_using_shadows) {
-	// Note, in stereo rendering:
-	// - p_cam_transform will be a transform in the middle of our two eyes
-	// - p_cam_projection is a wider frustrum that encompasses both eyes
-
+void RendererSceneCull::_render_scene(const RendererSceneRender::CameraData *p_camera_data, RID p_render_buffers, RID p_environment, RID p_force_camera_effects, uint32_t p_visible_layers, RID p_scenario, RID p_viewport, RID p_shadow_atlas, RID p_reflection_probe, int p_reflection_probe_pass, float p_screen_lod_threshold, bool p_using_shadows) {
 	Instance *render_reflection_probe = instance_owner.getornull(p_reflection_probe); //if null, not rendering to it

 	Scenario *scenario = scenario_owner.getornull(p_scenario);
@@ -2559,16 +2504,16 @@ void RendererSceneCull::_render_scene(const Transform3D &p_cam_transform, const

 	if (p_render_buffers.is_valid()) {
 		//no rendering code here, this is only to set up what needs to be done, request regions, etc.
-		scene_render->sdfgi_update(p_render_buffers, p_environment, p_cam_transform.origin); //update conditions for SDFGI (whether its used or not)
+		scene_render->sdfgi_update(p_render_buffers, p_environment, p_camera_data->main_transform.origin); //update conditions for SDFGI (whether its used or not)
 	}

 	RENDER_TIMESTAMP("Frustum Culling");

-	//rasterizer->set_camera(camera->transform, camera_matrix,ortho);
+	//rasterizer->set_camera(p_camera_data->main_transform, p_camera_data.main_projection, p_camera_data.is_ortogonal);

-	Vector<Plane> planes = p_cam_projection.get_projection_planes(p_cam_transform);
+	Vector<Plane> planes = p_camera_data->main_projection.get_projection_planes(p_camera_data->main_transform);

-	Plane near_plane(p_cam_transform.origin, -p_cam_transform.basis.get_axis(2).normalized());
+	Plane near_plane(p_camera_data->main_transform.origin, -p_camera_data->main_transform.basis.get_axis(2).normalized());

 	/* STEP 2 - CULL */

@@ -2606,7 +2551,7 @@ void RendererSceneCull::_render_scene(const Transform3D &p_cam_transform, const
 		scene_render->set_directional_shadow_count(lights_with_shadow.size());

 		for (int i = 0; i < lights_with_shadow.size(); i++) {
-			_light_instance_setup_directional_shadow(i, lights_with_shadow[i], p_cam_transform, p_cam_projection, p_cam_orthogonal, p_cam_vaspect);
+			_light_instance_setup_directional_shadow(i, lights_with_shadow[i], p_camera_data->main_transform, p_camera_data->main_projection, p_camera_data->is_ortogonal, p_camera_data->vaspect);
 		}
 	}

@@ -2647,11 +2592,11 @@ void RendererSceneCull::_render_scene(const Transform3D &p_cam_transform, const
 		cull_data.cull = &cull;
 		cull_data.scenario = scenario;
 		cull_data.shadow_atlas = p_shadow_atlas;
-		cull_data.cam_transform = p_cam_transform;
+		cull_data.cam_transform = p_camera_data->main_transform;
 		cull_data.visible_layers = p_visible_layers;
 		cull_data.render_reflection_probe = render_reflection_probe;
 		cull_data.occlusion_buffer = RendererSceneOcclusionCull::get_singleton()->buffer_get_ptr(p_viewport);
-		cull_data.camera_matrix = &p_cam_projection;
+		cull_data.camera_matrix = &p_camera_data->main_projection;
 //#define DEBUG_CULL_TIME
 #ifdef DEBUG_CULL_TIME
 		uint64_t time_from = OS::get_singleton()->get_ticks_usec();
@@ -2726,12 +2671,12 @@ void RendererSceneCull::_render_scene(const Transform3D &p_cam_transform, const

 			{ //compute coverage

-				Transform3D cam_xf = p_cam_transform;
-				float zn = p_cam_projection.get_z_near();
+				Transform3D cam_xf = p_camera_data->main_transform;
+				float zn = p_camera_data->main_projection.get_z_near();
 				Plane p(cam_xf.origin + cam_xf.basis.get_axis(2) * -zn, -cam_xf.basis.get_axis(2)); //camera near plane

 				// near plane half width and height
-				Vector2 vp_half_extents = p_cam_projection.get_viewport_half_extents();
+				Vector2 vp_half_extents = p_camera_data->main_projection.get_viewport_half_extents();

 				switch (RSG::storage->light_get_type(ins->base)) {
 					case RS::LIGHT_OMNI: {
@@ -2743,7 +2688,7 @@ void RendererSceneCull::_render_scene(const Transform3D &p_cam_transform, const
 							ins->transform.origin + cam_xf.basis.get_axis(0) * radius
 						};

-						if (!p_cam_orthogonal) {
+						if (!p_camera_data->is_ortogonal) {
 							//if using perspetive, map them to near plane
 							for (int j = 0; j < 2; j++) {
 								if (p.distance_to(points[j]) < 0) {
@@ -2771,7 +2716,7 @@ void RendererSceneCull::_render_scene(const Transform3D &p_cam_transform, const
 							base + cam_xf.basis.get_axis(0) * w
 						};

-						if (!p_cam_orthogonal) {
+						if (!p_camera_data->is_ortogonal) {
 							//if using perspetive, map them to near plane
 							for (int j = 0; j < 2; j++) {
 								if (p.distance_to(points[j]) < 0) {
@@ -2802,7 +2747,7 @@ void RendererSceneCull::_render_scene(const Transform3D &p_cam_transform, const
 			if (redraw && max_shadows_used < MAX_UPDATE_SHADOWS) {
 				//must redraw!
 				RENDER_TIMESTAMP(">Rendering Light " + itos(i));
-				light->shadow_dirty = _light_instance_update_shadow(ins, p_cam_transform, p_cam_projection, p_cam_orthogonal, p_cam_vaspect, p_shadow_atlas, scenario, p_screen_lod_threshold);
+				light->shadow_dirty = _light_instance_update_shadow(ins, p_camera_data->main_transform, p_camera_data->main_projection, p_camera_data->is_ortogonal, p_camera_data->vaspect, p_shadow_atlas, scenario, p_screen_lod_threshold);
 				RENDER_TIMESTAMP("<Rendering Light " + itos(i));
 			} else {
 				light->shadow_dirty = redraw;
@@ -2864,7 +2809,7 @@ void RendererSceneCull::_render_scene(const Transform3D &p_cam_transform, const
 	}

 	RENDER_TIMESTAMP("Render Scene ");
-	scene_render->render_scene(p_render_buffers, p_cam_transform, p_cam_projection, p_cam_orthogonal, frustum_cull_result.geometry_instances, frustum_cull_result.light_instances, frustum_cull_result.reflections, frustum_cull_result.voxel_gi_instances, frustum_cull_result.decals, frustum_cull_result.lightmaps, p_environment, camera_effects, p_shadow_atlas, occluders_tex, p_reflection_probe.is_valid() ? RID() : scenario->reflection_atlas, p_reflection_probe, p_reflection_probe_pass, p_screen_lod_threshold, render_shadow_data, max_shadows_used, render_sdfgi_data, cull.sdfgi.region_count, &sdfgi_update_data);
+	scene_render->render_scene(p_render_buffers, p_camera_data, frustum_cull_result.geometry_instances, frustum_cull_result.light_instances, frustum_cull_result.reflections, frustum_cull_result.voxel_gi_instances, frustum_cull_result.decals, frustum_cull_result.lightmaps, p_environment, camera_effects, p_shadow_atlas, occluders_tex, p_reflection_probe.is_valid() ? RID() : scenario->reflection_atlas, p_reflection_probe, p_reflection_probe_pass, p_screen_lod_threshold, render_shadow_data, max_shadows_used, render_sdfgi_data, cull.sdfgi.region_count, &sdfgi_update_data);

 	for (uint32_t i = 0; i < max_shadows_used; i++) {
 		render_shadow_data[i].instances.clear();
@@ -2911,7 +2856,11 @@ void RendererSceneCull::render_empty_scene(RID p_render_buffers, RID p_scenario,
 		environment = scenario->fallback_environment;
 	}
 	RENDER_TIMESTAMP("Render Empty Scene ");
-	scene_render->render_scene(p_render_buffers, Transform3D(), CameraMatrix(), true, PagedArray<RendererSceneRender::GeometryInstance *>(), PagedArray<RID>(), PagedArray<RID>(), PagedArray<RID>(), PagedArray<RID>(), PagedArray<RID>(), RID(), RID(), p_shadow_atlas, RID(), scenario->reflection_atlas, RID(), 0, 0, nullptr, 0, nullptr, 0, nullptr);
+
+	RendererSceneRender::CameraData camera_data;
+	camera_data.set_camera(Transform3D(), CameraMatrix(), true, false);
+
+	scene_render->render_scene(p_render_buffers, &camera_data, PagedArray<RendererSceneRender::GeometryInstance *>(), PagedArray<RID>(), PagedArray<RID>(), PagedArray<RID>(), PagedArray<RID>(), PagedArray<RID>(), RID(), RID(), p_shadow_atlas, RID(), scenario->reflection_atlas, RID(), 0, 0, nullptr, 0, nullptr, 0, nullptr);
 #endif
 }

@@ -2981,7 +2930,10 @@ bool RendererSceneCull::_render_reflection_probe_step(Instance *p_instance, int
 		}

 		RENDER_TIMESTAMP("Render Reflection Probe, Step " + itos(p_step));
-		_render_scene(xform, cm, false, false, RID(), environment, RID(), RSG::storage->reflection_probe_get_cull_mask(p_instance->base), p_instance->scenario->self, RID(), shadow_atlas, reflection_probe->instance, p_step, lod_threshold, use_shadows);
+		RendererSceneRender::CameraData camera_data;
+		camera_data.set_camera(xform, cm, false, false);
+
+		_render_scene(&camera_data, RID(), environment, RID(), RSG::storage->reflection_probe_get_cull_mask(p_instance->base), p_instance->scenario->self, RID(), shadow_atlas, reflection_probe->instance, p_step, lod_threshold, use_shadows);

 	} else {
 		//do roughness postprocess step until it believes it's done