SceneCaptureCamera_RayTracedLens.h

Go to the documentation of this file.

// Copyright (c) 2026 Computer Vision Center (CVC) at the Universitat Autonoma
// de Barcelona (UAB).
//
// This work is licensed under the terms of the MIT license.
// For a copy, see <https://opensource.org/licenses/MIT>.
 
#pragma once
 
#include "Carla/Actor/ActorDefinition.h"
#include "Carla/Sensor/SceneCaptureSensor.h"
#include "Carla/Util/CameraModelUtil.h"
 
#include <util/ue-header-guard-begin.h>
#include "Actor/ActorBlueprintFunctionLibrary.h"
#include <util/ue-header-guard-end.h>
 
#include "SceneCaptureCamera_RayTracedLens.generated.h"
 
/// A path-traced camera sensor ("sensor.camera.rt_lens") that renders through
/// a configurable lens model (see Util/CameraModelUtil.h for the model
/// contract) instead of the stock linear pinhole projection.
///
/// Unlike ASceneCaptureCamera_WideAngleLens, the lens distortion here is not
/// a 2D post-process warp applied to a rasterized cubemap: the lens model is
/// evaluated per-ray inside the path tracer itself, via genuine per-view
/// FPostProcessSettings fields (see RTLensEngineAdapter.h), so multiple
/// simultaneous rt_lens cameras with different lens models do not interfere
/// with each other. Consequently this sensor captures through a single
/// USceneCaptureComponent2D, exactly like ASceneCaptureCamera, and reuses the
/// same standard RGB readback path instead of the WideAngleLens family's
/// 6-face cubemap + compute-shader resample pipeline.
UCLASS()
class CARLA_API ASceneCaptureCamera_RayTracedLens : public ASceneCaptureSensor
{
  GENERATED_BODY()
 
public:
 
  static FActorDefinition GetSensorDefinition();
 
  ASceneCaptureCamera_RayTracedLens(const FObjectInitializer &ObjectInitializer);
 
  void Set(const FActorDescription &ActorDescription) override;
 
protected:
 
  void PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds) override;
 
  /// Restores r.RayTracing.NonBlockingPipelineCreation if this sensor left it
  /// at 0 (synchronous mode, see PostPhysTick): the override is process-wide
  /// and would otherwise outlive the sensor, making every other ray tracing
  /// pipeline in the process (Lumen, reflections) block on compiles. Another
  /// live rt_lens sensor re-applies it on its next tick.
  void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
 
  /// Forces path-tracing view mode for this sensor's capture. Hooked through
  /// the same extension point ASceneCaptureSensor::BeginPlay already calls
  /// for every subclass, so no BeginPlay override is needed here.
  void UpdatePostProcessConfig(FPostProcessConfig &InOutPostProcessConfig) override;
 
  bool RequiresRayTracing() const override
  {
    return true;
  }
 
  virtual void OnFirstClientConnected() override;
  virtual void OnLastClientDisconnected() override;
 
  /// Runs this tick's capture and this tick's readback in the order the world's
  /// synchronization mode requires, then returns. Every path-traced lens sensor
  /// must go through here so colour and AOV cameras stay on the same frame.
  ///
  /// EnqueueReadback is invoked at most once, only when clients are listening,
  /// with the bNonBlocking flag to hand to the ImageUtil::ReadImageData*
  /// overload; the callee just issues its own read of GetCaptureRenderTarget().
  ///
  /// Synchronous mode: capture first, then a BLOCKING readback of what was just
  /// rendered, delivered by FSensorManager's single per-tick
  /// ImageUtil::FlushBatchedReadbacks() GPU sync -- the frame the client gets
  /// for tick k is tick k's render and cannot be dropped.
  /// Asynchronous mode: readback first (of the previous tick's pixels), then
  /// capture, never waiting on the render thread -- one tick of latency and
  /// frames may drop under load, but the server cannot stall.
  ///
  /// Context is the capture metadata matching the pixels this readback will
  /// pull: this tick's in synchronous mode, the PREVIOUS tick's in async mode.
  void TickCaptureAndReadback(
      UWorld *World,
      ELevelTick TickType,
      float DeltaSeconds,
      TFunctionRef<void(bool bNonBlocking, const FSensorCaptureContext &Context)> EnqueueReadback);
 
  // Protected rather than private so subclasses that render the same lens into
  // a different output (ASceneCaptureCamera_RayTracedLensDistance,
  // ASceneCaptureCamera_RayTracedLensInstance) can reuse this class's
  // Set()/TickCaptureAndReadback() wholesale and only adjust these knobs --
  // PostPhysTick reasserts them every tick, so a subclass cannot override them
  // by writing PostProcessSettings directly.
 
  /// Pins the capture's auto exposure to the daylight histogram window
  /// (EV100 10-12, same as the viewport). Without this the path-traced
  /// view's eye adaptation can meter the raw sun-disk radiance and adapt
  /// far past the clamp the world volumes intend, blacking out the image.
  void SetRayTracedLensExposure();
 
  /// Resolved lens-model parameters, rebuilt in Set() from the
  /// camera_model / distortion_coeffs / lut / fx / fy / cx / cy /
  /// theta_max_deg / ca_shift_r / ca_shift_b Blueprint attributes and pushed
  /// into CaptureComponent2D->PostProcessSettings there via
  /// RTLensEngineAdapter::ApplyLensModel. Kept as a member (rather than a
  /// Set()-local) purely for inspection/debugging.
  FLensModelDescriptor LensModel;
 
  /// Path-tracer quality knobs. These map directly onto genuine per-view
  /// engine fields (FPostProcessSettings::PathTracingSamplesPerPixel /
  /// PathTracingEnableDenoiser, see Engine/Classes/Engine/Scene.h), so unlike
  /// LensModel they do not need the RTLensEngineAdapter seam.
  int32 SamplesPerPixel = 16;
  bool bEnableDenoiser = true;
 
private:
 
  /// Capture metadata for what is currently in the render target (see
  /// TickCaptureAndReadback).
  FSensorCaptureContext PendingReadbackContext;
  bool bHasPendingReadbackContext = false;
};