A path-traced instance/semantic sensor ("sensor.camera.rt_lens_instance"): the labelling twin of sensor.camera.rt_lens. More...

#include <SceneCaptureCamera_RayTracedLensInstance.h>

Inheritance diagram for ASceneCaptureCamera_RayTracedLensInstance:

Public Member Functions
	ASceneCaptureCamera_RayTracedLensInstance (const FObjectInitializer &ObjectInitializer)

void	Set (const FActorDescription &ActorDescription) override

Public Member Functions inherited from ASceneCaptureCamera_RayTracedLens
	ASceneCaptureCamera_RayTracedLens (const FObjectInitializer &ObjectInitializer)

Public Member Functions inherited from ASceneCaptureSensor
bool	ArePostProcessingEffectsEnabled () const

	ASceneCaptureSensor (const FObjectInitializer &ObjectInitializer)

void	Enable16BitFormat (bool Enable=false)

void	EnablePostProcessingEffects (bool Enable=true)

void	EnqueueRenderSceneImmediate ()
	Immediate enqueues render commands of the scene at the current time.

float	GetAOIntensity () const

float	GetAORadius () const

float	GetAperture () const

int	GetBladeCount () const

float	GetBloomIntensity () const

UTexture2D *	GetBloomKernelTexture () const

EBloomMethod	GetBloomMethod () const

float	GetBlueCorrection () const

USceneCaptureComponent2D_CARLA *	GetCaptureComponent () const

USceneCaptureComponent2D *	GetCaptureComponent2D ()

UTextureRenderTarget2D *	GetCaptureRenderTarget ()

float	GetChromAberrIntensity () const

float	GetChromAberrOffset () const

FVector4	GetColorContrast () const

FVector4	GetColorContrastHighlights () const

FVector4	GetColorContrastMidtones () const

FVector4	GetColorGamma () const

FVector4	GetColorSaturation () const

FVector4	GetColorSaturationHighlights () const

FVector4	GetColorSaturationMidtones () const

float	GetDepthBlurAmount () const

float	GetDepthBlurRadius () const

float	GetDepthOfFieldMinFstop () const

float	GetDetailStrength () const

float	GetDirtMaskIntensity () const

float	GetExposureCalibrationConstant () const

float	GetExposureCompensation () const

float	GetExposureMaxBrightness () const

EAutoExposureMethod	GetExposureMethod () const

float	GetExposureMinBrightness () const

float	GetExposureSpeedDown () const

float	GetExposureSpeedUp () const

float	GetFilmBlackClip () const

float	GetFilmGrainIntensity () const

float	GetFilmShoulder () const

float	GetFilmSlope () const

float	GetFilmToe () const

float	GetFilmWhiteClip () const

float	GetFocalDistance () const

float	GetFOVAngle () const

FVector4	GetGlobalGain () const

float	GetHighlightContrastScale () const

FVector4	GetHighlightsGamma () const

uint32	GetImageHeight () const

uint32	GetImageWidth () const

float	GetISO () const

float	GetLensFlareIntensity () const

ELocalExposureMethod	GetLocalExposureMethod () const

float	GetMotionBlurIntensity () const

float	GetMotionBlurMaxDistortion () const

float	GetMotionBlurMinObjectScreenSize () const

FRHIGPUReadbackPoolPtr	GetReadbackPool () const
	Per-sensor pool of recyclable RHI readback objects.

FLinearColor	GetSceneColorTint () const

float	GetSensorWidth () const

float	GetShadowContrastScale () const

const FCarlaShowOnlyFilter &	GetShowOnlyFilter () const

float	GetShutterSpeed () const

float	GetTargetGamma () const

float	GetToneCurveAmount () const

bool	GetUseRayTracing () const

float	GetVignetteIntensity () const

float	GetWhiteTemp () const

float	GetWhiteTint () const

bool	Is16BitFormatEnabled () const

bool	IsDLSSUpscaleEnabled () const

bool	IsShowOnlyEnabled () const

bool	ReadPixels (TArray< FColor > &BitMap) const
	Use for debugging purposes only.

void	SaveCaptureToDisk (const FString &FilePath) const
	Use for debugging purposes only.

void	Set (const FActorDescription &ActorDescription) override

void	SetAOIntensity (float Intensity)

void	SetAORadius (float Radius)

void	SetAperture (float Aperture)

void	SetBladeCount (int Count)

void	SetBloomConvolutionTexture (UTexture2D *Texture)

void	SetBloomIntensity (float Intensity)

void	SetBloomMethod (EBloomMethod Method)

void	SetBlueCorrection (float Val)

void	SetChromAberrIntensity (float Intensity)

void	SetChromAberrOffset (float ChromAberrOffset)

void	SetColorContrast (FVector4 ColorContrast)

void	SetColorContrastHighlights (FVector4 ColorContrast)

void	SetColorContrastMidtones (FVector4 ColorContrast)

void	SetColorGamma (FVector4 ColorGamma)

void	SetColorSaturation (FVector4 ColorSaturation)

void	SetColorSaturationHighlights (FVector4 ColorSaturation)

void	SetColorSaturationMidtones (FVector4 ColorSaturation)

void	SetDepthBlurAmount (float Amount)

void	SetDepthBlurRadius (float Radius)

void	SetDepthOfFieldMinFstop (float MinFstop)

void	SetDetailStrength (float Val)

void	SetDirtMaskIntensity (float Intensity)

void	SetDLSSUpscale (bool Enable, float ScreenPercentage)
	Render the capture at ScreenPercentage (25-100) of the target resolution and upscale it back with DLSS Super Resolution when available (bilinear otherwise).

void	SetExposureCalibrationConstant (float Constant)

void	SetExposureCompensation (float Compensation)

void	SetExposureMaxBrightness (float Brightness)

void	SetExposureMethod (EAutoExposureMethod Method)

void	SetExposureMinBrightness (float Brightness)

void	SetExposureSpeedDown (float Speed)

void	SetExposureSpeedUp (float Speed)

void	SetFilmBlackClip (float BlackClip)

void	SetFilmGrainIntensity (float Val)

void	SetFilmShoulder (float Shoulder)

void	SetFilmSlope (float Slope)

void	SetFilmToe (float Toe)

void	SetFilmWhiteClip (float WhiteClip)

void	SetFocalDistance (float Distance)

void	SetFOVAngle (float FOVAngle)

void	SetGlobalGain (FVector4 Gain)

void	SetHighlightContrastScale (float HighlightContrastScale)

void	SetHighlightsGamma (FVector4 HighlightsGamma)

void	SetImageSize (uint32 Width, uint32 Height)

void	SetISO (float ISO)

void	SetLensFlareIntensity (float Intensity)

void	SetLocalExposureMethod (ELocalExposureMethod Method)

void	SetMotionBlurIntensity (float Intensity)

void	SetMotionBlurMaxDistortion (float MaxDistortion)

void	SetMotionBlurMinObjectScreenSize (float ScreenSize)

void	SetSceneColorTint (FLinearColor SceneColorTint)

void	SetSensorWidth (float Width)

void	SetShadowCatcherTags (const FString &CommaSeparatedLabels)
	Labels whose actors are rendered alongside the show-only set so they receive its cast shadows ("Roads,Sidewalks").

void	SetShadowContrastScale (float ShadowContrastScale)

void	SetShowOnlyActorIds (const FString &CommaSeparatedIds)
	CARLA actor ids always included in the show-only set.

void	SetShowOnlyTags (const FString &CommaSeparatedLabels)
	Show-only render mode (see FCarlaShowOnlyFilter): comma separated semantic label names.

void	SetShutterSpeed (float Speed)

void	SetTargetGamma (float InTargetGamma)

void	SetToneCurveAmount (float ToneCurveAmount)

void	SetUseRayTracing (bool Enable)

void	SetVignetteIntensity (float VignetteIntensity)

void	SetWhiteTemp (float Temp)

void	SetWhiteTint (float Tint)

void	WaitForRenderThreadToFinish ()
	Blocks until the render thread has finished all it's tasks.

Public Member Functions inherited from ASensor
bool	AreClientsListening ()

	ASensor (const FObjectInitializer &ObjectInitializer)

void	ClearReadyToTick ()

std::optional< FActorAttribute >	GetAttribute (const FString Name)

const UCarlaEpisode &	GetEpisode () const

virtual void	GetMirrorState (carla::multigpu::SensorState &State) const
	Multi-GPU: the runtime state this sensor starts its next tick from (noise generator, sweep angle, history).

URandomEngine *	GetRandomEngine ()

float	GetReadyToTickDeltaSeconds () const

int32	GetSeed () const

auto	GetToken () const
	Return the token that allows subscribing to this sensor's stream.

bool	IsReadyToTick () const

bool	IsStreamReady ()

FDataStream	MoveDataStream ()

void	PostPhysTickInternal (UWorld *World, ELevelTick TickType, float DeltaSeconds)

void	SetDataStream (FDataStream InStream)
	Replace the FDataStream associated with this sensor.

void	SetEpisode (const UCarlaEpisode &InEpisode)

virtual void	SetMirrorState (const carla::multigpu::SensorState &State)

void	SetPrimaryTick (float DeltaSeconds)
	Multi-GPU secondary: tick in this engine frame, with the primary's delta.

void	SetSavingDataToDisk (bool bSavingData)

void	SetSeed (int32 InSeed)

void	Tick (const float DeltaTime) final

Static Public Member Functions
static FActorDefinition	GetSensorDefinition ()

Static Public Member Functions inherited from ASceneCaptureCamera_RayTracedLens
static FActorDefinition	GetSensorDefinition ()

Static Public Member Functions inherited from ASceneCaptureSensor
static bool	ApplyPostProcessVolumeToSensor (APostProcessVolume Origin, ASceneCaptureSensor Dest, bool bOverrideCurrentCamera=false)

static bool	ResolveCameraRayTracing (bool bSensorEnabled, bool bRequired)
	Applies the `carla.Camera.UseRayTracing` tier policy to a camera's own ray-tracing preference.

Static Public Member Functions inherited from ASensor
static FSensorCaptureContext	MakeCaptureContext (const ASensor &Sensor)
	Call on the game thread right after CaptureScene().

Protected Member Functions
void	BeginPlay () override

void	PostPhysTick (UWorld *World, ELevelTick TickType, float DeltaSeconds) override

Protected Member Functions inherited from ASceneCaptureCamera_RayTracedLens
void	EndPlay (const EEndPlayReason::Type EndPlayReason) override
	Restores r.RayTracing.NonBlockingPipelineCreation if this sensor left it at 0 (synchronous mode, see PostPhysTick): the override is process-wide and would otherwise outlive the sensor, making every other ray tracing pipeline in the process (Lumen, reflections) block on compiles.

virtual void	OnFirstClientConnected () override

virtual void	OnLastClientDisconnected () override

bool	RequiresRayTracing () const override
	True for sensors that cannot render without hardware ray tracing (path-traced lens cameras); the tier policy never switches them off.

void	SetRayTracedLensExposure ()
	Pins the capture's auto exposure to the daylight histogram window (EV100 10-12, same as the viewport).

void	TickCaptureAndReadback (UWorld *World, ELevelTick TickType, float DeltaSeconds, TFunctionRef< void(bool bNonBlocking, const FSensorCaptureContext &Context)> EnqueueReadback)
	Runs this tick's capture and this tick's readback in the order the world's synchronization mode requires, then returns.

void	UpdatePostProcessConfig (FPostProcessConfig &InOutPostProcessConfig) override
	Forces path-tracing view mode for this sensor's capture.

Protected Member Functions inherited from ASceneCaptureSensor
void	CaptureSceneExtended ()

bool	IsAnyGBufferClientListening () const
	Returns true if any GBuffer stream has an active listener.

virtual void	PrePhysTick (float DeltaSeconds) override

virtual void	SetUpSceneCaptureComponent (USceneCaptureComponent2D &SceneCapture)

bool	ShouldCaptureThisFrame ()
	Aggregate gate: main stream or any GBuffer stream has listeners, or CVarCarlaCameraForceAllGBuffers forces every frame.

void	UpdateExposureMotionBlur (UWorld *World, float DeltaTime)
	Motion blur from the auto exposure time (carla.Camera.ExposureMotionBlur).

Protected Member Functions inherited from ASensor
void	EndPlay (EEndPlayReason::Type EndPlayReason) override

template<typename SensorType >
FAsyncDataStream	GetDataStream (SensorType &&Self)
	Return the FDataStream associated with this sensor.

void	PostActorCreated () override

Private Member Functions
void	ApplyInstanceAOVConfig ()
	Re-asserts everything that makes this capture a tag AOV rather than a colour render.

Additional Inherited Members
Public Attributes inherited from ASceneCaptureSensor
struct {

FCameraGBufferUint8 CustomDepth

FCameraGBufferUint8 CustomStencil

FCameraGBufferUint8 GBufferA

FCameraGBufferUint8 GBufferB

FCameraGBufferUint8 GBufferC

FCameraGBufferUint8 GBufferD

FCameraGBufferUint8 GBufferE

FCameraGBufferUint8 GBufferF

FCameraGBufferUint8 SceneColor

FCameraGBufferUint8 SceneDepth

FCameraGBufferUint8 SceneStencil

FCameraGBufferUint8 SSAO

FCameraGBufferUint8 Velocity

}	CameraGBuffers

Static Protected Member Functions inherited from ASensor
template<typename SensorType , typename ElementType >
static void	SendDataToClient (SensorType &&Sensor, TArrayView< ElementType > SensorData, const FSensorCaptureContext &CaptureContext)

Protected Attributes inherited from ASceneCaptureCamera_RayTracedLens
bool	bEnableDenoiser = true

FLensModelDescriptor	LensModel
	Resolved lens-model parameters, rebuilt in Set() from the camera_model / distortion_coeffs / lut / fx / fy / cx / cy / theta_max_deg / ca_shift_r / ca_shift_b Blueprint attributes and pushed into CaptureComponent2D->PostProcessSettings there via RTLensEngineAdapter::ApplyLensModel.

int32	SamplesPerPixel = 16
	Path-tracer quality knobs.

Protected Attributes inherited from ASceneCaptureSensor
bool	bEnable16BitFormat = false
	Whether to change render target format to PF_A16B16G16R16, offering 16bit / channel.

bool	bEnableDLSSUpscale = false
	Render the capture at DLSSScreenPercentage of the target resolution and upscale back with DLSS Super Resolution (engine scene-capture temporal upscaler; bilinear fallback without NVIDIA hardware).

bool	bEnablePostProcessingEffects = true
	Whether to render the post-processing effects present in the scene.

bool	bUseRayTracing = true
	Whether this sensor uses hardware ray tracing for its scene capture.

TObjectPtr< USceneCaptureComponent2D_CARLA >	CaptureComponent2D = nullptr
	Scene capture component.

TObjectPtr< UTextureRenderTarget2D >	CaptureRenderTarget = nullptr
	Render target necessary for scene capture.

float	DLSSScreenPercentage = 50.0f
	Internal render resolution as a percentage of the target resolution when bEnableDLSSUpscale is set.

float	ExposureSeconds = 0.0f
	Exposure time of the last capture in seconds (exposure-linked motion blur).

uint32	ImageHeight = 600u
	Image height in pixels.

uint32	ImageWidth = 800u
	Image width in pixels.

double	LastCaptureTime = -1.0
	Simulation time of this sensor's previous capture (-1 before the first).

FRHIGPUReadbackPoolPtr	ReadbackPool
	Per-sensor pool of recyclable RHI readback objects.

FCarlaShowOnlyFilter	ShowOnlyFilter
	Show-only actor filter, applied to the capture component right before every capture (see FCarlaShowOnlyFilter).

float	TargetGamma = 2.2f

Protected Attributes inherited from ASensor
bool	bIsActive = false

bool	bSavingDataToDisk = false

URandomEngine *	RandomEngine = nullptr
	Random Engine used to provide noise for sensor output.

int32	Seed = 123456789
	Seed of the pseudo-random engine.

Detailed Description

A path-traced instance/semantic sensor ("sensor.camera.rt_lens_instance"): the labelling twin of sensor.camera.rt_lens.

It renders through exactly the same lens as ASceneCaptureCamera_RayTracedLens – same attributes, same Set(), same show_only_* filtering, evaluated per ray inside the path tracer – but instead of radiance it outputs, for every pixel, the CARLA tag of the primary hit, in the byte layout sensor.camera.instance_segmentation already uses:

R = semantic label (ATagger's crp::CityObjectLabel)
G = actor id, low byte
B = actor id, high byte

so the same client-side decode works for both, and the semantic image is just the R channel of this one.

The tag comes from the same place as the raster path's: ATagger writes it into custom primitive data slot 4 (Carla/Game/Tagger.cpp), which the raster segmentation pass reads as CustomPrimitiveData[1].xy (Engine/Shaders/Private/SegmentationSensor.usf). The path tracer's closest-hit shader reads the very same GPUScene field and carries it to the ray generation shader in the material payload, which records it into a dedicated R32_UINT render-graph texture next to the albedo/normal/depth AOVs (FPostProcessSettings::PathTracingLensAOV == 2, PathTracer.LensTag). The capture resolves that texture through ESceneCaptureSource::SCS_PathTracingLensTag.

Why the raster pass cannot simply be reused: ShowFlags.PathTracing makes FDeferredShadingSceneRenderer treat the frame as a ray-traced overlay and skip the depth prepass, base pass and Nanite raster entirely, so AddSegmentationSensorPass would silently produce an empty or stale buffer rather than fail.

Nothing on this route applies exposure, tone mapping, denoising or filtering, and the primary ray is fired through the exact pixel centre, so the ids are exact 8-bit values, never blended between two actors.

Definition at line 51 of file SceneCaptureCamera_RayTracedLensInstance.h.

Constructor & Destructor Documentation

◆ ASceneCaptureCamera_RayTracedLensInstance()

ASceneCaptureCamera_RayTracedLensInstance::ASceneCaptureCamera_RayTracedLensInstance ( const FObjectInitializer & ObjectInitializer )

Definition at line 53 of file SceneCaptureCamera_RayTracedLensInstance.cpp.

References ApplyInstanceAOVConfig(), and ASceneCaptureSensor::EnablePostProcessingEffects().

Member Function Documentation

◆ ApplyInstanceAOVConfig()

void ASceneCaptureCamera_RayTracedLensInstance::ApplyInstanceAOVConfig ( )

private

Re-asserts everything that makes this capture a tag AOV rather than a colour render.

Called from Set() and from every tick, because several code paths replace the capture component's PostProcessSettings wholesale (see the note in ASceneCaptureCamera_RayTracedLens::PostPhysTick).

Definition at line 65 of file SceneCaptureCamera_RayTracedLensInstance.cpp.

References ASceneCaptureCamera_RayTracedLens::bEnableDenoiser, ASceneCaptureSensor::CaptureComponent2D, GPathTracingLensAOVTag, ASceneCaptureCamera_RayTracedLens::SamplesPerPixel, and ASceneCaptureSensor::SetFocalDistance().

Referenced by ASceneCaptureCamera_RayTracedLensInstance(), BeginPlay(), PostPhysTick(), and Set().