CARLA
Loading...
Searching...
No Matches
SceneCaptureSensor.h
Go to the documentation of this file.
1// Copyright (c) 2026 Computer Vision Center (CVC) at the Universitat Autonoma
2// de Barcelona (UAB).
3//
4// This work is licensed under the terms of the MIT license.
5// For a copy, see <https://opensource.org/licenses/MIT>.
6
7#pragma once
8
11#include "Carla/Sensor/Sensor.h"
16
17#include <type_traits>
18
19#include <util/ue-header-guard-begin.h>
20#include "Async/Async.h"
21#if __has_include("GBufferView.h")
22#define CARLA_HAS_GBUFFER_API
23#include "GBufferView.h"
24#endif
25#include <util/ue-header-guard-end.h>
26
27#include "SceneCaptureSensor.generated.h"
28
29class UDrawFrustumComponent;
30class UStaticMeshComponent;
31class UTextureRenderTarget2D;
32class APostProcessVolume;
33
35{
36 /// Prevent this sensor to be spawned by users.
37 using not_spawnable = void;
38
40 {
41 Stream = std::move(InStream);
42 }
43
44 /// Replace the Stream associated with this sensor.
46 {
47 Stream = std::move(InStream);
48 }
49 /// Return the token that allows subscribing to this sensor's stream.
50 auto GetToken() const
51 {
52 bIsUsed = true;
53 return Stream.GetToken();
54 }
55
56 /// Dummy. Required for compatibility with other sensors only.
57 FTransform GetActorTransform() const
58 {
59 return {};
60 }
61 /// Return the FDataStream associated with this sensor.
62 ///
63 /// You need to provide a reference to self, this is necessary for template
64 /// deduction.
65 template <typename SensorT>
66 FAsyncDataStream GetDataStream(const SensorT &Self)
67 {
68 return Stream.MakeAsyncDataStream(Self, Self.GetEpisode().GetElapsedGameTime());
69 }
70
71 mutable bool bIsUsed = false;
73};
74
76{
77 /// Prevent this sensor to be spawned by users.
78 using not_spawnable = void;
79
81 {
82 Stream = std::move(InStream);
83 }
84
85 /// Replace the Stream associated with this sensor.
87 {
88 Stream = std::move(InStream);
89 }
90 /// Return the token that allows subscribing to this sensor's stream.
91 auto GetToken() const
92 {
93 bIsUsed = true;
94 return Stream.GetToken();
95 }
96 /// Dummy. Required for compatibility with other sensors only.
97 FTransform GetActorTransform() const
98 {
99 return {};
100 }
101 /// Return the FDataStream associated with this sensor.
102 ///
103 /// You need to provide a reference to self, this is necessary for template
104 /// deduction.
105 template <typename SensorT>
106 FAsyncDataStream GetDataStream(const SensorT &Self)
107 {
108 return Stream.MakeAsyncDataStream(Self, Self.GetEpisode().GetElapsedGameTime());
109 }
110
111 mutable bool bIsUsed = false;
113};
114
115/// Base class for sensors using a USceneCaptureComponent2D for rendering the
116/// scene. This class does not capture data, use
117/// `FPixelReader::SendPixelsInRenderThread<FColor>(*this)` in derived classes.
118///
119/// To access the USceneCaptureComponent2D override the
120/// SetUpSceneCaptureComponent function.
121///
122/// @warning All the setters should be called before BeginPlay.
123UCLASS(Abstract)
124class CARLA_API ASceneCaptureSensor : public ASensor
125{
126 GENERATED_BODY()
127
128 friend class ACarlaGameModeBase;
129 friend class FPixelReader;
130 friend class FPixelReader2;
131
132public:
133 ASceneCaptureSensor(const FObjectInitializer &ObjectInitializer);
134
135 void Set(const FActorDescription &ActorDescription) override;
136
137 void SetImageSize(uint32 Width, uint32 Height);
138
139 USceneCaptureComponent2D_CARLA *GetCaptureComponent() const
140 {
141 return CaptureComponent2D;
142 }
143
144 uint32 GetImageWidth() const
145 {
146 return ImageWidth;
147 }
148
149 uint32 GetImageHeight() const
150 {
151 return ImageHeight;
152 }
153
154 UFUNCTION(BlueprintCallable)
155 void EnablePostProcessingEffects(bool Enable = true)
156 {
157 bEnablePostProcessingEffects = Enable;
158 }
159
160 UFUNCTION(BlueprintCallable)
161 bool ArePostProcessingEffectsEnabled() const
162 {
163 return bEnablePostProcessingEffects;
164 }
165
166 /// Render the capture at ScreenPercentage (25-100) of the target resolution
167 /// and upscale it back with DLSS Super Resolution when available (bilinear
168 /// otherwise). 100 or Enable=false keeps the stock full-resolution path.
169 UFUNCTION(BlueprintCallable)
170 void SetDLSSUpscale(bool Enable, float ScreenPercentage)
171 {
172 bEnableDLSSUpscale = Enable;
173 DLSSScreenPercentage = FMath::Clamp(ScreenPercentage, 25.0f, 100.0f);
174 }
175
176 UFUNCTION(BlueprintCallable)
177 bool IsDLSSUpscaleEnabled() const
178 {
179 return bEnableDLSSUpscale;
180 }
181
182 UFUNCTION(BlueprintCallable)
183 void SetUseRayTracing(bool Enable);
184
185 UFUNCTION(BlueprintCallable)
186 bool GetUseRayTracing() const
187 {
188 return bUseRayTracing;
189 }
190
191 /// Applies the `carla.Camera.UseRayTracing` tier policy to a camera's own
192 /// ray-tracing preference. A camera that cannot render without ray tracing
193 /// passes `bRequired` and is never switched off by the policy.
194 static bool ResolveCameraRayTracing(bool bSensorEnabled, bool bRequired);
195
196 /// Show-only render mode (see FCarlaShowOnlyFilter): comma separated
197 /// semantic label names. Non-empty restricts the capture to the actors
198 /// carrying those labels (plus the shadow catchers and explicit ids).
199 UFUNCTION(BlueprintCallable)
200 void SetShowOnlyTags(const FString &CommaSeparatedLabels)
201 {
202 ShowOnlyFilter.SetShowOnlyTags(CommaSeparatedLabels);
203 }
204
205 /// Labels whose actors are rendered alongside the show-only set so they
206 /// receive its cast shadows ("Roads,Sidewalks"). No effect when the
207 /// show-only mode is off.
208 UFUNCTION(BlueprintCallable)
209 void SetShadowCatcherTags(const FString &CommaSeparatedLabels)
210 {
211 ShowOnlyFilter.SetShadowCatcherTags(CommaSeparatedLabels);
212 }
213
214 /// CARLA actor ids always included in the show-only set.
215 UFUNCTION(BlueprintCallable)
216 void SetShowOnlyActorIds(const FString &CommaSeparatedIds)
217 {
218 ShowOnlyFilter.SetShowOnlyActorIds(CommaSeparatedIds);
219 }
220
221 UFUNCTION(BlueprintCallable)
222 bool IsShowOnlyEnabled() const
223 {
224 return ShowOnlyFilter.IsEnabled();
225 }
226
228 {
229 return ShowOnlyFilter;
230 }
231
232 UFUNCTION(BlueprintCallable)
233 void Enable16BitFormat(bool Enable = false)
234 {
235 bEnable16BitFormat = Enable;
236 }
237
238 UFUNCTION(BlueprintCallable)
239 bool Is16BitFormatEnabled() const
240 {
241 return bEnable16BitFormat;
242 }
243
244 UFUNCTION(BlueprintCallable)
245 void SetFOVAngle(float FOVAngle);
246
247 UFUNCTION(BlueprintCallable)
248 float GetFOVAngle() const;
249
250 UFUNCTION(BlueprintCallable)
251 void SetTargetGamma(float InTargetGamma)
252 {
253 TargetGamma = InTargetGamma;
254 }
255
256 UFUNCTION(BlueprintCallable)
257 float GetTargetGamma() const
258 {
259 return TargetGamma;
260 }
261
262 UFUNCTION(BlueprintCallable)
263 void SetBloomMethod(EBloomMethod Method);
264
265 UFUNCTION(BlueprintCallable)
266 EBloomMethod GetBloomMethod() const;
267
268 UFUNCTION(BlueprintCallable)
269 FVector4 GetGlobalGain() const;
270
271 UFUNCTION(BlueprintCallable)
272 void SetGlobalGain(FVector4 Gain);
273
274 UFUNCTION(BlueprintCallable)
275 float GetBlueCorrection() const;
276
277 UFUNCTION(BlueprintCallable)
278 void SetBlueCorrection(float Val);
279
280 UFUNCTION(BlueprintCallable)
281 float GetDetailStrength() const;
282
283 UFUNCTION(BlueprintCallable)
284 void SetDetailStrength(float Val);
285
286 UFUNCTION(BlueprintCallable)
287 float GetFilmGrainIntensity() const;
288
289 UFUNCTION(BlueprintCallable)
290 void SetFilmGrainIntensity(float Val);
291
292 UFUNCTION(BlueprintCallable)
293 void SetAOIntensity(float Intensity);
294
295 UFUNCTION(BlueprintCallable)
296 float GetAOIntensity() const;
297
298 UFUNCTION(BlueprintCallable)
299 void SetAORadius(float Radius);
300
301 UFUNCTION(BlueprintCallable)
302 float GetAORadius() const;
303
304 UFUNCTION(BlueprintCallable)
305 void SetExposureMethod(EAutoExposureMethod Method);
306
307 UFUNCTION(BlueprintCallable)
308 EAutoExposureMethod GetExposureMethod() const;
309
310 UFUNCTION(BlueprintCallable)
311 void SetLocalExposureMethod(ELocalExposureMethod Method);
312
313 UFUNCTION(BlueprintCallable)
314 ELocalExposureMethod GetLocalExposureMethod() const;
315
316 UFUNCTION(BlueprintCallable)
317 UTexture2D *GetBloomKernelTexture() const;
318
319 UFUNCTION(BlueprintCallable)
320 void SetBloomConvolutionTexture(UTexture2D *Texture);
321
322 UFUNCTION(BlueprintCallable)
323 void SetExposureCompensation(float Compensation);
324
325 UFUNCTION(BlueprintCallable)
326 float GetExposureCompensation() const;
327
328 UFUNCTION(BlueprintCallable)
329 float GetDirtMaskIntensity() const;
330
331 UFUNCTION(BlueprintCallable)
332 void SetDirtMaskIntensity(float Intensity);
333
334 UFUNCTION(BlueprintCallable)
335 void SetShutterSpeed(float Speed);
336
337 UFUNCTION(BlueprintCallable)
338 float GetShutterSpeed() const;
339
340 UFUNCTION(BlueprintCallable)
341 void SetISO(float ISO);
342
343 UFUNCTION(BlueprintCallable)
344 float GetISO() const;
345
346 UFUNCTION(BlueprintCallable)
347 void SetAperture(float Aperture);
348
349 UFUNCTION(BlueprintCallable)
350 float GetAperture() const;
351
352 UFUNCTION(BlueprintCallable)
353 void SetSensorWidth(float Width);
354
355 UFUNCTION(BlueprintCallable)
356 float GetSensorWidth() const;
357
358 UFUNCTION(BlueprintCallable)
359 void SetFocalDistance(float Distance);
360
361 UFUNCTION(BlueprintCallable)
362 float GetFocalDistance() const;
363
364 UFUNCTION(BlueprintCallable)
365 void SetDepthBlurAmount(float Amount);
366
367 UFUNCTION(BlueprintCallable)
368 float GetDepthBlurAmount() const;
369
370 UFUNCTION(BlueprintCallable)
371 void SetDepthBlurRadius(float Radius);
372
373 UFUNCTION(BlueprintCallable)
374 float GetDepthBlurRadius() const;
375
376 UFUNCTION(BlueprintCallable)
377 void SetBladeCount(int Count);
378
379 UFUNCTION(BlueprintCallable)
380 int GetBladeCount() const;
381
382 UFUNCTION(BlueprintCallable)
383 void SetDepthOfFieldMinFstop(float MinFstop);
384
385 UFUNCTION(BlueprintCallable)
386 float GetDepthOfFieldMinFstop() const;
387
388 UFUNCTION(BlueprintCallable)
389 void SetFilmSlope(float Slope);
390
391 UFUNCTION(BlueprintCallable)
392 float GetFilmSlope() const;
393
394 UFUNCTION(BlueprintCallable)
395 void SetFilmToe(float Toe);
396
397 UFUNCTION(BlueprintCallable)
398 float GetFilmToe() const;
399
400 UFUNCTION(BlueprintCallable)
401 void SetFilmShoulder(float Shoulder);
402
403 UFUNCTION(BlueprintCallable)
404 float GetFilmShoulder() const;
405
406 UFUNCTION(BlueprintCallable)
407 void SetFilmBlackClip(float BlackClip);
408
409 UFUNCTION(BlueprintCallable)
410 float GetFilmBlackClip() const;
411
412 UFUNCTION(BlueprintCallable)
413 void SetFilmWhiteClip(float WhiteClip);
414
415 UFUNCTION(BlueprintCallable)
416 float GetFilmWhiteClip() const;
417
418 UFUNCTION(BlueprintCallable)
419 void SetExposureMinBrightness(float Brightness);
420
421 UFUNCTION(BlueprintCallable)
422 float GetExposureMinBrightness() const;
423
424 UFUNCTION(BlueprintCallable)
425 void SetExposureMaxBrightness(float Brightness);
426
427 UFUNCTION(BlueprintCallable)
428 float GetExposureMaxBrightness() const;
429
430 UFUNCTION(BlueprintCallable)
431 void SetExposureSpeedDown(float Speed);
432
433 UFUNCTION(BlueprintCallable)
434 float GetExposureSpeedDown() const;
435
436 UFUNCTION(BlueprintCallable)
437 void SetExposureSpeedUp(float Speed);
438
439 UFUNCTION(BlueprintCallable)
440 float GetExposureSpeedUp() const;
441
442 UFUNCTION(BlueprintCallable)
443 void SetExposureCalibrationConstant(float Constant);
444
445 UFUNCTION(BlueprintCallable)
446 float GetExposureCalibrationConstant() const;
447
448 UFUNCTION(BlueprintCallable)
449 void SetMotionBlurIntensity(float Intensity);
450
451 UFUNCTION(BlueprintCallable)
452 float GetMotionBlurIntensity() const;
453
454 UFUNCTION(BlueprintCallable)
455 void SetMotionBlurMaxDistortion(float MaxDistortion);
456
457 UFUNCTION(BlueprintCallable)
458 float GetMotionBlurMaxDistortion() const;
459
460 UFUNCTION(BlueprintCallable)
461 void SetMotionBlurMinObjectScreenSize(float ScreenSize);
462
463 UFUNCTION(BlueprintCallable)
464 float GetMotionBlurMinObjectScreenSize() const;
465
466 UFUNCTION(BlueprintCallable)
467 void SetLensFlareIntensity(float Intensity);
468
469 UFUNCTION(BlueprintCallable)
470 float GetLensFlareIntensity() const;
471
472 UFUNCTION(BlueprintCallable)
473 void SetBloomIntensity(float Intensity);
474
475 UFUNCTION(BlueprintCallable)
476 float GetBloomIntensity() const;
477
478 UFUNCTION(BlueprintCallable)
479 void SetWhiteTemp(float Temp);
480
481 UFUNCTION(BlueprintCallable)
482 float GetWhiteTemp() const;
483
484 UFUNCTION(BlueprintCallable)
485 void SetWhiteTint(float Tint);
486
487 UFUNCTION(BlueprintCallable)
488 float GetWhiteTint() const;
489
490 UFUNCTION(BlueprintCallable)
491 void SetChromAberrIntensity(float Intensity);
492
493 UFUNCTION(BlueprintCallable)
494 float GetChromAberrIntensity() const;
495
496 UFUNCTION(BlueprintCallable)
497 void SetChromAberrOffset(float ChromAberrOffset);
498
499 UFUNCTION(BlueprintCallable)
500 float GetChromAberrOffset() const;
501
502 UFUNCTION(BlueprintCallable)
503 void SetColorSaturation(FVector4 ColorSaturation);
504
505 UFUNCTION(BlueprintCallable)
506 FVector4 GetColorSaturation() const;
507
508 UFUNCTION(BlueprintCallable)
509 void SetColorSaturationMidtones(FVector4 ColorSaturation);
510
511 UFUNCTION(BlueprintCallable)
512 FVector4 GetColorSaturationMidtones() const;
513
514 UFUNCTION(BlueprintCallable)
515 void SetColorSaturationHighlights(FVector4 ColorSaturation);
516
517 UFUNCTION(BlueprintCallable)
518 FVector4 GetColorSaturationHighlights() const;
519
520 UFUNCTION(BlueprintCallable)
521 void SetColorContrast(FVector4 ColorContrast);
522
523 UFUNCTION(BlueprintCallable)
524 FVector4 GetColorContrast() const;
525
526 UFUNCTION(BlueprintCallable)
527 void SetColorContrastMidtones(FVector4 ColorContrast);
528
529 UFUNCTION(BlueprintCallable)
530 FVector4 GetColorContrastMidtones() const;
531
532 UFUNCTION(BlueprintCallable)
533 void SetColorContrastHighlights(FVector4 ColorContrast);
534
535 UFUNCTION(BlueprintCallable)
536 FVector4 GetColorContrastHighlights() const;
537
538 UFUNCTION(BlueprintCallable)
539 void SetColorGamma(FVector4 ColorGamma);
540
541 UFUNCTION(BlueprintCallable)
542 FVector4 GetColorGamma() const;
543
544 UFUNCTION(BlueprintCallable)
545 void SetHighlightsGamma(FVector4 HighlightsGamma);
546
547 UFUNCTION(BlueprintCallable)
548 FVector4 GetHighlightsGamma() const;
549
550 UFUNCTION(BlueprintCallable)
551 void SetToneCurveAmount(float ToneCurveAmount);
552
553 UFUNCTION(BlueprintCallable)
554 float GetToneCurveAmount() const;
555
556 UFUNCTION(BlueprintCallable)
557 void SetSceneColorTint(FLinearColor SceneColorTint);
558
559 UFUNCTION(BlueprintCallable)
560 FLinearColor GetSceneColorTint() const;
561
562 UFUNCTION(BlueprintCallable)
563 void SetVignetteIntensity(float VignetteIntensity);
564
565 UFUNCTION(BlueprintCallable)
566 float GetVignetteIntensity() const;
567
568 UFUNCTION(BlueprintCallable)
569 void SetHighlightContrastScale(float HighlightContrastScale);
570
571 UFUNCTION(BlueprintCallable)
572 float GetHighlightContrastScale() const;
573
574 UFUNCTION(BlueprintCallable)
575 void SetShadowContrastScale(float ShadowContrastScale);
576
577 UFUNCTION(BlueprintCallable)
578 float GetShadowContrastScale() const;
579
580 virtual void UpdatePostProcessConfig(
581 FPostProcessConfig &InOutPostProcessConfig);
582
583 /// Use for debugging purposes only.
584 UFUNCTION(BlueprintCallable)
585 bool ReadPixels(TArray<FColor> &BitMap) const
586 {
587 check(CaptureRenderTarget != nullptr);
588 return FPixelReader::WritePixelsToArray(*CaptureRenderTarget, BitMap);
589 }
590
591 /// Use for debugging purposes only.
592 UFUNCTION(BlueprintCallable)
593 void SaveCaptureToDisk(const FString &FilePath) const
594 {
595 check(CaptureRenderTarget != nullptr);
596 FPixelReader::SavePixelsToDisk(*CaptureRenderTarget, FilePath);
597 }
598
599 UFUNCTION(BlueprintCallable)
600 USceneCaptureComponent2D *GetCaptureComponent2D()
601 {
602 return CaptureComponent2D;
603 }
604
605 UFUNCTION(BlueprintCallable)
606 UTextureRenderTarget2D *GetCaptureRenderTarget()
607 {
608 return CaptureRenderTarget;
609 }
610
611 /// Per-sensor pool of recyclable RHI readback objects. May be null before
612 /// BeginPlay or after EndPlay; consumers fall back to a per-call alloc in
613 /// that case.
614 FRHIGPUReadbackPoolPtr GetReadbackPool() const { return ReadbackPool; }
615
616 /// Immediate enqueues render commands of the scene at the current time.
617 void EnqueueRenderSceneImmediate();
618
619 /// Blocks until the render thread has finished all it's tasks.
621 {
622 TRACE_CPUPROFILER_EVENT_SCOPE(ASceneCaptureSensor::WaitForRenderThreadToFinish);
623 // FlushRenderingCommands();
624 }
625
626 struct
627 {
641 } CameraGBuffers;
642
643 UFUNCTION(BlueprintCallable)
644 static bool ApplyPostProcessVolumeToSensor(APostProcessVolume *Origin, ASceneCaptureSensor *Dest, bool bOverrideCurrentCamera = false);
645
646protected:
647 /// True for sensors that cannot render without hardware ray tracing
648 /// (path-traced lens cameras); the tier policy never switches them off.
649 virtual bool RequiresRayTracing() const
650 {
651 return false;
652 }
653
655
656#ifdef CARLA_HAS_GBUFFER_API
657 virtual void SendGBufferTextures(FGBufferRequest &GBuffer);
658#endif
659
660 virtual void BeginPlay() override;
661
662 virtual void PrePhysTick(float DeltaSeconds) override;
663 virtual void PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaTime) override;
664
665 /// Motion blur from the auto exposure time (carla.Camera.ExposureMotionBlur).
666 void UpdateExposureMotionBlur(UWorld *World, float DeltaTime);
667
668 /// Simulation time of this sensor's previous capture (-1 before the first).
669 double LastCaptureTime = -1.0;
670
671 /// Exposure time of the last capture in seconds (exposure-linked motion blur).
672 float ExposureSeconds = 0.0f;
673
674 virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
675
677
678 /// Render target necessary for scene capture.
679 UPROPERTY(VisibleAnywhere)
680 TObjectPtr<UTextureRenderTarget2D> CaptureRenderTarget = nullptr;
681
682 /// Scene capture component.
683 UPROPERTY(VisibleAnywhere)
684 TObjectPtr<USceneCaptureComponent2D_CARLA> CaptureComponent2D = nullptr;
685
686 // Standard RGB/depth/segmentation cameras (MakeCameraDefinition) never
687 // register a "gamma" blueprint attribute -- only Wide Angle Lens and
688 // RayTraced Lens cameras do, defaulting it to 2.2 (sRGB). Client code that
689 // guards on bp.has_attribute('gamma') before setting it (e.g.
690 // manual_control.py's --gamma) silently no-ops for the standard camera, so
691 // this default is the only gamma that ever actually applies to it. 2.4
692 // reads visibly brighter/blown-out next to the editor/server's own ~2.2
693 // display gamma; match it instead.
694 UPROPERTY(EditAnywhere)
695 float TargetGamma = 2.2f;
696
697 /// Image width in pixels.
698 UPROPERTY(EditAnywhere)
699 uint32 ImageWidth = 800u;
700
701 /// Image height in pixels.
702 UPROPERTY(EditAnywhere)
703 uint32 ImageHeight = 600u;
704
705 /// Whether to render the post-processing effects present in the scene.
706 UPROPERTY(EditAnywhere)
707 bool bEnablePostProcessingEffects = true;
708
709 /// Whether this sensor uses hardware ray tracing for its scene capture.
710 /// Defaults to true so cameras follow upstream behaviour out of the box.
711 /// Setting false saves ~700 MiB-1 GiB on the GPU and is intended for
712 /// sensors that do not need RT (depth, semantic, lidar). The global CVar
713 /// carla.Camera.UseRayTracing forces on/off across every camera regardless
714 /// of the per-sensor attribute.
715 UPROPERTY(EditAnywhere)
716 bool bUseRayTracing = true;
717
718 /// Whether to change render target format to PF_A16B16G16R16, offering 16bit / channel
719 UPROPERTY(EditAnywhere)
720 bool bEnable16BitFormat = false;
721
722 /// Render the capture at DLSSScreenPercentage of the target resolution and
723 /// upscale back with DLSS Super Resolution (engine scene-capture temporal
724 /// upscaler; bilinear fallback without NVIDIA hardware). Off by default:
725 /// non-NVIDIA users keep the stock full-resolution path. Requires
726 /// bEnablePostProcessingEffects (the capture needs a temporal AA method for
727 /// DLSS to engage; without one the engine spatially upscales instead).
728 UPROPERTY(EditAnywhere)
729 bool bEnableDLSSUpscale = false;
730
731 /// Internal render resolution as a percentage of the target resolution when
732 /// bEnableDLSSUpscale is set. 50 = quarter of the pixels, 33 = one ninth.
733 UPROPERTY(EditAnywhere)
734 float DLSSScreenPercentage = 50.0f;
735
736 /// Per-sensor pool of recyclable RHI readback objects. Created in BeginPlay,
737 /// dropped in EndPlay; outstanding AsyncTasks keep it alive via shared
738 /// ownership.
740
741 /// Show-only actor filter, applied to the capture component right before
742 /// every capture (see FCarlaShowOnlyFilter).
743 FCarlaShowOnlyFilter ShowOnlyFilter;
744
745 /// Returns true if any GBuffer stream has an active listener.
746 bool IsAnyGBufferClientListening() const;
747
748 /// Aggregate gate: main stream or any GBuffer stream has listeners,
749 /// or CVarCarlaCameraForceAllGBuffers forces every frame.
750 /// Non-const because ASensor::AreClientsListening() is non-const.
751 bool ShouldCaptureThisFrame();
752
753private:
754 void ApplyRayTracingSetting();
755
756 /// Captures rendered as camera cuts since BeginPlay (carla.Camera.SpawnCameraCuts).
757 int32 NumCaptures = 0;
758
759#ifdef CARLA_HAS_GBUFFER_API
760 template <
761 typename SensorT,
762 typename CameraGBufferT>
763 static void SendGBuffer(
764 SensorT &Self,
765 CameraGBufferT &CameraGBuffer,
766 FGBufferRequest &GBufferData,
767 EGBufferTextureID TextureID)
768 {
769 using PixelType = typename std::conditional<
770 std::is_same<std::remove_reference_t<CameraGBufferT>, FCameraGBufferUint8>::value,
771 FColor,
772 FLinearColor>::type;
773 FIntPoint ViewSize;
774 TArray<PixelType> Pixels;
775 if (GBufferData.WaitForTextureTransfer(TextureID))
776 {
777 TRACE_CPUPROFILER_EVENT_SCOPE_STR("GBuffer Decode");
778 void *PixelData;
779 int32 SourcePitch;
780 FIntPoint SourceExtent;
781 GBufferData.MapTextureData(
782 TextureID,
783 PixelData,
784 SourcePitch,
785 SourceExtent);
786 auto Format = GBufferData.Readbacks[(size_t)TextureID]->GetFormat();
787 ViewSize = GBufferData.ViewRect.Size();
788 Pixels.AddUninitialized(ViewSize.X * ViewSize.Y);
789 FReadSurfaceDataFlags Flags = {};
790 Flags.SetLinearToGamma(true);
792 PixelData,
793 SourcePitch,
794 SourceExtent,
795 ViewSize,
796 Format,
797 Flags,
798 Pixels);
799 GBufferData.UnmapTextureData(TextureID);
800 }
801 else
802 {
803 ViewSize = GBufferData.ViewRect.Size();
804 Pixels.SetNum(ViewSize.X * ViewSize.Y);
805 for (auto &Pixel : Pixels)
806 Pixel = PixelType::Black;
807 }
808 auto GBufferStream = CameraGBuffer.GetDataStream(Self);
809 auto Buffer = GBufferStream.PopBufferFromPool();
810 Buffer.copy_from(
812 Pixels);
813 if (Buffer.empty())
814 {
815 return;
816 }
817 SCOPE_CYCLE_COUNTER(STAT_CarlaSensorStreamSend);
818 TRACE_CPUPROFILER_EVENT_SCOPE_STR("Stream Send");
819 GBufferStream.SerializeAndSend(
820 CameraGBuffer,
821 std::move(Buffer),
822 ViewSize.X,
823 ViewSize.Y,
824 Self.GetFOVAngle());
825 }
826#endif
827
828protected:
829#ifdef CARLA_HAS_GBUFFER_API
830 template <typename T>
831 void SendGBufferTexturesInternal(T &Self, FGBufferRequest &GBufferData)
832 {
833 for (size_t i = 0; i != FGBufferRequest::TextureCount; ++i)
834 {
835 if ((GBufferData.DesiredTexturesMask & (UINT64_C(1) << i)) == 0)
836 {
837 continue;
838 }
839 auto &C = CameraGBuffers;
840 EGBufferTextureID ID = (EGBufferTextureID)i;
841 switch (ID)
842 {
843 case EGBufferTextureID::SceneColor:
844 SendGBuffer(Self, C.SceneColor, GBufferData, ID);
845 break;
846 case EGBufferTextureID::SceneDepth:
847 SendGBuffer(Self, C.SceneDepth, GBufferData, ID);
848 break;
849 case EGBufferTextureID::SceneStencil:
850 SendGBuffer(Self, C.SceneStencil, GBufferData, ID);
851 break;
852 case EGBufferTextureID::GBufferA:
853 SendGBuffer(Self, C.GBufferA, GBufferData, ID);
854 break;
855 case EGBufferTextureID::GBufferB:
856 SendGBuffer(Self, C.GBufferB, GBufferData, ID);
857 break;
858 case EGBufferTextureID::GBufferC:
859 SendGBuffer(Self, C.GBufferC, GBufferData, ID);
860 break;
861 case EGBufferTextureID::GBufferD:
862 SendGBuffer(Self, C.GBufferD, GBufferData, ID);
863 break;
864 case EGBufferTextureID::GBufferE:
865 SendGBuffer(Self, C.GBufferE, GBufferData, ID);
866 break;
867 case EGBufferTextureID::GBufferF:
868 SendGBuffer(Self, C.GBufferF, GBufferData, ID);
869 break;
870 case EGBufferTextureID::Velocity:
871 SendGBuffer(Self, C.Velocity, GBufferData, ID);
872 break;
873 case EGBufferTextureID::SSAO:
874 SendGBuffer(Self, C.SSAO, GBufferData, ID);
875 break;
876 case EGBufferTextureID::CustomDepth:
877 SendGBuffer(Self, C.CustomDepth, GBufferData, ID);
878 break;
879 case EGBufferTextureID::CustomStencil:
880 SendGBuffer(Self, C.CustomStencil, GBufferData, ID);
881 break;
882 default:
883 abort();
884 }
885 }
886 }
887#endif
888};
TSharedPtr< FRHIGPUReadbackPool, ESPMode::ThreadSafe > FRHIGPUReadbackPoolPtr
Base class for the CARLA Game Mode.
Base class for sensors using a USceneCaptureComponent2D for rendering the scene.
FCameraGBufferUint8 CustomDepth
FCameraGBufferUint8 GBufferD
FCameraGBufferUint8 GBufferE
FCameraGBufferUint8 GBufferF
FCameraGBufferUint8 GBufferA
virtual void SetUpSceneCaptureComponent(USceneCaptureComponent2D &SceneCapture)
uint32 GetImageHeight() const
FCameraGBufferUint8 SceneDepth
FCameraGBufferUint8 SceneStencil
uint32 GetImageWidth() const
FCameraGBufferUint8 SSAO
FCameraGBufferUint8 GBufferC
virtual bool RequiresRayTracing() const
True for sensors that cannot render without hardware ray tracing (path-traced lens cameras); the tier...
FRHIGPUReadbackPoolPtr GetReadbackPool() const
Per-sensor pool of recyclable RHI readback objects.
void WaitForRenderThreadToFinish()
Blocks until the render thread has finished all it's tasks.
FCameraGBufferUint8 CustomStencil
FCameraGBufferUint8 Velocity
FCameraGBufferUint8 SceneColor
FCameraGBufferUint8 GBufferB
const FCarlaShowOnlyFilter & GetShowOnlyFilter() const
A streaming channel for sending sensor data to clients, supports sending data asynchronously.
auto GetToken() const
Return the token that allows subscribing to this stream.
Definition DataStream.h:53
FAsyncDataStreamTmpl< T > MakeAsyncDataStream(const SensorT &Sensor, double Timestamp)
Create a FAsyncDataStream object.
Definition DataStream.h:41
Utils for reading pixels from UTextureRenderTarget2D.
Definition PixelReader.h:42
static bool WritePixelsToArray(UTextureRenderTarget2D &RenderTarget, TArray< FColor > &BitMap)
Copy the pixels in RenderTarget into BitMap.
static TFuture< bool > SavePixelsToDisk(UTextureRenderTarget2D &RenderTarget, const FString &FilePath)
Asynchronously save the pixels in RenderTarget to disk.
bool DecodePixelsByFormat(const void *PixelData, int32 SourcePitch, FIntPoint Extent, EPixelFormat Format, FReadSurfaceDataFlags Flags, TArrayView< FLinearColor > Out)
Definition ImageUtil.cpp:38
A description of a Carla Actor with all its variation.
FTransform GetActorTransform() const
Dummy. Required for compatibility with other sensors only.
void SetStream(FDataMultiStream InStream)
Replace the Stream associated with this sensor.
void SetDataStream(FDataStream InStream)
auto GetToken() const
Return the token that allows subscribing to this sensor's stream.
void not_spawnable
Prevent this sensor to be spawned by users.
FAsyncDataStream GetDataStream(const SensorT &Self)
Return the FDataStream associated with this sensor.
auto GetToken() const
Return the token that allows subscribing to this sensor's stream.
void SetStream(FDataMultiStream InStream)
Replace the Stream associated with this sensor.
FAsyncDataStream GetDataStream(const SensorT &Self)
Return the FDataStream associated with this sensor.
void not_spawnable
Prevent this sensor to be spawned by users.
FTransform GetActorTransform() const
Dummy. Required for compatibility with other sensors only.
void SetDataStream(FDataStream InStream)
Show-only ("render layer") filter shared by every scene-capture camera.
typename detail::CompileTimeTypeMapImpl< sizeof...(Items), Items... >::template get< InKey > get