newton.sensors.SensorTiledCamera#

class newton.sensors.SensorTiledCamera(model, *, default_render_config=None, config=_DEPRECATED_CONFIG_UNSET, load_textures=True)[source]#

Bases: object

Warp-based tiled camera sensor for raytraced rendering across multiple worlds.

Renders up to six image channels per (world, camera) pair:

  • color – RGBA shaded image (uint32).

  • hdr_color – linear shaded RGB image (vec3f).

  • depth – ray-hit distance [m] (float32); negative means no hit.

  • normal – surface normal at hit point (vec3f).

  • albedo – unshaded surface color (uint32).

  • shape_index – shape id per pixel (uint32).

All output arrays have shape (world_count, camera_count, height, width). Use the flatten_* helpers to rearrange them into tiled RGBA buffers for display, with one tile per (world, camera) pair laid out in a grid.

Shapes without the VISIBLE flag are excluded.

Shape colors and base-color textures are interpreted as display/sRGB RGB, converted to linear RGB internally for shading, and packed according to RenderConfig.output_color_space at the output boundary.

Example

sensor = SensorTiledCamera(model)
rays = sensor.utils.compute_camera_rays_pinhole(width, height, camera_fovs=fov)
color = sensor.utils.create_color_image_output(width, height)

# BVHs are built for the initial state by ModelBuilder.finalize().
state = model.state()

# Before each frame that changes geometry, refit BVHs.
model.bvh_refit_shapes(state)
model.bvh_refit_particles(state)
sensor.update(state, camera_transforms, rays, color_image=color)

See RenderConfig for optional rendering settings and ClearData / DEFAULT_CLEAR_DATA / GRAY_CLEAR_DATA for image-clear presets.

class ClearData(clear_color=0, clear_depth=0.0, clear_shape_index=4294967295, clear_normal=(0.0, 0.0, 0.0), clear_albedo=0)#

Bases: object

Default values written to output images before rendering.

__init__(clear_color=0, clear_depth=0.0, clear_shape_index=4294967295, clear_normal=(0.0, 0.0, 0.0), clear_albedo=0)#
clear_albedo: int = 0#
clear_color: int = 0#
clear_depth: float = 0.0#
clear_normal: tuple[float, float, float] = (0.0, 0.0, 0.0)#
clear_shape_index: int = 4294967295#
class GaussianRenderMode(*values)#

Bases: IntEnum

Gaussian Render Mode

FAST = 0#

Fast Render Mode

QUALITY = 1#

Quality Render Mode, collect hits until minimum transmittance is reached

class RenderConfig(enable_global_world=True, enable_textures=False, enable_shadows=False, enable_ambient_lighting=True, enable_particles=True, enable_backface_culling=True, output_color_space=ColorSpace.SRGB, render_order=RenderOrder.PIXEL_PRIORITY, tile_width=16, tile_height=8, max_distance=1000.0, gaussians_mode=GaussianRenderMode.FAST, gaussians_min_transmittance=0.49, gaussians_max_num_hits=20)#

Bases: object

Raytrace render settings shared across all worlds.

__init__(enable_global_world=True, enable_textures=False, enable_shadows=False, enable_ambient_lighting=True, enable_particles=True, enable_backface_culling=True, output_color_space=ColorSpace.SRGB, render_order=RenderOrder.PIXEL_PRIORITY, tile_width=16, tile_height=8, max_distance=1000.0, gaussians_mode=GaussianRenderMode.FAST, gaussians_min_transmittance=0.49, gaussians_max_num_hits=20)#
enable_ambient_lighting: bool = True#

Enable ambient lighting for the scene.

enable_backface_culling: bool = True#

Cull back-facing triangles.

enable_global_world: bool = True#

Include shapes that belong to no specific world.

enable_particles: bool = True#

Enable standalone particle rendering.

Particles referenced by rendered triangle or tetrahedral deformable topology are rendered by the triangle mesh path and are not emitted as particle spheres.

enable_shadows: bool = False#

Enable shadow rays for directional lights.

enable_textures: bool = False#

Enable texture-mapped rendering for meshes.

gaussians_max_num_hits: int = 20#

Maximum Gaussian hits accumulated per ray.

gaussians_min_transmittance: float = 0.49#

Minimum transmittance before early-out during Gaussian rendering.

gaussians_mode: int = 0#

Gaussian splatting render mode (see GaussianRenderMode).

max_distance: float = 1000.0#

Maximum ray distance [m].

output_color_space: ColorSpace = 1#

Color space for packed color and albedo outputs.

Use ColorSpace.SRGB for display-encoded bytes or ColorSpace.LINEAR for linear RGB bytes.

render_order: int = 0#

Render traversal order (see RenderOrder).

tile_height: int = 8#

Tile height [px] for RenderOrder.TILED traversal.

tile_width: int = 16#

Tile width [px] for RenderOrder.TILED traversal.

class RenderLightType(*values)#

Bases: IntEnum

Light types supported by the Warp raytracer.

DIRECTIONAL = 1#

Directional Light.

SPOTLIGHT = 0#

Spotlight.

class RenderOrder(*values)#

Bases: IntEnum

Render Order

PIXEL_PRIORITY = 0#

Render the same pixel of every view before continuing to the next one

TILED = 2#

Render pixels in tiles, defined by tile_width x tile_height

VIEW_PRIORITY = 1#

Render all pixels of a whole view before continuing to the next one

class Utils(render_context, render_config=None)#

Bases: object

Utility functions for the RenderContext.

__init__(render_context, render_config=None)#
assign_checkerboard_material(*, shape_indices, resolution=64, checker_size=32)#

Assign a gray checkerboard texture material to selected shapes.

Parameters:

  • shape_indices (Sequence[int] | ndarray) – Shape indices that should use the checkerboard texture.

  • resolution (int) – Texture resolution in pixels (square texture).

  • checker_size (int) – Size of each checkerboard square in pixels.

assign_checkerboard_material_to_all_shapes(resolution=64, checker_size=32)#

Assign a gray checkerboard texture material to all shapes.

Deprecated since version 1.4: Use assign_checkerboard_material() with explicit shape indices instead.

Parameters:

  • resolution (int) – Texture resolution in pixels (square texture).

  • checker_size (int) – Size of each checkerboard square in pixels.

compute_camera_rays_fisheye_ftheta(width, height, optical_center_x, optical_center_y, *, image_width=None, image_height=None, nominal_width=None, nominal_height=None, k0=0.0, k1=1.0, k2=0.0, k3=0.0, k4=0.0, max_fov=2.0 * math.pi, out_rays=None, camera_index=0)#

Compute camera-space ray directions for F-theta fisheye cameras.

The F-theta radius polynomial r = k0 + k1 theta + k2 theta^2 + k3 theta^3 + k4 theta^4 must be monotonically increasing over the supported field of view, [0, min(max_fov / 2, pi)].

Parameters:

  • width (int) – Output image width [px].

  • height (int) – Output image height [px].

  • optical_center_x (float) – Optical center x-coordinate [px].

  • optical_center_y (float) – Optical center y-coordinate [px].

  • image_width (float | None) – Calibration image width [px]. If None, uses nominal_width, then width.

  • image_height (float | None) – Calibration image height [px]. If None, uses nominal_height, then height.

  • nominal_width (float | None) – Alias for image_width using F-theta model terminology. If both are provided, they must match.

  • nominal_height (float | None) – Alias for image_height using F-theta model terminology. If both are provided, they must match.

  • k0 (float) – Constant F-theta polynomial coefficient [px].

  • k1 (float) – Linear F-theta polynomial coefficient [px/rad].

  • k2 (float) – Quadratic F-theta polynomial coefficient [px/rad^2].

  • k3 (float) – Cubic F-theta polynomial coefficient [px/rad^3].

  • k4 (float) – Quartic F-theta polynomial coefficient [px/rad^4].

  • max_fov (float) – Maximum field of view [rad].

  • out_rays (wp.array(dtype=wp.vec3f, ndim=4) | None) – Optional output array to write into, shape (out_camera_count, height, width, 2). If None, allocates a new array.

  • camera_index (int) – Camera index in out_rays to write. Ignored when out_rays is None.

Returns:

out_rays if provided, otherwise a new array with

shape (1, height, width, 2) and dtype vec3f.

Return type:

camera_rays

compute_camera_rays_fisheye_kannala_brandt(width, height, optical_center_x, optical_center_y, *, image_width=None, image_height=None, nominal_width=None, nominal_height=None, k0=1.0, k1=0.0, k2=0.0, k3=0.0, max_fov=2.0 * math.pi, out_rays=None, camera_index=0)#

Compute camera-space ray directions for Kannala-Brandt fisheye cameras.

Uses the r = k0 theta + k1 theta^3 + k2 theta^5 + k3 theta^7 polynomial form.

The radius polynomial must be monotonically increasing over the supported field of view, [0, min(max_fov / 2, pi)].

Parameters:

  • width (int) – Output image width [px].

  • height (int) – Output image height [px].

  • optical_center_x (float) – Optical center x-coordinate [px].

  • optical_center_y (float) – Optical center y-coordinate [px].

  • image_width (float | None) – Calibration image width [px]. If None, uses nominal_width, then width.

  • image_height (float | None) – Calibration image height [px]. If None, uses nominal_height, then height.

  • nominal_width (float | None) – Alias for image_width using Kannala-Brandt model terminology. If both are provided, they must match.

  • nominal_height (float | None) – Alias for image_height using Kannala-Brandt model terminology. If both are provided, they must match.

  • k0 (float) – Linear Kannala-Brandt coefficient [px/rad].

  • k1 (float) – Cubic Kannala-Brandt coefficient [px/rad^3].

  • k2 (float) – Quintic Kannala-Brandt coefficient [px/rad^5].

  • k3 (float) – Septic Kannala-Brandt coefficient [px/rad^7].

  • max_fov (float) – Maximum field of view [rad].

  • out_rays (wp.array(dtype=wp.vec3f, ndim=4) | None) – Optional output array to write into, shape (out_camera_count, height, width, 2). If None, allocates a new array.

  • camera_index (int) – Camera index in out_rays to write. Ignored when out_rays is None.

Returns:

out_rays if provided, otherwise a new array with

shape (1, height, width, 2) and dtype vec3f.

Return type:

camera_rays

compute_camera_rays_fisheye_opencv(width, height, fx, fy, cx, cy, *, image_width=None, image_height=None, k1=0.0, k2=0.0, k3=0.0, k4=0.0, max_fov=2.0 * math.pi, out_rays=None, camera_index=0)#

Compute camera-space ray directions for OpenCV fisheye cameras.

The distorted radius polynomial r = theta * (1 + k1 theta^2 + k2 theta^4 + k3 theta^6 + k4 theta^8) must be monotonically increasing over the supported field of view, [0, min(max_fov / 2, pi)].

Parameters:

  • width (int) – Output image width [px].

  • height (int) – Output image height [px].

  • fx (float) – Horizontal focal length [px].

  • fy (float) – Vertical focal length [px].

  • cx (float) – Principal point x-coordinate [px].

  • cy (float) – Principal point y-coordinate [px].

  • image_width (float | None) – Calibration image width [px]. If None, uses width.

  • image_height (float | None) – Calibration image height [px]. If None, uses height.

  • k1 (float) – First OpenCV fisheye distortion coefficient.

  • k2 (float) – Second OpenCV fisheye distortion coefficient.

  • k3 (float) – Third OpenCV fisheye distortion coefficient.

  • k4 (float) – Fourth OpenCV fisheye distortion coefficient.

  • max_fov (float) – Maximum field of view [rad]. Pixels whose undistorted angle exceeds max_fov / 2 receive a zero ray.

  • out_rays (wp.array(dtype=wp.vec3f, ndim=4) | None) – Optional output array to write into, shape (out_camera_count, height, width, 2). If None, allocates a new array.

  • camera_index (int) – Camera index in out_rays to write. Ignored when out_rays is None.

Returns:

out_rays if provided, otherwise a new array with

shape (1, height, width, 2) and dtype vec3f.

Return type:

camera_rays

compute_camera_rays_pinhole(width, height, *, camera_fovs=None, focal_length=None, horizontal_aperture=None, vertical_aperture=None, horizontal_aperture_offset=0.0, vertical_aperture_offset=0.0, out_rays=None, camera_index=0)#

Compute camera-space ray directions for pinhole cameras.

Generates rays in camera space (origin at the camera center, direction normalized) for each pixel. Use either vertical field of view values or aperture/focal-length values.

Physical camera parameters accept any consistent length unit, such as USD-style millimeters. The unit must match across focal_length, apertures, and aperture offsets.

Leave focal_length, horizontal_aperture, and vertical_aperture as None to use camera_fovs. Supplying any of those aperture parameters selects aperture mode, requires all three, and ignores camera_fovs.

Parameters:

  • width (int) – Image width [px].

  • height (int) – Image height [px].

  • camera_fovs (float | list[float] | ndarray | wp.array(dtype=wp.float32, ndim=1) | None) – Vertical FOV angles [rad], scalar or shape (camera_count,). If None, aperture mode must be used; if no aperture parameters are provided, raises ValueError.

  • focal_length (float | list[float] | ndarray | wp.array(dtype=wp.float32, ndim=1) | None) – Focal length [mm or any consistent unit]. If None and the other aperture parameters are also None, uses camera_fovs.

  • horizontal_aperture (float | list[float] | ndarray | wp.array(dtype=wp.float32, ndim=1) | None) – Horizontal aperture [mm or any consistent unit]. If None and the other aperture parameters are also None, uses camera_fovs.

  • vertical_aperture (float | list[float] | ndarray | wp.array(dtype=wp.float32, ndim=1) | None) – Vertical aperture [mm or any consistent unit]. If None and the other aperture parameters are also None, uses camera_fovs.

  • horizontal_aperture_offset (float | list[float] | ndarray | wp.array(dtype=wp.float32, ndim=1)) – Horizontal aperture offset [mm or any consistent unit]. Defaults to 0.0 for a centered aperture.

  • vertical_aperture_offset (float | list[float] | ndarray | wp.array(dtype=wp.float32, ndim=1)) – Vertical aperture offset [mm or any consistent unit]. Defaults to 0.0 for a centered aperture.

  • out_rays (wp.array(dtype=wp.vec3f, ndim=4) | None) – Optional output array to write into, shape (out_camera_count, height, width, 2). If None, allocates a new array.

  • camera_index (int) – Camera index in out_rays at which to start writing. Ignored when out_rays is None.

Returns:

out_rays if provided, otherwise a new array with

shape (camera_count, height, width, 2) and dtype vec3f.

Return type:

camera_rays

compute_camera_rays_usd_pinhole(width, height, cameras, *, time=None, out_rays=None, camera_index=0)#

Compute camera-space ray directions for USD pinhole cameras.

Reads standard UsdGeom.Camera perspective attributes and forwards them to compute_camera_rays_pinhole(). The returned camera_rays array has no world axis, so rays are shared by every world in a render call. Pair these rays with per-world camera transforms only when the cameras at each camera index have matching intrinsics across all worlds.

Parameters:

  • width (int) – Image width [px].

  • height (int) – Image height [px].

  • cameras (Any | Sequence[Any]) – USD camera prim, UsdGeom.Camera, or a flat sequence of either. Per-world camera grids are not accepted because camera rays cannot vary by world.

  • time (Any | None) – Optional USD time code or numeric frame used for camera attributes.

  • out_rays (wp.array(dtype=wp.vec3f, ndim=4) | None) – Optional output array to write into, shape (out_camera_count, height, width, 2). If None, allocates a new array.

  • camera_index (int) – Camera index in out_rays at which to start writing. Ignored when out_rays is None.

Returns:

out_rays if provided, otherwise a new array with

shape (camera_count, height, width, 2) and dtype vec3f.

Return type:

camera_rays

compute_camera_transforms_usd(cameras, *, time=None, xform=None)#

Compute camera-to-world transforms from USD camera prims.

Transforms are rotated from each USD camera’s stage up-axis into the associated Model up-axis. Pass the same xform used for add_usd() when the USD scene was imported with a placement transform.

The returned transform array may vary by world, but compute_camera_rays_usd_pinhole() and update() use camera_rays without a world axis. When using a 2D per-world camera layout, every camera at the same camera index must therefore share the same intrinsics across worlds.

Parameters:

  • cameras (Any | Sequence[Any] | Sequence[Sequence[Any]]) – One or more USD camera prims or UsdGeom.Camera schemas. A 1D sequence shares the same transforms across all worlds. A 2D sequence indexed cameras[world_index][camera_index] assigns a distinct camera per world; the outer dimension must equal world_count and each row must have the same length. Note: the input is world-major (outer = world) but the returned array is camera-major (outer = camera), i.e., shape (camera_count, world_count).

  • time (Any | None) – Optional USD time code or numeric frame used for authored camera attributes and transforms.

  • xform (Any | None) – Optional scene placement transform to compose with each camera transform. Use the same value passed as ModelBuilder.add_usd(..., xform=xform) so cameras and imported geometry share the same model-space placement.

Returns:

Camera-to-world transforms, shape (camera_count, world_count).

Return type:

wp.array(dtype=wp.transformf, ndim=2)

compute_pinhole_camera_rays(width, height, camera_fovs=None, *, focal_length=None, horizontal_aperture=None, vertical_aperture=None, horizontal_aperture_offset=0.0, vertical_aperture_offset=0.0, out_rays=None, camera_index=0)#

Compute camera-space ray directions for pinhole cameras.

Deprecated since version 1.4: Use compute_camera_rays_pinhole() instead.

Physical camera parameters accept any consistent length unit, such as USD-style millimeters. The unit must match across focal_length, apertures, and aperture offsets.

Leave focal_length, horizontal_aperture, and vertical_aperture as None to use camera_fovs. Supplying any of those aperture parameters selects aperture mode, requires all three, and ignores camera_fovs.

Parameters:

  • width (int) – Image width [px].

  • height (int) – Image height [px].

  • camera_fovs (float | list[float] | ndarray | wp.array(dtype=wp.float32, ndim=1) | None) – Vertical FOV angles [rad], scalar or shape (camera_count,). If None, aperture mode must be used; if no aperture parameters are provided, raises ValueError.

  • focal_length (float | list[float] | ndarray | wp.array(dtype=wp.float32, ndim=1) | None) – Focal length [mm or any consistent unit]. If None and the other aperture parameters are also None, uses camera_fovs.

  • horizontal_aperture (float | list[float] | ndarray | wp.array(dtype=wp.float32, ndim=1) | None) – Horizontal aperture [mm or any consistent unit]. If None and the other aperture parameters are also None, uses camera_fovs.

  • vertical_aperture (float | list[float] | ndarray | wp.array(dtype=wp.float32, ndim=1) | None) – Vertical aperture [mm or any consistent unit]. If None and the other aperture parameters are also None, uses camera_fovs.

  • horizontal_aperture_offset (float | list[float] | ndarray | wp.array(dtype=wp.float32, ndim=1)) – Horizontal aperture offset [mm or any consistent unit]. Defaults to 0.0 for a centered aperture.

  • vertical_aperture_offset (float | list[float] | ndarray | wp.array(dtype=wp.float32, ndim=1)) – Vertical aperture offset [mm or any consistent unit]. Defaults to 0.0 for a centered aperture.

  • out_rays (wp.array(dtype=wp.vec3f, ndim=4) | None) – Optional output array to write into, shape (out_camera_count, height, width, 2). If None, allocates a new array.

  • camera_index (int) – Camera index in out_rays at which to start writing. Ignored when out_rays is None.

Returns:

out_rays if provided, otherwise a new array with

shape (camera_count, height, width, 2) and dtype vec3f.

Return type:

camera_rays

convert_ray_depth_to_forward_depth(depth_image, camera_transforms, camera_rays, out_depth=None)#

Convert ray-distance depth to forward (planar) depth.

Projects each pixel’s hit distance along its ray onto the camera’s forward axis, producing depth measured perpendicular to the image plane. The forward axis is derived from each camera transform by transforming camera-space (0, 0, -1) into world space.

Parameters:

  • depth_image (wp.array(dtype=wp.float32, ndim=4)) – Ray-distance depth [m] from update(), shape (world_count, camera_count, height, width).

  • camera_transforms (wp.array(dtype=wp.transformf, ndim=2)) – World-space camera transforms, shape (camera_count, world_count).

  • camera_rays (wp.array(dtype=wp.vec3f, ndim=4)) – Camera-space rays from compute_camera_rays_pinhole() or the fisheye camera ray helpers, shape (camera_count, height, width, 2).

  • out_depth (wp.array(dtype=wp.float32, ndim=4) | None) – Output forward-depth array [m] with the same shape as depth_image. If None, allocates a new one.

Returns:

Forward (planar) depth array, same shape as depth_image [m].

Return type:

wp.array(dtype=wp.float32, ndim=4)

create_albedo_image_output(width, height, camera_count=1)#

Create an albedo output array for update().

Parameters:

  • width (int) – Image width [px].

  • height (int) – Image height [px].

  • camera_count (int) – Number of cameras.

Returns:

Array of shape (world_count, camera_count, height, width), dtype uint32.

Return type:

wp.array(dtype=wp.uint32, ndim=4)

create_color_image_output(width, height, camera_count=1)#

Create a color output array for update().

Parameters:

  • width (int) – Image width [px].

  • height (int) – Image height [px].

  • camera_count (int) – Number of cameras.

Returns:

Array of shape (world_count, camera_count, height, width), dtype uint32.

Return type:

wp.array(dtype=wp.uint32, ndim=4)

create_default_light(enable_shadows=True, direction=None)#

Create a default directional light oriented at (-1, 1, -1).

Parameters:

  • enable_shadows (bool) – Enable shadow casting for this light.

  • direction (vec3f | None) – Normalized light direction. If None, defaults to (normalized (-1, 1, -1)).

create_depth_image_output(width, height, camera_count=1)#

Create a depth output array for update().

Parameters:

  • width (int) – Image width [px].

  • height (int) – Image height [px].

  • camera_count (int) – Number of cameras.

Returns:

Array of shape (world_count, camera_count, height, width), dtype float32.

Return type:

wp.array(dtype=wp.float32, ndim=4)

create_hdr_color_image_output(width, height, camera_count=1)#

Create a linear HDR color output array for update().

Parameters:

  • width (int) – Image width [px].

  • height (int) – Image height [px].

  • camera_count (int) – Number of cameras.

Returns:

Array of shape (world_count, camera_count, height, width), dtype vec3f.

Return type:

wp.array(dtype=wp.vec3f, ndim=4)

create_normal_image_output(width, height, camera_count=1)#

Create a normal output array for update().

Parameters:

  • width (int) – Image width [px].

  • height (int) – Image height [px].

  • camera_count (int) – Number of cameras.

Returns:

Array of shape (world_count, camera_count, height, width), dtype vec3f.

Return type:

wp.array(dtype=wp.vec3f, ndim=4)

create_shape_index_image_output(width, height, camera_count=1)#

Create a shape-index output array for update().

Parameters:

  • width (int) – Image width [px].

  • height (int) – Image height [px].

  • camera_count (int) – Number of cameras.

Returns:

Array of shape (world_count, camera_count, height, width), dtype uint32.

Return type:

wp.array(dtype=wp.uint32, ndim=4)

flatten_color_image_to_rgba(image, out_buffer=None, worlds_per_row=None)#

Flatten rendered color image to a tiled RGBA buffer.

Arranges (world_count * camera_count) tiles in a grid. Each tile shows one camera’s view of one world. Useful for writing a single pre-tiled image to disk; use to_rgba_from_color() with log_image() for in-viewer display.

Parameters:

  • image (wp.array(dtype=wp.uint32, ndim=4)) – Color output from update(), shape (world_count, camera_count, height, width).

  • out_buffer (wp.array(dtype=wp.uint8, ndim=3) | None) – Pre-allocated RGBA buffer. If None, allocates a new one.

  • worlds_per_row (int | None) – Tiles per row in the grid. If None, picks a square-ish layout.

flatten_depth_image_to_rgba(image, out_buffer=None, worlds_per_row=None, depth_range=None)#

Flatten rendered depth image to a tiled RGBA buffer.

Encodes depth as grayscale: inverts values (closer = brighter) and normalizes to the [50, 255] range. Background pixels (no hit) remain black. Useful for writing a single pre-tiled image to disk; use to_rgba_from_depth() with log_image() for in-viewer display.

Parameters:

  • image (wp.array(dtype=wp.float32, ndim=4)) – Depth output from update(), shape (world_count, camera_count, height, width).

  • out_buffer (wp.array(dtype=wp.uint8, ndim=3) | None) – Pre-allocated RGBA buffer. If None, allocates a new one.

  • worlds_per_row (int | None) – Tiles per row in the grid. If None, picks a square-ish layout.

  • depth_range (wp.array(dtype=wp.float32, ndim=1) | None) – Depth range to normalize to, shape (2,) [near, far]. If None, computes from image.

flatten_normal_image_to_rgba(image, out_buffer=None, worlds_per_row=None)#

Flatten rendered normal image to a tiled RGBA buffer.

Arranges (world_count * camera_count) tiles in a grid. Each tile shows one camera’s view of one world. Useful for writing a single pre-tiled image to disk; use to_rgba_from_normal() with log_image() for in-viewer display.

Parameters:

  • image (wp.array(dtype=wp.vec3f, ndim=4)) – Normal output from update(), shape (world_count, camera_count, height, width).

  • out_buffer (wp.array(dtype=wp.uint8, ndim=3) | None) – Pre-allocated RGBA buffer. If None, allocates a new one.

  • worlds_per_row (int | None) – Tiles per row in the grid. If None, picks a square-ish layout.

to_rgba_from_color(image)#

Reinterpret packed uint32 RGBA color sensor output as uint8 RGBA.

Returns a zero-copy view: each uint32 (R | G<<8 | B<<16 | A<<24) aliases 4 contiguous uint8 channels and the (world_count, camera_count) axes are flattened. The returned array shares memory with image; do not write into it.

The returned array plugs directly into log_image(). World is the slower-changing axis: tile i has world = i // camera_count and camera = i % camera_count.

Parameters:

image (wp.array(dtype=wp.uint32, ndim=4)) – Color sensor output, shape (world_count, camera_count, H, W), dtype uint32 (packed RGBA: R | G<<8 | B<<16 | A<<24). Must be contiguous; arrays returned by update() always satisfy this.

Returns:

Array of shape (world_count * camera_count, H, W, 4), dtype uint8, aliasing image.

Return type:

wp.array(dtype=wp.uint8, ndim=4)

to_rgba_from_depth(image, depth_range=None, out_buffer=None)#

Convert float32 depth sensor output to uint8 grayscale RGBA.

Closer pixels render brighter; miss pixels (depth <= 0; matches the default ClearData.clear_depth = 0.0 sentinel) render black. Alpha = 255.

Parameters:

  • image (wp.array(dtype=wp.float32, ndim=4)) – Depth output, shape (world_count, camera_count, H, W), dtype float32. Non-positive values denote ray misses.

  • depth_range (wp.array(dtype=wp.float32, ndim=1) | tuple[float, float] | None) – Optional (near, far) [m] for normalization. Accepts a 2-element wp.array[wp.float32] or a Python (near, far) tuple. If None, the per-frame range is computed on device by find_depth_range() (matches flatten_depth_image_to_rgba()).

  • out_buffer (wp.array(dtype=wp.uint8, ndim=4) | None) – Optional pre-allocated output of shape (world_count * camera_count, H, W, 4), dtype uint8.

Returns:

Array of shape (world_count * camera_count, H, W, 4), dtype uint8. Suitable for log_image().

Return type:

wp.array(dtype=wp.uint8, ndim=4)

to_rgba_from_normal(image, out_buffer=None)#

Convert vec3 normal sensor output to uint8 RGBA.

Parameters:

  • image (wp.array(dtype=wp.vec3f, ndim=4)) – Normal output, shape (world_count, camera_count, H, W), dtype vec3f.

  • out_buffer (wp.array(dtype=wp.uint8, ndim=4) | None) – Optional pre-allocated output of shape (world_count * camera_count, H, W, 4), dtype uint8.

Returns:

Array of shape (world_count * camera_count, H, W, 4), dtype uint8. Suitable for log_image().

Return type:

wp.array(dtype=wp.uint8, ndim=4)

to_rgba_from_shape_index(image, colors=None, out_buffer=None)#

Convert uint32 shape-index sensor output to uint8 RGBA.

Parameters:

  • image (wp.array(dtype=wp.uint32, ndim=4)) – Shape-index output, shape (world_count, camera_count, H, W), dtype uint32.

  • colors (wp.array(dtype=wp.uint8, ndim=2) | None) – Optional RGB palette of shape (num_entries, 3), dtype uint8. If provided, each pixel is colored by looking up its shape index in this palette (indices past the palette length render black). If None, a deterministic hash palette is used (good for debugging which shape hit which pixel without a predefined class map).

  • out_buffer (wp.array(dtype=wp.uint8, ndim=4) | None) – Optional pre-allocated output of shape (world_count * camera_count, H, W, 4), dtype uint8.

Returns:

Array of shape (world_count * camera_count, H, W, 4), dtype uint8. Suitable for log_image().

Return type:

wp.array(dtype=wp.uint8, ndim=4)

__init__(model, *, default_render_config=None, config=_DEPRECATED_CONFIG_UNSET, load_textures=True)#

Initialize the tiled camera sensor from a simulation model.

Builds the internal RenderContext, loads shape geometry (and optionally textures) from model, and exposes utils for creating output buffers, computing rays, and assigning materials.

Parameters:

  • model (Model) – Simulation model whose shapes will be rendered.

  • default_render_config (RenderConfig | None) – Rendering configuration. Pass a RenderConfig to control raytrace settings directly, or None to use defaults. Use RenderConfig.output_color_space to control whether packed color and albedo outputs are display-encoded or left linear.

  • config (RenderConfig | None) – Deprecated as of Newton 1.4; use default_render_config instead.

  • load_textures (bool) – Load texture data from the model. Set to False to skip texture loading when textures are not needed.

sync_transforms(state)#

Synchronize triangle-mesh points from the simulation state.

update() calls this automatically when state is not None.

Shape and particle BVHs on model are built for the initial state by finalize(). Before later frames that change geometry, refit them via bvh_refit_shapes() and bvh_refit_particles() prior to calling update().

Parameters:

state (newton.State) – The current simulation state containing particle positions.

update(state, camera_transforms=None, camera_rays=None, *, color_image=None, hdr_color_image=None, depth_image=None, shape_index_image=None, normal_image=None, albedo_image=None, clear_data=None, render_config=None, kernel_block_dim=64)#

Render output images for all worlds and cameras.

Each output array has shape (world_count, camera_count, height, width) where element [world_id, camera_id, y, x] corresponds to the ray in camera_rays[camera_id, y, x]. Each output channel is optional – pass None to skip that channel’s rendering entirely.

Shape and particle BVHs on model are built for the initial state by finalize(). Before later frames that change geometry, refit them for state via bvh_refit_shapes() and bvh_refit_particles() before calling this method.

Parameters:

  • state (newton.State) – Simulation state with body and particle transforms.

  • camera_transforms (wp.array(dtype=wp.transformf, ndim=2) | None) – Camera-to-world transforms, shape (camera_count, world_count).

  • camera_rays (wp.array(dtype=wp.vec3f, ndim=4) | None) – Camera-space rays from SensorTiledCamera.utils ray helpers, shape (camera_count, height, width, 2).

  • color_image (wp.array(dtype=wp.uint32, ndim=4) | None) – Output for packed RGBA color. The bytes are display/sRGB by default, or linear when self.default_render_config.output_color_space is newton.utils.ColorSpace.LINEAR. None to skip.

  • depth_image (wp.array(dtype=wp.float32, ndim=4) | None) – Output for ray-hit distance [m]. None to skip.

  • shape_index_image (wp.array(dtype=wp.uint32, ndim=4) | None) – Output for per-pixel shape id. None to skip.

  • normal_image (wp.array(dtype=wp.vec3f, ndim=4) | None) – Output for surface normals. None to skip.

  • albedo_image (wp.array(dtype=wp.uint32, ndim=4) | None) – Output for packed unshaded surface color, using the same output color space as color_image. None to skip.

  • clear_data (ClearData | None) – Values to clear output buffers with. Packed color and albedo clear values are specified as display/sRGB RGBA and converted to linear when linear output is requested. See DEFAULT_CLEAR_DATA, GRAY_CLEAR_DATA.

  • hdr_color_image (wp.array(dtype=wp.vec3f, ndim=4) | None) – Output for linear HDR color. None to skip.

  • render_config (RenderConfig | None) – Render settings for this update. If None, uses default_render_config.

  • kernel_block_dim (int) – Thread block dimension forwarded to wp.launch for the render megakernel.

DEFAULT_CLEAR_DATA = ClearData(clear_color=0, clear_depth=0.0, clear_shape_index=4294967295, clear_normal=(0.0, 0.0, 0.0), clear_albedo=0)#
GRAY_CLEAR_DATA = ClearData(clear_color=4284900966, clear_depth=0.0, clear_shape_index=4294967295, clear_normal=(0.0, 0.0, 0.0), clear_albedo=4278190080)#
property default_clear_data: ClearData#

The default clear data to use if none is passed to update().

Returns:

The default ClearData instance.

property default_render_config: RenderConfig#

The default render config to use if none is passed to update().

Returns:

The default RenderConfig instance.

property render_config: RenderConfig#

Deprecated alias for default_render_config.

Deprecated since version 1.4: Use default_render_config instead.

Returns:

The live default RenderConfig instance.

property utils: Utils#

Utility helpers for creating output buffers, computing rays, and assigning materials/lights.