Holistic Replay - Ghosting/Corruption in Resized Output

joshbarclay

Hi,

I'm working with holistic replay in DepthAI v3.1.0 and noticed an issue with resized camera outputs during replay.

Problem: When requesting a 300x300 BGR888p output from CAM_A during holistic replay, the frames show severe ghosting and color corruption artifacts. The same pipeline works perfectly in live mode.

Replay mode:

Live mode:

Reproduction

Recording Script: (1920x1080 at 5fps)

import cv2, depthai as dai
from pathlib import Path

OUT = "recordings"
Path(OUT).mkdir(parents=True, exist_ok=True)

with dai.Pipeline(True) as pipeline:
    # --- Sources you care about for your pipeline ---
    camA = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_A, sensorResolution=(1920,1080), sensorFps=5)
    outA = camA.requestOutput((1920,1080), fps=5)
    camB = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_B, sensorFps=5)
    outB = camB.requestOutput((640,400), fps=5)  
    camC = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_C, sensorFps=5)
    outC = camC.requestOutput((640,400), fps=5)

    # --- Holistic Record config (H.264, auto bitrate) ---
    cfg = dai.RecordConfig()
    cfg.outputDir = OUT
    cfg.videoEncoding.enabled = True
    cfg.videoEncoding.bitrate = 0
    cfg.videoEncoding.profile = dai.VideoEncoderProperties.Profile.H264_MAIN
    pipeline.enableHolisticRecord(cfg)

    qA = outA.createOutputQueue()
    qB = outB.createOutputQueue()
    qC = outC.createOutputQueue()

    pipeline.start()
    print("Recording... press 'q' to stop")
    while pipeline.isRunning():
        _ = qA.get(); _ = qB.get(); _ = qC.get()
        if cv2.waitKey(1) == ord('q'):
            break

Live/Replay Script:

"""
1. Live mode (works): python script.py --live
2. Replay mode (broken): python script.py --replay recordings/your_recording.tar
"""

import argparse
import depthai as dai
import cv2
import numpy as np


def convert_planar_to_interleaved(frame_msg):
    """Convert BGR888p (planar) to BGR888 (interleaved) for OpenCV display."""
    if frame_msg.getType() == dai.ImgFrame.Type.BGR888p:
        data = frame_msg.getData()
        w, h = frame_msg.getWidth(), frame_msg.getHeight()
        planar = np.frombuffer(data, dtype=np.uint8).reshape((3, h, w))
        return np.transpose(planar, (1, 2, 0))  # CHW → HWC
    else:
        return frame_msg.getCvFrame()


def main():
    ap = argparse.ArgumentParser(description="MRE for 300x300 replay corruption")
    ap.add_argument("--replay", type=str, help="Path to holistic replay .tar file")
    ap.add_argument("--live", action="store_true", help="Use live camera instead of replay")
    args = ap.parse_args()
    
    if not args.replay and not args.live:
        print("Error: Must specify either --replay <path> or --live")
        return
    
    if args.replay and args.live:
        print("Error: Cannot use both --replay and --live")
        return
    
    # Determine mode
    is_replay = args.replay is not None
    mode_name = "REPLAY" if is_replay else "LIVE"
    
    print(f"\n{'='*60}")
    print(f"Running in {mode_name} mode")
    if is_replay:
        print(f"Replay file: {args.replay}")
    print(f"{'='*60}\n")
    
    # Create pipeline based on mode
    if is_replay:
        # Replay mode: use context manager pattern
        p = dai.Pipeline(True)
    else:
        # Live mode: create device first, then pipeline
        device = dai.Device()
        p = dai.Pipeline(device)
    
    # Build identical pipeline for both modes
    cam = p.create(dai.node.Camera).build(
        boardSocket=dai.CameraBoardSocket.CAM_A,
        sensorResolution=(1920, 1080),
        sensorFps=5,
    )
    
    # Full resolution output (baseline - should work in both modes)
    out_full = cam.requestOutput((1920, 1080), fps=5)
    
    # 300x300 BGR888p output (BUGGY in replay mode)
    out_prev = cam.requestOutput(
        size=(300, 300),
        type=dai.ImgFrame.Type.BGR888p,
        resizeMode=dai.ImgResizeMode.STRETCH,
        fps=5,
        enableUndistortion=True,
    )
    
    # Enable replay if needed
    if is_replay:
        p.enableHolisticReplay(args.replay)
    
    # Create queues
    q_full = out_full.createOutputQueue()
    q_prev = out_prev.createOutputQueue()
    
    # Start pipeline
    p.start()
    
    print(f"{mode_name} started. Press 'q' to quit.\n")
    
    try:
        while p.isRunning():
            f_full = q_full.get()
            f_prev = q_prev.get()
            
            # Display full resolution (downscaled for viewing)
            cv2.imshow(f"{mode_name} - Full 1920x1080", 
                      cv2.resize(f_full.getCvFrame(), (640, 360)))
            
            # Display 300x300 with proper planar conversion
            frame_prev = convert_planar_to_interleaved(f_prev)
            cv2.imshow(f"{mode_name} - 300x300 BGR888p", 
                      frame_prev)
            
            if cv2.waitKey(1) == ord('q'):
                break
                
    finally:
        cv2.destroyAllWindows()


if __name__ == "__main__":
    main()

Is this a known issue with non-native resolution outputs during replay? Any workarounds available?

jakaskerl

joshbarclay
Can you recheck the conversion - convert_planar_to_interleaved? Also, does getCvFrame not work on the preview frame? It should do the proper conversion automatically.

Also, it's easier to find the cause if you manually specify the datatype you request from the camera.

Thanks
Jaka

joshbarclay

Sorry, I could have made this MRE script a little leaner. I've simplified the script to use getCvFrame() directly without the custom convert_planar_to_interleaved() function. I am still observing the same issue in out_prev regardless of the requestOutput parameters (i.e. resizeMode, type, enableUndistortion).

# Build identical pipeline for both modes
cam = p.create(dai.node.Camera).build(
    boardSocket=dai.CameraBoardSocket.CAM_A,
    sensorResolution=(1920, 1080),
    sensorFps=5,
)

# Full resolution output
out_full = cam.requestOutput((1920, 1080), fps=5)

# 300x300 output
out_prev = cam.requestOutput(
    size=(300, 300),
    resizeMode=dai.ImgResizeMode.STRETCH,
    # removed explicit type: type=dai.ImgFrame.Type.BGR888p,
    fps=5,
    enableUndistortion=True,
)

# Enable replay if needed
if is_replay:
    p.enableHolisticReplay(args.replay)

# Create queues
q_full = out_full.createOutputQueue()
q_prev = out_prev.createOutputQueue()

# Start pipeline
p.start()

try:
    while p.isRunning():
        f_full = q_full.get()
        f_prev = q_prev.get()
        
        # Display full resolution (downscaled for viewing)
        cv2.imshow(f"{mode_name} - Full 1920x1080", 
                    cv2.resize(f_full.getCvFrame(), (640, 360)))
        
        # Display 300x300 with proper planar conversion
        cv2.imshow(f"{mode_name} - 300x300", 
                    f_prev.getCvFrame())

Also, it's easier to find the cause if you manually specify the datatype you request from the camera.

Are you talking about specifying the type in the record.py script or the requestOutput for the replay_vs_live.py script?

Thanks,
Josh