vllm.model_executor.models.minicpmv4_7

Inference-only MiniCPM-V 4.7 model.

Same vision/LLM stack as MiniCPM-V 4.6, plus canvas 3D M-RoPE.

Functions:

• canvas_rope_delta –

  Offset applied to the 1-D positions of tokens decoded after the prompt.

_CanvasLayout

Bases: NamedTuple

Grid of one canvas measured in LLM tokens.

A canvas is a thumbnail plus the slices cut from it, tiled row-major.

Attributes:

• cols (int) –

  Slices sharing a canvas row. Zero when the canvas has no slices.

• slice_w (int) –

  Per-slice grid. Zero when the canvas has no slices.

• thumb_w (int) –

  Thumbnail grid.

• width (int) –

  Assembled canvas.

Source code in vllm/model_executor/models/minicpmv4_7.py

class _CanvasLayout(NamedTuple):
    """Grid of one canvas measured in LLM tokens.

    A canvas is a thumbnail plus the slices cut from it, tiled row-major.
    """

    thumb_h: int
    thumb_w: int
    """Thumbnail grid."""
    slice_h: int
    slice_w: int
    """Per-slice grid. Zero when the canvas has no slices."""
    cols: int
    """Slices sharing a canvas row. Zero when the canvas has no slices."""
    height: int
    width: int
    """Assembled canvas."""

cols instance-attribute

Slices sharing a canvas row. Zero when the canvas has no slices.

slice_w instance-attribute

Per-slice grid. Zero when the canvas has no slices.

thumb_w instance-attribute

Thumbnail grid.

width instance-attribute

Assembled canvas.

_assign_canvas_span(pos3d, *, thumbnail, span_end, slices, base, target_sizes)

Place one canvas at base, writing its 3-D positions into pos3d.

Shared by the video-frame and single-image paths; they differ only in how the per-canvas base advances, not in the geometry placed here.

Mutates pos3d in place, as the packed canvas computation has always done. Returns the distance to the next canvas base, so each caller keeps only its own base advancement.

Source code in vllm/model_executor/models/minicpmv4_7.py

def _assign_canvas_span(
    pos3d: torch.Tensor,
    *,
    thumbnail: tuple[int, int, int],
    span_end: int,
    slices: list[tuple[int, int, int]],
    base: int,
    target_sizes: torch.Tensor,
) -> int:
    """Place one canvas at ``base``, writing its 3-D positions into ``pos3d``.

    Shared by the video-frame and single-image paths; they differ only in how
    the per-canvas base advances, not in the geometry placed here.

    Mutates ``pos3d`` in place, as the packed canvas computation has always
    done. Returns the distance to the next canvas base, so each caller keeps
    only its own base advancement.
    """
    t_bs, t_be, _ = thumbnail
    device = pos3d.device
    span_start = t_bs - 1  # the <im_start> token
    n_thumb = t_be - t_bs
    layout = _canvas_layout(thumbnail, slices, target_sizes)
    cols, canvas_H, canvas_W = layout.cols, layout.height, layout.width

    # base coat
    pos3d[0, 0, span_start:span_end] = base
    pos3d[1, 0, span_start:span_end] = base
    pos3d[2, 0, span_start:span_end] = base

    # <im_start> -> halo
    halo_lo = max(base - 1, 0)
    pos3d[1, 0, span_start] = halo_lo
    pos3d[2, 0, span_start] = halo_lo

    # <im_end> -> halo (right after thumbnail)
    if t_be < span_end:
        pos3d[1, 0, t_be] = base + canvas_H
        pos3d[2, 0, t_be] = base + canvas_W

    # thumbnail visual tokens
    llm_th, llm_tw = layout.thumb_h, layout.thumb_w
    if llm_th > 0 and llm_tw > 0 and llm_th * llm_tw == n_thumb:
        if slices and canvas_H > 0 and canvas_W > 0:
            h_c = torch.linspace(0, canvas_H - 1, llm_th, device=device).round().long()
            w_c = torch.linspace(0, canvas_W - 1, llm_tw, device=device).round().long()
        else:
            h_c = torch.arange(llm_th, device=device)
            w_c = torch.arange(llm_tw, device=device)
        h_idx = h_c.view(-1, 1).expand(-1, llm_tw).reshape(-1)
        w_idx = w_c.view(1, -1).expand(llm_th, -1).reshape(-1)
        pos3d[0, 0, t_bs:t_be] = base
        pos3d[1, 0, t_bs:t_be] = h_idx + base
        pos3d[2, 0, t_bs:t_be] = w_idx + base
    else:
        fallback = torch.arange(n_thumb, device=device, dtype=torch.long) + base
        for d in range(3):
            pos3d[d, 0, t_bs:t_be] = fallback

    # slice visual tokens + slice specials
    for k, (s_bs, s_be, s_gi) in enumerate(slices):
        n_s = s_be - s_bs
        sh_k, sw_k = compute_llm_grid(n_s, *_target_hw(target_sizes, s_gi))
        h_off = (k // cols) * layout.slice_h
        w_off = (k % cols) * layout.slice_w

        slice_start_pos = s_bs - 1
        if slice_start_pos >= span_start:
            pos3d[1, 0, slice_start_pos] = base + h_off
            pos3d[2, 0, slice_start_pos] = base + w_off

        slice_end_pos = s_be
        if sh_k > 0 and sw_k > 0:
            end_h = h_off + sh_k - 1
            end_w = w_off + sw_k - 1
        else:
            end_h = h_off
            end_w = w_off
        if slice_end_pos < span_end:
            pos3d[1, 0, slice_end_pos] = base + end_h
            pos3d[2, 0, slice_end_pos] = base + end_w

        if sh_k > 0 and sw_k > 0 and sh_k * sw_k == n_s:
            h_idx = (
                torch.arange(sh_k, device=device)
                .view(-1, 1)
                .expand(-1, sw_k)
                .reshape(-1)
            )
            w_idx = (
                torch.arange(sw_k, device=device)
                .view(1, -1)
                .expand(sh_k, -1)
                .reshape(-1)
            )
            pos3d[0, 0, s_bs:s_be] = base
            pos3d[1, 0, s_bs:s_be] = h_idx + h_off + base
            pos3d[2, 0, s_bs:s_be] = w_idx + w_off + base
        else:
            fallback = torch.arange(n_s, device=device, dtype=torch.long) + base
            for d in range(3):
                pos3d[d, 0, s_bs:s_be] = fallback

    # \n between slice rows -> W = right edge + 1.
    # H stays within the ended row (does NOT cross to next row): the \n sits at
    # the right-outside of the row that just ended, at H = that row's last row
    # and W = one column past the rightmost slice.
    for k in range(len(slices) - 1):
        gap_s = slices[k][1]
        gap_e = slices[k + 1][0]
        if gap_e - gap_s <= 2:
            continue
        boundary_h = ((k // cols) + 1) * layout.slice_h - 1
        right_edge_w = cols * layout.slice_w
        for nl_pos in range(gap_s + 1, gap_e - 1):
            pos3d[1, 0, nl_pos] = base + boundary_h
            pos3d[2, 0, nl_pos] = base + right_edge_w

    return max(canvas_H, canvas_W) + 1

_canvas_layout(thumbnail, slices, target_sizes)

Measure the canvas for one (thumbnail, slices) group.

thumbnail and each slice are (start, end, target_index) token spans.

Source code in vllm/model_executor/models/minicpmv4_7.py

def _canvas_layout(
    thumbnail: tuple[int, int, int],
    slices: list[tuple[int, int, int]],
    target_sizes: torch.Tensor,
) -> _CanvasLayout:
    """Measure the canvas for one ``(thumbnail, slices)`` group.

    ``thumbnail`` and each slice are ``(start, end, target_index)`` token spans.
    """
    t_bs, t_be, t_gi = thumbnail
    thumb_h, thumb_w = compute_llm_grid(t_be - t_bs, *_target_hw(target_sizes, t_gi))
    if not slices:
        return _CanvasLayout(
            thumb_h, thumb_w, 0, 0, 0, max(thumb_h, 1), max(thumb_w, 1)
        )

    cols = len(slices)
    for k in range(len(slices) - 1):
        if slices[k + 1][0] - slices[k][1] > 2:
            cols = k + 1
            break
    rows = len(slices) // cols if cols > 0 else 1
    if rows * cols != len(slices):
        rows, cols = 1, len(slices)

    s0_bs, s0_be, s0_gi = slices[0]
    slice_h, slice_w = compute_llm_grid(s0_be - s0_bs, *_target_hw(target_sizes, s0_gi))
    return _CanvasLayout(
        thumb_h, thumb_w, slice_h, slice_w, cols, rows * slice_h, cols * slice_w
    )

_sequential_positions(seq_len)

Positions for the fallback where canvas M-RoPE cannot be computed.

Source code in vllm/model_executor/models/minicpmv4_7.py

def _sequential_positions(seq_len: int) -> torch.Tensor:
    """Positions for the fallback where canvas M-RoPE cannot be computed."""
    return torch.arange(seq_len).unsqueeze(0).expand(3, -1)

_target_hw(target_sizes, index)

Patch-grid height/width recorded for one image or slice.

Source code in vllm/model_executor/models/minicpmv4_7.py

def _target_hw(target_sizes: torch.Tensor, index: int) -> tuple[int, int]:
    """Patch-grid height/width recorded for one image or slice."""
    return int(target_sizes[index, 0]), int(target_sizes[index, 1])

canvas_rope_delta(position_ids, seq_len)

Offset applied to the 1-D positions of tokens decoded after the prompt.

The first decoded token sits one past the last prefill position.

Source code in vllm/model_executor/models/minicpmv4_7.py

def canvas_rope_delta(position_ids: torch.Tensor, seq_len: int) -> int:
    """Offset applied to the 1-D positions of tokens decoded after the prompt.

    The first decoded token sits one past the last prefill position.
    """
    return int(position_ids.amax().item()) + 1 - seq_len