SmolVLA: A vision-language-action model for affordable and efficient robotics

# Size and cost figures, as stated in the paper (Sections 3.1, 4.3, 4.4 and 5.1).
total, expert = 450e6, 100e6
print(f"action expert share of the main model: {expert / total:.0%}; the rest ({(total - expert) / 1e6:.0f} M) is the cut-down vision-language model")
print(f"the paper keeps the first N = 16 layers, which it calls N = L/2, so the language model has L = {2 * 16} layers")

print()
print("how much larger are the comparison models?")
for name, size in (("pi0 as stated in Section 4.4 and Table 2 (3.3B)", 3.3e9), ("pi0 as labelled in Tables 2 and 3 for Meta-World and real world (3.5B)", 3.5e9), ("OpenVLA (7B)", 7e9)):
    print(f"  {name:72s} {size / total:5.1f} times")
print('the abstract says "10x larger"; the text says about 7x for pi0')

print()
gpu_hours, gpus = 30_000, 4
print(f"project total of {gpu_hours} GPU hours; on {gpus} GPUs that would be {gpu_hours / gpus:.0f} hours = {gpu_hours / gpus / 24:.0f} days")
print("the paper gives no GPU hours for the main pretraining run alone, and no inference latency, memory or CPU figures")
print("it states 'about 40% faster to train and 6 times less memory' than pi0, without a table")

Output:

action expert share of the main model: 22%; the rest (350 M) is the cut-down vision-language model
the paper keeps the first N = 16 layers, which it calls N = L/2, so the language model has L = 32 layers

how much larger are the comparison models?
  pi0 as stated in Section 4.4 and Table 2 (3.3B)                            7.3 times
  pi0 as labelled in Tables 2 and 3 for Meta-World and real world (3.5B)     7.8 times
  OpenVLA (7B)                                                              15.6 times
the abstract says "10x larger"; the text says about 7x for pi0

project total of 30000 GPU hours; on 4 GPUs that would be 7500 hours = 312 days
the paper gives no GPU hours for the main pretraining run alone, and no inference latency, memory or CPU figures
it states 'about 40% faster to train and 6 times less memory' than pi0, without a table

import numpy as np

# Visual token reduction (Section 3.1): "pixel shuffle" limits the visual tokens to 64 per frame.
# The paper does not give the patch size or the shuffle factor. We assume 16 x 16 patches and a factor of 4,
# which is one way to get 64 tokens from a 512 x 512 image.
image, patch, factor = 512, 16, 4
grid = image // patch                       # patches along each side
tokens = grid * grid
print(f"{image} x {image} image, {patch} x {patch} patches: {grid} x {grid} = {tokens} tokens")

dim = 8                                     # a tiny feature size, to keep the example small
x = np.random.default_rng(0).normal(size=(grid, grid, dim))

def pixel_shuffle(x, f):
    # move each f x f block of neighbouring tokens into the feature dimension: fewer tokens, wider features
    g, _, d = x.shape
    return x.reshape(g // f, f, g // f, f, d).transpose(0, 2, 1, 3, 4).reshape(g // f, g // f, f * f * d)

y = pixel_shuffle(x, factor)
print(f"after pixel shuffle with factor {factor}: {y.shape[0]} x {y.shape[1]} = {y.shape[0] * y.shape[1]} tokens, each with {y.shape[2]} features (was {dim})")
print("numbers stored before and after:", x.size, y.size, "(nothing is thrown away)")

# the shuffle can be undone exactly
def unshuffle(y, f):
    g, _, d = y.shape
    return y.reshape(g, g, f, f, d // (f * f)).transpose(0, 2, 1, 3, 4).reshape(g * f, g * f, d // (f * f))
print("exactly reversible:", bool(np.array_equal(unshuffle(y, factor), x)))

Output:

512 x 512 image, 16 x 16 patches: 32 x 32 = 1024 tokens
after pixel shuffle with factor 4: 8 x 8 = 64 tokens, each with 128 features (was 8)
numbers stored before and after: 8192 8192 (nothing is thrown away)
exactly reversible: True

import numpy as np

# The action expert's two kinds of attention layer (Section 3.1), in a tiny version with random weights.
rng = np.random.default_rng(0)
d, n_actions, n_vlm = 4, 6, 8
Wq, Wk, Wv = (rng.normal(size=(d, d)) for _ in range(3))
vlm = rng.normal(size=(n_vlm, d))            # features from the vision-language model
actions = rng.normal(size=(n_actions, d))    # noisy action tokens

def attend(queries, keys_values, mask=None):
    q, k, v = queries @ Wq, keys_values @ Wk, keys_values @ Wv
    scores = q @ k.T / np.sqrt(d)
    if mask is not None:
        scores = np.where(mask, scores, -1e9)
    w = np.exp(scores - scores.max(axis=1, keepdims=True))
    return (w / w.sum(axis=1, keepdims=True)) @ v

causal = np.tril(np.ones((n_actions, n_actions), dtype=bool))
print("causal mask (row = action token, column = token it may look at):")
for row in causal.astype(int):
    print("  ", " ".join(map(str, row)))

# Change only the LAST action token and see which outputs move.
changed = actions.copy()
changed[-1] += 5.0
for name, mask in (("causal self-attention", causal), ("bidirectional self-attention", None)):
    before, after = attend(actions, actions, mask), attend(changed, changed, mask)
    moved = np.abs(after - before).max(axis=1) > 1e-9
    print(f"{name:30s} outputs that change when the last token changes: {moved.astype(int).tolist()}")

# Cross-attention: the keys and values come from the VLM, so action tokens do not exchange information with each other.
before, after = attend(actions, vlm), attend(changed, vlm)
moved = np.abs(after - before).max(axis=1) > 1e-9
print(f"{'cross-attention':30s} outputs that change when the last token changes: {moved.astype(int).tolist()}")

Output:

causal mask (row = action token, column = token it may look at):
   1 0 0 0 0 0
   1 1 0 0 0 0
   1 1 1 0 0 0
   1 1 1 1 0 0
   1 1 1 1 1 0
   1 1 1 1 1 1
causal self-attention          outputs that change when the last token changes: [0, 0, 0, 0, 0, 1]
bidirectional self-attention   outputs that change when the last token changes: [1, 1, 1, 1, 1, 1]
cross-attention                outputs that change when the last token changes: [0, 0, 0, 0, 0, 1]

import math

# Section 3.3: an action queue of n actions, a control step of dt seconds, a policy latency l seconds.
n = 50                  # chunk size (Section 4.3)
dt = 1 / 30             # 30 frames per second (Section 3.3)
print(f"one chunk lasts n * dt = {n * dt:.2f} s")

print()
print("latency | smallest g that avoids an empty queue | idle share of a synchronous cycle")
for latency in (0.1, 0.3, 0.5, 1.0, 1.5):
    g_min = (latency / dt) / n                       # the paper's condition: g >= (E[l] / dt) / n
    idle = latency / (n * dt + latency)              # synchronous: run a chunk, then wait for the next
    print(f"{latency:5.1f} s | {g_min:37.2f} | {idle:5.0%}")

g = 0.7
print()
print(f"with g = {g}: the longest latency that can be hidden is g * n * dt = {g * n * dt:.2f} s")
print(f"with g = {g}: a new observation is sent every (1 - g) * n * dt = {(1 - g) * n * dt:.2f} s")
for name, gg in (("g = 0 (synchronous)", 0.0), ("g = 0.7", 0.7), ("g = 1", 1.0)):
    # observations sent per second when the latency is 0.3 s (no similarity filter)
    interval = n * dt + 0.3 if gg == 0 else max((1 - gg) * n * dt, dt)
    print(f"{name:20s} server calls per second: {1 / interval:5.2f}")

Output:

one chunk lasts n * dt = 1.67 s

latency | smallest g that avoids an empty queue | idle share of a synchronous cycle
  0.1 s |                                  0.06 |    6%
  0.3 s |                                  0.18 |   15%
  0.5 s |                                  0.30 |   23%
  1.0 s |                                  0.60 |   37%
  1.5 s |                                  0.90 |   47%

with g = 0.7: the longest latency that can be hidden is g * n * dt = 1.17 s
with g = 0.7: a new observation is sent every (1 - g) * n * dt = 0.50 s
g = 0 (synchronous)  server calls per second:  0.51
g = 0.7              server calls per second:  2.00
g = 1                server calls per second: 30.00

# The pretraining data (Table 1 and Appendix A.1). The counts of dataset names are ours, from the printed list.
episodes = 22_900
frames = 10_600_000
datasets_table = 481
datasets_listed = 532            # distinct names printed in Appendix A.1
fps = 30                         # the real-world frame rate quoted in Section 3.3; community datasets may differ
openvla_episodes = 1_000_000     # "around 1 million trajectories" (Section 5.1)

print(f"frames per episode: {frames / episodes:.0f}  ({frames / episodes / fps:.1f} s at {fps} frames per second)")
print(f"total length: {frames / fps / 3600:.0f} hours at {fps} frames per second")
print(f"OpenVLA episodes / SmolVLA episodes: {openvla_episodes / episodes:.0f}")
print(f"Table 1 caption says about 10M episodes; the table says {episodes} episodes -> factor {10_000_000 / episodes:.0f}")
print(f"datasets in Table 1: {datasets_table}; names printed in Appendix A.1: {datasets_listed}; difference {datasets_listed - datasets_table}")

samples = 200_000 * 256          # steps x global batch size (Section 4.3)
print(f"pretraining samples: {samples / 1e6:.1f} million = {samples / frames:.1f} passes over {frames / 1e6:.1f} million frames")

print()
print("dataset names in Appendix A.1 that contain a keyword (our count, names only):")
for word, count in (("stack", 9), ("sort", 12), ("pick", 66), ("place", 50), ("lego", 20), ("so101", 0)):
    print(f"  {word:6s} {count:3d}")
print("the four real-world tasks are pick-place, stacking, sorting and pick-place with a lego brick on another arm")

Output:

frames per episode: 463  (15.4 s at 30 frames per second)
total length: 98 hours at 30 frames per second
OpenVLA episodes / SmolVLA episodes: 44
Table 1 caption says about 10M episodes; the table says 22900 episodes -> factor 437
datasets in Table 1: 481; names printed in Appendix A.1: 532; difference 51
pretraining samples: 51.2 million = 4.8 passes over 10.6 million frames

dataset names in Appendix A.1 that contain a keyword (our count, names only):
  stack    9
  sort    12
  pick    66
  place   50
  lego    20
  so101    0
the four real-world tasks are pick-place, stacking, sorting and pick-place with a lego brick on another arm

import math

# How much can a LIBERO success rate move by chance? 10 tasks per suite and 10 trials per task (Section 4.1).
trials_suite, trials_average = 100, 400

def se(p, n):
    # standard error of a success rate p out of n independent trials, in percentage points
    return 100 * math.sqrt(p * (1 - p) / n)

print(f"one suite (n = {trials_suite}) at 80%: standard error {se(0.8, trials_suite):.1f} points")
print(f"four-suite average (n = {trials_average}) at 80%: standard error {se(0.8, trials_average):.1f} points; 95% interval about +-{1.96 * se(0.8, trials_average):.1f}")

def gap(a, b, name):
    pa, pb = a / 100, b / 100
    s = math.sqrt((se(pa, trials_average)) ** 2 + (se(pb, trials_average)) ** 2)
    print(f"{name:52s} {a:6.2f} vs {b:6.2f}: gap {a - b:+5.1f}, {abs(a - b) / s:3.1f} standard errors")

print()
print("gaps behind the paper's main statements (independent trials assumed)")
gap(87.3, 86.0, "SmolVLA 0.45B against pi0 3.3B, LIBERO (Table 2)")
gap(85.5, 79.0, "CA+SA against CA (Table 6)")
gap(79.0, 74.5, "CA against SA (Table 6)")
gap(74.5, 67.5, "causal against bidirectional (Table 7)")
gap(80.25, 75.25, "flow matching against regression (Table 10)")
gap(84.0, 80.3, "chunk size 10 against 50 (Table 12)")
gap(80.3, 51.8, "update every step against every 50 steps (Table 13)")

# The same-looking configuration, as it appears in different tables
same = {"Table 6, CA": 79.0, "Table 8, N = 32": 80.3, "Table 9, x1.00": 82.3, "Table 10, flow matching": 80.25,
        "Table 11, prefix + CA": 80.3, "Table 12, chunk 50": 80.3, "Table 13, 1 step": 80.3}
vals = sorted(set(same.values()))
print()
print("rows that look like the same configuration:", vals)
print(f"spread between runs: {max(vals) - min(vals):.1f} points")

Output:

one suite (n = 100) at 80%: standard error 4.0 points
four-suite average (n = 400) at 80%: standard error 2.0 points; 95% interval about +-3.9

gaps behind the paper's main statements (independent trials assumed)
SmolVLA 0.45B against pi0 3.3B, LIBERO (Table 2)      87.30 vs  86.00: gap  +1.3, 0.5 standard errors
CA+SA against CA (Table 6)                            85.50 vs  79.00: gap  +6.5, 2.4 standard errors
CA against SA (Table 6)                               79.00 vs  74.50: gap  +4.5, 1.5 standard errors
causal against bidirectional (Table 7)                74.50 vs  67.50: gap  +7.0, 2.2 standard errors
flow matching against regression (Table 10)           80.25 vs  75.25: gap  +5.0, 1.7 standard errors
chunk size 10 against 50 (Table 12)                   84.00 vs  80.30: gap  +3.7, 1.4 standard errors
update every step against every 50 steps (Table 13)   80.30 vs  51.80: gap +28.5, 8.9 standard errors

rows that look like the same configuration: [79.0, 80.25, 80.3, 82.3]
spread between runs: 3.3 points

# Meta-World (Table 2): success rates per difficulty group, as printed, and the printed average.
rows = {
    "SmolVLA (0.24B)":   ([86.43, 46.36, 35.0, 60.0], 56.95),
    "SmolVLA (0.45B)":   ([82.5, 41.8, 45.0, 60.0], 57.3),
    "SmolVLA (2.25B)":   ([87.14, 51.82, 70.0, 64.0], 68.24),
    "pi0 (3.5B-Paligemma)": ([80.4, 40.9, 36.7, 44.0], 50.5),
    "pi0 (3.5B)":        ([71.8, 48.2, 41.7, 30.0], 47.9),
    "Diffusion Policy":  ([23.1, 10.7, 1.9, 6.1], 10.5),
    "TinyVLA":           ([77.6, 21.5, 11.4, 15.8], 31.6),
}
# 10 trials per task. If the groups had 28, 11, 6 and 5 tasks, their trial counts would be 280, 110, 60 and 50.
tasks = [28, 11, 6, 5]
trials = [10 * t for t in tasks]
print("printed average is the plain mean of the four groups:")
for name, (groups, printed) in rows.items():
    print(f"  {name:22s} mean {sum(groups) / 4:6.2f}  printed {printed}")

print()
print("do the printed percentages equal whole numbers of successes out of 280, 110, 60 and 50?")
for name, (groups, printed) in rows.items():
    ok = [abs(round(g * n / 100) * 100 / n - g) < 0.06 for g, n in zip(groups, trials)]
    print(f"  {name:22s} {ok}")

print()
print("average over the 50 tasks instead of over the four groups (weights 28, 11, 6, 5, as the first five rows imply):")
for name, (groups, printed) in rows.items():
    weighted = sum(g * t for g, t in zip(groups, tasks)) / sum(tasks)
    print(f"  {name:22s} groups {printed:6.2f}   tasks {weighted:6.2f}")

Output:

printed average is the plain mean of the four groups:
  SmolVLA (0.24B)        mean  56.95  printed 56.95
  SmolVLA (0.45B)        mean  57.33  printed 57.3
  SmolVLA (2.25B)        mean  68.24  printed 68.24
  pi0 (3.5B-Paligemma)   mean  50.50  printed 50.5
  pi0 (3.5B)             mean  47.92  printed 47.9
  Diffusion Policy       mean  10.45  printed 10.5
  TinyVLA                mean  31.57  printed 31.6

do the printed percentages equal whole numbers of successes out of 280, 110, 60 and 50?
  SmolVLA (0.24B)        [True, True, True, True]
  SmolVLA (0.45B)        [True, True, True, True]
  SmolVLA (2.25B)        [True, True, True, True]
  pi0 (3.5B-Paligemma)   [True, True, True, True]
  pi0 (3.5B)             [True, True, True, True]
  Diffusion Policy       [False, False, False, False]
  TinyVLA                [False, False, False, False]

average over the 50 tasks instead of over the four groups (weights 28, 11, 6, 5, as the first five rows imply):
  SmolVLA (0.24B)        groups  56.95   tasks  68.80
  SmolVLA (0.45B)        groups  57.30   tasks  66.80
  SmolVLA (2.25B)        groups  68.24   tasks  75.00
  pi0 (3.5B-Paligemma)   groups  50.50   tasks  62.83
  pi0 (3.5B)             groups  47.90   tasks  58.82
  Diffusion Policy       groups  10.50   tasks  16.13
  TinyVLA                groups  31.60   tasks  51.13

import math

# Real-world results as printed (success in percent). Tables 3 and 5 and Figure 5.
t3 = {"ACT (single task)": (70, 50, 25), "pi0 (multi-task)": (100, 40, 45), "SmolVLA (pretrained, multi-task)": (75, 90, 70)}
t5 = {"SmolVLA, single task, no pretraining": (55, 45, 20), "SmolVLA, multi-task, no pretraining": (80, 40, 35), "SmolVLA, multi-task, pretrained": (75, 90, 70)}
tasks = ("Pick-Place", "Stacking", "Sorting")

print("Table 3, averages recomputed, and the best method per task")
for name, row in t3.items():
    print(f"  {name:34s} mean {sum(row) / 3:5.1f}")
for i, task in enumerate(tasks):
    best = max(t3, key=lambda k: t3[k][i])
    print(f"  best on {task:10s}: {best} ({t3[best][i]})")

print()
print("effect of pretraining and multi-task training (Table 5), points per task")
a, b, c = t5.values()
for i, task in enumerate(tasks):
    print(f"  {task:10s} multi-task over single-task {b[i] - a[i]:+4d}   pretraining over none {c[i] - b[i]:+4d}")

print()
print("synchronous against asynchronous inference (Figure 5)")
sync, asyn = (75, 90, 70), (80, 90, 50)
print(f"  success per task: sync {sync}  async {asyn}  differences {[x - y for x, y in zip(asyn, sync)]}; averages {sum(sync) / 3:.1f} and {sum(asyn) / 3:.1f}")

def t_two_sided_p(t, df):
    # Student t two-sided p-value by numerical integration of the density
    pdf = lambda x: math.gamma((df + 1) / 2) / (math.sqrt(df * math.pi) * math.gamma(df / 2)) * (1 + x * x / df) ** (-(df + 1) / 2)
    steps, top = 20000, abs(t)
    h = top / steps
    area = sum((pdf(i * h) + pdf((i + 1) * h)) / 2 * h for i in range(steps))
    return 1 - 2 * area

def welch(m1, s1, n1, m2, s2, n2):
    se = math.sqrt(s1 ** 2 / n1 + s2 ** 2 / n2)
    t = (m1 - m2) / se
    df = se ** 4 / ((s1 ** 2 / n1) ** 2 / (n1 - 1) + (s2 ** 2 / n2) ** 2 / (n2 - 1))
    return t, df, t_two_sided_p(t, df)

t, df, p = welch(13.75, 2.42, 10, 9.70, 2.95, 10)
print(f"  time per task, sync 13.75 s against async 9.70 s (n = 10): t = {t:.2f}, df = {df:.1f}, p = {p:.3f}; {100 * (13.75 - 9.70) / 13.75:.0f}% faster")
t, df, p = welch(3.8, 1.3, 5, 1.8, 0.45, 5)
print(f"  cubes in a fixed time, async 3.8 against sync 1.8 (n = 5): t = {t:.2f}, df = {df:.1f}, p = {p:.3f}")

print()
print("do the two speed experiments agree?")
print(f"  speed-up from the task times: {13.75 / 9.70:.2f}x")
print(f"  speed-up from the fixed-time counts: {3.8 / 1.8:.2f}x")
print(f"  a 60 s limit would allow {60 / 13.75:.1f} (sync) and {60 / 9.70:.1f} (async) tasks at those times; the paper reports {1.8} and {3.8} per run")
print("  the time limit is given only as 'e.g., 60 seconds'")

Output:

Table 3, averages recomputed, and the best method per task
  ACT (single task)                  mean  48.3
  pi0 (multi-task)                   mean  61.7
  SmolVLA (pretrained, multi-task)   mean  78.3
  best on Pick-Place: pi0 (multi-task) (100)
  best on Stacking  : SmolVLA (pretrained, multi-task) (90)
  best on Sorting   : SmolVLA (pretrained, multi-task) (70)

effect of pretraining and multi-task training (Table 5), points per task
  Pick-Place multi-task over single-task  +25   pretraining over none   -5
  Stacking   multi-task over single-task   -5   pretraining over none  +50
  Sorting    multi-task over single-task  +15   pretraining over none  +35

synchronous against asynchronous inference (Figure 5)
  success per task: sync (75, 90, 70)  async (80, 90, 50)  differences [5, 0, -20]; averages 78.3 and 73.3
  time per task, sync 13.75 s against async 9.70 s (n = 10): t = 3.36, df = 17.3, p = 0.004; 29% faster
  cubes in a fixed time, async 3.8 against sync 1.8 (n = 5): t = 3.25, df = 4.9, p = 0.023

do the two speed experiments agree?
  speed-up from the task times: 1.42x
  speed-up from the fixed-time counts: 2.11x
  a 60 s limit would allow 4.4 (sync) and 6.2 (async) tasks at those times; the paper reports 1.8 and 3.8 per run
  the time limit is given only as 'e.g., 60 seconds'

# Ablation tables, typed in as printed (LIBERO success rate, percent). Columns: Spatial, Object, Goal, Long, average.
table11 = {("states to the VLM (prefix)", "cross-attention"): (89, 94, 85, 53, 80.3),
           ("states to the expert (suffix)", "cross-attention"): (86, 82, 78, 47, 73.3),
           ("states to the VLM (prefix)", "self-attention"): (62, 74, 57, 20, 53.3),
           ("states to the expert (suffix)", "self-attention"): (80, 92, 80, 47, 74.8)}
table8 = {"first 16 layers (N = 16)": 78.5, "every second layer (Skip % 2)": 75.5, "smaller VLM (VLM-256M)": 75.8}
table9 = {"x1.00": 82.3, "x0.75": 77.5, "x0.50": 80.3, "x0.25": 73.8}

print("Text: 'including state information in the VLM leads to significantly better performance for both the CA and SA variants' (Table 11)")
for attention in ("cross-attention", "self-attention"):
    prefix = table11[("states to the VLM (prefix)", attention)][4]
    suffix = table11[("states to the expert (suffix)", attention)][4]
    print(f"  {attention:16s} prefix {prefix:5.1f}  suffix {suffix:5.1f}  prefix minus suffix {prefix - suffix:+5.1f}")

print()
print("Text: 'skipping every second layer performs better than training a smaller VLM' (Table 8)")
print(f"  every second layer {table8['every second layer (Skip % 2)']}  against smaller VLM {table8['smaller VLM (VLM-256M)']}  difference {table8['every second layer (Skip % 2)'] - table8['smaller VLM (VLM-256M)']:+.1f}")
print(f"  (first 16 layers {table8['first 16 layers (N = 16)']} against smaller VLM {table8['smaller VLM (VLM-256M)']}: {table8['first 16 layers (N = 16)'] - table8['smaller VLM (VLM-256M)']:+.1f})")

print()
print("Caption of Table 9: 'Larger capacities yield better success rates'")
widths = list(table9)
for a, b in zip(widths, widths[1:]):
    print(f"  {a} -> {b}: {table9[b] - table9[a]:+.1f}")
best = max(table9, key=table9.get)
print(f"  the chosen width x0.75 scores {table9['x0.75']}; x0.50 scores {table9['x0.50']}; the best is {best} at {table9[best]}")

Output:

Text: 'including state information in the VLM leads to significantly better performance for both the CA and SA variants' (Table 11)
  cross-attention  prefix  80.3  suffix  73.3  prefix minus suffix  +7.0
  self-attention   prefix  53.3  suffix  74.8  prefix minus suffix -21.5

Text: 'skipping every second layer performs better than training a smaller VLM' (Table 8)
  every second layer 75.5  against smaller VLM 75.8  difference -0.3
  (first 16 layers 78.5 against smaller VLM 75.8: +2.7)

Caption of Table 9: 'Larger capacities yield better success rates'
  x1.00 -> x0.75: -4.8
  x0.75 -> x0.50: +2.8
  x0.50 -> x0.25: -6.5
  the chosen width x0.75 scores 77.5; x0.50 scores 80.3; the best is x1.00 at 82.3