1.4 — VLA Architecture and the Perception-to-Action Pipeline

import numpy as np

# A fake colour image, 224 pixels tall and 224 wide, with 3 colour values (red, green, blue) per pixel.
# np.arange fills it with 0, 1, 2, ... so every number is different and easy to follow.
image = np.arange(224 * 224 * 3, dtype=np.float32).reshape(224, 224, 3)

patch_size = 16                  # each patch is a square of 16 x 16 pixels
grid = 224 // patch_size         # 224 / 16 = 14 patches along each side

# Step 1: split the rows into 14 groups of 16 and the columns into 14 groups of 16.
# The shape becomes (14, 16, 14, 16, 3) = (patch row, row inside patch, patch column, column inside patch, colour).
blocks = image.reshape(grid, patch_size, grid, patch_size, 3)

# Step 2: move the two "which patch" axes to the front.
# The shape becomes (14, 14, 16, 16, 3): the first two numbers now pick one patch.
blocks = blocks.transpose(0, 2, 1, 3, 4)

# Step 3: flatten every patch into one list of numbers (16 x 16 x 3 = 768),
# and list all 14 x 14 = 196 patches one after another.
patches = blocks.reshape(grid * grid, patch_size * patch_size * 3)

print("image:", image.shape)
print("patches:", patches.shape, "=", grid, "x", grid, "patches, each", patch_size * patch_size * 3, "numbers")

# Check: the first patch should be exactly the top-left 16 x 16 corner of the image.
top_left_corner = image[:patch_size, :patch_size].reshape(-1)
print("first patch equals the top-left 16x16 block:", bool(np.array_equal(patches[0], top_left_corner)))

Output:

image: (224, 224, 3)
patches: (196, 768) = 14 x 14 patches, each 768 numbers
first patch equals the top-left 16x16 block: True

import numpy as np
rng = np.random.default_rng(0)       # fixed seed, so the "random" numbers are the same every run

# --- The image side ---------------------------------------------------------------
n_patches = 196                      # from the previous example: 14 x 14 patches
vision_dim = 64                      # width of each feature vector the image encoder produces
lm_dim = 128                         # width of the vectors the language model works with

# Pretend output of the image encoder: one feature vector per patch (random numbers here).
image_features = rng.normal(size=(n_patches, vision_dim))

# The projector is a matrix that converts vectors of width 64 into vectors of width 128.
# (Real projectors are trained; this one is random, so it only shows the shapes.)
projector = rng.normal(size=(vision_dim, lm_dim)) / np.sqrt(vision_dim)
image_tokens = image_features @ projector            # (196, 64) x (64, 128) -> (196, 128)

# --- The instruction side ---------------------------------------------------------
# A tiny vocabulary: each word gets a number, and each number gets a vector of width 128.
vocab = {"pick": 0, "up": 1, "the": 2, "red": 3, "block": 4}
embedding_table = rng.normal(size=(len(vocab), lm_dim))

words = "pick up the red block".split()               # ["pick", "up", "the", "red", "block"]
word_numbers = [vocab[word] for word in words]        # [0, 1, 2, 3, 4]
text_tokens = embedding_table[word_numbers]           # look up one vector per word -> (5, 128)

# --- Join them ----------------------------------------------------------------------
# Both now have width 128, so they can be stacked into one sequence: image tokens first, then text.
sequence = np.concatenate([image_tokens, text_tokens])

print("image tokens:", image_tokens.shape)
print("text tokens:", text_tokens.shape)
print("sequence into the backbone:", sequence.shape)

Output:

image tokens: (196, 128)
text tokens: (5, 128)
sequence into the backbone: (201, 128)

import numpy as np
rng = np.random.default_rng(1)       # fixed seed, so the "random" numbers are the same every run

n_bins = 256                         # each action number is described by one of 256 possible bins

# The robot command has 7 numbers: x, y, z movement, 3 rotations, and the gripper (open to closed).
# These are the smallest and largest allowed values for each of the 7 (made-up limits).
low  = np.array([-0.05, -0.05, -0.05, -0.5, -0.5, -0.5, 0.0])
high = np.array([ 0.05,  0.05,  0.05,  0.5,  0.5,  0.5, 1.0])

# Pretend model output: for each of the 7 numbers, a score for each of the 256 bins (random here).
scores = rng.normal(size=(7, n_bins))

# Step 1: for each of the 7 numbers, pick the bin with the highest score.
# These bin numbers (0 to 255) are the "action tokens" the model writes.
tokens = scores.argmax(axis=1)

# Step 2: turn each bin number back into a real value.
# Bin k covers a small slice of [low, high]; we use the middle of that slice.
fraction = (tokens + 0.5) / n_bins           # a number between 0 and 1: how far along the range
action = low + fraction * (high - low)       # scale to the real range of each number

print("tokens:", tokens.tolist())
print("action:", np.round(action, 3).tolist())

# Check: every decoded value must lie between its low and high limit.
inside = np.all((action >= low) & (action <= high))
print("every value inside its limits:", bool(inside))

Output:

tokens: [183, 17, 115, 92, 156, 128, 28]
action: [0.022, -0.043, -0.005, -0.139, 0.111, 0.002, 0.111]
every value inside its limits: True