11.2 Memory Layouts and Data Access

Memory layouts define the structure, size, and alignment of native data for safe and efficient access.

Primitive Layouts

Basic Value Layouts:

// Java primitive equivalents
ValueLayout.JAVA_BYTE      // byte, 1 byte
ValueLayout.JAVA_SHORT     // short, 2 bytes
ValueLayout.JAVA_CHAR      // char, 2 bytes
ValueLayout.JAVA_INT       // int, 4 bytes
ValueLayout.JAVA_LONG      // long, 8 bytes
ValueLayout.JAVA_FLOAT     // float, 4 bytes
ValueLayout.JAVA_DOUBLE    // double, 8 bytes
ValueLayout.JAVA_BOOLEAN   // boolean, 1 byte
ValueLayout.ADDRESS        // pointer, platform-dependent

// Example usage
try (Arena arena = Arena.ofConfined()) {
    // Allocate and set each type
    MemorySegment byteVal = arena.allocate(ValueLayout.JAVA_BYTE);
    byteVal.set(ValueLayout.JAVA_BYTE, 0, (byte) 127);

    MemorySegment intVal = arena.allocate(ValueLayout.JAVA_INT);
    intVal.set(ValueLayout.JAVA_INT, 0, 42);

    MemorySegment doubleVal = arena.allocate(ValueLayout.JAVA_DOUBLE);
    doubleVal.set(ValueLayout.JAVA_DOUBLE, 0, 3.14159);

    MemorySegment addrVal = arena.allocate(ValueLayout.ADDRESS);
    addrVal.set(ValueLayout.ADDRESS, 0, MemorySegment.NULL);
}

Endianness:

// Platform endianness (default)
ValueLayout.JAVA_INT  // Native byte order

// Explicit endianness
ValueLayout intBigEndian = ValueLayout.JAVA_INT
    .withOrder(ByteOrder.BIG_ENDIAN);

ValueLayout intLittleEndian = ValueLayout.JAVA_INT
    .withOrder(ByteOrder.LITTLE_ENDIAN);

// Example
try (Arena arena = Arena.ofConfined()) {
    MemorySegment segment = arena.allocate(4);

    segment.set(intLittleEndian, 0, 0x12345678);

    // Read as bytes
    for (int i = 0; i < 4; i++) {
        byte b = segment.get(ValueLayout.JAVA_BYTE, i);
        System.out.printf("Byte %d: 0x%02x%n", i, b);
    }
    // Output (little-endian): 0x78, 0x56, 0x34, 0x12
}

Struct Layouts

Defining C Structures:

// C struct:
// struct Person {
//     int age;
//     float height;
//     char name[32];
// };

MemoryLayout PERSON_LAYOUT = MemoryLayout.structLayout(
    ValueLayout.JAVA_INT.withName("age"),
    ValueLayout.JAVA_FLOAT.withName("height"),
    MemoryLayout.sequenceLayout(32, ValueLayout.JAVA_BYTE).withName("name")
);

// Access fields
try (Arena arena = Arena.ofConfined()) {
    MemorySegment person = arena.allocate(PERSON_LAYOUT);

    // Using VarHandle
    VarHandle ageHandle = PERSON_LAYOUT.varHandle(
        MemoryLayout.PathElement.groupElement("age")
    );
    VarHandle heightHandle = PERSON_LAYOUT.varHandle(
        MemoryLayout.PathElement.groupElement("height")
    );

    ageHandle.set(person, 0L, 30);
    heightHandle.set(person, 0L, 5.9f);

    // Name field (byte array)
    MemorySegment nameSlice = person.asSlice(
        PERSON_LAYOUT.byteOffset(MemoryLayout.PathElement.groupElement("name")),
        32
    );

    String name = "Alice";
    MemorySegment nameBytes = arena.allocateUtf8String(name);
    MemorySegment.copy(nameBytes, 0, nameSlice, 0, name.length());

    // Read back
    int age = (int) ageHandle.get(person, 0L);
    float height = (float) heightHandle.get(person, 0L);

    System.out.println("Age: " + age);
    System.out.println("Height: " + height);
}

Struct with Padding:

// C struct with alignment:
// struct Data {
//     char flag;      // 1 byte
//     // 3 bytes padding
//     int value;      // 4 bytes
//     short count;    // 2 bytes
//     // 2 bytes padding
//     long timestamp; // 8 bytes
// };

MemoryLayout DATA_LAYOUT = MemoryLayout.structLayout(
    ValueLayout.JAVA_BYTE.withName("flag"),
    MemoryLayout.paddingLayout(3),  // Explicit padding for alignment
    ValueLayout.JAVA_INT.withName("value"),
    ValueLayout.JAVA_SHORT.withName("count"),
    MemoryLayout.paddingLayout(2),  // Explicit padding
    ValueLayout.JAVA_LONG.withName("timestamp")
);

System.out.println("Total size: " + DATA_LAYOUT.byteSize());  // 24 bytes
System.out.println("Alignment: " + DATA_LAYOUT.byteAlignment());  // 8 bytes

Nested Structures:

// C structs:
// struct Point {
//     int x;
//     int y;
// };
//
// struct Rectangle {
//     struct Point topLeft;
//     struct Point bottomRight;
// };

MemoryLayout POINT_LAYOUT = MemoryLayout.structLayout(
    ValueLayout.JAVA_INT.withName("x"),
    ValueLayout.JAVA_INT.withName("y")
);

MemoryLayout RECTANGLE_LAYOUT = MemoryLayout.structLayout(
    POINT_LAYOUT.withName("topLeft"),
    POINT_LAYOUT.withName("bottomRight")
);

try (Arena arena = Arena.ofConfined()) {
    MemorySegment rect = arena.allocate(RECTANGLE_LAYOUT);

    // Access nested fields
    VarHandle tlxHandle = RECTANGLE_LAYOUT.varHandle(
        MemoryLayout.PathElement.groupElement("topLeft"),
        MemoryLayout.PathElement.groupElement("x")
    );
    VarHandle tlyHandle = RECTANGLE_LAYOUT.varHandle(
        MemoryLayout.PathElement.groupElement("topLeft"),
        MemoryLayout.PathElement.groupElement("y")
    );

    tlxHandle.set(rect, 0L, 10);
    tlyHandle.set(rect, 0L, 20);

    int tlx = (int) tlxHandle.get(rect, 0L);
    int tly = (int) tlyHandle.get(rect, 0L);

    System.out.println("TopLeft: (" + tlx + ", " + tly + ")");
}

Array and Sequence Layouts

Fixed-Size Arrays:

// C: int numbers[10];
MemoryLayout INT_ARRAY_LAYOUT = MemoryLayout.sequenceLayout(
    10,
    ValueLayout.JAVA_INT
);

try (Arena arena = Arena.ofConfined()) {
    MemorySegment array = arena.allocate(INT_ARRAY_LAYOUT);

    // Fill array
    for (int i = 0; i < 10; i++) {
        array.setAtIndex(ValueLayout.JAVA_INT, i, i * 10);
    }

    // Read array
    for (int i = 0; i < 10; i++) {
        int value = array.getAtIndex(ValueLayout.JAVA_INT, i);
        System.out.println("numbers[" + i + "] = " + value);
    }
}

Multi-Dimensional Arrays:

// C: int matrix[3][4];
MemoryLayout MATRIX_LAYOUT = MemoryLayout.sequenceLayout(3,
    MemoryLayout.sequenceLayout(4, ValueLayout.JAVA_INT)
);

try (Arena arena = Arena.ofConfined()) {
    MemorySegment matrix = arena.allocate(MATRIX_LAYOUT);

    // Access using flattened index
    VarHandle elementHandle = MATRIX_LAYOUT.varHandle(
        MemoryLayout.PathElement.sequenceElement(),
        MemoryLayout.PathElement.sequenceElement()
    );

    // Fill matrix[i][j] = i * 10 + j
    for (int i = 0; i < 3; i++) {
        for (int j = 0; j < 4; j++) {
            elementHandle.set(matrix, 0L, (long) i, (long) j, i * 10 + j);
        }
    }

    // Read back
    for (int i = 0; i < 3; i++) {
        for (int j = 0; j < 4; j++) {
            int value = (int) elementHandle.get(matrix, 0L, (long) i, (long) j);
            System.out.printf("matrix[%d][%d] = %d%n", i, j, value);
        }
    }
}

Variable-Length Data:

// Reading variable-length array from native code
public class VariableLengthArray {
    // C function returns pointer to int array and sets count
    // int* get_numbers(int* count);

    public static int[] readNativeArray(MemorySegment arrayPtr, int count) {
        int[] result = new int[count];

        for (int i = 0; i < count; i++) {
            result[i] = arrayPtr.getAtIndex(ValueLayout.JAVA_INT, i);
        }

        return result;
    }
}

Union Layouts

Defining Unions:

// C union:
// union Value {
//     int intValue;
//     float floatValue;
//     char bytes[4];
// };

MemoryLayout VALUE_UNION = MemoryLayout.unionLayout(
    ValueLayout.JAVA_INT.withName("intValue"),
    ValueLayout.JAVA_FLOAT.withName("floatValue"),
    MemoryLayout.sequenceLayout(4, ValueLayout.JAVA_BYTE).withName("bytes")
);

try (Arena arena = Arena.ofConfined()) {
    MemorySegment value = arena.allocate(VALUE_UNION);

    // Set as int
    VarHandle intHandle = VALUE_UNION.varHandle(
        MemoryLayout.PathElement.groupElement("intValue")
    );
    intHandle.set(value, 0L, 0x12345678);

    // Read as float (same memory)
    VarHandle floatHandle = VALUE_UNION.varHandle(
        MemoryLayout.PathElement.groupElement("floatValue")
    );
    float floatValue = (float) floatHandle.get(value, 0L);

    // Read as bytes
    for (int i = 0; i < 4; i++) {
        byte b = value.get(ValueLayout.JAVA_BYTE, i);
        System.out.printf("byte[%d] = 0x%02x%n", i, b);
    }
}

VarHandle for Efficient Access

Creating VarHandles:

MemoryLayout PERSON = MemoryLayout.structLayout(
    ValueLayout.JAVA_INT.withName("id"),
    ValueLayout.JAVA_FLOAT.withName("salary"),
    ValueLayout.JAVA_BYTE.withName("status")
);

// Create VarHandles once, reuse many times
VarHandle idHandle = PERSON.varHandle(
    MemoryLayout.PathElement.groupElement("id")
);
VarHandle salaryHandle = PERSON.varHandle(
    MemoryLayout.PathElement.groupElement("salary")
);
VarHandle statusHandle = PERSON.varHandle(
    MemoryLayout.PathElement.groupElement("status")
);

// Efficient access
try (Arena arena = Arena.ofConfined()) {
    MemorySegment person = arena.allocate(PERSON);

    // Set values
    idHandle.set(person, 0L, 12345);
    salaryHandle.set(person, 0L, 75000.0f);
    statusHandle.set(person, 0L, (byte) 1);

    // Get values
    int id = (int) idHandle.get(person, 0L);
    float salary = (float) salaryHandle.get(person, 0L);
    byte status = (byte) statusHandle.get(person, 0L);
}

Atomic Operations:

try (Arena arena = Arena.ofShared()) {
    MemorySegment counter = arena.allocate(ValueLayout.JAVA_INT);

    VarHandle handle = MemoryLayout.structLayout(
        ValueLayout.JAVA_INT.withName("value")
    ).varHandle(MemoryLayout.PathElement.groupElement("value"));

    // Atomic operations
    handle.set(counter, 0L, 0);

    // Atomic increment
    int oldValue = (int) handle.getAndAdd(counter, 0L, 1);
    System.out.println("Old: " + oldValue);

    // Compare and set
    boolean success = handle.compareAndSet(counter, 0L, 1, 2);
    System.out.println("CAS success: " + success);

    int current = (int) handle.getVolatile(counter, 0L);
    System.out.println("Current: " + current);
}

Real-World Example: Parsing Binary Protocol

import java.lang.foreign.*;
import java.lang.invoke.VarHandle;

public class BinaryProtocolParser {
    // Protocol message:
    // struct Message {
    //     uint16_t magic;       // 0xABCD
    //     uint16_t version;     // 1
    //     uint32_t type;        // Message type
    //     uint32_t length;      // Payload length
    //     uint64_t timestamp;   // Unix timestamp
    //     char payload[N];      // Variable length
    // };

    private static final MemoryLayout MESSAGE_HEADER = MemoryLayout.structLayout(
        ValueLayout.JAVA_SHORT.withName("magic"),
        ValueLayout.JAVA_SHORT.withName("version"),
        ValueLayout.JAVA_INT.withName("type"),
        ValueLayout.JAVA_INT.withName("length"),
        ValueLayout.JAVA_LONG.withName("timestamp")
    );

    private static final VarHandle MAGIC_HANDLE = MESSAGE_HEADER.varHandle(
        MemoryLayout.PathElement.groupElement("magic")
    );
    private static final VarHandle VERSION_HANDLE = MESSAGE_HEADER.varHandle(
        MemoryLayout.PathElement.groupElement("version")
    );
    private static final VarHandle TYPE_HANDLE = MESSAGE_HEADER.varHandle(
        MemoryLayout.PathElement.groupElement("type")
    );
    private static final VarHandle LENGTH_HANDLE = MESSAGE_HEADER.varHandle(
        MemoryLayout.PathElement.groupElement("length")
    );
    private static final VarHandle TIMESTAMP_HANDLE = MESSAGE_HEADER.varHandle(
        MemoryLayout.PathElement.groupElement("timestamp")
    );

    public record Message(
        short magic,
        short version,
        int type,
        int length,
        long timestamp,
        byte[] payload
    ) {}

    public static Message parseMessage(MemorySegment buffer) {
        // Read header
        short magic = (short) MAGIC_HANDLE.get(buffer, 0L);
        short version = (short) VERSION_HANDLE.get(buffer, 0L);
        int type = (int) TYPE_HANDLE.get(buffer, 0L);
        int length = (int) LENGTH_HANDLE.get(buffer, 0L);
        long timestamp = (long) TIMESTAMP_HANDLE.get(buffer, 0L);

        // Validate
        if (magic != (short) 0xABCD) {
            throw new IllegalArgumentException("Invalid magic number");
        }

        // Read payload
        long headerSize = MESSAGE_HEADER.byteSize();
        MemorySegment payloadSegment = buffer.asSlice(headerSize, length);

        byte[] payload = new byte[length];
        MemorySegment.copy(payloadSegment, ValueLayout.JAVA_BYTE, 0,
                          payload, 0, length);

        return new Message(magic, version, type, length, timestamp, payload);
    }

    public static MemorySegment createMessage(
            Arena arena,
            int type,
            long timestamp,
            byte[] payload) {

        long totalSize = MESSAGE_HEADER.byteSize() + payload.length;
        MemorySegment buffer = arena.allocate(totalSize);

        // Write header
        MAGIC_HANDLE.set(buffer, 0L, (short) 0xABCD);
        VERSION_HANDLE.set(buffer, 0L, (short) 1);
        TYPE_HANDLE.set(buffer, 0L, type);
        LENGTH_HANDLE.set(buffer, 0L, payload.length);
        TIMESTAMP_HANDLE.set(buffer, 0L, timestamp);

        // Write payload
        MemorySegment payloadSegment = buffer.asSlice(
            MESSAGE_HEADER.byteSize(),
            payload.length
        );
        MemorySegment.copy(payload, 0, payloadSegment, ValueLayout.JAVA_BYTE, 0, payload.length);

        return buffer;
    }
}

Best Practices

1. Cache VarHandles:

// Good - create once, reuse
private static final VarHandle ID_HANDLE = LAYOUT.varHandle(...);

// Avoid - creating repeatedly
for (int i = 0; i < 1000; i++) {
    VarHandle handle = LAYOUT.varHandle(...);  // Expensive!
}

2. Use Named Fields:

// Good - clear field names
MemoryLayout.structLayout(
    ValueLayout.JAVA_INT.withName("userId"),
    ValueLayout.JAVA_LONG.withName("timestamp")
);

// Avoid - anonymous fields
MemoryLayout.structLayout(
    ValueLayout.JAVA_INT,
    ValueLayout.JAVA_LONG
);

3. Document Alignment:

// Document expected C struct alignment
// Matches: struct __attribute__((packed)) Data { ... };
MemoryLayout DATA = MemoryLayout.structLayout(
    ValueLayout.JAVA_BYTE.withName("flag"),
    ValueLayout.JAVA_INT.withName("value")
);  // No padding

// Or with explicit padding:
MemoryLayout DATA_ALIGNED = MemoryLayout.structLayout(
    ValueLayout.JAVA_BYTE.withName("flag"),
    MemoryLayout.paddingLayout(3),  // Document padding
    ValueLayout.JAVA_INT.withName("value")
);

4. Validate Layout Sizes:

// Verify against C sizeof()
long expectedSize = 24;  // From C: sizeof(struct Data)
long actualSize = DATA_LAYOUT.byteSize();

if (actualSize != expectedSize) {
    throw new IllegalStateException(
        "Layout size mismatch: expected " + expectedSize + 
        ", got " + actualSize
    );
}

5. Handle Platform Differences:

// Use ADDRESS for pointers, not JAVA_LONG
ValueLayout.ADDRESS  // Correct on 32-bit and 64-bit

// Check pointer size
boolean is64Bit = ValueLayout.ADDRESS.byteSize() == 8;