13.1 Linker Fundamentals and Symbol Resolution

Master the Linker API for connecting Java code to native functions through symbol resolution and dynamic loading.

Understanding the Linker

Core Concepts:

import java.lang.foreign.*;
import java.lang.invoke.MethodHandle;

public class LinkerBasics {
    public void demonstrateLinker() {
        // Get platform-specific linker
        Linker linker = Linker.nativeLinker();

        System.out.println("Linker: " + linker.getClass().getName());
        System.out.println("Platform: " + System.getProperty("os.name"));

        // Linker handles:
        // 1. Symbol lookup
        // 2. Downcall creation (Java -> Native)
        // 3. Upcall creation (Native -> Java)
    }
}

Linker Capabilities:

public class LinkerCapabilities {
    public void exploreCapabilities() {
        Linker linker = Linker.nativeLinker();

        // 1. Default symbol lookup
        SymbolLookup defaultLookup = linker.defaultLookup();

        // 2. Create downcall handles
        // (demonstrated in detail below)

        // 3. Create upcall stubs
        // (for callbacks, demonstrated later)

        // 4. Canonical layouts for primitive types
        System.out.println("Address size: " + 
            ValueLayout.ADDRESS.byteSize() + " bytes");
    }
}

Symbol Lookup Strategies

Strategy 1: Default Lookup (System Libraries):

public class DefaultLookupExample {
    public void useDefaultLookup() {
        Linker linker = Linker.nativeLinker();
        SymbolLookup lookup = linker.defaultLookup();

        // Find common system functions
        var strlen = lookup.find("strlen");
        var malloc = lookup.find("malloc");
        var free = lookup.find("free");
        var getpid = lookup.find("getpid");

        System.out.println("strlen found: " + strlen.isPresent());
        System.out.println("malloc found: " + malloc.isPresent());
        System.out.println("free found: " + free.isPresent());
        System.out.println("getpid found: " + getpid.isPresent());

        // Not all functions available on all platforms
        var windowsFunc = lookup.find("GetCurrentProcessId");
        System.out.println("Windows func found: " + windowsFunc.isPresent());
    }
}

Strategy 2: Loader Lookup (Dynamically Loaded Libraries):

public class LoaderLookupExample {
    public void useLoaderLookup() {
        // Load library using System.loadLibrary or System.load
        try {
            System.loadLibrary("z");  // zlib compression

            // Get loader lookup
            SymbolLookup lookup = SymbolLookup.loaderLookup();

            // Find symbols from loaded library
            var compress = lookup.find("compress");
            var uncompress = lookup.find("uncompress");
            var zlibVersion = lookup.find("zlibVersion");

            System.out.println("compress found: " + compress.isPresent());
            System.out.println("uncompress found: " + uncompress.isPresent());
            System.out.println("zlibVersion found: " + zlibVersion.isPresent());

        } catch (UnsatisfiedLinkError e) {
            System.err.println("Failed to load library: " + e.getMessage());
        }
    }
}

Strategy 3: Library Lookup (Explicit Path):

import java.nio.file.*;

public class LibraryLookupExample {
    public void useLibraryLookup() {
        try (Arena arena = Arena.ofConfined()) {
            // Platform-specific library names
            String libName = getLibraryName();
            Path libPath = findLibrary(libName);

            if (libPath != null && Files.exists(libPath)) {
                // Load specific library
                SymbolLookup lookup = SymbolLookup.libraryLookup(libPath, arena);

                // Find symbols
                var symbol = lookup.find("some_function");
                System.out.println("Function found: " + symbol.isPresent());
            } else {
                System.err.println("Library not found: " + libName);
            }
        }
    }

    private String getLibraryName() {
        String os = System.getProperty("os.name").toLowerCase();
        if (os.contains("win")) {
            return "mylib.dll";
        } else if (os.contains("mac")) {
            return "libmylib.dylib";
        } else {
            return "libmylib.so";
        }
    }

    private Path findLibrary(String name) {
        // Search common library paths
        String[] searchPaths = {
            "/usr/lib",
            "/usr/local/lib",
            "/lib",
            System.getProperty("user.home") + "/lib"
        };

        for (String searchPath : searchPaths) {
            Path path = Paths.get(searchPath, name);
            if (Files.exists(path)) {
                return path;
            }
        }

        return null;
    }
}

Strategy 4: Composed Lookup:

public class ComposedLookupExample {
    public void useComposedLookup() {
        Linker linker = Linker.nativeLinker();

        // Combine multiple lookups
        SymbolLookup defaultLookup = linker.defaultLookup();

        // Load additional libraries
        System.loadLibrary("z");
        SymbolLookup loaderLookup = SymbolLookup.loaderLookup();

        // Compose lookups (searches in order)
        SymbolLookup combined = name -> {
            // Try loader lookup first
            var symbol = loaderLookup.find(name);
            if (symbol.isPresent()) {
                return symbol;
            }
            // Fall back to default
            return defaultLookup.find(name);
        };

        // Use combined lookup
        var strlen = combined.find("strlen");
        var compress = combined.find("compress");

        System.out.println("strlen from default: " + strlen.isPresent());
        System.out.println("compress from zlib: " + compress.isPresent());
    }
}

Symbol Address Handling

Working with Symbol Addresses:

public class SymbolAddresses {
    public void demonstrateAddresses() {
        Linker linker = Linker.nativeLinker();
        SymbolLookup lookup = linker.defaultLookup();

        // Find symbol
        MemorySegment strlen = lookup.find("strlen")
            .orElseThrow(() -> new UnsatisfiedLinkError("strlen not found"));

        // Get address
        long address = strlen.address();
        System.out.println("strlen address: 0x" + Long.toHexString(address));

        // Symbols are MemorySegments with zero size
        System.out.println("strlen size: " + strlen.byteSize());  // 0

        // Can compare addresses
        MemorySegment strlen2 = lookup.find("strlen").orElseThrow();
        System.out.println("Same address: " + (strlen.address() == strlen2.address()));
    }
}

Platform-Specific Symbol Resolution

Handling Platform Differences:

public class PlatformSpecificSymbols {
    private final Linker linker = Linker.nativeLinker();
    private final SymbolLookup lookup = linker.defaultLookup();

    /**
     * Get process ID (platform-specific function names)
     */
    public int getProcessId() throws Throwable {
        String os = System.getProperty("os.name").toLowerCase();
        String functionName;

        if (os.contains("win")) {
            functionName = "GetCurrentProcessId";
        } else {
            functionName = "getpid";
        }

        MemorySegment symbol = lookup.find(functionName)
            .orElseThrow(() -> new UnsatisfiedLinkError(
                "Process ID function not found: " + functionName
            ));

        MethodHandle handle = linker.downcallHandle(
            symbol,
            FunctionDescriptor.of(ValueLayout.JAVA_INT)
        );

        return (int) handle.invoke();
    }

    /**
     * Sleep function (platform-specific)
     */
    public void sleep(int seconds) throws Throwable {
        String os = System.getProperty("os.name").toLowerCase();

        if (os.contains("win")) {
            // Windows: Sleep(DWORD dwMilliseconds)
            MemorySegment sleepSym = lookup.find("Sleep")
                .orElseThrow(() -> new UnsatisfiedLinkError("Sleep not found"));

            MethodHandle sleep = linker.downcallHandle(
                sleepSym,
                FunctionDescriptor.ofVoid(ValueLayout.JAVA_INT)
            );

            sleep.invoke(seconds * 1000);  // Convert to milliseconds

        } else {
            // POSIX: unsigned int sleep(unsigned int seconds)
            MemorySegment sleepSym = lookup.find("sleep")
                .orElseThrow(() -> new UnsatisfiedLinkError("sleep not found"));

            MethodHandle sleep = linker.downcallHandle(
                sleepSym,
                FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_INT)
            );

            sleep.invoke(seconds);
        }
    }
}

Symbol Lookup Caching

Efficient Symbol Management:

import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

public class SymbolCache {
    private final Linker linker = Linker.nativeLinker();
    private final SymbolLookup lookup;
    private final Map<String, MemorySegment> cache = new ConcurrentHashMap<>();

    public SymbolCache(SymbolLookup lookup) {
        this.lookup = lookup;
    }

    /**
     * Get symbol with caching
     */
    public MemorySegment getSymbol(String name) {
        return cache.computeIfAbsent(name, symbolName -> 
            lookup.find(symbolName)
                .orElseThrow(() -> new UnsatisfiedLinkError(
                    "Symbol not found: " + symbolName
                ))
        );
    }

    /**
     * Check if symbol exists
     */
    public boolean hasSymbol(String name) {
        if (cache.containsKey(name)) {
            return true;
        }

        var symbol = lookup.find(name);
        symbol.ifPresent(s -> cache.put(name, s));

        return symbol.isPresent();
    }

    /**
     * Preload symbols
     */
    public void preloadSymbols(String... names) {
        for (String name : names) {
            try {
                getSymbol(name);
            } catch (UnsatisfiedLinkError e) {
                System.err.println("Failed to preload: " + name);
            }
        }
    }

    public int getCacheSize() {
        return cache.size();
    }
}

Real-World Example: Dynamic Library Manager

import java.lang.foreign.*;
import java.lang.invoke.MethodHandle;
import java.nio.file.*;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;

public class DynamicLibraryManager implements AutoCloseable {
    private final Linker linker = Linker.nativeLinker();
    private final Map<String, LibraryInfo> libraries = new ConcurrentHashMap<>();
    private final Arena arena = Arena.ofShared();

    public static class LibraryInfo {
        public final String name;
        public final Path path;
        public final SymbolLookup lookup;
        public final Map<String, MemorySegment> symbols = new ConcurrentHashMap<>();

        public LibraryInfo(String name, Path path, SymbolLookup lookup) {
            this.name = name;
            this.path = path;
            this.lookup = lookup;
        }
    }

    /**
     * Load library from path
     */
    public void loadLibrary(String name, Path libraryPath) {
        if (libraries.containsKey(name)) {
            throw new IllegalStateException("Library already loaded: " + name);
        }

        if (!Files.exists(libraryPath)) {
            throw new IllegalArgumentException("Library not found: " + libraryPath);
        }

        try {
            SymbolLookup lookup = SymbolLookup.libraryLookup(libraryPath, arena);
            LibraryInfo info = new LibraryInfo(name, libraryPath, lookup);
            libraries.put(name, info);

            System.out.println("Loaded library: " + name + " from " + libraryPath);

        } catch (Exception e) {
            throw new RuntimeException("Failed to load library: " + name, e);
        }
    }

    /**
     * Load system library by name
     */
    public void loadSystemLibrary(String name) {
        if (libraries.containsKey(name)) {
            throw new IllegalStateException("Library already loaded: " + name);
        }

        try {
            System.loadLibrary(name);
            SymbolLookup lookup = SymbolLookup.loaderLookup();
            LibraryInfo info = new LibraryInfo(name, null, lookup);
            libraries.put(name, info);

            System.out.println("Loaded system library: " + name);

        } catch (UnsatisfiedLinkError e) {
            throw new RuntimeException("Failed to load system library: " + name, e);
        }
    }

    /**
     * Find symbol in specific library
     */
    public Optional<MemorySegment> findSymbol(String libraryName, String symbolName) {
        LibraryInfo library = libraries.get(libraryName);
        if (library == null) {
            return Optional.empty();
        }

        // Check cache first
        MemorySegment cached = library.symbols.get(symbolName);
        if (cached != null) {
            return Optional.of(cached);
        }

        // Lookup and cache
        Optional<MemorySegment> symbol = library.lookup.find(symbolName);
        symbol.ifPresent(s -> library.symbols.put(symbolName, s));

        return symbol;
    }

    /**
     * Find symbol in any loaded library
     */
    public Optional<MemorySegment> findSymbol(String symbolName) {
        for (LibraryInfo library : libraries.values()) {
            Optional<MemorySegment> symbol = findSymbol(library.name, symbolName);
            if (symbol.isPresent()) {
                return symbol;
            }
        }
        return Optional.empty();
    }

    /**
     * Create downcall handle
     */
    public MethodHandle createDowncall(
            String libraryName,
            String symbolName,
            FunctionDescriptor descriptor
    ) {
        MemorySegment symbol = findSymbol(libraryName, symbolName)
            .orElseThrow(() -> new UnsatisfiedLinkError(
                "Symbol not found: " + symbolName + " in " + libraryName
            ));

        return linker.downcallHandle(symbol, descriptor);
    }

    /**
     * List all loaded libraries
     */
    public Set<String> getLoadedLibraries() {
        return new HashSet<>(libraries.keySet());
    }

    /**
     * Get library information
     */
    public Optional<LibraryInfo> getLibraryInfo(String name) {
        return Optional.ofNullable(libraries.get(name));
    }

    /**
     * Enumerate symbols in library (if supported)
     */
    public Set<String> getCachedSymbols(String libraryName) {
        LibraryInfo library = libraries.get(libraryName);
        if (library == null) {
            return Collections.emptySet();
        }
        return new HashSet<>(library.symbols.keySet());
    }

    @Override
    public void close() {
        libraries.clear();
        arena.close();
    }

    // Example usage
    public static void main(String[] args) {
        try (DynamicLibraryManager manager = new DynamicLibraryManager()) {
            // Load system libraries
            manager.loadSystemLibrary("c");  // Standard C library

            // Try to load zlib
            try {
                manager.loadSystemLibrary("z");
            } catch (RuntimeException e) {
                System.out.println("zlib not available");
            }

            // List loaded libraries
            System.out.println("Loaded libraries: " + manager.getLoadedLibraries());

            // Find strlen in libc
            Optional<MemorySegment> strlen = manager.findSymbol("c", "strlen");
            System.out.println("strlen found: " + strlen.isPresent());

            if (strlen.isPresent()) {
                // Create downcall
                MethodHandle strlenHandle = manager.createDowncall(
                    "c",
                    "strlen",
                    FunctionDescriptor.of(
                        ValueLayout.JAVA_LONG,
                        ValueLayout.ADDRESS
                    )
                );

                // Use it
                try (Arena arena = Arena.ofConfined()) {
                    MemorySegment str = arena.allocateUtf8String("Hello, FFM!");
                    long len = (long) strlenHandle.invoke(str);
                    System.out.println("String length: " + len);
                }
            }

            // Show cached symbols
            System.out.println("Cached symbols in 'c': " + 
                manager.getCachedSymbols("c"));

        } catch (Throwable e) {
            e.printStackTrace();
        }
    }
}

Best Practices

1. Cache Symbol Lookups:

// Good - lookup once, reuse
private static final MemorySegment STRLEN_SYMBOL;
static {
    Linker linker = Linker.nativeLinker();
    STRLEN_SYMBOL = linker.defaultLookup().find("strlen").orElseThrow();
}

// Bad - lookup every time
public void badApproach() throws Throwable {
    Linker linker = Linker.nativeLinker();
    MemorySegment strlen = linker.defaultLookup().find("strlen").orElseThrow();
    // ...
}

2. Handle Missing Symbols Gracefully:

// Good - provide fallback
Optional<MemorySegment> symbol = lookup.find("optional_function");
if (symbol.isPresent()) {
    // Use native function
} else {
    // Use Java fallback
    System.out.println("Native function not available, using fallback");
}

// Bad - unchecked exception
MemorySegment symbol = lookup.find("optional_function").get();  // May throw

3. Use Appropriate Lookup Strategy:

// Good - explicit about which library
System.loadLibrary("mylib");
SymbolLookup lookup = SymbolLookup.loaderLookup();

// Or for system functions
SymbolLookup lookup = Linker.nativeLinker().defaultLookup();

// Bad - unclear which library provides symbol
SymbolLookup lookup = ???  // Where does this come from?

4. Document Platform Dependencies:

/**
 * Uses POSIX sleep() on Unix-like systems,
 * Sleep() on Windows.
 * 
 * @throws UnsatisfiedLinkError if platform not supported
 */
public void platformSleep(int seconds) throws Throwable {
    // Platform-specific implementation
}

5. Validate Symbols at Startup:

public class LibraryValidator {
    public static void validateRequiredSymbols() {
        Linker linker = Linker.nativeLinker();
        SymbolLookup lookup = linker.defaultLookup();

        String[] required = {"strlen", "malloc", "free"};
        List<String> missing = new ArrayList<>();

        for (String symbol : required) {
            if (lookup.find(symbol).isEmpty()) {
                missing.add(symbol);
            }
        }

        if (!missing.isEmpty()) {
            throw new UnsatisfiedLinkError(
                "Required symbols not found: " + missing
            );
        }
    }
}

These patterns ensure robust, maintainable symbol resolution across platforms.