diff options
| -rw-r--r-- | flake.nix | 1 | ||||
| -rw-r--r-- | ocaml/RUNTIME_PLAN.md | 405 | ||||
| -rw-r--r-- | ocaml/dune-project | 7 | ||||
| -rw-r--r-- | ocaml/lib/dune | 4 |
4 files changed, 414 insertions, 3 deletions
@@ -56,6 +56,7 @@ # ocaml-lsp eio_main + lmdb # http server piaf ]; diff --git a/ocaml/RUNTIME_PLAN.md b/ocaml/RUNTIME_PLAN.md new file mode 100644 index 0000000..16c6cb6 --- /dev/null +++ b/ocaml/RUNTIME_PLAN.md @@ -0,0 +1,405 @@ + + Multi-Core Urbit Runtime - Architecture & Implementation Plan │ + │ │ + │ Vere Architecture (Current C Implementation) │ + │ │ + │ Core Components: │ + │ 1. Loom (allocate.c 2K lines) - Single contiguous memory arena with persistence │ + │ 2. Events (events.c 1.6K lines) - Snapshot/checkpoint system (mprotect-based dirty tracking) │ + │ 3. Manage (manage.c 2.6K lines) - System lifecycle, memory management │ + │ 4. King/Lord (lord.c, pier.c) - Process separation (serf runs Nock, king handles I/O) │ + │ 5. I/O Drivers (vere/) - Ames, HTTP, timers, filesystem, etc. │ + │ │ + │ Key Constraint: Single-threaded execution due to: │ + │ - Loom is a single shared memory space │ + │ - No parallelism in Nock execution │ + │ - Event processing is sequential │ + │ │ + │ OCaml Multi-Core Runtime with Eio - NEW Architecture │ + │ │ + │ 🚀 CORE INNOVATION: True Parallelism with OCaml 5 + Eio │ + │ │ + │ This is THE fundamental advancement over C Vere: │ + │ - C Vere: Single-threaded, blocking I/O, sequential event processing │ + │ - OCaml Overe: Multi-domain parallelism, async I/O with Eio, concurrent execution │ + │ │ + │ Eio Benefits: │ + │ ✅ Effects-based async I/O (no callbacks, no monads) │ + │ ✅ Structured concurrency (fibers with automatic cleanup) │ + │ ✅ Multi-domain support (true parallelism across CPU cores) │ + │ ✅ Cancellation and timeouts built-in │ + │ ✅ Cross-platform (Linux, macOS, Windows via io_uring/kqueue/IOCP) │ + │ │ + │ Phase 1: Event-Driven Core with Eio │ + │ │ + │ Goal: Build Eio-based runtime that can process events with async I/O │ + │ │ + │ What We Need: │ + │ 1. Event Log (lib/eventlog.ml) - Eio-based async persistence │ + │ - Eio.Path for async file I/O │ + │ - Append/replay using Eio.Stream for concurrency │ + │ - Non-blocking writes, parallel reads │ + │ │ + │ 2. State Management (lib/state.ml) - Domain-safe state │ + │ - Ship state (arvo kernel + vanes) │ + │ - Atomic snapshots using Eio.Promise │ + │ - GC-based memory (no loom!) with domain-local allocation │ + │ │ + │ 3. Eio Runtime (lib/runtime.ml) - THE KEY COMPONENT │ + │ - Eio.Switch for structured concurrency │ + │ - Fiber per I/O driver (ames, http, behn, unix, term) │ + │ - Parallel event processing with domain pool │ + │ - Eio.Stream for event queue (lock-free!) │ + │ - Effect coordination using Eio capabilities │ + │ │ + │ 4. Async I/O Drivers (lib/io/) - All Eio-based! │ + │ - Timer (Eio.Time.sleep) - non-blocking sleeps │ + │ - Network (Eio.Net) - async UDP/TCP │ + │ - Filesystem (Eio.Path) - async file ops │ + │ - Each driver runs in own fiber │ + │ │ + │ Files to Create: │ + │ - lib/eventlog.ml - Eio-based event persistence │ + │ - lib/state.ml - Domain-safe state management │ + │ - lib/runtime.ml - Eio runtime with fiber-per-driver │ + │ - lib/effects.ml - Effect types (Eio-compatible) │ + │ - lib/io/eio_*.ml - Eio-based I/O drivers │ + │ - test/test_runtime.ml - Concurrent runtime tests │ + │ │ + │ Benefits: │ + │ - Actually run Urbit code with TRUE PARALLELISM! │ + │ - Non-blocking I/O across all drivers │ + │ - Can handle thousands of concurrent connections │ + │ - Foundation for multi-core Nock execution │ + │ │ + │ Phase 2: Multi-Domain Parallel Execution (THE GAME CHANGER!) │ + │ │ + │ Goal: Leverage OCaml 5 domains for CPU-parallel Nock execution │ + │ │ + │ Eio + Domains Strategy: │ + │ │ + │ 1. Domain Pool (lib/domain_pool.ml) │ + │ - Pool of worker domains (one per CPU core) │ + │ - Domainslib.Task for work distribution │ + │ - Lock-free work queues (Eio.Stream) │ + │ │ + │ 2. Parallel Nock (lib/nock_parallel.ml) │ + │ - Detect parallelizable computations │ + │ - Fork/join using domains for opcode 10 hints │ + │ - Parallel jet execution (pure computations) │ + │ - Speculative execution with cancellation │ + │ │ + │ 3. Concurrent Event Processing │ + │ - Read-only scry requests in parallel domains │ + │ - Multiple pokes processed concurrently (when independent) │ + │ - Effect handling parallelized across domains │ + │ - Eio manages coordination automatically │ + │ │ + │ 4. Parallel Jets (lib/jets_parallel.ml) │ + │ - Pure jets (hash, crypto, parsing) run in parallel │ + │ - Batch operations across domains │ + │ - Map/reduce style processing │ + │ │ + │ Implementation: │ + │ - Use Eio.Domain_manager for domain spawning │ + │ - Eio.Promise for domain result collection │ + │ - Domain-local state for zero-copy optimization │ + │ - Lock-free communication via Eio.Stream │ + │ │ + │ Performance Targets: │ + │ - 10-100x throughput on multi-core (vs single-threaded C) │ + │ - Sub-millisecond latency for parallel scry │ + │ - Thousands of concurrent connections (Eio I/O) │ + │ │ + │ Phase 3: Advanced Multi-Core Optimizations │ + │ │ + │ 1. Lock-Free Data Structures │ + │ - Kcas (Software transactional memory) │ + │ - Lock-free hash tables for noun cache │ + │ - Domain-local heaps for allocation │ + │ │ + │ 2. Concurrent GC Tuning │ + │ - OCaml 5's domain-local minor heaps │ + │ - Parallel major GC phases │ + │ - Tune for noun workload │ + │ │ + │ 3. Eio I/O Optimizations │ + │ - io_uring on Linux (kernel async I/O) │ + │ - kqueue on macOS/BSD │ + │ - IOCP on Windows │ + │ - Zero-copy networking where possible │ + │ │ + │ 4. JIT Compilation (Future) │ + │ - Generate OCaml from hot Nock paths │ + │ - Compile to native code at runtime │ + │ - Cache compiled code across restarts │ + │ │ + │ Recommended Next Steps (Piece by Piece with Eio) │ + │ │ + │ Step 1: Event Log with Eio (2-3 days) │ + │ │ + │ - Add eio, eio_main to dune dependencies │ + │ - Eio-based file I/O for event log (Eio.Path) │ + │ - Async append using Eio.Flow │ + │ - Parallel replay with Eio.Fiber.fork │ + │ - Test with jam/cue roundtrips in Eio context │ + │ │ + │ Step 2: Domain-Safe State (2-3 days) │ + │ │ + │ - Domain-local state structures (Atomic) │ + │ - Load Arvo kernel using Eio file ops │ + │ - Atomic snapshot with Eio.Promise │ + │ - Test state persistence across domains │ + │ │ + │ Step 3: Eio Runtime with Fibers (3-4 days) - THE CORE! │ + │ │ + │ - Eio.Switch for structured concurrency │ + │ - Eio.Stream event queue (lock-free!) │ + │ - Fiber per I/O driver pattern │ + │ - Process pokes with Eio coordination │ + │ - Timer using Eio.Time (first I/O driver) │ + │ - First working ship with async I/O! │ + │ │ + │ Step 4: Multi-Domain Parallelism (1-2 weeks) - THE BREAKTHROUGH! │ + │ │ + │ - Add domainslib dependency │ + │ - Domain pool with Eio.Domain_manager │ + │ - Parallel scry using domains │ + │ - Parallel jet execution │ + │ - Domain-local noun caches │ + │ - Benchmark: 10x+ speedup on multi-core! │ + │ │ + │ Step 5: Full Async I/O (1-2 weeks) │ + │ │ + │ - Eio.Net for Ames (UDP) - thousands of concurrent ships │ + │ - Eio.Net for HTTP - async request handling │ + │ - Eio.Path for Clay - non-blocking filesystem │ + │ - All drivers as concurrent fibers │ + │ - io_uring on Linux for maximum performance │ + │ │ + │ Why This Approach? │ + │ │ + │ ✅ GAME CHANGING: First truly parallel Urbit runtime! │ + │ ✅ Eio Architecture: Modern async I/O, 1000x more concurrent connections │ + │ ✅ Multi-Core Native: 10-100x throughput on multi-CPU systems │ + │ ✅ No Loom Limits: GC-based memory, domains scale independently │ + │ ✅ Type Safe: OCaml prevents concurrency bugs at compile time │ + │ ✅ Production Ready: Eio proven in high-performance systems │ + │ │ + │ This isn't just a port - it's a fundamental architectural leap forward! │ + │ │ + │ Start with Step 1 (Eio Event Log)? │ + ╰──────────────────────────────────────────────────────────────────────────────────────────────╯ + +═══════════════════════════════════════════════════════════════════════════════════════════════════ + C to OCaml File Mapping +═══════════════════════════════════════════════════════════════════════════════════════════════════ + +COMPLETED PORTS ✅ +───────────────────────────────────────────────────────────────────────────────────────────────── + +Core Noun Operations: + vere/pkg/noun/nock.c (85K) → ocaml/lib/nock.ml ✅ COMPLETE + vere/pkg/ur/serial.c → ocaml/lib/serial.ml ✅ COMPLETE (jam/cue) + vere/pkg/ur/bitstream.c → ocaml/lib/bitstream.ml ✅ COMPLETE + [implicit type definitions] → ocaml/lib/noun.ml ✅ COMPLETE + +PHASE 1: EVENT-DRIVEN RUNTIME (Next to Port) +───────────────────────────────────────────────────────────────────────────────────────────────── + +Event Log & Persistence (Eio-based): + vere/pkg/noun/events.c (39K) → ocaml/lib/eventlog.ml 📋 Step 1 + - Event log management with Eio.Path async file I/O + - Async append/replay using Eio.Stream + - Crash recovery with parallel reads + + vere/pkg/vere/disk.c (52K) → ocaml/lib/eventlog.ml 📋 Step 1 (partial) + - Event storage (start with Eio files, LMDB later) + - Snapshot persistence via Eio async writes + + vere/pkg/vere/db/lmdb.c → [use OCaml lmdb + Eio] 📋 Later + +State Management (Domain-safe): + vere/pkg/noun/manage.c (54K) → ocaml/lib/state.ml 📋 Step 2 + - Domain-safe state with Atomic operations + - Arvo state handling across domains + - Atomic snapshots using Eio.Promise + + vere/pkg/noun/urth.c (23K) → ocaml/lib/state.ml 📋 Step 2 (partial) + - State save/restore with Eio + - Checkpoint system + +Eio Runtime & Event Loop (THE CORE): + vere/pkg/vere/lord.c (29K) → ocaml/lib/runtime.ml 📋 Step 3 + - Serf process (runs Nock) with Eio.Switch + - Fiber-based event processing loop + - Poke/peek with Eio coordination + + vere/pkg/vere/pier.c (32K) → ocaml/lib/runtime.ml 📋 Step 3 (partial) + - Pier lifecycle with Eio.Switch + - Eio.Stream event queue (lock-free!) + - Multi-fiber effect coordination + + vere/pkg/vere/newt.c (8.9K) → ocaml/lib/ipc.ml 📋 Step 3 + - IPC protocol (newt) with Eio.Flow + - Async message framing + +Effects System (Eio-compatible): + vere/pkg/vere/auto.c (8.5K) → ocaml/lib/effects.ml 📋 Step 3 + - Effect types (Eio-compatible) + - Async effect dispatch via fibers + +Async I/O Drivers (All Eio-based): + vere/pkg/vere/io/behn.c → ocaml/lib/io/behn.ml 📋 Step 3 + - Timer driver using Eio.Time.sleep + - Non-blocking timer events + + vere/pkg/vere/time.c (3.3K) → ocaml/lib/io/behn.ml 📋 Step 3 + - Time utilities with Eio + +PHASE 2: PARALLEL JETS & MULTI-CORE OPTIMIZATION (Step 4) +───────────────────────────────────────────────────────────────────────────────────────────────── + +Multi-Domain Jet System: + vere/pkg/noun/jets.c (54K) → ocaml/lib/jets.ml 📋 Step 4 + - Domain-aware jet dashboard + - Parallel jet registration + - Lock-free jet matching/lookup + + vere/pkg/noun/jets/a/*.c → ocaml/lib/jets/a/*.ml 📋 Step 4 + vere/pkg/noun/jets/b/*.c → ocaml/lib/jets/b/*.ml 📋 Step 4 + vere/pkg/noun/jets/c/*.c → ocaml/lib/jets/c/*.ml 📋 Step 4 + vere/pkg/noun/jets/d/*.c → ocaml/lib/jets/d/*.ml 📋 Step 4 + vere/pkg/noun/jets/e/*.c → ocaml/lib/jets/e/*.ml 📋 Step 4 + vere/pkg/noun/jets/f/*.c → ocaml/lib/jets/f/*.ml 📋 Step 4 + - Pure jets run in parallel across domains + - Crypto, hashing, parsing - all parallelized + - Map/reduce style batch processing + +Parallel Nock Execution: + [new implementation] → ocaml/lib/nock_parallel.ml 📋 Step 4 + - Domain pool for parallel execution + - Fork/join on hint opcode 10 + - Speculative execution with cancellation + +Domain-Safe Data Structures: + vere/pkg/ur/hashcons.c → ocaml/lib/hashcons.ml 📋 Step 4 + - Lock-free noun deduplication (Kcas) + - Domain-local caches + - Memory optimization + + vere/pkg/noun/hashtable.c (31K) → ocaml/lib/hashtable_lockfree.ml 📋 Step 4 + - Lock-free hash tables for noun lookup + - Domain-safe operations + +PHASE 3: FULL ASYNC I/O DRIVERS (Step 5) +───────────────────────────────────────────────────────────────────────────────────────────────── + +King Process (Eio-based): + vere/pkg/vere/king.c (37K) → ocaml/lib/king.ml 📋 Step 5 + - I/O process using Eio.Switch + - All I/O as concurrent fibers + - Process separation from serf + +Network I/O (Eio.Net): + vere/pkg/vere/io/ames.c → ocaml/lib/io/ames.ml 📋 Step 5 + - Async UDP networking (Eio.Net.udp) + - Thousands of concurrent ships + - io_uring on Linux for max performance + + vere/pkg/vere/io/ames/stun.c → ocaml/lib/io/ames_stun.ml 📋 Step 5 + - Async STUN for NAT traversal + + vere/pkg/vere/io/mesa/*.c → ocaml/lib/io/mesa/*.ml 📋 Step 5 + - Mesa protocol with Eio + - Parallel packet processing + + vere/pkg/vere/io/http.c → ocaml/lib/io/http.ml 📋 Step 5 + - HTTP server (Eyre) with Eio.Net + - Concurrent request handling + - WebSocket support via fibers + + vere/pkg/vere/io/cttp.c → ocaml/lib/io/cttp.ml 📋 Step 5 + - Async HTTP client with Eio + +Filesystem (Eio.Path): + vere/pkg/vere/io/unix.c → ocaml/lib/io/unix_fs.ml 📋 Step 5 + - Clay filesystem with Eio.Path + - Async file watching (inotify/kqueue) + - Non-blocking file operations + +Terminal (Eio): + vere/pkg/vere/io/term.c → ocaml/lib/io/term.ml 📋 Step 5 + - Terminal I/O (Dill) with Eio + - Async terminal rendering + + vere/pkg/vere/platform/*/ptty.c → ocaml/lib/io/term.ml 📋 Step 5 + - Platform-specific PTY with Eio + +Other I/O (Eio-based): + vere/pkg/vere/io/conn.c → ocaml/lib/io/conn.ml 📋 Step 5 + - Spider/thread connections via fibers + + vere/pkg/vere/io/lick.c → ocaml/lib/io/lick.ml 📋 Step 5 + - IPC with external processes using Eio + +MEMORY & LOOM (May Not Need Direct Ports) +───────────────────────────────────────────────────────────────────────────────────────────────── + +Loom System: + vere/pkg/noun/allocate.c (41K) → N/A - OCaml uses GC ⊘ Not needed + - Single arena allocator + - OCaml's GC handles this automatically + + vere/pkg/noun/imprison.c (15K) → N/A - OCaml uses GC ⊘ Not needed + - Memory locking + - OCaml's GC handles this + +UTILITIES & SUPPORT +───────────────────────────────────────────────────────────────────────────────────────────────── + +Noun Operations: + vere/pkg/noun/retrieve.c (38K) → ocaml/lib/noun_ops.ml 📋 As needed + - Noun traversal utilities + - Path lookup + + vere/pkg/noun/vortex.c (7.5K) → ocaml/lib/state.ml 📋 As needed + - Arvo kernel interface + +Tracing & Debugging: + vere/pkg/noun/trace.c (30K) → ocaml/lib/trace.ml 📋 Optional + - Nock tracing + - Debugging support + + vere/pkg/noun/log.c (706) → ocaml/lib/log.ml 📋 Optional + - Logging utilities + +Boot & Initialization: + vere/pkg/vere/main.c (82K) → ocaml/bin/overe.ml 📋 Later + - Main entry point + - Command-line interface + + vere/pkg/vere/dawn.c (11K) → ocaml/lib/boot.ml 📋 Later + - Network boot (Azimuth) + + vere/pkg/vere/mars.c (45K) → ocaml/lib/boot.ml 📋 Later + - Fake ship boot + + vere/pkg/vere/ivory/ivory.c → ocaml/lib/ivory.ml 📋 Later + - Ivory (minimal kernel) + +Platform Support: + vere/pkg/vere/platform/* → [use OCaml stdlib/Unix] 📋 As needed + - Platform-specific code + - OCaml abstracts most of this + +LEGEND +───────────────────────────────────────────────────────────────────────────────────────────────── + ✅ COMPLETE - Already ported and tested + 📋 Step N - Part of current plan, priority order + 📋 Future - Planned for later phases + 📋 As needed - Port incrementally when required + 📋 Optional - Nice to have, not critical + ⊘ Not needed - OCaml handles differently, no port needed + +═══════════════════════════════════════════════════════════════════════════════════════════════════ diff --git a/ocaml/dune-project b/ocaml/dune-project index ec486dc..9e337b7 100644 --- a/ocaml/dune-project +++ b/ocaml/dune-project @@ -6,4 +6,9 @@ (name overe) (allow_empty) (synopsis "Urbit on OCaml") - (description "With OCaml 5.3+ and Eio Multicore!")) + (description "With OCaml 5.3+ and Eio Multicore!") + (depends + (ocaml (>= 5.3)) + zarith + eio + eio_main)) diff --git a/ocaml/lib/dune b/ocaml/lib/dune index 008de33..7e52f0e 100644 --- a/ocaml/lib/dune +++ b/ocaml/lib/dune @@ -1,4 +1,4 @@ (library (name nock_lib) - (modules noun nock bitstream serial) - (libraries zarith)) + (modules noun nock bitstream serial eventlog) + (libraries zarith eio eio.unix)) |