diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 1f286fc..e81885b 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -230,6 +230,11 @@ async fn main() -> anyhow::Result<()> { println!(" redundancy Show replica counts for your posts"); println!(" worm Worm lookup (find peer beyond 3-hop map)"); println!(" connections Show mesh connections"); + println!(" slots Show mesh / temp-referral slot usage"); + println!(" uniques [node_id_hex] Uniques index counts, or where an ID is known"); + println!(" convection Adaptive convection weights + pool-mined anchors"); + println!(" convect Ask an anchor for peers (convection)"); + println!(" uniques-pull Exchange uniques indexes with mesh peers"); println!(" social-routes Show social routing cache"); println!(" name Set your display name"); println!(" register [keywords...] Register default persona in the network registry (30d, re-run to renew)"); @@ -248,13 +253,22 @@ async fn main() -> anyhow::Result<()> { // Start background tasks (v2: mesh connections) let _accept_handle = node.start_accept_loop(); - let _pull_handle = node.start_pull_cycle(); // 60s tiered pull cycle + let _sync_handle = node.start_sync_cycle(); // 60s: uniques-index exchange + content sync let _diff_handle = node.start_diff_cycle(120); // 2 min routing diff let _rebalance_handle = node.start_rebalance_cycle(600); // 10 min rebalance - let _growth_handle = node.start_growth_loop(); // reactive mesh growth + // Reactive mesh growth. `ITSGOIN_TEST_NO_GROWTH` suppresses it so an + // integration test can hold an exact topology (on loopback the growth loop + // otherwise collapses any chain into a full mesh within seconds, and a + // full mesh has no N2 at all — every peer is a direct). Test gate only. + let _growth_handle = if std::env::var("ITSGOIN_TEST_NO_GROWTH").is_ok() { + eprintln!("[test gate] growth loop disabled (ITSGOIN_TEST_NO_GROWTH)"); + None + } else { + Some(node.start_growth_loop()) + }; let _recovery_handle = node.start_recovery_loop(); // reactive anchor reconnect on mesh loss let _checkin_handle = node.start_social_checkin_cycle(3600); // 1 hour social checkin - let _anchor_handle = node.start_anchor_register_cycle(600); // 10 min anchor register + let _convection_handle = node.start_convection_loop(); // stochastic anchor convection + anchor self-probe let _upnp_tcp_handle = node.start_upnp_tcp_renewal_cycle(); // UPnP TCP lease renewal let _http_handle = node.start_http_server(); // HTTP post delivery (if publicly reachable) let _bootstrap_check = node.start_bootstrap_connectivity_check(); // 24h isolation check @@ -858,7 +872,7 @@ async fn main() -> anyhow::Result<()> { println!("(no mesh connections)"); } else { println!("Mesh connections ({}):", conns.len()); - for (nid, slot_kind, connected_at) in conns { + for (nid, slot, connected_at) in conns { let name = node.get_display_name(&nid).await.unwrap_or(None); let id_short = &hex::encode(nid)[..12]; let label = name.map(|n| format!("{} ({})", n, id_short)) @@ -870,7 +884,95 @@ async fn main() -> anyhow::Result<()> { .as_millis() as u64; (now.saturating_sub(connected_at)) / 1000 }; - println!(" {} [{:?}] connected {}s ago", label, slot_kind, duration_secs); + println!(" {} [{}] connected {}s ago", label, slot, duration_secs); + } + } + } + + // v0.8 Iteration C observability: slot pool + uniques index. + "slots" => { + let conns = node.list_connections().await; + let mesh = conns.iter().filter(|(_, s, _)| s.is_mesh()).count(); + let temp = conns.len() - mesh; + println!("mesh {}/{} temp-referral {}/{}", + mesh, node.device_profile().mesh_slots(), + temp, node.device_profile().temp_referral_slots()); + for (nid, slot, _) in conns { + println!(" {} {}", &hex::encode(nid)[..12], slot); + } + } + + "uniques" => { + let storage = node.storage.get().await; + if parts.len() > 1 { + match itsgoin_core::parse_node_id_hex(parts[1]) { + Ok(target) => { + let mut any = false; + for bounce in 2u8..=4 { + for reporter in storage.find_reporters_at(&target, bounce).unwrap_or_default() { + println!(" N{} via {}", bounce, &hex::encode(reporter)[..12]); + any = true; + } + } + if !any { + println!("(not in the uniques index)"); + } + } + Err(e) => println!("Bad node id: {}", e), + } + } else { + println!("N2 {} N3 {} N4 {} (anchor density {:.2})", + storage.count_distinct_n2().unwrap_or(0), + storage.count_distinct_n3().unwrap_or(0), + storage.count_distinct_n4().unwrap_or(0), + storage.anchor_density().unwrap_or(0.0)); + } + } + + // The v0.8 "pull": exchange uniques indexes with every mesh peer. + "uniques-pull" => { + let n = node.uniques_pull().await; + println!("uniques-pull: exchanged with {} mesh peers", n); + } + + // On-demand convection against a named anchor. + "convect" => { + if parts.len() < 2 { + println!("Usage: convect "); + } else { + match itsgoin_core::parse_node_id_hex(parts[1]) { + Ok(anchor) => match node.convection_request(anchor).await { + Ok((_, true, ms)) => println!("convection: refused in {}ms", ms), + Ok((n, false, ms)) => println!("convection: {} new peers in {}ms", n, ms), + Err(e) => println!("convection failed: {}", e), + }, + Err(e) => println!("Bad node id: {}", e), + } + } + } + + // Convection diagnostics: the adaptive action weights, the + // pool-mined anchor directory, and a manual entry-class request. + "convection" => { + let (density, bias, mesh) = node.network.conn_handle().convection_state().await; + let cap = node.device_profile().mesh_slots(); + let fill = if cap == 0 { 1.0 } else { mesh as f64 / cap as f64 }; + let w = itsgoin_core::connection::convection_weights(density, bias, fill); + let total = w.nothing + w.anchor + w.mesh; + println!("anchor_density {:.3} anchor_bias {:.2} mesh {}/{}", density, bias, mesh, cap); + println!("weights: nothing {:.3} anchor {:.3} mesh {:.3} (P: {:.0}% / {:.0}% / {:.0}%)", + w.nothing, w.anchor, w.mesh, + 100.0 * w.nothing / total, 100.0 * w.anchor / total, 100.0 * w.mesh / total); + let anchors = { + let storage = node.storage.get().await; + storage.list_pool_anchors(16).unwrap_or_default() + }; + if anchors.is_empty() { + println!("pool anchors: (none)"); + } else { + println!("pool anchors ({}):", anchors.len()); + for (nid, addrs) in anchors { + println!(" {} {}", &hex::encode(nid)[..12], addrs.join(",")); } } } diff --git a/crates/core/src/connection.rs b/crates/core/src/connection.rs index ea7ed6b..f1367ac 100644 --- a/crates/core/src/connection.rs +++ b/crates/core/src/connection.rs @@ -1,4 +1,4 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::net::SocketAddr; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; @@ -11,26 +11,40 @@ use crate::blob::BlobStore; use crate::content::verify_post_id; use crate::crypto; use crate::protocol::{ - read_message_type, read_payload, write_typed_message, AnchorReferral, - AnchorReferralRequestPayload, AnchorReferralResponsePayload, AnchorRegisterPayload, + read_message_type, read_payload, write_typed_message, + ConvectionReferral, ConvectionRequestPayload, ConvectionResponsePayload, BlobHeaderDiffPayload, BlobHeaderRequestPayload, BlobHeaderResponsePayload, BlobRequestPayload, BlobResponsePayload, CircleProfileUpdatePayload, InitialExchangePayload, - MessageType, NodeListUpdatePayload, PostDownstreamRegisterPayload, - ProfileUpdatePayload, PullSyncRequestPayload, PullSyncResponsePayload, + MessageType, PostDownstreamRegisterPayload, + ProfileUpdatePayload, ContentSyncRequestPayload, ContentSyncResponsePayload, UniquesPullPayload, RefuseRedirectPayload, RelayIntroducePayload, RelayIntroduceResultPayload, SessionRelayPayload, SocialAddressUpdatePayload, SocialCheckinPayload, SyncPost, WormQueryPayload, WormResponsePayload, ReplicationRequestPayload, ReplicationResponsePayload, ALPN, }; use crate::storage::StoragePool; +use crate::protocol::{ + pack_ids, unpack_ids, AnchorEntry, UniquesAnnouncePayload, UniquesSlice, +}; use crate::types::{ - DeviceProfile, NodeId, PeerSlotKind, PeerWithAddress, PostId, PostVisibility, ReachMethod, - SessionReachMethod, SocialRouteEntry, SocialStatus, WormId, WormResult, + DeviceProfile, IdClass, MeshSlot, NodeId, PeerWithAddress, PostId, PostVisibility, + ReachEntry, ReachMethod, SessionReachMethod, SocialRouteEntry, SocialStatus, UniquesPools, + WormId, WormResult, }; const MAX_PAYLOAD: usize = 64 * 1024 * 1024; // 64 MB +/// Read ceiling for the uniques opcodes (0x01 announce, 0x40/0x41 pull). +/// +/// `MAX_PAYLOAD` exists for FILE transfer. Applying it to the index exchange +/// let one announce decode into ~1.5M packed IDs and the same number of row +/// inserts, per bounce, per reporter, across ~20 reporters — a storage/CPU +/// amplifier reachable by any mesh peer. design.html §layers budgets N2 <= 400 +/// and N3 <= 8,000, i.e. well under 400 KB of packed IDs; 2 MB is an order of +/// magnitude of headroom over the design ceiling and 32x under the file cap. +const MAX_UNIQUES_PAYLOAD: usize = 2 * 1024 * 1024; // 2 MB + const WORM_FAN_OUT_TIMEOUT_MS: u64 = 500; const WORM_BLOOM_TIMEOUT_MS: u64 = 1500; const WORM_TOTAL_TIMEOUT_MS: u64 = 3000; @@ -47,10 +61,40 @@ const RELAY_PIPE_IDLE_MS: u64 = 120_000; // 2 min /// How long reconnect watchers live before expiry (30 days) const WATCHER_EXPIRY_MS: i64 = 30 * 24 * 60 * 60 * 1000; -/// Grace period before removing disconnected peers from referral list -const REFERRAL_DISCONNECT_GRACE_MS: u64 = 120_000; // 2 min -/// Soft cap on referral list size (affects max_uses tiering) -const REFERRAL_LIST_CAP: usize = 50; +// --- Anchor convection (design.html §anchors) --- +// +// No registration database. The anchor keeps a small ROLLING WINDOW of recent +// callers, fed by the convection request itself. A caller gets the 2 most +// recent viable prior callers; its own address then goes to the next 2 callers +// and rotates out. That is the whole service. + +/// How many recent callers the rolling window holds. Small on purpose: the +/// point is *recency*, not coverage — a stale address is worse than none, +/// because it costs the caller a hole-punch timeout. +const CONVECTION_WINDOW_SIZE: usize = 16; +/// How many callers an entry is handed to before it rotates out ("2 before, +/// 2 after"). +const CONVECTION_HANDOUT_CAP: u32 = 2; +/// How many referrals a served request gets. +const CONVECTION_REFERRALS: usize = 2; +/// An entry older than this is not worth handing out — the caller has probably +/// moved or gone. Bounds the window in time as well as in count. +const CONVECTION_ENTRY_MAX_AGE_MS: u64 = 15 * 60 * 1000; // 15 min +/// How often the anchor-density prior is recomputed (it changes slowly, and +/// the disconnect path must never pay for a COUNT DISTINCT). +const ANCHOR_DENSITY_REFRESH_MS: u64 = 60_000; // 1 min +/// How long a refusing anchor stays in the penalty box. +const ANCHOR_REFUSAL_PENALTY_MS: u64 = 5 * 60 * 1000; // 5 min +/// A young anchor with an empty window may supplement referrals from its own +/// live mesh peers. Those peers never called and never offered themselves for +/// redistribution, so the supplement gets its own hand-out budget: without one +/// it spent no window budget at all and the same peer could be disclosed to +/// every caller, turning unconditional entry-class service into a topology +/// enumeration oracle. +const CONVECTION_SUPPLEMENT_CAP: u32 = 2; +const CONVECTION_SUPPLEMENT_WINDOW_MS: u64 = 15 * 60 * 1000; // 15 min +/// How many self-reported addresses a caller may add beyond the observed one. +const CONVECTION_MAX_SELF_REPORTED: usize = 2; /// Zombie connection timeout: no stream activity for this long = dead const ZOMBIE_TIMEOUT_MS: u64 = 600_000; // 10 minutes @@ -565,10 +609,23 @@ pub fn normalize_addr(addr: std::net::SocketAddr) -> std::net::SocketAddr { } } +/// How long a temporary referral slot lives before it is reaped, if it has not +/// graduated into a freed mesh slot by then. Expiry is enforced in the rebalance +/// dead-sweep — no separate loop. +pub const TEMP_REFERRAL_TTL_MS: u64 = 5 * 60 * 1000; + +/// Default depth assumed for a peer that hasn't told us its retention horizon. +const DEFAULT_PEER_DEPTH: u8 = 4; + pub struct MeshConnection { pub node_id: NodeId, pub connection: iroh::endpoint::Connection, - pub slot_kind: PeerSlotKind, + /// Mesh vs temporary referral. Temp slots live ABOVE the mesh cap, carry no + /// knowledge exchange, and are never persisted to `mesh_peers`. + pub slot: MeshSlot, + /// Deepest bounce this peer retains (from their announce). We skip building + /// the terminal pool for peers that advertise 3. + pub peer_depth: u8, pub connected_at: u64, /// Remote address as seen from our side of the QUIC connection pub remote_addr: Option, @@ -586,18 +643,309 @@ pub struct SessionConnection { pub remote_addr: Option, } +/// Result of a transitional content sync (0x46/0x47). pub struct PullSyncStats { pub posts_received: usize, } -/// Entry in the anchor's referral list — connection-backed, self-pruning. -struct ReferralEntry { +/// One caller in the anchor's rolling convection window. +/// +/// Deliberately in-memory only: an anchor that restarts should forget, because +/// everything it knew is by then a cold address. Liveness is NOT stored — it is +/// resolved against `connections`/`sessions` at hand-out time, so an entry can +/// never claim a peer is connected when it isn't. +#[derive(Clone)] +pub(crate) struct ConvectionEntry { + pub(crate) node_id: NodeId, + pub(crate) addresses: Vec, + pub(crate) arrived_at: u64, + /// Times this entry has been handed to a later caller (cap + /// `CONVECTION_HANDOUT_CAP`, then it rotates out). + pub(crate) handouts: u32, +} + +/// May this address go into the convection window / a referral? +/// +/// Publicly routable only (bugs-fixed #8: an unroutable referral address costs +/// the next caller a hole-punch timeout), with one TEST-ONLY escape hatch: +/// `ITSGOIN_TEST_ALLOW_LOOPBACK_REFERRALS` lets the whole topology run on +/// 127.0.0.1. Same debug-gate pattern as `ITSGOIN_TEST_ALLOW_REGISTRY_GENESIS`. +pub(crate) fn convection_addr_ok(sa: &SocketAddr) -> bool { + if crate::network::is_publicly_routable(sa) { + return true; + } + static ALLOW_LOOPBACK: std::sync::OnceLock = std::sync::OnceLock::new(); + // `== "1"`, not `.is_ok()`: an empty or accidental value must not silently + // open loopback referral circulation in a shipped build. Matches the + // `ITSGOIN_TEST_ALLOW_REGISTRY_GENESIS` check. + *ALLOW_LOOPBACK + .get_or_init(|| std::env::var("ITSGOIN_TEST_ALLOW_LOOPBACK_REFERRALS").as_deref() == Ok("1")) + && sa.ip().is_loopback() +} + +/// Everything the uniques-index exchange needs from connection state, captured +/// under a brief lock so the storage half runs with the lock released. +pub(crate) struct UniquesCtx { + storage: Arc, + our_node_id: NodeId, + max_bounce: u8, + anchor_addr: Option, + seq: u64, + pub(crate) is_mesh: bool, + pub(crate) peer_depth: u8, + /// Snapshot of who we are connected to — used only to keep already-meshed + /// peers out of the bounce-2 growth-candidate pool. + connected: HashSet, +} + +impl UniquesCtx { + pub(crate) async fn build(&self, peer_depth: u8) -> Option { + let storage = self.storage.get().await; + build_uniques_payload( + &storage, + &self.our_node_id, + self.anchor_addr.as_deref(), + self.seq, + self.max_bounce, + peer_depth, + ) + .ok() + } + + pub(crate) async fn apply( + &self, + reporter: &NodeId, + payload: &UniquesAnnouncePayload, + ) -> anyhow::Result { + let storage = self.storage.get().await; + apply_uniques_to_storage( + &storage, + &self.our_node_id, + reporter, + payload, + self.max_bounce, + |nid| self.connected.contains(nid), + ) + } +} + +/// Everything a convection response needs, gathered under a brief lock so the +/// write and the coordinated introductions happen outside it. +pub struct ConvectionGathered { + pub response: ConvectionResponsePayload, + /// Referral targets that are live on us right now — each gets an + /// anchor-initiated RelayIntroduce naming the caller as requester. + pub introductions: Vec<(NodeId, iroh::endpoint::Connection)>, + pub requester: NodeId, + /// The caller's usable (observed, routable) addresses, for the intro push. + pub requester_addresses: Vec, + pub nat_mapping: Option, + pub nat_filtering: Option, +} + +/// What the stochastic per-disconnect trigger decided to do (round-4/5). +/// +/// There is no threshold and no jitter timer: the randomness itself is what +/// de-synchronises the network's response to a shared disconnect event. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ConvectionAction { + /// Sit this one out. Churn is normal; not every disconnect needs a reaction. + Nothing, + /// Ask a random known anchor for an introduction. + AskAnchor, + /// Ask via a connected mesh peer (this is the existing growth loop, whose + /// introduction arm already exists). + AskMesh, +} + +/// Adaptive action weights for the stochastic trigger (round-5 ruling). +/// +/// The anchor-vs-mesh balance is NOT fixed at design time — the willing-anchor +/// ratio is unknowable before deployment. Two runtime inputs set it: +/// * the local anchor-density prior — anchor entries are the only +/// address-bearing rows in the uniques pools, so their share of known +/// node-class uniques is a free local statistic, no census needed; +/// * refusal feedback — an anchor refusing a top-up shifts weight toward +/// mesh introductions and other anchors; successes drift it back (AIMD). +#[derive(Debug, Clone, Copy)] +pub struct ConvectionWeights { + pub nothing: f64, + pub anchor: f64, + pub mesh: f64, +} + +/// Pure, directly-testable action choice. +/// +/// * `roll` — uniform in [0, 1). +/// * `density` — anchor-flagged share of known node-class uniques, [0, 1]. +/// * `anchor_bias` — AIMD feedback term, (0, 1]. Halved on refusal, drifts back +/// up on success. +/// * `fill_ratio` — mesh_count / mesh_slots, [0, 1]. Leaning toward action when +/// far under target is the "lean toward action" half of the ruling: at +/// fill_ratio 0 the do-nothing weight is 0, so we always act. +pub fn convection_weights(density: f64, anchor_bias: f64, fill_ratio: f64) -> ConvectionWeights { + let density = density.clamp(0.0, 1.0); + let anchor_bias = anchor_bias.clamp(0.05, 1.0); + let fill_ratio = fill_ratio.clamp(0.0, 1.0); + ConvectionWeights { + // Quadratic so a nearly-full mesh mostly ignores churn while a badly + // depleted one almost always reacts. + nothing: 2.0 * fill_ratio * fill_ratio, + // Abundant anchors (IPv6 world) → anchor-led growth keeps convection + // windows fresh. Scarce or unhelpful anchors → this collapses toward 0 + // and mesh introductions carry growth. Same code, both worlds. + anchor: anchor_bias * (0.25 + 1.75 * density), + mesh: 1.0, + } +} + +pub fn choose_convection_action( + roll: f64, + density: f64, + anchor_bias: f64, + fill_ratio: f64, +) -> ConvectionAction { + let w = convection_weights(density, anchor_bias, fill_ratio); + let total = w.nothing + w.anchor + w.mesh; + if total <= 0.0 { + return ConvectionAction::Nothing; + } + let point = roll.clamp(0.0, 1.0) * total; + if point < w.nothing { + ConvectionAction::Nothing + } else if point < w.nothing + w.anchor { + ConvectionAction::AskAnchor + } else { + ConvectionAction::AskMesh + } +} + +/// How a mesh disconnect is answered. Recovery PRE-EMPTS the dice: below 2 mesh +/// peers there is nothing stochastic to decide. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DisconnectResponse { + /// mesh < 2 — act immediately and unconditionally. + Recover, + /// Healthy mesh with room — roll the stochastic action. + Roll, + /// Healthy and full — nothing to do. + Idle, +} + +pub fn disconnect_response(mesh_count: usize, available_mesh_slots: usize) -> DisconnectResponse { + if mesh_count < 2 { + DisconnectResponse::Recover + } else if available_mesh_slots > 0 { + DisconnectResponse::Roll + } else { + DisconnectResponse::Idle + } +} + +// ---- Rolling convection window, as free functions over the deque ---- +// +// Split out of `ConnectionManager` so the rotation rules are directly testable +// without standing up an iroh endpoint. + +/// Admit or refresh a caller. A fresh arrival replaces any older entry and +/// resets its hand-out budget — this caller is available again. +pub(crate) fn convection_window_admit( + window: &mut VecDeque, node_id: NodeId, addresses: Vec, - #[allow(dead_code)] - registered_at: u64, - use_count: u32, - disconnected_at: Option, + now: u64, +) { + if addresses.is_empty() { + return; + } + window.retain(|e| e.node_id != node_id); + window.push_back(ConvectionEntry { node_id, addresses, arrived_at: now, handouts: 0 }); + convection_window_prune(window, now); + while window.len() > CONVECTION_WINDOW_SIZE { + window.pop_front(); + } +} + +/// Drop entries that aged out or used up their hand-out budget. +pub(crate) fn convection_window_prune(window: &mut VecDeque, now: u64) { + window.retain(|e| { + e.handouts < CONVECTION_HANDOUT_CAP + && now.saturating_sub(e.arrived_at) <= CONVECTION_ENTRY_MAX_AGE_MS + }); +} + +/// How many entries could be handed out right now. +pub(crate) fn convection_window_viable( + window: &VecDeque, + exclude: &NodeId, + now: u64, +) -> usize { + window + .iter() + .filter(|e| { + e.node_id != *exclude + && e.handouts < CONVECTION_HANDOUT_CAP + && now.saturating_sub(e.arrived_at) <= CONVECTION_ENTRY_MAX_AGE_MS + }) + .count() +} + +/// Take up to `count` referrals: most recent first, still-connected preferred. +/// Each pick spends one hand-out; an entry that reaches the cap rotates out. +pub(crate) fn convection_window_pick( + window: &mut VecDeque, + exclude: &NodeId, + count: usize, + now: u64, + is_live: impl Fn(&NodeId) -> bool, +) -> Vec { + convection_window_prune(window, now); + + // Newest first. + let ordered: Vec = window + .iter() + .rev() + .filter(|e| e.node_id != *exclude) + .map(|e| e.node_id) + .collect(); + // Still-connected callers first: their address is provably current, and + // they are the case where an introduction can be coordinated instead of a + // cold address handed over. + let mut chosen: Vec = ordered.iter().copied().filter(|n| is_live(n)).collect(); + for n in ordered { + if chosen.len() >= count { + break; + } + if !chosen.contains(&n) { + chosen.push(n); + } + } + chosen.truncate(count); + + let mut picked = Vec::with_capacity(chosen.len()); + for n in &chosen { + if let Some(e) = window.iter_mut().find(|e| e.node_id == *n) { + e.handouts += 1; + picked.push(e.clone()); + } + } + convection_window_prune(window, now); + picked +} + +/// Should this request be served? ENTRY is unconditional (round-5 ruling). +pub fn convection_should_serve( + class: crate::protocol::ConvectionClass, + viable_referrals: usize, + saturated: bool, +) -> bool { + if class.is_entry() { + // A node that cannot get in cannot come back, and a network that + // refuses re-entry partitions permanently. Always served — even with + // an empty window, where the mesh-peer supplement takes over. + return true; + } + viable_referrals > 0 && !saturated } /// Data gathered under brief lock for anchor probe I/O. @@ -659,14 +1007,16 @@ pub struct ConnectionManager { blob_store: Arc, /// Dedup map for worm queries: worm_id → timestamp_ms seen_worms: HashMap, - /// Last broadcast N1 set (for computing diffs) - last_n1_set: HashSet, - /// Last broadcast N2 set (for computing diffs) - last_n2_set: HashSet, - /// Max local (diverse) mesh slots - local_slots: usize, - /// Max wide (bloom-sourced) mesh slots - wide_slots: usize, + /// Digest of the last uniques announce we broadcast. The announce is always + /// a full snapshot; this is what lets us skip the send when nothing changed, + /// which recovers most of what incremental diffing would have bought. + last_announce_digest: u64, + /// The single mesh pool cap (v0.8: 20 desktop / 15 mobile) + mesh_slots: usize, + /// Temporary referral slots, allocated strictly ABOVE `mesh_slots` + temp_referral_slots: usize, + /// Deepest bounce WE retain (4 desktop / 3 mobile) + max_bounce: u8, /// Session connections: short-lived, tracked, separate from mesh slots sessions: HashMap, /// Max session slots @@ -688,12 +1038,25 @@ pub struct ConnectionManager { /// Peers known to be unreachable directly: node_id → last_failed_at_ms /// Learned over the session — cleared on restart, updated on connect success/failure unreachable_peers: HashMap, - /// Anchor-side referral list: connected peers available for referral - referral_list: HashMap, + /// Anchor-side rolling window of recent callers. Newest at the back. + /// In-memory only — no registration DB, no persistence. + convection_window: VecDeque, + /// Cached anchor-density prior for the stochastic trigger, refreshed on a + /// throttle so the hot disconnect path never pays for the query. + anchor_density: f64, + anchor_density_at_ms: u64, + /// AIMD feedback term on the anchor arm of the action choice. + anchor_bias: f64, + /// Anchors that recently refused us: node_id → penalty expiry (ms). + /// Shifts the *choice of anchor*, while `anchor_bias` shifts the choice of + /// arm — a single unhelpful anchor should not make us stop using anchors. + refused_anchors: HashMap, /// Channel to signal the growth loop to wake up and seek diverse peers growth_tx: Option>, /// Channel to signal the recovery loop when mesh drops below threshold recovery_tx: Option>, + /// Channel to signal the convection loop to ask an anchor for peers. + convection_tx: Option>, activity_log: Arc>, /// UPnP external address (prepended to self-reported addresses in anchor registration) upnp_external_addr: Option, @@ -711,7 +1074,10 @@ pub struct ConnectionManager { last_probe_success_ms: u64, /// Anchor probe: consecutive failure count probe_failure_streak: u8, - /// When this ConnectionManager was created + /// When this ConnectionManager was created. Kept for diagnostics; the + /// 2-hour uptime bar it used to gate anchor candidacy on was the retired + /// popularity model (round-2 ruling: candidacy = reachability + opt-in). + #[allow(dead_code)] started_at_ms: u64, /// Dedup map for anchor probes: probe_id → timestamp_ms seen_probes: HashMap<[u8; 16], u64>, @@ -719,9 +1085,11 @@ pub struct ConnectionManager { pub(crate) http_capable: bool, /// External HTTP address (ip:port) if known pub(crate) http_addr: Option, - /// Sticky N1 entries: NodeIds to report in N1 share until expiry (ms). - /// Used to advertise the bootstrap anchor for 24h after isolation recovery. - sticky_n1: HashMap, + /// Hand-out ledger for convection referrals SUPPLEMENTED from our live + /// mesh (node id → (count, window start ms)). Window entries have their own + /// `handouts` counter; supplemented peers had none, so a single peer could + /// be named to unlimited callers. + supplement_handouts: HashMap, /// NodeIds with an outgoing connect attempt currently in flight. /// Used by `try_begin_connect` to suppress duplicate concurrent outgoing /// connects from racing paths (auto-reconnect, rebalance, relay @@ -778,10 +1146,10 @@ impl ConnectionManager { diff_seq: AtomicU64::new(0), blob_store, seen_worms: HashMap::new(), - last_n1_set: HashSet::new(), - last_n2_set: HashSet::new(), - local_slots: profile.local_slots(), - wide_slots: profile.wide_slots(), + last_announce_digest: 0, + mesh_slots: profile.mesh_slots(), + temp_referral_slots: profile.temp_referral_slots(), + max_bounce: profile.max_bounce(), sessions: HashMap::new(), session_slots: profile.session_slots(), seen_intros: HashMap::new(), @@ -790,9 +1158,14 @@ impl ConnectionManager { session_relay_enabled, device_profile: profile, unreachable_peers: HashMap::new(), - referral_list: HashMap::new(), + convection_window: VecDeque::new(), + anchor_density: 0.0, + anchor_density_at_ms: 0, + anchor_bias: 1.0, + refused_anchors: HashMap::new(), growth_tx: None, recovery_tx: None, + convection_tx: None, activity_log, upnp_external_addr, observed_external_addr: std::sync::Mutex::new(None), @@ -808,7 +1181,7 @@ impl ConnectionManager { seen_probes: HashMap::new(), http_capable: false, http_addr: None, - sticky_n1: HashMap::new(), + supplement_handouts: HashMap::new(), pending_connects: Arc::new(std::sync::Mutex::new(HashSet::new())), } } @@ -854,52 +1227,37 @@ impl ConnectionManager { crate::types::NatProfile::new(self.nat_mapping, self.nat_filtering) } - /// Whether this node is a candidate for anchor status (has UPnP/public, enough connections, uptime) + /// Whether this node is a candidate for anchor status. + /// + /// v0.8 (round-2 ruling): candidacy is **stable direct reachability**, full + /// stop. The old ">=50 connections and 2h uptime" bar was the retired + /// popularity model — an anchor's job is to be dialable for network + /// (re)entry, and a node with 3 peers and a public address does that job + /// exactly as well as one with 80. Mobile is still excluded (battery and + /// roaming addresses, not popularity). pub fn is_anchor_candidate(&self) -> bool { - let now = now_ms(); - // Must have UPnP mapping or public IPv6 let has_public_addr = self.upnp_external_addr.is_some() || self.endpoint.addr().ip_addrs().any(|s| crate::network::is_publicly_routable(&s)); - if !has_public_addr { - return false; - } - // Must have enough connections (at least 50) - if self.connections.len() < 50 { - return false; - } - // Must have been running for at least 2 hours - let two_hours_ms = 2 * 60 * 60 * 1000; - if now.saturating_sub(self.started_at_ms) < two_hours_ms { - return false; - } - // Not mobile - if self.device_profile == crate::types::DeviceProfile::Mobile { - return false; - } - true + has_public_addr && self.device_profile != crate::types::DeviceProfile::Mobile } - /// Whether an anchor probe is due (candidate minus probe-success check, last probe > 30 min ago) + /// Whether an anchor self-verification probe is due: we are a candidate and + /// the last successful probe was more than 30 minutes ago. + /// + /// Driven by the convection loop's maintenance tick (and, on address + /// change, by the reachability watcher) now that the register cycle it used + /// to hang off is retired. pub fn probe_due(&self) -> bool { - let now = now_ms(); - let has_public_addr = self.upnp_external_addr.is_some() - || self.endpoint.addr().ip_addrs().any(|s| crate::network::is_publicly_routable(&s)); - if !has_public_addr { + if !self.is_anchor_candidate() { return false; } - if self.connections.len() < 50 { + // A probe needs a witness from the index and a reporter to route + // through — with no peers at all there is nobody to ask. + if self.connections.is_empty() { return false; } - let two_hours_ms = 2 * 60 * 60 * 1000; - if now.saturating_sub(self.started_at_ms) < two_hours_ms { - return false; - } - if self.device_profile == crate::types::DeviceProfile::Mobile { - return false; - } - // Last probe must be > 30 min ago let thirty_min_ms = 30 * 60 * 1000; - now.saturating_sub(self.last_probe_success_ms) > thirty_min_ms + now_ms().saturating_sub(self.last_probe_success_ms) > thirty_min_ms } /// Initiate an anchor self-verification probe. @@ -1324,9 +1682,18 @@ impl ConnectionManager { && !self.connections.contains_key(nid) && !self.is_likely_unreachable(nid) }) - .map(|(nid, reporter_count, in_n3)| { - let score = 1.0 / (reporter_count as f64) - + if !in_n3 { 0.3 } else { 0.0 }; + // Depth is a GRADED bonus, not a boolean. With the horizon widened + // to 2..4, `+0.3 unless deeper` let a bounce-4 ID seen by ONE + // reporter (1.0) outrank a genuine bounce-2 ID seen by three + // (0.33 + 0.3 = 0.63) — i.e. the growth loop preferentially picked + // the deepest, least-resolvable candidates. + .map(|(nid, reporter_count, shallowest)| { + let depth_bonus = match shallowest { + 0..=2 => 0.3, + 3 => 0.1, + _ => 0.0, + }; + let score = 1.0 / (reporter_count.max(1) as f64) + depth_bonus; (nid, score) }) .collect(); @@ -1335,69 +1702,104 @@ impl ConnectionManager { scored } - /// How many local slots are available for the growth loop to fill. - pub fn available_local_slots(&self) -> usize { - let local_count = self.count_kind(PeerSlotKind::Local); - self.local_slots.saturating_sub(local_count) + /// Live count of MESH-slot connections (temp referral slots excluded). + pub fn mesh_count(&self) -> usize { + self.connections.values().filter(|pc| pc.slot.is_mesh()).count() } - /// Accept an incoming connection, returning false if slots are full. + /// Live count of temporary referral slots. + pub fn temp_count(&self) -> usize { + self.connections.values().filter(|pc| pc.slot.is_temp()).count() + } + + /// How many mesh slots are free for the growth loop to fill. + /// + /// v0.8: this counts the WHOLE pool. Under the old Local/Wide split the + /// growth gate counted Local only while every outbound path hardcoded + /// Local — so growth stopped at the Local cap and the Wide slots could + /// only ever be filled by inbound connections. Collapsing to one pool + /// removes that bug structurally. + pub fn available_mesh_slots(&self) -> usize { + self.mesh_slots.saturating_sub(self.mesh_count()) + } + + /// Decide which slot class a new connection may occupy. + /// `None` = no room at all, refuse. + fn allocate_slot(&self, peer_id: &NodeId) -> Option { + if let Some(existing) = self.connections.get(peer_id) { + // Replacing a live connection keeps its class. + return Some(existing.slot); + } + decide_slot( + self.mesh_count(), + self.mesh_slots, + self.temp_count(), + self.temp_referral_slots, + now_ms(), + ) + } + + /// Accept an incoming connection. Returns the slot class it was given, or + /// `None` if both the mesh pool and the temp referral band are full. + /// + /// The caller MUST propagate `slot.is_mesh()` into the initial exchange: + /// a temp referral slot carries no uniques announce in either direction. pub fn accept_connection( &mut self, conn: iroh::endpoint::Connection, remote_node_id: NodeId, remote_addr: Option, - ) -> bool { + ) -> Option { if self.connections.contains_key(&remote_node_id) { debug!(peer = hex::encode(remote_node_id), "Replacing existing connection"); } - let total_slots = self.local_slots + self.wide_slots; - let total = self.connections.len(); - if total >= total_slots && !self.connections.contains_key(&remote_node_id) { - debug!(peer = hex::encode(remote_node_id), "Slots full, rejecting"); - return false; - } - - let now = now_ms(); - let slot_kind = if self.count_kind(PeerSlotKind::Local) < self.local_slots { - PeerSlotKind::Local - } else { - PeerSlotKind::Wide + let slot = match self.allocate_slot(&remote_node_id) { + Some(s) => s, + None => { + debug!(peer = hex::encode(remote_node_id), "Slots full (mesh + temp), rejecting"); + return None; + } }; + let now = now_ms(); self.connections.insert( remote_node_id, MeshConnection { node_id: remote_node_id, connection: conn, - slot_kind, + slot, + peer_depth: DEFAULT_PEER_DEPTH, connected_at: now, remote_addr: remote_addr.map(normalize_addr), last_activity: Arc::new(AtomicU64::new(now)), }, ); - // If peer was on referral list and reconnected, clear disconnected_at - self.mark_referral_reconnected(&remote_node_id); - - true + Some(slot) } /// Register an already-established connection into the mesh. /// The QUIC connect must happen OUTSIDE the conn_mgr lock to avoid blocking /// all other tasks (diff cycle, accept loop, keepalive, rebalance) during /// the potentially long connection timeout. + /// + /// The slot class is DECIDED here rather than supplied by the caller: every + /// outbound path used to pass `Local` unconditionally and none of them + /// checked capacity, so only inbound connections were ever capped. At 20 + /// slots that overshoot would be load-bearing. + /// Returns the slot the connection was placed in, or `None` if refused. pub async fn register_connection( &mut self, peer_id: NodeId, conn: iroh::endpoint::Connection, addrs: &[std::net::SocketAddr], - slot_kind: PeerSlotKind, - ) { + ) -> Option { let now = now_ms(); let connect_addr = addrs.first().copied(); + let slot = self.allocate_slot(&peer_id)?; + // Direct connect succeeded — mark peer as directly reachable self.mark_reachable(&peer_id); @@ -1406,27 +1808,31 @@ impl ConnectionManager { MeshConnection { node_id: peer_id, connection: conn, - slot_kind, + slot, + peer_depth: DEFAULT_PEER_DEPTH, connected_at: now, remote_addr: connect_addr.map(normalize_addr), last_activity: Arc::new(AtomicU64::new(now)), }, ); - // If peer was on referral list and reconnected, clear disconnected_at - self.mark_referral_reconnected(&peer_id); - - // Persist address to peers table so it survives restart + // Persist address to peers table so it survives restart. + // bugs-fixed #2: ALWAYS do this, temp referral slots included — the + // `peers` table is addresses, `mesh_peers` is knowledge membership. if !addrs.is_empty() { let storage = self.storage.get().await; let _ = storage.upsert_peer(&peer_id, addrs, None); drop(storage); } - // Record in mesh_peers table + touch social route + // Record in mesh_peers table + touch social route. + // Temp referral slots are deliberately NOT written to mesh_peers — + // that omission is what keeps them out of our uniques announce. { let storage = self.storage.get().await; - let _ = storage.add_mesh_peer(&peer_id, slot_kind, 0); + if slot.is_mesh() { + let _ = storage.add_mesh_peer(&peer_id); + } if storage.has_social_route(&peer_id).unwrap_or(false) { let _ = storage.touch_social_route_connect(&peer_id, addrs, ReachMethod::Direct); // Gather watcher data before dropping storage @@ -1443,7 +1849,8 @@ impl ConnectionManager { } } - info!(peer = hex::encode(peer_id), kind = %slot_kind, "Connected to peer"); + info!(peer = hex::encode(peer_id), slot = %slot, "Connected to peer"); + Some(slot) } /// Establish an outgoing mesh connection with a 15s timeout on the QUIC connect. @@ -1454,12 +1861,18 @@ impl ConnectionManager { peer_id: NodeId, conn: iroh::endpoint::Connection, addrs: &[std::net::SocketAddr], - slot_kind: PeerSlotKind, - ) { - if self.connections.contains_key(&peer_id) { - return; // Already connected + ) -> Option { + if let Some(pc) = self.connections.get(&peer_id) { + return Some(pc.slot); // Already connected + } + self.register_connection(peer_id, conn, addrs).await + } + + /// Record a peer's advertised retention depth (from their announce). + pub fn set_peer_depth(&mut self, peer_id: &NodeId, depth: u8) { + if let Some(pc) = self.connections.get_mut(peer_id) { + pc.peer_depth = depth.clamp(2, 4); } - self.register_connection(peer_id, conn, addrs, slot_kind).await; } /// QUIC connect with 15s timeout — call this OUTSIDE the conn_mgr lock. @@ -1476,9 +1889,16 @@ impl ConnectionManager { Ok(conn) } - /// Pull posts from a peer — standalone version that doesn't require conn_mgr lock. - /// Takes a cloned connection and storage Arc. - pub async fn pull_from_peer_unlocked( + /// TRANSITIONAL content sync from a peer (0x46/0x47) — standalone version + /// that doesn't require the conn_mgr lock. Takes a cloned connection and + /// storage Arc. + /// + /// This is NOT the pull. A pull is the uniques-index exchange + /// (`uniques_pull_unlocked`). Content moves here only until Iteration D + /// gives the CDN a carrier for non-public visibilities — `PostFetchRequest` + /// (0xD4) serves Public posts only, so DMs, group posts, Friends and + /// FoFClosed posts have no other path today. + pub async fn content_sync_unlocked( conn: iroh::endpoint::Connection, storage: &Arc, peer_id: &NodeId, @@ -1503,20 +1923,20 @@ impl ConnectionManager { } } - let request = PullSyncRequestPayload { + let request = ContentSyncRequestPayload { follows: query_list, since_ms: follows_sync, }; let (mut send, mut recv) = conn.open_bi().await?; - write_typed_message(&mut send, MessageType::PullSyncRequest, &request).await?; + write_typed_message(&mut send, MessageType::ContentSyncRequest, &request).await?; send.finish()?; let msg_type = read_message_type(&mut recv).await?; - if msg_type != MessageType::PullSyncResponse { - anyhow::bail!("expected PullSyncResponse, got {:?}", msg_type); + if msg_type != MessageType::ContentSyncResponse { + anyhow::bail!("expected ContentSyncResponse, got {:?}", msg_type); } - let response: PullSyncResponsePayload = read_payload(&mut recv, MAX_PAYLOAD).await?; + let response: ContentSyncResponsePayload = read_payload(&mut recv, MAX_PAYLOAD).await?; let mut posts_received = 0; let mut new_post_ids: Vec = Vec::new(); @@ -1646,459 +2066,133 @@ impl ConnectionManager { Ok(updated) } - /// Do the initial exchange after connecting: N1/N2 node lists + profile + deletes + peer addresses (both directions). - pub async fn do_initial_exchange( - &self, - conn: &iroh::endpoint::Connection, - remote_node_id: NodeId, - ) -> anyhow::Result<()> { - // Build our payload - let our_payload = { - let storage = self.storage.get().await; - let n1 = storage.build_n1_share()?; - let n2 = storage.build_n2_share()?; - // Profile keyed by network id (used for N1/N2/N3 routing). - // Strip persona display data before sending so peers don't learn - // a human-readable name for our network id. - let profile = storage.get_profile(&self.our_node_id)? - .map(|p| p.sanitized_for_network_broadcast()); - let deletes = storage.list_delete_records()?; - let post_ids = storage.list_post_ids()?; - let peer_addresses = storage.build_peer_addresses_for(&self.our_node_id)?; - let our_profile = crate::types::NatProfile::from_nat_type(self.nat_type); - InitialExchangePayload { - n1_node_ids: n1, - n2_node_ids: n2, - profile, - deletes, - post_ids, - peer_addresses, - anchor_addr: self.build_anchor_advertised_addr(), - your_observed_addr: None, - nat_type: Some(self.nat_type.to_string()), - nat_mapping: Some(our_profile.mapping.to_string()), - nat_filtering: Some(our_profile.filtering.to_string()), - http_capable: self.http_capable, - http_addr: self.http_addr.clone(), - device_role: None, - cache_pressure: None, - duplicate_active: None, - } + /// Snapshot everything the uniques exchange needs from connection state, so + /// the (bulk, slow) storage half can run with the ConnectionManager mutex + /// released. See `handle_uniques_pull`. + pub(crate) fn uniques_ctx(&self, peer: &NodeId) -> UniquesCtx { + let (is_mesh, peer_depth) = match self.connections.get(peer) { + Some(pc) => (pc.slot.is_mesh(), pc.peer_depth), + // Not a mesh connection at all (ephemeral/session) — no knowledge + // crosses. + None => (false, DEFAULT_PEER_DEPTH), }; - - // Open bi-stream for initial exchange - let (mut send, mut recv) = conn.open_bi().await?; - - // Send our payload - write_typed_message(&mut send, MessageType::InitialExchange, &our_payload).await?; - send.finish()?; - - // Read their payload - let msg_type = read_message_type(&mut recv).await?; - if msg_type != MessageType::InitialExchange { - anyhow::bail!("expected InitialExchange, got {:?}", msg_type); + UniquesCtx { + storage: Arc::clone(&self.storage), + our_node_id: self.our_node_id, + max_bounce: self.max_bounce, + anchor_addr: self.build_anchor_advertised_addr(), + seq: self.diff_seq.fetch_add(1, Ordering::Relaxed) + 1, + is_mesh, + peer_depth, + connected: self.connections.keys().copied().collect(), } - let their_payload: InitialExchangePayload = read_payload(&mut recv, MAX_PAYLOAD).await?; - - // Process their data - let storage = self.storage.get().await; - - // Their N1 → our N2 (tagged to this reporter) - // Filter out our own ID and already-connected peers (they'd waste candidate slots) - let filtered_n1: Vec = their_payload.n1_node_ids.iter() - .filter(|nid| **nid != self.our_node_id && !self.connections.contains_key(*nid)) - .copied() - .collect(); - storage.set_peer_n1(&remote_node_id, &filtered_n1)?; - debug!(peer = hex::encode(remote_node_id), raw = their_payload.n1_node_ids.len(), stored = filtered_n1.len(), "Stored peer N1 as our N2 (filtered self+mesh)"); - - // Their N2 → our N3 (tagged to this reporter) - let filtered_n2: Vec = their_payload.n2_node_ids.iter() - .filter(|nid| **nid != self.our_node_id) - .copied() - .collect(); - storage.set_peer_n2(&remote_node_id, &filtered_n2)?; - debug!(peer = hex::encode(remote_node_id), raw = their_payload.n2_node_ids.len(), stored = filtered_n2.len(), "Stored peer N2 as our N3 (filtered self)"); - - // Store their profile - if let Some(profile) = their_payload.profile { - let _ = storage.store_profile(&profile); - } - - // Process delete records - for dr in their_payload.deletes { - if crypto::verify_delete_signature(&dr.author, &dr.post_id, &dr.signature) { - let _ = storage.store_delete(&dr); - let _ = storage.apply_delete(&dr); - } - } - - // Store their peer_addresses (N+10:Addresses) - for pa in &their_payload.peer_addresses { - if let Ok(nid) = crate::parse_node_id_hex(&pa.n) { - let addrs: Vec = pa.a.iter().filter_map(|a| a.parse().ok()).collect(); - if !addrs.is_empty() { - let _ = storage.upsert_peer(&nid, &addrs, None); - } - } - } - - // Record replicas from overlapping post_ids - let our_post_ids: HashSet<[u8; 32]> = storage.list_post_ids()?.into_iter().collect(); - for pid in &their_payload.post_ids { - if our_post_ids.contains(pid) { - let _ = storage.record_replica(pid, &remote_node_id); - } - } - - // Process anchor's advertised address - if let Some(ref anchor_addr_str) = their_payload.anchor_addr { - if let Ok(sock) = anchor_addr_str.parse::() { - let _ = storage.upsert_known_anchor(&remote_node_id, &[sock]); - let _ = storage.upsert_peer(&remote_node_id, &[sock], None); - info!(peer = hex::encode(remote_node_id), addr = %sock, "Stored anchor's advertised address"); - } - } - - // Store observed address (STUN-like feedback from peer) - if let Some(ref observed) = their_payload.your_observed_addr { - info!(observed_addr = %observed, reporter = hex::encode(remote_node_id), "Peer reports our address as"); - if let Ok(addr) = observed.parse::() { - *self.observed_external_addr.lock().unwrap() = Some(addr); - } - } - - // Store peer's NAT type - if let Some(ref nat_str) = their_payload.nat_type { - let nat = crate::types::NatType::from_str_label(nat_str); - let _ = storage.set_peer_nat_type(&remote_node_id, nat); - } - - // Store peer's NAT profile (mapping + filtering) if provided - if their_payload.nat_mapping.is_some() || their_payload.nat_filtering.is_some() { - let mapping = their_payload.nat_mapping.as_deref() - .map(crate::types::NatMapping::from_str_label) - .unwrap_or(crate::types::NatMapping::Unknown); - let filtering = their_payload.nat_filtering.as_deref() - .map(crate::types::NatFiltering::from_str_label) - .unwrap_or(crate::types::NatFiltering::Unknown); - let profile = crate::types::NatProfile::new(mapping, filtering); - let _ = storage.set_peer_nat_profile(&remote_node_id, &profile); - } - - Ok(()) } - /// Handle the responder side of initial exchange. - pub async fn handle_initial_exchange( - &self, - mut send: iroh::endpoint::SendStream, + /// Responder for 0x40 — the uniques-index exchange. + /// + /// design.html §sync: a pull is not a post transfer. It is the peer's list + /// of unique IDs they have a live path to — "if you want these IDs, talk to + /// me and I'll help you find them". Symmetric: their pools arrive in the + /// request, ours go back in the response, so one round trip refreshes both + /// indexes. Content travels via the CDN (and, until Iteration D, via the + /// transitional 0x46/0x47 content sync). + pub async fn handle_uniques_pull( + conn_mgr: &Arc>, mut recv: iroh::endpoint::RecvStream, + mut send: iroh::endpoint::SendStream, remote_node_id: NodeId, ) -> anyhow::Result<()> { - // Read their payload (message type byte already consumed by caller) - let their_payload: InitialExchangePayload = read_payload(&mut recv, MAX_PAYLOAD).await?; + let request: UniquesPullPayload = read_payload(&mut recv, MAX_UNIQUES_PAYLOAD).await?; - // Build and send our payload - let our_payload = { - let storage = self.storage.get().await; - let n1 = storage.build_n1_share()?; - let n2 = storage.build_n2_share()?; - // Profile keyed by network id (used for N1/N2/N3 routing). - // Strip persona display data before sending so peers don't learn - // a human-readable name for our network id. - let profile = storage.get_profile(&self.our_node_id)? - .map(|p| p.sanitized_for_network_broadcast()); - let deletes = storage.list_delete_records()?; - let post_ids = storage.list_post_ids()?; - let peer_addresses = storage.build_peer_addresses_for(&self.our_node_id)?; - let our_profile = crate::types::NatProfile::from_nat_type(self.nat_type); - InitialExchangePayload { - n1_node_ids: n1, - n2_node_ids: n2, - profile, - deletes, - post_ids, - peer_addresses, - anchor_addr: self.build_anchor_advertised_addr(), - your_observed_addr: None, // no remote_addr available in this method - nat_type: Some(self.nat_type.to_string()), - nat_mapping: Some(our_profile.mapping.to_string()), - nat_filtering: Some(our_profile.filtering.to_string()), - http_capable: self.http_capable, - http_addr: self.http_addr.clone(), - device_role: None, - cache_pressure: None, - duplicate_active: None, - } + // Gather the SMALL bits of connection state under a brief lock, then + // drop it and do every storage operation against the pool directly — + // the `gather_anchor_probe_data` / `run_anchor_probe_unlocked` pattern. + // + // Both the build and the apply are bulk SQLite work (the apply is a + // per-row upsert loop sized in thousands). Doing them under the + // ConnectionManager mutex blocked slot allocation, disconnects, + // referral serving and relay introductions for the whole write, once + // per pull per peer across ~20 mesh peers. + // + // Knowledge boundary: a temp referral slot exchanges no knowledge in + // either direction. We answer (so the caller isn't left hanging) but + // with `uniques: None`, and we drop whatever they sent. + let ctx = { + let cm = conn_mgr.lock().await; + cm.uniques_ctx(&remote_node_id) }; - write_typed_message(&mut send, MessageType::InitialExchange, &our_payload).await?; + let ours = if ctx.is_mesh { + let depth = request.uniques.as_ref().map(|u| u.depth).unwrap_or(ctx.peer_depth); + ctx.build(depth).await + } else { + None + }; + + // Write BEFORE applying theirs — applying is a storage write of up to + // a few thousand rows and the caller should not wait on it. + let response = UniquesPullPayload { uniques: ours }; + write_typed_message(&mut send, MessageType::UniquesPullResponse, &response).await?; send.finish()?; - // Process their data - let storage = self.storage.get().await; - - // Their N1 → our N2 (filter out self + already-connected peers) - let filtered_n1: Vec = their_payload.n1_node_ids.iter() - .filter(|nid| **nid != self.our_node_id && !self.connections.contains_key(*nid)) - .copied() - .collect(); - storage.set_peer_n1(&remote_node_id, &filtered_n1)?; - debug!(peer = hex::encode(remote_node_id), raw = their_payload.n1_node_ids.len(), stored = filtered_n1.len(), "Stored peer N1 as our N2 (filtered self+mesh)"); - - // Their N2 → our N3 (filter out self) - let filtered_n2: Vec = their_payload.n2_node_ids.iter() - .filter(|nid| **nid != self.our_node_id) - .copied() - .collect(); - storage.set_peer_n2(&remote_node_id, &filtered_n2)?; - - if let Some(profile) = their_payload.profile { - let _ = storage.store_profile(&profile); - } - - for dr in their_payload.deletes { - if crypto::verify_delete_signature(&dr.author, &dr.post_id, &dr.signature) { - let _ = storage.store_delete(&dr); - let _ = storage.apply_delete(&dr); - } - } - - // Store their peer_addresses (N+10:Addresses) - for pa in &their_payload.peer_addresses { - if let Ok(nid) = crate::parse_node_id_hex(&pa.n) { - let addrs: Vec = pa.a.iter().filter_map(|a| a.parse().ok()).collect(); - if !addrs.is_empty() { - let _ = storage.upsert_peer(&nid, &addrs, None); + if ctx.is_mesh { + if let Some(theirs) = request.uniques { + let applied = ctx.apply(&remote_node_id, &theirs).await?; + let mut cm = conn_mgr.lock().await; + cm.set_peer_depth(&remote_node_id, theirs.depth); + if applied > 0 { + debug!(peer = hex::encode(remote_node_id), applied, "Uniques pull: applied peer index"); + cm.notify_growth(); } } } - - let our_post_ids: HashSet<[u8; 32]> = storage.list_post_ids()?.into_iter().collect(); - for pid in &their_payload.post_ids { - if our_post_ids.contains(pid) { - let _ = storage.record_replica(pid, &remote_node_id); - } - } - Ok(()) } - /// Compute N1/N2 changes and broadcast NodeListUpdate to all connected peers. - pub async fn broadcast_routing_diff(&mut self) -> anyhow::Result { - let seq = self.diff_seq.fetch_add(1, Ordering::Relaxed) + 1; - - let (current_n1, current_n2) = { - let storage = self.storage.get().await; - let n1: HashSet = storage.build_n1_share()?.into_iter().collect(); - let n2: HashSet = storage.build_n2_share()?.into_iter().collect(); - (n1, n2) - }; - - let n1_added: Vec = current_n1.difference(&self.last_n1_set).copied().collect(); - let n1_removed: Vec = self.last_n1_set.difference(¤t_n1).copied().collect(); - let n2_added: Vec = current_n2.difference(&self.last_n2_set).copied().collect(); - let n2_removed: Vec = self.last_n2_set.difference(¤t_n2).copied().collect(); - - if n1_added.is_empty() && n1_removed.is_empty() && n2_added.is_empty() && n2_removed.is_empty() { - return Ok(0); - } - - let payload = NodeListUpdatePayload { - seq, - n1_added, - n1_removed, - n2_added, - n2_removed, - }; - let mut sent_count = 0; - - for (peer_id, pc) in &self.connections { - let result = async { - let mut send = pc.connection.open_uni().await?; - write_typed_message(&mut send, MessageType::NodeListUpdate, &payload).await?; - send.finish()?; - anyhow::Ok(()) - } - .await; - - match result { - Ok(()) => sent_count += 1, - Err(e) => { - debug!( - peer = hex::encode(peer_id), - error = %e, - "Failed to send node list update" - ); - } - } - } - - // Update last sets - self.last_n1_set = current_n1; - self.last_n2_set = current_n2; - - Ok(sent_count) - } - - /// Process a node list update from a peer: their N1 changes → our N2, their N2 changes → our N3. - pub async fn process_routing_diff( - &self, - reporter: &NodeId, - diff: NodeListUpdatePayload, + /// Client side of the uniques-index exchange — standalone, no conn_mgr lock + /// held across I/O. Returns how many index rows the peer's pools produced. + pub async fn uniques_pull_unlocked( + conn_mgr: &Arc>, + conn: iroh::endpoint::Connection, + peer_id: &NodeId, ) -> anyhow::Result { - let storage = self.storage.get().await; - let mut count = 0; - - // Their N1 added → add to our N2 (filter self + already-connected) - if !diff.n1_added.is_empty() { - let filtered: Vec = diff.n1_added.iter() - .filter(|nid| **nid != self.our_node_id && !self.connections.contains_key(*nid)) - .copied() - .collect(); - if !filtered.is_empty() { - storage.add_peer_n1(reporter, &filtered)?; - } - count += filtered.len(); - } - - // Their N1 removed → remove from our N2 - if !diff.n1_removed.is_empty() { - storage.remove_peer_n1(reporter, &diff.n1_removed)?; - count += diff.n1_removed.len(); - } - - // Their N2 added → add to our N3 (filter self) - if !diff.n2_added.is_empty() { - let filtered: Vec = diff.n2_added.iter() - .filter(|nid| **nid != self.our_node_id) - .copied() - .collect(); - if !filtered.is_empty() { - storage.add_peer_n2(reporter, &filtered)?; - } - count += filtered.len(); - } - - // Their N2 removed → remove from our N3 - if !diff.n2_removed.is_empty() { - storage.remove_peer_n2(reporter, &diff.n2_removed)?; - count += diff.n2_removed.len(); - } - - // Update last_diff_seq - let _ = storage.update_mesh_peer_seq(reporter, diff.seq); - - Ok(count) - } - - /// Pull posts from a connected peer. - pub async fn pull_from_peer(&self, peer_id: &NodeId) -> anyhow::Result { - let pc = self - .connections - .get(peer_id) - .ok_or_else(|| anyhow::anyhow!("not connected to {}", hex::encode(peer_id)))?; - - let (our_follows, follows_sync, our_personas) = { - let storage = self.storage.get().await; - ( - storage.list_follows()?, - storage.get_follows_with_last_sync().unwrap_or_default(), - storage.list_posting_identities().unwrap_or_default(), - ) + let ctx = { + let cm = conn_mgr.lock().await; + cm.uniques_ctx(peer_id) }; - - // Merged pull: include every posting identity so DMs match recipient. - let mut query_list = our_follows; - for pi in &our_personas { - if !query_list.contains(&pi.node_id) { - query_list.push(pi.node_id); - } + if !ctx.is_mesh { + anyhow::bail!("uniques pull only runs on mesh slots"); } - let request = PullSyncRequestPayload { - follows: query_list, - since_ms: follows_sync, - }; + // Built with the lock RELEASED — see `handle_uniques_pull`. + let ours = ctx.build(ctx.peer_depth).await; - let (mut send, mut recv) = pc.connection.open_bi().await?; - write_typed_message(&mut send, MessageType::PullSyncRequest, &request).await?; + let (mut send, mut recv) = conn.open_bi().await?; + write_typed_message( + &mut send, + MessageType::UniquesPullRequest, + &UniquesPullPayload { uniques: ours }, + ) + .await?; send.finish()?; let msg_type = read_message_type(&mut recv).await?; - if msg_type != MessageType::PullSyncResponse { - anyhow::bail!("expected PullSyncResponse, got {:?}", msg_type); + if msg_type != MessageType::UniquesPullResponse { + anyhow::bail!("expected UniquesPullResponse, got {:?}", msg_type); } - let response: PullSyncResponsePayload = read_payload(&mut recv, MAX_PAYLOAD).await?; + let response: UniquesPullPayload = read_payload(&mut recv, MAX_UNIQUES_PAYLOAD).await?; - let mut posts_received = 0; - let mut new_post_ids: Vec = Vec::new(); - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64; - - // Brief lock 1: store posts - let mut synced_authors: HashSet = HashSet::new(); - { - let storage = self.storage.get().await; - for sp in &response.posts { - if storage.is_deleted(&sp.id)? { - continue; - } - if verify_post_id(&sp.id, &sp.post) { - match crate::control::receive_post(&storage, &sp.id, &sp.post, &sp.visibility, sp.intent.as_ref()) { - Ok(true) => { - new_post_ids.push(sp.id); - posts_received += 1; - synced_authors.insert(sp.post.author); - } - Ok(false) => { - synced_authors.insert(sp.post.author); - } - Err(e) => { - warn!(post_id = hex::encode(sp.id), error = %e, "rejecting post"); - } - } - } - } + let theirs = match response.uniques { + Some(t) => t, + None => return Ok(0), + }; + let applied = ctx.apply(peer_id, &theirs).await?; + let mut cm = conn_mgr.lock().await; + cm.set_peer_depth(peer_id, theirs.depth); + if applied > 0 { + cm.notify_growth(); } - // Lock RELEASED - - // Brief lock 2: upstream + last_sync - { - let storage = self.storage.get().await; - for pid in &new_post_ids { - let _ = storage.touch_file_holder( - pid, - peer_id, - &[], - crate::storage::HolderDirection::Received, - ); - } - for author in &synced_authors { - let _ = storage.update_follow_last_sync(author, now_ms); - } - } - - // Register as downstream with the sender for new posts (cap at 50 to avoid flooding) - if !new_post_ids.is_empty() { - let conn = pc.connection.clone(); - tokio::spawn(async move { - for post_id in new_post_ids.into_iter().take(50) { - let payload = PostDownstreamRegisterPayload { post_id }; - if let Ok(mut send) = conn.open_uni().await { - let _ = write_typed_message(&mut send, MessageType::PostDownstreamRegister, &payload).await; - let _ = send.finish(); - } - } - }); - } - - Ok(PullSyncStats { - posts_received, - }) + Ok(applied) } /// Fetch engagement headers (reactions, comments, policies) for posts due for check from a peer. @@ -2208,18 +2302,20 @@ impl ConnectionManager { Ok(updated) } - /// Handle an incoming pull request from a peer. - /// Handle a pull sync request — no conn_mgr lock needed, only storage. + /// Handle a TRANSITIONAL content sync request (0x46) — no conn_mgr lock + /// needed, only storage. `should_send_post` is the ONLY non-public content + /// gate in the tree and it is wired only here; do not lose it when + /// Iteration D moves content onto the CDN. /// (`_our_node_id` is the NETWORK id; own-post checks inside use posting /// identities from storage instead.) - pub async fn handle_pull_request_unlocked( + pub async fn handle_content_sync_request_unlocked( storage: &StoragePool, _our_node_id: NodeId, remote_node_id: NodeId, mut recv: iroh::endpoint::RecvStream, mut send: iroh::endpoint::SendStream, ) -> anyhow::Result<()> { - let request: PullSyncRequestPayload = read_payload(&mut recv, MAX_PAYLOAD).await?; + let request: ContentSyncRequestPayload = read_payload(&mut recv, MAX_PAYLOAD).await?; let their_follows: HashSet = request.follows.into_iter().collect(); @@ -2269,11 +2365,11 @@ impl ConnectionManager { posts_to_send }; - let response = PullSyncResponsePayload { + let response = ContentSyncResponsePayload { posts, }; - write_typed_message(&mut send, MessageType::PullSyncResponse, &response).await?; + write_typed_message(&mut send, MessageType::ContentSyncResponse, &response).await?; send.finish()?; Ok(()) @@ -2302,35 +2398,24 @@ impl ConnectionManager { } } - // N2 lookup: ask tagged reporter for address - let n2_reporters = { + // Referral cascade over the whole stored horizon, shallowest first: + // N2 (bounce 2) → N3 → N4. N4 is USED here even though it is never + // re-announced — that is precisely the "terminal, but not useless" + // property from design.html §layers. + let reporters: Vec = { let storage = self.storage.get().await; - storage.find_in_n2(target)? - }; - for reporter in &n2_reporters { - if let Some(pc) = self.connections.get(reporter) { - let result = async { - let (mut send, mut recv) = pc.connection.open_bi().await?; - let req = crate::protocol::AddressRequestPayload { target: *target }; - write_typed_message(&mut send, MessageType::AddressRequest, &req).await?; - send.finish()?; - let _resp_type = read_message_type(&mut recv).await?; - let resp: crate::protocol::AddressResponsePayload = - read_payload(&mut recv, MAX_PAYLOAD).await?; - anyhow::Ok(resp.address) - }.await; - if let Ok(Some(addr)) = result { - return Ok(Some(addr)); + let mut seen: HashSet = HashSet::new(); + let mut ordered = Vec::new(); + for bounce in 2u8..=4 { + for r in storage.find_reporters_at(target, bounce).unwrap_or_default() { + if seen.insert(r) { + ordered.push(r); + } } } - } - - // N3 lookup: ask tagged reporter (chains one more hop) - let n3_reporters = { - let storage = self.storage.get().await; - storage.find_in_n3(target)? + ordered }; - for reporter in &n3_reporters { + for reporter in &reporters { if let Some(pc) = self.connections.get(reporter) { let result = async { let (mut send, mut recv) = pc.connection.open_bi().await?; @@ -2529,7 +2614,7 @@ impl ConnectionManager { // Check N2/N3 let storage = self.storage.get().await; - let found_entries = storage.find_any_in_n2_n3(all_needles)?; + let found_entries = storage.find_any_reachable(all_needles)?; if let Some((found_id, _reporter, _level)) = found_entries.first() { drop(storage); let address = self.resolve_address(found_id).await.unwrap_or(None); @@ -2870,7 +2955,7 @@ impl ConnectionManager { } if found.is_none() { let storage = self.storage.get().await; - let entries = storage.find_any_in_n2_n3(&all_needles)?; + let entries = storage.find_any_reachable(&all_needles)?; if let Some((found_id, _reporter, _level)) = entries.first() { drop(storage); let address = self.resolve_address(found_id).await.unwrap_or(None); @@ -2988,38 +3073,26 @@ impl ConnectionManager { Ok(()) } - /// Pick a random wide-connected peer to include as a referral in worm responses. - /// Returns (node_id, address_string) if a suitable peer with a known address is found. + /// Pick a random mesh peer with a known address, to include as a referral + /// in worm responses. + /// + /// v0.8: "prefer Wide" is meaningless with one pool. Temporary referral + /// slots are excluded — we should never point a stranger at a peer we + /// ourselves hold only provisionally. async fn pick_random_wide_referral(&self, exclude: &NodeId) -> Option<(NodeId, String)> { - // Prefer wide-slot peers, but any connected peer with a known address works - let candidates: Vec<(NodeId, PeerSlotKind)> = self + let candidates: Vec = self .connections .iter() - .filter(|(nid, _)| *nid != exclude && **nid != self.our_node_id) - .map(|(nid, pc)| (*nid, pc.slot_kind)) + .filter(|(nid, pc)| *nid != exclude && **nid != self.our_node_id && pc.slot.is_mesh()) + .map(|(nid, _)| *nid) .collect(); if candidates.is_empty() { return None; } - // Prefer wide peers - let wide: Vec<_> = candidates - .iter() - .filter(|(_, kind)| *kind == PeerSlotKind::Wide) - .collect(); - - // Shuffle candidates to pick randomly - let ordered: Vec<&(NodeId, PeerSlotKind)> = if !wide.is_empty() { - wide - } else { - candidates.iter().collect() - }; - - // Try each candidate until we find one with a known address let storage = self.storage.get().await; - for candidate in ordered { - let nid = candidate.0; + for nid in candidates { if let Ok(Some(rec)) = storage.get_peer_record(&nid) { if let Some(addr) = rec.addresses.first() { return Some((nid, addr.to_string())); @@ -3030,44 +3103,116 @@ impl ConnectionManager { None } - /// Disconnect a peer, cleaning up N2/N3 entries. + /// Disconnect a peer. + /// + /// The departing peer's index rows are DELIBERATELY RETAINED. Round-4/5 + /// ruling and design.html §anchors: "Pool knowledge is overwritten memory: + /// a slot's contributed entries are wiped only when a new handshake + /// replaces them — so a node knocked down from 20 peers to 4 still holds 16 + /// dead peers' pools full of anchor addresses to reconnect through." + /// Wiping on disconnect defeated the exact recovery scenario the design was + /// written for: a mass disconnect emptied the index, so `list_pool_anchors` + /// (phase 0 of the anchor mine) and `score_n2_candidates_batch` both + /// returned nothing precisely when the node needed them most, and + /// `anchor_density` collapsed toward 0 — steering the adaptive weights + /// toward mesh introductions at the moment the mesh was gone. + /// + /// Retention is safe without any new bookkeeping: `set_reach` DELETEs per + /// (reporter, bounce) before inserting, so a returning peer's rows are + /// REPLACED rather than duplicated, and the 5h `prune_reach` sweep in + /// `rebalance_slots` ages out anything genuinely stale. pub async fn disconnect_peer(&mut self, peer_id: &NodeId) { + // The slot class decides whether this teardown rolls the growth dice. + // Captured BEFORE the remove. + let was_mesh = self + .connections + .get(peer_id) + .map(|pc| pc.slot.is_mesh()) + .unwrap_or(false); if let Some(pc) = self.connections.remove(peer_id) { drop(pc); } - // Mark disconnected in referral list (anchor-side) - self.mark_referral_disconnected(peer_id); - let storage = self.storage.get().await; - // Remove their N2 contributions (their N1 share → our N2) - let _ = storage.clear_peer_n2(peer_id); - // Remove their N3 contributions (their N2 share → our N3) - let _ = storage.clear_peer_n3(peer_id); // Remove from active mesh peers (but keep in peers table for reconnection) let _ = storage.remove_mesh_peer(peer_id); // Mark social route as disconnected if storage.has_social_route(peer_id).unwrap_or(false) { let _ = storage.set_social_route_status(peer_id, SocialStatus::Disconnected); } + // Piggyback the anchor-density prior on a guard we already hold, on a + // throttle — the stochastic roll below must never trigger a + // COUNT DISTINCT of its own on this hot path. + let fresh_density = if self.anchor_density_refresh_due() { + storage.anchor_density().ok() + } else { + None + }; + + // Release the storage guard BEFORE graduation — StoragePool is a + // non-reentrant tokio Mutex, so re-locking under it would deadlock. + drop(storage); + if let Some(d) = fresh_density { + self.set_anchor_density(d); + } + + // A freed mesh slot is the trigger for temp referral graduation. + // Doing it here rather than only in the 10-minute rebalance means a + // temp peer is promoted the moment room appears. + self.graduate_temp_referrals().await; let remaining = self.connections.len(); debug!(peer = hex::encode(peer_id), remaining, "Disconnected peer"); self.log_activity(ActivityLevel::Info, ActivityCategory::Connection, format!("Disconnected, {} remaining", remaining), Some(*peer_id)); - // If mesh is completely empty, trigger immediate recovery - if self.connections.is_empty() { - info!("Mesh empty, triggering recovery"); - self.log_activity(ActivityLevel::Warn, ActivityCategory::Connection, "Mesh empty".into(), None); - self.notify_recovery(); + // The ruling is "on EACH MESH-PEER disconnect". A temp referral slot + // expiring (5 min TTL, up to 10 of them on desktop) is not mesh churn, + // and letting temp churn roll the dice would inflate convection and + // growth traffic well past the rate the design intends. Graduation + // above still runs for every teardown — a freed mesh slot is a freed + // mesh slot however it was freed. + if !was_mesh { + return; } - // Signal growth loop to fill the empty slot (don't wait 10min for rebalance) - let total_slots = self.local_slots + self.wide_slots; - if remaining < total_slots { - self.notify_growth(); + // RECOVERY stays non-stochastic: below 2 mesh peers a node cannot + // function, so it always acts immediately. The threshold counts MESH + // peers, not connections — a pool of temp referral slots is not a mesh. + match disconnect_response(self.mesh_count(), self.available_mesh_slots()) { + DisconnectResponse::Recover => { + info!(mesh = self.mesh_count(), "Mesh below 2, triggering recovery"); + self.log_activity(ActivityLevel::Warn, ActivityCategory::Connection, "Mesh below 2".into(), None); + self.notify_recovery(); + return; + } + DisconnectResponse::Idle => return, + DisconnectResponse::Roll => {} } + // STOCHASTIC per-disconnect action (round-4/5). No threshold, no jitter + // timer: the randomness itself de-synchronises the network's response + // to a shared disconnect event, which is what a timer with jitter was + // only approximating. Weights are adaptive — anchor-density prior plus + // refusal feedback — and lean toward acting the further under target we + // are. + // + // The dice are rolled here, under the lock (pure state read + RNG); the + // resulting network action is dispatched by a loop on the other end of + // a channel, never inline. `disconnect_peer` is on the hot path of + // every teardown. + match self.roll_convection_action() { + ConvectionAction::Nothing => { + debug!("Convection: rolled do-nothing on disconnect"); + } + ConvectionAction::AskAnchor => { + debug!("Convection: rolled anchor introduction on disconnect"); + self.notify_convection(); + } + ConvectionAction::AskMesh => { + debug!("Convection: rolled mesh introduction on disconnect"); + self.notify_growth(); + } + } } /// Notify watchers that a previously disconnected peer has reconnected. @@ -3128,18 +3273,78 @@ impl ConnectionManager { Ok(reply) } - /// Rebalance connection slots: remove dead connections, prune stale N2/N3 entries. + /// Promote temporary referral slots into freed mesh slots, oldest first. + /// + /// Graduation is in-place: the connection already lives in `connections`, + /// so there is no new QUIC connect, no reconnect, and no protocol round + /// trip. It becomes a knowledge-exchanging peer at the next announce, which + /// is exactly when `mesh_peers` starts feeding `build_own_uniques`. + /// + /// This can NEVER evict an established mesh peer: it only ever fills slots + /// that are already free. + pub async fn graduate_temp_referrals(&mut self) -> Vec { + let free = self.available_mesh_slots(); + if free == 0 { + return Vec::new(); + } + let mut temps: Vec<(NodeId, u64)> = self + .connections + .values() + .filter(|pc| pc.slot.is_temp()) + .map(|pc| (pc.node_id, pc.connected_at)) + .collect(); + if temps.is_empty() { + return Vec::new(); + } + temps.sort_by_key(|(_, at)| *at); + + let mut promoted = Vec::new(); + for (peer_id, _) in temps.into_iter().take(free) { + if let Some(pc) = self.connections.get_mut(&peer_id) { + pc.slot = MeshSlot::Mesh; + promoted.push(peer_id); + } + } + if !promoted.is_empty() { + let storage = self.storage.get().await; + for peer_id in &promoted { + let _ = storage.add_mesh_peer(peer_id); + info!(peer = hex::encode(peer_id), "Temp referral graduated to mesh slot"); + } + drop(storage); + for peer_id in &promoted { + self.log_activity( + ActivityLevel::Info, + ActivityCategory::Rebalance, + "Temp referral graduated to mesh".into(), + Some(*peer_id), + ); + } + } + promoted + } + + /// Rebalance connection slots: reap dead/zombie/expired connections, prune + /// the stale uniques index, graduate temp referral slots, refill the mesh. /// Returns (newly_connected, pending_connects). Caller should QUIC-connect the pending /// list outside the lock, then register them. - pub async fn rebalance_slots(&mut self) -> anyhow::Result<(Vec, Vec<(NodeId, iroh::EndpointAddr, String, PeerSlotKind)>)> { + pub async fn rebalance_slots(&mut self) -> anyhow::Result<(Vec, Vec<(NodeId, iroh::EndpointAddr, String)>)> { self.log_activity(ActivityLevel::Info, ActivityCategory::Rebalance, "Rebalance started".into(), None); - // 1. Remove dead + zombie connections + // 1. Remove dead + zombie + expired-temp connections. + // Temp referral expiry is one extra condition in this sweep — that + // IS the whole expiry mechanism, no separate loop. let mut dead = Vec::new(); let now = now_ms(); for (peer_id, pc) in &self.connections { if pc.connection.close_reason().is_some() { dead.push(*peer_id); + } else if let MeshSlot::TempReferral { expires_at_ms } = pc.slot { + if now >= expires_at_ms { + info!(peer = hex::encode(peer_id), "Temp referral slot expired"); + self.log_activity(ActivityLevel::Info, ActivityCategory::Rebalance, "Temp referral expired".into(), Some(*peer_id)); + dead.push(*peer_id); + } } else if now.saturating_sub(pc.last_activity.load(Ordering::Relaxed)) > ZOMBIE_TIMEOUT_MS { let idle_secs = now.saturating_sub(pc.last_activity.load(Ordering::Relaxed)) / 1000; info!(peer = hex::encode(peer_id), idle_secs, "Zombie connection detected (no activity)"); @@ -3154,38 +3359,29 @@ impl ConnectionManager { self.disconnect_peer(peer_id).await; } - // 2. Prune stale N2/N3 entries (5 hours) + stale watchers (30 days) + // 2. Graduate temp referral slots into freed mesh slots. + // Oldest temp first. This NEVER evicts an established mesh peer: + // graduation is gated purely on a mesh slot already being free, and + // no path in the tree evicts a mesh peer at all. + self.graduate_temp_referrals().await; + + // 3. Prune the stale uniques index (5 hours, every bounce incl. N4) + // + stale watchers (30 days) { let storage = self.storage.get().await; - let pruned = storage.prune_n2_n3(5 * 60 * 60 * 1000)?; + let pruned = storage.prune_reach(5 * 60 * 60 * 1000)?; if pruned > 0 { - info!(pruned, "Pruned stale N2/N3 entries"); + info!(pruned, "Pruned stale uniques entries"); } let _ = storage.prune_stale_watchers(WATCHER_EXPIRY_MS); } - // 3. Diversity scoring: find low-diversity peers for potential eviction - { - let storage = self.storage.get().await; - let connected: Vec = self.connections.keys().copied().collect(); - let mut zero_diversity = Vec::new(); - for peer_id in &connected { - let unique = storage.count_unique_n2_for_reporter(peer_id, &[]).unwrap_or(0); - if unique == 0 { - zero_diversity.push(*peer_id); - } - } - if !zero_diversity.is_empty() { - debug!(count = zero_diversity.len(), "Peers with zero unique N2 contributions"); - } - } - let newly_connected: Vec = Vec::new(); - let mut pending_connects: Vec<(NodeId, iroh::EndpointAddr, String, PeerSlotKind)> = Vec::new(); + let mut pending_connects: Vec<(NodeId, iroh::EndpointAddr, String)> = Vec::new(); - // Priority 1+2: Fill empty local slots with diverse candidates - let local_count = self.count_kind(PeerSlotKind::Local); - if local_count < self.local_slots { + // Priority 1+2: Refill free MESH slots with diverse candidates + let mesh_count = self.mesh_count(); + if mesh_count < self.mesh_slots { let candidates: Vec<(NodeId, Option)> = { let storage = self.storage.get().await; let mut cands = Vec::new(); @@ -3205,10 +3401,10 @@ impl ConnectionManager { cands }; - let slots_available = self.local_slots - local_count; + let slots_available = self.mesh_slots - mesh_count; let to_connect = candidates.len().min(slots_available); if to_connect > 0 { - debug!(candidates = candidates.len(), connecting = to_connect, "Filling local slots"); + debug!(candidates = candidates.len(), connecting = to_connect, "Filling mesh slots"); } for (peer_id, addr_str) in candidates.into_iter().take(slots_available) { @@ -3231,7 +3427,7 @@ impl ConnectionManager { addr = addr.with_ip_addr(sock); } // Collect for connection outside the lock - pending_connects.push((peer_id, addr, addr_s, PeerSlotKind::Local)); + pending_connects.push((peer_id, addr, addr_s)); } } } @@ -3242,13 +3438,18 @@ impl ConnectionManager { // 5. Reap idle session connections (keeps anchor + referral sessions alive) self.reap_idle_sessions(SESSION_IDLE_TIMEOUT_MS).await; - // 6. Prune stale relay intro dedup entries + // 6. Prune stale relay intro dedup entries + the convection penalty box + // and the supplement hand-out ledger. All three are network-driven + // maps keyed by node id, so all three need a sweep. { let now = now_ms(); if self.seen_intros.len() > 500 { self.seen_intros .retain(|_, ts| now - *ts < RELAY_INTRO_DEDUP_EXPIRY_MS); } + self.refused_anchors.retain(|_, until| now < *until); + self.supplement_handouts + .retain(|_, (_, since)| now.saturating_sub(*since) < CONVECTION_SUPPLEMENT_WINDOW_MS); } if !dead.is_empty() { @@ -3258,7 +3459,7 @@ impl ConnectionManager { self.log_activity(ActivityLevel::Info, ActivityCategory::Rebalance, "Complete, no changes".into(), None); } - // Backstop: signal growth loop to fill any remaining local slots + // Backstop: signal growth loop to fill any remaining mesh slots self.notify_growth(); Ok((newly_connected, pending_connects)) @@ -3282,17 +3483,13 @@ impl ConnectionManager { &self.connections } - pub fn connection_info(&self) -> Vec<(NodeId, PeerSlotKind, u64)> { + pub fn connection_info(&self) -> Vec<(NodeId, MeshSlot, u64)> { self.connections .values() - .map(|pc| (pc.node_id, pc.slot_kind, pc.connected_at)) + .map(|pc| (pc.node_id, pc.slot, pc.connected_at)) .collect() } - fn count_kind(&self, kind: PeerSlotKind) -> usize { - self.connections.values().filter(|pc| pc.slot_kind == kind).count() - } - // ---- Session connection management ---- /// Add a session connection. Evicts oldest idle session if at capacity. @@ -3358,10 +3555,13 @@ impl ConnectionManager { // Build set of sessions that have a reason to stay alive let mut keep_alive: std::collections::HashSet = std::collections::HashSet::new(); - // Anchor side: peers on our referral list need their session kept - for nid in self.referral_list.keys() { - if self.sessions.contains_key(nid) && !self.connections.contains_key(nid) { - keep_alive.insert(*nid); + // Anchor side: a caller sitting in the convection window is exactly the + // session we need alive — it is the end we coordinate introductions + // through. Reaping it would make the anchor hand out cold addresses + // instead. + for e in &self.convection_window { + if self.sessions.contains_key(&e.node_id) && !self.connections.contains_key(&e.node_id) { + keep_alive.insert(e.node_id); } } @@ -3535,7 +3735,13 @@ impl ConnectionManager { } } - // Step 2: Fallback — full N3 scan for target + // Step 2: Fallback — full N3 scan for target. + // + // Deliberately NOT extended to N4 (bounce 4) in Iteration C: the + // forward chain only ever forwards a relay request via N2 reporters + // (`handle_relay_introduce` + its actor twin), so a ttl=2 request would + // dead-end. The ttl semantics and the forward lookup have to be widened + // together — left for whoever does the relay-depth work. if candidates.len() < 3 { if let Ok(reporters) = storage.find_in_n3(target) { for reporter in reporters { @@ -3730,7 +3936,7 @@ impl ConnectionManager { if let Some(session) = cm.sessions.get(&requester) { let session_conn = session.connection.clone(); drop(cm); // release lock before async work - match initial_exchange_connect(&storage_clone, &our_node_id, &session_conn, requester, None, our_nat_type, our_http_capable, our_http_addr.clone(), None, None).await { + match initial_exchange_connect(&storage_clone, &our_node_id, &session_conn, requester, None, our_nat_type, our_http_capable, our_http_addr.clone(), None, None, DEFAULT_PEER_DEPTH, false).await { Ok(ExchangeResult::Accepted { .. }) => { tracing::info!(peer = hex::encode(requester), "Target-side: initial exchange after hole punch"); } @@ -4276,7 +4482,7 @@ impl ConnectionManager { Ok(conn) => { let mut cm = cm_arc.lock().await; if !cm.connections.contains_key(&remote_node_id) { - cm.register_new_connection(remote_node_id, conn, &[addr], PeerSlotKind::Local).await; + cm.register_new_connection(remote_node_id, conn, &[addr]).await; info!(peer = hex::encode(remote_node_id), "Auto-reconnected after unexpected disconnect"); cm.log_activity(ActivityLevel::Info, ActivityCategory::Connection, format!("Auto-reconnected to {}", &hex::encode(remote_node_id)[..8]), Some(remote_node_id)); @@ -4299,251 +4505,432 @@ impl ConnectionManager { } } - // ---- Anchor referral methods ---- + // ---- Anchor convection service (design.html §anchors) ---- + // + // The registration model is gone: there is no 0xC0, no referral database, + // no use-count tiering, no disconnect grace period. An anchor keeps a small + // rolling window of recent callers, fed by the convection request itself. + // A caller receives the 2 most recent viable prior callers; its own address + // is then handed to the next 2 callers and rotates out. - /// Anchor-side: register a peer in the referral list. - /// Uses the observed remote address from the peers table (the NAT-mapped public IP - /// seen by the anchor from the QUIC connection) rather than self-reported addresses, - /// which are often private/LAN IPs for NAT'd peers. - pub async fn handle_anchor_register(&mut self, payload: AnchorRegisterPayload) { - if !self.is_anchor.load(Ordering::Relaxed) { + /// The address we OBSERVED for a caller (their NAT-mapped public IP as seen + /// from our side of the QUIC connection). Preferred over anything + /// self-reported, which for a NAT'd peer is usually a LAN address. + /// + /// Reads the live connection first and only then the `peers` row, which + /// keeps the DB off this path entirely in the common case. + fn convection_observed_addr(&self, nid: &NodeId) -> Option { + self.connections.get(nid).and_then(|pc| pc.remote_addr) + .or_else(|| self.sessions.get(nid).and_then(|s| s.remote_addr)) + } + + /// Admit (or refresh) a caller in the rolling window. + /// + /// Addresses are normalised (bugs-fixed #4) and filtered to publicly + /// routable (bugs-fixed #8) before they can ever be handed out — a private + /// or IPv4-mapped-IPv6 address costs the next caller a punch timeout. A + /// caller with no usable address is simply not admitted. + pub(crate) fn convection_admit( + &mut self, + node_id: NodeId, + self_reported: &[String], + observed: Option, + ) { + let addresses = self.convection_usable_addrs(&node_id, self_reported, observed); + if addresses.is_empty() { + debug!(peer = hex::encode(node_id), "Convection: no routable address, not admitting"); return; } - if !self.connections.contains_key(&payload.node_id) && !self.sessions.contains_key(&payload.node_id) { - debug!(peer = hex::encode(payload.node_id), "AnchorRegister from non-connected/session peer, ignoring"); - return; - } - - // Prefer observed remote address (NAT-mapped public IP) over self-reported - let addresses = { - let storage = self.storage.get().await; - let observed = storage.get_peer_record(&payload.node_id) - .ok().flatten() - .map(|r| r.addresses).unwrap_or_default(); - if observed.is_empty() { - // Fall back to self-reported addresses - payload.addresses - } else { - // Use observed addresses (public IP as seen by anchor) - observed.iter().map(|a| a.to_string()).collect() - } - }; - - let now = now_ms(); - self.referral_list.insert(payload.node_id, ReferralEntry { - node_id: payload.node_id, - addresses, - registered_at: now, - use_count: 0, - disconnected_at: None, - }); + convection_window_admit(&mut self.convection_window, node_id, addresses, now_ms()); debug!( - peer = hex::encode(payload.node_id), - list_size = self.referral_list.len(), - "Anchor: registered peer in referral list" + peer = hex::encode(node_id), + window = self.convection_window.len(), + "Convection: admitted caller to rolling window" ); } - /// Anchor-side: pick referrals from the list, applying tiered usage and self-pruning. - pub fn pick_referrals(&mut self, exclude: &NodeId, count: usize) -> Vec { - let now = now_ms(); - - // Prune: remove entries where disconnected_at is >2 min ago - self.referral_list.retain(|_, entry| { - match entry.disconnected_at { - Some(disc_at) if now.saturating_sub(disc_at) > REFERRAL_DISCONNECT_GRACE_MS => false, - _ => true, + /// The addresses of a caller that are actually worth circulating: + /// normalised (bugs-fixed #4) and publicly routable (bugs-fixed #8), + /// observed first because that is the one that provably works. Empty means + /// "we have nothing usable for this caller" — it is neither admitted to the + /// window nor named as the requester of a coordinated introduction. + fn convection_usable_addrs( + &self, + node_id: &NodeId, + self_reported: &[String], + observed: Option, + ) -> Vec { + if *node_id == self.our_node_id { + return Vec::new(); + } + let mut out: Vec = Vec::new(); + let push = |sa: SocketAddr, out: &mut Vec| { + let sa = normalize_addr(sa); + if convection_addr_ok(&sa) { + let s = sa.to_string(); + if !out.contains(&s) { + out.push(s); + } } - }); - - // Tiered usage policy - let list_len = self.referral_list.len(); - let max_uses: u32 = if list_len < REFERRAL_LIST_CAP { - 3 - } else if list_len == REFERRAL_LIST_CAP { - 2 - } else { - 1 }; + let observed = observed.or_else(|| self.convection_observed_addr(node_id)); + if let Some(sa) = observed { + push(sa, &mut out); + } + // SELF-REPORTED addresses are a caller's unverified claim. `convection_- + // addr_ok` only proves an address is public — not that the caller owns + // it — so an unbounded self-reported list let one cheap request make the + // anchor advertise arbitrary third-party IPs under the caller's node id. + // Capped, and once we have an observed address (essentially always, for + // a caller connected over QUIC) restricted to the SAME IP: a different + // port is the legitimate EDM case, a different host is not. + let mut extra = 0usize; + for a in self_reported { + if extra >= CONVECTION_MAX_SELF_REPORTED { + break; + } + if let Ok(sa) = a.parse::() { + if let Some(obs) = observed { + if normalize_addr(sa).ip() != normalize_addr(obs).ip() { + continue; + } + } + let before = out.len(); + push(sa, &mut out); + if out.len() > before { + extra += 1; + } + } + } + out + } - // Filter eligible entries - let mut eligible: Vec<&NodeId> = self.referral_list.iter() - .filter(|(nid, entry)| { - *nid != exclude - && entry.use_count < max_uses - && entry.disconnected_at.is_none() - }) - .map(|(nid, _)| nid) + /// The ONLY address the anchor may instruct a third party to punch at: the + /// one it observed on the caller's own QUIC connection. + /// + /// `serve_convection` pushes an anchor-INITIATED `RelayIntroduce` at up to + /// two live peers naming these addresses. Feeding self-reported claims into + /// that turned one cheap `ConvectionRequest` into "make two third parties + /// start hole-punching at an attacker-chosen victim" — an amplifier the old + /// retired 0xC0 register path did not have — it never pushed anything. + /// Self-reported entries still ride the referral payload as a secondary + /// hint, so the referred peer can try them itself rather than the anchor + /// instructing others to. + fn convection_introduce_addrs( + &self, + node_id: &NodeId, + observed: Option, + ) -> Vec { + if *node_id == self.our_node_id { + return Vec::new(); + } + observed + .or_else(|| self.convection_observed_addr(node_id)) + .map(normalize_addr) + .filter(convection_addr_ok) + .map(|sa| vec![sa.to_string()]) + .unwrap_or_default() + } + + /// Take up to `count` referrals out of the window. + pub(crate) fn convection_pick( + &mut self, + exclude: &NodeId, + count: usize, + ) -> Vec { + let now = now_ms(); + let live: HashSet = self + .connections + .keys() + .chain(self.sessions.keys()) + .copied() .collect(); + convection_window_pick(&mut self.convection_window, exclude, count, now, |n| live.contains(n)) + } - // Sort: lowest use_count first, then most recent registration - eligible.sort_by(|a, b| { - let ea = &self.referral_list[*a]; - let eb = &self.referral_list[*b]; - ea.use_count.cmp(&eb.use_count) - .then(eb.registered_at.cmp(&ea.registered_at)) - }); + /// Can we serve a TOP-UP right now? (ENTRY class never consults this.) + /// + /// Two honest signals, both cheap: nothing fresh to circulate, or we are + /// already saturated — mesh full AND the session table full, so every + /// referral we serve comes back to us as load we cannot absorb. + fn convection_saturated(&self) -> bool { + self.mesh_count() >= self.mesh_slots && self.sessions.len() >= self.session_slots + } - eligible.truncate(count); - let picked_ids: Vec = eligible.into_iter().copied().collect(); + /// Everything the convection response needs, gathered under a brief lock. + /// + /// The old handler held the conn_mgr lock across `write_typed_message` + + /// `send.finish()` — the same lock-contention class as the three known + /// TODOs. Gathering here and writing outside closes that. + pub fn gather_convection( + &mut self, + payload: &ConvectionRequestPayload, + observed: Option, + ) -> ConvectionGathered { + let requester = payload.requester; - let mut result = Vec::with_capacity(picked_ids.len()); - for nid in &picked_ids { - if let Some(entry) = self.referral_list.get_mut(nid) { - entry.use_count += 1; - result.push(AnchorReferral { - node_id: entry.node_id, - addresses: entry.addresses.clone(), + // ENTRY class is ALWAYS served: a node with <2 connections cannot + // function, and a network that refuses re-entry is a network that + // partitions permanently. TOP-UP is served capacity-permitting and + // refused with a single small message — no timeout burned. + let viable = convection_window_viable(&self.convection_window, &requester, now_ms()); + if !convection_should_serve(payload.class, viable, self.convection_saturated()) { + // Still admit them: their address is fresh material for the next + // caller. Top-ups are the convection MEDIUM, not freeloading. + self.convection_admit(requester, &payload.requester_addresses, observed); + debug!(requester = hex::encode(requester), "Convection: refusing top-up (no capacity)"); + return ConvectionGathered { + response: ConvectionResponsePayload { referrals: vec![], refused: true }, + introductions: Vec::new(), + requester, + requester_addresses: Vec::new(), + nat_mapping: None, + nat_filtering: None, + }; + } + + // Can we actually coordinate an introduction? Only if we have a + // routable address for the CALLER to name as the requester. Deciding + // this up front matters: `introduced` tells the caller to skip its own + // introduction round trip, so claiming it without pushing one would + // strand the caller on a failed direct dial. + let requester_addrs = self.convection_introduce_addrs(&requester, observed); + let can_introduce = !requester_addrs.is_empty(); + + let picked = self.convection_pick(&requester, CONVECTION_REFERRALS); + + let mut referrals: Vec = Vec::with_capacity(picked.len()); + let mut introductions: Vec<(NodeId, iroh::endpoint::Connection)> = Vec::new(); + for entry in picked { + // Both ends live on us → coordinate an introduction rather than + // handing over a cold address. The peer starts punching outward + // before the caller has even read our response. + let conn = self + .connections + .get(&entry.node_id) + .map(|pc| pc.connection.clone()) + .or_else(|| self.sessions.get(&entry.node_id).map(|s| s.connection.clone())); + let introduced = conn.is_some() && can_introduce; + if introduced { + if let Some(c) = conn { + introductions.push((entry.node_id, c)); + } + } + debug!( + requester = hex::encode(requester), + referred = hex::encode(entry.node_id), + introduced, + "Convection: referring peer" + ); + referrals.push(ConvectionReferral { + node_id: entry.node_id, + addresses: entry.addresses, + introduced, + }); + } + + // Supplement from live mesh peers when the window is sparse (a young + // anchor genuinely has no prior callers yet). CONSENT-SHAPED: a mesh + // peer never called us and never offered its address for + // redistribution, unlike a window entry whose address arrived in its + // own `ConvectionRequestPayload.requester_addresses`. So: + // + // * we hand out NO address — the referral carries an empty vec and + // relies on the anchor-initiated RelayIntroduce, which makes the + // TARGET disclose its own addresses on its own terms; + // * therefore only peers we can actually introduce qualify (an + // address-free referral we cannot introduce is useless anyway); + // * and each supplemented peer gets its own hand-out budget, so + // unconditional entry-class service cannot be turned into a + // "shuffle + repeat" enumeration oracle over the anchor's mesh. + if referrals.len() < CONVECTION_REFERRALS && can_introduce { + let now = now_ms(); + let already: HashSet = referrals.iter().map(|r| r.node_id).collect(); + let mut mesh_candidates: Vec = self + .connections + .iter() + .filter(|(nid, pc)| { + **nid != requester + && **nid != self.our_node_id + && !already.contains(*nid) + && pc.slot.is_mesh() + }) + .map(|(nid, _)| *nid) + .filter(|nid| match self.supplement_handouts.get(nid) { + Some((count, since)) => { + now.saturating_sub(*since) >= CONVECTION_SUPPLEMENT_WINDOW_MS + || *count < CONVECTION_SUPPLEMENT_CAP + } + None => true, + }) + .collect(); + use rand::seq::SliceRandom; + mesh_candidates.shuffle(&mut rand::rng()); + for nid in mesh_candidates.into_iter().take(CONVECTION_REFERRALS - referrals.len()) { + if let Some(pc) = self.connections.get(&nid) { + introductions.push((nid, pc.connection.clone())); + } + let slot = self.supplement_handouts.entry(nid).or_insert((0, now)); + if now.saturating_sub(slot.1) >= CONVECTION_SUPPLEMENT_WINDOW_MS { + *slot = (0, now); + } + slot.0 += 1; + referrals.push(ConvectionReferral { + node_id: nid, + addresses: Vec::new(), + introduced: true, }); } } - // Self-prune: remove entries that reached max_uses - self.referral_list.retain(|_, entry| entry.use_count < max_uses); - - result - } - - /// Mark a peer as disconnected in the referral list. - pub fn mark_referral_disconnected(&mut self, node_id: &NodeId) { - if let Some(entry) = self.referral_list.get_mut(node_id) { - entry.disconnected_at = Some(now_ms()); - } - } - - /// Mark a peer as reconnected in the referral list. - pub fn mark_referral_reconnected(&mut self, node_id: &NodeId) { - if let Some(entry) = self.referral_list.get_mut(node_id) { - entry.disconnected_at = None; - } - } - - /// Anchor-side: handle a referral request (bi-stream). - /// Supplements from mesh peers when the explicit referral list is sparse. - pub async fn handle_anchor_referral_request( - &mut self, - payload: AnchorReferralRequestPayload, - mut send: iroh::endpoint::SendStream, - ) -> anyhow::Result<()> { - // Also register the requester (they provide addresses, they're connected) - self.handle_anchor_register(AnchorRegisterPayload { - node_id: payload.requester, - addresses: payload.requester_addresses, - }).await; - - let mut referrals = self.pick_referrals(&payload.requester, 3); - - // Auto-refer from mesh peers when referral list is sparse - if referrals.len() < 3 { - let referred_ids: std::collections::HashSet = referrals.iter().map(|r| r.node_id).collect(); - let mut mesh_candidates: Vec<_> = self.connections.iter() - .filter(|(nid, _)| **nid != payload.requester && !referred_ids.contains(*nid) && **nid != self.our_node_id) - .filter_map(|(nid, pc)| pc.remote_addr.map(|a| (*nid, a.to_string()))) - .collect(); - // Shuffle for variety - use rand::seq::SliceRandom; - mesh_candidates.shuffle(&mut rand::rng()); - for (nid, addr) in mesh_candidates.into_iter().take(3 - referrals.len()) { - referrals.push(AnchorReferral { node_id: nid, addresses: vec![addr] }); - } - } + // The request IS the registration. Admitted AFTER picking so a caller + // is never referred to itself, and so its address goes to the NEXT two + // callers rather than this one. + self.convection_admit(requester, &payload.requester_addresses, observed); info!( - requester = hex::encode(payload.requester), - referral_count = referrals.len(), - "Anchor: serving referrals" + requester = hex::encode(requester), + class = if payload.class.is_entry() { "entry" } else { "top-up" }, + referrals = referrals.len(), + introductions = introductions.len(), + window = self.convection_window.len(), + "Convection: serving referrals" ); - let response = AnchorReferralResponsePayload { referrals }; - write_typed_message(&mut send, MessageType::AnchorReferralResponse, &response).await?; + let profile = self.our_nat_profile(); + ConvectionGathered { + response: ConvectionResponsePayload { referrals, refused: false }, + introductions, + requester, + requester_addresses: requester_addrs, + nat_mapping: Some(profile.mapping.to_string()), + nat_filtering: Some(profile.filtering.to_string()), + } + } + + /// Write the convection response and fire any coordinated introductions. + /// Static: runs entirely OUTSIDE the conn_mgr lock. + pub async fn serve_convection( + gathered: ConvectionGathered, + mut send: iroh::endpoint::SendStream, + ) -> anyhow::Result<()> { + // Response first — a refusal must reach the caller immediately, and a + // served caller should start dialling while the introductions fly. + write_typed_message(&mut send, MessageType::ConvectionResponse, &gathered.response).await?; send.finish()?; + + if gathered.introductions.is_empty() || gathered.requester_addresses.is_empty() { + return Ok(()); + } + + // Anchor-INITIATED introduction. Today's introductions are always + // requester-initiated; here the anchor holds both ends, so it pushes a + // RelayIntroduce toward the referral naming the caller as requester. + // The target's handler responds with its addresses and starts punching + // (bugs-fixed #3 registration path), so the caller's direct dial lands. + // + // This is signalling, NOT a session relay — no bytes are piped. + for (target, conn) in gathered.introductions { + let payload = RelayIntroducePayload { + intro_id: rand::random(), + target, + requester: gathered.requester, + requester_addresses: gathered.requester_addresses.clone(), + ttl: 0, + nat_mapping: gathered.nat_mapping.clone(), + nat_filtering: gathered.nat_filtering.clone(), + }; + tokio::spawn(async move { + let result = async { + let (mut s, mut r) = conn.open_bi().await?; + write_typed_message(&mut s, MessageType::RelayIntroduce, &payload).await?; + s.finish()?; + let _ = read_message_type(&mut r).await?; + let _: RelayIntroduceResultPayload = read_payload(&mut r, MAX_PAYLOAD).await?; + anyhow::Ok(()) + }; + match tokio::time::timeout(std::time::Duration::from_secs(10), result).await { + Ok(Ok(())) => debug!(target = hex::encode(target), "Convection: coordinated introduction"), + Ok(Err(e)) => debug!(target = hex::encode(target), error = %e, "Convection: introduction push failed"), + Err(_) => debug!(target = hex::encode(target), "Convection: introduction push timed out"), + } + }); + } Ok(()) } - /// Client-side: request referrals from an anchor peer (mesh or session). - pub async fn request_anchor_referrals( - &mut self, - anchor_peer: &NodeId, - ) -> anyhow::Result> { - let conn = if let Some(pc) = self.connections.get(anchor_peer) { - pc.connection.clone() - } else if let Some(session) = self.sessions.get(anchor_peer) { - session.connection.clone() - } else { - anyhow::bail!("anchor peer not connected (mesh or session)"); - }; + // ---- Stochastic growth trigger (round-4/5) ---- - let our_addrs: Vec = self.endpoint.addr().ip_addrs() - .map(|s| s.to_string()) - .collect(); - - let request = AnchorReferralRequestPayload { - requester: self.our_node_id, - requester_addresses: our_addrs, - }; - - let (mut send, mut recv) = conn.open_bi().await?; - write_typed_message(&mut send, MessageType::AnchorReferralRequest, &request).await?; - send.finish()?; - - let msg_type = read_message_type(&mut recv).await?; - if msg_type != MessageType::AnchorReferralResponse { - anyhow::bail!("expected AnchorReferralResponse, got {:?}", msg_type); - } - let response: AnchorReferralResponsePayload = read_payload(&mut recv, 4096).await?; - - // Touch session last_active to prevent idle reaping - if let Some(session) = self.sessions.get_mut(anchor_peer) { - session.last_active_at = now_ms(); - } - - Ok(response.referrals) + /// Is the cached anchor-density prior stale? Split from the setter because + /// the storage guard borrows `self`, so a caller holding it cannot also + /// take `&mut self`. + fn anchor_density_refresh_due(&self) -> bool { + now_ms().saturating_sub(self.anchor_density_at_ms) >= ANCHOR_DENSITY_REFRESH_MS } - /// Client-side: register our address with an anchor peer (mesh or session). - pub async fn send_anchor_register(&mut self, anchor_peer: &NodeId) -> anyhow::Result<()> { - let conn = if let Some(pc) = self.connections.get(anchor_peer) { - pc.connection.clone() - } else if let Some(session) = self.sessions.get(anchor_peer) { - session.connection.clone() + fn set_anchor_density(&mut self, density: f64) { + self.anchor_density = density.clamp(0.0, 1.0); + self.anchor_density_at_ms = now_ms(); + } + + /// Roll the dice for one mesh disconnect. Pure state read + RNG — no I/O, + /// no network dispatch, safe to call under the lock. + pub fn roll_convection_action(&self) -> ConvectionAction { + let fill_ratio = if self.mesh_slots == 0 { + 1.0 } else { - anyhow::bail!("anchor peer not connected (mesh or session)"); + self.mesh_count() as f64 / self.mesh_slots as f64 }; + let roll: f64 = rand::random(); + choose_convection_action(roll, self.anchor_density, self.anchor_bias, fill_ratio) + } - let mut our_addrs: Vec = self.endpoint.addr().ip_addrs() - .map(|s| s.to_string()) - .collect(); - // Prepend UPnP external address (most useful for remote peers) - if let Some(ref ext) = self.upnp_external_addr { - let ext_str = ext.to_string(); - if !our_addrs.contains(&ext_str) { - our_addrs.insert(0, ext_str); - } + /// An anchor refused us: multiplicative decrease on the anchor arm, plus a + /// penalty box for that specific anchor so we try a different one next. + pub fn record_convection_refusal(&mut self, anchor: &NodeId) { + self.anchor_bias = (self.anchor_bias * 0.5).max(0.1); + // This is the ONLY insertion site, so sweeping here bounds the map with + // no extra timer. Without it the penalty box grew one permanent entry + // per anchor that ever refused us — a network-driven map with no + // eviction, over an anchor population the design expects to be 20-30% + // of all nodes. (`rebalance_slots` sweeps it too, for long idles.) + let now = now_ms(); + self.refused_anchors.retain(|_, until| now < *until); + self.refused_anchors.insert(*anchor, now + ANCHOR_REFUSAL_PENALTY_MS); + debug!(anchor = hex::encode(anchor), bias = self.anchor_bias, "Convection: refusal feedback"); + } + + /// An anchor served us: additive increase, drifting back toward parity. + pub fn record_convection_success(&mut self, anchor: &NodeId) { + self.anchor_bias = (self.anchor_bias + 0.15).min(1.0); + self.refused_anchors.remove(anchor); + } + + /// Is this anchor in the penalty box? + pub fn is_anchor_penalized(&self, anchor: &NodeId) -> bool { + match self.refused_anchors.get(anchor) { + Some(&until) => now_ms() < until, + None => false, } - // Prepend stable anchor advertised address - if let Some(anchor_addr) = self.build_anchor_advertised_addr() { - if !our_addrs.contains(&anchor_addr) { - our_addrs.insert(0, anchor_addr); - } + } + + pub fn anchor_bias(&self) -> f64 { + self.anchor_bias + } + + pub fn cached_anchor_density(&self) -> f64 { + self.anchor_density + } + + pub fn set_convection_tx(&mut self, tx: tokio::sync::mpsc::Sender<()>) { + self.convection_tx = Some(tx); + } + + /// Wake the convection loop (the "ask a random known anchor" arm). + pub fn notify_convection(&self) { + if let Some(tx) = &self.convection_tx { + let _ = tx.try_send(()); } - - let payload = AnchorRegisterPayload { - node_id: self.our_node_id, - addresses: our_addrs, - }; - - let mut send = conn.open_uni().await?; - write_typed_message(&mut send, MessageType::AnchorRegister, &payload).await?; - send.finish()?; - - // Touch session last_active to prevent idle reaping - if let Some(session) = self.sessions.get_mut(anchor_peer) { - session.last_active_at = now_ms(); - } - - debug!(anchor = hex::encode(anchor_peer), "Registered with anchor"); - - Ok(()) } /// Handle an incoming NAT filter probe request (anchor side). @@ -4634,15 +5021,25 @@ impl ConnectionManager { let msg_type = read_message_type(recv).await?; match msg_type { - MessageType::NodeListUpdate => { - let diff: NodeListUpdatePayload = read_payload(recv, MAX_PAYLOAD).await?; - let has_n1_additions = !diff.n1_added.is_empty(); - let cm = conn_mgr.lock().await; - let count = cm.process_routing_diff(&remote_node_id, diff).await?; - if count > 0 { - debug!(peer = hex::encode(remote_node_id), count, "Applied node list update"); + MessageType::UniquesAnnounce => { + let announce: UniquesAnnouncePayload = read_payload(recv, MAX_UNIQUES_PAYLOAD).await?; + // Same gather-outside-lock discipline as the pull handlers: + // the apply is a bulk per-row write and must not block slot + // allocation, disconnects or referral serving. + let ctx = { + let mut cm = conn_mgr.lock().await; + cm.set_peer_depth(&remote_node_id, announce.depth); + cm.uniques_ctx(&remote_node_id) + }; + // A temp referral slot carries no knowledge — drop the announce. + if !ctx.is_mesh { + debug!(peer = hex::encode(remote_node_id), "Ignoring announce from temp referral slot"); + return Ok(()); } - if has_n1_additions { + let count = ctx.apply(&remote_node_id, &announce).await?; + if count > 0 { + debug!(peer = hex::encode(remote_node_id), count, "Applied uniques announce"); + let cm = conn_mgr.lock().await; cm.notify_growth(); } } @@ -4988,11 +5385,6 @@ impl ConnectionManager { } } } - MessageType::AnchorRegister => { - let payload: AnchorRegisterPayload = read_payload(recv, 4096).await?; - let mut cm = conn_mgr.lock().await; - cm.handle_anchor_register(payload).await; - } MessageType::MeshKeepalive => { // No-op — last_activity already updated on stream accept } @@ -5046,22 +5438,28 @@ impl ConnectionManager { msg_type: MessageType, ) -> anyhow::Result<()> { match msg_type { - MessageType::PullSyncRequest => { + MessageType::ContentSyncRequest => { let (storage, our_node_id) = { let cm = conn_mgr.lock().await; (Arc::clone(&cm.storage), *cm.our_node_id()) }; // Lock RELEASED — handler does its own brief storage locks + network I/O - ConnectionManager::handle_pull_request_unlocked(&storage, our_node_id, remote_node_id, recv, send).await?; + ConnectionManager::handle_content_sync_request_unlocked(&storage, our_node_id, remote_node_id, recv, send).await?; + } + MessageType::UniquesPullRequest => { + ConnectionManager::handle_uniques_pull(conn_mgr, recv, send, remote_node_id).await?; } MessageType::InitialExchange => { - let (storage, our_node_id, anchor_addr, our_nat_type, our_http_capable, our_http_addr, is_duplicate) = { + let (storage, our_node_id, anchor_addr, our_nat_type, our_http_capable, our_http_addr, is_duplicate, our_depth, carry_knowledge) = { let cm = conn_mgr.lock().await; // Duplicate identity detection: is this NodeId already mesh-connected? let dup = cm.connections.contains_key(&remote_node_id); - (cm.storage_ref(), *cm.our_node_id(), cm.build_anchor_advertised_addr(), cm.nat_type(), cm.http_capable, cm.http_addr.clone(), dup) + // Knowledge boundary: only a MESH slot exchanges uniques. + // An unregistered peer (session/relay path) carries none either. + let carry = cm.connections.get(&remote_node_id).map(|pc| pc.slot.is_mesh()).unwrap_or(false); + (cm.storage_ref(), *cm.our_node_id(), cm.build_anchor_advertised_addr(), cm.nat_type(), cm.http_capable, cm.http_addr.clone(), dup, cm.max_bounce, carry) }; - initial_exchange_accept(&storage, &our_node_id, send, recv, remote_node_id, anchor_addr, None, our_nat_type, our_http_capable, our_http_addr, None, None, is_duplicate) + initial_exchange_accept(&storage, &our_node_id, send, recv, remote_node_id, anchor_addr, None, our_nat_type, our_http_capable, our_http_addr, None, None, is_duplicate, our_depth, carry_knowledge) .await?; } MessageType::AddressRequest => { @@ -5165,12 +5563,15 @@ impl ConnectionManager { let cm = conn_mgr.lock().await; Arc::clone(&cm.blob_store) }; - // Also snapshot wide-referral candidates (node_id, slot_kind) - let wide_candidates: Vec<(NodeId, PeerSlotKind)> = { + // Snapshot referral candidates — MESH slots only. We never + // point a stranger at a peer we hold in a temp slot. + let wide_candidates: Vec = { let cm = conn_mgr.lock().await; cm.connections.iter() - .filter(|(nid, _)| **nid != remote_node_id && **nid != cm.our_node_id) - .map(|(nid, pc)| (*nid, pc.slot_kind)) + .filter(|(nid, pc)| { + **nid != remote_node_id && **nid != cm.our_node_id && pc.slot.is_mesh() + }) + .map(|(nid, _)| *nid) .collect() }; tokio::spawn(async move { @@ -5447,7 +5848,7 @@ impl ConnectionManager { if let Some(session) = cm.sessions.get(&requester) { let session_conn = session.connection.clone(); drop(cm); - let _ = initial_exchange_connect(&storage, &our_node_id, &session_conn, requester, None, our_nat_type, our_http_capable, our_http_addr, None, None).await; + let _ = initial_exchange_connect(&storage, &our_node_id, &session_conn, requester, None, our_nat_type, our_http_capable, our_http_addr, None, None, DEFAULT_PEER_DEPTH, false).await; } } }); @@ -5530,10 +5931,20 @@ impl ConnectionManager { let cm = Arc::clone(conn_mgr); Self::handle_session_relay(cm, recv, send, remote_node_id).await?; } - MessageType::AnchorReferralRequest => { - let payload: AnchorReferralRequestPayload = read_payload(&mut recv, 4096).await?; - let mut cm = conn_mgr.lock().await; - cm.handle_anchor_referral_request(payload, send).await?; + MessageType::ConvectionRequest => { + let mut payload: ConvectionRequestPayload = read_payload(&mut recv, 4096).await?; + // The QUIC peer identity is authoritative. Without this a + // caller could enrol a THIRD party's node id in our window and + // have us circulate an address it does not own. + payload.requester = remote_node_id; + // Gather under a brief lock, write outside it. The old handler + // held the lock across write + finish. + let gathered = { + let mut cm = conn_mgr.lock().await; + let observed = cm.convection_observed_addr(&remote_node_id); + cm.gather_convection(&payload, observed) + }; + ConnectionManager::serve_convection(gathered, send).await?; } MessageType::AnchorProbeRequest => { let payload: crate::protocol::AnchorProbeRequestPayload = read_payload(&mut recv, 4096).await?; @@ -6077,7 +6488,7 @@ pub enum ConnResponse { NodeId(NodeId), OptConnection(Option), Peers(Vec), - ConnectionInfo(Vec<(NodeId, PeerSlotKind, u64)>), + ConnectionInfo(Vec<(NodeId, MeshSlot, u64)>), SessionInfo(Vec<(NodeId, SessionReachMethod, u64)>), NatProfile(crate::types::NatProfile), NatType(crate::types::NatType), @@ -6089,26 +6500,33 @@ pub enum ConnResponse { Storage(Arc), BlobStore(Arc), ActiveRelayPipes(Arc), - Referrals(Vec), OptSocketAddr(Option), IsAnchorAtomicBool(Arc), ActivityLogRef(Arc>), ScoreVec(Vec<(NodeId, f64)>), OptPeerWithAddress(Option), - ConnectionMap(Vec<(NodeId, iroh::endpoint::Connection, PeerSlotKind, Arc)>), - DiffData(DiffSnapshot), + ConnectionMap(Vec<(NodeId, iroh::endpoint::Connection, MeshSlot, Arc)>), + AnnounceData(AnnounceSnapshot), Unit, } -/// Snapshot of data needed for routing diff computation. -pub struct DiffSnapshot { +/// Snapshot for broadcasting the uniques announce outside the lock. +/// +/// One built payload plus the per-peer connections it goes to. Only MESH-slot +/// peers appear in `connections`: a temporary referral slot carries no +/// knowledge exchange in either direction. +pub struct AnnounceSnapshot { pub our_node_id: NodeId, - pub connections: Vec<(NodeId, iroh::endpoint::Connection)>, - pub diff_seq: u64, - pub n1_added: Vec, - pub n1_removed: Vec, - pub n2_added: Vec, - pub n2_removed: Vec, + /// (peer, connection, peer's advertised retention depth) + pub connections: Vec<(NodeId, iroh::endpoint::Connection, u8)>, + pub seq: u64, + /// Our announce, built with the full terminal pool. Peers that advertise + /// depth < 4 get it stripped at send time. + pub payload: Option, + /// True when the pools are unchanged since the last broadcast — the caller + /// skips the send. (The announce is always a full snapshot, so this is what + /// stands in for incremental diffing.) + pub unchanged: bool, } /// Commands sent to the ConnectionActor via the ConnHandle channel. @@ -6121,11 +6539,16 @@ pub enum ConnCommand { ConnectionCount { reply: oneshot::Sender, }, + /// Mesh-slot count only (temp referral slots excluded) — the number the + /// recovery threshold and the ENTRY/TOP-UP class bit are defined against. + MeshCount { + reply: oneshot::Sender, + }, ConnectedPeers { reply: oneshot::Sender>, }, ConnectionInfo { - reply: oneshot::Sender>, + reply: oneshot::Sender>, }, GetConnection { peer: NodeId, @@ -6137,7 +6560,7 @@ pub enum ConnCommand { }, /// Get all mesh connections (node_id, connection, slot_kind, last_activity) GetConnectionMap { - reply: oneshot::Sender)>>, + reply: oneshot::Sender)>>, }, OurNodeId { reply: oneshot::Sender, @@ -6168,7 +6591,7 @@ pub enum ConnCommand { SessionPeerIds { reply: oneshot::Sender>, }, - AvailableLocalSlots { + AvailableMeshSlots { reply: oneshot::Sender, }, IsLikelyUnreachable { @@ -6237,14 +6660,13 @@ pub enum ConnCommand { conn: iroh::endpoint::Connection, remote_node_id: NodeId, remote_addr: Option, - reply: oneshot::Sender, + reply: oneshot::Sender>, }, RegisterConnection { peer_id: NodeId, conn: iroh::endpoint::Connection, addrs: Vec, - slot_kind: PeerSlotKind, - reply: oneshot::Sender<()>, + reply: oneshot::Sender>, }, DisconnectPeer { peer: NodeId, @@ -6278,10 +6700,6 @@ pub enum ConnCommand { SetNatFiltering { filtering: crate::types::NatFiltering, }, - AddStickyN1 { - peer: NodeId, - duration_ms: u64, - }, SetGrowthTx { tx: tokio::sync::mpsc::Sender<()>, }, @@ -6315,33 +6733,36 @@ pub enum ConnCommand { RecordProbeSuccess, RecordProbeFailure, - // --- Referral management --- - HandleAnchorRegister { - payload: AnchorRegisterPayload, - reply: oneshot::Sender<()>, + // --- Anchor convection (adaptive weights, round-5) --- + RecordConvectionRefusal { + anchor: NodeId, }, - PickReferrals { - exclude: NodeId, - count: usize, - reply: oneshot::Sender>, + RecordConvectionSuccess { + anchor: NodeId, }, - MarkReferralDisconnected { - node_id: NodeId, + IsAnchorPenalized { + anchor: NodeId, + reply: oneshot::Sender, }, - MarkReferralReconnected { - node_id: NodeId, + ConvectionState { + reply: oneshot::Sender<(f64, f64, usize)>, + }, + SetConvectionTx { + tx: tokio::sync::mpsc::Sender<()>, }, - // --- Diff/routing --- - ProcessRoutingDiff { + // --- Uniques announce --- + ProcessUniquesAnnounce { from_peer: NodeId, - payload: NodeListUpdatePayload, + payload: UniquesAnnouncePayload, reply: oneshot::Sender>, }, - /// Get snapshot of connections + diff data for external broadcast - GetDiffData { - reply: oneshot::Sender, + /// Get snapshot of mesh connections + our built announce for broadcast + GetAnnounceData { + reply: oneshot::Sender, }, + /// Reset the change detector so the next announce is sent unconditionally. + ForceAnnounce, // --- Relay/address lookups (state reads, no network I/O) --- FindRelaysFor { @@ -6381,7 +6802,7 @@ pub enum ConnCommand { target: NodeId, reply: oneshot::Sender>>, }, - PullFromPeer { + ContentSyncFromPeer { peer: NodeId, reply: oneshot::Sender>, }, @@ -6411,19 +6832,29 @@ pub struct ConnHandle { http_addr: Arc>>, /// CDN device role (set once at startup by Network) device_role_val: Arc>>, + /// Deepest bounce this device retains (4 desktop / 3 mobile). Hoisted out + /// of the manager so the accept path can read it without a lock. + max_bounce: u8, } impl ConnHandle { /// Create a ConnHandle from a command channel sender. - pub fn new(tx: mpsc::Sender) -> Self { + pub fn new(tx: mpsc::Sender, max_bounce: u8) -> Self { Self { tx, http_capable: Arc::new(AtomicBool::new(false)), http_addr: Arc::new(std::sync::Mutex::new(None)), device_role_val: Arc::new(std::sync::Mutex::new(None)), + max_bounce, } } + /// Deepest bounce we retain — goes on the wire so peers can skip building + /// the terminal pool for us. + pub fn max_bounce(&self) -> u8 { + self.max_bounce + } + /// Set HTTP capability info (called once after HTTP server starts). pub fn set_http_info(&self, capable: bool, addr: Option) { self.http_capable.store(capable, Ordering::Relaxed); @@ -6464,13 +6895,20 @@ impl ConnHandle { rx.await.unwrap_or(0) } + /// Mesh-slot count (temp referral slots excluded). + pub async fn mesh_count(&self) -> usize { + let (tx, rx) = oneshot::channel(); + let _ = self.tx.send(ConnCommand::MeshCount { reply: tx }).await; + rx.await.unwrap_or(0) + } + pub async fn connected_peers(&self) -> Vec { let (tx, rx) = oneshot::channel(); let _ = self.tx.send(ConnCommand::ConnectedPeers { reply: tx }).await; rx.await.unwrap_or_default() } - pub async fn connection_info(&self) -> Vec<(NodeId, PeerSlotKind, u64)> { + pub async fn connection_info(&self) -> Vec<(NodeId, MeshSlot, u64)> { let (tx, rx) = oneshot::channel(); let _ = self.tx.send(ConnCommand::ConnectionInfo { reply: tx }).await; rx.await.unwrap_or_default() @@ -6489,7 +6927,7 @@ impl ConnHandle { } /// Get all mesh connections as (node_id, connection, slot_kind, last_activity). - pub async fn get_connection_map(&self) -> Vec<(NodeId, iroh::endpoint::Connection, PeerSlotKind, Arc)> { + pub async fn get_connection_map(&self) -> Vec<(NodeId, iroh::endpoint::Connection, MeshSlot, Arc)> { let (tx, rx) = oneshot::channel(); let _ = self.tx.send(ConnCommand::GetConnectionMap { reply: tx }).await; rx.await.unwrap_or_default() @@ -6546,9 +6984,9 @@ impl ConnHandle { rx.await.unwrap_or_default() } - pub async fn available_local_slots(&self) -> usize { + pub async fn available_mesh_slots(&self) -> usize { let (tx, rx) = oneshot::channel(); - let _ = self.tx.send(ConnCommand::AvailableLocalSlots { reply: tx }).await; + let _ = self.tx.send(ConnCommand::AvailableMeshSlots { reply: tx }).await; rx.await.unwrap_or(0) } @@ -6662,12 +7100,15 @@ impl ConnHandle { // === Mutation operations === + /// Accept an inbound connection. `None` = refused (mesh + temp both full); + /// otherwise the slot class it was placed in — the caller must propagate + /// `is_mesh()` into the initial exchange as the knowledge-carry flag. pub async fn accept_connection( &self, conn: iroh::endpoint::Connection, remote_node_id: NodeId, remote_addr: Option, - ) -> bool { + ) -> Option { let (tx, rx) = oneshot::channel(); let _ = self.tx.send(ConnCommand::AcceptConnection { conn, @@ -6675,25 +7116,25 @@ impl ConnHandle { remote_addr, reply: tx, }).await; - rx.await.unwrap_or(false) + rx.await.ok().flatten() } + /// Register an outbound connection. The slot class is decided by the + /// manager (mesh if there is room, else a temp referral slot, else `None`). pub async fn register_connection( &self, peer_id: NodeId, conn: iroh::endpoint::Connection, addrs: Vec, - slot_kind: PeerSlotKind, - ) { + ) -> Option { let (tx, rx) = oneshot::channel(); let _ = self.tx.send(ConnCommand::RegisterConnection { peer_id, conn, addrs, - slot_kind, reply: tx, }).await; - let _ = rx.await; + rx.await.ok().flatten() } pub async fn disconnect_peer(&self, peer: &NodeId) { @@ -6746,11 +7187,6 @@ impl ConnHandle { let _ = self.tx.try_send(ConnCommand::MarkUnreachable { peer: *peer }); } - /// Add a node to the sticky N1 set (reported in N1 share until expiry). - pub fn add_sticky_n1(&self, peer: &NodeId, duration_ms: u64) { - let _ = self.tx.try_send(ConnCommand::AddStickyN1 { peer: *peer, duration_ms }); - } - pub fn set_nat_filtering(&self, filtering: crate::types::NatFiltering) { let _ = self.tx.try_send(ConnCommand::SetNatFiltering { filtering }); } @@ -6807,35 +7243,43 @@ impl ConnHandle { // === Referral management === - pub async fn handle_anchor_register(&self, payload: AnchorRegisterPayload) { + /// Refusal feedback: multiplicative decrease on the anchor arm + penalty + /// box for that anchor. + pub fn record_convection_refusal(&self, anchor: &NodeId) { + let _ = self.tx.try_send(ConnCommand::RecordConvectionRefusal { anchor: *anchor }); + } + + /// Success feedback: additive increase, drifting back toward parity. + pub fn record_convection_success(&self, anchor: &NodeId) { + let _ = self.tx.try_send(ConnCommand::RecordConvectionSuccess { anchor: *anchor }); + } + + pub async fn is_anchor_penalized(&self, anchor: &NodeId) -> bool { let (tx, rx) = oneshot::channel(); - let _ = self.tx.send(ConnCommand::HandleAnchorRegister { payload, reply: tx }).await; - let _ = rx.await; + let _ = self.tx.send(ConnCommand::IsAnchorPenalized { anchor: *anchor, reply: tx }).await; + rx.await.unwrap_or(false) } - pub async fn pick_referrals(&self, exclude: &NodeId, count: usize) -> Vec { + /// (anchor_density, anchor_bias, mesh_count) — diagnostics + tests. + pub async fn convection_state(&self) -> (f64, f64, usize) { let (tx, rx) = oneshot::channel(); - let _ = self.tx.send(ConnCommand::PickReferrals { exclude: *exclude, count, reply: tx }).await; - rx.await.unwrap_or_default() + let _ = self.tx.send(ConnCommand::ConvectionState { reply: tx }).await; + rx.await.unwrap_or((0.0, 1.0, 0)) } - pub fn mark_referral_disconnected(&self, node_id: &NodeId) { - let _ = self.tx.try_send(ConnCommand::MarkReferralDisconnected { node_id: *node_id }); - } - - pub fn mark_referral_reconnected(&self, node_id: &NodeId) { - let _ = self.tx.try_send(ConnCommand::MarkReferralReconnected { node_id: *node_id }); + pub async fn set_convection_tx(&self, tx: tokio::sync::mpsc::Sender<()>) { + let _ = self.tx.send(ConnCommand::SetConvectionTx { tx }).await; } // === Diff/routing === - pub async fn process_routing_diff( + pub async fn process_uniques_announce( &self, from_peer: &NodeId, - payload: NodeListUpdatePayload, + payload: UniquesAnnouncePayload, ) -> anyhow::Result { let (tx, rx) = oneshot::channel(); - let _ = self.tx.send(ConnCommand::ProcessRoutingDiff { + let _ = self.tx.send(ConnCommand::ProcessUniquesAnnounce { from_peer: *from_peer, payload, reply: tx, @@ -6866,20 +7310,23 @@ impl ConnHandle { let _ = self.tx.try_send(ConnCommand::TouchSessionIfExists { peer: *peer }); } - pub async fn get_diff_data(&self) -> DiffSnapshot { + pub async fn get_announce_data(&self) -> AnnounceSnapshot { let (tx, rx) = oneshot::channel(); - let _ = self.tx.send(ConnCommand::GetDiffData { reply: tx }).await; - rx.await.unwrap_or_else(|_| DiffSnapshot { + let _ = self.tx.send(ConnCommand::GetAnnounceData { reply: tx }).await; + rx.await.unwrap_or_else(|_| AnnounceSnapshot { our_node_id: [0u8; 32], connections: vec![], - diff_seq: 0, - n1_added: vec![], - n1_removed: vec![], - n2_added: vec![], - n2_removed: vec![], + seq: 0, + payload: None, + unchanged: true, }) } + /// Force the next `get_announce_data` to send even if nothing changed. + pub async fn force_announce(&self) { + let _ = self.tx.send(ConnCommand::ForceAnnounce).await; + } + // === Complex operations === pub async fn rebalance_slots(&self) -> anyhow::Result> { @@ -6946,9 +7393,11 @@ impl ConnHandle { rx.await.map_err(|_| anyhow::anyhow!("actor dropped"))? } - pub async fn pull_from_peer(&self, peer: &NodeId) -> anyhow::Result { + /// TRANSITIONAL content sync (0x46/0x47) — the real path: uses + /// `control::receive_post`, `touch_file_holder`, and PostDownstreamRegister. + pub async fn content_sync_from_peer(&self, peer: &NodeId) -> anyhow::Result { let (tx, rx) = oneshot::channel(); - let _ = self.tx.send(ConnCommand::PullFromPeer { peer: *peer, reply: tx }).await; + let _ = self.tx.send(ConnCommand::ContentSyncFromPeer { peer: *peer, reply: tx }).await; rx.await.map_err(|_| anyhow::anyhow!("actor dropped"))? } @@ -6985,7 +7434,7 @@ impl ConnectionActor { pub async fn spawn_with_arc(cm: Arc>) -> ConnHandle { let (tx, rx) = mpsc::channel(256); // Hoist frequently-needed Arcs so handlers can skip the conn_mgr lock - let (storage, blob_store, endpoint, our_node_id, activity_log, is_anchor) = { + let (storage, blob_store, endpoint, our_node_id, activity_log, is_anchor, max_bounce) = { let cm_guard = cm.lock().await; ( Arc::clone(&cm_guard.storage), @@ -6994,11 +7443,12 @@ impl ConnectionActor { cm_guard.our_node_id, Arc::clone(&cm_guard.activity_log), Arc::clone(&cm_guard.is_anchor), + cm_guard.max_bounce, ) }; let actor = ConnectionActor { cm, rx, storage, blob_store, endpoint, our_node_id, activity_log, is_anchor }; tokio::spawn(actor.run()); - ConnHandle::new(tx) + ConnHandle::new(tx, max_bounce) } /// Spawn the actor owning the ConnectionManager directly (Phase 5+). @@ -7043,42 +7493,28 @@ impl ConnectionActor { } } - // N2 lookup: brief lock to get reporters + their connections - let n2_queries: Vec = { + // Referral cascade over the whole stored horizon (bounce 2 → 3 → 4), + // shallowest first. Must stay in step with + // `ConnectionManager::resolve_address` — these two copies have diverged + // before. + let queries: Vec = { let s = storage.get().await; - let reporters = s.find_in_n2(target)?; - drop(s); - let cm_guard = cm.lock().await; - reporters.iter() - .filter_map(|r| cm_guard.connections.get(r).map(|pc| pc.connection.clone())) - .collect() - }; - for conn in &n2_queries { - let result = tokio::time::timeout(std::time::Duration::from_secs(5), async { - let (mut send, mut recv) = conn.open_bi().await?; - let req = crate::protocol::AddressRequestPayload { target: *target }; - write_typed_message(&mut send, MessageType::AddressRequest, &req).await?; - send.finish()?; - let _resp_type = read_message_type(&mut recv).await?; - let resp: crate::protocol::AddressResponsePayload = read_payload(&mut recv, MAX_PAYLOAD).await?; - anyhow::Ok(resp.address) - }).await; - if let Ok(Ok(Some(addr))) = result { - return Ok(Some(addr)); + let mut seen: HashSet = HashSet::new(); + let mut ordered = Vec::new(); + for bounce in 2u8..=4 { + for r in s.find_reporters_at(target, bounce).unwrap_or_default() { + if seen.insert(r) { + ordered.push(r); + } + } } - } - - // N3 lookup: same pattern - let n3_queries: Vec = { - let s = storage.get().await; - let reporters = s.find_in_n3(target)?; drop(s); let cm_guard = cm.lock().await; - reporters.iter() + ordered.iter() .filter_map(|r| cm_guard.connections.get(r).map(|pc| pc.connection.clone())) .collect() }; - for conn in &n3_queries { + for conn in &queries { let result = tokio::time::timeout(std::time::Duration::from_secs(5), async { let (mut send, mut recv) = conn.open_bi().await?; let req = crate::protocol::AddressRequestPayload { target: *target }; @@ -7196,7 +7632,7 @@ impl ConnectionActor { } { let s = ctx.storage.get().await; - let found_entries = s.find_any_in_n2_n3(all_needles)?; + let found_entries = s.find_any_reachable(all_needles)?; if let Some((found_id, _reporter, _level)) = found_entries.first() { drop(s); let address = Self::resolve_address_unlocked(&ctx.storage, &ctx.cm, &ctx.endpoint, found_id).await.ok().flatten(); @@ -7293,14 +7729,14 @@ impl ConnectionActor { None } - /// Pick a random wide referral from a pre-snapshotted list of (node_id, slot_kind). - async fn pick_wide_referral(storage: &Arc, candidates: &[(NodeId, PeerSlotKind)], exclude: &NodeId) -> Option<(NodeId, String)> { - let filtered: Vec<(NodeId, PeerSlotKind)> = candidates.iter().filter(|(nid, _)| nid != exclude).copied().collect(); + /// Pick a random mesh referral from a pre-snapshotted candidate list. + /// v0.8: one pool, so there is no "prefer wide". Temp referral slots were + /// already filtered out when the snapshot was taken. + async fn pick_wide_referral(storage: &Arc, candidates: &[NodeId], exclude: &NodeId) -> Option<(NodeId, String)> { + let filtered: Vec = candidates.iter().filter(|nid| *nid != exclude).copied().collect(); if filtered.is_empty() { return None; } - let wide: Vec<(NodeId, PeerSlotKind)> = filtered.iter().filter(|(_, kind)| *kind == PeerSlotKind::Wide).copied().collect(); - let ordered = if !wide.is_empty() { wide } else { filtered }; let s = storage.get().await; - for (nid, _) in &ordered { + for nid in &filtered { if let Ok(Some(rec)) = s.get_peer_record(nid) { if let Some(addr) = rec.addresses.first() { return Some((*nid, addr.to_string())); } } @@ -7312,7 +7748,7 @@ impl ConnectionActor { async fn handle_worm_query_unlocked( ctx: WormContext, blob_store: Arc, - wide_candidates: Vec<(NodeId, PeerSlotKind)>, + wide_candidates: Vec, payload: WormQueryPayload, mut send: iroh::endpoint::SendStream, from_peer: NodeId, @@ -7359,7 +7795,7 @@ impl ConnectionActor { } if found.is_none() { let s = ctx.storage.get().await; - let entries = s.find_any_in_n2_n3(&all_needles)?; + let entries = s.find_any_reachable(&all_needles)?; if let Some((found_id, _, _)) = entries.first() { drop(s); let address = Self::resolve_address_unlocked(&ctx.storage, &ctx.cm, &ctx.endpoint, found_id).await.ok().flatten(); @@ -7432,6 +7868,10 @@ impl ConnectionActor { let cm = self.cm.lock().await; let _ = reply.send(cm.connection_count()); } + ConnCommand::MeshCount { reply } => { + let cm = self.cm.lock().await; + let _ = reply.send(cm.mesh_count()); + } ConnCommand::ConnectedPeers { reply } => { let cm = self.cm.lock().await; let _ = reply.send(cm.connected_peers()); @@ -7452,7 +7892,7 @@ impl ConnectionActor { ConnCommand::GetConnectionMap { reply } => { let cm = self.cm.lock().await; let map: Vec<_> = cm.connections_ref().iter().map(|(nid, pc)| { - (*nid, pc.connection.clone(), pc.slot_kind, Arc::clone(&pc.last_activity)) + (*nid, pc.connection.clone(), pc.slot, Arc::clone(&pc.last_activity)) }).collect(); let _ = reply.send(map); } @@ -7492,9 +7932,9 @@ impl ConnectionActor { let cm = self.cm.lock().await; let _ = reply.send(cm.session_peer_ids()); } - ConnCommand::AvailableLocalSlots { reply } => { + ConnCommand::AvailableMeshSlots { reply } => { let cm = self.cm.lock().await; - let _ = reply.send(cm.available_local_slots()); + let _ = reply.send(cm.available_mesh_slots()); } ConnCommand::IsLikelyUnreachable { peer, reply } => { let cm = self.cm.lock().await; @@ -7577,10 +8017,10 @@ impl ConnectionActor { let r = cm.accept_connection(conn, remote_node_id, remote_addr); let _ = reply.send(r); } - ConnCommand::RegisterConnection { peer_id, conn, addrs, slot_kind, reply } => { + ConnCommand::RegisterConnection { peer_id, conn, addrs, reply } => { let mut cm = self.cm.lock().await; - cm.register_connection(peer_id, conn, &addrs, slot_kind).await; - let _ = reply.send(()); + let r = cm.register_connection(peer_id, conn, &addrs).await; + let _ = reply.send(r); } ConnCommand::DisconnectPeer { peer, reply } => { let mut cm = self.cm.lock().await; @@ -7619,11 +8059,6 @@ impl ConnectionActor { let mut cm = self.cm.lock().await; cm.nat_filtering = filtering; } - ConnCommand::AddStickyN1 { peer, duration_ms } => { - let mut cm = self.cm.lock().await; - let expiry = now_ms() + duration_ms; - cm.sticky_n1.insert(peer, expiry); - } ConnCommand::SetGrowthTx { tx } => { let mut cm = self.cm.lock().await; cm.set_growth_tx(tx); @@ -7689,85 +8124,83 @@ impl ConnectionActor { } // --- Referral management --- - ConnCommand::HandleAnchorRegister { payload, reply } => { + ConnCommand::RecordConvectionRefusal { anchor } => { let mut cm = self.cm.lock().await; - cm.handle_anchor_register(payload).await; - let _ = reply.send(()); + cm.record_convection_refusal(&anchor); } - ConnCommand::PickReferrals { exclude, count, reply } => { + ConnCommand::RecordConvectionSuccess { anchor } => { let mut cm = self.cm.lock().await; - let r = cm.pick_referrals(&exclude, count); - let _ = reply.send(r); + cm.record_convection_success(&anchor); } - ConnCommand::MarkReferralDisconnected { node_id } => { - let mut cm = self.cm.lock().await; - cm.mark_referral_disconnected(&node_id); + ConnCommand::IsAnchorPenalized { anchor, reply } => { + let cm = self.cm.lock().await; + let _ = reply.send(cm.is_anchor_penalized(&anchor)); } - ConnCommand::MarkReferralReconnected { node_id } => { + ConnCommand::ConvectionState { reply } => { + let cm = self.cm.lock().await; + let _ = reply.send((cm.cached_anchor_density(), cm.anchor_bias(), cm.mesh_count())); + } + ConnCommand::SetConvectionTx { tx } => { let mut cm = self.cm.lock().await; - cm.mark_referral_reconnected(&node_id); + cm.set_convection_tx(tx); } - // --- Diff/routing --- - ConnCommand::ProcessRoutingDiff { from_peer, payload, reply } => { - let cm = self.cm.lock().await; - let r = cm.process_routing_diff(&from_peer, payload).await; + // --- Uniques announce --- + ConnCommand::ProcessUniquesAnnounce { from_peer, payload, reply } => { + let ctx = { + let mut cm = self.cm.lock().await; + cm.set_peer_depth(&from_peer, payload.depth); + cm.uniques_ctx(&from_peer) + }; + // Lock released for the bulk write. + let r = if ctx.is_mesh { + ctx.apply(&from_peer, &payload).await + } else { + Ok(0) + }; let _ = reply.send(r); } - ConnCommand::GetDiffData { reply } => { + ConnCommand::ForceAnnounce => { let mut cm = self.cm.lock().await; - // Prune expired sticky N1 entries first - let now = now_ms(); - cm.sticky_n1.retain(|_, expiry| *expiry > now); - let sticky_peers: Vec = cm.sticky_n1.keys().copied().collect(); - // Compute diff snapshot - let storage = cm.storage.get().await; - let current_n1: HashSet = { - let mut set = HashSet::new(); - for nid in cm.connections_ref().keys() { - set.insert(*nid); - } - if let Ok(routes) = storage.list_social_routes() { - for route in &routes { - if route.status == crate::types::SocialStatus::Online { - set.insert(route.node_id); - } - } - } - for nid in &sticky_peers { - set.insert(*nid); - } - set - }; - let current_n2: HashSet = storage.build_n2_share() - .unwrap_or_default() - .into_iter() + cm.last_announce_digest = 0; + } + ConnCommand::GetAnnounceData { reply } => { + let mut cm = self.cm.lock().await; + let our_node_id = *cm.our_node_id(); + let anchor_addr = cm.build_anchor_advertised_addr(); + let our_depth = cm.max_bounce; + let seq = cm.diff_seq.fetch_add(1, Ordering::Relaxed) + 1; + + // Knowledge boundary: MESH slots only. + let conns: Vec<(NodeId, iroh::endpoint::Connection, u8)> = cm + .connections_ref() + .iter() + .filter(|(_, pc)| pc.slot.is_mesh()) + .map(|(nid, pc)| (*nid, pc.connection.clone(), pc.peer_depth)) .collect(); + + let storage = cm.storage.get().await; + let payload = build_uniques_payload( + &storage, + &our_node_id, + anchor_addr.as_deref(), + seq, + our_depth, + 4, + ) + .ok(); drop(storage); - let n1_added: Vec = current_n1.difference(&cm.last_n1_set).copied().collect(); - let n1_removed: Vec = cm.last_n1_set.difference(¤t_n1).copied().collect(); - let n2_added: Vec = current_n2.difference(&cm.last_n2_set).copied().collect(); - let n2_removed: Vec = cm.last_n2_set.difference(¤t_n2).copied().collect(); + let digest = payload.as_ref().map(announce_digest).unwrap_or(0); + let unchanged = digest != 0 && digest == cm.last_announce_digest; + cm.last_announce_digest = digest; - cm.last_n1_set = current_n1; - cm.last_n2_set = current_n2; - - let seq = cm.diff_seq.fetch_add(1, Ordering::Relaxed); - - let conns: Vec<(NodeId, iroh::endpoint::Connection)> = cm.connections_ref() - .iter() - .map(|(nid, pc)| (*nid, pc.connection.clone())) - .collect(); - - let _ = reply.send(DiffSnapshot { - our_node_id: *cm.our_node_id(), + let _ = reply.send(AnnounceSnapshot { + our_node_id, connections: conns, - diff_seq: seq, - n1_added, - n1_removed, - n2_added, - n2_removed, + seq, + payload, + unchanged, }); } @@ -7806,7 +8239,7 @@ impl ConnectionActor { // outgoing-connect slot per peer so we don't race with // auto-reconnect / relay-introduction paths for the same // target; skip peers already mid-connect. - for (peer_id, addr, _addr_s, slot_kind) in pending_connects { + for (peer_id, addr, _addr_s) in pending_connects { let _connect_guard = { let cm = self.cm.lock().await; match cm.try_begin_connect(peer_id) { @@ -7825,7 +8258,7 @@ impl ConnectionActor { match ConnectionManager::connect_to_unlocked(&endpoint, addr).await { Ok(conn) => { let mut cm = self.cm.lock().await; - cm.register_new_connection(peer_id, conn, &addrs, slot_kind).await; + let _ = cm.register_new_connection(peer_id, conn, &addrs).await; info!(peer = hex::encode(peer_id), "Auto-connected to peer"); newly_connected.push(peer_id); } @@ -7918,7 +8351,7 @@ impl ConnectionActor { let r = Self::resolve_address_unlocked(&self.storage, &self.cm, &self.endpoint, &target).await; let _ = reply.send(r); } - ConnCommand::PullFromPeer { peer, reply } => { + ConnCommand::ContentSyncFromPeer { peer, reply } => { // Brief lock: grab connection clone + our_node_id let gather = { let cm = self.cm.lock().await; @@ -7928,7 +8361,7 @@ impl ConnectionActor { let r = match gather { Some((conn, our_node_id)) => { // All I/O outside the lock, storage accessed via hoisted Arc - ConnectionManager::pull_from_peer_unlocked(conn, &self.storage, &peer, our_node_id).await + ConnectionManager::content_sync_unlocked(conn, &self.storage, &peer, our_node_id).await } None => Err(anyhow::anyhow!("not connected to {}", hex::encode(peer))), }; @@ -7977,8 +8410,293 @@ impl ConnectionActor { } } +/// The slot-allocation rule, isolated so it is testable without an endpoint. +/// +/// Mesh first, then the temporary referral band that sits strictly ABOVE the +/// mesh cap, then refuse. Nothing here can evict anything: an established mesh +/// peer is never displaced by an arriving stranger. +pub fn decide_slot( + mesh_count: usize, + mesh_slots: usize, + temp_count: usize, + temp_slots: usize, + now: u64, +) -> Option { + if mesh_count < mesh_slots { + Some(MeshSlot::Mesh) + } else if temp_count < temp_slots { + Some(MeshSlot::TempReferral { expires_at_ms: now + TEMP_REFERRAL_TTL_MS }) + } else { + None + } +} + +// ---- Uniques announce: build / apply (design.html §layers) ---- + +/// Content hash of an announce, ignoring `seq`. Lets the broadcaster skip a +/// send when the pools have not changed. +pub fn announce_digest(p: &UniquesAnnouncePayload) -> u64 { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + for slice in &p.fwd { + slice.nodes.hash(&mut h); + slice.authors.hash(&mut h); + for a in &slice.anchors { + a.i.hash(&mut h); + a.a.hash(&mut h); + } + } + p.term.nodes.hash(&mut h); + p.term.authors.hash(&mut h); + for a in &p.term.anchors { + a.i.hash(&mut h); + a.a.hash(&mut h); + } + let d = h.finish(); + if d == 0 { 1 } else { d } +} + +/// Serialise a slice of the pool for the wire. +/// +/// Node-class and author-class IDs go into separate packed blobs; anchors go +/// into the only address-bearing array. A receiver therefore cannot mistake a +/// persona ID for a connectable node ID, and cannot find an address on a +/// non-anchor entry — both invariants are structural, not conventional. +pub fn entries_to_slice(entries: &[ReachEntry]) -> UniquesSlice { + let mut nodes: Vec = Vec::new(); + let mut authors: Vec = Vec::new(); + let mut anchors: Vec = Vec::new(); + for e in entries { + // `id_class == Node` is required STRUCTURALLY, not incidentally: the + // class and the anchor flag are stored in independent columns, so an + // author-class row carrying `is_anchor = 1` would otherwise serialise + // as an address-bearing entry — a persona ID published next to a device + // address, the exact linkage §21 forbids. An author-class entry falls + // through to the bare `authors` array regardless of its flag. + if e.id_class == IdClass::Node && e.is_anchor && !e.addresses.is_empty() { + // bugs-fixed #8: only publicly-routable addresses go on the wire. + let addrs: Vec = e + .addresses + .iter() + .filter(|a| { + a.parse::() + .map(|s| crate::network::is_publicly_routable(&s)) + .unwrap_or(false) + }) + .cloned() + .collect(); + if !addrs.is_empty() { + anchors.push(AnchorEntry { i: hex::encode(e.id), a: addrs }); + continue; + } + // No routable address left — degrade to a bare node entry. + nodes.push(e.id); + continue; + } + match e.id_class { + IdClass::Node => nodes.push(e.id), + IdClass::Author => authors.push(e.id), + } + } + UniquesSlice { nodes: pack_ids(&nodes), authors: pack_ids(&authors), anchors } +} + +/// Deserialise a wire slice back into entries, truncated to `cap`. +/// +/// RECEIVE-SIDE ADDRESS FILTERING is mandatory here and was previously absent, +/// leaving the ingest asymmetric with `entries_to_slice`: a peer could hand us +/// `127.0.0.1:22`, `10.0.0.1:4433` or `192.168.1.1:443`, we would store it and +/// later dial it. That is attacker-chosen internal QUIC probing from the +/// victim's own host, and — per the v0.7.3 EDM lesson — every such `connect()` +/// permanently enters iroh's per-endpoint path store and is probed in the +/// background forever. An anchor entry left with no usable address degrades to +/// a bare node entry rather than being stored address-free-but-anchor-flagged. +pub fn slice_to_entries_capped(slice: &UniquesSlice, cap: usize) -> Vec { + let mut out = Vec::new(); + for id in unpack_ids(&slice.nodes) { + if out.len() >= cap { + return out; + } + out.push(ReachEntry::node(id)); + } + for id in unpack_ids(&slice.authors) { + if out.len() >= cap { + return out; + } + out.push(ReachEntry::author(id)); + } + for a in &slice.anchors { + if out.len() >= cap { + return out; + } + if let Ok(id) = crate::parse_node_id_hex(&a.i) { + // bugs-fixed #4: normalise IPv4-mapped IPv6 before storing. + // bugs-fixed #8 (receive side): routable addresses only. + let addrs: Vec = a + .a + .iter() + .filter_map(|s| s.parse::().ok()) + .map(normalize_addr) + .filter(convection_addr_ok) + .map(|s| s.to_string()) + .collect(); + if addrs.is_empty() { + out.push(ReachEntry::node(id)); + } else { + out.push(ReachEntry::anchor(id, addrs)); + } + } + } + out +} + +/// Uncapped convenience wrapper (tests and the local build path). +pub fn slice_to_entries(slice: &UniquesSlice) -> Vec { + slice_to_entries_capped(slice, usize::MAX) +} + +/// Build our announce payload from storage. +pub fn build_uniques_payload( + storage: &crate::storage::Storage, + our_node_id: &NodeId, + our_anchor_addr: Option<&str>, + seq: u64, + our_depth: u8, + peer_depth: u8, +) -> anyhow::Result { + let pools: UniquesPools = storage.build_uniques_pools(our_node_id, our_anchor_addr)?; + let fwd: Vec = pools.fwd.iter().map(|e| entries_to_slice(e)).collect(); + // A peer that only retains 3 bounces has nowhere to put the terminal pool — + // skip building it for them. Biggest single saving on a cellular link. + let term = if peer_depth >= 4 { + entries_to_slice(&pools.term) + } else { + UniquesSlice::default() + }; + Ok(UniquesAnnouncePayload { seq, full: true, fwd, term, depth: our_depth }) +} + +/// Apply a received announce, shifting pool 1 one bounce deeper and landing +/// pool 2 at the terminal bounce 4. +pub fn apply_uniques_to_storage( + storage: &crate::storage::Storage, + our_node_id: &NodeId, + reporter: &NodeId, + payload: &UniquesAnnouncePayload, + our_max_bounce: u8, + is_connected: impl Fn(&NodeId) -> bool, +) -> anyhow::Result { + // Every reader of `fwd` assumes exactly 3 slices (N0, N1, N2). Nothing + // enforced it, so a peer sending 6 slices wrote rows at bounce 5 and 6 — + // depths the rest of the system has no semantics for (invisible to + // `list_reach_entries_at(2)/(3)` and to `resolve_address`'s 2..=4 cascade, + // but very visible to `find_any_reachable`, `score_n2_candidates_batch`, + // `anchor_density` and `list_pool_anchors`). + if payload.fwd.len() != 3 { + anyhow::bail!("uniques announce: fwd must carry exactly 3 slices, got {}", payload.fwd.len()); + } + + let mut stored = 0usize; + + // fwd[0] is the reporter itself. Nothing to store at bounce 1 (they are a + // direct by definition), but this is where their own anchor address rides. + // This is the ONE first-party claim in the payload — the sender is on the + // other end of the QUIC connection we are reading — so it is the only one + // allowed to touch `peers`, and even then by UNION, never by replacement. + if let Some(self_slice) = payload.fwd.first() { + for e in slice_to_entries_capped(self_slice, 8) { + if e.is_anchor && e.id == *reporter { + persist_direct_anchor_claim(storage, &e); + } + } + } + + // fwd[i] → bounce i + 1, for i >= 1 (so bounce 2 and 3). + for (i, slice) in payload.fwd.iter().enumerate().skip(1) { + let bounce = (i + 1) as u8; + if bounce > our_max_bounce || bounce > 4 { + continue; + } + let cap = crate::storage::uniques_accept_cap(bounce); + let raw = slice_to_entries_capped(slice, cap); + let truncated = raw.len() >= cap; + let entries: Vec = raw + .into_iter() + .filter(|e| { + e.id != *our_node_id + // A peer we are already meshed with is not a growth + // candidate; keeping it out saves candidate-slot churn. + && !(bounce == 2 && e.id_class == IdClass::Node && is_connected(&e.id)) + }) + .collect(); + if truncated { + warn!( + reporter = hex::encode(reporter), + bounce, cap, "Uniques announce truncated to the §layers budget" + ); + } + stored += entries.len(); + storage.set_reach(reporter, bounce, &entries)?; + } + + // term → bounce 4. Dropped entirely when we only retain 3 bounces. + // + // Third-party anchor entries at ANY depth stay in `reachable` only. They + // are an INDEX ("someone says this ID is an anchor at this address"), not + // an address record: nothing has proven the address belongs to the node ID. + // Mirroring them into `peers`/`known_anchors` made a single hostile mesh + // peer able to (a) overwrite our stored address for any node — including + // the DNS-resolved bootstrap anchor (bugs-fixed #5, remotely triggerable), + // (b) get thousands of entries pointing at one victim IP dialled by us and + // by everyone we re-announce to, and (c) re-inflate `known_anchors` from + // gossip, undoing its round-4 demotion to a bootstrap cache. + // `list_pool_anchors` mines `reachable` directly, so the anchor directory + // is unaffected. + if our_max_bounce >= 4 { + let cap = crate::storage::uniques_accept_cap(4); + let raw = slice_to_entries_capped(&payload.term, cap); + if raw.len() >= cap { + warn!( + reporter = hex::encode(reporter), + bounce = 4, cap, "Uniques announce truncated to the §layers budget" + ); + } + let entries: Vec = + raw.into_iter().filter(|e| e.id != *our_node_id).collect(); + stored += entries.len(); + storage.set_reach(reporter, 4, &entries)?; + } + + let _ = storage.update_mesh_peer_seq(reporter, payload.seq); + Ok(stored) +} + +/// The reporter's claim about ITS OWN anchor address (payload `fwd[0]`). +/// +/// First-party and from a peer we are connected to right now — the same trust +/// class as `InitialExchangePayload.anchor_addr`. It may therefore mark the +/// peer as an anchor and add the address, but it may NOT replace addresses we +/// already have (one of which is the address we actually connected on), and it +/// does NOT seed `known_anchors`: that cache is now reserved for anchors we +/// have completed a handshake to, which is what makes its `last_seen_ms` +/// ordering meaningful again. +fn persist_direct_anchor_claim(storage: &crate::storage::Storage, e: &ReachEntry) { + let socks: Vec = e + .addresses + .iter() + .filter_map(|a| a.parse::().ok()) + .map(normalize_addr) + .filter(convection_addr_ok) + .collect(); + if socks.is_empty() { + return; + } + let _ = storage.add_peer_addresses(&e.id, &socks); + let _ = storage.set_peer_anchor(&e.id, true); +} + /// Standalone initial exchange (connector side) — does NOT require conn_mgr lock. -/// Opens a bi-stream, sends our N1/N2/profile/deletes/post_ids, reads theirs. +/// Opens a bi-stream, sends our uniques announce/profile/deletes/post_ids, reads theirs. pub async fn initial_exchange_connect( storage: &Arc, our_node_id: &NodeId, @@ -7990,20 +8708,42 @@ pub async fn initial_exchange_connect( our_http_addr: Option, our_device_role: Option, our_cache_pressure: Option, + our_depth: u8, + carry_knowledge: bool, ) -> anyhow::Result { let our_payload = { let storage = storage.get().await; - let n1 = storage.build_n1_share()?; - let n2 = storage.build_n2_share()?; + // A temporary referral slot carries NO uniques exchange — explicit + // `None` on the wire rather than empty vectors, so the peer can tell + // "nothing to share" from "not sharing". + let uniques = if carry_knowledge { + Some(build_uniques_payload( + &storage, + our_node_id, + anchor_addr.as_deref(), + 0, + our_depth, + DEFAULT_PEER_DEPTH, + )?) + } else { + None + }; // Profile keyed by network id; strip persona display before send. let profile = storage.get_profile(our_node_id)? .map(|p| p.sanitized_for_network_broadcast()); let deletes = storage.list_delete_records()?; let post_ids = storage.list_post_ids()?; - let peer_addresses = storage.build_peer_addresses_for(our_node_id)?; + // Knowledge boundary: a temporary referral slot carries NO knowledge + // exchange (design.html §connections). Peer addresses ARE knowledge — + // up to 10 of our mesh peers with their addresses — so they are gated + // on the same flag as the uniques pools, not sent unconditionally. + let peer_addresses = if carry_knowledge { + storage.build_peer_addresses_for(our_node_id)? + } else { + Vec::new() + }; InitialExchangePayload { - n1_node_ids: n1, - n2_node_ids: n2, + uniques, profile, deletes, post_ids, @@ -8040,7 +8780,7 @@ pub async fn initial_exchange_connect( if dup { tracing::warn!(peer = hex::encode(remote_node_id), "Anchor reports duplicate identity active on network"); } - process_exchange_payload(storage, our_node_id, &remote_node_id, &their_payload).await?; + process_exchange_payload(storage, our_node_id, &remote_node_id, &their_payload, our_depth, carry_knowledge).await?; Ok(ExchangeResult::Accepted { duplicate_active: dup }) }; @@ -8069,22 +8809,44 @@ pub async fn initial_exchange_accept( our_device_role: Option, our_cache_pressure: Option, duplicate_detected: bool, + our_depth: u8, + carry_knowledge: bool, ) -> anyhow::Result<()> { let their_payload: InitialExchangePayload = read_payload(&mut recv, MAX_PAYLOAD).await?; let our_payload = { let storage = storage.get().await; - let n1 = storage.build_n1_share()?; - let n2 = storage.build_n2_share()?; + // Temporary referral slot => no uniques in either direction. + // Their advertised depth lets us skip building the terminal pool. + let peer_depth = their_payload.uniques.as_ref().map(|u| u.depth).unwrap_or(DEFAULT_PEER_DEPTH); + let uniques = if carry_knowledge { + Some(build_uniques_payload( + &storage, + our_node_id, + anchor_addr.as_deref(), + 0, + our_depth, + peer_depth, + )?) + } else { + None + }; // Profile keyed by network id; strip persona display before send. let profile = storage.get_profile(our_node_id)? .map(|p| p.sanitized_for_network_broadcast()); let deletes = storage.list_delete_records()?; let post_ids = storage.list_post_ids()?; - let peer_addresses = storage.build_peer_addresses_for(our_node_id)?; + // Knowledge boundary: a temporary referral slot carries NO knowledge + // exchange (design.html §connections). Peer addresses ARE knowledge — + // up to 10 of our mesh peers with their addresses — so they are gated + // on the same flag as the uniques pools, not sent unconditionally. + let peer_addresses = if carry_knowledge { + storage.build_peer_addresses_for(our_node_id)? + } else { + Vec::new() + }; InitialExchangePayload { - n1_node_ids: n1, - n2_node_ids: n2, + uniques, profile, deletes, post_ids, @@ -8109,7 +8871,7 @@ pub async fn initial_exchange_accept( write_typed_message(&mut send, MessageType::InitialExchange, &our_payload).await?; send.finish()?; - process_exchange_payload(storage, our_node_id, &remote_node_id, &their_payload).await?; + process_exchange_payload(storage, our_node_id, &remote_node_id, &their_payload, our_depth, carry_knowledge).await?; Ok(()) } @@ -8119,23 +8881,34 @@ async fn process_exchange_payload( our_node_id: &NodeId, remote_node_id: &NodeId, payload: &InitialExchangePayload, + our_max_bounce: u8, + store_knowledge: bool, ) -> anyhow::Result<()> { let storage = storage.get().await; - // Filter out our own ID from their N1 before storing as our N2 - let filtered_n1: Vec = payload.n1_node_ids.iter() - .filter(|nid| *nid != our_node_id) - .copied() - .collect(); - storage.set_peer_n1(remote_node_id, &filtered_n1)?; - debug!(peer = hex::encode(remote_node_id), raw = payload.n1_node_ids.len(), stored = filtered_n1.len(), "Stored peer N1 as our N2 (filtered self)"); - - let filtered_n2: Vec = payload.n2_node_ids.iter() - .filter(|nid| *nid != our_node_id) - .copied() - .collect(); - storage.set_peer_n2(remote_node_id, &filtered_n2)?; - debug!(peer = hex::encode(remote_node_id), raw = payload.n2_node_ids.len(), stored = filtered_n2.len(), "Stored peer N2 as our N3 (filtered self)"); + // Uniques announce: shift pool 1 one bounce deeper, land pool 2 at N4. + // `uniques == None` means the sender put us in a temporary referral slot; + // `store_knowledge == false` means WE put them in one. Either way the + // connection carries no knowledge, in either direction. + match (&payload.uniques, store_knowledge) { + (Some(announce), true) => { + let n = apply_uniques_to_storage( + &storage, + our_node_id, + remote_node_id, + announce, + our_max_bounce, + |_| false, + )?; + debug!(peer = hex::encode(remote_node_id), stored = n, "Applied uniques announce"); + } + _ => { + debug!( + peer = hex::encode(remote_node_id), + "No uniques exchanged (temporary referral slot)" + ); + } + } if let Some(ref profile) = payload.profile { let _ = storage.store_profile(profile); @@ -8148,11 +8921,20 @@ async fn process_exchange_payload( } } + // Third-party address hearsay: filter to routable (bugs-fixed #8, receive + // side) and MERGE rather than replace (bugs-fixed #5 — a peer must not be + // able to overwrite the address we are successfully using for a node). for pa in &payload.peer_addresses { if let Ok(nid) = crate::parse_node_id_hex(&pa.n) { - let addrs: Vec = pa.a.iter().filter_map(|a| a.parse().ok()).collect(); + let addrs: Vec = pa + .a + .iter() + .filter_map(|a| a.parse::().ok()) + .map(normalize_addr) + .filter(convection_addr_ok) + .collect(); if !addrs.is_empty() { - let _ = storage.upsert_peer(&nid, &addrs, None); + let _ = storage.add_peer_addresses(&nid, &addrs); } } } @@ -8164,12 +8946,17 @@ async fn process_exchange_payload( } } - // Process anchor's advertised address + // Process anchor's advertised address. First-party (we are connected to + // them) but still self-declared: UNION it in, never replace the address we + // dialled. `known_anchors` is written on a proven connect, not here. if let Some(ref anchor_addr_str) = payload.anchor_addr { if let Ok(sock) = anchor_addr_str.parse::() { - let _ = storage.upsert_known_anchor(remote_node_id, &[sock]); - let _ = storage.upsert_peer(remote_node_id, &[sock], None); - info!(peer = hex::encode(remote_node_id), addr = %sock, "Stored anchor's advertised address"); + let sock = normalize_addr(sock); + if convection_addr_ok(&sock) { + let _ = storage.add_peer_addresses(remote_node_id, &[sock]); + let _ = storage.set_peer_anchor(remote_node_id, true); + info!(peer = hex::encode(remote_node_id), addr = %sock, "Stored anchor's advertised address"); + } } } @@ -8982,3 +9769,806 @@ mod tests { assert!(set.lock().unwrap().is_empty(), "all guards dropped — set should be empty"); } } + +#[cfg(test)] +mod uniques_tests { + //! v0.8 Iteration C: slot pool + two-pool uniques announce. + + use super::{ + announce_digest, apply_uniques_to_storage, build_uniques_payload, decide_slot, + entries_to_slice, slice_to_entries, MAX_UNIQUES_PAYLOAD, TEMP_REFERRAL_TTL_MS, + }; + use crate::protocol::{unpack_ids, AnchorEntry, UniquesAnnouncePayload, UniquesSlice}; + use crate::storage::Storage; + use crate::types::{IdClass, MeshSlot, NodeId, ReachEntry}; + + fn temp_storage() -> Storage { + let dir = std::env::temp_dir().join(format!("itsgoin-uniq-{}", rand::random::())); + std::fs::create_dir_all(&dir).unwrap(); + Storage::open(dir.join("test.db")).unwrap() + } + + fn nid(b: u8) -> NodeId { + [b; 32] + } + + // ---- Slot accounting ---- + + #[test] + fn mesh_slots_fill_before_temp_referral_slots() { + // Under the cap: always a mesh slot. + assert_eq!(decide_slot(0, 20, 0, 10, 0), Some(MeshSlot::Mesh)); + assert_eq!(decide_slot(19, 20, 0, 10, 0), Some(MeshSlot::Mesh)); + + // At the cap: temp referral, ABOVE the cap — the mesh pool is untouched. + let s = decide_slot(20, 20, 0, 10, 1_000).unwrap(); + assert_eq!(s, MeshSlot::TempReferral { expires_at_ms: 1_000 + TEMP_REFERRAL_TTL_MS }); + assert!(s.is_temp()); + + // Temp band full too: refuse. Note the total (20 + 10) exceeds the mesh + // cap by exactly the referral band — that is the point. + assert_eq!(decide_slot(20, 20, 10, 10, 0), None); + + // Mobile numbers. + assert_eq!(decide_slot(14, 15, 4, 4, 0), Some(MeshSlot::Mesh)); + assert_eq!(decide_slot(15, 15, 4, 4, 0), None); + } + + #[test] + fn graduation_requires_an_already_free_mesh_slot() { + // A temp peer can only become mesh when a mesh slot is genuinely free. + // Full mesh => the arriving peer is temp, and nothing is evicted. + assert!(decide_slot(20, 20, 0, 10, 0).unwrap().is_temp()); + // One mesh peer drops => the next allocation is mesh again. + assert!(decide_slot(19, 20, 1, 10, 0).unwrap().is_mesh()); + } + + // ---- Wire shape ---- + + #[test] + fn anchor_entries_carry_addresses_everything_else_is_bare() { + let plain = nid(1); + let persona = nid(2); + let anchor = nid(3); + + let slice = entries_to_slice(&[ + ReachEntry::node(plain), + ReachEntry::author(persona), + ReachEntry::anchor(anchor, vec!["203.0.113.5:4433".into()]), + ]); + + assert_eq!(unpack_ids(&slice.nodes), vec![plain]); + assert_eq!(unpack_ids(&slice.authors), vec![persona]); + assert_eq!(slice.anchors.len(), 1); + assert_eq!(slice.anchors[0].i, hex::encode(anchor)); + assert_eq!(slice.anchors[0].a, vec!["203.0.113.5:4433".to_string()]); + + // The wire type has no address field on the bare arrays at all, so a + // non-anchor entry structurally cannot carry one. Round-trip proves it. + let back = slice_to_entries(&slice); + for e in &back { + if e.id == anchor { + assert!(e.is_anchor && !e.addresses.is_empty()); + } else { + assert!(!e.is_anchor, "{:?} must not be an anchor", e.id); + assert!(e.addresses.is_empty(), "non-anchor entries are bare"); + } + } + + // Classes survive the round trip — a persona never becomes connectable. + assert_eq!( + back.iter().find(|e| e.id == persona).unwrap().id_class, + IdClass::Author + ); + assert_eq!(back.iter().find(|e| e.id == plain).unwrap().id_class, IdClass::Node); + } + + #[test] + fn unroutable_anchor_addresses_are_filtered_off_the_wire() { + // bugs-fixed #8: private/loopback ranges never get announced. + let anchor = nid(3); + let slice = entries_to_slice(&[ReachEntry::anchor( + anchor, + vec!["10.0.0.5:4433".into(), "192.168.1.9:4433".into()], + )]); + assert!(slice.anchors.is_empty(), "private addresses must not be announced"); + // It degrades to a bare node entry rather than vanishing. + assert_eq!(unpack_ids(&slice.nodes), vec![anchor]); + } + + // ---- Build / apply ---- + + #[test] + fn built_announce_places_pools_correctly_and_omits_n4() { + let s = temp_storage(); + let us = nid(0); + let reporter = nid(1); + let mesh_peer = nid(2); + let n2 = nid(3); + let n3 = nid(4); + let n4 = nid(5); + + s.add_mesh_peer(&mesh_peer).unwrap(); + s.set_reach(&reporter, 2, &[ReachEntry::node(n2)]).unwrap(); + s.set_reach(&reporter, 3, &[ReachEntry::node(n3)]).unwrap(); + s.set_reach(&reporter, 4, &[ReachEntry::node(n4)]).unwrap(); + + let p = build_uniques_payload(&s, &us, None, 7, 4, 4).unwrap(); + assert!(p.full); + assert_eq!(p.seq, 7); + assert_eq!(p.depth, 4); + assert_eq!(p.fwd.len(), 3); + + assert_eq!(unpack_ids(&p.fwd[0].nodes), vec![us], "fwd[0] is N0 = ourselves"); + assert!(unpack_ids(&p.fwd[1].nodes).contains(&mesh_peer), "fwd[1] is our own uniques"); + assert_eq!(unpack_ids(&p.fwd[2].nodes), vec![n2], "fwd[2] is our N2"); + assert_eq!(unpack_ids(&p.term.nodes), vec![n3], "term is our N3"); + + // The whole point: N4 is terminal and never re-announced. + let all: Vec = p + .fwd + .iter() + .chain(std::iter::once(&p.term)) + .flat_map(|sl| { + unpack_ids(&sl.nodes) + .into_iter() + .chain(unpack_ids(&sl.authors)) + .collect::>() + }) + .collect(); + assert!(!all.contains(&n4), "N4 must never appear in an announcement"); + } + + #[test] + fn shallow_peer_depth_strips_the_terminal_pool() { + let s = temp_storage(); + let us = nid(0); + s.set_reach(&nid(1), 3, &[ReachEntry::node(nid(9))]).unwrap(); + + let full = build_uniques_payload(&s, &us, None, 1, 4, 4).unwrap(); + assert!(!full.term.is_empty()); + + // Peer advertised depth 3 — nowhere to put N4, so don't build it. + let trimmed = build_uniques_payload(&s, &us, None, 1, 4, 3).unwrap(); + assert!(trimmed.term.is_empty()); + // Pool 1 is unaffected — only the terminal pool is skipped. + assert_eq!(unpack_ids(&trimmed.fwd[0].nodes), vec![us]); + } + + #[test] + fn apply_shifts_pool1_one_bounce_deeper_and_lands_pool2_at_n4() { + let s = temp_storage(); + let us = nid(0); + let reporter = nid(1); + + let their_n1 = nid(10); + let their_n2 = nid(11); + let their_n3 = nid(12); + + let payload = UniquesAnnouncePayload { + seq: 3, + full: true, + fwd: vec![ + entries_to_slice(&[ReachEntry::node(reporter)]), + entries_to_slice(&[ReachEntry::node(their_n1)]), + entries_to_slice(&[ReachEntry::node(their_n2)]), + ], + term: entries_to_slice(&[ReachEntry::node(their_n3)]), + depth: 4, + }; + + let stored = apply_uniques_to_storage(&s, &us, &reporter, &payload, 4, |_| false).unwrap(); + assert_eq!(stored, 3); + + assert_eq!(s.find_in_n2(&their_n1).unwrap(), vec![reporter], "fwd[1] -> our N2"); + assert_eq!(s.find_in_n3(&their_n2).unwrap(), vec![reporter], "fwd[2] -> our N3"); + assert_eq!(s.find_in_n4(&their_n3).unwrap(), vec![reporter], "term -> our N4"); + + // N4 is populated and usable... + let hits = s.find_any_reachable(&[their_n3]).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].2, 4); + + // ...but it does not leak back out into what WE announce. + let ours = build_uniques_payload(&s, &us, None, 1, 4, 4).unwrap(); + let announced: Vec = ours + .fwd + .iter() + .chain(std::iter::once(&ours.term)) + .flat_map(|sl| unpack_ids(&sl.nodes)) + .collect(); + assert!(announced.contains(&their_n1)); + assert!(announced.contains(&their_n2)); + assert!( + !announced.contains(&their_n3), + "an ID learned at N4 must never be forwarded (no N5)" + ); + } + + #[test] + fn shallow_device_drops_the_terminal_pool_on_receipt() { + let s = temp_storage(); + let us = nid(0); + let reporter = nid(1); + let deep = nid(12); + + let payload = UniquesAnnouncePayload { + seq: 1, + full: true, + fwd: vec![UniquesSlice::default(), UniquesSlice::default(), UniquesSlice::default()], + term: entries_to_slice(&[ReachEntry::node(deep)]), + depth: 4, + }; + + // max_bounce = 3 (mobile): the terminal pool is discarded, not stored. + apply_uniques_to_storage(&s, &us, &reporter, &payload, 3, |_| false).unwrap(); + assert_eq!(s.count_distinct_n4().unwrap(), 0); + + // max_bounce = 4 (desktop): stored. + apply_uniques_to_storage(&s, &us, &reporter, &payload, 4, |_| false).unwrap(); + assert_eq!(s.count_distinct_n4().unwrap(), 1); + } + + #[test] + fn apply_filters_self_and_already_connected_peers() { + let s = temp_storage(); + let us = nid(0); + let reporter = nid(1); + let already = nid(2); + let fresh = nid(3); + + let payload = UniquesAnnouncePayload { + seq: 1, + full: true, + fwd: vec![ + UniquesSlice::default(), + entries_to_slice(&[ + ReachEntry::node(us), + ReachEntry::node(already), + ReachEntry::node(fresh), + ]), + UniquesSlice::default(), + ], + term: UniquesSlice::default(), + depth: 4, + }; + + apply_uniques_to_storage(&s, &us, &reporter, &payload, 4, |n| *n == already).unwrap(); + assert!(s.find_in_n2(&us).unwrap().is_empty(), "never index ourselves"); + assert!(s.find_in_n2(&already).unwrap().is_empty(), "already meshed, not a candidate"); + assert_eq!(s.find_in_n2(&fresh).unwrap(), vec![reporter]); + } + + /// Third-party anchor entries are an INDEX, not an address record. They are + /// mineable from the pools and nothing else — no `peers` row, no + /// `known_anchors` seed, no `is_anchor` latch — because nothing has proven + /// the address belongs to that node ID. + #[test] + fn received_anchor_entries_are_index_only_never_authoritative() { + let s = temp_storage(); + let us = nid(0); + let reporter = nid(1); + let anchor = nid(7); + + let payload = UniquesAnnouncePayload { + seq: 1, + full: true, + fwd: vec![ + UniquesSlice::default(), + entries_to_slice(&[ReachEntry::anchor(anchor, vec!["198.51.100.9:4433".into()])]), + UniquesSlice::default(), + ], + term: UniquesSlice::default(), + depth: 4, + }; + apply_uniques_to_storage(&s, &us, &reporter, &payload, 4, |_| false).unwrap(); + + // Mineable from the pools — this IS the anchor directory. + let mined = s.list_pool_anchors(10).unwrap(); + assert_eq!(mined.len(), 1); + assert_eq!(mined[0].0, anchor); + + // But authoritative address state is untouched: a hostile peer must not + // be able to overwrite our address for any node (bugs-fixed #5, + // remotely triggerable) or re-inflate the bootstrap cache from gossip. + assert!(s.get_peer_record(&anchor).unwrap().is_none(), "no peers row from hearsay"); + assert!(s.list_known_anchors().unwrap().is_empty(), "bootstrap cache untouched"); + assert!(!s.is_peer_anchor(&anchor).unwrap(), "no is_anchor latch from hearsay"); + } + + /// Round-4/5: "a node knocked down from 20 peers to 4 still holds 16 dead + /// peers' pools full of anchor addresses to reconnect through." Disconnect + /// must NOT wipe the reporter's rows; a new handshake overwrites them. + #[test] + fn a_departed_reporters_pool_is_retained_until_a_new_handshake_replaces_it() { + let s = temp_storage(); + let reporter = nid(1); + let anchor = nid(7); + let other = nid(8); + + s.set_reach( + &reporter, + 2, + &[ + ReachEntry::anchor(anchor, vec!["198.51.100.9:4433".into()]), + ReachEntry::node(other), + ], + ) + .unwrap(); + + // The peer leaves. `disconnect_peer` no longer clears anything, so the + // mine and the growth-candidate reserve survive the exact mass- + // disconnect the retention rule was written for. + assert_eq!(s.list_pool_anchors(10).unwrap().len(), 1); + assert!(!s.score_n2_candidates_batch().unwrap().is_empty()); + assert!(s.anchor_density().unwrap() > 0.0); + + // It comes back with a different view: set_reach DELETEs per + // (reporter, bounce) first, so rows are replaced, never duplicated. + s.set_reach(&reporter, 2, &[ReachEntry::node(other)]).unwrap(); + assert!(s.list_pool_anchors(10).unwrap().is_empty(), "overwritten by the new handshake"); + assert_eq!(s.find_in_n2(&other).unwrap(), vec![reporter]); + } + + /// THE terminal-pool invariant, round-tripped through an ANCHOR entry — + /// the case the earlier tests missed. An anchor learned at bounce 4 must + /// appear in NO slice of the next announce. Before the fix it was mirrored + /// into `peers.is_anchor` and then re-emitted by `build_own_uniques` at + /// pool-1 depth 1, resetting its horizon on every hop forever. + #[test] + fn an_anchor_known_only_at_n4_is_never_re_announced() { + let s = temp_storage(); + let us = nid(0); + let reporter = nid(1); + let deep_anchor = nid(9); + + let payload = UniquesAnnouncePayload { + seq: 1, + full: true, + fwd: vec![UniquesSlice::default(), UniquesSlice::default(), UniquesSlice::default()], + term: entries_to_slice(&[ReachEntry::anchor( + deep_anchor, + vec!["198.51.100.9:4433".into()], + )]), + depth: 4, + }; + apply_uniques_to_storage(&s, &us, &reporter, &payload, 4, |_| false).unwrap(); + assert_eq!(s.find_in_n4(&deep_anchor).unwrap(), vec![reporter], "stored at N4"); + + let out = build_uniques_payload(&s, &us, None, 2, 4, 4).unwrap(); + for (i, slice) in out.fwd.iter().enumerate() { + let ids: Vec = slice_to_entries(slice).into_iter().map(|e| e.id).collect(); + assert!(!ids.contains(&deep_anchor), "N4 anchor leaked into fwd[{}]", i); + } + let term_ids: Vec = + slice_to_entries(&out.term).into_iter().map(|e| e.id).collect(); + assert!(!term_ids.contains(&deep_anchor), "N4 anchor leaked into the terminal pool"); + } + + /// `fwd` is assumed to be exactly 3 slices by every reader. A 6-slice + /// payload used to write rows at bounce 5 and 6 — depths with no semantics + /// anywhere, invisible to the announce builder but very visible to + /// `find_any_reachable`, the growth scorer and `anchor_density`. + #[test] + fn a_payload_with_the_wrong_fwd_length_is_rejected() { + let s = temp_storage(); + let us = nid(0); + let reporter = nid(1); + let mut fwd = vec![UniquesSlice::default(); 6]; + fwd[4] = entries_to_slice(&[ReachEntry::node(nid(42))]); + let payload = UniquesAnnouncePayload { + seq: 1, + full: true, + fwd, + term: UniquesSlice::default(), + depth: 4, + }; + assert!(apply_uniques_to_storage(&s, &us, &reporter, &payload, 4, |_| false).is_err()); + let max: i64 = s.max_stored_bounce().unwrap(); + assert_eq!(max, 0, "nothing written at all"); + } + + /// The §layers size budget must be ENFORCED, not merely documented: with + /// `MAX_PAYLOAD` at 64 MB a single announce could decode into ~1.5M IDs and + /// the same number of row inserts, per bounce, per reporter. + #[test] + fn oversized_slices_are_truncated_and_a_built_announce_stays_under_the_ceiling() { + let s = temp_storage(); + let us = nid(0); + let reporter = nid(1); + + let cap = crate::storage::uniques_accept_cap(2); + let flood: Vec = (0..(cap + 5_000)) + .map(|i| { + let mut id = [0u8; 32]; + id[0] = (i % 251) as u8; + id[1] = ((i / 251) % 251) as u8; + id[2] = (i / (251 * 251)) as u8; + ReachEntry::node(id) + }) + .collect(); + let payload = UniquesAnnouncePayload { + seq: 1, + full: true, + fwd: vec![ + UniquesSlice::default(), + entries_to_slice(&flood), + UniquesSlice::default(), + ], + term: UniquesSlice::default(), + depth: 4, + }; + let stored = apply_uniques_to_storage(&s, &us, &reporter, &payload, 4, |_| false).unwrap(); + assert!(stored <= cap, "accepted {} rows, cap is {}", stored, cap); + + // ...and what WE build back out of that index stays under the wire cap. + let out = build_uniques_payload(&s, &us, None, 2, 4, 4).unwrap(); + let bytes = serde_json::to_vec(&out).unwrap().len(); + assert!(bytes < MAX_UNIQUES_PAYLOAD, "built announce is {} bytes", bytes); + assert!(out.fwd[2].len() <= crate::storage::UNIQUES_BUILD_CAP_SHALLOW); + } + + /// bugs-fixed #8, receive side. The send side always filtered; the receive + /// side did not, so a peer could seed us with `127.0.0.1:22` / `10.x` / + /// `192.168.x` and we would store and later dial them. + #[test] + fn unroutable_addresses_are_dropped_on_receipt() { + let bad = UniquesSlice { + nodes: crate::protocol::pack_ids(&[]), + authors: crate::protocol::pack_ids(&[]), + anchors: vec![AnchorEntry { + i: hex::encode(nid(5)), + a: vec!["127.0.0.1:22".into(), "10.0.0.1:4433".into(), "192.168.1.1:443".into()], + }], + }; + let entries = slice_to_entries(&bad); + assert_eq!(entries.len(), 1); + assert!(!entries[0].is_anchor, "degrades to a bare node entry"); + assert!(entries[0].addresses.is_empty()); + } + + /// The address invariant must be STRUCTURAL. `id_class` and `is_anchor` are + /// independent columns, so an author-class row carrying the anchor flag + /// must still serialise as a bare persona ID — never next to a device + /// address (design.html §21). + #[test] + fn an_author_class_entry_never_serialises_as_an_address_bearing_anchor() { + let e = ReachEntry { + id: nid(3), + id_class: IdClass::Author, + is_anchor: true, + addresses: vec!["198.51.100.9:4433".into()], + }; + let slice = entries_to_slice(&[e]); + assert!(slice.anchors.is_empty(), "no address-bearing entry for a persona ID"); + assert_eq!(unpack_ids(&slice.authors), vec![nid(3)]); + assert!(unpack_ids(&slice.nodes).is_empty()); + } + + #[test] + fn announce_digest_ignores_seq_but_tracks_content() { + let s = temp_storage(); + let us = nid(0); + let a = build_uniques_payload(&s, &us, None, 1, 4, 4).unwrap(); + let b = build_uniques_payload(&s, &us, None, 99, 4, 4).unwrap(); + assert_eq!(announce_digest(&a), announce_digest(&b), "seq must not affect the digest"); + + s.add_mesh_peer(&nid(5)).unwrap(); + let c = build_uniques_payload(&s, &us, None, 1, 4, 4).unwrap(); + assert_ne!(announce_digest(&a), announce_digest(&c)); + } +} + +#[cfg(test)] +mod convection_tests { + //! v0.8 Iteration C: anchor convection + the stochastic growth trigger. + //! + //! The window rules and the action choice are pure functions precisely so + //! they can be tested without standing up an iroh endpoint. + + use super::{ + choose_convection_action, convection_should_serve, convection_weights, + convection_window_admit, convection_window_pick, convection_window_viable, + disconnect_response, ConvectionAction, ConvectionEntry, DisconnectResponse, + CONVECTION_ENTRY_MAX_AGE_MS, CONVECTION_HANDOUT_CAP, CONVECTION_REFERRALS, + CONVECTION_WINDOW_SIZE, + }; + use crate::protocol::ConvectionClass; + use crate::storage::Storage; + use crate::types::{NodeId, ReachEntry}; + use std::collections::VecDeque; + + fn nid(b: u8) -> NodeId { + [b; 32] + } + + fn addr(b: u8) -> Vec { + vec![format!("203.0.113.{}:4433", b)] + } + + fn window() -> VecDeque { + VecDeque::new() + } + + fn nobody_live(_: &NodeId) -> bool { + false + } + + fn temp_storage() -> Storage { + let dir = std::env::temp_dir().join(format!("itsgoin-conv-{}", rand::random::())); + std::fs::create_dir_all(&dir).unwrap(); + Storage::open(dir.join("test.db")).unwrap() + } + + // ---- Rolling window: 2 before, 2 after ---- + + #[test] + fn a_caller_gets_the_two_most_recent_prior_callers() { + let mut w = window(); + // Callers arrive 1, 2, 3. + for i in 1..=3u8 { + convection_window_admit(&mut w, nid(i), addr(i), 1_000 + i as u64); + } + // Caller 4 arrives and asks. It must get the two MOST RECENT priors + // (3 and 2), not the oldest and not all three. + let picked = convection_window_pick(&mut w, &nid(4), CONVECTION_REFERRALS, 2_000, nobody_live); + let got: Vec = picked.iter().map(|e| e.node_id[0]).collect(); + assert_eq!(got, vec![3, 2], "most recent first, exactly 2"); + } + + #[test] + fn an_entry_is_handed_to_two_callers_then_rotates_out() { + let mut w = window(); + convection_window_admit(&mut w, nid(1), addr(1), 1_000); + convection_window_admit(&mut w, nid(2), addr(2), 1_001); + + // Caller A takes both. + let a = convection_window_pick(&mut w, &nid(9), 2, 2_000, nobody_live); + assert_eq!(a.len(), 2); + assert_eq!(w.len(), 2, "one hand-out each, still under the cap"); + + // Caller B takes both again — that is hand-out #2 for each. + let b = convection_window_pick(&mut w, &nid(9), 2, 2_001, nobody_live); + assert_eq!(b.len(), 2); + assert_eq!(CONVECTION_HANDOUT_CAP, 2); + assert!(w.is_empty(), "both entries rotated out after 2 hand-outs"); + + // Caller C gets nothing from the window. + let c = convection_window_pick(&mut w, &nid(9), 2, 2_002, nobody_live); + assert!(c.is_empty()); + } + + #[test] + fn a_caller_is_never_referred_to_itself() { + let mut w = window(); + convection_window_admit(&mut w, nid(1), addr(1), 1_000); + let picked = convection_window_pick(&mut w, &nid(1), 2, 1_500, nobody_live); + assert!(picked.is_empty()); + assert_eq!(w.len(), 1, "and its own entry is not spent"); + } + + #[test] + fn still_connected_callers_are_preferred_over_more_recent_cold_ones() { + let mut w = window(); + // 1 is older but still connected; 2 and 3 are newer and gone. + convection_window_admit(&mut w, nid(1), addr(1), 1_000); + convection_window_admit(&mut w, nid(2), addr(2), 2_000); + convection_window_admit(&mut w, nid(3), addr(3), 3_000); + + let live = |n: &NodeId| n[0] == 1; + let picked = convection_window_pick(&mut w, &nid(9), 2, 4_000, live); + let got: Vec = picked.iter().map(|e| e.node_id[0]).collect(); + assert_eq!(got[0], 1, "the live caller leads even though it is oldest"); + assert_eq!(got[1], 3, "then the most recent of the rest"); + } + + #[test] + fn a_refreshed_caller_gets_a_new_handout_budget_and_moves_to_the_back() { + let mut w = window(); + convection_window_admit(&mut w, nid(1), addr(1), 1_000); + convection_window_admit(&mut w, nid(2), addr(2), 1_001); + // Spend one hand-out on caller 1. + let _ = convection_window_pick(&mut w, &nid(2), 1, 1_500, |n: &NodeId| n[0] == 1); + assert_eq!(w.iter().find(|e| e.node_id == nid(1)).unwrap().handouts, 1); + + // Caller 1 calls again — it is fresh material once more. + convection_window_admit(&mut w, nid(1), addr(1), 2_000); + let e = w.iter().find(|e| e.node_id == nid(1)).unwrap(); + assert_eq!(e.handouts, 0); + assert_eq!(w.back().unwrap().node_id, nid(1), "newest sits at the back"); + assert_eq!(w.len(), 2, "refreshed, not duplicated"); + } + + #[test] + fn the_window_is_bounded_in_both_count_and_age() { + let mut w = window(); + for i in 1..=(CONVECTION_WINDOW_SIZE as u8 + 5) { + convection_window_admit(&mut w, nid(i), addr(i), 1_000 + i as u64); + } + assert_eq!(w.len(), CONVECTION_WINDOW_SIZE); + assert_eq!(w.front().unwrap().node_id[0], 6, "oldest fell off the front"); + + // A stale entry is not worth handing out — it costs the next caller a + // punch timeout. + let mut w2 = window(); + convection_window_admit(&mut w2, nid(1), addr(1), 0); + let late = CONVECTION_ENTRY_MAX_AGE_MS + 1; + assert_eq!(convection_window_viable(&w2, &nid(9), late), 0); + assert!(convection_window_pick(&mut w2, &nid(9), 2, late, nobody_live).is_empty()); + } + + #[test] + fn an_address_less_caller_is_not_admitted() { + let mut w = window(); + convection_window_admit(&mut w, nid(1), vec![], 1_000); + assert!(w.is_empty(), "nothing to hand out means nothing to store"); + } + + // ---- Request classes ---- + + #[test] + fn entry_is_always_served_and_top_up_is_refusable() { + // ENTRY: served with an empty window, served while saturated. A node + // that cannot get in cannot come back. + assert!(convection_should_serve(ConvectionClass::Entry, 0, false)); + assert!(convection_should_serve(ConvectionClass::Entry, 0, true)); + assert!(convection_should_serve(ConvectionClass::Entry, 5, true)); + + // TOP-UP: refused when there is nothing fresh to circulate, or when we + // are saturated. Served otherwise — top-ups are the convection medium, + // not freeloading, so they are not throttled beyond that. + assert!(!convection_should_serve(ConvectionClass::TopUp, 0, false)); + assert!(!convection_should_serve(ConvectionClass::TopUp, 5, true)); + assert!(convection_should_serve(ConvectionClass::TopUp, 1, false)); + + // And the class bit is derived purely from "can this node function". + assert!(ConvectionClass::for_mesh_count(1).is_entry()); + assert!(!ConvectionClass::for_mesh_count(2).is_entry()); + } + + // ---- Adaptive weights ---- + + #[test] + fn the_anchor_density_prior_sets_the_anchor_arm() { + let sparse = convection_weights(0.0, 1.0, 0.5); + let dense = convection_weights(0.30, 1.0, 0.5); + assert!( + dense.anchor > sparse.anchor, + "abundant anchors (IPv6 world) => anchor-led growth: {} vs {}", + dense.anchor, + sparse.anchor + ); + // Same code in both worlds: with no anchors known the anchor arm is + // small but never zero — we still have to be able to discover one. + assert!(sparse.anchor > 0.0); + assert!(dense.mesh > 0.0, "and the mesh arm never disappears either"); + } + + #[test] + fn refusal_feedback_shifts_weight_away_from_the_anchor_arm() { + let density = 0.30; + let served = convection_weights(density, 1.0, 0.5); + // AIMD: a refusal halves the bias. + let after_refusal = convection_weights(density, 0.5, 0.5); + let after_two = convection_weights(density, 0.25, 0.5); + assert!(after_refusal.anchor < served.anchor); + assert!(after_two.anchor < after_refusal.anchor); + assert_eq!(after_refusal.mesh, served.mesh, "mesh arm is untouched…"); + // …so the SHARE going to mesh rises, which is the point. + let share = |w: super::ConvectionWeights| w.mesh / (w.nothing + w.anchor + w.mesh); + assert!(share(after_refusal) > share(served)); + + // A roll that picked the anchor while healthy now picks mesh. + let roll = 0.45; + assert_eq!(choose_convection_action(roll, density, 1.0, 0.5), ConvectionAction::AskAnchor); + assert_eq!(choose_convection_action(roll, density, 0.1, 0.5), ConvectionAction::AskMesh); + } + + #[test] + fn the_action_leans_toward_acting_the_further_under_target_we_are() { + // Far under target: do-nothing weight is zero, so EVERY roll acts. + for i in 0..20 { + let roll = i as f64 / 20.0; + assert_ne!( + choose_convection_action(roll, 0.2, 1.0, 0.0), + ConvectionAction::Nothing, + "empty mesh must always act (roll {})", + roll + ); + } + // Near target: do-nothing dominates. + let full = convection_weights(0.2, 1.0, 1.0); + assert!(full.nothing > full.anchor + full.mesh); + + // No threshold anywhere: the choice varies continuously with the roll, + // and all three arms are reachable at a mid fill ratio. + let mut seen = std::collections::HashSet::new(); + for i in 0..100 { + seen.insert(choose_convection_action(i as f64 / 100.0, 0.3, 1.0, 0.6)); + } + assert_eq!(seen.len(), 3, "all three arms reachable: {:?}", seen); + } + + // ---- Recovery is not stochastic ---- + + #[test] + fn recovery_pre_empts_the_dice_and_never_rolls() { + // Below 2 mesh peers: always recover, whatever the slot situation. + for mesh in 0..2usize { + for free in [0usize, 5, 20] { + assert_eq!( + disconnect_response(mesh, free), + DisconnectResponse::Recover, + "mesh {} free {}", + mesh, + free + ); + } + } + // At or above 2: stochastic when there is room, idle when there isn't. + assert_eq!(disconnect_response(2, 1), DisconnectResponse::Roll); + assert_eq!(disconnect_response(19, 1), DisconnectResponse::Roll); + assert_eq!(disconnect_response(20, 0), DisconnectResponse::Idle); + } + + // ---- Anchors mined from the pools ---- + + #[test] + fn anchor_density_and_the_anchor_directory_come_from_the_pools() { + let s = temp_storage(); + let reporter = nid(200); + // 4 node-class uniques, 1 of them anchor-flagged with an address. + let entries = vec![ + ReachEntry::anchor(nid(1), vec!["203.0.113.1:4433".into()]), + ReachEntry::node(nid(2)), + ReachEntry::node(nid(3)), + ReachEntry::node(nid(4)), + // Author-class IDs are NOT connect targets and must not skew the + // prior or appear in the directory. + ReachEntry::author(nid(5)), + ]; + s.set_reach(&reporter, 2, &entries).unwrap(); + + let density = s.anchor_density().unwrap(); + assert!( + (density - 0.25).abs() < 1e-9, + "1 anchor of 4 node-class uniques, authors excluded: got {}", + density + ); + + // The pools double as the anchor directory — no separate registry, and + // only anchor entries carry an address to mine. + let anchors = s.list_pool_anchors(16).unwrap(); + assert_eq!(anchors.len(), 1); + assert_eq!(anchors[0].0, nid(1)); + assert_eq!(anchors[0].1, vec!["203.0.113.1:4433".to_string()]); + + // Zero knowledge => zero prior, and the weights still work. + let empty = temp_storage(); + assert_eq!(empty.anchor_density().unwrap(), 0.0); + assert!(empty.list_pool_anchors(16).unwrap().is_empty()); + assert!(convection_weights(0.0, 1.0, 0.5).anchor > 0.0); + } + + #[test] + fn a_disconnected_reporters_anchor_addresses_survive_for_the_cold_start_mine() { + // Round-4/5: "cold reconnect mines the RETAINED pools for anchor + // addresses" — the pool itself is the thing that survives, because + // disconnect no longer wipes it. That, not a peers/known_anchors + // mirror, is what keeps the adaptive anchor-density prior meaningful + // after churn (`anchor_density` counts `reachable` rows only). + let s = temp_storage(); + let reporter = nid(200); + let anchor = nid(1); + let sa: std::net::SocketAddr = "203.0.113.7:4433".parse().unwrap(); + s.set_reach(&reporter, 2, &[ReachEntry::anchor(anchor, vec![sa.to_string()])]).unwrap(); + + // The reporter departs. Nothing is cleared. + assert_eq!(s.list_pool_anchors(16).unwrap().len(), 1, "the mine survives the disconnect"); + assert!(s.anchor_density().unwrap() > 0.0, "and so does the adaptive prior"); + + // Only the 5h staleness sweep removes it. + s.prune_reach(0).unwrap(); + assert!(s.list_pool_anchors(16).unwrap().is_empty()); + } +} diff --git a/crates/core/src/group_key_distribution.rs b/crates/core/src/group_key_distribution.rs index 115afb4..d835757 100644 --- a/crates/core/src/group_key_distribution.rs +++ b/crates/core/src/group_key_distribution.rs @@ -14,7 +14,7 @@ use crate::content::compute_post_id; use crate::crypto; use crate::storage::Storage; use crate::types::{ - GroupKeyDistributionContent, GroupKeyRecord, GroupMemberKey, NodeId, Post, PostId, + GroupKeyDistributionContent, GroupKeyRecord, NodeId, Post, PostId, PostVisibility, PostingIdentity, VisibilityIntent, }; diff --git a/crates/core/src/network.rs b/crates/core/src/network.rs index 6710dca..16274b9 100644 --- a/crates/core/src/network.rs +++ b/crates/core/src/network.rs @@ -10,16 +10,15 @@ use tracing::{debug, error, info, warn}; use crate::activity::{ActivityCategory, ActivityLevel, ActivityLog}; use crate::blob::BlobStore; use crate::connection::{initial_exchange_accept, initial_exchange_connect, ConnHandle, ConnectionActor, ConnectionManager, ExchangeResult}; -use crate::content::verify_post_id; use crate::protocol::{ read_message_type, read_payload, write_typed_message, BlobRequestPayload, BlobResponsePayload, MessageType, ProfileUpdatePayload, - PullSyncRequestPayload, PullSyncResponsePayload, RefuseRedirectPayload, + RefuseRedirectPayload, ALPN, }; use crate::storage::StoragePool; use crate::types::{ - DeviceProfile, DeviceRole, NodeId, PeerSlotKind, Post, PostId, + DeviceProfile, DeviceRole, NodeId, MeshSlot, Post, PostId, PostVisibility, PublicProfile, SessionReachMethod, WormResult, }; @@ -734,13 +733,16 @@ impl Network { recv: iroh::endpoint::RecvStream, is_mesh: &AtomicBool, ) -> bool { - // Try to allocate a slot - let accepted = conn_handle.accept_connection( + // Try to allocate a slot. `None` = both the mesh pool and the temp + // referral band are full. + let slot = conn_handle.accept_connection( conn.clone(), remote_node_id, Some(remote_sock), ).await; - info!(peer = hex::encode(remote_node_id), accepted, "try_mesh_upgrade: accept_connection result"); - if !accepted { + info!(peer = hex::encode(remote_node_id), accepted = slot.is_some(), "try_mesh_upgrade: accept_connection result"); + let slot = match slot { + Some(s) => s, + None => { let redirect = conn_handle.pick_random_redirect_peer(&remote_node_id).await; let payload = RefuseRedirectPayload { reason: "slots full".to_string(), @@ -752,13 +754,19 @@ impl Network { debug!(peer = hex::encode(remote_node_id), "Refused mesh connection (slots full)"); conn_handle.add_session(remote_node_id, conn.clone(), crate::types::SessionReachMethod::Direct, Some(remote_sock)).await; conn_handle.log_activity(ActivityLevel::Info, ActivityCategory::Connection, format!("Refused {} mesh (slots full), added session", &hex::encode(remote_node_id)[..8]), Some(remote_node_id)); - return false; - } + return false; + } + }; { let s = storage.get().await; + // bugs-fixed #2: address always persisted, temp slots included. let _ = s.upsert_peer(&remote_node_id, &[remote_sock], None); - let _ = s.add_mesh_peer(&remote_node_id, PeerSlotKind::Local, 0); + // But mesh_peers ONLY for a real mesh slot — that omission is what + // keeps temp referral peers out of our uniques announce. + if slot.is_mesh() { + let _ = s.add_mesh_peer(&remote_node_id); + } if s.has_social_route(&remote_node_id).unwrap_or(false) { let _ = s.touch_social_route_connect( &remote_node_id, @@ -774,9 +782,10 @@ impl Network { let our_nat_type = conn_handle.nat_type().await; let our_http_capable = conn_handle.is_http_capable(); let our_http_addr = conn_handle.http_addr(); - match initial_exchange_accept(storage, &our_node_id, send, recv, remote_node_id, anchor_addr, Some(remote_sock), our_nat_type, our_http_capable, our_http_addr, conn_handle.device_role(), None, false).await { + let our_depth = conn_handle.max_bounce(); + match initial_exchange_accept(storage, &our_node_id, send, recv, remote_node_id, anchor_addr, Some(remote_sock), our_nat_type, our_http_capable, our_http_addr, conn_handle.device_role(), None, false, our_depth, slot.is_mesh()).await { Ok(()) => { - info!(peer = hex::encode(remote_node_id), "Initial exchange complete (upgraded to mesh)"); + info!(peer = hex::encode(remote_node_id), slot = %slot, "Initial exchange complete (upgraded to mesh)"); conn_handle.log_activity(ActivityLevel::Info, ActivityCategory::Connection, format!("Upgraded {} to mesh", &hex::encode(remote_node_id)[..8]), Some(remote_node_id)); } Err(e) => { @@ -819,12 +828,15 @@ impl Network { let conn = ConnectionManager::connect_to_unlocked(&self.endpoint, addr).await?; // Register the established connection - self.conn_handle.register_connection(peer_id, conn.clone(), addrs, PeerSlotKind::Local).await; + let slot = match self.conn_handle.register_connection(peer_id, conn.clone(), addrs).await { + Some(s) => s, + None => anyhow::bail!("no slot available (mesh + temp full)"), + }; let anchor_addr = self.conn_handle.build_anchor_advertised_addr().await; let our_nat_type = self.conn_handle.nat_type().await; // Initial exchange WITHOUT holding conn_mgr lock - match initial_exchange_connect(&self.storage, &self.our_node_id, &conn, peer_id, anchor_addr, our_nat_type, self.is_http_capable(), self.http_addr(), Some(self.device_role), None).await? { + match initial_exchange_connect(&self.storage, &self.our_node_id, &conn, peer_id, anchor_addr, our_nat_type, self.is_http_capable(), self.http_addr(), Some(self.device_role), None, self.conn_handle.max_bounce(), slot.is_mesh()).await? { ExchangeResult::Accepted { duplicate_active } => { if duplicate_active { self.duplicate_detected.store(true, std::sync::atomic::Ordering::Relaxed); @@ -870,27 +882,38 @@ impl Network { } } - /// Pull from all connected peers. - pub async fn pull_from_all(&self) -> anyhow::Result { + /// TRANSITIONAL content sync from every connected peer, plus the + /// engagement fetch that Iteration A's registry and greetings ride on. + /// + /// Routes through the actor path (`content_sync_unlocked`), NOT a second + /// local implementation: the old `Network::pull_from_peer` stored posts + /// with `store_post_with_visibility` instead of `control::receive_post`, so + /// it silently dropped `intent` (control posts — deletes, revocations, key + /// distributions — were never processed) and never called + /// `touch_file_holder`, which is why posts received on the startup pull + /// registered no CDN holder. + pub async fn content_sync_all(&self) -> anyhow::Result { let peers = self.conn_handle.connected_peers().await; let mut total_posts = 0; let mut success = 0; for peer_id in peers { - // Uses Network::pull_from_peer which doesn't hold conn_mgr lock during I/O - let result = self.pull_from_peer(&peer_id).await; + let result = self.conn_handle.content_sync_from_peer(&peer_id).await; match result { Ok(stats) => { total_posts += stats.posts_received; success += 1; - // Also fetch engagement data + // Engagement fetch is what carries Iteration A's registry + // entries and greeting comments (BlobHeaderRequest/Response + // + BlobHeaderDiff). It MUST stay driven on a timer — this + // is the pull side of that safety net. let _ = self.conn_handle.fetch_engagement_from_peer(&peer_id).await; if stats.posts_received > 0 { info!( peer = hex::encode(peer_id), posts = stats.posts_received, - "Pulled posts" + "Content sync: received posts" ); } } @@ -898,7 +921,7 @@ impl Network { debug!( peer = hex::encode(peer_id), error = %e, - "Pull failed" + "Content sync failed" ); } } @@ -910,82 +933,85 @@ impl Network { }) } - /// Broadcast routing diff to all connected peers. - /// Uses ConnHandle to get diff data, then sends outside the lock. - pub async fn broadcast_diff(&self) -> anyhow::Result { - use crate::protocol::{NodeListUpdatePayload, write_typed_message, MessageType}; - - let snapshot = self.conn_handle.get_diff_data().await; - - if snapshot.n1_added.is_empty() && snapshot.n1_removed.is_empty() - && snapshot.n2_added.is_empty() && snapshot.n2_removed.is_empty() - { - return Ok(0); + /// Run the uniques-index exchange (0x40/0x41) against every MESH peer. + /// + /// This is what a "pull" means in v0.8: an exchange of the IDs each side + /// has a live path to. Symmetric, so one round trip refreshes both indexes. + /// Temp referral slots are skipped — they carry no knowledge. + pub async fn uniques_pull_all(&self) -> usize { + let conns = self.conn_handle.get_connection_map().await; + let mut exchanged = 0; + for (peer_id, conn, slot, _) in conns { + if !slot.is_mesh() { + continue; + } + let fut = crate::connection::ConnectionManager::uniques_pull_unlocked( + &self.conn_mgr, + conn, + &peer_id, + ); + match tokio::time::timeout(std::time::Duration::from_secs(20), fut).await { + Ok(Ok(n)) => { + exchanged += 1; + if n > 0 { + debug!(peer = hex::encode(peer_id), rows = n, "Uniques pull: index updated"); + } + } + Ok(Err(e)) => debug!(peer = hex::encode(peer_id), error = %e, "Uniques pull failed"), + Err(_) => debug!(peer = hex::encode(peer_id), "Uniques pull timed out"), + } } + exchanged + } - let payload = NodeListUpdatePayload { - seq: snapshot.diff_seq, - n1_added: snapshot.n1_added, - n1_removed: snapshot.n1_removed, - n2_added: snapshot.n2_added, - n2_removed: snapshot.n2_removed, + /// Broadcast our uniques announce to every MESH peer. + /// + /// Temporary referral slots are excluded — they carry no knowledge + /// exchange. Peers that advertised `depth < 4` get the terminal pool + /// stripped before send (the biggest saving on a cellular link). + /// + /// The announce is always a full snapshot; `snapshot.unchanged` is what + /// stands in for incremental diffing, so an idle mesh sends nothing. + pub async fn broadcast_uniques(&self) -> anyhow::Result { + use crate::protocol::{write_typed_message, MessageType, UniquesSlice}; + + let snapshot = self.conn_handle.get_announce_data().await; + let payload = match snapshot.payload { + Some(p) if !snapshot.unchanged => p, + _ => return Ok(0), }; let mut sent = 0; - for (peer_id, conn) in &snapshot.connections { + for (peer_id, conn, peer_depth) in &snapshot.connections { + let per_peer = crate::protocol::UniquesAnnouncePayload { + seq: payload.seq, + full: payload.full, + fwd: payload.fwd.clone(), + term: if *peer_depth >= 4 { payload.term.clone() } else { UniquesSlice::default() }, + depth: payload.depth, + }; let result = async { let mut send = conn.open_uni().await?; - write_typed_message(&mut send, MessageType::NodeListUpdate, &payload).await?; + write_typed_message(&mut send, MessageType::UniquesAnnounce, &per_peer).await?; send.finish()?; anyhow::Ok(()) }.await; if result.is_ok() { sent += 1; } else { - debug!(peer = hex::encode(peer_id), "Failed to send routing diff"); + debug!(peer = hex::encode(peer_id), "Failed to send uniques announce"); } } Ok(sent) } - /// Broadcast full N1/N2 state to all mesh peers (periodic catch-up for missed diffs). + /// Periodic catch-up: force the next announce even if the pools are + /// unchanged, so a peer that missed one resynchronises. + /// (The old `broadcast_full_state` is subsumed — every announce is already + /// a full snapshot.) pub async fn broadcast_full_state(&self) -> anyhow::Result { - use crate::protocol::{NodeListUpdatePayload, write_typed_message, MessageType}; - - let snapshot = self.conn_handle.get_diff_data().await; - - // Build full state: all current N1 and N2 as "added", nothing removed - let all_n1 = self.conn_handle.connected_peers().await; - let all_n2 = { - let storage = self.storage.get().await; - storage.build_n2_share().unwrap_or_default() - }; - - if all_n1.is_empty() && all_n2.is_empty() { - return Ok(0); - } - - let payload = NodeListUpdatePayload { - seq: snapshot.diff_seq, - n1_added: all_n1, - n1_removed: vec![], - n2_added: all_n2, - n2_removed: vec![], - }; - - let mut sent = 0; - for (_peer_id, conn) in &snapshot.connections { - let result = async { - let mut send = conn.open_uni().await?; - write_typed_message(&mut send, MessageType::NodeListUpdate, &payload).await?; - send.finish()?; - anyhow::Ok(()) - }.await; - if result.is_ok() { - sent += 1; - } - } - Ok(sent) + self.conn_handle.force_announce().await; + self.broadcast_uniques().await } /// Push a profile update to all audience members (ephemeral-capable). @@ -1191,7 +1217,7 @@ impl Network { } /// Get connection info for display. - pub async fn connection_info(&self) -> Vec<(NodeId, PeerSlotKind, u64)> { + pub async fn connection_info(&self) -> Vec<(NodeId, MeshSlot, u64)> { self.conn_handle.connection_info().await } @@ -1226,11 +1252,15 @@ impl Network { } match self.connect_to_peer(peer_id, addr.clone()).await { - Ok(()) => Ok(()), + Ok(()) => { + self.promote_proven_anchor(peer_id, &addr).await; + Ok(()) + } Err(e) if e.to_string().contains("mesh refused") => { // Anchor refused mesh — reconnect as session for registration - let conn = ConnectionManager::connect_to_unlocked(&self.endpoint, addr).await?; + let conn = ConnectionManager::connect_to_unlocked(&self.endpoint, addr.clone()).await?; self.conn_handle.add_session(peer_id, conn, crate::types::SessionReachMethod::Direct, None).await; + self.promote_proven_anchor(peer_id, &addr).await; self.conn_handle.log_activity( ActivityLevel::Info, ActivityCategory::Anchor, @@ -1243,6 +1273,28 @@ impl Network { } } + /// An anchor we have COMPLETED a QUIC handshake to. iroh has proven the + /// node ID owns this path, so — and only so — the entry may enter the + /// authoritative address state: `peers.is_anchor` and the `known_anchors` + /// bootstrap cache. + /// + /// This is the counterpart to removing the announce-path mirror. Hearsay + /// anchor claims live in `reachable` only (mineable via `list_pool_anchors`, + /// which is what "the pools ARE the anchor directory" means); proven ones + /// live here, which is also what makes `known_anchors.last_seen_ms` + /// ordering meaningful again — and keeps the DNS-refreshed bootstrap anchor + /// from being evicted by a flood of gossip-sourced entries. + async fn promote_proven_anchor(&self, peer_id: NodeId, addr: &iroh::EndpointAddr) { + let socks: Vec = addr.ip_addrs().copied().collect(); + if socks.is_empty() { + return; + } + let storage = self.storage.get().await; + let _ = storage.upsert_peer(&peer_id, &socks, None); + let _ = storage.set_peer_anchor(&peer_id, true); + let _ = storage.upsert_known_anchor(&peer_id, &socks); + } + /// Register an already-established QUIC connection as a mesh peer. /// Sends InitialExchange first — if the remote accepts (responds with /// InitialExchange), registers as mesh and spawns the stream loop. @@ -1259,13 +1311,15 @@ impl Network { let anchor_addr = self.conn_handle.build_anchor_advertised_addr().await; let our_nat_type = self.conn_handle.nat_type().await; - match initial_exchange_connect(&self.storage, &self.our_node_id, &conn, peer_id, anchor_addr, our_nat_type, self.is_http_capable(), self.http_addr(), Some(self.device_role), None).await { + // Decide the slot BEFORE the exchange: the slot class is what says + // whether this handshake carries a uniques announce. + // bugs-fixed #3: a connection is registered before anything can drop it. + let slot = match self.conn_handle.register_connection(peer_id, conn.clone(), vec![]).await { + Some(s) => s, + None => anyhow::bail!("no slot available (mesh + temp full)"), + }; + match initial_exchange_connect(&self.storage, &self.our_node_id, &conn, peer_id, anchor_addr, our_nat_type, self.is_http_capable(), self.http_addr(), Some(self.device_role), None, self.conn_handle.max_bounce(), slot.is_mesh()).await { Ok(ExchangeResult::Accepted { .. }) => { - self.conn_handle.register_connection(peer_id, conn.clone(), vec![], PeerSlotKind::Local).await; - { - let s = self.storage.get().await; - let _ = s.add_mesh_peer(&peer_id, PeerSlotKind::Local, 0); - } // Spawn the per-connection stream loop let conn_data = self.conn_handle.get_connection_map().await; @@ -1280,10 +1334,16 @@ impl Network { Ok(ExchangeResult::Refused { redirect }) => { let redir_info = redirect.as_ref().map(|r| r.n.clone()); info!(peer = hex::encode(peer_id), redirect = ?redir_info, "Mesh refused after hole punch, keeping as session"); + self.conn_handle.disconnect_peer(&peer_id).await; self.conn_handle.add_session(peer_id, conn, crate::types::SessionReachMethod::HolePunch, None).await; anyhow::bail!("mesh refused: slots full"); } - Err(e) => Err(e), + Err(e) => { + // Roll back the slot we reserved — otherwise a failed handshake + // holds a mesh slot until the zombie sweep. + self.conn_handle.disconnect_peer(&peer_id).await; + Err(e) + } } } @@ -1344,7 +1404,9 @@ impl Network { None, ); - if remaining == 0 { + // Recovery threshold is mesh < 2 (design.html §lifecycle), not zero: + // a single remaining peer is a partition waiting to happen. + if remaining < 2 { self.conn_handle.notify_recovery(); } else { self.conn_handle.notify_growth(); @@ -1364,7 +1426,7 @@ impl Network { for peer_id in &newly_connected { let conn = self.conn_handle.get_connection(peer_id).await; if let Some(conn) = conn { - match initial_exchange_connect(&self.storage, &self.our_node_id, &conn, *peer_id, anchor_addr.clone(), our_nat_type, self.is_http_capable(), self.http_addr(), Some(self.device_role), None).await { + match initial_exchange_connect(&self.storage, &self.our_node_id, &conn, *peer_id, anchor_addr.clone(), our_nat_type, self.is_http_capable(), self.http_addr(), Some(self.device_role), None, self.conn_handle.max_bounce(), true).await { Ok(ExchangeResult::Accepted { .. }) => {} Ok(ExchangeResult::Refused { redirect }) => { debug!(peer = hex::encode(peer_id), "Auto-connect refused, disconnecting"); @@ -1444,10 +1506,10 @@ impl Network { loop { // Check slots + pick candidate via ConnHandle (no lock contention) - let available = self.conn_handle.available_local_slots().await; + let available = self.conn_handle.available_mesh_slots().await; if available == 0 { - debug!("Growth loop: local slots full"); - self.log_activity(ActivityLevel::Info, ActivityCategory::Growth, "Local slots full".into(), None); + debug!("Growth loop: mesh slots full"); + self.log_activity(ActivityLevel::Info, ActivityCategory::Growth, "Mesh slots full".into(), None); break; } let candidate = { @@ -1476,25 +1538,39 @@ impl Network { self.log_activity(ActivityLevel::Info, ActivityCategory::Growth, format!("Trying candidate {} (score {:.1})", &hex::encode(candidate_id)[..8], score), Some(candidate_id)); // Resolve address via ConnHandle (no lock during I/O) + let asked_a_reporter; let addr_str = { let local_addr = self.conn_handle.resolve_peer_addr_local(&candidate_id).await; if let Some(endpoint_addr) = local_addr { + asked_a_reporter = true; endpoint_addr.ip_addrs().next().map(|a| a.to_string()) } else { - // Network resolution: get reporter connections, resolve outside lock + // Network resolution: get reporter connections, resolve + // outside lock. The horizon here must match the horizon + // the CANDIDATE SCORER uses (2..=4, same as + // `ConnectionManager::resolve_address`) — when it did + // not, the growth loop scored N4 candidates it then had + // no reporter to ask about, and burned a + // `consecutive_failures` slot plus an UNREACHABLE_EXPIRY + // suppression on every one of them. let reporters_and_conns = { let storage = self.storage.get().await; - let n2 = storage.find_in_n2(&candidate_id).unwrap_or_default(); - let n3 = storage.find_in_n3(&candidate_id).unwrap_or_default(); + let mut reporter_set: std::collections::HashSet = + std::collections::HashSet::new(); + for bounce in 2..=4u8 { + for r in storage.find_reporters_at(&candidate_id, bounce).unwrap_or_default() { + reporter_set.insert(r); + } + } drop(storage); let conn_map = self.conn_handle.get_connection_map().await; - let reporter_set: std::collections::HashSet = n2.into_iter().chain(n3).collect(); conn_map.into_iter() .filter(|(nid, _, _, _)| reporter_set.contains(nid)) .map(|(_, conn, _, _)| conn) .collect::>() }; + asked_a_reporter = !reporters_and_conns.is_empty(); let mut resolved = None; for conn in reporters_and_conns { let result: anyhow::Result> = async { @@ -1521,10 +1597,19 @@ impl Network { None => { debug!( peer = hex::encode(candidate_id), - "Growth loop: no address, marking unreachable" + asked_a_reporter, + "Growth loop: no address" ); self.log_activity(ActivityLevel::Warn, ActivityCategory::Growth, format!("No address for {}", &hex::encode(candidate_id)[..8]), Some(candidate_id)); - self.conn_handle.mark_unreachable(&candidate_id); + // Only a candidate we could actually ask about, and + // that came back with nothing, has earned an + // UNREACHABLE_EXPIRY suppression. Failing for want of a + // connected reporter says nothing about the candidate — + // suppressing it there would hide it even after a + // shallower reporter shows up. + if asked_a_reporter { + self.conn_handle.mark_unreachable(&candidate_id); + } consecutive_failures += 1; if consecutive_failures >= 3 { debug!("Growth loop: 3 consecutive failures, backing off"); @@ -1560,7 +1645,7 @@ impl Network { consecutive_failures = 0; // Broadcast diff so peers learn about our new connection - let _ = self.broadcast_diff().await; + let _ = self.broadcast_uniques().await; // Brief pause to let InitialExchange update N2/N3 before next pick tokio::time::sleep(std::time::Duration::from_millis(500)).await; @@ -1618,7 +1703,7 @@ impl Network { if introduced { consecutive_failures = 0; - let _ = self.broadcast_diff().await; + let _ = self.broadcast_uniques().await; tokio::time::sleep(std::time::Duration::from_millis(500)).await; } else { self.conn_handle.mark_unreachable(&candidate_id); @@ -1635,64 +1720,6 @@ impl Network { } } -/// Pull posts from a peer (persistent if available, ephemeral otherwise). - pub async fn pull_from_peer(&self, peer_id: &NodeId) -> anyhow::Result { - let conn = self.get_connection(peer_id).await?; - let (our_follows, follows_sync, our_personas) = { - let storage = self.storage.get().await; - ( - storage.list_follows()?, - storage.get_follows_with_last_sync().unwrap_or_default(), - storage.list_posting_identities().unwrap_or_default(), - ) - }; - // Merged pull: include every posting identity we hold so DMs addressed - // to any of our personas match on recipient. Our network NodeId is - // never an author nor a wrapped_key recipient — including it would - // never match and would leak the network↔posting boundary. - let mut query_list = our_follows; - for pi in &our_personas { - if !query_list.contains(&pi.node_id) { - query_list.push(pi.node_id); - } - } - let (mut send, mut recv) = conn.open_bi().await?; - write_typed_message( - &mut send, - MessageType::PullSyncRequest, - &PullSyncRequestPayload { - follows: query_list, - since_ms: follows_sync, - }, - ) - .await?; - send.finish()?; - let msg_type = read_message_type(&mut recv).await?; - if msg_type != MessageType::PullSyncResponse { - anyhow::bail!("expected PullSyncResponse, got {:?}", msg_type); - } - let response: PullSyncResponsePayload = read_payload(&mut recv, 64 * 1024 * 1024).await?; - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64; - let storage = self.storage.get().await; - let mut posts_received = 0; - for sp in &response.posts { - if !storage.is_deleted(&sp.id)? && verify_post_id(&sp.id, &sp.post) { - if storage.store_post_with_visibility(&sp.id, &sp.post, &sp.visibility)? { - posts_received += 1; - } - // Protocol v4: update last_sync_ms for the author - let _ = storage.update_follow_last_sync(&sp.post.author, now_ms); - } - } - Ok(PullStats { - peers_pulled: 1, - posts_received, - }) - } - /// Send a uni-stream message. Uses persistent connection if available, ephemeral otherwise. pub async fn send_to_peer_uni( &self, @@ -1940,41 +1967,146 @@ impl Network { Some(addr) } - // ---- Anchor referral delegation ---- + // ---- Anchor convection delegation (design.html §anchors) ---- - /// Request peer referrals from an anchor peer (mesh or session). - /// Does NOT hold conn_mgr lock during network I/O. - pub async fn request_anchor_referrals( + /// Ask an anchor for peers. The request itself enrols us in that anchor's + /// rolling window — there is no registration message; 0xC0 is retired. + /// + /// Does NOT hold the conn_mgr lock during network I/O. A refusal comes back + /// as a single small message, so the `refused` case costs one round trip, + /// not a timeout. + pub async fn request_convection( &self, anchor: &NodeId, - ) -> anyhow::Result> { + class: crate::protocol::ConvectionClass, + ) -> anyhow::Result { let conn = self.conn_handle.get_any_connection(anchor).await .ok_or_else(|| anyhow::anyhow!("anchor peer not connected (mesh or session)"))?; + // Self-reported addresses, best first: the stable advertised address + // (if we are an anchor ourselves), then the UPnP external mapping, then + // whatever the endpoint knows. The anchor prefers what it OBSERVES over + // all of these; these only help when it cannot observe us. let endpoint = self.conn_handle.endpoint().await; - let our_addrs: Vec = endpoint.addr().ip_addrs() + let mut our_addrs: Vec = endpoint.addr().ip_addrs() + .filter(|s| is_publicly_routable(s)) .map(|s| s.to_string()) .collect(); + if let Some(ext) = self.conn_handle.upnp_external_addr().await { + let ext_str = ext.to_string(); + if !our_addrs.contains(&ext_str) { + our_addrs.insert(0, ext_str); + } + } + if let Some(anchor_addr) = self.conn_handle.build_anchor_advertised_addr().await { + if !our_addrs.contains(&anchor_addr) { + our_addrs.insert(0, anchor_addr); + } + } - let request = crate::protocol::AnchorReferralRequestPayload { + let request = crate::protocol::ConvectionRequestPayload { requester: self.our_node_id, requester_addresses: our_addrs, + class, }; let (mut send, mut recv) = conn.open_bi().await?; - crate::protocol::write_typed_message(&mut send, crate::protocol::MessageType::AnchorReferralRequest, &request).await?; + crate::protocol::write_typed_message(&mut send, crate::protocol::MessageType::ConvectionRequest, &request).await?; send.finish()?; let msg_type = crate::protocol::read_message_type(&mut recv).await?; - if msg_type != crate::protocol::MessageType::AnchorReferralResponse { - anyhow::bail!("expected AnchorReferralResponse, got {:?}", msg_type); + if msg_type != crate::protocol::MessageType::ConvectionResponse { + anyhow::bail!("expected ConvectionResponse, got {:?}", msg_type); } - let response: crate::protocol::AnchorReferralResponsePayload = crate::protocol::read_payload(&mut recv, 4096).await?; + let response: crate::protocol::ConvectionResponsePayload = + crate::protocol::read_payload(&mut recv, 4096).await?; // Touch session to prevent idle reaping self.conn_handle.touch_session_if_exists(anchor); - Ok(response.referrals) + // Adaptive feedback (round 5): a refusal shifts weight toward mesh + // introductions and other anchors; a served request drifts it back. + if response.refused { + debug!(anchor = hex::encode(anchor), "Convection: refused (top-up, no capacity)"); + self.conn_handle.record_convection_refusal(anchor); + } else { + self.conn_handle.record_convection_success(anchor); + } + + // The NAT filter probe used to ride the register message. It has to + // ride something, and this is now the only message we send an anchor. + if self.conn_handle.nat_filtering().await == crate::types::NatFiltering::Unknown { + if let Err(e) = self.request_nat_filter_probe(anchor).await { + debug!(error = %e, "NAT filter probe request failed"); + } + } + + Ok(response) + } + + /// Act on one convection response: connect to each referral, landing it in + /// whatever slot class is free (mesh first, then the temp referral band + /// above the cap). Returns how many connections were established. + /// + /// `introduced` referrals skip the extra introduction round trip — the + /// anchor already pushed a RelayIntroduce toward them naming us, so they + /// are punching outward while we dial. + pub async fn act_on_convection( + &self, + anchor: &NodeId, + response: &crate::protocol::ConvectionResponsePayload, + ) -> usize { + let mut connected = 0; + for referral in &response.referrals { + if referral.node_id == self.our_node_id { + continue; + } + if self.conn_handle.is_connected(&referral.node_id).await { + continue; + } + let mut ok = false; + // bugs-fixed #8 (receive side): a referral address is a third + // party's claim. Dialling `127.0.0.1:22` or `192.168.1.1:443` + // because an anchor said so is attacker-directed internal probing + // from our own host, and every such connect() enters iroh's + // per-endpoint path store permanently (v0.7.3 EDM lesson). + let usable: Option<&String> = referral.addresses.iter().find(|a| { + a.parse::() + .map(|sa| crate::connection::convection_addr_ok(&sa)) + .unwrap_or(false) + }); + if let Some(addr_str) = usable { + let connect_str = format!("{}@{}", hex::encode(referral.node_id), addr_str); + if let Ok((rid, raddr)) = crate::parse_connect_string(&connect_str) { + let fut = self.connect_to_peer(rid, raddr); + match tokio::time::timeout(std::time::Duration::from_secs(15), fut).await { + Ok(Ok(())) => { + info!(peer = hex::encode(rid), "Convection: connected to referred peer"); + ok = true; + } + Ok(Err(e)) => debug!(error = %e, peer = hex::encode(rid), "Convection: direct connect failed"), + Err(_) => debug!(peer = hex::encode(rid), "Convection: direct connect timed out"), + } + } + } + if !ok && !referral.introduced { + // No coordinated introduction was possible — ask for one. + match self.connect_via_introduction_as_mesh(referral.node_id, *anchor).await { + Ok(()) => { + info!(peer = hex::encode(referral.node_id), "Convection: connected via introduction"); + ok = true; + } + Err(e) => debug!(error = %e, peer = hex::encode(referral.node_id), "Convection: introduction failed"), + } + } + if ok { + connected += 1; + } + } + if connected > 0 { + self.notify_growth().await; + } + connected } /// Request NAT filter probe from an anchor (determines address-restricted vs port-restricted). @@ -2028,55 +2160,6 @@ impl Network { Ok(()) } - /// Register our address with an anchor peer (mesh or session). - /// Also runs NAT filter probe if filtering is still Unknown. - /// Does NOT hold conn_mgr lock during network I/O. - pub async fn send_anchor_register(&self, anchor: &NodeId) -> anyhow::Result<()> { - let conn = self.conn_handle.get_any_connection(anchor).await - .ok_or_else(|| anyhow::anyhow!("anchor peer not connected (mesh or session)"))?; - - let endpoint = self.conn_handle.endpoint().await; - let mut our_addrs: Vec = endpoint.addr().ip_addrs() - .map(|s| s.to_string()) - .collect(); - // Prepend UPnP external address (most useful for remote peers) - if let Some(ext) = self.conn_handle.upnp_external_addr().await { - let ext_str = ext.to_string(); - if !our_addrs.contains(&ext_str) { - our_addrs.insert(0, ext_str); - } - } - // Prepend stable anchor advertised address - if let Some(anchor_addr) = self.conn_handle.build_anchor_advertised_addr().await { - if !our_addrs.contains(&anchor_addr) { - our_addrs.insert(0, anchor_addr); - } - } - - let payload = crate::protocol::AnchorRegisterPayload { - node_id: self.our_node_id, - addresses: our_addrs, - }; - - let mut send = conn.open_uni().await?; - crate::protocol::write_typed_message(&mut send, crate::protocol::MessageType::AnchorRegister, &payload).await?; - send.finish()?; - - // Touch session to prevent idle reaping - self.conn_handle.touch_session_if_exists(anchor); - - debug!(anchor = hex::encode(anchor), "Registered with anchor"); - - // Also run NAT filter probe if filtering is still Unknown - let needs_probe = self.conn_handle.nat_filtering().await == crate::types::NatFiltering::Unknown; - if needs_probe { - if let Err(e) = self.request_nat_filter_probe(anchor).await { - debug!(error = %e, "NAT filter probe request failed"); - } - } - Ok(()) - } - /// Check if a peer is a known anchor. pub async fn is_anchor_peer(&self, node_id: &NodeId) -> bool { let storage = self.storage.get().await; diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index aa33950..e9e9c4c 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -13,7 +13,7 @@ use crate::network::Network; use crate::storage::StoragePool; use crate::types::{ Attachment, Circle, - DeviceProfile, DeviceRole, NodeId, PeerRecord, PeerSlotKind, PeerWithAddress, Post, PostId, + DeviceProfile, DeviceRole, NodeId, PeerRecord, MeshSlot, PeerWithAddress, Post, PostId, PostVisibility, PublicProfile, ReachMethod, RevocationMode, SessionReachMethod, SocialRelation, SocialRouteEntry, SocialStatus, VisibilityIntent, WormResult, }; @@ -55,11 +55,12 @@ pub struct Node { bootstrap_anchors: tokio::sync::Mutex>, /// True if an anchor reported another instance of this identity is already active pub duplicate_detected: Arc, - #[allow(dead_code)] profile: DeviceProfile, pub activity_log: Arc>, pub last_rebalance_ms: Arc, - pub last_anchor_register_ms: Arc, + /// Last time the convection loop acted (replaces the retired anchor + /// register cycle's timer). + pub last_convection_ms: Arc, /// CDN replication budget: bytes remaining we're willing to pull and cache this hour replication_budget_remaining: Arc, /// CDN delivery budget: bytes remaining we're willing to serve this hour @@ -250,6 +251,127 @@ async fn probe_anchors_batched( result } +/// Gather anchor candidates, POOL FIRST (round-4 ruling). +/// +/// Phase 0: anchor-flagged entries mined out of the uniques pools. Anchor +/// entries are the only address-bearing rows in the index, so the +/// pools double as the anchor directory — including the retained +/// pools of peers that have since disconnected (slot knowledge is +/// overwritten memory, wiped when a new handshake takes the slot, +/// not when a peer leaves). +/// Phase 1: `known_anchors` — DEMOTED to a bootstrap cache. Its `success_count` +/// ordering was a v0.7.x scarcity artifact and no longer means +/// anything now that pools supply anchors in bulk. +/// Phase 2: peers flagged `is_anchor`. +/// +/// Anchors currently in the refusal penalty box are pushed to the back rather +/// than dropped: a refusal is a load signal, not a blacklist. +async fn gather_anchor_candidates( + storage: &Arc, + network: &crate::network::Network, + self_node_id: NodeId, + limit: usize, +) -> Vec<(NodeId, Vec)> { + let mut out: Vec<(NodeId, Vec)> = Vec::new(); + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + seen.insert(self_node_id); + + { + let s = storage.get().await; + // Phase 0 — pool-mined. + for (nid, addrs) in s.list_pool_anchors(limit).unwrap_or_default() { + if !seen.insert(nid) { + continue; + } + let socks: Vec = + addrs.iter().filter_map(|a| a.parse().ok()).collect(); + if !socks.is_empty() { + out.push((nid, socks)); + } + } + // Phase 1 — bootstrap cache. + for (nid, addrs) in s.list_known_anchors().unwrap_or_default() { + if seen.insert(nid) && !addrs.is_empty() { + out.push((nid, addrs)); + } + } + // Phase 2 — anchor-flagged peers. + for r in s.list_anchor_peers().unwrap_or_default() { + if seen.insert(r.node_id) && !r.addresses.is_empty() { + out.push((r.node_id, r.addresses)); + } + } + } + + // De-prioritise (do not drop) anchors that recently refused us. + let mut penalized = Vec::new(); + let mut fresh = Vec::new(); + for entry in out { + if network.conn_handle().is_anchor_penalized(&entry.0).await { + penalized.push(entry); + } else { + fresh.push(entry); + } + } + fresh.extend(penalized); + fresh.truncate(limit); + fresh +} + +/// One convection exchange against one anchor: connect if needed, ask, act. +/// +/// Replaces the four hand-rolled `request_anchor_referrals` → `connect_to_peer` +/// → `connect_via_introduction` blocks (bootstrap x2, recovery, register cycle) +/// that had drifted apart. Returns how many peer connections it produced. +async fn run_convection( + network: &Arc, + anchor_nid: NodeId, + anchor_addrs: &[std::net::SocketAddr], + class: crate::protocol::ConvectionClass, + self_node_id: NodeId, +) -> usize { + if anchor_nid == self_node_id { + return 0; + } + if !network.is_peer_connected_or_session(&anchor_nid).await { + let endpoint_id = match iroh::EndpointId::from_bytes(&anchor_nid) { + Ok(eid) => eid, + Err(_) => return 0, + }; + let mut addr = iroh::EndpointAddr::from(endpoint_id); + for sa in anchor_addrs { + addr = addr.with_ip_addr(*sa); + } + if let Err(e) = network.connect_to_anchor(anchor_nid, addr).await { + debug!(error = %e, anchor = hex::encode(anchor_nid), "Convection: anchor connect failed"); + return 0; + } + } + + // A refusal is one small message — the 10s ceiling is for the connect leg, + // never for the refusal itself. + let response = match tokio::time::timeout( + std::time::Duration::from_secs(10), + network.request_convection(&anchor_nid, class), + ).await { + Ok(Ok(r)) => r, + Ok(Err(e)) => { + debug!(error = %e, anchor = hex::encode(anchor_nid), "Convection request failed"); + return 0; + } + Err(_) => { + debug!(anchor = hex::encode(anchor_nid), "Convection request timed out"); + return 0; + } + }; + + if response.refused { + // Feedback already recorded inside request_convection. + return 0; + } + network.act_on_convection(&anchor_nid, &response).await +} + async fn probe_one_anchor( network: &crate::network::Network, storage: &Arc, @@ -368,7 +490,7 @@ impl Node { // Startup sweep: clear stale N2/N3 and mesh_peers from prior session { let s = storage.get().await; - let n_cleared = s.clear_all_n2_n3().unwrap_or(0); + let n_cleared = s.clear_all_reach().unwrap_or(0); let m_cleared = s.clear_all_mesh_peers().unwrap_or(0); if n_cleared > 0 || m_cleared > 0 { info!(n2_n3 = n_cleared, mesh_peers = m_cleared, "Startup sweep: cleared stale entries"); @@ -428,10 +550,8 @@ impl Node { // Open blob store let blob_store = Arc::new(BlobStore::open(&data_dir)?); - // Activity log + timer atomics + // Activity log let activity_log = Arc::new(std::sync::Mutex::new(ActivityLog::new())); - let last_rebalance_ms = Arc::new(AtomicU64::new(0)); - let last_anchor_register_ms = Arc::new(AtomicU64::new(0)); // Start network (single ALPN, connection manager) let network = Arc::new( @@ -459,7 +579,7 @@ impl Node { // Build the node (fast path — no network I/O beyond endpoint creation) let activity_log_ref = Arc::clone(&activity_log); let last_rebalance_ms = Arc::new(AtomicU64::new(0)); - let last_anchor_register_ms = Arc::new(AtomicU64::new(0)); + let last_convection_ms = Arc::new(AtomicU64::new(0)); let role = network.device_role(); let (replication_budget, delivery_budget) = (role.replication_limit(), role.delivery_limit()); @@ -471,7 +591,7 @@ impl Node { )); blob_store.set_delivery_budget(delivery_budget); - let mut node = Self { + let node = Self { data_dir: data_dir.clone(), storage: Arc::clone(&storage), network: Arc::clone(&network), @@ -484,7 +604,7 @@ impl Node { profile, activity_log: activity_log_ref, last_rebalance_ms, - last_anchor_register_ms, + last_convection_ms, replication_budget_remaining, delivery_budget_remaining, budget_last_reset_ms, @@ -643,7 +763,7 @@ impl Node { Ok(()) => { info!(peer = hex::encode(nid), "Bootstrap: connected"); // Pull posts from the bootstrap peer - match network.pull_from_all().await { + match network.content_sync_all().await { Ok(stats) => { info!( "Bootstrap pull: {} posts from {} peers", @@ -666,52 +786,25 @@ impl Node { } } - // Request referrals from anchor (10s timeout) - match tokio::time::timeout(std::time::Duration::from_secs(10), network.request_anchor_referrals(&nid)).await { - Ok(Ok(referrals)) if !referrals.is_empty() => { - info!(count = referrals.len(), "Bootstrap: got anchor referrals"); - // Spawn referral connections in background — don't block startup - let net = Arc::clone(&network); - let my_id = node_id; - let anchor = nid; - tokio::spawn(async move { - for referral in referrals { - if referral.node_id == my_id { - continue; - } - if let Some(addr_str) = referral.addresses.first() { - let connect_str = format!( - "{}@{}", - hex::encode(referral.node_id), - addr_str, - ); - if let Ok((rid, raddr)) = crate::parse_connect_string(&connect_str) { - let connect_fut = async { - match net.connect_to_peer(rid, raddr).await { - Ok(()) => { info!(peer = hex::encode(rid), "Connected to referred peer"); Ok(()) }, - Err(e) => { - debug!(error = %e, peer = hex::encode(rid), "One-sided connect failed, requesting introduction from anchor"); - match net.connect_via_introduction(rid, anchor).await { - Ok(()) => { info!(peer = hex::encode(rid), "Connected to referred peer via hole punch"); Ok(()) }, - Err(e2) => Err(e2), - } - } - } - }; - match tokio::time::timeout(std::time::Duration::from_secs(15), connect_fut).await { - Ok(Ok(())) => {}, - Ok(Err(e)) => debug!(error = %e, peer = hex::encode(rid), "Bootstrap referral connect failed"), - Err(_) => debug!(peer = hex::encode(rid), "Bootstrap referral connect timed out"), - } - } - } - } - net.notify_growth().await; - }); - } - Ok(Ok(_)) => debug!("Bootstrap: no referrals from anchor (first to register)"), - Ok(Err(e)) => debug!(error = %e, "Bootstrap: referral request failed"), - Err(_) => debug!("Bootstrap: referral request timed out"), + // Convection, ENTRY class — we have no mesh + // yet, so this is always served. Spawned so + // startup isn't blocked on peer connects. + { + let net = Arc::clone(&network); + let my_id = node_id; + let anchor = nid; + let anchor_addrs = ip_addrs.clone(); + tokio::spawn(async move { + let n = run_convection( + &net, + anchor, + &anchor_addrs, + crate::protocol::ConvectionClass::Entry, + my_id, + ).await; + info!(connected = n, "Bootstrap: convection complete"); + net.notify_growth().await; + }); } break; } @@ -792,10 +885,9 @@ impl Node { { let conn_count = network.connection_count().await; if conn_count < 5 { - let known = { - let s = storage.get().await; - s.list_known_anchors().unwrap_or_default() - }; + // Pool-mined anchors FIRST (round-4), then the known_anchors + // bootstrap cache, then anchor-flagged peers. + let known = gather_anchor_candidates(storage, network, node_id, 32).await; // Split into discovered anchors (priority) and bootstrap anchors (fallback) let (discovered, bootstrap_known): (Vec<_>, Vec<_>) = known.into_iter() .partition(|(nid, _)| !bootstrap_anchor_ids.contains(nid)); @@ -834,49 +926,23 @@ impl Node { Ok(Err(e)) => warn!(error = %e, "NAT filter probe failed during bootstrap"), Err(_) => warn!("NAT filter probe timed out during bootstrap"), } - match tokio::time::timeout(std::time::Duration::from_secs(10), network.request_anchor_referrals(&anchor_nid)).await { - Ok(Ok(referrals)) if !referrals.is_empty() => { - info!(count = referrals.len(), "Got anchor referrals"); - let net = Arc::clone(&network); - let my_id = node_id; - let anchor = anchor_nid; - tokio::spawn(async move { - for referral in referrals { - if referral.node_id == my_id { - continue; - } - if let Some(addr_str) = referral.addresses.first() { - let connect_str = format!( - "{}@{}", - hex::encode(referral.node_id), - addr_str, - ); - if let Ok((rid, raddr)) = crate::parse_connect_string(&connect_str) { - let connect_fut = async { - match net.connect_to_peer(rid, raddr).await { - Ok(()) => { info!(peer = hex::encode(rid), "Connected to referred peer"); Ok(()) }, - Err(_) => { - match net.connect_via_introduction(rid, anchor).await { - Ok(()) => { info!(peer = hex::encode(rid), "Connected via hole punch"); Ok(()) }, - Err(e) => Err(e), - } - } - } - }; - match tokio::time::timeout(std::time::Duration::from_secs(15), connect_fut).await { - Ok(Ok(())) => {}, - Ok(Err(e)) => debug!(error = %e, peer = hex::encode(rid), "Referral connect failed"), - Err(_) => debug!(peer = hex::encode(rid), "Referral connect timed out"), - } - } - } - } - net.notify_growth().await; - }); - } - Ok(Ok(_)) => debug!("No referrals from anchor"), - Ok(Err(e)) => debug!(error = %e, "Referral request failed"), - Err(_) => debug!("Referral request timed out"), + // Convection, ENTRY class — same single code path as + // bootstrap and recovery. + { + let net = Arc::clone(&network); + let my_id = node_id; + let anchor = anchor_nid; + tokio::spawn(async move { + let n = run_convection( + &net, + anchor, + &[], + crate::protocol::ConvectionClass::Entry, + my_id, + ).await; + info!(connected = n, "Startup: convection complete"); + net.notify_growth().await; + }); } } } @@ -893,11 +959,11 @@ impl Node { self.activity_log.lock().unwrap().recent(limit) } - /// Get timer state: (last_rebalance_ms, last_anchor_register_ms). + /// Get timer state: (last_rebalance_ms, last_convection_ms). pub fn timer_state(&self) -> (u64, u64) { ( self.last_rebalance_ms.load(AtomicOrdering::Relaxed), - self.last_anchor_register_ms.load(AtomicOrdering::Relaxed), + self.last_convection_ms.load(AtomicOrdering::Relaxed), ) } @@ -1299,19 +1365,13 @@ impl Node { /// Prefers social peers, then wide. async fn current_recent_peers(&self) -> Vec { let conns = self.network.connection_info().await; - let mut social: Vec = Vec::new(); - let mut wide: Vec = Vec::new(); - for (nid, kind, _) in conns { - if nid == self.node_id { - continue; - } - match kind { - PeerSlotKind::Local => social.push(nid), - PeerSlotKind::Wide => wide.push(nid), - } - } - let mut result = social; - result.extend(wide); + // v0.8: one mesh pool. Temp referral slots are excluded — a peer we + // hold only provisionally should not be advertised as our neighborhood. + let mut result: Vec = conns + .into_iter() + .filter(|(nid, slot, _)| *nid != self.node_id && slot.is_mesh()) + .map(|(nid, _, _)| nid) + .collect(); result.truncate(10); result } @@ -4189,7 +4249,7 @@ impl Node { let storage = self.storage.get().await; let _ = storage.update_follow_last_sync(&peer_id, 0); } - let stats = self.network.conn_handle().pull_from_peer(&peer_id).await?; + let stats = self.network.conn_handle().content_sync_from_peer(&peer_id).await?; // Also fetch engagement data (reactions, comments) for posts we hold let engagement = self.network.conn_handle().fetch_engagement_from_peer(&peer_id).await.unwrap_or(0); info!( @@ -4209,7 +4269,7 @@ impl Node { pub async fn sync_with_addr(&self, addr: iroh::EndpointAddr) -> anyhow::Result<()> { let peer_id = *addr.id.as_bytes(); self.network.connect_to_peer(peer_id, addr).await?; - let stats = self.network.conn_handle().pull_from_peer(&peer_id).await?; + let stats = self.network.conn_handle().content_sync_from_peer(&peer_id).await?; info!( peer = hex::encode(peer_id), posts = stats.posts_received, @@ -4220,7 +4280,7 @@ impl Node { /// Pull from all connected peers pub async fn sync_all(&self) -> anyhow::Result<()> { - let stats = self.network.pull_from_all().await?; + let stats = self.network.content_sync_all().await?; info!( "Pull complete: {} posts from {} peers", stats.posts_received, stats.peers_pulled @@ -4253,8 +4313,13 @@ impl Node { self.bootstrap_anchors.lock().await.clone() } - /// Get connection info for display: (node_id, slot_kind, connected_at) - pub async fn list_connections(&self) -> Vec<(NodeId, PeerSlotKind, u64)> { + /// This device's slot/depth budget. + pub fn device_profile(&self) -> DeviceProfile { + self.profile + } + + /// Get connection info for display: (node_id, slot, connected_at) + pub async fn list_connections(&self) -> Vec<(NodeId, MeshSlot, u64)> { self.network.connection_info().await } @@ -4273,53 +4338,73 @@ impl Node { tokio::spawn(async move { network.run_accept_loop().await }) } - /// Start pull cycle: Protocol v4 tiered pull — 60s ticks, full pull on first tick, - /// then only pull for stale authors (last_sync_ms > 4 hours old). - pub fn start_pull_cycle(self: &Arc) -> tokio::task::JoinHandle<()> { + /// Start the sync cycle — two cadences on one 60s tick. + /// + /// (1) THE PULL, i.e. the uniques-index exchange (0x40/0x41). design.html + /// §sync: a pull is not a post transfer, it is "if you want these IDs, + /// talk to me and I'll help you find them". Runs on the slow tick + /// because the pools are large and mostly static; the push-side + /// announce (0x01) already covers fast changes. + /// + /// (2) TRANSITIONAL content sync (0x46/0x47) for stale authors. Folds into + /// the update-cadence scheduler + CDN replication in Iteration D. + /// Until then it is the only carrier for non-public visibilities. + pub fn start_sync_cycle(self: &Arc) -> tokio::task::JoinHandle<()> { let node = Arc::clone(self); tokio::spawn(async move { let mut interval = tokio::time::interval(std::time::Duration::from_secs(60)); - let mut is_first_tick = true; + let mut tick: u64 = 0; loop { interval.tick().await; + tick += 1; - if is_first_tick { - // Full pull on startup - let _ = node.network.pull_from_all().await; - is_first_tick = false; - // Prefetch after initial sync + if tick == 1 { + // Startup: full content sync + engagement fetch, then + // prefetch blobs for what arrived. + let _ = node.network.content_sync_all().await; let peers = node.network.conn_handle().connected_peers().await; for peer_id in peers { node.prefetch_blobs_from_peer(&peer_id).await; } + let n = node.network.uniques_pull_all().await; + tracing::debug!(peers = n, "Startup uniques-index exchange"); continue; } - // Tiered: only pull for stale authors (4-hour default) + // (1) Uniques-index exchange every 5 minutes. + if tick % 5 == 0 { + let n = node.network.uniques_pull_all().await; + if n > 0 { + tracing::debug!(peers = n, "Uniques-index exchange"); + } + } + + // (2) Tiered content sync: only when some author is stale. let stale_authors = { let storage = node.storage.get().await; storage.get_stale_follows(4 * 3600 * 1000).unwrap_or_default() }; - if stale_authors.is_empty() { continue; // Most ticks skip — no stale authors } - // Find a connected peer and pull + // Every connected peer, not just `peers.first()`: one arbitrary + // peer is very unlikely to hold a given stale author's posts, + // so the tiered tick silently did almost nothing. let peers = node.network.conn_handle().connected_peers().await; - if let Some(peer_id) = peers.first() { - match node.network.conn_handle().pull_from_peer(peer_id).await { - Ok(stats) => { - if stats.posts_received > 0 { - tracing::debug!( - posts = stats.posts_received, - "Tiered pull complete" - ); - node.prefetch_blobs_from_peer(peer_id).await; - } + for peer_id in &peers { + match node.network.conn_handle().content_sync_from_peer(peer_id).await { + Ok(stats) if stats.posts_received > 0 => { + tracing::debug!( + peer = hex::encode(peer_id), + posts = stats.posts_received, + "Tiered content sync complete" + ); + node.prefetch_blobs_from_peer(peer_id).await; } - Err(e) => tracing::debug!(error = %e, "Tiered pull failed"), + Ok(_) => {} + Err(e) => tracing::debug!(error = %e, "Tiered content sync failed"), } } } @@ -4351,7 +4436,7 @@ impl Node { } } } else { - match network.broadcast_diff().await { + match network.broadcast_uniques().await { Ok(count) => { if count > 0 { tracing::debug!(count, "Broadcast routing diff"); @@ -4401,8 +4486,12 @@ impl Node { }) } - /// Start recovery loop: triggered when the mesh empties completely. - /// Immediately reconnects to anchors and requests referrals. + /// Start recovery loop: triggered when the mesh drops below 2 peers. + /// + /// Recovery is deliberately NOT stochastic (round-4 ruling). A node with + /// fewer than 2 mesh peers cannot function, so it always acts immediately; + /// only *growth* rolls dice. Anchors are gathered pool-first, then the + /// bootstrap cache. pub fn start_recovery_loop(&self) -> tokio::task::JoinHandle<()> { let network = Arc::clone(&self.network); let storage = Arc::clone(&self.storage); @@ -4416,86 +4505,31 @@ impl Node { network.set_recovery_tx(tx).await; while rx.recv().await.is_some() { tracing::info!("Recovery triggered: reconnecting to anchors"); - log_evt(ActivityLevel::Warn, ActivityCategory::Recovery, "Recovery triggered: mesh empty".into(), None); + log_evt(ActivityLevel::Warn, ActivityCategory::Recovery, "Recovery triggered: mesh below 2".into(), None); // Debounce: wait briefly for more disconnects to settle tokio::time::sleep(std::time::Duration::from_secs(2)).await; // Drain any queued signals while rx.try_recv().is_ok() {} - // Gather anchors: known_anchors table, then anchor peers fallback - let anchors: Vec<(crate::types::NodeId, Vec)> = { - let s = storage.get().await; - let known = s.list_known_anchors().unwrap_or_default(); - if !known.is_empty() { - known - } else { - s.list_anchor_peers().unwrap_or_default() - .into_iter() - .map(|r| (r.node_id, r.addresses)) - .collect() - } - }; - + let anchors = gather_anchor_candidates(&storage, &network, node_id, 8).await; + let mut connected = 0usize; for (anchor_nid, anchor_addrs) in &anchors { - if *anchor_nid == node_id { continue; } - // Connect to anchor (mesh or session fallback) - if !network.is_peer_connected_or_session(anchor_nid).await { - let endpoint_id = match iroh::EndpointId::from_bytes(anchor_nid) { - Ok(eid) => eid, - Err(_) => continue, - }; - let mut addr = iroh::EndpointAddr::from(endpoint_id); - for sa in anchor_addrs { - addr = addr.with_ip_addr(*sa); - } - match network.connect_to_anchor(*anchor_nid, addr).await { - Ok(()) => { - log_evt(ActivityLevel::Info, ActivityCategory::Recovery, "Connected to anchor".into(), Some(*anchor_nid)); - } - Err(e) => { - tracing::debug!(error = %e, "Recovery: anchor connect failed"); - log_evt(ActivityLevel::Warn, ActivityCategory::Recovery, format!("Anchor connect failed: {}", e), Some(*anchor_nid)); - continue; - } - } - } - // Register with anchor - let _ = network.send_anchor_register(anchor_nid).await; - // Request referrals - match network.request_anchor_referrals(anchor_nid).await { - Ok(referrals) => { - for referral in referrals { - if referral.node_id == node_id { continue; } - if let Some(addr_str) = referral.addresses.first() { - let connect_str = format!( - "{}@{}", hex::encode(referral.node_id), addr_str, - ); - if let Ok((rid, raddr)) = crate::parse_connect_string(&connect_str) { - match network.connect_to_peer(rid, raddr).await { - Ok(()) => { - tracing::info!(peer = hex::encode(rid), "Recovery: connected to referred peer"); - log_evt(ActivityLevel::Info, ActivityCategory::Recovery, "Connected to referred peer".into(), Some(rid)); - } - Err(_) => { - match network.connect_via_introduction(rid, *anchor_nid).await { - Ok(()) => { - tracing::info!(peer = hex::encode(rid), "Recovery: connected via hole punch"); - log_evt(ActivityLevel::Info, ActivityCategory::Recovery, "Connected via hole punch".into(), Some(rid)); - } - Err(e) => { - tracing::debug!(error = %e, peer = hex::encode(rid), "Recovery: hole punch failed"); - log_evt(ActivityLevel::Warn, ActivityCategory::Recovery, format!("Hole punch failed: {}", e), Some(rid)); - } - } - } - } - } - } - } - } - Err(e) => tracing::debug!(error = %e, "Recovery: referral request failed"), + // ENTRY class: always served, by definition of the ruling. + // There is no separate registration — the request enrols us. + connected += run_convection( + &network, + *anchor_nid, + anchor_addrs, + crate::protocol::ConvectionClass::Entry, + node_id, + ).await; + if network.conn_handle().mesh_count().await >= 2 { + break; } } + if connected > 0 { + log_evt(ActivityLevel::Info, ActivityCategory::Recovery, format!("Convection produced {} connections", connected), None); + } let conn_count = network.connection_count().await; tracing::info!(connections = conn_count, "Recovery complete"); log_evt(ActivityLevel::Info, ActivityCategory::Recovery, format!("Recovery complete, {} connections", conn_count), None); @@ -4503,6 +4537,117 @@ impl Node { }) } + /// Run one convection exchange against a specific anchor, on demand. + /// + /// Diagnostics + integration testing: the automatic paths pick the anchor + /// themselves (recovery pool-first, the stochastic arm at random), which is + /// correct but untestable. Returns `(peers_connected, refused, elapsed_ms)` + /// — the elapsed time is the point of the cheap-refusal contract. + pub async fn convection_request(&self, anchor: NodeId) -> anyhow::Result<(usize, bool, u128)> { + let started = std::time::Instant::now(); + let mesh = self.network.conn_handle().mesh_count().await; + let class = crate::protocol::ConvectionClass::for_mesh_count(mesh); + let addrs: Vec = { + let s = self.storage.get().await; + s.get_peer_record(&anchor).ok().flatten().map(|r| r.addresses).unwrap_or_default() + }; + if !self.network.is_peer_connected_or_session(&anchor).await { + let eid = iroh::EndpointId::from_bytes(&anchor)?; + let mut ea = iroh::EndpointAddr::from(eid); + for sa in &addrs { + ea = ea.with_ip_addr(*sa); + } + self.network.connect_to_anchor(anchor, ea).await?; + } + let response = self.network.request_convection(&anchor, class).await?; + let refused = response.refused; + let connected = if refused { + 0 + } else { + self.network.act_on_convection(&anchor, &response).await + }; + Ok((connected, refused, started.elapsed().as_millis())) + } + + /// Run the uniques-index exchange (the v0.8 "pull") against every mesh peer. + pub async fn uniques_pull(&self) -> usize { + self.network.uniques_pull_all().await + } + + /// Start the convection loop: the "ask a random known anchor" arm of the + /// per-disconnect stochastic action (round-4/5). + /// + /// The dice are rolled inside `disconnect_peer` under the conn_mgr lock + /// (pure state read + RNG); this loop is where the resulting network I/O + /// happens, so nothing blocks a teardown. It also carries the anchor + /// self-verification probe, which lost its home when the register cycle + /// retired. + pub fn start_convection_loop(&self) -> tokio::task::JoinHandle<()> { + let network = Arc::clone(&self.network); + let storage = Arc::clone(&self.storage); + let node_id = self.node_id; + let alog = Arc::clone(&self.activity_log); + let timer = Arc::clone(&self.last_convection_ms); + let (tx, mut rx) = tokio::sync::mpsc::channel::<()>(1); + tokio::spawn(async move { + let log_evt = |level: ActivityLevel, cat: ActivityCategory, msg: String, peer: Option| { + if let Ok(mut log) = alog.try_lock() { log.log(level, cat, msg, peer); } + }; + network.conn_handle().set_convection_tx(tx).await; + // Slow maintenance tick: the anchor self-verification probe used to + // hang off the register cycle. + let mut probe_tick = tokio::time::interval(std::time::Duration::from_secs(600)); + probe_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + tokio::select! { + signal = rx.recv() => { + if signal.is_none() { break; } + // Coalesce a burst of disconnects into one action. + while rx.try_recv().is_ok() {} + + let mesh = network.conn_handle().mesh_count().await; + let class = crate::protocol::ConvectionClass::for_mesh_count(mesh); + let mut anchors = gather_anchor_candidates(&storage, &network, node_id, 12).await; + if anchors.is_empty() { + // No anchor to ask — fall through to the mesh arm + // rather than doing nothing. + network.notify_growth().await; + continue; + } + // "a RANDOM known anchor" — not the best-ranked one. + // Ranking anchors was a scarcity artifact; spreading + // load is what keeps convection windows fresh. + use rand::seq::SliceRandom; + anchors.shuffle(&mut rand::rng()); + let (anchor_nid, anchor_addrs) = anchors.remove(0); + timer.store( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64, + AtomicOrdering::Relaxed, + ); + let n = run_convection(&network, anchor_nid, &anchor_addrs, class, node_id).await; + if n > 0 { + log_evt(ActivityLevel::Info, ActivityCategory::Anchor, format!("Convection: {} new peers", n), Some(anchor_nid)); + } else { + // Nothing came back — let the mesh arm try. + network.notify_growth().await; + } + } + _ = probe_tick.tick() => { + if network.conn_handle().probe_due().await { + log_evt(ActivityLevel::Info, ActivityCategory::Anchor, "Initiating anchor self-verification probe".into(), None); + if let Err(e) = network.conn_handle().initiate_anchor_probe().await { + tracing::debug!(error = %e, "Anchor probe error"); + } + } + } + } + } + }) + } + /// Start social checkin cycle: every interval_secs, refresh stale social routes. /// Uses ephemeral connections if not persistently connected. pub fn start_social_checkin_cycle(&self, interval_secs: u64) -> tokio::task::JoinHandle<()> { @@ -4548,165 +4693,6 @@ impl Node { }) } - /// Register with all connected anchor peers. Returns count registered. - pub async fn register_with_anchors(&self) -> usize { - let conns = self.network.connection_info().await; - let mut count = 0; - for (nid, _, _) in &conns { - if self.network.is_anchor_peer(nid).await { - match self.network.send_anchor_register(nid).await { - Ok(()) => { - count += 1; - info!(anchor = hex::encode(nid), "Registered with anchor"); - } - Err(e) => debug!(error = %e, anchor = hex::encode(nid), "Anchor register failed"), - } - } - } - count - } - - /// Start anchor register cycle: periodically re-register with anchors and request referrals - /// when connection count is low. - pub fn start_anchor_register_cycle(&self, interval_secs: u64) -> tokio::task::JoinHandle<()> { - let network = Arc::clone(&self.network); - let storage = Arc::clone(&self.storage); - let node_id = self.node_id; - let alog = Arc::clone(&self.activity_log); - let timer = Arc::clone(&self.last_anchor_register_ms); - tokio::spawn(async move { - let log_evt = |level: ActivityLevel, cat: ActivityCategory, msg: String, peer: Option| { - if let Ok(mut log) = alog.try_lock() { log.log(level, cat, msg, peer); } - }; - let mut interval = - tokio::time::interval(std::time::Duration::from_secs(interval_secs)); - loop { - interval.tick().await; - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64; - timer.store(now, AtomicOrdering::Relaxed); - - // Re-register with connected anchors (mesh + session) - let conns = network.connection_info().await; - let session_peers = network.session_peer_ids().await; - let mut registered_anchors = std::collections::HashSet::new(); - // Mesh-connected anchors - for (nid, _, _) in &conns { - if network.is_anchor_peer(nid).await { - match network.send_anchor_register(nid).await { - Ok(()) => { - log_evt(ActivityLevel::Info, ActivityCategory::Anchor, "Re-registered with anchor".into(), Some(*nid)); - registered_anchors.insert(*nid); - } - Err(e) => { - tracing::debug!(error = %e, "Anchor re-register failed"); - log_evt(ActivityLevel::Warn, ActivityCategory::Anchor, format!("Re-register failed: {}", e), Some(*nid)); - } - } - } - } - // Session-connected anchors (e.g. anchor with full mesh) - for nid in &session_peers { - if registered_anchors.contains(nid) { continue; } - if network.is_anchor_peer(nid).await { - match network.send_anchor_register(nid).await { - Ok(()) => { - log_evt(ActivityLevel::Info, ActivityCategory::Anchor, "Re-registered with anchor (session)".into(), Some(*nid)); - } - Err(e) => { - tracing::debug!(error = %e, "Anchor session re-register failed"); - } - } - } - } - - // If few connections, try requesting referrals from known anchors - let conn_count = network.connection_count().await; - if conn_count < 10 { - log_evt(ActivityLevel::Info, ActivityCategory::Anchor, format!("Low connections ({}), requesting referrals", conn_count), None); - let known = { - let s = storage.get().await; - s.list_known_anchors().unwrap_or_default() - }; - for (anchor_nid, anchor_addrs) in known { - if anchor_nid == node_id { - continue; - } - // Connect if not already connected (mesh or session) - if !network.is_peer_connected_or_session(&anchor_nid).await { - let endpoint_id = match iroh::EndpointId::from_bytes(&anchor_nid) { - Ok(eid) => eid, - Err(_) => continue, - }; - let mut addr = iroh::EndpointAddr::from(endpoint_id); - for sa in &anchor_addrs { - addr = addr.with_ip_addr(*sa); - } - if let Err(e) = network.connect_to_anchor(anchor_nid, addr).await { - tracing::debug!(error = %e, "Anchor cycle: connect failed"); - continue; - } - } - match network.request_anchor_referrals(&anchor_nid).await { - Ok(referrals) => { - for referral in referrals { - if referral.node_id == node_id { - continue; - } - if let Some(addr_str) = referral.addresses.first() { - let connect_str = format!( - "{}@{}", - hex::encode(referral.node_id), - addr_str, - ); - if let Ok((rid, raddr)) = crate::parse_connect_string(&connect_str) { - match network.connect_to_peer(rid, raddr).await { - Ok(()) => { - tracing::info!(peer = hex::encode(rid), "Anchor cycle: connected to referred peer"); - log_evt(ActivityLevel::Info, ActivityCategory::Anchor, "Connected to referred peer".into(), Some(rid)); - } - Err(_) => { - match network.connect_via_introduction(rid, anchor_nid).await { - Ok(()) => { - tracing::info!(peer = hex::encode(rid), "Anchor cycle: connected via hole punch"); - log_evt(ActivityLevel::Info, ActivityCategory::Anchor, "Connected via hole punch".into(), Some(rid)); - } - Err(e) => { - tracing::debug!(error = %e, peer = hex::encode(rid), "Anchor cycle: hole punch failed"); - log_evt(ActivityLevel::Warn, ActivityCategory::Anchor, format!("Hole punch failed: {}", e), Some(rid)); - } - } - } - } - } - } - } - } - Err(e) => tracing::debug!(error = %e, "Anchor cycle: referral request failed"), - } - } - } - - // Anchor self-verification probe - { - let probe_due = network.conn_handle().probe_due().await; - if probe_due { - log_evt(ActivityLevel::Info, ActivityCategory::Anchor, "Initiating anchor self-verification probe".into(), None); - match network.conn_handle().initiate_anchor_probe().await { - Ok(true) => {}, // success already logged inside - Ok(false) => {}, // failure already logged inside - Err(e) => { - tracing::debug!(error = %e, "Anchor probe error"); - } - } - } - } - } - }) - } - /// Start bootstrap connectivity check: 24 hours after startup, verify the bootstrap /// anchor is within our network knowledge (N1/N2/N3). If not, we may be in an isolated /// segment — reconnect to bootstrap and request referrals to bridge back. @@ -4731,20 +4717,18 @@ impl Node { continue; } - // Check if bootstrap is in N1 (mesh), N2, or N3 + // Is the bootstrap anywhere in our N1-N4 horizon? N4 counts: + // it is used for search and resolution, it is only never + // re-announced. let is_reachable = { let connected = node.network.is_connected(&bootstrap_nid).await; if connected { true } else { let storage = node.storage.get().await; - let in_n2 = storage.find_in_n2(&bootstrap_nid).unwrap_or_default(); - if !in_n2.is_empty() { - true - } else { - let in_n3 = storage.find_in_n3(&bootstrap_nid).unwrap_or_default(); - !in_n3.is_empty() - } + storage.find_any_reachable(std::slice::from_ref(&bootstrap_nid)) + .map(|r| !r.is_empty()) + .unwrap_or(false) } }; @@ -4762,26 +4746,16 @@ impl Node { continue; } - // Report bootstrap in our N1 for 24 hours so peers learn about it - node.network.conn_handle().add_sticky_n1(&bootstrap_nid, 24 * 60 * 60 * 1000); - - match node.network.request_anchor_referrals(&bootstrap_nid).await { - Ok(referrals) => { - tracing::info!(count = referrals.len(), "Bootstrap connectivity: got referrals"); - for referral in referrals { - if referral.node_id == node.node_id { continue; } - if let Some(addr_str) = referral.addresses.first() { - let connect_str = format!("{}@{}", hex::encode(referral.node_id), addr_str); - if let Ok((rid, raddr)) = crate::parse_connect_string(&connect_str) { - let _ = node.network.connect_to_peer(rid, raddr).await; - } - } - } - } - Err(e) => { - tracing::warn!(error = %e, "Bootstrap connectivity: referral request failed"); - } - } + // ENTRY class: an isolated segment is exactly the case the + // always-served class exists for. + let n = run_convection( + &node.network, + bootstrap_nid, + &[], + crate::protocol::ConvectionClass::Entry, + node.node_id, + ).await; + tracing::info!(connected = n, "Bootstrap connectivity: convection complete"); } }) } @@ -4867,8 +4841,9 @@ impl Node { if nid == self.node_id { continue; } - // Prefer social peers - if kind != PeerSlotKind::Local && result.len() >= 10 { + // Temp referral slots are never advertised as part of our + // neighborhood. + if !kind.is_mesh() { continue; } let addrs: Vec = storage.get_peer_record(&nid) @@ -5327,7 +5302,9 @@ impl Node { }; let storage = self.storage.get().await; - storage.store_comment(&comment)?; + // `store_own_comment`, not `store_comment`: WE authored this, so its + // author (our persona) must stay out of our own uniques announce. + storage.store_own_comment(&comment)?; // v0.8 (A3): refresh the aggregated header so pulls serve this // comment without waiting for a diff roundtrip. let _ = storage.rebuild_blob_header_from_db(&post_id, &our_node_id, now); @@ -5595,7 +5572,7 @@ impl Node { // Store locally. { let storage = self.storage.get().await; - storage.store_comment(&comment)?; + storage.store_own_comment(&comment)?; let _ = storage.rebuild_blob_header_from_db(&post_id, &post_author, now); } @@ -6464,7 +6441,11 @@ impl Node { { let s = self.storage.get().await; - s.store_comment(&comment)?; + // Greeting: the author is a per-greeting THROWAWAY id. We are the + // first node in the network that could announce it, at bounce 1, + // the moment it is created — first-announcer identifies the + // greeter. `store_own_comment` keeps it out of our announce. + s.store_own_comment(&comment)?; let _ = s.rebuild_blob_header_from_db(&target_post_id, &target_post.author, now); } @@ -6704,7 +6685,7 @@ impl Node { let _ = s.upsert_registry_entry_newest_wins( &crate::registry::REGISTRY_POST_ID, posting_id, now, ); - s.store_comment(&comment)?; + s.store_own_comment(&comment)?; let _ = s.rebuild_blob_header_from_db( &crate::registry::REGISTRY_POST_ID, ®istry_post.author, now, ); diff --git a/crates/core/src/protocol.rs b/crates/core/src/protocol.rs index ea48075..1e354b8 100644 --- a/crates/core/src/protocol.rs +++ b/crates/core/src/protocol.rs @@ -27,13 +27,25 @@ pub struct SyncPost { #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[repr(u8)] pub enum MessageType { - NodeListUpdate = 0x01, + UniquesAnnounce = 0x01, InitialExchange = 0x02, AddressRequest = 0x03, AddressResponse = 0x04, RefuseRedirect = 0x05, - PullSyncRequest = 0x40, - PullSyncResponse = 0x41, + /// The uniques-index exchange (design.html §sync). A pull is no longer a + /// post transfer: it is "if you want these IDs, talk to me and I'll help + /// you find them". Symmetric — request carries our pools, response carries + /// theirs, one round trip updates both sides. + UniquesPullRequest = 0x40, + UniquesPullResponse = 0x41, + /// TRANSITIONAL (Iteration D folds this into the update-cadence scheduler + + /// CDN replication). Content still has to move somewhere while + /// PostFetchRequest (0xD4) serves Public posts only: DMs, group posts, + /// Friends and FoFClosed posts have no other carrier. Splitting it off + /// 0x40/0x41 keeps §sync honest — a *pull* is not a post transfer, because + /// post transfer is not a pull. + ContentSyncRequest = 0x46, + ContentSyncResponse = 0x47, ProfileUpdate = 0x50, WormQuery = 0x60, WormResponse = 0x61, @@ -49,9 +61,11 @@ pub enum MessageType { RelayIntroduceResult = 0xB1, SessionRelay = 0xB2, CircleProfileUpdate = 0xB4, - AnchorRegister = 0xC0, - AnchorReferralRequest = 0xC1, - AnchorReferralResponse = 0xC2, + // 0xC0 retired in v0.8 — anchors keep a rolling window of recent callers + // fed by the request itself; there is no registration message and no + // registration database. + ConvectionRequest = 0xC1, + ConvectionResponse = 0xC2, AnchorProbeRequest = 0xC3, AnchorProbeResult = 0xC4, PortScanHeartbeat = 0xC5, @@ -73,13 +87,15 @@ pub enum MessageType { impl MessageType { pub fn from_byte(b: u8) -> Option { match b { - 0x01 => Some(Self::NodeListUpdate), + 0x01 => Some(Self::UniquesAnnounce), 0x02 => Some(Self::InitialExchange), 0x03 => Some(Self::AddressRequest), 0x04 => Some(Self::AddressResponse), 0x05 => Some(Self::RefuseRedirect), - 0x40 => Some(Self::PullSyncRequest), - 0x41 => Some(Self::PullSyncResponse), + 0x40 => Some(Self::UniquesPullRequest), + 0x41 => Some(Self::UniquesPullResponse), + 0x46 => Some(Self::ContentSyncRequest), + 0x47 => Some(Self::ContentSyncResponse), 0x50 => Some(Self::ProfileUpdate), 0x60 => Some(Self::WormQuery), 0x61 => Some(Self::WormResponse), @@ -94,9 +110,8 @@ impl MessageType { 0xB1 => Some(Self::RelayIntroduceResult), 0xB2 => Some(Self::SessionRelay), 0xB4 => Some(Self::CircleProfileUpdate), - 0xC0 => Some(Self::AnchorRegister), - 0xC1 => Some(Self::AnchorReferralRequest), - 0xC2 => Some(Self::AnchorReferralResponse), + 0xC1 => Some(Self::ConvectionRequest), + 0xC2 => Some(Self::ConvectionResponse), 0xC3 => Some(Self::AnchorProbeRequest), 0xC4 => Some(Self::AnchorProbeResult), 0xC5 => Some(Self::PortScanHeartbeat), @@ -124,13 +139,152 @@ impl MessageType { // --- Payload structs --- -/// Initial exchange: N1/N2 node lists + profile + deletes + post_ids + peer addresses +/// A packed set of 32-byte IDs: base64 of the raw concatenated bytes. +/// +/// `NodeId = [u8; 32]` serialised through serde_json becomes a JSON array of 32 +/// decimal numbers (~100-130 bytes/ID — a 3-4x blowup). The uniques pools can +/// carry thousands of IDs, so they ride packed instead. +pub type PackedIds = String; + +/// Encode a set of IDs into a packed base64 blob. +pub fn pack_ids(ids: &[NodeId]) -> PackedIds { + use base64::Engine; + let mut raw = Vec::with_capacity(ids.len() * 32); + for id in ids { + raw.extend_from_slice(id); + } + base64::engine::general_purpose::STANDARD.encode(raw) +} + +/// Decode a packed base64 blob back into IDs. Trailing partial IDs are dropped. +pub fn unpack_ids(packed: &str) -> Vec { + use base64::Engine; + let raw = match base64::engine::general_purpose::STANDARD.decode(packed) { + Ok(r) => r, + Err(_) => return Vec::new(), + }; + raw.chunks_exact(32) + .map(|c| { + let mut id = [0u8; 32]; + id.copy_from_slice(c); + id + }) + .collect() +} + +/// How many 32-byte IDs a packed base64 blob holds, WITHOUT decoding it. +/// +/// Standard base64 emits 4 chars per 3 input bytes (padded), so decoded length +/// is `chars / 4 * 3` minus padding; integer division by 32 absorbs the padding +/// slack for any non-empty blob. Exact for every blob `pack_ids` produces. +pub fn packed_id_count(packed: &str) -> usize { + (packed.len() / 4) * 3 / 32 +} + +/// The ONLY entry type in a uniques pool that carries an address. +/// +/// design.html §layers: anchor entries carry addresses so the pools double as +/// the anchor directory; every other entry is a bare ID (there is no point +/// sharing an address that needs an introduction anyway). Addresses MUST be +/// filtered through `is_publicly_routable()` before they go on the wire. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AnchorEntry { + /// hex node id (anchors are always node-class) + pub i: String, + /// e.g. ["1.2.3.4:4433"] + pub a: Vec, +} + +/// One slice of a uniques pool. Node-class and author-class IDs live in +/// separate arrays so a receiver structurally cannot mistake a persona ID for +/// a connectable node ID. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct UniquesSlice { + /// Connectable network IDs, bare. + #[serde(default)] + pub nodes: PackedIds, + /// Posting/persona/throwaway IDs, bare. CDN-resolvable only — never a + /// connect target. + #[serde(default)] + pub authors: PackedIds, + /// Address-bearing anchor entries (node-class by definition). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub anchors: Vec, +} + +impl UniquesSlice { + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() && self.authors.is_empty() && self.anchors.is_empty() + } + /// Number of IDs carried (for logging / budget checks). + /// + /// `pack_ids` concatenates ALL IDs and base64-encodes the blob ONCE, so the + /// encoded length is `4 * ceil(32n/3)` characters — about 42.67 per ID, not + /// 44 (44 is the encoding of exactly ONE 32-byte ID). Counting `len()/44` + /// undercounts by ~3% at scale and by 33% for a 3-ID blob. Since this backs + /// budget checks, it is computed from the DECODED byte length instead. + pub fn len(&self) -> usize { + packed_id_count(&self.nodes) + packed_id_count(&self.authors) + self.anchors.len() + } +} + +/// 0x01 — the two-pool uniques announce (design.html §layers). +/// +/// POOL 1 (`fwd`) is FORWARDABLE and indexed by OUR bounce: +/// fwd[0] = N0 — ourselves (and our own anchor address, if we are an anchor) +/// fwd[1] = N1 — our own uniques: mesh peers, social directs, CDN file +/// authors, CDN file-holder peers, merged so the receiver +/// cannot tell which source an entry came from +/// fwd[2] = N2 — what our N1 reported to us +/// The receiver stores `fwd[i]` at bounce `i + 1` (their N1..N3), tagged to us +/// as reporter, and MAY re-announce it shifted one deeper. +/// +/// POOL 2 (`term`) is TERMINAL: our N3, deduplicated against every fwd slice. +/// The receiver stores it as their N4 and USES it for search / address +/// resolution, but NEVER re-announces it. The structural separation — a +/// distinct field, not a flag in a merged list — is what makes "never +/// re-announce" un-forgettable at the announce builder: the builder simply +/// never reads bounce-4 rows. There is no N5 anywhere. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UniquesAnnouncePayload { + pub seq: u64, + /// Always true in v0.8 — the announce is a full snapshot that replaces this + /// reporter's rows. (Incremental diffing over four layers with anchor + /// metadata was not worth the divergence risk; the sender skips the send + /// entirely when nothing changed, which recovers most of the saving.) + pub full: bool, + /// Pool 1, forwardable. Always length 3 (bounces 0, 1, 2). + pub fwd: Vec, + /// Pool 2, terminal. Empty when the recipient advertised `depth < 4`. + #[serde(default)] + pub term: UniquesSlice, + /// Deepest bounce the SENDER retains (4 desktop, 3 mobile). Lets the peer + /// skip building/sending the terminal pool for us — the single biggest + /// bandwidth saving on a cellular link. + pub depth: u8, +} + +impl UniquesAnnouncePayload { + pub fn empty(seq: u64, depth: u8) -> Self { + Self { + seq, + full: true, + fwd: vec![UniquesSlice::default(), UniquesSlice::default(), UniquesSlice::default()], + term: UniquesSlice::default(), + depth, + } + } +} + +/// Initial exchange: uniques announce + profile + deletes + post_ids + peer addresses #[derive(Debug, Serialize, Deserialize)] pub struct InitialExchangePayload { - /// Our connections + social contacts NodeIds (no addresses) - pub n1_node_ids: Vec, - /// Our deduplicated N2 NodeIds (no addresses) - pub n2_node_ids: Vec, + /// Our uniques pools (full snapshot). `None` means "this connection carries + /// no knowledge" — i.e. the sender placed us in a temporary referral slot. + /// Explicit on the wire rather than a pair of empty vectors so the receiver + /// can tell "nothing to share" from "not sharing". + #[serde(default)] + pub uniques: Option, /// Our profile pub profile: Option, /// Our delete records @@ -162,28 +316,32 @@ pub struct InitialExchangePayload { pub duplicate_active: Option, } -/// Incremental N1/N2 changes +/// 0x40/0x41 — the uniques-index exchange, both directions. +/// +/// The requester sends its pools; the responder answers with its own. One +/// round trip refreshes both indexes, which is what makes an index *exchange* +/// rather than a fetch. Content does not ride this path. #[derive(Debug, Serialize, Deserialize)] -pub struct NodeListUpdatePayload { - pub seq: u64, - pub n1_added: Vec, - pub n1_removed: Vec, - pub n2_added: Vec, - pub n2_removed: Vec, +pub struct UniquesPullPayload { + /// The sender's two pools. `None` = "I carry no knowledge on this link" + /// (temp referral slot), mirroring `InitialExchangePayload.uniques`. + #[serde(default)] + pub uniques: Option, } -/// Pull-based post sync request +/// 0x46 — TRANSITIONAL content sync request. Identical in shape to the v0.7 +/// pull request; only the opcode moved. See `MessageType::ContentSyncRequest`. #[derive(Debug, Serialize, Deserialize)] -pub struct PullSyncRequestPayload { +pub struct ContentSyncRequestPayload { /// Our follows (for the responder to filter) pub follows: Vec, /// Per-author last-sync timestamps (Vec of tuples for serde compat) pub since_ms: Vec<(NodeId, u64)>, } -/// Pull-based post sync response +/// 0x47 — TRANSITIONAL content sync response. #[derive(Debug, Serialize, Deserialize)] -pub struct PullSyncResponsePayload { +pub struct ContentSyncResponsePayload { pub posts: Vec, } @@ -387,33 +545,63 @@ pub struct CircleProfileUpdatePayload { pub updated_at: u64, } -// --- Anchor referral payloads --- +// --- Anchor convection payloads (design.html §anchors) --- -/// Node registers its address with an anchor (uni-stream) -#[derive(Debug, Serialize, Deserialize)] -pub struct AnchorRegisterPayload { - pub node_id: NodeId, - pub addresses: Vec, +/// The one request-class bit (round-5 ruling). +/// +/// ENTRY = bootstrap or recovery, i.e. the caller has fewer than 2 mesh +/// connections. Always served — a node that cannot get in cannot come back. +/// TOP_UP = growth with a working mesh. Served capacity-permitting and refused +/// CHEAPLY (a single response message, no timeout burned). Top-ups are the +/// convection medium — they are what keeps referral windows fresh — so they +/// are not throttled aggressively; the refusal doubles as the adaptive load +/// signal the caller feeds back into its action weights. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum ConvectionClass { + Entry, + TopUp, } -/// Node requests peer referrals from an anchor (bi-stream request) +impl ConvectionClass { + /// Entry class is decided purely by "can this node function right now". + pub fn for_mesh_count(mesh_count: usize) -> Self { + if mesh_count < 2 { Self::Entry } else { Self::TopUp } + } + pub fn is_entry(self) -> bool { + matches!(self, Self::Entry) + } +} + +/// 0xC1 — "I'm joining / I need peers". The request itself is what enrols the +/// caller in the anchor's rolling window; there is no separate registration. #[derive(Debug, Serialize, Deserialize)] -pub struct AnchorReferralRequestPayload { +pub struct ConvectionRequestPayload { pub requester: NodeId, pub requester_addresses: Vec, + pub class: ConvectionClass, } -/// Anchor responds with peer referrals (bi-stream response) +/// 0xC2 — up to 2 recent prior callers, or a cheap refusal. #[derive(Debug, Serialize, Deserialize)] -pub struct AnchorReferralResponsePayload { - pub referrals: Vec, +pub struct ConvectionResponsePayload { + pub referrals: Vec, + /// Cheap refusal of a TOP_UP request: one message, immediately, so the + /// caller never burns a timeout. Never set for ENTRY-class requests. + #[serde(default)] + pub refused: bool, } -/// A single peer referral from an anchor +/// A single referral out of the anchor's rolling caller window. #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AnchorReferral { +pub struct ConvectionReferral { pub node_id: NodeId, pub addresses: Vec, + /// The anchor had BOTH ends live and has pushed a RelayIntroduce toward + /// this peer naming the caller as requester — so the peer is already + /// punching outward. Dial straight through instead of paying for a second + /// introduction round trip. + #[serde(default)] + pub introduced: bool, } // --- Anchor probe payloads --- @@ -634,13 +822,15 @@ mod tests { #[test] fn message_type_roundtrip() { let types = [ - MessageType::NodeListUpdate, + MessageType::UniquesAnnounce, MessageType::InitialExchange, MessageType::AddressRequest, MessageType::AddressResponse, MessageType::RefuseRedirect, - MessageType::PullSyncRequest, - MessageType::PullSyncResponse, + MessageType::UniquesPullRequest, + MessageType::UniquesPullResponse, + MessageType::ContentSyncRequest, + MessageType::ContentSyncResponse, MessageType::ProfileUpdate, MessageType::WormQuery, MessageType::WormResponse, @@ -655,9 +845,8 @@ mod tests { MessageType::RelayIntroduceResult, MessageType::SessionRelay, MessageType::CircleProfileUpdate, - MessageType::AnchorRegister, - MessageType::AnchorReferralRequest, - MessageType::AnchorReferralResponse, + MessageType::ConvectionRequest, + MessageType::ConvectionResponse, MessageType::AnchorProbeRequest, MessageType::AnchorProbeResult, MessageType::PortScanHeartbeat, @@ -779,56 +968,105 @@ mod tests { } #[test] - fn anchor_register_payload_roundtrip() { - let payload = AnchorRegisterPayload { - node_id: [1u8; 32], - addresses: vec!["192.168.1.5:4433".to_string(), "10.0.0.1:4433".to_string()], - }; - let json = serde_json::to_string(&payload).unwrap(); - let decoded: AnchorRegisterPayload = serde_json::from_str(&json).unwrap(); - assert_eq!(decoded.node_id, [1u8; 32]); - assert_eq!(decoded.addresses.len(), 2); - assert_eq!(decoded.addresses[0], "192.168.1.5:4433"); + fn convection_request_carries_the_class_bit() { + for class in [ConvectionClass::Entry, ConvectionClass::TopUp] { + let payload = ConvectionRequestPayload { + requester: [2u8; 32], + requester_addresses: vec!["10.0.0.2:4433".to_string()], + class, + }; + let json = serde_json::to_string(&payload).unwrap(); + let decoded: ConvectionRequestPayload = serde_json::from_str(&json).unwrap(); + assert_eq!(decoded.requester, [2u8; 32]); + assert_eq!(decoded.requester_addresses, vec!["10.0.0.2:4433"]); + assert_eq!(decoded.class, class); + } + // The class is derived purely from "can this node function right now". + assert!(ConvectionClass::for_mesh_count(0).is_entry()); + assert!(ConvectionClass::for_mesh_count(1).is_entry()); + assert!(!ConvectionClass::for_mesh_count(2).is_entry()); + assert!(!ConvectionClass::for_mesh_count(20).is_entry()); } #[test] - fn anchor_referral_request_payload_roundtrip() { - let payload = AnchorReferralRequestPayload { - requester: [2u8; 32], - requester_addresses: vec!["10.0.0.2:4433".to_string()], - }; - let json = serde_json::to_string(&payload).unwrap(); - let decoded: AnchorReferralRequestPayload = serde_json::from_str(&json).unwrap(); - assert_eq!(decoded.requester, [2u8; 32]); - assert_eq!(decoded.requester_addresses, vec!["10.0.0.2:4433"]); - } - - #[test] - fn anchor_referral_response_payload_roundtrip() { - let payload = AnchorReferralResponsePayload { + fn convection_response_roundtrip_including_cheap_refusal() { + let payload = ConvectionResponsePayload { referrals: vec![ - AnchorReferral { + ConvectionReferral { node_id: [3u8; 32], addresses: vec!["10.0.0.3:4433".to_string()], + introduced: false, }, - AnchorReferral { + ConvectionReferral { node_id: [4u8; 32], - addresses: vec!["10.0.0.4:4433".to_string(), "192.168.1.4:4433".to_string()], + addresses: vec!["10.0.0.4:4433".to_string()], + introduced: true, }, ], + refused: false, }; let json = serde_json::to_string(&payload).unwrap(); - let decoded: AnchorReferralResponsePayload = serde_json::from_str(&json).unwrap(); + let decoded: ConvectionResponsePayload = serde_json::from_str(&json).unwrap(); assert_eq!(decoded.referrals.len(), 2); - assert_eq!(decoded.referrals[0].node_id, [3u8; 32]); - assert_eq!(decoded.referrals[0].addresses, vec!["10.0.0.3:4433"]); - assert_eq!(decoded.referrals[1].node_id, [4u8; 32]); - assert_eq!(decoded.referrals[1].addresses.len(), 2); + assert!(!decoded.referrals[0].introduced); + assert!(decoded.referrals[1].introduced); + assert!(!decoded.refused); - // Empty referrals - let empty = AnchorReferralResponsePayload { referrals: vec![] }; - let json2 = serde_json::to_string(&empty).unwrap(); - let decoded2: AnchorReferralResponsePayload = serde_json::from_str(&json2).unwrap(); + // A refusal is one small message with no referrals — the caller reads + // it immediately instead of burning a timeout. + let refusal = ConvectionResponsePayload { referrals: vec![], refused: true }; + let json2 = serde_json::to_string(&refusal).unwrap(); + assert!(json2.len() < 64, "refusal must stay tiny: {}", json2); + let decoded2: ConvectionResponsePayload = serde_json::from_str(&json2).unwrap(); + assert!(decoded2.refused); assert!(decoded2.referrals.is_empty()); } + + #[test] + fn uniques_pull_is_an_index_exchange_not_a_post_transfer() { + let payload = UniquesPullPayload { + uniques: Some(UniquesAnnouncePayload::empty(7, 4)), + }; + let json = serde_json::to_string(&payload).unwrap(); + let decoded: UniquesPullPayload = serde_json::from_str(&json).unwrap(); + let u = decoded.uniques.unwrap(); + assert_eq!(u.seq, 7); + assert_eq!(u.depth, 4); + assert_eq!(u.fwd.len(), 3); + // No post field exists on this payload at all — content rides 0x46/0x47. + assert!(!json.contains("posts")); + + // A temp referral slot carries no knowledge in either direction. + let none = UniquesPullPayload { uniques: None }; + let json2 = serde_json::to_string(&none).unwrap(); + let decoded2: UniquesPullPayload = serde_json::from_str(&json2).unwrap(); + assert!(decoded2.uniques.is_none()); + } + + /// `len()` backs budget checks, so it must be EXACT — not the old + /// `bytes / 44` approximation, which assumed one base64 encoding per ID + /// when `pack_ids` encodes the whole concatenated blob once. + #[test] + fn uniques_slice_len_counts_packed_ids_exactly() { + for n in [0usize, 1, 2, 3, 7, 100, 1000] { + let ids: Vec = (0..n) + .map(|i| { + let mut id = [0u8; 32]; + id[0] = (i % 251) as u8; + id[1] = (i / 251) as u8; + id + }) + .collect(); + let packed = pack_ids(&ids); + assert_eq!(packed_id_count(&packed), n, "packed_id_count for n={}", n); + assert_eq!(unpack_ids(&packed).len(), n); + + let slice = UniquesSlice { + nodes: packed.clone(), + authors: packed.clone(), + anchors: vec![AnchorEntry { i: "aa".into(), a: vec!["1.2.3.4:1".into()] }], + }; + assert_eq!(slice.len(), n * 2 + 1, "UniquesSlice::len for n={}", n); + } + } } diff --git a/crates/core/src/storage.rs b/crates/core/src/storage.rs index 6fd2ae6..ffa7e61 100644 --- a/crates/core/src/storage.rs +++ b/crates/core/src/storage.rs @@ -7,9 +7,9 @@ use crate::types::{ Attachment, Circle, CircleProfile, CommentPolicy, DeleteRecord, FollowVisibility, GossipPeerInfo, GroupEpoch, GroupId, GroupKeyRecord, GroupMemberKey, InlineComment, ManifestEntry, NodeId, PeerRecord, - PeerSlotKind, PeerWithAddress, Post, PostId, PostVisibility, PostingIdentity, - PublicProfile, Reaction, ReachMethod, SocialRelation, SocialRouteEntry, SocialStatus, - ThreadMeta, VisibilityIntent, + IdClass, PeerWithAddress, Post, PostId, PostVisibility, PostingIdentity, + PublicProfile, ReachEntry, Reaction, ReachMethod, SocialRelation, SocialRouteEntry, + SocialStatus, ThreadMeta, UniquesPools, VisibilityIntent, }; /// Direction for file_holders entries: whether we sent the file to this peer, @@ -92,11 +92,39 @@ impl StoragePool { /// Current schema version. Bump this when making schema or data changes /// that require migration. Old databases with a lower version will be migrated. /// If the gap is too large (major version mismatch), the DB is reset instead. -const SCHEMA_VERSION: u32 = 2; +const SCHEMA_VERSION: u32 = 3; /// Minimum schema version we can migrate from. Anything older gets a full reset. const MIN_MIGRATABLE_VERSION: u32 = 1; +/// How recently we must have been in contact with an anchor-flagged peer for +/// its address to ride our own bounce-1 slice. `peers.is_anchor` is a one-way +/// latch; this is what bounds it in time. +const ANCHOR_ENRICH_MAX_AGE_MS: i64 = 24 * 60 * 60 * 1000; // 24h + +// ---- v0.8 §layers size budget for the uniques pools ---- +// +// design.html §layers publishes N2 <= 400 and N3 <= 8,000 as the design +// ceiling. These are the ENFORCED numbers, set generously above the design +// budget so a large-but-honest peer degrades gracefully (truncation) rather +// than being refused, while a hostile peer cannot make us write six figures of +// rows per announce. + +/// Max entries we will BUILD into one shallow announce slice (fwd[1], fwd[2]). +pub const UNIQUES_BUILD_CAP_SHALLOW: usize = 4_000; +/// Max entries we will BUILD into the terminal slice (our N3, budget 8,000). +pub const UNIQUES_BUILD_CAP_TERM: usize = 12_000; + +/// Max entries we will ACCEPT at each bounce from one reporter — 2x our own +/// build caps, so an honest peer is never truncated and a hostile one is. +pub fn uniques_accept_cap(bounce: u8) -> usize { + match bounce { + 0 | 1 | 2 => 8_000, + 3 => 8_000, + _ => 24_000, + } +} + impl Storage { pub fn open(path: impl AsRef) -> anyhow::Result { let conn = Connection::open(path.as_ref())?; @@ -218,24 +246,25 @@ impl Storage { target_id BLOB PRIMARY KEY, failed_at INTEGER NOT NULL ); - CREATE TABLE IF NOT EXISTS reachable_n2 ( - reporter_node_id BLOB NOT NULL, - reachable_node_id BLOB NOT NULL, - updated_at INTEGER NOT NULL, - PRIMARY KEY (reporter_node_id, reachable_node_id) + -- v0.8 (Iteration C): reachable_n2 + reachable_n3 collapse into one + -- bounce-parametrised table so N4 does not need a third near-identical + -- copy of 12 methods. bounce ∈ {2,3,4}; bounce 1 (our own uniques) is + -- computed from mesh_peers/social_routes/CDN, never stored here. + -- addresses is non-empty ONLY when is_anchor = 1 (design.html §layers). + CREATE TABLE IF NOT EXISTS reachable ( + reporter_node_id BLOB NOT NULL, + reachable_id BLOB NOT NULL, + bounce INTEGER NOT NULL, + id_class INTEGER NOT NULL DEFAULT 0, + is_anchor INTEGER NOT NULL DEFAULT 0, + addresses TEXT NOT NULL DEFAULT '[]', + updated_at INTEGER NOT NULL, + PRIMARY KEY (reporter_node_id, reachable_id) ); - CREATE INDEX IF NOT EXISTS idx_n2_reachable ON reachable_n2(reachable_node_id); - CREATE TABLE IF NOT EXISTS reachable_n3 ( - reporter_node_id BLOB NOT NULL, - reachable_node_id BLOB NOT NULL, - updated_at INTEGER NOT NULL, - PRIMARY KEY (reporter_node_id, reachable_node_id) - ); - CREATE INDEX IF NOT EXISTS idx_n3_reachable ON reachable_n3(reachable_node_id); + CREATE INDEX IF NOT EXISTS idx_reachable_id ON reachable(reachable_id, bounce); + CREATE INDEX IF NOT EXISTS idx_reachable_anchor ON reachable(is_anchor, bounce); CREATE TABLE IF NOT EXISTS mesh_peers ( node_id BLOB NOT NULL PRIMARY KEY, - slot_kind TEXT NOT NULL, - priority INTEGER NOT NULL DEFAULT 0, connected_at INTEGER NOT NULL, last_diff_seq INTEGER NOT NULL DEFAULT 0 ); @@ -364,6 +393,12 @@ impl Storage { deleted_at INTEGER, -- v0.8 (A3): absolute expiry (ms). NULL/0 = legacy no-TTL. expires_at INTEGER, + -- v0.8 (C): 1 = WE authored this comment locally. Locally + -- authored comment authors (our personas, our per-greeting + -- throwaway IDs) must never enter our own uniques announce — + -- announcing them at bounce 1 next to our node ID is a + -- persona-to-device linkage (design.html §21). + origin_local INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (author, post_id, timestamp_ms) ); CREATE INDEX IF NOT EXISTS idx_comments_post ON comments(post_id); @@ -671,6 +706,34 @@ impl Storage { "ALTER TABLE routing_peers RENAME TO mesh_peers;" ); + // v0.8 Iteration C: mesh_peers loses slot_kind (Local/Wide collapsed into + // one pool) and priority (the preferred-tier rank, dead since Iteration B). + // init_tables() uses CREATE TABLE IF NOT EXISTS, so an existing DB keeps + // the old 5-column table and never sees the new DDL — the drop+create has + // to happen HERE (migrate runs after init_tables, before migrate_data). + // Safe to drop outright: node.rs clears the whole table on every open. + let has_slot_kind = self.conn.prepare( + "SELECT COUNT(*) FROM pragma_table_info('mesh_peers') WHERE name='slot_kind'" + )?.query_row([], |row| row.get::<_, i64>(0))?; + if has_slot_kind > 0 { + self.conn.execute_batch( + "DROP TABLE IF EXISTS mesh_peers; + CREATE TABLE mesh_peers ( + node_id BLOB NOT NULL PRIMARY KEY, + connected_at INTEGER NOT NULL, + last_diff_seq INTEGER NOT NULL DEFAULT 0 + );" + )?; + } + + // v0.8 Iteration C: reachable_n2/reachable_n3 → one `reachable` table + // parametrised by bounce (2/3/4). The N-layer index is rebuilt from + // scratch on every handshake, so there is nothing worth carrying over. + let _ = self.conn.execute_batch( + "DROP TABLE IF EXISTS reachable_n2; + DROP TABLE IF EXISTS reachable_n3;" + ); + // Add post_id/author/created_at/last_accessed_at columns to blobs if missing // (blobs are recoverable from filesystem + posts, so drop-and-recreate is safe) let has_blob_post_id = self.conn.prepare( @@ -811,6 +874,18 @@ impl Storage { )?; } + // v0.8 (C): mark locally-authored comments so `build_own_uniques` can + // exclude their authors. Our own personas and per-greeting throwaway + // IDs must never ride our N1 slice — see §21 persona/device split. + let has_origin_local = self.conn.prepare( + "SELECT COUNT(*) FROM pragma_table_info('comments') WHERE name='origin_local'" + )?.query_row([], |row| row.get::<_, i64>(0))?; + if has_origin_local == 0 { + self.conn.execute_batch( + "ALTER TABLE comments ADD COLUMN origin_local INTEGER NOT NULL DEFAULT 0;" + )?; + } + // v0.6.2: add canonical_root_post_id to group_keys. When set, the // record is a group (many-way, anchored at a public root post); // when NULL it's a traditional circle (one-way, admin-only). @@ -1822,8 +1897,10 @@ impl Storage { /// Get a random N2 stranger (node in reachable_n2 but not in our connections) /// Returns (witness_node_id, reporter_node_id) for anchor probe pub fn random_n2_stranger(&self, our_connections: &std::collections::HashSet) -> anyhow::Result> { + // id_class = 0: an author/persona ID cannot act as a probe witness. let mut stmt = self.conn.prepare( - "SELECT reachable_node_id, reporter_node_id FROM reachable_n2 ORDER BY RANDOM() LIMIT 10" + "SELECT reachable_id, reporter_node_id FROM reachable + WHERE bounce = 2 AND id_class = 0 ORDER BY RANDOM() LIMIT 10" )?; let rows = stmt.query_map([], |row| { let rn: Vec = row.get(0)?; @@ -2143,9 +2220,84 @@ impl Storage { Ok(records) } - // ---- Known anchors (persistent anchor cache for NAT traversal) ---- + /// Anchor-flagged peers we have actually been in contact with recently. + /// + /// `build_own_uniques` uses this (never the unbounded `list_anchor_peers`) + /// so a stale one-way `is_anchor` latch cannot keep re-advertising an + /// address we have not touched in months. + pub fn list_anchor_peers_seen_since(&self, cutoff_ms: i64) -> anyhow::Result> { + let mut stmt = self.conn.prepare( + "SELECT node_id, addresses, last_seen, introduced_by, is_anchor, first_seen + FROM peers WHERE is_anchor = 1 AND last_seen >= ?1 ORDER BY last_seen DESC", + )?; + let mut records = Vec::new(); + let mut rows = stmt.query(params![cutoff_ms])?; + while let Some(row) = rows.next()? { + records.push(row_to_peer_record(row)?); + } + Ok(records) + } - /// Upsert a known anchor. Increments success_count on conflict. Auto-prunes to 5. + /// Merge addresses into a peer row WITHOUT dropping what is already there. + /// + /// `upsert_peer` REPLACES the address list, which is right when we have + /// just connected on those addresses and wrong for anything a peer merely + /// claims: a claimed address that replaced a working one is bugs-fixed #5 + /// (stale/wrong addresses block reconnection), remotely triggerable. Union + /// semantics keep every address we have ever had for the node, newest + /// first, capped so the row cannot be inflated without bound. + pub fn add_peer_addresses( + &self, + node_id: &NodeId, + addresses: &[SocketAddr], + ) -> anyhow::Result<()> { + const MAX_PEER_ADDRESSES: usize = 8; + if addresses.is_empty() { + return Ok(()); + } + // EXISTING FIRST, claims appended. `upsert_peer` writes the addresses we + // actually connected on, so position 0 stays the proven one — callers + // that take `.first()` keep working, and no volume of claims can push + // a working address out under the cap (bugs-fixed #5). + let mut merged: Vec = self + .get_peer_record(node_id)? + .map(|r| r.addresses.iter().map(|a| a.to_string()).collect()) + .unwrap_or_default(); + for a in addresses { + let s = a.to_string(); + if !merged.contains(&s) { + merged.push(s); + } + } + merged.truncate(MAX_PEER_ADDRESSES); + let now = now_ms(); + let addrs_json = serde_json::to_string(&merged)?; + self.conn.execute( + "INSERT INTO peers (node_id, addresses, last_seen, introduced_by, first_seen) + VALUES (?1, ?2, ?3, NULL, ?3) + ON CONFLICT(node_id) DO UPDATE SET addresses = ?2, last_seen = ?3", + params![node_id.as_slice(), addrs_json, now], + )?; + Ok(()) + } + + // ---- Known anchors — the BOOTSTRAP CACHE ---- + // + // v0.8 (round-4 ruling): this table is demoted. The uniques pools are the + // anchor directory (anchor entries are the only address-bearing rows, so + // `list_pool_anchors` supplies anchors in bulk and for free); `known_anchors` + // exists only to survive a cold start with an empty index. `success_count` + // ranking was a v0.7.x scarcity artifact — under pool-mined abundance, + // "the anchor that answered most often" is not a better anchor, it is just + // the one we hammered. Ordering is now by recency, which at least means + // "most likely to still be at that address". + + /// Cap on the bootstrap cache. Raised from 5 because `apply_uniques_- + /// to_storage` now mirrors every received anchor entry here — at 5 the + /// table would thrash on every announce. + pub const KNOWN_ANCHOR_CACHE_CAP: usize = 32; + + /// Upsert a known anchor. Auto-prunes to `KNOWN_ANCHOR_CACHE_CAP`. pub fn upsert_known_anchor(&self, node_id: &NodeId, addresses: &[SocketAddr]) -> anyhow::Result<()> { let addr_json = serde_json::to_string( &addresses.iter().map(|a| a.to_string()).collect::>(), @@ -2158,18 +2310,19 @@ impl Storage { addresses = ?2, last_seen_ms = ?3, success_count = success_count + 1", params![node_id.as_slice(), addr_json, now], )?; - self.prune_known_anchors(5)?; + self.prune_known_anchors(Self::KNOWN_ANCHOR_CACHE_CAP)?; Ok(()) } - /// List known anchors, ordered by success_count descending. + /// List the bootstrap cache, freshest first. (Not success_count — see the + /// section comment above.) pub fn list_known_anchors(&self) -> anyhow::Result)>> { let mut stmt = self.conn.prepare( "SELECT node_id, addresses FROM known_anchors - ORDER BY success_count DESC LIMIT 5", + ORDER BY last_seen_ms DESC LIMIT ?1", )?; let mut result = Vec::new(); - let mut rows = stmt.query([])?; + let mut rows = stmt.query(params![Self::KNOWN_ANCHOR_CACHE_CAP as i64])?; while let Some(row) = rows.next()? { let node_id = blob_to_nodeid(row.get(0)?)?; let addr_json: String = row.get(1)?; @@ -2210,7 +2363,9 @@ impl Storage { Ok(()) } - /// Prune known anchors to keep at most `max` entries (by highest success_count). + /// Prune the bootstrap cache to at most `max` entries, dropping the + /// stalest first. (The 3-day stale-anchor self-prune on failed probes — + /// `delete_known_anchor` via `maybe_prune_stale_anchor` — is unchanged.) pub fn prune_known_anchors(&self, max: usize) -> anyhow::Result { let count: i64 = self.conn.query_row( "SELECT COUNT(*) FROM known_anchors", @@ -2224,7 +2379,7 @@ impl Storage { self.conn.execute( "DELETE FROM known_anchors WHERE node_id IN ( SELECT node_id FROM known_anchors - ORDER BY success_count ASC, last_seen_ms ASC + ORDER BY last_seen_ms ASC LIMIT ?1 )", params![excess as i64], @@ -3365,276 +3520,562 @@ impl Storage { Ok(count > 0) } - // ---- Reach: N2/N3 ---- + // ---- Reach: the bounce-parametrised uniques index ---- - /// Replace a peer's entire N1 set in reachable_n2 (their N1 share → our N2). - pub fn set_peer_n1(&self, reporter: &NodeId, node_ids: &[NodeId]) -> anyhow::Result<()> { - let now = now_ms(); + /// Replace a reporter's ENTIRE contribution at one bounce. + /// + /// Dedup is free at the store: the composite PK `(reporter, reachable_id)` + /// gives per-reporter set-merge, and the conflict clause keeps the + /// SHALLOWEST bounce we have ever heard from this reporter — so an ID can + /// never occupy two depths from one reporter (the ambiguity the old + /// n2/n3-table pair papered over by sorting). + /// + /// DELETE + inserts run in ONE transaction. Thousands of individually + /// committed inserts (an announce is sized in thousands of rows, once per + /// peer per cycle across ~20 peers) is the dominant cost of the whole + /// uniques exchange; one commit makes it a rounding error. + pub fn set_reach( + &self, + reporter: &NodeId, + bounce: u8, + entries: &[ReachEntry], + ) -> anyhow::Result<()> { + let tx = self.conn.unchecked_transaction()?; self.conn.execute( - "DELETE FROM reachable_n2 WHERE reporter_node_id = ?1", - params![reporter.as_slice()], + "DELETE FROM reachable WHERE reporter_node_id = ?1 AND bounce = ?2", + params![reporter.as_slice(), bounce as i64], )?; - let mut stmt = self.conn.prepare( - "INSERT OR REPLACE INTO reachable_n2 (reporter_node_id, reachable_node_id, updated_at) VALUES (?1, ?2, ?3)", - )?; - for nid in node_ids { - stmt.execute(params![reporter.as_slice(), nid.as_slice(), now])?; - } + self.add_reach_inner(reporter, bounce, entries)?; + tx.commit()?; Ok(()) } - /// Add NodeIds to a peer's N1 set in reachable_n2. - pub fn add_peer_n1(&self, reporter: &NodeId, node_ids: &[NodeId]) -> anyhow::Result<()> { + /// Merge entries into a reporter's contribution at one bounce. + pub fn add_reach( + &self, + reporter: &NodeId, + bounce: u8, + entries: &[ReachEntry], + ) -> anyhow::Result<()> { + if entries.is_empty() { + return Ok(()); + } + let tx = self.conn.unchecked_transaction()?; + self.add_reach_inner(reporter, bounce, entries)?; + tx.commit()?; + Ok(()) + } + + fn add_reach_inner( + &self, + reporter: &NodeId, + bounce: u8, + entries: &[ReachEntry], + ) -> anyhow::Result<()> { + if entries.is_empty() { + return Ok(()); + } let now = now_ms(); let mut stmt = self.conn.prepare( - "INSERT OR REPLACE INTO reachable_n2 (reporter_node_id, reachable_node_id, updated_at) VALUES (?1, ?2, ?3)", + "INSERT INTO reachable + (reporter_node_id, reachable_id, bounce, id_class, is_anchor, addresses, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ON CONFLICT(reporter_node_id, reachable_id) DO UPDATE SET + bounce = MIN(bounce, excluded.bounce), + id_class = excluded.id_class, + is_anchor = MAX(is_anchor, excluded.is_anchor), + addresses = CASE WHEN excluded.is_anchor = 1 THEN excluded.addresses ELSE addresses END, + updated_at = excluded.updated_at", )?; - for nid in node_ids { - stmt.execute(params![reporter.as_slice(), nid.as_slice(), now])?; + for e in entries { + let addrs_json = if e.is_anchor && !e.addresses.is_empty() { + serde_json::to_string(&e.addresses).unwrap_or_else(|_| "[]".into()) + } else { + "[]".to_string() + }; + stmt.execute(params![ + reporter.as_slice(), + e.id.as_slice(), + bounce as i64, + e.id_class.as_i64(), + if e.is_anchor { 1i64 } else { 0i64 }, + addrs_json, + now, + ])?; } Ok(()) } - /// Remove NodeIds from a peer's N1 set in reachable_n2. - pub fn remove_peer_n1(&self, reporter: &NodeId, node_ids: &[NodeId]) -> anyhow::Result<()> { - let mut stmt = self.conn.prepare( - "DELETE FROM reachable_n2 WHERE reporter_node_id = ?1 AND reachable_node_id = ?2", - )?; - for nid in node_ids { - stmt.execute(params![reporter.as_slice(), nid.as_slice()])?; - } - Ok(()) - } - - /// Remove all N2 entries from a specific reporter (on disconnect). - pub fn clear_peer_n2(&self, reporter: &NodeId) -> anyhow::Result { + /// Remove all entries a reporter contributed (on disconnect). + pub fn clear_peer_reach(&self, reporter: &NodeId) -> anyhow::Result { let deleted = self.conn.execute( - "DELETE FROM reachable_n2 WHERE reporter_node_id = ?1", + "DELETE FROM reachable WHERE reporter_node_id = ?1", params![reporter.as_slice()], )?; Ok(deleted) } - /// Replace a peer's N2-reported entries in reachable_n3 (their N2 share → our N3). - pub fn set_peer_n2(&self, reporter: &NodeId, node_ids: &[NodeId]) -> anyhow::Result<()> { - let now = now_ms(); - self.conn.execute( - "DELETE FROM reachable_n3 WHERE reporter_node_id = ?1", - params![reporter.as_slice()], - )?; + /// Which reporters place this ID at exactly this bounce? + pub fn find_reporters_at(&self, id: &NodeId, bounce: u8) -> anyhow::Result> { let mut stmt = self.conn.prepare( - "INSERT OR REPLACE INTO reachable_n3 (reporter_node_id, reachable_node_id, updated_at) VALUES (?1, ?2, ?3)", + "SELECT reporter_node_id FROM reachable WHERE reachable_id = ?1 AND bounce = ?2", )?; - for nid in node_ids { - stmt.execute(params![reporter.as_slice(), nid.as_slice(), now])?; + let mut result = Vec::new(); + let mut rows = stmt.query(params![id.as_slice(), bounce as i64])?; + while let Some(row) = rows.next()? { + result.push(blob_to_nodeid(row.get(0)?)?); } - Ok(()) + Ok(result) } - /// Add to N3 from a peer's N2 changes. - pub fn add_peer_n2(&self, reporter: &NodeId, node_ids: &[NodeId]) -> anyhow::Result<()> { - let now = now_ms(); - let mut stmt = self.conn.prepare( - "INSERT OR REPLACE INTO reachable_n3 (reporter_node_id, reachable_node_id, updated_at) VALUES (?1, ?2, ?3)", - )?; - for nid in node_ids { - stmt.execute(params![reporter.as_slice(), nid.as_slice(), now])?; - } - Ok(()) - } - - /// Remove from N3. - pub fn remove_peer_n2(&self, reporter: &NodeId, node_ids: &[NodeId]) -> anyhow::Result<()> { - let mut stmt = self.conn.prepare( - "DELETE FROM reachable_n3 WHERE reporter_node_id = ?1 AND reachable_node_id = ?2", - )?; - for nid in node_ids { - stmt.execute(params![reporter.as_slice(), nid.as_slice()])?; - } - Ok(()) - } - - /// Remove all N3 entries from a specific reporter. - pub fn clear_peer_n3(&self, reporter: &NodeId) -> anyhow::Result { - let deleted = self.conn.execute( - "DELETE FROM reachable_n3 WHERE reporter_node_id = ?1", - params![reporter.as_slice()], - )?; - Ok(deleted) - } - - /// Which reporters have this node in N2? + /// Which reporters have this ID in our N2 (bounce 2)? pub fn find_in_n2(&self, node_id: &NodeId) -> anyhow::Result> { - let mut stmt = self.conn.prepare( - "SELECT reporter_node_id FROM reachable_n2 WHERE reachable_node_id = ?1", - )?; - let mut result = Vec::new(); - let mut rows = stmt.query(params![node_id.as_slice()])?; - while let Some(row) = rows.next()? { - result.push(blob_to_nodeid(row.get(0)?)?); - } - Ok(result) + self.find_reporters_at(node_id, 2) } - /// Which reporters have this node in N3? + /// Which reporters have this ID in our N3 (bounce 3)? pub fn find_in_n3(&self, node_id: &NodeId) -> anyhow::Result> { - let mut stmt = self.conn.prepare( - "SELECT reporter_node_id FROM reachable_n3 WHERE reachable_node_id = ?1", - )?; - let mut result = Vec::new(); - let mut rows = stmt.query(params![node_id.as_slice()])?; - while let Some(row) = rows.next()? { - result.push(blob_to_nodeid(row.get(0)?)?); - } - Ok(result) + self.find_reporters_at(node_id, 3) } - /// Batch lookup: find any of the given node IDs in N2 or N3. - /// Returns Vec<(target, reporter, level)> where level is 2 or 3, sorted by level ASC. - pub fn find_any_in_n2_n3(&self, ids: &[NodeId]) -> anyhow::Result> { + /// Which reporters have this ID in our N4 (bounce 4)? + /// + /// N4 is USED (search, address resolution) but NEVER re-announced — + /// `build_uniques_pools` simply does not read bounce-4 rows. + pub fn find_in_n4(&self, node_id: &NodeId) -> anyhow::Result> { + self.find_reporters_at(node_id, 4) + } + + /// Batch lookup across the whole stored horizon (bounces 2..4). + /// Returns Vec<(target, reporter, bounce)> sorted shallowest-first. + /// Node-class only — an author ID is not a routing target. + pub fn find_any_reachable(&self, ids: &[NodeId]) -> anyhow::Result> { if ids.is_empty() { return Ok(vec![]); } + let mut stmt = self.conn.prepare( + "SELECT reporter_node_id, bounce FROM reachable + WHERE reachable_id = ?1 AND id_class = 0 ORDER BY bounce ASC", + )?; let mut results = Vec::new(); - // Check N2 first (closer) for id in ids { - let reporters = self.find_in_n2(id)?; - for r in reporters { - results.push((*id, r, 2u8)); + let mut rows = stmt.query(params![id.as_slice()])?; + while let Some(row) = rows.next()? { + let reporter = blob_to_nodeid(row.get(0)?)?; + let bounce: i64 = row.get(1)?; + results.push((*id, reporter, bounce as u8)); } } - // Then N3 - for id in ids { - let reporters = self.find_in_n3(id)?; - for r in reporters { - results.push((*id, r, 3u8)); - } - } - results.sort_by_key(|&(_, _, level)| level); + results.sort_by_key(|&(_, _, bounce)| bounce); Ok(results) } - /// All NodeIds this peer can reach (from N2 table). - pub fn list_n2_for_reporter(&self, reporter: &NodeId) -> anyhow::Result> { + /// Everything a reporter contributed at one bounce. + pub fn list_reach_for_reporter( + &self, + reporter: &NodeId, + bounce: u8, + ) -> anyhow::Result> { let mut stmt = self.conn.prepare( - "SELECT reachable_node_id FROM reachable_n2 WHERE reporter_node_id = ?1", + "SELECT reachable_id FROM reachable WHERE reporter_node_id = ?1 AND bounce = ?2", )?; let mut result = Vec::new(); - let mut rows = stmt.query(params![reporter.as_slice()])?; + let mut rows = stmt.query(params![reporter.as_slice(), bounce as i64])?; while let Some(row) = rows.next()? { result.push(blob_to_nodeid(row.get(0)?)?); } Ok(result) } - /// Build N1 share: merge mesh peers (connections) + social contacts NodeIds (deduplicated). - pub fn build_n1_share(&self) -> anyhow::Result> { - let mut ids = std::collections::HashSet::new(); - // Add mesh peers (connections) - let mesh_peers = self.list_mesh_peers()?; - for (nid, _, _) in mesh_peers { - ids.insert(nid); + /// Distinct IDs stored at a bounce, with their class/anchor metadata. + /// + /// One row per ID is selected EXPLICITLY (anchor rows first, then freshest) + /// rather than relying on SQLite's bare-column-with-MAX() quirk to keep + /// `id_class`, `is_anchor` and `addresses` from the same physical row. A + /// future query rewrite or a second aggregate would silently break that + /// pairing, and the thing it pairs is "is this ID address-bearing" — the + /// invariant the whole announce format rests on. + /// + /// `limit` bounds the result deterministically so the announce builder + /// cannot emit an unbounded slice; ordering is stable across cycles + /// (anchors, then freshest) so the announce digest does not thrash. + pub fn list_reach_entries_at(&self, bounce: u8) -> anyhow::Result> { + self.list_reach_entries_at_limited(bounce, usize::MAX) + } + + pub fn list_reach_entries_at_limited( + &self, + bounce: u8, + limit: usize, + ) -> anyhow::Result> { + let mut stmt = self.conn.prepare( + "SELECT reachable_id, id_class, is_anchor, addresses, updated_at + FROM reachable WHERE bounce = ?1 + ORDER BY reachable_id ASC, is_anchor DESC, updated_at DESC", + )?; + let mut result: Vec = Vec::new(); + let mut last: Option = None; + let mut rows = stmt.query(params![bounce as i64])?; + while let Some(row) = rows.next()? { + let id = blob_to_nodeid(row.get(0)?)?; + if last == Some(id) { + continue; // keep only the winning row per ID + } + last = Some(id); + let id_class = IdClass::from_i64(row.get::<_, i64>(1)?); + let is_anchor = row.get::<_, i64>(2)? != 0; + let addrs_json: String = row.get(3)?; + let addresses: Vec = serde_json::from_str(&addrs_json).unwrap_or_default(); + result.push(ReachEntry { id, id_class, is_anchor, addresses }); } - // Add only ONLINE social routes (not disconnected) - let routes = self.list_social_routes()?; - for route in routes { - if route.status == crate::types::SocialStatus::Online { - ids.insert(route.node_id); + if result.len() > limit { + // Anchors first (they are the anchor directory), then the rest — + // deterministic, so trimming does not thrash the digest. + result.sort_by(|a, b| b.is_anchor.cmp(&a.is_anchor).then(a.id.cmp(&b.id))); + result.truncate(limit); + } + Ok(result) + } + + /// Distinct IDs stored at a bounce (bare). + pub fn list_reach_ids_at(&self, bounce: u8) -> anyhow::Result> { + let mut stmt = self.conn.prepare( + "SELECT DISTINCT reachable_id FROM reachable WHERE bounce = ?1", + )?; + let mut result = Vec::new(); + let mut rows = stmt.query(params![bounce as i64])?; + while let Some(row) = rows.next()? { + result.push(blob_to_nodeid(row.get(0)?)?); + } + Ok(result) + } + + /// Deepest bounce present in the index (0 when empty). Test/diagnostic + /// helper: nothing may ever be stored past bounce 4. + pub fn max_stored_bounce(&self) -> anyhow::Result { + Ok(self.conn.query_row( + "SELECT ifnull(MAX(bounce), 0) FROM reachable", + [], + |row| row.get(0), + )?) + } + + /// Count of distinct IDs stored at a bounce. + pub fn count_distinct_at(&self, bounce: u8) -> anyhow::Result { + let count: i64 = self.conn.query_row( + "SELECT COUNT(DISTINCT reachable_id) FROM reachable WHERE bounce = ?1", + params![bounce as i64], + |row| row.get(0), + )?; + Ok(count as usize) + } + + /// Anchor entries mined from the uniques pools, shallowest and freshest first. + /// + /// Round-4 ruling: because anchor entries are the only address-bearing rows, + /// the pools double as the anchor directory. `known_anchors` demotes to a + /// bootstrap cache. SEAM FOR WRITER 2 (convection): this is the query the + /// stochastic growth trigger should draw a "random known anchor" from. + pub fn list_pool_anchors(&self, limit: usize) -> anyhow::Result)>> { + let mut stmt = self.conn.prepare( + "SELECT reachable_id, addresses, MIN(bounce) FROM reachable + WHERE is_anchor = 1 AND id_class = 0 + GROUP BY reachable_id + ORDER BY MIN(bounce) ASC, MAX(updated_at) DESC + LIMIT ?1", + )?; + let mut result = Vec::new(); + let mut rows = stmt.query(params![limit as i64])?; + while let Some(row) = rows.next()? { + let id = blob_to_nodeid(row.get(0)?)?; + let addrs_json: String = row.get(1)?; + let addrs: Vec = serde_json::from_str(&addrs_json).unwrap_or_default(); + if !addrs.is_empty() { + result.push((id, addrs)); } } - Ok(ids.into_iter().collect()) - } - - /// Build N2 share (reach): deduplicated unique NodeIds from all N2 entries. - pub fn build_n2_share(&self) -> anyhow::Result> { - let mut stmt = self.conn.prepare( - "SELECT DISTINCT reachable_node_id FROM reachable_n2", - )?; - let mut result = Vec::new(); - let mut rows = stmt.query([])?; - while let Some(row) = rows.next()? { - result.push(blob_to_nodeid(row.get(0)?)?); - } Ok(result) } - /// Count distinct reachable NodeIds in the N2 table. + /// Anchor-flagged share of the distinct node-class IDs we know about — a + /// free local statistic (no census) because anchor entries self-identify by + /// carrying an address. SEAM FOR WRITER 2: round-5 adaptive action-weight + /// prior for the stochastic convection trigger. + pub fn anchor_density(&self) -> anyhow::Result { + let (anchors, total): (i64, i64) = self.conn.query_row( + "SELECT COUNT(DISTINCT CASE WHEN is_anchor = 1 THEN reachable_id END), + COUNT(DISTINCT reachable_id) + FROM reachable WHERE id_class = 0", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + if total == 0 { + return Ok(0.0); + } + Ok(anchors as f64 / total as f64) + } + + /// Our own bounce-1 uniques: every ID we have a live path to, merged from + /// all four sources BEFORE announcing so a receiver cannot tell which source + /// an entry came from. + /// + /// * mesh peers (node class) — temp referral slots are excluded + /// structurally: they are never + /// written to `mesh_peers` + /// * social directs (node class) — ALL of them, not just Online: + /// the ruling is "everyone's + /// directs"; a held route is a + /// findable ID either way + /// * CDN file-holder peers (node class) + /// * CDN file authors (AUTHOR class) — posting identities. No address, + /// no device linkage. Kept in a + /// separate class so they never + /// enter the connect/hole-punch + /// paths. + /// * comment authors (AUTHOR class) — OTHER people's throwaway/ + /// greeting IDs stay in + /// deliberately (anonymity noise, + /// round-1 ruling); ours are + /// excluded, see below. + /// + /// NEVER included: + /// * our own posting identities (all personas, not just the default) and + /// any author ID on a locally-authored comment (per-greeting throwaway + /// IDs). Our node ID and — when we are an anchor — our address ride + /// `fwd[0]` of the very same announce, so emitting our personas at + /// bounce 1 would hand every mesh peer a signed-by-transport + /// persona→device mapping. Worse, our own personas are the invariant + /// across every announce we ever send while everything else churns, so + /// the set is trivially separable by diffing two announces. design.html + /// §21: "Posting identities never map to device addresses on the wire." + /// They remain discoverable through everyone ELSE who holds our + /// content — we simply must not be the reporter. + /// * anchors we have merely HEARD about. A third-party anchor report is + /// an index row at its true bounce, not a bounce-1 unique of ours; + /// re-emitting it at depth 1 would reset its horizon on every hop and + /// let a terminal (N4) entry propagate forever. + pub fn build_own_uniques(&self) -> anyhow::Result> { + let mut nodes: std::collections::HashSet = std::collections::HashSet::new(); + let mut authors: std::collections::HashSet = std::collections::HashSet::new(); + + for nid in self.list_mesh_peers()? { + nodes.insert(nid); + } + for route in self.list_social_routes()? { + nodes.insert(route.node_id); + } + { + let mut stmt = self.conn.prepare("SELECT DISTINCT peer_id FROM file_holders")?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + if let Ok(nid) = blob_to_nodeid(row.get(0)?) { + nodes.insert(nid); + } + } + } + { + let mut stmt = self.conn.prepare("SELECT DISTINCT node_id FROM post_replicas")?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + if let Ok(nid) = blob_to_nodeid(row.get(0)?) { + nodes.insert(nid); + } + } + } + { + let mut stmt = self.conn.prepare("SELECT DISTINCT author FROM cdn_manifests")?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + if let Ok(nid) = blob_to_nodeid(row.get(0)?) { + authors.insert(nid); + } + } + } + { + // origin_local = 0: comments that reached us from the network. + // Ours are filtered at the store, not here, so nobody has to + // remember the rule at each of the four local authoring sites. + let mut stmt = self.conn + .prepare("SELECT DISTINCT author FROM comments WHERE origin_local = 0")?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + if let Ok(nid) = blob_to_nodeid(row.get(0)?) { + authors.insert(nid); + } + } + } + + // Our own personas — ALL of them, not just the default posting ID — + // never ride our own announce. See the doc comment above. + for pi in self.list_posting_identities()? { + authors.remove(&pi.node_id); + nodes.remove(&pi.node_id); + } + + // A node-class ID wins over an author-class one: it is the strictly + // more useful classification and cannot mislead a connect attempt. + for n in &nodes { + authors.remove(n); + } + + // Anchor enrichment, NOT anchor import. An ID already in `nodes` (we + // have a live path to it) that we have PROVEN is an anchor by + // connecting to it carries its address; nothing new is added to the + // set. `is_anchor` is only ever set from a completed handshake or a + // connected peer's own self-declaration — never from a third party's + // announce — so a bounce-4 hearsay anchor can never surface here at + // bounce 1. + let anchor_ids: std::collections::HashSet = self + .list_anchor_peers_seen_since(now_ms() - ANCHOR_ENRICH_MAX_AGE_MS)? + .into_iter() + .map(|r| r.node_id) + .filter(|id| nodes.contains(id)) + .collect(); + + let mut out: Vec = Vec::with_capacity(nodes.len() + authors.len()); + for id in nodes { + let addresses = if anchor_ids.contains(&id) { + self.get_peer_record(&id) + .ok() + .flatten() + .map(|r| { + r.addresses + .iter() + .filter(|a| crate::network::is_publicly_routable(a)) + .map(|a| a.to_string()) + .collect::>() + }) + .unwrap_or_default() + } else { + Vec::new() + }; + let is_anchor = !addresses.is_empty(); + out.push(ReachEntry { id, id_class: IdClass::Node, is_anchor, addresses }); + } + for id in authors { + out.push(ReachEntry { + id, + id_class: IdClass::Author, + is_anchor: false, + addresses: Vec::new(), + }); + } + Ok(out) + } + + /// Build the two announce pools (design.html §layers). + /// + /// Slices are built shallow-to-deep against one running set of already- + /// emitted IDs, so an ID announced at a shallower layer is never repeated + /// deeper — dedup at the ANNOUNCE side, complementing the store-side + /// `MIN(bounce)` upsert. Duplication counts are never transmitted (that + /// would leak topology); only membership is used and the set is discarded. + /// + /// The terminal pool reads bounce 3. Bounce 4 (our N4) is NEVER read here — + /// that omission is the whole "terminal, never re-announced" rule, and it + /// is enforced structurally rather than by a filter someone can forget. + pub fn build_uniques_pools( + &self, + our_node_id: &NodeId, + our_anchor_addr: Option<&str>, + ) -> anyhow::Result { + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + + // fwd[0] = N0: ourselves. The only place our own anchor address rides. + let self_entry = ReachEntry { + id: *our_node_id, + id_class: IdClass::Node, + is_anchor: our_anchor_addr.is_some(), + addresses: our_anchor_addr.map(|a| vec![a.to_string()]).unwrap_or_default(), + }; + seen.insert(*our_node_id); + let fwd0 = vec![self_entry]; + + let take = |entries: Vec, + seen: &mut std::collections::HashSet| + -> Vec { + let mut out = Vec::new(); + for e in entries { + if seen.insert(e.id) { + out.push(e); + } + } + out + }; + + // Each slice is capped (§layers size budget). The trim is deterministic + // — anchors first, then by ID — so a stable index produces a stable + // digest and the announce is skipped rather than resent every cycle. + let mut fwd1 = take(self.build_own_uniques()?, &mut seen); + if fwd1.len() > UNIQUES_BUILD_CAP_SHALLOW { + fwd1.sort_by(|a, b| b.is_anchor.cmp(&a.is_anchor).then(a.id.cmp(&b.id))); + fwd1.truncate(UNIQUES_BUILD_CAP_SHALLOW); + } + let fwd2 = take( + self.list_reach_entries_at_limited(2, UNIQUES_BUILD_CAP_SHALLOW)?, + &mut seen, + ); + let term = take( + self.list_reach_entries_at_limited(3, UNIQUES_BUILD_CAP_TERM)?, + &mut seen, + ); + + Ok(UniquesPools { fwd: vec![fwd0, fwd1, fwd2], term }) + } + + /// Count distinct reachable IDs at bounce 2 (our N2). pub fn count_distinct_n2(&self) -> anyhow::Result { - let count: i64 = self.conn.query_row( - "SELECT COUNT(DISTINCT reachable_node_id) FROM reachable_n2", - [], - |row| row.get(0), - )?; - Ok(count as usize) + self.count_distinct_at(2) } - /// Count distinct reachable NodeIds in the N3 table. + /// Count distinct reachable IDs at bounce 3 (our N3). pub fn count_distinct_n3(&self) -> anyhow::Result { - let count: i64 = self.conn.query_row( - "SELECT COUNT(DISTINCT reachable_node_id) FROM reachable_n3", - [], - |row| row.get(0), - )?; - Ok(count as usize) + self.count_distinct_at(3) } - /// List distinct reachable NodeIds in the N3 table. + /// Count distinct reachable IDs at bounce 4 (our N4 — the terminal layer). + pub fn count_distinct_n4(&self) -> anyhow::Result { + self.count_distinct_at(4) + } + + /// List distinct reachable IDs at bounce 3. pub fn list_distinct_n3(&self) -> anyhow::Result> { - let mut stmt = self.conn.prepare( - "SELECT DISTINCT reachable_node_id FROM reachable_n3", - )?; - let mut result = Vec::new(); - let mut rows = stmt.query([])?; - while let Some(row) = rows.next()? { - result.push(blob_to_nodeid(row.get(0)?)?); - } - Ok(result) + self.list_reach_ids_at(3) } - /// Diversity score: how many unique NodeIds does this reporter contribute + /// List distinct reachable IDs at bounce 4. + pub fn list_distinct_n4(&self) -> anyhow::Result> { + self.list_reach_ids_at(4) + } + + /// Diversity score: how many IDs does this reporter contribute at bounce 2 /// that no other reporter provides? pub fn count_unique_n2_for_reporter( &self, reporter: &NodeId, - exclude_reporters: &[NodeId], + _exclude_reporters: &[NodeId], ) -> anyhow::Result { - // Get this reporter's N2 set let reporter_set: std::collections::HashSet = - self.list_n2_for_reporter(reporter)?.into_iter().collect(); - + self.list_reach_for_reporter(reporter, 2)?.into_iter().collect(); if reporter_set.is_empty() { return Ok(0); } - - // Get all other reporters' N2 sets (excluding specified reporters) - let exclude_set: std::collections::HashSet = - exclude_reporters.iter().copied().collect(); let mut other_nodes = std::collections::HashSet::new(); - let mut stmt = self.conn.prepare( - "SELECT reachable_node_id FROM reachable_n2 WHERE reporter_node_id != ?1", + "SELECT reachable_id FROM reachable WHERE reporter_node_id != ?1 AND bounce = 2", )?; let mut rows = stmt.query(params![reporter.as_slice()])?; while let Some(row) = rows.next()? { - let rn: Vec = row.get(0)?; - // Check if the reporter of this entry is excluded - // (simplified: we just exclude the reporter itself) - if let Ok(nid) = blob_to_nodeid(rn) { + if let Ok(nid) = blob_to_nodeid(row.get(0)?) { other_nodes.insert(nid); } } - - let unique = reporter_set.difference(&other_nodes).count(); - let _ = exclude_set; // used for future filtering - Ok(unique) + Ok(reporter_set.difference(&other_nodes).count()) } - /// Remove stale N2/N3 entries. - /// Clear ALL N2/N3 entries (startup sweep after unclean shutdown). - pub fn clear_all_n2_n3(&self) -> anyhow::Result { - let d1 = self.conn.execute("DELETE FROM reachable_n2", [])?; - let d2 = self.conn.execute("DELETE FROM reachable_n3", [])?; - Ok(d1 + d2) + /// Clear the whole uniques index (startup sweep after unclean shutdown). + pub fn clear_all_reach(&self) -> anyhow::Result { + Ok(self.conn.execute("DELETE FROM reachable", [])?) } /// Clear ALL mesh_peers entries (no connections exist at startup). @@ -3643,39 +4084,56 @@ impl Storage { Ok(deleted) } - pub fn prune_n2_n3(&self, max_age_ms: u64) -> anyhow::Result { + /// Age out stale uniques rows — same discipline at every bounce, N4 included. + pub fn prune_reach(&self, max_age_ms: u64) -> anyhow::Result { let cutoff = now_ms() - max_age_ms as i64; - let d1 = self.conn.execute( - "DELETE FROM reachable_n2 WHERE updated_at < ?1", + Ok(self.conn.execute( + "DELETE FROM reachable WHERE updated_at < ?1", params![cutoff], - )?; - let d2 = self.conn.execute( - "DELETE FROM reachable_n3 WHERE updated_at < ?1", - params![cutoff], - )?; - Ok(d1 + d2) + )?) } - /// Score all N2 candidates for growth loop diversity selection. - /// Returns (node_id, reporter_count, in_n3) for each unique N2 candidate. - /// Lower reporter_count = more unique neighborhood = higher diversity value. - pub fn score_n2_candidates_batch(&self) -> anyhow::Result> { + /// How many candidates the SQL shortlist returns. Large enough that the + /// Rust-side exclusions (self, already-connected, likely-unreachable) can + /// never empty it in practice, small enough that the growth loop — which + /// uses exactly ONE candidate per pass and is woken on every announce that + /// stores rows — is not a full scan + in-memory sort of six figures of + /// rows. `idx_reachable_id(reachable_id, bounce)` supports the grouping. + pub const GROWTH_SHORTLIST: usize = 64; + + /// Score growth candidates for the growth loop's diversity selection. + /// + /// Returns (node_id, reporter_count, shallowest_bounce) over the whole + /// stored horizon, already ORDERED and LIMITED in SQL: shallowest first + /// (cheapest to resolve and to reach), then fewest reporters (most unique + /// neighborhood = highest diversity value). + /// + /// `shallowest_bounce` is returned GRADED rather than as a boolean: a + /// bounce-4 ID seen by one reporter must not outrank a genuine bounce-2 ID + /// seen by three, which is exactly what `1.0/reporter_count + 0.3` did when + /// the horizon widened past N2. + /// + /// Author-class IDs are EXCLUDED: a persona ID has no address and can never + /// answer a connect, so scoring it would burn hole-punch timeouts and mark + /// phantom peers unreachable. + pub fn score_n2_candidates_batch(&self) -> anyhow::Result> { let mut stmt = self.conn.prepare( - "SELECT n2.reachable_node_id, - COUNT(DISTINCT n2.reporter_node_id) as reporter_count, - CASE WHEN n3.reachable_node_id IS NOT NULL THEN 1 ELSE 0 END as in_n3 - FROM reachable_n2 n2 - LEFT JOIN (SELECT DISTINCT reachable_node_id FROM reachable_n3) n3 - ON n2.reachable_node_id = n3.reachable_node_id - GROUP BY n2.reachable_node_id", + "SELECT reachable_id, + COUNT(DISTINCT reporter_node_id) AS reporter_count, + MIN(bounce) AS shallowest + FROM reachable + WHERE id_class = 0 + GROUP BY reachable_id + ORDER BY shallowest ASC, reporter_count ASC + LIMIT ?1", )?; let mut results = Vec::new(); - let mut rows = stmt.query([])?; + let mut rows = stmt.query(params![Self::GROWTH_SHORTLIST as i64])?; while let Some(row) = rows.next()? { let nid = blob_to_nodeid(row.get(0)?)?; let reporter_count: usize = row.get::<_, i64>(1)? as usize; - let in_n3: bool = row.get::<_, i64>(2)? != 0; - results.push((nid, reporter_count, in_n3)); + let shallowest: i64 = row.get(2)?; + results.push((nid, reporter_count, shallowest.clamp(0, 255) as u8)); } Ok(results) } @@ -3692,24 +4150,20 @@ impl Storage { // ---- Mesh Peers ---- - /// Add a mesh peer connection record. - pub fn add_mesh_peer( - &self, - node_id: &NodeId, - slot_kind: PeerSlotKind, - priority: i32, - ) -> anyhow::Result<()> { + /// Record a MESH-slot peer. + /// + /// Temporary referral slots must NEVER be written here. `build_own_uniques` + /// reads this table to construct our announce, so simply not writing the row + /// is what keeps temp peers out of the uniques exchange — at the store + /// layer, with no filtering anyone can forget. (`get_lateral_blob_sources` + /// also treats membership here as "reachable", a second reason.) + pub fn add_mesh_peer(&self, node_id: &NodeId) -> anyhow::Result<()> { let now = now_ms(); self.conn.execute( - "INSERT INTO mesh_peers (node_id, slot_kind, priority, connected_at, last_diff_seq) - VALUES (?1, ?2, ?3, ?4, 0) - ON CONFLICT(node_id) DO UPDATE SET slot_kind = ?2, priority = ?3, connected_at = ?4", - params![ - node_id.as_slice(), - slot_kind.to_string(), - priority, - now, - ], + "INSERT INTO mesh_peers (node_id, connected_at, last_diff_seq) + VALUES (?1, ?2, 0) + ON CONFLICT(node_id) DO UPDATE SET connected_at = ?2", + params![node_id.as_slice(), now], )?; Ok(()) } @@ -3723,32 +4177,19 @@ impl Storage { Ok(()) } - /// List all mesh peers: (node_id, slot_kind_str, priority). - pub fn list_mesh_peers(&self) -> anyhow::Result> { + /// List all mesh-slot peers, most recently connected first. + pub fn list_mesh_peers(&self) -> anyhow::Result> { let mut stmt = self.conn.prepare( - "SELECT node_id, slot_kind, priority FROM mesh_peers ORDER BY connected_at DESC", + "SELECT node_id FROM mesh_peers ORDER BY connected_at DESC", )?; let mut result = Vec::new(); let mut rows = stmt.query([])?; while let Some(row) = rows.next()? { - let node_id = blob_to_nodeid(row.get(0)?)?; - let slot_kind: String = row.get(1)?; - let priority: i32 = row.get(2)?; - result.push((node_id, slot_kind, priority)); + result.push(blob_to_nodeid(row.get(0)?)?); } Ok(result) } - /// Count mesh peers of a given slot kind. - pub fn count_mesh_peers_by_kind(&self, slot_kind: PeerSlotKind) -> anyhow::Result { - let count: i64 = self.conn.query_row( - "SELECT COUNT(*) FROM mesh_peers WHERE slot_kind = ?1", - params![slot_kind.to_string()], - |row| row.get(0), - )?; - Ok(count as usize) - } - /// Update last_diff_seq for a mesh peer. pub fn update_mesh_peer_seq(&self, node_id: &NodeId, seq: u64) -> anyhow::Result<()> { self.conn.execute( @@ -3950,14 +4391,31 @@ impl Storage { } /// Build peer_addresses for a contact from their profile's recent_peers. + /// + /// PUBLICLY ROUTABLE ONLY. design.html §21 sanctions this payload class on + /// the reasoning that these are "network endpoints an observer at that + /// vantage could already see us talking to" — that holds for public + /// addresses and not for a peer's LAN address (192.168.x, 10.x picked up + /// from an earlier exchange), which discloses local topology to someone who + /// could not otherwise observe it. A peer left with no routable address is + /// dropped rather than sent as a bare, useless entry. pub fn build_peer_addresses_for(&self, node_id: &NodeId) -> anyhow::Result> { let recent_peers = self.get_recent_peers(node_id)?; let mut result = Vec::new(); for rp in recent_peers.iter().take(10) { let addrs: Vec = self .get_peer_record(rp)? - .map(|r| r.addresses.iter().map(|a| a.to_string()).collect()) + .map(|r| { + r.addresses + .iter() + .filter(|a| crate::connection::convection_addr_ok(a)) + .map(|a| a.to_string()) + .collect() + }) .unwrap_or_default(); + if addrs.is_empty() { + continue; + } result.push(PeerWithAddress { n: hex::encode(rp), a: addrs, @@ -5766,6 +6224,25 @@ impl Storage { Ok(()) } + /// Store a comment WE authored. Identical to `store_comment` except it + /// marks the row `origin_local = 1`, which keeps its author ID (our + /// persona, or a per-greeting throwaway) out of `build_own_uniques` — see + /// design.html §21: we must never be the node that announces our own + /// posting identities at bounce 1 beside our node ID and address. + pub fn store_own_comment(&self, comment: &InlineComment) -> anyhow::Result<()> { + self.store_comment(comment)?; + self.conn.execute( + "UPDATE comments SET origin_local = 1 + WHERE author = ?1 AND post_id = ?2 AND timestamp_ms = ?3", + params![ + comment.author.as_slice(), + comment.post_id.as_slice(), + comment.timestamp_ms as i64, + ], + )?; + Ok(()) + } + /// Edit a comment (must match author + post_id + timestamp_ms). pub fn edit_comment(&self, author: &NodeId, post_id: &PostId, timestamp_ms: u64, new_content: &str) -> anyhow::Result { let updated = self.conn.execute( @@ -6544,8 +7021,12 @@ mod tests { assert!(s.is_relay_cooldown(&target, 1).unwrap()); } + fn nodes(ids: &[NodeId]) -> Vec { + ids.iter().map(|id| ReachEntry::node(*id)).collect() + } + #[test] - fn n2_n3_crud() { + fn reach_crud() { let s = temp_storage(); let reporter_a = make_node_id(1); let reporter_b = make_node_id(2); @@ -6553,68 +7034,285 @@ mod tests { let node_y = make_node_id(11); let node_z = make_node_id(12); - // Set reporter_a's N1 (their connections) → our N2 - s.set_peer_n1(&reporter_a, &[node_x, node_y]).unwrap(); - let found = s.find_in_n2(&node_x).unwrap(); - assert_eq!(found, vec![reporter_a]); + s.set_reach(&reporter_a, 2, &nodes(&[node_x, node_y])).unwrap(); + assert_eq!(s.find_in_n2(&node_x).unwrap(), vec![reporter_a]); - // Set reporter_b's N1 → our N2 - s.set_peer_n1(&reporter_b, &[node_y, node_z]).unwrap(); - let found = s.find_in_n2(&node_y).unwrap(); - assert_eq!(found.len(), 2); // Both reporters have node_y + s.set_reach(&reporter_b, 2, &nodes(&[node_y, node_z])).unwrap(); + assert_eq!(s.find_in_n2(&node_y).unwrap().len(), 2); - // Build N2 share (deduplicated) - let n2_share = s.build_n2_share().unwrap(); - assert_eq!(n2_share.len(), 3); // node_x, node_y, node_z + assert_eq!(s.list_reach_ids_at(2).unwrap().len(), 3); - // Clear reporter_a's N2 contributions - let cleared = s.clear_peer_n2(&reporter_a).unwrap(); - assert_eq!(cleared, 2); - let found = s.find_in_n2(&node_x).unwrap(); - assert!(found.is_empty()); + // Disconnect cleanup wipes every bounce for that reporter. + s.set_reach(&reporter_a, 4, &nodes(&[node_z])).unwrap(); + let cleared = s.clear_peer_reach(&reporter_a).unwrap(); + assert_eq!(cleared, 3); + assert!(s.find_in_n2(&node_x).unwrap().is_empty()); + assert!(s.find_in_n4(&node_z).unwrap().is_empty()); - // N3 operations - s.set_peer_n2(&reporter_a, &[node_z]).unwrap(); - let found = s.find_in_n3(&node_z).unwrap(); - assert_eq!(found, vec![reporter_a]); - - s.clear_peer_n3(&reporter_a).unwrap(); - let found = s.find_in_n3(&node_z).unwrap(); - assert!(found.is_empty()); + // N3 + N4 land at their own bounces, reporter-tagged. + s.set_reach(&reporter_a, 3, &nodes(&[node_z])).unwrap(); + assert_eq!(s.find_in_n3(&node_z).unwrap(), vec![reporter_a]); + s.set_reach(&reporter_b, 4, &nodes(&[node_x])).unwrap(); + assert_eq!(s.find_in_n4(&node_x).unwrap(), vec![reporter_b]); + assert_eq!(s.count_distinct_n4().unwrap(), 1); } + /// Store-side dedup: one reporter cannot place one ID at two depths — the + /// shallowest sighting wins. #[test] - fn n1_share_build() { + fn reach_store_dedup_keeps_shallowest_bounce() { let s = temp_storage(); - let peer_a = make_node_id(1); - let follow_b = make_node_id(2); - let addr: std::net::SocketAddr = "10.0.0.1:4433".parse().unwrap(); + let reporter = make_node_id(1); + let target = make_node_id(10); - // Add a mesh peer - s.add_mesh_peer(&peer_a, PeerSlotKind::Local, 0).unwrap(); - // Add a follow with social route - s.add_follow(&follow_b).unwrap(); + s.set_reach(&reporter, 4, &nodes(&[target])).unwrap(); + assert_eq!(s.find_in_n4(&target).unwrap(), vec![reporter]); + + // Same reporter now reports it shallower. + s.add_reach(&reporter, 2, &nodes(&[target])).unwrap(); + assert_eq!(s.find_in_n2(&target).unwrap(), vec![reporter]); + assert!(s.find_in_n4(&target).unwrap().is_empty(), "must not occupy two depths"); + + // A deeper re-report does not push it back down. + s.add_reach(&reporter, 3, &nodes(&[target])).unwrap(); + assert_eq!(s.find_in_n2(&target).unwrap(), vec![reporter]); + } + + /// §layers privacy invariant: addresses exist ONLY on anchor entries. + #[test] + fn reach_addresses_only_on_anchors() { + let s = temp_storage(); + let reporter = make_node_id(1); + let plain = make_node_id(10); + let anchor = make_node_id(11); + + // A non-anchor entry that arrives carrying addresses must be blanked. + let sneaky = ReachEntry { + id: plain, + id_class: IdClass::Node, + is_anchor: false, + addresses: vec!["1.2.3.4:4433".to_string()], + }; + s.set_reach( + &reporter, + 2, + &[sneaky, ReachEntry::anchor(anchor, vec!["5.6.7.8:4433".to_string()])], + ) + .unwrap(); + + let leaked: i64 = s + .conn + .query_row( + "SELECT COUNT(*) FROM reachable WHERE is_anchor = 0 AND addresses != '[]'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(leaked, 0, "non-anchor entries must never carry an address"); + + let mined = s.list_pool_anchors(10).unwrap(); + assert_eq!(mined.len(), 1); + assert_eq!(mined[0].0, anchor); + assert_eq!(mined[0].1, vec!["5.6.7.8:4433".to_string()]); + + // Anchor density is a free local statistic (2 distinct node IDs, 1 anchor). + assert!((s.anchor_density().unwrap() - 0.5).abs() < 1e-9); + } + + /// Author-class IDs must never be offered as connect targets. + #[test] + fn author_class_excluded_from_growth_candidates() { + let s = temp_storage(); + let reporter = make_node_id(1); + let node = make_node_id(10); + let persona = make_node_id(11); + + s.set_reach(&reporter, 2, &[ReachEntry::node(node), ReachEntry::author(persona)]) + .unwrap(); + + let scored = s.score_n2_candidates_batch().unwrap(); + let ids: Vec = scored.iter().map(|(id, _, _)| *id).collect(); + assert!(ids.contains(&node)); + assert!(!ids.contains(&persona), "persona IDs are not connectable"); + + // ...but they ARE findable, which is the whole point of carrying them. + assert!(s.list_reach_ids_at(2).unwrap().contains(&persona)); + } + + /// The announce builder must never read bounce-4 rows: N4 is terminal. + #[test] + fn n4_never_appears_in_a_built_announce() { + let s = temp_storage(); + let us = make_node_id(0); + let reporter = make_node_id(1); + let n2_id = make_node_id(10); + let n3_id = make_node_id(11); + let n4_id = make_node_id(12); + + s.set_reach(&reporter, 2, &nodes(&[n2_id])).unwrap(); + s.set_reach(&reporter, 3, &nodes(&[n3_id])).unwrap(); + s.set_reach(&reporter, 4, &nodes(&[n4_id])).unwrap(); + + let pools = s.build_uniques_pools(&us, None).unwrap(); + let all: Vec = pools + .fwd + .iter() + .flatten() + .chain(pools.term.iter()) + .map(|e| e.id) + .collect(); + + assert!(all.contains(&n2_id), "our N2 rides pool 1"); + assert!(all.contains(&n3_id), "our N3 rides pool 2 (terminal)"); + assert!(!all.contains(&n4_id), "N4 must NEVER be re-announced"); + + // And it is in the right pool. + assert!(pools.fwd[2].iter().any(|e| e.id == n2_id)); + assert!(pools.term.iter().any(|e| e.id == n3_id)); + } + + /// Announce-side dedup: an ID announced shallow is not repeated deeper. + #[test] + fn announce_dedup_across_pool_slices() { + let s = temp_storage(); + let us = make_node_id(0); + let reporter = make_node_id(1); + let dup = make_node_id(10); + + // Same ID known at bounce 2 AND bounce 3 (via different reporters). + s.set_reach(&reporter, 2, &nodes(&[dup])).unwrap(); + s.set_reach(&make_node_id(2), 3, &nodes(&[dup])).unwrap(); + + let pools = s.build_uniques_pools(&us, None).unwrap(); + let count = pools + .fwd + .iter() + .flatten() + .chain(pools.term.iter()) + .filter(|e| e.id == dup) + .count(); + assert_eq!(count, 1, "an ID must appear in exactly one pool slice"); + assert!(pools.fwd[2].iter().any(|e| e.id == dup), "shallowest slice wins"); + + // Our own node id is never repeated deeper either. + let self_count = pools + .fwd + .iter() + .flatten() + .chain(pools.term.iter()) + .filter(|e| e.id == us) + .count(); + assert_eq!(self_count, 1); + } + + /// Our own uniques merge every source, and anchor addresses only attach to + /// peers actually flagged as anchors. + #[test] + fn own_uniques_merge_sources_and_flag_anchors() { + let s = temp_storage(); + let mesh = make_node_id(1); + let direct = make_node_id(2); + let anchor = make_node_id(3); + let addr: std::net::SocketAddr = "203.0.113.7:4433".parse().unwrap(); + + s.add_mesh_peer(&mesh).unwrap(); s.upsert_social_route(&SocialRouteEntry { - node_id: follow_b, - addresses: vec![addr], + node_id: direct, + addresses: vec![], peer_addresses: vec![], relation: SocialRelation::Follow, + // Deliberately Disconnected: the ruling is "everyone's directs". status: SocialStatus::Disconnected, last_connected_ms: 0, last_seen_ms: 1000, reach_method: ReachMethod::Direct, - }).unwrap(); + }) + .unwrap(); + // An anchor we have a LIVE PATH to (here: a social direct) carries its + // address. Anchor flagging ENRICHES a unique; it never imports one. + s.upsert_peer(&direct, &[addr], None).unwrap(); + s.set_peer_anchor(&direct, true).unwrap(); - // Disconnected routes should NOT be in N1 share - let n1 = s.build_n1_share().unwrap(); - assert!(n1.contains(&peer_a)); - assert!(!n1.contains(&follow_b), "Disconnected social route should not be in N1"); + // An anchor-flagged peers row with no live path — e.g. one we merely + // heard about — must NOT become a bounce-1 unique of ours. + s.upsert_peer(&anchor, &[addr], None).unwrap(); + s.set_peer_anchor(&anchor, true).unwrap(); - // Set to Online — now it should be included - s.set_social_route_status(&follow_b, SocialStatus::Online).unwrap(); - let n1 = s.build_n1_share().unwrap(); - assert!(n1.contains(&peer_a)); - assert!(n1.contains(&follow_b), "Online social route should be in N1"); + let own = s.build_own_uniques().unwrap(); + let ids: Vec = own.iter().map(|e| e.id).collect(); + assert!(ids.contains(&mesh)); + assert!(ids.contains(&direct), "held routes count even when offline"); + assert!( + !ids.contains(&anchor), + "an anchor we merely have a row for is not a live path of ours" + ); + + let anchor_entry = own.iter().find(|e| e.id == direct).unwrap(); + assert!(anchor_entry.is_anchor); + assert_eq!(anchor_entry.addresses, vec![addr.to_string()]); + + let mesh_entry = own.iter().find(|e| e.id == mesh).unwrap(); + assert!(!mesh_entry.is_anchor); + assert!(mesh_entry.addresses.is_empty(), "non-anchors are bare IDs"); + } + + /// design.html §21: "Posting identities never map to device addresses on + /// the wire." Our node ID (and our anchor address) ride `fwd[0]` of the + /// same announce, so a persona of ours in `fwd[1]` IS that mapping. + #[test] + fn own_personas_and_local_comment_authors_never_ride_our_announce() { + let s = temp_storage(); + let us = make_node_id(0); + // Plural personas per the A1 rule — not just the default posting id. + let persona = add_persona(&s, 66); + let persona2 = add_persona(&s, 67); + let throwaway = make_node_id(77); + let foreign_author = make_node_id(88); + + // CDN manifests we authored (node.rs stores these with OUR posting id). + s.store_cdn_manifest(&make_post_id(1), "{}", &persona, 1).unwrap(); + s.store_cdn_manifest(&make_post_id(2), "{}", &persona2, 1).unwrap(); + + let mk = |author: NodeId, ts: u64| crate::types::InlineComment { + author, + post_id: [9u8; 32], + content: "hi".into(), + timestamp_ms: ts, + signature: vec![1, 2, 3], + ref_post_id: None, + deleted_at: None, + pub_x_index: None, + group_sig: None, + encrypted_payload: None, + expires_at_ms: 0, + }; + // Our greeting: a per-greeting throwaway ID we minted. We would be the + // FIRST node in the network to announce it, at bounce 1, the instant it + // was created — first-announcer identifies the greeter. + s.store_own_comment(&mk(throwaway, 100)).unwrap(); + // Someone else's comment reached us from the network — that author is + // legitimate uniques material (anonymity noise, round-1 ruling). + s.store_comment(&mk(foreign_author, 200)).unwrap(); + + let own = s.build_own_uniques().unwrap(); + let ids: Vec = own.iter().map(|e| e.id).collect(); + assert!(!ids.contains(&persona), "our own persona must never be in our N1"); + assert!(!ids.contains(&persona2), "every persona, not just the default"); + assert!(!ids.contains(&throwaway), "our greeting throwaway must never be in our N1"); + assert!(ids.contains(&foreign_author), "other people's authors still count"); + + // ...and it holds through the pool builder, which is what goes on the wire. + let pools = s.build_uniques_pools(&us, None).unwrap(); + let all: Vec = pools + .fwd + .iter() + .flatten() + .chain(pools.term.iter()) + .map(|e| e.id) + .collect(); + assert!(!all.contains(&persona)); + assert!(!all.contains(&persona2)); + assert!(!all.contains(&throwaway)); } #[test] @@ -6626,9 +7324,9 @@ mod tests { let shared_node = make_node_id(11); // reporter_a has unique_node + shared_node - s.set_peer_n1(&reporter_a, &[unique_node, shared_node]).unwrap(); + s.set_reach(&reporter_a, 2, &nodes(&[unique_node, shared_node])).unwrap(); // reporter_b only has shared_node - s.set_peer_n1(&reporter_b, &[shared_node]).unwrap(); + s.set_reach(&reporter_b, 2, &nodes(&[shared_node])).unwrap(); // reporter_a contributes 1 unique node (unique_node) let unique = s.count_unique_n2_for_reporter(&reporter_a, &[]).unwrap(); @@ -6639,41 +7337,40 @@ mod tests { assert_eq!(unique, 0); } + /// N4 is USED for search even though it is never re-announced. #[test] - fn find_any_in_n2_n3() { + fn find_any_reachable_spans_the_whole_horizon() { let s = temp_storage(); let reporter = make_node_id(1); let node_n2 = make_node_id(10); let node_n3 = make_node_id(11); - let node_nowhere = make_node_id(12); + let node_n4 = make_node_id(12); + let node_nowhere = make_node_id(13); - s.set_peer_n1(&reporter, &[node_n2]).unwrap(); - s.set_peer_n2(&reporter, &[node_n3]).unwrap(); + s.set_reach(&reporter, 2, &nodes(&[node_n2])).unwrap(); + s.set_reach(&make_node_id(2), 3, &nodes(&[node_n3])).unwrap(); + s.set_reach(&make_node_id(3), 4, &nodes(&[node_n4])).unwrap(); - let results = s.find_any_in_n2_n3(&[node_n2, node_n3, node_nowhere]).unwrap(); - assert_eq!(results.len(), 2); - assert_eq!(results[0].2, 2); // N2 first - assert_eq!(results[1].2, 3); // N3 second + let results = s + .find_any_reachable(&[node_n2, node_n3, node_n4, node_nowhere]) + .unwrap(); + assert_eq!(results.len(), 3); + assert_eq!(results[0].2, 2); + assert_eq!(results[1].2, 3); + assert_eq!(results[2].2, 4); } #[test] fn mesh_peers_crud() { - use crate::types::PeerSlotKind; let s = temp_storage(); let nid = make_node_id(1); - s.add_mesh_peer(&nid, PeerSlotKind::Local, 4).unwrap(); + s.add_mesh_peer(&nid).unwrap(); let peers = s.list_mesh_peers().unwrap(); - assert_eq!(peers.len(), 1); - assert_eq!(peers[0].0, nid); - assert_eq!(peers[0].1, "local"); - assert_eq!(peers[0].2, 4); - - assert_eq!(s.count_mesh_peers_by_kind(PeerSlotKind::Local).unwrap(), 1); - assert_eq!(s.count_mesh_peers_by_kind(PeerSlotKind::Wide).unwrap(), 0); + assert_eq!(peers, vec![nid]); s.remove_mesh_peer(&nid).unwrap(); - assert_eq!(s.count_mesh_peers_by_kind(PeerSlotKind::Local).unwrap(), 0); + assert!(s.list_mesh_peers().unwrap().is_empty()); } // ---- Social routes tests ---- @@ -7099,16 +7796,19 @@ mod tests { #[test] fn slot_counts() { - assert_eq!(DeviceProfile::Desktop.local_slots(), 71); - assert_eq!(DeviceProfile::Mobile.local_slots(), 7); - assert_eq!( - DeviceProfile::Desktop.local_slots() + DeviceProfile::Desktop.wide_slots(), - 91 - ); - assert_eq!( - DeviceProfile::Mobile.local_slots() + DeviceProfile::Mobile.wide_slots(), - 12 - ); + // v0.8: ONE mesh pool. Local/Wide collapsed. + assert_eq!(DeviceProfile::Desktop.mesh_slots(), 20); + assert_eq!(DeviceProfile::Mobile.mesh_slots(), 15); + + // Temp referral slots sit ABOVE the mesh cap, +4..+10 (round-3 ruling). + for p in [DeviceProfile::Desktop, DeviceProfile::Mobile] { + let t = p.temp_referral_slots(); + assert!((4..=10).contains(&t), "temp referral band out of range: {t}"); + } + + // Device-tiered depth: mobile caps the stored horizon at 3 bounces. + assert_eq!(DeviceProfile::Desktop.max_bounce(), 4); + assert_eq!(DeviceProfile::Mobile.max_bounce(), 3); } // ---- Circle Profile tests ---- @@ -7264,29 +7964,33 @@ mod tests { s.upsert_known_anchor(&a2, &[addr]).unwrap(); s.upsert_known_anchor(&a3, &[addr]).unwrap(); - // a1 gets extra success bumps - s.upsert_known_anchor(&a1, &[addr]).unwrap(); + // Re-contacting a1 refreshes it. Ordering is by RECENCY, not by + // success_count: under pool-mined anchor abundance "the anchor that + // answered most often" is not a better anchor, just the one we + // hammered (round-4 ruling). + std::thread::sleep(std::time::Duration::from_millis(2)); s.upsert_known_anchor(&a1, &[addr]).unwrap(); let anchors = s.list_known_anchors().unwrap(); assert_eq!(anchors.len(), 3); - // a1 should be first (highest success_count = 3) - assert_eq!(anchors[0].0, a1); + assert_eq!(anchors[0].0, a1, "freshest anchor first"); } #[test] - fn known_anchors_prune() { + fn known_anchors_prune_to_the_bootstrap_cache_cap() { let s = temp_storage(); let addr: SocketAddr = "10.0.0.1:4433".parse().unwrap(); - for i in 0..10u8 { + // The old cap was 5. `apply_uniques_to_storage` now mirrors every + // received anchor entry into this table, so at 5 it would thrash on + // every announce. + for i in 0..(Storage::KNOWN_ANCHOR_CACHE_CAP as u8 + 8) { let nid = make_node_id(i + 1); s.upsert_known_anchor(&nid, &[addr]).unwrap(); } - // Auto-prune should keep only 5 let anchors = s.list_known_anchors().unwrap(); - assert_eq!(anchors.len(), 5); + assert_eq!(anchors.len(), Storage::KNOWN_ANCHOR_CACHE_CAP); } #[test] diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index 3518413..faba589 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -649,26 +649,67 @@ impl NatProfile { } /// Device profile — determines connection slot budget +/// +/// v0.8 (Iteration C): the Local/Wide split is gone. There is ONE mesh pool +/// (`mesh_slots`) plus a strictly-additional band of temporary referral slots +/// (`temp_referral_slots`) that live ABOVE the mesh cap and carry no knowledge +/// exchange. See design.html §connections. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DeviceProfile { - /// Desktop: 71 local + 20 wide = 91 mesh connections + /// Desktop: 20 mesh + up to 10 temp referral Desktop, - /// Mobile: 7 local + 5 wide = 12 mesh connections + /// Mobile: 15 mesh + up to 4 temp referral Mobile, } impl DeviceProfile { - pub fn local_slots(&self) -> usize { + /// The single mesh pool size (v0.8 narrow mesh, deep knowledge). + /// + /// `ITSGOIN_TEST_MESH_SLOTS` overrides it so integration tests can reach + /// the temp-referral boundary with a handful of nodes instead of 21. Test + /// gate only — same pattern as `ITSGOIN_TEST_ALLOW_REGISTRY_GENESIS`. + /// + /// Read ONCE into a `OnceLock`: this is called from `ConnectionManager` + /// construction and from every path that wants a fresh cap, and an env + /// lookup per call is a syscall-shaped cost on a hot path (and lets the + /// mesh cap change under a running node, which nothing expects). + pub fn mesh_slots(&self) -> usize { + static OVERRIDE: std::sync::OnceLock> = std::sync::OnceLock::new(); + let over = *OVERRIDE.get_or_init(|| { + std::env::var("ITSGOIN_TEST_MESH_SLOTS") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|n| *n > 0) + }); + if let Some(n) = over { + return n; + } match self { - DeviceProfile::Desktop => 71, - DeviceProfile::Mobile => 7, + DeviceProfile::Desktop => 20, + DeviceProfile::Mobile => 15, } } - pub fn wide_slots(&self) -> usize { + /// Temporary referral slots, allocated ABOVE `mesh_slots`. These exist only + /// to facilitate introductions: they carry no uniques exchange, are never + /// persisted to `mesh_peers`, and either graduate into a freed mesh slot or + /// expire. Ruling: +4..+10 above the cap. + pub fn temp_referral_slots(&self) -> usize { match self { - DeviceProfile::Desktop => 20, - DeviceProfile::Mobile => 5, + DeviceProfile::Desktop => 10, + DeviceProfile::Mobile => 4, + } + } + + /// Deepest bounce this device stores in the `reachable` index. + /// Desktop keeps the full 4-bounce horizon; mobile caps at 3 (it drops the + /// terminal pool on receipt and advertises `depth = 3` so senders skip + /// building it). Alternative considered and NOT implemented: Bloom-compress + /// N4 on mobile (see `apply_uniques_announce` doc comment). + pub fn max_bounce(&self) -> u8 { + match self { + DeviceProfile::Desktop => 4, + DeviceProfile::Mobile => 3, } } @@ -708,24 +749,103 @@ impl std::fmt::Display for SessionReachMethod { } } -/// Slot kind for mesh connections -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum PeerSlotKind { - /// Diverse local connections (Desktop: 71, Mobile: 7) - Local, - /// Bloom-sourced random distant connections (Desktop: 20, Mobile: 5) - Wide, +/// Which class of slot a live connection occupies. +/// +/// v0.8: `Mesh` is the real pool (capped at `DeviceProfile::mesh_slots`) and is +/// the ONLY class that participates in the uniques announce. `TempReferral` +/// slots sit above the cap, exist purely to broker introductions, and are never +/// written to `mesh_peers` — which is what structurally keeps them out of our +/// own announcements. They graduate into a freed mesh slot or expire. +/// +/// NOT on the wire — purely local accounting. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MeshSlot { + Mesh, + TempReferral { expires_at_ms: u64 }, } -impl std::fmt::Display for PeerSlotKind { +impl MeshSlot { + pub fn is_mesh(&self) -> bool { + matches!(self, MeshSlot::Mesh) + } + pub fn is_temp(&self) -> bool { + !self.is_mesh() + } +} + +impl std::fmt::Display for MeshSlot { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - PeerSlotKind::Local => write!(f, "local"), - PeerSlotKind::Wide => write!(f, "wide"), + MeshSlot::Mesh => write!(f, "mesh"), + MeshSlot::TempReferral { .. } => write!(f, "temp"), } } } +/// Class of an ID carried in a uniques pool. +/// +/// A `Node` ID is a network identity we may connect to / hole-punch / relay to. +/// An `Author` ID is a posting identity (persona, throwaway, greeting ID). It +/// has NO address and NO device linkage (Iteration A stripped +/// `AuthorManifest.author_addresses`), so it must NEVER enter the address +/// resolution cascade or the growth-candidate scorer — it resolves only through +/// CDN holder search. The pools keep the two classes in separate arrays so a +/// receiver structurally cannot confuse them. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IdClass { + Node = 0, + Author = 1, +} + +impl IdClass { + pub fn as_i64(self) -> i64 { + self as i64 + } + pub fn from_i64(v: i64) -> Self { + if v == 1 { IdClass::Author } else { IdClass::Node } + } +} + +/// One entry in the uniques index / announce pools. +/// +/// `addresses` is non-empty ONLY when `is_anchor` — that is the §layers privacy +/// invariant, enforced at the store (`add_reach` blanks the column otherwise) +/// and at the wire builder (only `AnchorEntry` has an address field at all). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReachEntry { + pub id: NodeId, + pub id_class: IdClass, + pub is_anchor: bool, + pub addresses: Vec, +} + +impl ReachEntry { + /// A bare node-class entry (no address). + pub fn node(id: NodeId) -> Self { + Self { id, id_class: IdClass::Node, is_anchor: false, addresses: Vec::new() } + } + /// A bare author-class entry (no address, never a connect target). + pub fn author(id: NodeId) -> Self { + Self { id, id_class: IdClass::Author, is_anchor: false, addresses: Vec::new() } + } + /// An anchor entry — the only kind that carries an address. + pub fn anchor(id: NodeId, addresses: Vec) -> Self { + Self { id, id_class: IdClass::Node, is_anchor: true, addresses } + } +} + +/// The two announce pools, pre-wire. +/// +/// `fwd` is always length 3 (our N0, N1, N2) and is FORWARDABLE — the receiver +/// stores `fwd[i]` at bounce `i + 1` and may re-announce it shifted. +/// `term` is our N3 and is TERMINAL — the receiver stores it at bounce 4, uses +/// it, and never re-announces it. +#[derive(Debug, Clone, Default)] +pub struct UniquesPools { + pub fwd: Vec>, + pub term: Vec, +} + // --- Social Routing Cache --- /// How we last reached a social contact diff --git a/crates/core/src/upnp.rs b/crates/core/src/upnp.rs index 643a985..379647b 100644 --- a/crates/core/src/upnp.rs +++ b/crates/core/src/upnp.rs @@ -69,7 +69,7 @@ use std::net::SocketAddr; use std::num::NonZeroU16; use std::time::Duration; -use tracing::{debug, info, warn}; +use tracing::{debug, info}; /// An active port mapping. While this value is held, the underlying /// `portmapper::Client` keeps the lease alive in a background task. diff --git a/crates/tauri-app/src/lib.rs b/crates/tauri-app/src/lib.rs index 85991a9..a6440b5 100644 --- a/crates/tauri-app/src/lib.rs +++ b/crates/tauri-app/src/lib.rs @@ -6,7 +6,7 @@ use tracing::info; use itsgoin_core::identity::IdentityManager; use itsgoin_core::node::Node; -use itsgoin_core::types::{NodeId, PeerSlotKind, Post, PostId, PostVisibility, VisibilityIntent}; +use itsgoin_core::types::{NodeId, Post, PostId, PostVisibility, VisibilityIntent}; /// The active Node, swappable on identity switch. type AppNode = Arc>>; @@ -1363,7 +1363,7 @@ async fn list_peers(state: State<'_, AppNode>) -> Result, String> { .into_iter() .map(|(nid, _, _)| nid) .collect(); - let (social_ids, n2_ids, n3_ids) = { + let (social_ids, n2_ids, n3_ids, n4_ids) = { let storage = node.storage.get().await; let social: std::collections::HashSet<_> = storage .list_social_routes() @@ -1372,7 +1372,7 @@ async fn list_peers(state: State<'_, AppNode>) -> Result, String> { .map(|r| r.node_id) .collect(); let n2: std::collections::HashSet<_> = storage - .build_n2_share() + .list_reach_ids_at(2) .unwrap_or_default() .into_iter() .collect(); @@ -1381,7 +1381,12 @@ async fn list_peers(state: State<'_, AppNode>) -> Result, String> { .unwrap_or_default() .into_iter() .collect(); - (social, n2, n3) + let n4: std::collections::HashSet<_> = storage + .list_distinct_n4() + .unwrap_or_default() + .into_iter() + .collect(); + (social, n2, n3, n4) }; let mut dtos = Vec::with_capacity(records.len()); @@ -1406,6 +1411,8 @@ async fn list_peers(state: State<'_, AppNode>) -> Result, String> { "n2" } else if n3_ids.contains(&rec.node_id) { "n3" + } else if n4_ids.contains(&rec.node_id) { + "n4" } else { "known" }; @@ -2066,12 +2073,12 @@ async fn list_connections(state: State<'_, AppNode>) -> Result, node_id_hex: String) -> Resul #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct NetworkSummaryDto { - local_count: usize, - wide_count: usize, + mesh_count: usize, + temp_count: usize, + mesh_cap: usize, total_connections: usize, n2_distinct: usize, n3_distinct: usize, + n4_distinct: usize, has_public_v6: bool, has_public_v4: bool, has_upnp: bool, @@ -2507,26 +2516,24 @@ struct NetworkSummaryDto { async fn get_network_summary(state: State<'_, AppNode>) -> Result { let node = get_node(&state).await; let conns = node.list_connections().await; - let mut local = 0usize; - let mut wide = 0usize; - for (_nid, slot_kind, _at) in &conns { - match slot_kind { - PeerSlotKind::Local => local += 1, - PeerSlotKind::Wide => wide += 1, - } - } - let (n2, n3) = { + let mesh = conns.iter().filter(|(_, slot, _)| slot.is_mesh()).count(); + let temp = conns.len() - mesh; + let (n2, n3, n4) = { let storage = node.storage.get().await; - let n2 = storage.count_distinct_n2().unwrap_or(0); - let n3 = storage.count_distinct_n3().unwrap_or(0); - (n2, n3) + ( + storage.count_distinct_n2().unwrap_or(0), + storage.count_distinct_n3().unwrap_or(0), + storage.count_distinct_n4().unwrap_or(0), + ) }; Ok(NetworkSummaryDto { - local_count: local, - wide_count: wide, + mesh_count: mesh, + temp_count: temp, + mesh_cap: node.device_profile().mesh_slots(), total_connections: conns.len(), n2_distinct: n2, n3_distinct: n3, + n4_distinct: n4, has_public_v6: node.network.has_public_v6(), has_public_v4: node.network.is_anchor(), has_upnp: node.network.has_upnp(), @@ -2748,15 +2755,14 @@ struct ActivityLogDto { events: Vec, rebalance_last_ms: u64, rebalance_interval_secs: u64, - anchor_register_last_ms: u64, - anchor_register_interval_secs: u64, + convection_last_ms: u64, } #[tauri::command] async fn get_activity_log(state: State<'_, AppNode>) -> Result { let node = get_node(&state).await; let events = node.get_activity_log(200); - let (rebalance_last, anchor_last) = node.timer_state(); + let (rebalance_last, convection_last) = node.timer_state(); let dto_events: Vec = events.into_iter().map(|e| { ActivityEventDto { timestamp_ms: e.timestamp_ms, @@ -2770,8 +2776,7 @@ async fn get_activity_log(state: State<'_, AppNode>) -> Result) -> Result async fn request_referrals(state: State<'_, AppNode>) -> Result { let node = get_node(&state).await; let node_id = node.node_id; - // Try known_anchors table first (populated by anchor register cycle), - // fall back to anchor peers from the peers table (is_anchor = true) + let mesh = node.network.conn_handle().mesh_count().await; + // Manual trigger uses the same class rule as the automatic path: below 2 + // mesh peers this is an ENTRY request and is always served. + let class = itsgoin_core::protocol::ConvectionClass::for_mesh_count(mesh); + + // Pool-mined anchors first (the uniques pools ARE the anchor directory), + // then the known_anchors bootstrap cache, then anchor-flagged peers. let anchors: Vec<(NodeId, Vec)> = { let storage = node.storage.get().await; - let known = storage.list_known_anchors().unwrap_or_default(); - if !known.is_empty() { - known - } else { - storage.list_anchor_peers().unwrap_or_default() - .into_iter() - .map(|r| (r.node_id, r.addresses)) - .collect() + let mut out: Vec<(NodeId, Vec)> = Vec::new(); + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + for (nid, addrs) in storage.list_pool_anchors(16).unwrap_or_default() { + let socks: Vec = addrs.iter().filter_map(|a| a.parse().ok()).collect(); + if !socks.is_empty() && seen.insert(nid) { + out.push((nid, socks)); + } } + for (nid, addrs) in storage.list_known_anchors().unwrap_or_default() { + if seen.insert(nid) { + out.push((nid, addrs)); + } + } + for r in storage.list_anchor_peers().unwrap_or_default() { + if seen.insert(r.node_id) { + out.push((r.node_id, r.addresses)); + } + } + out }; if anchors.is_empty() { return Ok("No known anchors".to_string()); } - let mut total = 0usize; - let mut reachable = 0usize; + + let mut connected = 0usize; + let mut asked = 0usize; + let mut refused = 0usize; for (anchor_nid, anchor_addrs) in &anchors { if *anchor_nid == node_id { continue; } - // Connect to anchor if not already connected - if !node.network.is_peer_connected(anchor_nid).await { + if !node.network.is_peer_connected_or_session(anchor_nid).await { let endpoint_id = match itsgoin_core::EndpointId::from_bytes(anchor_nid) { Ok(eid) => eid, Err(_) => continue, @@ -2820,40 +2841,26 @@ async fn request_referrals(state: State<'_, AppNode>) -> Result for sa in anchor_addrs { addr = addr.with_ip_addr(*sa); } - if let Err(_) = node.network.connect_to_peer(*anchor_nid, addr).await { + if node.network.connect_to_anchor(*anchor_nid, addr).await.is_err() { continue; } } - match node.network.request_anchor_referrals(anchor_nid).await { - Ok(referrals) => { - reachable += 1; - for referral in &referrals { - if referral.node_id == node_id { - continue; - } - if let Some(addr_str) = referral.addresses.first() { - let connect_str = format!( - "{}@{}", - hex::encode(referral.node_id), - addr_str, - ); - if let Ok((rid, raddr)) = itsgoin_core::parse_connect_string(&connect_str) { - match node.network.connect_to_peer(rid, raddr).await { - Ok(()) => {} - Err(_) => { - // Direct connect failed (NAT) — try hole punch via anchor - let _ = node.network.connect_via_introduction(rid, *anchor_nid).await; - } - } - total += 1; - } - } + match node.network.request_convection(anchor_nid, class).await { + Ok(response) => { + asked += 1; + if response.refused { + refused += 1; + } else { + connected += node.network.act_on_convection(anchor_nid, &response).await; } } Err(_) => {} } } - Ok(format!("Got {} referrals from {} anchors", total, reachable)) + Ok(format!( + "Convection: {} new peers from {} anchors ({} refused)", + connected, asked, refused + )) } #[tauri::command] @@ -3197,13 +3204,13 @@ async fn switch_identity( // Start background tasks on the new node new_node.start_accept_loop(); - new_node.start_pull_cycle(); + new_node.start_sync_cycle(); new_node.start_diff_cycle(120); new_node.start_rebalance_cycle(600); new_node.start_growth_loop(); new_node.start_recovery_loop(); new_node.start_social_checkin_cycle(3600); - new_node.start_anchor_register_cycle(600); + new_node.start_convection_loop(); new_node.start_upnp_tcp_renewal_cycle(); new_node.start_http_server(); new_node.start_bootstrap_connectivity_check(); @@ -3321,13 +3328,13 @@ async fn clear_duplicate_flag(state: State<'_, AppNode>) -> Result<(), String> { node.network.duplicate_detected.store(false, std::sync::atomic::Ordering::Relaxed); // Start the sync tasks that were skipped during bootstrap node.start_accept_loop(); - node.start_pull_cycle(); + node.start_sync_cycle(); node.start_diff_cycle(120); node.start_rebalance_cycle(600); node.start_growth_loop(); node.start_recovery_loop(); node.start_social_checkin_cycle(3600); - node.start_anchor_register_cycle(600); + node.start_convection_loop(); node.start_upnp_tcp_renewal_cycle(); node.start_http_server(); node.start_bootstrap_connectivity_check(); @@ -3569,13 +3576,13 @@ pub fn run() { // Start all background networking tasks boot_node.start_accept_loop(); - boot_node.start_pull_cycle(); + boot_node.start_sync_cycle(); boot_node.start_diff_cycle(120); boot_node.start_rebalance_cycle(600); boot_node.start_growth_loop(); boot_node.start_recovery_loop(); boot_node.start_social_checkin_cycle(3600); - boot_node.start_anchor_register_cycle(600); + boot_node.start_convection_loop(); boot_node.start_upnp_tcp_renewal_cycle(); boot_node.start_http_server(); boot_node.start_bootstrap_connectivity_check(); diff --git a/deploy.sh b/deploy.sh index f89cdca..17f632d 100755 --- a/deploy.sh +++ b/deploy.sh @@ -18,8 +18,19 @@ SSH_OPTS="-o StrictHostKeyChecking=no" KEYSTORE="itsgoin.keystore" KS_ALIAS="itsgoin" -VERSION=$(grep '"version"' crates/tauri-app/tauri.conf.json | head -1 | sed 's/.*"\([0-9.]*\)".*/\1/') -echo "=== Deploying v${VERSION} ===" +# Captures the FULL version string including any pre-release suffix +# (e.g. 0.8.0-alpha). The old digits-and-dots-only pattern silently +# truncated at the dash, so every ${VERSION} filename below missed the +# artifacts Tauri had actually produced. +VERSION=$(grep '"version"' crates/tauri-app/tauri.conf.json | head -1 | sed 's/.*"version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/') +# Pre-release versions announce on their own channel so stable clients +# are not told an alpha is "the" current release. +case "$VERSION" in + *-*) ANN_CHANNEL="${VERSION#*-}" ;; + *) ANN_CHANNEL="stable" ;; +esac + +echo "=== Deploying v${VERSION} (announce channel: ${ANN_CHANNEL}) ===" # Builds run SERIALLY — parallel cargo invocations write to the same # target/ directory, which causes intermittent failures (linuxdeploy @@ -76,10 +87,11 @@ sshpass -p "$SSH_PASS" ssh $SSH_OPTS "$SSH_HOST" " kill \$(cat ~/itsgoin-anchor.pid 2>/dev/null) 2>/dev/null sleep 1 mv ~/bin/itsgoin.new ~/bin/itsgoin && chmod +x ~/bin/itsgoin + ~/bin/itsgoin ~/itsgoin-anchor-data --publish-registry 2>&1 | tail -3 ~/bin/itsgoin ~/itsgoin-anchor-data \ --announce \ --ann-category release \ - --ann-channel stable \ + --ann-channel ${ANN_CHANNEL} \ --ann-version ${VERSION} \ --ann-url https://itsgoin.com/download.html \ --ann-title 'ItsGoin v${VERSION} available' \ diff --git a/scripts/a3_integration_test.sh b/scripts/a3_integration_test.sh index 7682704..32c887e 100755 --- a/scripts/a3_integration_test.sh +++ b/scripts/a3_integration_test.sh @@ -27,6 +27,12 @@ check() { # check else FAIL=$((FAIL+1)); echo "FAIL: $desc"; fi } strip_ansi() { sed 's/\x1b\[[0-9;]*m//g' "$1" | grep -v "WARN.*netlink\|WARN.*buffer_tool"; } +# Same noise filter for the tail-window checks below. The netlink/mDNS watcher +# logs in unpredictable bursts, and a burst landing between a REPL command and +# its assertion pushes the answer out of a raw `tail -N` window — a false +# failure with nothing wrong in the node. +quiet_tail() { grep -av "netlink\|buffer_tool\|iroh_quinn\|swarm_discovery" "$1" | tail -"$2"; } +export -f quiet_tail sq() { sqlite3 "/tmp/itsgoin-test$1/itsgoin.db" "$2"; } cleanup() { @@ -91,7 +97,7 @@ sleep 3 echo "search rust" >&15 sleep 8 check "node3 re-search finds nothing after signed delete" \ - bash -c 'tail -20 /tmp/itsgoin-cli3.log | grep -q "no registry matches"' + bash -c 'quiet_tail /tmp/itsgoin-cli3.log 20 | grep -q "no registry matches"' echo "== step 3: greeting roundtrip ==" # node2's bio post id (latest Profile post) from its DB. @@ -128,7 +134,8 @@ sleep 4 echo "search rust" >&15 sleep 8 check "node3 sees exactly one (newest) entry for node2's persona" \ - bash -c 'tail -6 /tmp/itsgoin-cli3.log | grep -c "rust" | grep -q "^1$" && tail -6 /tmp/itsgoin-cli3.log | grep -q AliceV2' + bash -c 'quiet_tail /tmp/itsgoin-cli3.log 6 | grep -c "rust" | grep -q "^1$" && + quiet_tail /tmp/itsgoin-cli3.log 6 | grep -q AliceV2' echo echo "== results: $PASS passed, $FAIL failed ==" diff --git a/scripts/c_topology_test.sh b/scripts/c_topology_test.sh new file mode 100755 index 0000000..30a432a --- /dev/null +++ b/scripts/c_topology_test.sh @@ -0,0 +1,278 @@ +#!/usr/bin/env bash +# v0.8 Iteration C integration test — TOPOLOGY. +# +# Run from the repo root after `cargo build -p itsgoin-cli`, AFTER +# scripts/a3_integration_test.sh (C does not replace A). Drives interactive +# REPLs over FIFOs, asserts by grepping logs and by sqlite3 against node DBs. +# +# SCENARIO 1 Two-pool uniques exchange over a chain: pool 1 shifts one bounce +# deeper per hop, pool 2 lands at N4, and N4 is USED but NEVER +# re-announced (no N5 anywhere). Plus the §layers privacy +# invariant: only anchor entries carry an address. +# SCENARIO 2 Anchor convection: the rolling window hands each caller the 2 +# most recent prior callers, produces a REAL connection, rotates +# entries out after 2 hand-outs, coordinates a RelayIntroduce when +# both ends are live, and refuses a top-up CHEAPLY. +# SCENARIO 3 Temp referral slots: allocated ABOVE the mesh cap, carrying no +# knowledge, never evicting an established mesh peer, graduating +# into a freed mesh slot. +# STANDING 0xC0 AnchorRegister is gone; no automatic session relay anywhere. +# +# Exit code 0 = all checks passed. Logs: /tmp/itsgoin-ctop{1..7}.log +set -u +cd "$(dirname "$0")/.." +BIN=target/debug/itsgoin +[ -x "$BIN" ] || { echo "build first: cargo build -p itsgoin-cli"; exit 2; } + +PASS=0; FAIL=0 +check() { # check + local desc="$1"; shift + if "$@" >/dev/null 2>&1; then PASS=$((PASS+1)); echo "PASS: $desc"; + else FAIL=$((FAIL+1)); echo "FAIL: $desc"; fi +} +strip_ansi() { sed 's/\x1b\[[0-9;]*m//g' "$1" | grep -v "WARN.*netlink\|WARN.*buffer_tool"; } +sq() { sqlite3 "/tmp/itsgoin-ctop$1/itsgoin.db" "$2"; } + +# Is `id` (hex) present in node N's uniques index at `bounce`? +at_bounce() { # at_bounce + [ "$(sq "$1" "SELECT count(*) FROM reachable WHERE lower(hex(reachable_id))='$2' AND bounce=$3")" != "0" ] +} +anywhere() { # anywhere + [ "$(sq "$1" "SELECT count(*) FROM reachable WHERE lower(hex(reachable_id))='$2'")" != "0" ] +} +# `check` runs its command via `bash -c`, which does not inherit shell functions. +export -f strip_ansi sq at_bounce anywhere + +NODES="1 2 3 4 5 6 7" +FDS="" +cleanup() { + for fd in 21 22 23 24 25 26 27; do eval "exec $fd>&-" 2>/dev/null || true; done + kill $(pgrep -f 'itsgoin.*itsgoin-ctop') 2>/dev/null + rm -f /tmp/itsgoin-ctopcmd* +} +trap cleanup EXIT + +# Test gates: +# - loopback referrals: the convection address filter is publicly-routable-only +# (bugs-fixed #8), which would make every 127.0.0.1 referral unusable. +# - mesh cap 2: reaches the temp-referral boundary with 4 nodes, not 21. +# - no growth loop: on loopback the growth loop collapses any chain into a +# full mesh in seconds, and a full mesh has no N2 at all (every peer is a +# direct). Topology here is script-controlled on purpose. +export ITSGOIN_TEST_ALLOW_LOOPBACK_REFERRALS=1 +export ITSGOIN_TEST_ALLOW_REGISTRY_GENESIS=1 +export ITSGOIN_TEST_NO_GROWTH=1 +export RUST_LOG="${RUST_LOG:-itsgoin_core=debug,info,iroh=warn,swarm_discovery=warn}" + +echo "== setup ==" +kill $(pgrep -f 'itsgoin.*itsgoin-ctop') 2>/dev/null; sleep 1 +for i in $NODES; do + rm -rf /tmp/itsgoin-ctop$i /tmp/itsgoin-ctopcmd$i /tmp/itsgoin-ctop$i.log + mkdir -p /tmp/itsgoin-ctop$i + mkfifo /tmp/itsgoin-ctopcmd$i +done + +start_node() { # start_node [extra env assignments...] + local i="$1" port="$2"; shift 2 + ( exec env "$@" "$BIN" /tmp/itsgoin-ctop$i --bind 127.0.0.1:$port \ + < /tmp/itsgoin-ctopcmd$i > /tmp/itsgoin-ctop$i.log 2>&1 ) & +} + +# Nodes 1-6: chain + convection. Node 7 joins late (fresh index). +for i in 1 2 3 4 5 6 7; do start_node $i $((18430 + i)) ITSGOIN_TEST_NO_GROWTH=1; done +exec 21>/tmp/itsgoin-ctopcmd1 22>/tmp/itsgoin-ctopcmd2 23>/tmp/itsgoin-ctopcmd3 \ + 24>/tmp/itsgoin-ctopcmd4 25>/tmp/itsgoin-ctopcmd5 26>/tmp/itsgoin-ctopcmd6 \ + 27>/tmp/itsgoin-ctopcmd7 +sleep 7 + +for i in $NODES; do + eval "n${i}=\$(strip_ansi /tmp/itsgoin-ctop$i.log | grep -m1 'Node ID:' | awk '{print \$3}')" +done +echo "n1=$n1 n2=$n2 n3=$n3 n4=$n4 n5=$n5" +[ -n "$n5" ] || { echo "FAIL: nodes did not start"; exit 1; } + +############################################################################## +echo +echo "== scenario 1: two-pool uniques exchange, N4 terminal ==" +############################################################################## +# Chain 1 - 2 - 3 - 4 - 5. Built inward so each hop's initial exchange already +# carries the deeper knowledge where possible. +echo "connect $n1@127.0.0.1:18431" >&22; sleep 3 +echo "connect $n2@127.0.0.1:18432" >&23; sleep 3 +echo "connect $n3@127.0.0.1:18433" >&24; sleep 3 +echo "connect $n4@127.0.0.1:18434" >&25; sleep 4 + +# Drive propagation explicitly: each uniques-pull round moves knowledge one +# bounce along the chain. This IS the v0.8 pull — an index exchange, no posts. +for round in 1 2 3 4 5; do + for fd in 21 22 23 24 25; do echo "uniques-pull" >&$fd; done + sleep 3 +done + +check "uniques pull is an index exchange (peers exchanged, no post transfer)" \ + bash -c 'grep -q "uniques-pull: exchanged with [1-9]" /tmp/itsgoin-ctop3.log' + +# Pool 1 shifts one bounce deeper per hop, tagged to the reporter. +check "node1 holds node3 at N2 (pool 1 shifted one bounce)" at_bounce 1 "$n3" 2 +check "node1 holds node4 at N3" at_bounce 1 "$n4" 3 +check "node1 holds node5 at N4 (terminal pool)" at_bounce 1 "$n5" 4 +check "node5 holds node1 at N4 from the other end of the chain" at_bounce 5 "$n1" 4 + +# N4 is USED — address resolution and search read it. +check "node1 can look up its N4 entry (used, not merely stored)" \ + bash -c 'grep -q "N4 via" <(echo "uniques '"$n5"'" >&21; sleep 3; tail -40 /tmp/itsgoin-ctop1.log)' + +# §layers privacy invariant: ONLY anchor entries carry an address. +check "no non-anchor index row carries an address (on any node)" \ + bash -c 'for i in 1 2 3 4 5; do + c=$(sqlite3 /tmp/itsgoin-ctop$i/itsgoin.db \ + "SELECT count(*) FROM reachable WHERE is_anchor=0 AND addresses NOT IN ('"''"',\"[]\")"); + [ "$c" = "0" ] || exit 1; done' + +# Per-reporter dedup: one row per (reporter, id) — an ID can never sit at two +# depths from the same reporter. +check "one index row per (reporter, id) — no duplicate depths" \ + bash -c 'for i in 1 2 3 4 5; do + d=$(sqlite3 /tmp/itsgoin-ctop$i/itsgoin.db \ + "SELECT count(*) FROM (SELECT reporter_node_id, reachable_id FROM reachable + GROUP BY reporter_node_id, reachable_id HAVING count(*) > 1)"); + [ "$d" = "0" ] || exit 1; done' + +# Nothing is stored past bounce 4. There is no N5 table and no N5 row. +check "stored horizon never exceeds 4 bounces" \ + bash -c 'for i in 1 2 3 4 5; do + m=$(sqlite3 /tmp/itsgoin-ctop$i/itsgoin.db "SELECT ifnull(max(bounce),0) FROM reachable"); + [ "$m" -le 4 ] || exit 1; done' + +# THE KEY ASSERTION: a fresh node joining node1 must NOT learn node5. node1 +# knows node5 only at N4, and the terminal pool is never re-announced. +echo "connect $n1@127.0.0.1:18431" >&27; sleep 4 +for round in 1 2 3; do echo "uniques-pull" >&27; echo "uniques-pull" >&21; sleep 3; done +check "N4 is never re-announced: node7 never learns node5" \ + bash -c '! /usr/bin/env bash -c "[ \"\$(sqlite3 /tmp/itsgoin-ctop7/itsgoin.db \ + \"SELECT count(*) FROM reachable WHERE lower(hex(reachable_id))=\\\"'"$n5"'\\\"\")\" != \"0\" ]"' +check "node7 DID learn node2 (the exchange itself works)" anywhere 7 "$n2" + +############################################################################## +echo +echo "== scenario 2: anchor convection ==" +############################################################################## +# Node 6 plays the anchor: directly dialable, and it is the one every caller +# asks. Callers are node1 and node5 — the two ENDS of the chain, so they have +# no path to each other and a referral that lands is unambiguous. +echo "connect $n6@127.0.0.1:18436" >&21; sleep 3 +echo "convect $n6" >&21; sleep 5 # node1 admitted; window was empty +echo "connect $n6@127.0.0.1:18436" >&25; sleep 3 +echo "convect $n6" >&25; sleep 10 # node5 is referred to node1 + +check "anchor serves referrals out of the rolling window" \ + bash -c 'grep -q "Convection: serving referrals" /tmp/itsgoin-ctop6.log' +check "the anchor admitted callers to the window without any registration msg" \ + bash -c 'grep -q "Convection: admitted caller to rolling window" /tmp/itsgoin-ctop6.log' +check "the anchor referred the PRIOR caller to the next one" \ + bash -c 'grep "Convection: referring peer" /tmp/itsgoin-ctop6.log | grep -q "'"$n1"'"' +check "a convection referral produced a real peer connection" \ + bash -c 'grep -q "Convection: connected to referred peer\|Convection: connected via introduction" \ + /tmp/itsgoin-ctop5.log' +check "node5 is now actually mesh-connected to node1 (chain ends joined)" \ + bash -c 'echo "connections" >&25; sleep 3; strip_ansi /tmp/itsgoin-ctop5.log | tail -20 | grep -q "'"${n1:0:12}"'"' + +# Both ends live on the anchor => coordinate an introduction, not a cold address. +check "anchor coordinates RelayIntroduce when both ends are live" \ + bash -c 'grep -q "Convection: coordinated introduction" /tmp/itsgoin-ctop6.log' + +# Rotation: an entry is handed to exactly 2 callers, then leaves the window. +echo "convect $n6" >&24; sleep 6 +echo "convect $n6" >&23; sleep 6 +echo "convect $n6" >&27; sleep 6 +check "each window entry is handed out at most twice, then rotates out" \ + bash -c 'n=$(grep "Convection: referring peer" /tmp/itsgoin-ctop6.log | grep -c "referred=\"\?'"$n1"'"); \ + [ "$n" -le 2 ]' +check "later callers are served the more RECENT callers" \ + bash -c 'grep "Convection: referring peer" /tmp/itsgoin-ctop6.log | tail -3 | grep -q "'"$n5"'\|'"$n4"'"' + +# CHEAP REFUSAL: node3 is an interior chain node (2 mesh peers) so its request +# is TOP-UP class. It asks node5, whose own convection window is empty — there +# is nothing fresh to circulate, so the refusal must be one message, not a +# burned timeout. +echo "connect $n5@127.0.0.1:18435" >&23; sleep 4 +echo "convect $n5" >&23; sleep 6 +check "top-up against an empty window is refused" \ + bash -c 'grep -q "convection: refused" /tmp/itsgoin-ctop3.log' +check "the refusal is CHEAP (single message, well under a timeout)" \ + bash -c 'ms=$(grep -o "convection: refused in [0-9]*ms" /tmp/itsgoin-ctop3.log | tail -1 | + grep -o "[0-9]*" | head -1); [ -n "$ms" ] && [ "$ms" -lt 3000 ]' +check "entry class is served even where top-up was refused" \ + bash -c 'grep "Convection: serving referrals" /tmp/itsgoin-ctop6.log | grep -q entry' +check "the refusing anchor lands in the caller's penalty box (refusal feedback)" \ + bash -c 'echo "convection" >&23; sleep 3; strip_ansi /tmp/itsgoin-ctop3.log | tail -20 | + grep -oE "anchor_bias [0-9.]+" | tail -1 | awk "{ exit !(\$2 < 1.0) }"' + +############################################################################## +echo +echo "== scenario 3: temp referral slots (mesh cap 2) ==" +############################################################################## +# Restart nodes 1-4 with a mesh cap of 2 so the temp-referral band is reachable. +for fd in 21 22 23 24 25 26 27; do echo "quit" >&$fd 2>/dev/null; done +sleep 3 +kill $(pgrep -f 'itsgoin.*itsgoin-ctop') 2>/dev/null; sleep 2 +for fd in 21 22 23 24 25 26 27; do eval "exec $fd>&-" 2>/dev/null || true; done + +for i in 1 2 3 4; do + rm -rf /tmp/itsgoin-ctop$i /tmp/itsgoin-ctopcmd$i /tmp/itsgoin-ctop$i.log + mkdir -p /tmp/itsgoin-ctop$i; mkfifo /tmp/itsgoin-ctopcmd$i + start_node $i $((18440 + i)) ITSGOIN_TEST_MESH_SLOTS=2 \ + ITSGOIN_TEST_ALLOW_LOOPBACK_REFERRALS=1 ITSGOIN_TEST_NO_GROWTH=1 +done +exec 21>/tmp/itsgoin-ctopcmd1 22>/tmp/itsgoin-ctopcmd2 23>/tmp/itsgoin-ctopcmd3 \ + 24>/tmp/itsgoin-ctopcmd4 +sleep 7 +for i in 1 2 3 4; do + eval "m${i}=\$(strip_ansi /tmp/itsgoin-ctop$i.log | grep -m1 'Node ID:' | awk '{print \$3}')" +done + +# Fill node1's 2 mesh slots, then send a third peer at it. +echo "connect $m1@127.0.0.1:18441" >&22; sleep 4 +echo "connect $m1@127.0.0.1:18441" >&23; sleep 4 +echo "slots" >&21; sleep 3 +check "mesh pool fills to the cap" \ + bash -c 'strip_ansi /tmp/itsgoin-ctop1.log | grep -q "mesh 2/2"' + +echo "connect $m1@127.0.0.1:18441" >&24; sleep 5 +echo "slots" >&21; sleep 3 +check "the 4th peer lands in a temp referral slot ABOVE the cap" \ + bash -c 'strip_ansi /tmp/itsgoin-ctop1.log | tail -30 | grep -q "mesh 2/2 temp-referral 1/"' +check "an established mesh peer was never evicted to make room" \ + bash -c 'strip_ansi /tmp/itsgoin-ctop1.log | tail -30 | grep -q "mesh 2/2"' +check "the temp slot carries NO knowledge (never a reporter in our index)" \ + bash -c '[ "$(sqlite3 /tmp/itsgoin-ctop1/itsgoin.db \ + "SELECT count(*) FROM reachable WHERE lower(hex(reporter_node_id))=\"'"$m4"'\"")" = "0" ]' +check "and a temp peer is never written to mesh_peers" \ + bash -c '[ "$(sqlite3 /tmp/itsgoin-ctop1/itsgoin.db \ + "SELECT count(*) FROM mesh_peers WHERE lower(hex(node_id))=\"'"$m4"'\"")" = "0" ]' + +# Free a mesh slot: the temp peer graduates into it. The wait covers the mesh +# keepalive interval (30s) — an abruptly-gone QUIC peer is noticed when the +# next keepalive write fails, not instantly. +echo "quit" >&22; sleep 45 +echo "slots" >&21; sleep 5 +check "the temp referral slot GRADUATES into the freed mesh slot" \ + bash -c 'strip_ansi /tmp/itsgoin-ctop1.log | tail -20 | grep -q "temp-referral 0/"' +check "the graduated peer now exchanges knowledge (written to mesh_peers)" \ + bash -c '[ "$(sqlite3 /tmp/itsgoin-ctop1/itsgoin.db \ + "SELECT count(*) FROM mesh_peers WHERE lower(hex(node_id))=\"'"$m4"'\"")" != "0" ]' + +############################################################################## +echo +echo "== standing assertions ==" +############################################################################## +check "0xC0 AnchorRegister is gone from the tree" \ + bash -c '! grep -rn "AnchorRegister" crates/ --include=*.rs' +check "no automatic session relay anywhere in the convection path" \ + bash -c '! grep -riE "session relay" /tmp/itsgoin-ctop*.log' +check "no registration cycle writes known_anchors (bootstrap cache only)" \ + bash -c '! grep -rn "start_anchor_register_cycle" crates/ --include=*.rs' + +echo +echo "== results: $PASS passed, $FAIL failed ==" +[ "$FAIL" -eq 0 ] diff --git a/website/design.html b/website/design.html index a21631c..daa150c 100644 --- a/website/design.html +++ b/website/design.html @@ -129,11 +129,11 @@ - + - - - + + +
CycleIntervalPurposeStatus
Update cadence schedulerDescending scale (minutes → days) per author, keyed on freshness × relationship tier, jitteredCDN update checks + uniques-list refresh; timing shaped to look like uniform network overhead. See update cadence.Rework — today: 60s pull tick with a 4h stale-author threshold
Routing diff120s (2 min)Announce network-knowledge changes to mesh peers (target: uniques-list diffs; today: N1/N2 share-list diffs)Rework
Routing diff120s (2 min)Announce network knowledge to mesh peers as a full two-pool uniques snapshot; skipped when the content digest is unchangedImplemented
Rebalance600s (10 min)Clean dead connections, signal growthRework — preferred-peer Priority 0 deleted (Iteration B); remaining priorities still target the wide model
Growth loopReactive (signal-driven on knowledge receipt)Fill mesh slots toward 20 with the most diverse candidates; secondary peer-finding path alongside convectionRework — mechanism is live; targets 101 today and only counts local slots
Convection triggerStochastic, per-disconnectOn each mesh disconnect, random choice: do nothing / ask a random known anchor for an introduction / ask a mesh peer. See anchors.Planned
Recovery loopReactive (mesh drops below 2)Emergency reconnect: mine retained uniques pools for anchor addresses, then known_anchors cache, then any connected anchor peersRework — today's loop still sends AnchorRegister
Growth loopReactive (signal-driven on knowledge receipt)Fill the single mesh pool toward 20 with the most diverse candidates (fewest reporters wins, shallower bounce breaks ties); secondary peer-finding path alongside convectionImplemented
Convection triggerStochastic, per-disconnectOn each mesh disconnect (temp referral expiry does not count), random choice: do nothing / ask a random known anchor for an introduction / ask a mesh peer. See anchors.Implemented
Recovery loopReactive (mesh drops below 2)Emergency reconnect: mine retained uniques pools for anchor addresses, then known_anchors cache, then any connected anchor peers. Non-stochastic — below 2 it always acts.Implemented
Anchor reachability watcherEvent-driven (portmapper watch channel)Clears anchor candidacy after >5min of no port mapping; restores it when the mapping comes back. Network roams self-heal without restart. Mobile never auto-anchors.Implemented
@@ -163,30 +163,32 @@

A port mapping or public IPv6 address is a claim, not proof. The node selects a witness from its network knowledge that is NOT a current peer — a genuine stranger with no prior connection, no cached address, no warm path — reached via the peer that reported it (AnchorProbeRequest, 0xC3). The witness performs a raw cold QUIC connect to the candidate's external address (15s timeout), deliberately skipping the entire resolution cascade — hole punch, introduction, or relay success would be a false positive.

Asymmetric return path: on success the witness reports back over its fresh direct connection (AnchorProbeResult, 0xC4); on failure there is by definition no direct path, so the result routes back through the reporting peer.

- v0.8 change: Probe scheduling currently rides the anchor register cycle, which is removed. The target triggers re-probes from the reachability watcher (on mapping acquisition/restoration) and on failed inbound connections. + v0.8 (Iteration C): the anchor register cycle the probe used to ride is gone. The probe now rides the convection loop's 10-minute maintenance tick. Note the consequence: is_anchor_candidate no longer requires a successful probe — candidacy is reachability + opt-in, and the probe is confirmation that runs alongside, not a gate in front. Triggering re-probes from the reachability watcher (on mapping acquisition/restoration) and on failed inbound connections remains Planned.
-

Connection convection Rework

+

Connection convection Implemented

The anchor's referral service is a rotating referral chain — "connection convection." The anchor keeps only a small rolling window of recent callers; there is no peer database and no persistent registration.

  1. A caller connects and announces "I'm joining / I need peers."
  2. The anchor returns the addresses of the 2 most recent viable prior callers.
  3. The caller's own address is handed out to the next 2 callers, then rotates out of the window.
+

Window parameters (previously open, now decided in code): the window holds 16 entries; each entry is handed to 2 callers before rotating out; an entry older than 15 minutes is not handed out at all — a stale address is worse than none, because it costs the next caller a hole-punch timeout. A caller with no routable address is admitted to nothing. Self-reported addresses are a caller's unverified claim: at most 2 are kept, and when the anchor has an observed address for the caller (essentially always over QUIC) self-reported entries are kept only if they share its IP — a different port is the legitimate endpoint-dependent-mapping case, a different host is not. Only the observed address is ever named in an anchor-initiated introduction; self-reported ones ride the referral payload as a hint the referred peer may try itself.

+

Sparse-window supplement: a young anchor with an empty window may fill referrals from its own live mesh peers — but those peers never called and never offered themselves for redistribution, so the supplement is consent-shaped. It discloses no address (the referral carries an empty address list and relies on the coordinated introduction, so the target discloses its own addresses on its own terms), it is used only for peers the anchor can actually introduce, and each supplemented peer carries its own hand-out budget (2 per 15 minutes). Without that budget, unconditional entry-class service plus a reshuffle would be an enumeration oracle over the anchor's entire mesh membership.

Request classes: convection requests carry a one-bit class — entry (bootstrap or recovery: fewer than 2 connections) versus top-up (growth with a working mesh). Anchors always serve entry; top-ups are served capacity-permitting and refused cheaply (a single refusal message, no timeout burned), so deprioritizing the non-needy is harmless. The refusal doubles as the load signal the adaptive weights consume. Top-ups are not freeloading: every caller freshens the anchor's referral window, so top-up traffic is the convection medium — a network where nobody tops up hands its next genuine joiner stale referrals.

-

Referral quality: the anchor prefers callers it is still connected to — and when both the referred peer and the caller are live on the anchor, it coordinates a RelayIntroduce hole-punch introduction between them instead of handing out a cold address. Referred peers land in the caller's temporary referral slots (above the 20-peer mesh cap, no knowledge exchange) and either graduate into mesh slots or expire.

-

Trigger — stochastic, per-disconnect: each time a mesh peer disconnects, the node randomly chooses one of three actions: do nothing, request an introduction from a random known anchor (mined from the uniques pools — see bookkeeping below), or request an introduction via a connected mesh peer (the growth loop's diversity-scored path — see connection lifecycle). The randomness itself de-synchronizes the network's reaction to shared disconnection events — no jitter timer, no threshold. Because every caller becomes a referral for the next callers, ordinary churn keeps the chain circulating: convection. With IPv6 making anchors abundant (an estimated 20–30% of nodes), anchors rebuild the mesh among themselves and with their callers, absorbing even large disconnection events. The action-choice weights are adaptive, not fixed. The anchor ratio is locally observable — anchor entries are the ones carrying addresses in the pools, so anchor density among known uniques is a free statistic needing no census — and it sets the prior: anchor-rich networks lean anchor-heavy (which also keeps convection windows fresh), anchor-poor lean mesh-ward. Refusals then steer: an anchor refusing a growth request shifts the caller's weights toward mesh introductions and other anchors, drifting back as requests succeed. How many operators will announce addresses versus hide them (or lack public ones) is unknowable before deployment — the adaptive weights are designed to find that ratio rather than assume it. (The node-ID–to–persona ratio stays deliberately unknowable; this statistic never needs it.)

+

Referral quality: the anchor prefers callers it is still connected to — and when both the referred peer and the caller are live on the anchor, it coordinates a RelayIntroduce hole-punch introduction between them instead of handing out a cold address. A referred peer lands in whatever slot class is free: a mesh slot first, and only the temporary referral band above the cap when the mesh is full — from there it graduates into a freed mesh slot or expires.

+

Trigger — stochastic, per-disconnect Implemented: each time a mesh peer disconnects — a temp referral slot expiring is not mesh churn and does not roll the dice — the node randomly chooses one of three actions: do nothing, request an introduction from a random known anchor (mined from the uniques pools — see bookkeeping below), or request an introduction via a connected mesh peer (the growth loop's diversity-scored path — see connection lifecycle). The randomness itself de-synchronizes the network's reaction to shared disconnection events — no jitter timer, no threshold. Because every caller becomes a referral for the next callers, ordinary churn keeps the chain circulating: convection. With IPv6 making anchors abundant (an estimated 20–30% of nodes), anchors rebuild the mesh among themselves and with their callers, absorbing even large disconnection events. The action-choice weights are adaptive, not fixed. The anchor ratio is locally observable — anchor entries are the ones carrying addresses in the pools, so anchor density among known uniques is a free statistic needing no census — and it sets the prior: anchor-rich networks lean anchor-heavy (which also keeps convection windows fresh), anchor-poor lean mesh-ward. Refusals then steer: an anchor refusing a growth request shifts the caller's weights toward mesh introductions and other anchors, drifting back as requests succeed. How many operators will announce addresses versus hide them (or lack public ones) is unknowable before deployment — the adaptive weights are designed to find that ratio rather than assume it. (The node-ID–to–persona ratio stays deliberately unknowable; this statistic never needs it.)

- Current code implements the older registration model this replaces: an in-memory referral list with tiered use caps (3 uses under 50 entries, fewer as the list grows), a 2-minute disconnect grace, and least-used-first selection. The convection window replaces all of it. + Done (Iteration C): the older registration model is gone — 0xC0 AnchorRegister and the periodic register cycle are removed along with the in-memory referral list, its tiered use caps and least-used-first selection. The rolling window replaced all of it, and recovery (below 2 mesh peers) stays non-stochastic: it always acts immediately.
-

Anchor bookkeeping Rework

+

Anchor bookkeeping Implemented

Target: the uniques pools ARE the anchor directory. Anchor entries carry addresses in the N0–N3 pools (see Network Knowledge), so a node needing a connection mines its retained pools for a random anchor-flagged entry. Pool knowledge is overwritten memory: a slot's contributed entries are wiped only when a new handshake replaces them — so a node knocked down from 20 peers to 4 still holds 16 dead peers' pools full of anchor addresses to reconnect through. No separately curated anchor list is needed.

-

The shipped known_anchors table (node_id, addresses, last_seen, success_count — a count of successful connects accreted during v0.7.x bootstrap hardening, ranked descending, capped at 5) survives as the bootstrap cache. Its careful ranking solved an anchor-scarcity problem that pool-mined abundance retires.

+

The shipped known_anchors table survives as the bootstrap cache, capped at 32 entries and ordered by last_seen_ms (freshest first). Its old success_count ranking solved an anchor-scarcity problem that pool-mined abundance retires. It is written only for anchors we have actually connected to — never from a peer's announce — which is both what keeps gossip from re-inflating a table the ruling deliberately demoted, and what makes its recency ordering mean something again (a cache refreshed by hearsay would evict the DNS-resolved bootstrap anchor first).

Selection order:

    -
  1. Uniques pools Planned — random anchor-flagged entry, including entries from disconnected peers' retained pools.
    2. -
  2. known_anchors bootstrap cache Implemented — ordered by success_count descending. Failed probes to entries not seen for 3 days are deleted on the spot (self-healing against stale data dirs).
    3. +
  3. Uniques pools Implemented — anchor-flagged entries, shallowest and freshest first, including entries from disconnected peers' retained pools.
    4. +
  4. known_anchors bootstrap cache Implemented — ordered by last_seen_ms descending. Failed probes to entries not seen for 3 days are deleted on the spot (self-healing against stale data dirs).
  5. Hardcoded default anchor(s) — only if everything above is empty or exhausted. A brand-new node hits the hardcoded anchor once on first bootstrap, populates its caches from that session, and the hardcoded list recedes to pure fallback.
@@ -194,7 +196,7 @@
  • Bootstrap: First startup with an empty peers table — batched probing as described in Identity & Bootstrap, then a convection referral request.
  • Recovery: When the mesh drops below 2 connections. Reconnect via anchor addresses mined from the retained uniques pools, then the known_anchors cache, then any still-connected anchor peers.
    • -
  • Convection top-up Planned: the per-disconnect stochastic action described above. This is the ONLY ongoing anchor contact — there is no periodic registration.
    • +
  • Convection top-up Implemented: the per-disconnect stochastic action described above. This is the ONLY ongoing anchor contact — there is no periodic registration.
  • itsgoin.net node: A permanent, well-connected ItsGoin node runs on itsgoin.net as part of the share-link redirect infrastructure (see share links). It participates as a standard anchor — it bootstraps new nodes and serves convection referrals — and is not special-cased in the protocol. Its value comes from permanent uptime, not from any privileged role.
@@ -203,12 +205,12 @@
-

4. Connections & Slot Architecture Rework

+

4. Connections & Slot Architecture Implemented

Connection types

  • Mesh connection — long-lived routing slot in a single ~20-slot pool. Structural backbone for search and propagation; participates in the N1–N4 knowledge system's uniques announce (two pools: forwardable N0–N2 + terminal N3 — see Network Knowledge). DB table: mesh_peers.
    • -
  • Temp referral connection Planned — short-lived slot above the mesh cap, created by anchor convection or peer introductions. Carries NO knowledge exchange; exists purely to facilitate introductions. Graduates into a mesh slot or expires.
    • +
  • Temp referral connection Implemented — short-lived slot above the mesh cap, created by anchor convection or peer introductions. Carries NO knowledge exchange; exists purely to facilitate introductions. Graduates into a mesh slot or expires.
  • Cadence-driven session — connection to a social or file-layer peer that isn't in the mesh pool, opened on demand by the update-cadence cycle and closed when idle (see Update Cadence & Keep-Alive). There is no standing keep-alive pool — only the shaped check cycle recurs.
  • Session connection — short-lived, held open for active interaction (DM conversations, group activity, anchor matchmaking). Tracks remote_addr so a relay can inject observed addresses during introductions.
  • Ephemeral connection — single request/response, no slot allocation.
    • @@ -217,8 +219,8 @@

    Slot architecture

    - - + +
    Slot kindDesktopMobileStatusPurpose
    Mesh (single pool)2015ReworkLong-lived routing backbone; every slot equal, filled by growth loop + inbound
    Temp referral+4–+10+4–+10PlannedIntroduction facilitation above the cap; no knowledge exchange; graduate or expire
    Mesh (single pool)2015ImplementedLong-lived routing backbone; every slot equal, filled by growth loop + inbound
    Temp referral+10+4ImplementedIntroduction facilitation above the cap; no knowledge exchange; graduate or expire. The +4..+10 ruling is a range; these are the shipped points in it.
    Sessions (interactive)205ImplementedActive DM, group interaction, anchor matchmaking. At capacity, the oldest idle session is evicted for a new one.
    Relay pipes102ImplementedSession-relay byte pipes. Serving others is opt-in and default OFF (see Relay & NAT Traversal).
    @@ -230,12 +232,12 @@

MeshConnection struct

-

Each mesh connection tracks: node_id, connection (QUIC), remote_addr (captured from Incoming before accept), last_activity (AtomicU64, updated on every stream accept), connected_at. The per-slot slot_kind discriminator (Preferred/Local/Wide) goes away with the pool merge; a boolean or small enum distinguishes mesh slots from temp referral slots.

+

Each mesh connection tracks: node_id, connection (QUIC), remote_addr (captured from Incoming before accept), last_activity (AtomicU64, updated on every stream accept), connected_at. The per-slot slot_kind discriminator (Preferred/Local/Wide) is gone with the pool merge; a small enum distinguishes Mesh from TempReferral { expires_at_ms }. Allocation is mesh-first, then the temp band, then refuse — nothing in the rule can evict anything, so an established mesh peer is never displaced by an arriving stranger.

-

Temp referral slots Planned

-

Nodes hold +4 to +10 temporary connections above the 20-slot mesh cap. These exist so that introductions (anchor convection referrals, growth-loop introductions) never fail for want of a free slot, without paying the knowledge-state cost of a full mesh peer:

+

Temp referral slots Implemented

+

Nodes hold temporary connections above the mesh cap — +10 on desktop, +4 on mobile, within the +4..+10 ruling. These exist so that introductions (anchor convection referrals, growth-loop introductions) never fail for want of a free slot, without paying the knowledge-state cost of a full mesh peer:

    -
  • No knowledge exchange — temp referral peers do not participate in the uniques announce exchange. Lightweight by construction.
    • +
  • No knowledge exchange — temp referral peers do not participate in the uniques announce exchange, in either direction, and the boundary is structural: a temp peer is never written to mesh_peers, which is the table the announce builder reads. The knowledge gate covers the whole initial exchange, not just the uniques pools: a temp referral peer is also sent an empty peer-address list, since up to 10 of our mesh peers with their addresses is knowledge by any reading. On the wire "no knowledge" is an explicit null, not empty vectors, so the peer can tell "nothing to share" from "not sharing".
  • Graduate or expire — when a mesh slot is free, a temp connection can be promoted into it (and only then joins knowledge exchange). Otherwise it expires after its introduction purpose is served.
  • Graduation requires a free mesh slot — a temp referral never evicts an established mesh peer.
@@ -249,8 +251,8 @@
  • Full session/ephemeral interaction still works — messages, probes, routing participation; blacklisted nodes never consume each other's mesh slots
    • -

    --max-mesh <n> CLI flag Planned

    -

    Topology control for testing. Forces a node to cap its mesh connections below the 20 default, keeping it permanently in N2 of other nodes. --max-mesh 0 = pure N2 participant (free rider — consumes routing knowledge without carrying mesh load); --max-mesh 20 = default full behavior. Reuses RefuseRedirect (0x05) — no new protocol machinery. Testing affordance only.

    +

    Mesh-cap override Implemented

    +

    Topology control for testing, shipped as the ITSGOIN_TEST_MESH_SLOTS environment override rather than the originally-planned --max-mesh <n> CLI flag. It caps a node's mesh pool below the 20 default so the integration suite can reach the temp-referral boundary with four nodes instead of twenty-one, and so a node can be held permanently in other nodes' N2. Read once into a OnceLock at first use — the cap cannot change under a running node. Refusals reuse RefuseRedirect (0x05) — no new protocol machinery. Testing affordance only.

    Keepalive Implemented

      @@ -277,7 +279,7 @@ v0.8 change: growth targets the whole 20-slot pool. (Today the loop stops when the Local sub-pool is full and Wide slots only fill from inbound — a casualty of the old taxonomy that dies with it.) Rework
      - v0.8 change — stochastic growth Planned: threshold triggers are replaced by a per-disconnect random choice among {do nothing | convection request to a random known anchor | introduction via a connected mesh peer}. The mesh path uses the diversity-scored candidate machinery below; the anchor path defers candidate choice to the anchor's convection window. Randomness replaces jitter for de-synchronization (see Anchors & Connection Convection). + v0.8 — stochastic growth Implemented: threshold triggers are replaced by a per-disconnect random choice among {do nothing | convection request to a random known anchor | introduction via a connected mesh peer}. The mesh path uses the diversity-scored candidate machinery below; the anchor path defers candidate choice to the anchor's convection window. Randomness replaces jitter for de-synchronization (see Anchors & Connection Convection).

      Candidate selection (N2 diversity scoring) Implemented:

      @@ -313,7 +315,7 @@

      Trigger: fires when the mesh drops below 2 connections — the one non-stochastic case: at the edge of isolation the node always acts, immediately.

      1. Debounce 2 seconds (let cascading disconnects settle), coalesce queued signals
        2. -
      2. Gather anchors: mine the retained uniques pools for anchor-flagged addresses (Planned — disconnected peers' pools survive as overwritten memory until new handshakes replace them), then the known_anchors bootstrap cache, then anchor-flagged entries in the peers table
        3. +
      3. Gather anchors: mine the retained uniques pools for anchor-flagged addresses (Implemented — disconnected peers' pools survive as overwritten memory until new handshakes replace them), then the known_anchors bootstrap cache, then anchor-flagged entries in the peers table. Anchors that recently refused us are pushed to the back rather than dropped — a refusal is a load signal, not a blacklist.
      4. Make a convection call to each anchor in turn: receive referrals, try direct connect to each, fall back to anchor-coordinated hole punch for unreachable referrals (15s per attempt)
      5. Signal the growth loop as soon as any connection lands; stale anchors are pruned per the 3-day rule (see Anchors & Connection Convection)
      @@ -337,18 +339,18 @@

    Processing: their N1 → our N2 table (tagged to reporter), their N2 → our N3 table (tagged to reporter). Store profile, apply deletes, record replica overlaps. New N1 knowledge signals the growth loop immediately.

    - v0.8 change: the knowledge-share component becomes the uniques announce (two pools: forwardable N0–N2 + terminal N3; stored N4 is never transmitted), with the receiver storing each entry shifted one bounce deeper, our terminal pool becoming their N4 (see Network Knowledge). The handshake's other fields carry forward unchanged. Rework + v0.8 (Iteration C, done): the knowledge-share component is now the uniques announce (two pools: forwardable N0–N2 + terminal N3; stored N4 is never transmitted), with the receiver storing each entry shifted one bounce deeper, our terminal pool becoming their N4 (see Network Knowledge). The handshake's other fields carry forward unchanged, except that a temporary referral slot now sends null uniques and an empty peer-address list. Implemented

    Incremental Routing Diffs (every 120s + 4h full state) Rework

    NodeListUpdate (0x01) carries N1 added/removed and N2 added/removed, sent to connected peers on a 120s cycle; every 4 hours a full-state re-broadcast catches missed diffs. Diffs are also pushed eagerly after each growth-loop connection. Receiver processing mirrors the initial exchange: their N1 adds → our N2 adds, their N2 adds → our N3 adds; diffs with N1 additions signal the growth loop.

    - v0.8 change: diffs become uniques-announce diffs under the N1–N4 model, and their cadence folds into the update-cadence system's traffic shaping (see Update Cadence & Keep-Alive). Rework + v0.8 (Iteration C, done): the 2-minute broadcast is now a full uniques announce under the N1–N4 model, skipped entirely when the content digest is unchanged. Folding its cadence into the update-cadence system's traffic shaping (see Update Cadence & Keep-Alive) is Planned for Iteration D.
    -

    6. Network Knowledge: N1–N4 Uniques Rework

    +

    6. Network Knowledge: N1–N4 Uniques Implemented

    Every node maintains a layered index of uniques — IDs it can help other nodes reach. The index spans four bounces (N1–N4) and is exchanged only between directly-connected mesh peers. Collectively, these per-node indexes form the network's distributed search index: a pull (see Sync Protocol) is an exchange of uniques lists — "if you want these IDs, talk to me and I'll help you find them" — and answering "who can get me to ID X?" is a walk down reporter chains. Content never travels with the index; it lives in the CDN (see Files & Storage).

    @@ -365,25 +367,30 @@
  • CDN file peers — holders we exchanged files with

    • Sources are merged before announcing — a receiver cannot tell whether we know an ID from a mesh slot, a friendship, or a file transfer. Anonymous and throwaway IDs (temporary poster IDs in comment chains, greeting senders) are deliberately kept in the lists: they are network noise that protects anonymity and future hooks for anonymous encrypted messaging over temp IDs. Comment TTL expiry (see Delete Propagation) retires them naturally — when the content dies, the throwaway ID falls out of everyone's uniques.

    +

    Our own posting identities are never in our own uniques Implemented. Our node ID — and, when we are an anchor, our address — ride slice 0 of the same announce, so listing one of our personas at bounce 1 would be a signed-by-transport persona→device mapping handed to all ~20 mesh peers. It is also the easiest possible mapping to extract: our own personas are the invariant across every announce we send while everything else churns. Every persona is excluded (not just the default posting ID), as is the author of any comment we wrote locally — which covers the per-greeting throwaway IDs we mint, where we would otherwise be the first node in the network to announce the ID, at bounce 1, the instant the greeting was created. Our personas stay discoverable through everyone else who holds our content; we simply must not be the reporter. See Graph Privacy.

    Temp referral slots (see Connections) are the one class of direct connection that does not enter the exchange: they carry no knowledge state in either direction until they graduate into mesh slots.

    The four layers

    - - - - + + + +
    LayerBounceSourceAnnounced?Stored in
    N11Our own uniques (merged from all sources above)Yesmesh_peers + social_routes + file_holders
    N22Peers' announced N1, tagged to reporterYesreachable_n2
    N33Peers' announced N2, tagged to reporterYesreachable_n3
    N44Peers' announced N3, tagged to reporterNever — terminatesreachable_n4 Planned
    N11Our own uniques (merged from all sources above)YesDerived at build time from mesh_peers + social_routes + file_holders + post_replicas + cdn_manifests + comments
    N22Peers' announced N1, tagged to reporterYesreachable, bounce = 2
    N33Peers' announced N2, tagged to reporterYesreachable, bounce = 3
    N44Peers' announced N3, tagged to reporterNever — terminatesreachable, bounce = 4
    +

    There is one reachable table, keyed (reporter_node_id, reachable_id), with bounce, id_class, is_anchor, addresses and updated_at columns — not a table per layer. The composite key gives per-reporter set-merge for free, and the upsert keeps the shallowest bounce ever heard from that reporter, so one ID can never occupy two depths from one source (the ambiguity the old reachable_n2/reachable_n3 pair papered over by sorting).

    Announce protocol Rework

    • To/from N1 only. Uniques are exchanged exclusively with direct mesh peers — never forwarded on a node's behalf.
      • -
    • Full exchange on connect, incremental seq-numbered add/remove diffs afterward (today's NodeListUpdate 0x01 is the seed; it grows an N3 layer).
      • +
    • Every announce is a full snapshot Implemented (full: true) that replaces the sender's rows wholesale. Incremental seq-numbered diffing across four layers with anchor metadata was rejected as not worth the divergence risk; instead the sender hashes the pools (ignoring seq) and skips the send entirely when nothing changed, which recovers most of the saving. Slice trimming is deterministic (anchors first, then by ID) so a stable index yields a stable digest rather than thrashing it.
    • An announcement carries two explicitly separated pools. Pool 1 — the forwardable pool: our N0–N2 uniques (self + own uniques + two bounces), deduplicated internally. Pool 2 — the terminal pool: our N3 uniques, deduplicated against pool 1. The receiver stores each entry shifted one bounce deeper (pool 1 → their N1–N3, pool 2 → their N4), all tagged to us as reporter.
    • The terminal pool is used, never forwarded. Entries received via pool 2 (stored as N4) may be searched and resolved through, but never appear in the receiver's own announcements. The explicit terminal tag is what stops eternal re-propagation — some data must be differentiated as use-only, and there is no N5 anywhere in the network. (Carrying stored N4 in the payload was considered and rejected: dedup math put the resulting stored horizon near ~760k entries per node.)
      • -
    • Anchor entries carry addresses; everything else is a bare ID. There is no point sharing an address that cannot be connected without an introduction, so pool entries carry addresses only when flagged as anchors (directly reachable). The uniques index thereby doubles as the network's anchor directory — and with IPv6 pushing an estimated 20–30% of nodes into anchor candidacy, every node's pools hold a deep reserve of re-entry addresses.
      • -
    • Disconnect cleanup: when a peer drops, every entry it reported is removed. Stale entries are pruned after 5 hours regardless.
      • +
    • Node-class and author-class IDs travel in separate packed arrays Implemented. A slice is { nodes, authors, anchors }: two base64 blobs of concatenated 32-byte IDs plus the address-bearing anchor array. The separation is load-bearing, not cosmetic — a receiver structurally cannot mistake a persona ID for a connectable node ID (so persona IDs never enter connect/hole-punch paths and never burn punch timeouts), and the anchor array is the only place an address can appear. An author-class entry is emitted as a bare ID even if its stored row carries an anchor flag.
      • +
    • Anchor entries carry addresses; everything else is a bare ID. There is no point sharing an address that cannot be connected without an introduction, so pool entries carry addresses only when flagged as anchors (directly reachable). The uniques index thereby doubles as the network's anchor directory — and with IPv6 pushing an estimated 20–30% of nodes into anchor candidacy, every node's pools hold a deep reserve of re-entry addresses. A received anchor entry is an index row, not an address record: nothing has proven the address belongs to that node ID, so it stays in the uniques index (where the anchor mine reads it) and never mutates authoritative address state. Only an anchor we have completed a QUIC handshake to — iroh having proven the node ID owns that path — is promoted into peers.is_anchor and the known_anchors bootstrap cache. Without that rule a single mesh peer could overwrite our stored address for any node (including the DNS-resolved bootstrap anchor), aim thousands of entries at one victim IP, and have every node one bounce out re-announce them.
      • +
    • Sizes are enforced, not merely budgeted Implemented. The uniques opcodes read against a dedicated 2 MB ceiling rather than the 64 MB file-transfer cap; built slices are capped (4,000 shallow / 12,000 terminal); received slices are truncated to a per-bounce accept cap (8,000 at N2 and N3, 24,000 at N4) with a log rather than rejected, so a large-but-honest peer degrades gracefully and a hostile one cannot make us write six figures of rows per announce. A payload whose forwardable pool is not exactly three slices is rejected outright — nothing may ever be stored past bounce 4.
      • +
    • Addresses are filtered on receipt as well as on send Implemented. Inbound anchor addresses are normalized (IPv4-mapped IPv6 → v4) and dropped unless publicly routable; an anchor entry left with no usable address degrades to a bare node entry. Storing a peer-supplied 127.0.0.1 or RFC1918 address would turn the index into attacker-directed internal probing from our own host — and every such dial enters iroh's per-endpoint path store permanently.
      • +
    • Disconnect retention Implemented: when a peer drops, its reported entries are kept. Pool knowledge is overwritten memory — a reporter's rows are replaced (per reporter, per bounce) only when a new handshake from that reporter lands, which is exactly what lets a node knocked down from 20 peers to 4 mine 16 dead peers' pools for anchor addresses to reconnect through (see Anchors). Staleness is bounded by the 5-hour prune, not by disconnection. Wiping on disconnect would empty the index — and collapse the anchor-density prior that steers the adaptive convection weights — at precisely the moment the node needs both most.
    • Refreshes ride the regular update cadence (see Update Cadence & Keep-Alive) so index traffic blends into uniform network overhead.
    @@ -402,7 +409,8 @@

    Because N1 includes directs and CDN contacts (not just the 20 mesh slots), per-node branching runs above 20 and real N4 lists can exceed the pure-mesh bound — the "160k+" is a planning floor, not a ceiling. With reporter tags and SQLite index overhead, expect low tens of MB worst-case on disk. That is acceptable on desktop; if list sizes blow past the budget in practice, the depth default shrinks (<20 peers @ 3 bounces was the explicit fallback in the design ruling).

    Device-tiered depth

    -

    Depth is a per-device choice, not a protocol constant. Desktop and anchor nodes hold the full N1–N4. Mobile may hold only 3 bounces (~8k entries, ~256 KB) or keep N4 as a Bloom filter (~10 bits/entry at 1% false-positive → ~200 KB for 160k entries). A Bloom N4 answers the only question N4 exists for — "can this peer help me reach ID X?" — without supporting enumeration, which N4 never needs since it is never announced.

    +

    Depth is a per-device choice, not a protocol constant. Desktop and anchor nodes hold the full N1–N4. Mobile holds 3 bounces Implemented: it drops the terminal pool on receipt and advertises depth = 3 in its announce, so senders skip building the terminal slice for it — the single biggest bandwidth saving on a cellular link. Mobile still announces its own N3, so it remains a full participant in the network's depth; it only declines to store the terminal layer.

    +

    Bloom-compressed N4 Planned was the alternative (~10 bits/entry at 1% false-positive → ~200 KB for 160k entries) and is not implemented. A Bloom N4 would answer the only question N4 exists for — "can this peer help me reach ID X?" — without supporting enumeration, which N4 never needs since it is never announced. Dropping is the simpler of the two and is what ships.

    Indexed access

    A node has indexed access to a target if the target appears anywhere in its N1–N4: the target is then resolvable by reporter-chain queries alone, with no worm search or relay introduction. The update-cadence system (see Update Cadence & Keep-Alive) uses indexed access to decide whether extra session connections are needed for social and file operations. (This replaces the old "<N4 access" term, which meant "within 3 hops" back when N4 didn't exist as a layer.)

    @@ -425,7 +433,7 @@ 5Relay introduction15sHole punch coordinated via an intermediary (see Relay & NAT Traversal) 6Session relay—Opt-in only, default OFF on both the serving and using side (see Relay & NAT Traversal) -

    Peers that fail all direct steps are marked likely-unreachable, so later attempts skip straight to introduction rather than burning timeouts on dead addresses. The cascade grows one lookup deeper when N4 lands (a 3-link reporter chain); everything else carries over unchanged.

    +

    Peers that fail all direct steps are marked likely-unreachable, so later attempts skip straight to introduction rather than burning timeouts on dead addresses — but only when a reporter was actually available to ask; failing for want of a connected reporter says nothing about the candidate and must not suppress it. Resolution and growth-candidate scoring now share one horizon (bounces 2–4), and depth is a graded penalty in the scorer so a bounce-4 sighting can never outrank a bounce-2 one. The third reporter-chain link — resolving an N4 hit through the reporter's reporter — has not landed Planned: today an N4 entry is resolved by asking its direct reporter, exactly like N2 and N3.

    @@ -1059,10 +1067,10 @@

    Retired type bytes, never reused: 0x42–0x45 (PostNotification/PostPush/AudienceRequest/AudienceResponse, v0.6.2), 0x95 (BlobDeleteNotice — holders evict via LRU, no notice needed), 0xA0 (GroupKeyDistribute — group seeds travel as encrypted posts).

    -

    Pull = uniques-index exchange Rework

    +

    Pull = uniques-index exchange Implemented

    In v0.8 a pull is not a post transfer. A pull exchanges uniques lists: the set of unique IDs each side can help the other reach — mesh peers, direct contacts, CDN file authors, and CDN file peers, including throwaway/anonymous IDs (deliberate noise that protects real identities). The semantics are “if you want one of these IDs, talk to me and I'll help you find their CDN holders to connect to and search.” The union of these lists across the mesh forms a distributed search index — the same index that powers person-discovery for the registry. What a node knows and announces at each depth (N1–N4, termination rules, dedup, size budgets) is specified in Network Knowledge; this section covers only the wire exchange.

    Content itself never rides the pull path: posts and blobs travel through the CDN (PostFetchRequest/BlobRequest against holders resolved through the index — see Content Propagation). The old “never pull from mesh peers” principle dissolves cleanly: the mesh exchanges the index, the CDN carries the content.

    -

    The exact request/response payload shape (announce diffs, Bloom compression for mobile N4) is still open — see Network Knowledge.

    +

    The pull opcodes (0x40/0x41) now carry only the index — that is what makes "a pull is not a post transfer" literally true of them. Post bodies still move between mesh peers over a separate, explicitly transitional content-sync pair (0x46 ContentSyncRequest / 0x47 ContentSyncResponse) Rework, which exists until the CDN cadence work in Iteration D can carry everything. Read caps differ accordingly: the uniques opcodes read against a 2 MB ceiling, not the 64 MB file-transfer cap.

    What it replaces: today PullSyncRequest (0x40) carries a merged query list plus per-author since_ms timestamps (Self Last Encounter), and the responder returns posts newer than each timestamp after should_send_post() visibility filtering, plus visibility updates for own posts the requester already holds. The cycle ticks every 60s: a full pull from all connected peers on the first tick, then pulls only for authors stale beyond 4 hours. This per-author post-pull dies with v0.8; its staleness-tiering idea survives inside the update cadence system.
    @@ -1402,9 +1410,9 @@

    What is never shared

    • Follows are never shared in gossip or profiles. Implemented
      • -
    • Uniques announcements carry bare IDs — except anchors. The uniques announce (see Network Knowledge) ships NodeIds without addresses; the sole exception is anchor-flagged entries, whose addresses ride the pools by design (self-declared reachable infrastructure — publishing them links nothing to any person). Rework (today's N1/N2 share lists are fully address-free; the uniques format adds the anchor exception)
      • -
    • N4 is never re-announced — knowledge terminates at the fourth bounce, so an observer M hops away cannot enumerate your neighborhood through transitive gossip. Planned
      • -
    • Posting identities never map to device addresses on the wire. Addresses attach to network identities only; the persona split (see Identity Architecture) keeps the two namespaces unlinkable. Rework — today's AuthorManifest.author_addresses ships the device's real addresses under a posting-identity author. This is a privacy bug; the v0.8 manifest carries no device addresses, and the fix must land before or with the discovery registry (location-anonymity dependency).
      • +
    • Uniques announcements carry bare IDs — except anchors. The uniques announce (see Network Knowledge) ships NodeIds without addresses; the sole exception is anchor-flagged entries, whose addresses ride the pools by design (self-declared reachable infrastructure — publishing them links nothing to any person). Node-class and author-class IDs travel in separate packed arrays, and the anchor array is the only address-bearing field, so the exception is structural rather than conventional. Implemented
      • +
    • N4 is never re-announced — knowledge terminates at the fourth bounce, so an observer M hops away cannot enumerate your neighborhood through transitive gossip. The announce builder simply never reads bounce-4 rows, and third-party anchor reports are never promoted into our own bounce-1 slice — that promotion would have reset the horizon of every address-bearing entry on every hop, forever. Implemented
      • +
    • Posting identities never map to device addresses on the wire. Addresses attach to network identities only; the persona split (see Identity Architecture) keeps the two namespaces unlinkable. Implemented — AuthorManifest.author_addresses is gone (Iteration A). The same invariant is enforced a second time in the uniques announce (Iteration C): the wire separates node-class from author-class IDs into distinct arrays so an address can never attach to a persona, and our own personas — every one of them, plus the author of any comment we wrote locally, which covers per-greeting throwaway IDs — are excluded from the uniques we announce, because our node ID and anchor address ride the same payload. Our personas stay discoverable through other holders of our content; we must simply never be the reporter. See Network Knowledge.

    Where addresses do travel, and why

    @@ -1416,7 +1424,7 @@

    Uniques lists as cover traffic

    -

    The N1 uniques list deliberately merges mesh peers, social contacts, CDN file authors, CDN file peers, and throwaway/anonymous IDs into one flat set. An observer diffing your announcements over time is looking at a list dominated by high-churn CDN-derived entries and deliberate throwaway-ID noise; the stable social core does not stand out the way it did when the share list was just mesh-plus-contacts. Throwaway IDs additionally age out of the network via comment TTL (above), so the noise floor is self-renewing rather than monotonically accumulating. Planned (today's N1 share merges only mesh peers + social contacts)

    +

    The N1 uniques list deliberately merges mesh peers, social contacts, CDN file authors, CDN file peers, and throwaway/anonymous IDs into one flat set. An observer diffing your announcements over time is looking at a list dominated by high-churn CDN-derived entries and deliberate throwaway-ID noise; the stable social core does not stand out the way it did when the share list was just mesh-plus-contacts. Throwaway IDs additionally age out of the network via comment TTL (above), so the noise floor is self-renewing rather than monotonically accumulating. Our own personas and our own greeting throwaways are the deliberate exception — they are excluded, because we would be the reporter. Implemented

    The timing adversary Planned

    With content encrypted and recipient lists off the wire, the remaining deanonymizer is request-timing correlation: an observer who can watch traffic sees who fetches what, when. If your device checks an author's CDN seconds after every new post, you are that author's reader; if fresh content always radiates outward from one node first, that node is the author. The v0.8 defenses are behavioral, not cryptographic:

    @@ -1892,8 +1900,14 @@ WORM_DEDUP10sIn-flight worm dedup WORM_COOLDOWN300s (5 min)Miss cooldown before retry - REFERRAL_DISCONNECT_GRACE120s (2 min)Rework Current-code registration model: anchor keeps a disconnected caller referable during this grace; retired by the convection window (see Anchors) - REFERRAL_LIST_CAP50Rework Current-code registration model: soft cap on the anchor's registration list, driving per-entry use tiering; retired by the convection window (replacement window size still open) + CONVECTION_WINDOW_SIZE16Implemented Rolling caller window on the anchor. Small on purpose: the point is recency, not coverage. (Retired REFERRAL_DISCONNECT_GRACE 120s and REFERRAL_LIST_CAP 50 from the registration model.) + CONVECTION_HANDOUT_CAP2Implemented Hand-outs before a window entry rotates out ("2 before, 2 after") + CONVECTION_REFERRALS2Implemented Referrals returned per served request + CONVECTION_ENTRY_MAX_AGE15 minImplemented Older window entries are not handed out — a stale address costs the next caller a punch timeout + CONVECTION_SUPPLEMENT_CAP2 per 15 minImplemented Hand-out budget for mesh peers used to supplement a sparse window (address-free, introduction-only) + ANCHOR_REFUSAL_PENALTY5 minImplemented How long a refusing anchor is de-prioritised; the penalty map is swept on every insert + UNIQUES read cap2 MBImplemented Dedicated ceiling for 0x01/0x40/0x41, distinct from the 64 MB file-transfer cap + UNIQUES build / accept caps4,000 & 12,000 / 8,000 & 24,000Implemented Per-slice build caps (shallow, terminal) and per-bounce accept caps (N2–N3, N4); received slices are truncated, not rejected KNOWLEDGE_STALE_PRUNEImmediate on disconnect + 5h fallbackRemove reach entries tagged to disconnected reporters; age fallback for stragglers. Boot sweep clears all non-mesh entries. @@ -1933,8 +1947,8 @@ HTTP_HEADER_TIMEOUT5sHTTP request header read timeout HTTP_REDIRECT_PROBE200msTCP liveness probe of a holder before issuing a 302 - CONVECTION_ACTIONstochastic, per-disconnectPlanned On each mesh disconnect: random {nothing | anchor introduction | mesh-peer introduction}; weights adaptive = local anchor-density prior + refusal feedback - CONVECTION_CLASSentry / top-up (1 bit)Planned Entry (<2 connections) always served; top-up refused cheaply under load — refusal feeds the adaptive weights + CONVECTION_ACTIONstochastic, per-disconnectImplemented On each mesh disconnect: random {nothing | anchor introduction | mesh-peer introduction}; weights adaptive = local anchor-density prior + refusal feedback + CONVECTION_CLASSentry / top-up (1 bit)Implemented Entry (<2 connections) always served; top-up refused cheaply under load — refusal feeds the adaptive weights COMMENT_TTLrand(30–365 days)Planned Comment expiry, fixed at creation and carried with the comment @@ -1950,7 +1964,7 @@ Mesh slots~20 (Desktop) / 15 (Mobile)Single pool, no tiers. Rework — current code runs 101/15 across a Preferred/Local/Wide split - Temp referral slots+4 to +10 above the capPlanned Introduction facilitation only; no uniques exchange; graduate or expire + Temp referral slots+10 desktop / +4 mobile, above the capImplemented Introduction facilitation only; no uniques or peer-address exchange; graduate or expire (5 min TTL) Knowledge depth4 bounces (N4 terminates)Planned Mobile may hold 3 bounces or Bloom-compress N4 @@ -1986,12 +2000,12 @@ AreaStatus Mesh & knowledge - Single ~20-slot mesh pool (tiers eliminated)Rework — code runs Local/Wide, 91 desktop / 12 mobile (preferred tier deleted, Iteration B) + Single ~20-slot mesh pool (tiers eliminated)Implemented — Done (Iteration C): one pool, 20 desktop / 15 mobile Preferred-peer elimination (slots, MeshPrefer, prunes, watchers)Implemented — Done (Iteration B, Roadmap item 3) - Temp referral slots (+4..+10, no knowledge exchange)Planned - Uniques announce, payload N1–N3 (N4 stored but terminates, dedup at store + announce)Rework — code exchanges N1–N3 share lists - Pull as uniques-index exchange (distributed search index)Rework — code pulls posts via since_ms - Growth loop (signal-driven, diversity scoring, stochastic anchor path)Rework — reactive core + scoring shipped; whole-pool counting and the per-disconnect stochastic action are target + Temp referral slots (+4..+10, no knowledge exchange)Implemented — Done (Iteration C): +10 desktop / +4 mobile, above the cap, never evicting a mesh peer + Uniques announce, payload N1–N3 (N4 stored but terminates, dedup at store + announce)Implemented — Done (Iteration C): two-pool announce, terminal N3, single reachable table, size caps enforced + Pull as uniques-index exchange (distributed search index)Implemented — Done (Iteration C); post bodies moved to the transitional 0x46/0x47 content sync + Growth loop (signal-driven, diversity scoring, stochastic anchor path)Implemented — Done (Iteration C): whole-pool counting, bounded SQL shortlist, graded depth penalty, per-disconnect stochastic action Worm search (point queries: posts, blobs, nodes; no keyword flooding)Implemented Social routing cache + checkins + reconnect watchersImplemented — preferred_tree field retires with preferred peers Update-cadence scheduler (freshness × relationship, jitter, dedup)Rework — freshness-only engagement tiers exist @@ -1999,7 +2013,7 @@ Anchors & connectivity Reachability-based anchor candidacy (portmapper watcher, bidirectional)Implemented Anchor opt-in gatePlanned — reachable desktops auto-anchor today - Connection convection (2-before/2-after rolling referrals, per-disconnect stochastic trigger)Rework — register loop + tiered referral list exist; convection semantics are target + Connection convection (2-before/2-after rolling referrals, per-disconnect stochastic trigger)Implemented — Done (Iteration C): rolling window, entry/top-up classes, adaptive weights, 0xC0 register loop retired Batched bootstrap probes + 3-day stale-anchor self-pruneImplemented Anchor self-verification probe (3rd-party witness)Implemented — scheduling re-keys to the watcher in v0.8 Port mapping (UPnP-IGD + NAT-PMP + PCP via portmapper, all platforms)Implemented @@ -2027,7 +2041,7 @@ FoF visibility Layers 1–5 (vouch, gated comments, FoFClosed, rotation/burn, unlock cache)Implemented Sync & content - Wire protocol (length-prefixed JSON, 64 MB payload)Rework — 40 types today; six (0x51/0x52/0x71/0xA1/0xA2/0xB3) deleted in the clean-break purge (Roadmap item 3); 0xC0 AnchorRegister stays live until anchor convection (Roadmap item 5) + Wire protocol (length-prefixed JSON, 64 MB payload)Implemented — six types (0x51/0x52/0x71/0xA1/0xA2/0xB3) deleted in the clean-break purge (Iteration B); 0xC0 retired with anchor convection (Iteration C); the uniques opcodes read against a 2 MB cap Merged pull with recipient matching (all posting identities)Implemented — payload semantics change with the uniques redefinition Manifest machinery (push, refresh, header diffs)Implemented AuthorManifest privacy (no device addresses, 2–5 neighborhood)Rework — code ships addresses + 10+10 neighborhood @@ -2076,11 +2090,11 @@

    4. N1–N4 uniques knowledge layer

    -

    Narrow the mesh to ~20 slots (single pool), add temp referral slots (+4..+10, no knowledge exchange), and replace N1–N3 share lists with the two-pool uniques announce: forwardable N0–N2 + terminal N3, received entries stored shifted one bounce deeper, N4 held but never re-announced. Anchor entries carry addresses (the pools double as the anchor directory); everything else is bare IDs. Redefine pulls as uniques-index exchange. Device-tiered depth and Bloom-compressed N4 for mobile as needed.

    +

    Done (Iteration C) — with size caps enforced, receive-side address filtering, and our own personas excluded from our own announce. Bloom-compressed N4 remains unbuilt; mobile drops the terminal pool instead. Narrow the mesh to ~20 slots (single pool), add temp referral slots (+4..+10, no knowledge exchange), and replace N1–N3 share lists with the two-pool uniques announce: forwardable N0–N2 + terminal N3, received entries stored shifted one bounce deeper, N4 held but never re-announced. Anchor entries carry addresses (the pools double as the anchor directory); everything else is bare IDs. Redefine pulls as uniques-index exchange. Device-tiered depth and Bloom-compressed N4 for mobile as needed.

    5. Anchor convection

    -

    Retire the register loop. Anchor candidacy = reachability watcher + opt-in (opt-in gate is low priority — IPv6-abundant anchors are unlikely to be overwhelmed); the anchor service becomes a rolling referral window (caller gets 2 recent prior callers, is referred to the next 2), with RelayIntroduce coordination when both ends are live, and entry/top-up request classes (entry always served; top-up refused cheaply under load). Clients act stochastically on each mesh disconnect — random {do nothing | anchor introduction | mesh-peer introduction} with adaptive weights (anchor-density prior + refusal feedback). Anchors are mined from the uniques pools; known_anchors recedes to a bootstrap cache.

    +

    Done (Iteration C) — except the opt-in gate, which stays low priority per the ruling. Third-party anchor claims are index rows only; known_anchors and peers.is_anchor are written from proven handshakes. Retire the register loop. Anchor candidacy = reachability watcher + opt-in (opt-in gate is low priority — IPv6-abundant anchors are unlikely to be overwhelmed); the anchor service becomes a rolling referral window (caller gets 2 recent prior callers, is referred to the next 2), with RelayIntroduce coordination when both ends are live, and entry/top-up request classes (entry always served; top-up refused cheaply under load). Clients act stochastically on each mesh disconnect — random {do nothing | anchor introduction | mesh-peer introduction} with adaptive weights (anchor-density prior + refusal feedback). Anchors are mined from the uniques pools; known_anchors recedes to a bootstrap cache.

    6. Post concealment (2–5 budget)