diff --git a/drivers/net/e1000/src/e1000.rs b/drivers/net/e1000/src/e1000.rs index 3c78c58d6..c538a38f5 100644 --- a/drivers/net/e1000/src/e1000.rs +++ b/drivers/net/e1000/src/e1000.rs @@ -2072,7 +2072,7 @@ pub fn probe(device: BusDevice, cap: Cap) -> Result<(), nar // gigabit packet rates. The coalescer excludes loopback and preserves // control/options boundaries; the RX pump flushes it whenever the ring is // drained to empty. - narf_net::tcp_stack::set_gro(true); + let _ = narf_net::iface::set_software_gro(e1000_ifname(), true); // Opt into zero-copy TX (P-F): the stack builds egress frames straight into // a pooled DMA buffer and hands it to `e1000_send_frame_zc`, which DMAs it // without the per-frame slice copy `e1000_send_frame` does. The 64 KiB @@ -2136,7 +2136,7 @@ async fn e1000_rx_pump(device: Arc) { } // Empty ring is the NAPI-complete point for software GRO. This is a - // no-op while GRO remains disabled. + // no-op when this CPU holds no GRO flows. narf_net::tcp_stack::gro_flush(); let missed = device.read_mpc(); if missed > 0 { diff --git a/drivers/net/igc/src/igc.rs b/drivers/net/igc/src/igc.rs index 802b1b559..91273a36e 100644 --- a/drivers/net/igc/src/igc.rs +++ b/drivers/net/igc/src/igc.rs @@ -26,7 +26,11 @@ //! | 0x0008 | STATUS | Device Status | //! | 0x0010 | EEC | EEPROM/Flash Control (I225 specific) | //! | 0x0014 | EERD | EEPROM Read | -//! | 0x00D0 | IMS | Interrupt Mask Set/Read | +//! | 0x1500 | ICR | Interrupt Cause Read | +//! | 0x1508 | IMS | Interrupt Mask Set/Read | +//! | 0x1524 | EIMS | Extended Interrupt Mask Set/Read | +//! | 0x1680 | EITR0 | Interrupt Throttle, vector 0 | +//! | 0x1700 | IVAR0 | Queue interrupt-vector allocation | //! | 0x0100 | RCTL | Receive Control | //! | 0x0400 | TCTL | Transmit Control | //! | 0x2800 | RDBAL/H/LEN/H/T as e1000 | @@ -36,7 +40,7 @@ use core::sync::atomic::{compiler_fence, AtomicU64, Ordering}; -use alloc::sync::Arc; +use alloc::{sync::Arc, vec::Vec}; use narf_bus::{enable_msix, map_bar, BusDevice, BusDeviceCap, MmioRegion, MsixTable}; use narf_capabilities::{Cap, Write}; use narf_io::{alloc_coherent, DmaBuffer}; @@ -96,15 +100,24 @@ pub const IGC_I221_V: u16 = 0x125E; const REG_CTRL: u64 = 0x0000; const REG_STATUS: u64 = 0x0008; /// IGC ICR — Interrupt Cause Read. Reading clears all pending cause -/// bits (Linux `drivers/net/ethernet/intel/igc/igc_defines.h::IGC_ICR`). -const REG_ICR: u64 = 0x00C0; -const REG_IMS: u64 = 0x00D0; +/// bits (Linux `drivers/net/ethernet/intel/igc/igc_regs.h::IGC_ICR`). +const REG_ICR: u64 = 0x1500; +const REG_IMS: u64 = 0x1508; /// IGC IMC — Interrupt Mask Clear (write-1-to-clear). -const REG_IMC: u64 = 0x00D8; +const REG_IMC: u64 = 0x150C; +/// Extended MSI-X interrupt mask registers. +const REG_EIMS: u64 = 0x1524; +const REG_EIMC: u64 = 0x1528; +const REG_EIAC: u64 = 0x152C; +const REG_EIAM: u64 = 0x1530; /// IGC GPIE — General Purpose Interrupt Enable. Bit 4 (NSICR) + /// bit 31 (PBA_support) tell the chip to deliver IRQs as MSI-X /// rather than legacy IMS (Linux `igc_defines.h::IGC_GPIE`). const REG_GPIE: u64 = 0x1514; +/// Per-vector interrupt-throttle register for MSI-X table entry 0. +const REG_EITR0: u64 = 0x1680; +/// Queue-0 RX/TX interrupt-vector allocation register. +const REG_IVAR0: u64 = 0x1700; /// IGC IVAR_MISC — table mapping "other" causes to MSI-X vector /// number (Linux: `IGC_IVAR_MISC`). We pin the "other" causes to /// vector index 0 in single-vector MSI-X mode. @@ -156,6 +169,22 @@ pub const IMS_RXDMT0: u32 = 1 << 4; pub const IMS_RXT0: u32 = 1 << 7; pub const IMS_DEFAULT: u32 = IMS_TXDW | IMS_LSC | IMS_RXO | IMS_RXDMT0 | IMS_RXT0; +/// One shared MSI-X table/vector entry handles RX0, TX0, and other causes. +const MSIX_VECTOR0_MASK: u32 = 1; +const IVAR_VALID: u32 = 0x80; +/// Linux starts igc queue vectors at 648 256-ns units (~6,000 IRQ/s). +const EITR_START: u32 = 648; +const EITR_CNT_IGNR: u32 = 1 << 31; + +const fn ivar0_shared_vector(vector: u8) -> u32 { + let entry = (vector as u32) | IVAR_VALID; + entry | (entry << 8) +} + +const fn ivar_misc_shared_vector(vector: u8) -> u32 { + ((vector as u32) | IVAR_VALID) << 8 +} + // GPIE bits. Single-vector MSI-X needs GPIE.NSICR set so the chip // stops auto-clearing the entire ICR on read (we want the "extended" // MSI-X cause encoding) and GPIE.MULTIPLE_MSIX set so the chip @@ -183,9 +212,13 @@ const TXD_STAT_DD: u8 = 1 << 0; // Ring sizes — small but valid (must be a multiple of 8 per // I225 datasheet §7.2.7). const TX_RING_LEN: usize = 8; -const RX_RING_LEN: usize = 8; +const RX_RING_LEN: usize = 256; const FRAME_SIZE: usize = 2048; +fn previous_rx_slot(next_head: usize) -> usize { + (next_head + RX_RING_LEN - 1) % RX_RING_LEN +} + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum IgcError { BarMapFailed, @@ -518,13 +551,14 @@ impl Igc { return Err(IgcError::ResetTimeout); } - // 2. Mask all interrupt causes during bring-up. IMC is - // write-1-to-clear (Linux: IGC_IMC); writing all-ones - // leaves IMS = 0 and prevents stale IRQs while we program - // the rings. Re-enabled in step 8 once MSI-X is bound. + // 2. Mask legacy and extended interrupt causes during bring-up. + // I225/I226 place these registers in the 0x1500 extended block, + // not at the legacy e1000 0x00c0 aliases. Re-enabled after MSI-X + // routing is fully programmed. // SAFETY: same. unsafe { mmio.write32(REG_IMC, !0u32); + mmio.write32(REG_EIMC, !0u32); // Clear any pending causes by reading ICR (read-to-clear). let _ = mmio.read32(REG_ICR); } @@ -644,12 +678,11 @@ impl Igc { narf_interrupts::install_handler(v, igc_isr); } - // 9. Program GPIE for single-vector MSI-X delivery. Linux: - // `igc_configure_msix` — sets GPIE.NSICR + MULTIPLE_MSIX + - // EIAME + PBA so MSI-X delivery + ICR semantics agree. - // Then unmask the standard RX/TX/LSC/RXO cause set in IMS. - // Skip if no IRQ vector was bound — keep "all masked" so - // polled callers don't see spurious cause bits. + // 9. Program the I225/I226 extended interrupt block. Queue 0 RX and + // TX occupy IVAR0 bytes 0 and 1; "other" causes occupy IVAR_MISC + // byte 1. All three target MSI-X table entry 0. EIAC/EIAM make the + // vector auto-clear + auto-mask on delivery; the RX pump writes + // EIMS after it drains and rechecks the ring. if irq_vector.is_some() { // SAFETY: identity-mapped MMIO. unsafe { @@ -657,11 +690,12 @@ impl Igc { REG_GPIE, GPIE_NSICR | GPIE_MULTIPLE_MSIX | GPIE_EIAME | GPIE_PBA, ); - // Route the "other" causes (link-status etc) to MSI-X - // table[0]. Single-vector mode pins everything at - // index 0. IVAR_MISC bit 7 is the valid bit for the - // low byte; vector index lives in bits[6:0]. - mmio.write32(REG_IVAR_MISC, 0x0000_0080); + mmio.write32(REG_IVAR0, ivar0_shared_vector(0)); + mmio.write32(REG_IVAR_MISC, ivar_misc_shared_vector(0)); + mmio.write32(REG_EITR0, EITR_START | EITR_CNT_IGNR); + mmio.write32(REG_EIAC, MSIX_VECTOR0_MASK); + mmio.write32(REG_EIAM, MSIX_VECTOR0_MASK); + mmio.write32(REG_EIMS, MSIX_VECTOR0_MASK); mmio.write32(REG_IMS, IMS_DEFAULT); } // INTX_DISABLE is already set by the probe path (igc has @@ -711,13 +745,15 @@ impl Igc { let mut msix = enable_msix(cap, device).map_err(|_| IgcError::BarMapFailed)?; let v = narf_interrupts::vector::alloc().map_err(|_| IgcError::BarMapFailed)?; let _ = msix.alloc_vector().ok_or(IgcError::BarMapFailed)?; - // Deliver to APIC id 0 (BSP). On aarch64 this routes through - // the GIC ITS doorbell with EventID=v. + // Route to the CPU performing probe. On aarch64 the helper returns + // the ITS collection target used by the bus MSI layer. + // SAFETY: interrupt topology is initialized before subsystem probes. + let target = unsafe { narf_interrupts::current_cpu_target_id() }; // SAFETY: caller holds the BusDeviceCap; we own the MSI-X // table (no other writer); we issue this write before the // global enable so the device can't fire stale data. // SAFETY: Valid MMIO bounds or trusted driver environment - let _ = unsafe { msix.program_vector(0, 0, v) }.map_err(|_| IgcError::BarMapFailed)?; + let _ = unsafe { msix.program_vector(0, target, v) }.map_err(|_| IgcError::BarMapFailed)?; // SAFETY: cfg-space write to a known cap-list offset. unsafe { msix.enable() }.map_err(|_| IgcError::BarMapFailed)?; Ok((msix, v)) @@ -930,49 +966,87 @@ impl Igc { len } - /// Zero-copy receive (P-F): hand the descriptor's own DMA buffer to the - /// stack as a `Frame` and swap a fresh buffer into its slot — no copy into - /// a scratch buffer and then a freshly-allocated frame, unlike the pump's - /// old `rx` + `Frame::new` + `copy_from_slice`. Returns `None` when no - /// frame is pending, or when the refill allocation fails (the frame is then - /// left in its descriptor for the next poll rather than dropped). - pub fn rx_recv_frame(&self) -> Option { + /// Drain up to `budget` completed RX descriptors, swapping a fresh DMA + /// buffer into each slot and publishing the whole rearmed run with one RDT + /// write. The returned frames own the device-filled buffers. + fn rx_drain_frames(&self, budget: usize) -> Vec { + let mut frames = Vec::new(); + let limit = budget.min(RX_RING_LEN); + if limit == 0 { + return frames; + } let mut head = self.rx_head.lock(); - let idx = *head as usize; + let mut idx = *head as usize; let ring_phys = self.rx_ring_buf.dma_addr().raw(); - let desc_ptr = narf_memory::PhysAddr::new(ring_phys + (idx * 16) as u64) - .kernel_mut_ptr::(); - // SAFETY: identity-mapped DMA; idx < RX_RING_LEN. - let desc = unsafe { core::ptr::read_volatile(desc_ptr) }; - if desc.status_error & ADV_RXD_STAT_DD == 0 { - return None; + while frames.len() < limit { + let desc_ptr = narf_memory::PhysAddr::new(ring_phys + (idx * 16) as u64) + .kernel_mut_ptr::(); + // SAFETY: identity-mapped DMA; idx < RX_RING_LEN. + let desc = unsafe { core::ptr::read_volatile(desc_ptr) }; + if desc.status_error & ADV_RXD_STAT_DD == 0 { + break; + } + let len = (desc.length as usize).min(FRAME_SIZE); + // Refill before handing the filled buffer up so the chip never + // sees an empty slot. On allocation failure, leave this descriptor + // completed and retry it on the next pump pass. + let replacement = match alloc_coherent(FRAME_SIZE, DomainId::DRIVER_0) { + Ok(replacement) => replacement, + Err(_) => break, + }; + let repl_phys = replacement.dma_addr().raw(); + let filled = { + let mut slots = self.rx_buf_pool.lock(); + core::mem::replace(&mut slots[idx], replacement) + }; + let read_form = AdvRxDescRead { + pkt_addr: repl_phys, + hdr_addr: 0, + }; + // SAFETY: identity-mapped DMA; AdvRxDescRead is the same 16 bytes + // as AdvRxDescWb and `idx` is ring-bounded. + unsafe { + core::ptr::write_volatile(desc_ptr as *mut AdvRxDescRead, read_form); + } + frames.push(Frame::new(filled, len as u32)); + idx = (idx + 1) % RX_RING_LEN; } - let len = (desc.length as usize).min(FRAME_SIZE); - // Refill before handing the filled buffer up so the chip never sees an - // empty slot. On allocation failure, bail without advancing. - let replacement = alloc_coherent(FRAME_SIZE, DomainId::DRIVER_0).ok()?; - let repl_phys = replacement.dma_addr().raw(); - let filled = { - let mut slots = self.rx_buf_pool.lock(); - core::mem::replace(&mut slots[idx], replacement) - }; - // Re-hand the slot in the read form (hdr_addr = 0 clears the wb DD bit). - let read_form = AdvRxDescRead { - pkt_addr: repl_phys, - hdr_addr: 0, - }; - // SAFETY: identity-mapped DMA; AdvRxDescRead is the same 16 bytes as - // AdvRxDescWb (the chip selects interpretation via SRRCTL.DESCTYPE). - unsafe { - core::ptr::write_volatile(desc_ptr as *mut AdvRxDescRead, read_form); + if !frames.is_empty() { + *head = idx as u16; + compiler_fence(Ordering::SeqCst); + // SAFETY: identity-mapped MMIO; every descriptor through this + // final slot has been returned to read form above. + unsafe { + self.mmio.write32(REG_RDT, previous_rx_slot(idx) as u32); + } } - *head = ((idx + 1) % RX_RING_LEN) as u16; - compiler_fence(Ordering::SeqCst); - // SAFETY: identity-mapped MMIO. - unsafe { - self.mmio.write32(REG_RDT, (idx % RX_RING_LEN) as u32); + frames + } + + /// Zero-copy single-frame receive compatibility wrapper. + pub fn rx_recv_frame(&self) -> Option { + self.rx_drain_frames(1).into_iter().next() + } + + /// Whether queue 0 has a completed descriptor at the software head. + fn rx_has_pending(&self) -> bool { + let idx = *self.rx_head.lock() as usize; + let ring_phys = self.rx_ring_buf.dma_addr().raw(); + let desc_ptr = + narf_memory::PhysAddr::new(ring_phys + (idx * 16) as u64).kernel_ptr::(); + // SAFETY: identity-mapped DMA; idx < RX_RING_LEN. + unsafe { core::ptr::read_volatile(desc_ptr) }.status_error & ADV_RXD_STAT_DD != 0 + } + + /// Re-enable the shared MSI-X vector after a NAPI-style drain. + fn rearm_msix(&self) { + if self.irq_vector.is_some() { + compiler_fence(Ordering::SeqCst); + // SAFETY: mapped BAR0; vector 0 is the only configured entry. + unsafe { + self.mmio.write32(REG_EIMS, MSIX_VECTOR0_MASK); + } } - Some(Frame::new(filled, len as u32)) } } @@ -1172,15 +1246,46 @@ fn spawn_pumps( } async fn igc_rx_pump(device: Arc, mut rx_prod: Producer) { - // Zero-copy (P-F): take the device-filled DMA buffer straight out of the RX - // descriptor as a Frame (refilling the slot), with no copy through a scratch - // buffer + freshly-allocated frame. + const RX_DRAIN_BUDGET: usize = 64; + const IRQ_FALLBACK_MS: u64 = 10; + let poll_fallback_cycles = narf_time::ns_to_cycles(1_000_000); + + // Snapshot the IRQ fire count before the final descriptor check so an edge + // racing the empty-ring transition cannot be lost. Under load, drain a + // bounded burst and publish its RDT once; on quiescence, rearm MSI-X, + // recheck DD, then park on the prebuilt waiter. Systems without a usable + // vector retain a bounded 1-ms polling fallback. loop { - match device.rx_recv_frame() { - Some(frame) => { - let _ = rx_prod.send(frame).await; + let irq_waiter = device.irq_vector.map(|vector| { + narf_interrupts::wait_for_irq_until( + vector, + narf_time::Deadline::after_ms(IRQ_FALLBACK_MS), + ) + }); + let frames = device.rx_drain_frames(RX_DRAIN_BUDGET); + let drained = frames.len(); + for frame in frames { + if rx_prod.send(frame).await.is_err() { + return; } - None => narf_scheduler::yield_now().await, + } + if drained == RX_DRAIN_BUDGET { + narf_scheduler::yield_now().await; + continue; + } + + device.rearm_msix(); + if device.rx_has_pending() { + // Either a frame raced the rearm or refill allocation temporarily + // failed. Let peers run once before retrying the bounded drain. + narf_scheduler::yield_now().await; + continue; + } + match irq_waiter { + Some(waiter) => { + let _ = waiter.await; + } + None => narf_time::sleep_cycles(poll_fallback_cycles).await, } } } @@ -1193,3 +1298,47 @@ async fn igc_tx_pump(device: Arc, mut tx_cons: Consumer) let _ = device.tx_frame(&frame); } } + +#[cfg(any(test, feature = "builtin"))] +mod tests { + use narf_kernel_test::{kernel_test_in, TestResult}; + + use super::{ + ivar0_shared_vector, ivar_misc_shared_vector, previous_rx_slot, EITR_CNT_IGNR, EITR_START, + REG_EIAC, REG_EIAM, REG_EIMC, REG_EIMS, REG_EITR0, REG_ICR, REG_IMC, REG_IMS, REG_IVAR0, + REG_IVAR_MISC, RX_RING_LEN, + }; + + fn smoke_igc_extended_interrupt_layout() -> TestResult { + if REG_ICR != 0x1500 + || REG_IMS != 0x1508 + || REG_IMC != 0x150c + || REG_EIMS != 0x1524 + || REG_EIMC != 0x1528 + || REG_EIAC != 0x152c + || REG_EIAM != 0x1530 + || REG_EITR0 != 0x1680 + || REG_IVAR0 != 0x1700 + || REG_IVAR_MISC != 0x1740 + { + return TestResult::Fail("igc extended interrupt register offset drift"); + } + if ivar0_shared_vector(0) != 0x0000_8080 || ivar_misc_shared_vector(0) != 0x0000_8000 { + return TestResult::Fail("igc queue/other causes do not target shared vector 0"); + } + if EITR_START != 648 || EITR_CNT_IGNR != 1 << 31 { + return TestResult::Fail("igc initial interrupt throttle drift"); + } + TestResult::Pass + } + kernel_test_in!("drivers/net/igc", smoke_igc_extended_interrupt_layout); + + fn smoke_igc_batched_rx_tail_wrap() -> TestResult { + if RX_RING_LEN != 256 || previous_rx_slot(1) != 0 || previous_rx_slot(0) != RX_RING_LEN - 1 + { + return TestResult::Fail("igc batched RDT publication wraps incorrectly"); + } + TestResult::Pass + } + kernel_test_in!("drivers/net/igc", smoke_igc_batched_rx_tail_wrap); +} diff --git a/drivers/net/specification/high-throughput-datapath.md b/drivers/net/specification/high-throughput-datapath.md index d7481cd51..91a4b961d 100644 --- a/drivers/net/specification/high-throughput-datapath.md +++ b/drivers/net/specification/high-throughput-datapath.md @@ -123,12 +123,12 @@ progress toward line rate. | Driver | Speed | Today | Target work | |---------|-------|-------|-------------| -| e1000 | 1 G | 256-entry pipelined zero-copy TX; 16 KiB IPv4 TSO; unified RX dispatch | multi-queue/GRO if justified; reference-simple, not a high-speed target | -| igc | 2.5 G | basic | batched datapath | +| e1000 | 1 G | 256-entry pipelined zero-copy TX; 64 KiB IPv4 TSO; batched RX + software GRO | reference-simple, not a high-speed target | +| igc | 2.5 G | 256-entry, IRQ-driven, 64-frame batched RX | async/batched TX; multi-queue/RSS | | ixgbe | 10 G | TSO + csum descriptors present, unused by stack | drive TSO/csum from stack; mq/RSS; NAPI | | i40e | 40 G | caps advertise TSO/csum | same + deep rings | | mlx5 | 100 G | RSS RQT present | full mq/RSS + TSO/GRO + moderation; the 100G reference | -| virtio-net | host-bound | mq, NAPI, RSS, mergeable bufs | batched drain + GRO; the in-VM high-throughput path | +| virtio-net | host-bound | mq, NAPI, RSS, mergeable bufs, per-interface GRO | the in-VM high-throughput path | ## 4. Phased implementation (each phase lands and is testable on its own) @@ -144,7 +144,9 @@ progress toward line rate. extensions (P-B). _Landed:_ the `net::gro` engine plus live wiring into `tcp_stack::rx_handler` — per-CPU, iface-tagged, sitting after XDP/AF_PACKET and before L3, flushed at NAPI-complete (`drain_pump` idle, virtio park). - Opt-in (`set_gro`) and loopback-bypassed, so the default path is unchanged. + Opt-in per interface (`iface::set_software_gro`) and loopback-bypassed, so + enabling one driver's poll loop cannot change another interface's receive + semantics. - **P4 — Multi-queue + RSS + NAPI/moderation.** Per-queue contract, per-core pollers, RSS steering, shared NAPI helper, ITR (P-D, P-E). _Landed:_ the first-class contract (`Interface`/`HwNic` `num_rx_queues`/`num_tx_queues`/ diff --git a/drivers/net/specification/spec.md b/drivers/net/specification/spec.md index a31d3df0e..017642633 100644 --- a/drivers/net/specification/spec.md +++ b/drivers/net/specification/spec.md @@ -61,8 +61,10 @@ batch. It retains a bounded polling fallback for lost or unavailable interrupts. The shared completion vector starts with the legacy e1000 Linux policy of 20,000 interrupts/second (ITR/EITR0 interval 195, about 50 microseconds) instead of generating an interrupt for every received frame. -Registering an e1000 enables the stack's bounded software GRO path; the driver -flushes held flows at RX-ring quiescence. +Registering an e1000 opts that interface into the stack's bounded software GRO +path; the driver flushes held flows at RX-ring quiescence. GRO selection is +per-interface, so another NIC sharing the CPU bypasses GRO unless its own RX +poll loop also opts in and supplies a completion flush. For Ethernet/IPv4/TCP, e1000 advertises a bounded 64 KiB TSO frame. The public `TxContextDesc::ipv4_tso` constructor encodes the 8254x context descriptor; @@ -81,6 +83,16 @@ feature assigns the first registered NIC QEMU SLIRP's `10.0.2.15/24` lease and `10.0.2.2` gateway at `Stage::Late`; production networking leaves address selection to userspace. +IGC uses the I225/I226 extended interrupt block (`ICR` 0x1500, `EIMS`/`EIMC`, +`EIAC`/`EIAM`, `EITR0`, `IVAR0` and `IVAR_MISC`) rather than the legacy e1000 +interrupt aliases. Its current single queue has 256 RX descriptors. Queue-0 +RX, TX and other causes share MSI-X entry 0; the initial EITR interval is 648 +256-ns units (Linux's approximately 6,000 interrupts/second starting point). +The RX worker snapshots its IRQ waiter before checking descriptors, drains at +most 64 frames per round, rearms all consumed descriptors with one RDT write, +then reenables and rechecks the vector before parking. A 10-ms lost-interrupt +deadline and 1-ms no-vector polling fallback preserve forward progress. + ### 3.1 i40e runtime profile Each X710/XL710/XXV710/X722 PF negotiates 1–8 64-entry TX/RX pairs. diff --git a/drivers/virtio/specification/spec.md b/drivers/virtio/specification/spec.md index d29faf697..538f7ad5a 100644 --- a/drivers/virtio/specification/spec.md +++ b/drivers/virtio/specification/spec.md @@ -39,6 +39,10 @@ blk/net/console/rng. native parallelism the moment we touch real hardware. `class_blk` and `class_net` modules consume the per-queue rings and present the unified block/net subsystem interfaces. +- The primary virtio-net interface opts into the kernel stack's + per-interface software GRO stage. Every RX forwarder flushes the current + CPU's held flows after its queue drains and before it parks; secondary + frame-ring-only interfaces do not alter the kernel-stack setting. - virtio-gpu exposes `offered_features()`, `host_offers_virgl()`, and `virgl_enabled()`. The first two report the pre-negotiation host offer; `virgl_enabled()` is true only after immutable negotiation accepts diff --git a/drivers/virtio/src/net_pci.rs b/drivers/virtio/src/net_pci.rs index 6903ba6d0..aa1cb759c 100644 --- a/drivers/virtio/src/net_pci.rs +++ b/drivers/virtio/src/net_pci.rs @@ -1587,6 +1587,10 @@ fn register_net_interface(idx: usize, bus_info: alloc::string::String) { let name = narf_net::iface::register_static("eth%d", mac, vnet0_send_fn); // `virtnet_get_drvinfo`: driver "virtio_net", VIRTNET_DRIVER_VERSION. narf_net::iface::set_driver_info(name, "virtio_net", Some("1.0.0"), bus_info); + // The primary RX forwarder drains complete bursts and calls + // `gro_flush` before it parks, so this interface can safely opt into + // the stack's per-interface software GRO path. + let _ = narf_net::iface::set_software_gro(name, true); *PRIMARY_IFNAME.lock() = Some(name); // Opt into zero-copy TX (P-F), sharing this controller's own TX pool so // the stack's `tx_acquire` and the TX reaper use one free-list. 12-byte @@ -1803,7 +1807,8 @@ fn register_net_interface(idx: usize, bus_info: alloc::string::String) { // delivered. Only pair/idx 0 feeds GRO (it owns the synchronous // tap into the stack), so only it needs draining. This runs on // the forwarder's own CPU — the same CPU the offers ran on — so - // it drains that CPU's per-CPU coalescer. No-op when GRO is off. + // it drains that CPU's per-CPU coalescer. No-op when that CPU + // holds no GRO flows. if idx == 0 { narf_net::tcp_stack::gro_flush(); } diff --git a/net/specification/spec.md b/net/specification/spec.md index c12675f65..0b7f1d27c 100644 --- a/net/specification/spec.md +++ b/net/specification/spec.md @@ -122,6 +122,17 @@ when a device advertises the generic limit. is false. These fields describe the in-kernel frame-ring contract; they do not change the stack-daemon attach wire ABI. +Software GRO is an explicit per-interface receive property: + +```rust +pub fn iface::set_software_gro(name: &str, enabled: bool) -> bool; +``` + +New interfaces default to disabled. A driver opts in only when every RX path +feeding that interface flushes the current CPU's GRO table at poll completion. +Enabling one interface never changes receive semantics for another interface; +`NetIfaceSnapshot::software_gro` records the selected state. + ### 3.3 Control-plane operations ```rust diff --git a/net/src/iface.rs b/net/src/iface.rs index fa5fa0939..c063d1b61 100644 --- a/net/src/iface.rs +++ b/net/src/iface.rs @@ -127,6 +127,10 @@ pub struct NetIfaceEntry { /// checksum/TSO offload via [`crate::TxMeta`] instead of doing the work /// in software. pub offloads: crate::OffloadCapabilities, + /// Whether this interface opts into the kernel stack's software GRO stage. + /// This is deliberately per-interface: one GRO-capable driver must not + /// change receive semantics for another driver's poll loop. + pub software_gro: bool, /// Zero-copy TX hook (P-F): `Some` once the driver opts in via /// [`enable_zero_copy_tx`]. When set, the stack builds egress frames /// directly into a buffer from `tx_pool` and hands ownership here instead @@ -187,6 +191,22 @@ pub fn set_offloads(name: &str, offloads: crate::OffloadCapabilities) { } } +/// Enable or disable software GRO for one registered interface. +/// +/// Drivers opt in only when their RX path flushes GRO at poll completion. +/// Returns `false` when `name` is not registered. +pub fn set_software_gro(name: &str, enabled: bool) -> bool { + let mut g = IFACES.lock(); + let Some(entry) = g + .as_mut() + .and_then(|ifaces| ifaces.iter_mut().find(|entry| entry.name == name)) + else { + return false; + }; + entry.software_gro = enabled; + true +} + /// Opt an interface into zero-copy TX (P-F) with an iface-owned recycling pool /// of `cap` buffers of `buf_len` bytes and no head-room — the shape for a /// driver (e1000) whose TX is synchronous and recycles via [`tx_release`]. The @@ -562,6 +582,7 @@ fn register_inner( offloads: old .as_ref() .map_or_else(Default::default, |old| old.offloads), + software_gro: old.as_ref().is_some_and(|old| old.software_gro), // A re-probe preserves any zero-copy TX opt-in (hook + pool + headroom) // so the stack keeps the same recycling pool across a driver reset. send_frame: old.as_ref().and_then(|old| old.send_frame), @@ -1014,6 +1035,8 @@ pub struct NetIfaceSnapshot { /// Offloads this interface's send path honors; the stack requests them /// via [`crate::TxMeta`] rather than computing in software. pub offloads: crate::OffloadCapabilities, + /// Whether ingress on this interface uses software GRO. + pub software_gro: bool, /// Zero-copy TX hook, mirrored from the entry. `Some` ⇒ the stack may hand /// an owned [`crate::Frame`] to [`Self::xmit_frame`]; private so every /// transmit still passes through the snapshot's tap-aware methods. @@ -1136,6 +1159,7 @@ fn snapshot(entry: &NetIfaceEntry) -> NetIfaceSnapshot { duplex: entry.duplex, stats: entry.stats.clone(), offloads: entry.offloads, + software_gro: entry.software_gro, send_frame: entry.send_frame, } } @@ -1628,7 +1652,7 @@ pub fn drain_pump() -> bool { // A round that drained nothing is the end of a receive burst (the // NAPI-complete point): flush any TCP segments GRO is still holding so // coalesced data is never stranded waiting for a next segment that may not - // come. A no-op when GRO is disabled or this CPU holds nothing. + // come. A no-op when this CPU holds no GRO flows. if !any { crate::tcp_stack::gro_flush(); } @@ -1680,4 +1704,23 @@ mod zc_tx_tests { TestResult::Pass } kernel_test_in!("net/iface", smoke_iface_zero_copy_tx_acquire); + + fn smoke_iface_software_gro_is_per_device() -> TestResult { + let enabled = register("gro-enabled-selftest", [2, 0, 0, 0, 0, 2], dummy_send); + let disabled = register("gro-disabled-selftest", [2, 0, 0, 0, 0, 3], dummy_send); + if !set_software_gro(&enabled, true) { + return TestResult::Fail("software GRO opt-in rejected a registered interface"); + } + if lookup(&enabled).is_none_or(|iface| !iface.software_gro) { + return TestResult::Fail("software GRO opt-in was not recorded"); + } + if lookup(&disabled).is_none_or(|iface| iface.software_gro) { + return TestResult::Fail("software GRO leaked to another interface"); + } + if set_software_gro("missing-gro-selftest", true) { + return TestResult::Fail("software GRO accepted an unknown interface"); + } + TestResult::Pass + } + kernel_test_in!("net/iface", smoke_iface_software_gro_is_per_device); } diff --git a/net/src/tcp_stack.rs b/net/src/tcp_stack.rs index 13d6e12bc..be67b3a76 100644 --- a/net/src/tcp_stack.rs +++ b/net/src/tcp_stack.rs @@ -287,11 +287,10 @@ pub fn rx_handler(iface_name: &str, frame: &mut [u8]) { // state across frames, so two paths bypass it and dispatch inline — a // byte-for-byte match for the pre-GRO code: // * loopback, which has no poll loop to flush held flows; and - // * the disabled default (GRO stays opt-in until exercised on real - // multi-queue hardware). + // * interfaces that have not opted in (GRO stays disabled by default). // Linux ref: `napi_gro_receive` sits exactly here — after the `packet_rcv` // AF_PACKET taps and before the protocol handlers. - if gro_rx::enabled() && iface_name != "lo" { + if ingress.as_ref().is_some_and(|dev| dev.software_gro) && iface_name != "lo" { gro_rx::offer(iface_name, net_ns_id, frame); } else { dispatch_l3(_iface_name, net_ns_id, frame); @@ -323,25 +322,15 @@ fn dispatch_l3(iface_name: Option<&str>, net_ns_id: u64, frame: &[u8]) { mod gro_rx { use super::{dispatch_l3, iface, IrqSafeSpinLock}; use crate::gro::Gro; - use core::sync::atomic::{AtomicBool, Ordering}; const MAX_CPUS: usize = narf_lib::percpu::MAX_CPUS; static TABLES: [IrqSafeSpinLock; MAX_CPUS] = [const { IrqSafeSpinLock::new(Gro::new()) }; MAX_CPUS]; - static ENABLED: AtomicBool = AtomicBool::new(false); fn cpu() -> usize { narf_lib::percpu::current_cpu().min(MAX_CPUS - 1) } - pub fn set_enabled(on: bool) { - ENABLED.store(on, Ordering::Relaxed); - } - - pub fn enabled() -> bool { - ENABLED.load(Ordering::Relaxed) - } - /// Offer one received frame to this CPU's coalescer and dispatch whatever /// it releases now (0 when the frame was merged/held, 1–2 on a flush). The /// lock is dropped before dispatch so the protocol handlers never run under @@ -366,16 +355,6 @@ mod gro_rx { } } -/// Enable or disable software GRO (receive coalescing). Off by default. -pub fn set_gro(on: bool) { - gro_rx::set_enabled(on); -} - -/// Whether software GRO is currently enabled. -pub fn gro_enabled() -> bool { - gro_rx::enabled() -} - /// Flush this CPU's held GRO flows. Drivers call this at NAPI-complete / when /// the RX pump parks, so coalesced data is never stranded waiting for the next /// segment that may not arrive.