diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ead9dbcd..c1f82faf 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -8,7 +8,11 @@ env: jobs: build: - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v5 diff --git a/Cargo.lock b/Cargo.lock index 50803d0f..22cc4c15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -258,6 +258,7 @@ dependencies = [ "bytes", "flume", "internet-checksum", + "libc", "log", "pnet", "rand", @@ -596,7 +597,6 @@ dependencies = [ "pretty_env_logger", "socket2", "tokio", - "tokio-tun", "tokio-util", ] diff --git a/README.md b/README.md index 9e9dae2b..1a17fdb5 100644 --- a/README.md +++ b/README.md @@ -12,17 +12,22 @@ A lightweight and fast UDP to TCP obfuscator. * [Overview](#overview) * [Usage](#usage) * [1. Enable Kernel IP forwarding](#1-enable-kernel-ip-forwarding) + * [On Linux](#on-linux) + * [On macOS](#on-macos) * [2. Add required firewall rules](#2-add-required-firewall-rules) * [Client](#client) * [Using nftables](#using-nftables) * [Using iptables](#using-iptables) + * [Using pf (macOS)](#using-pf-macos) * [Server](#server) * [Using nftables](#using-nftables) * [Using iptables](#using-iptables) + * [Using pf (macOS)](#using-pf-macos) * [3. Run Phantun binaries as non-root (Optional)](#3-run-phantun-binaries-as-non-root-optional) * [4. Start Phantun daemon](#4-start-phantun-daemon) * [Server](#server) * [Client](#client) +* [macOS](#macos) * [MTU overhead](#mtu-overhead) * [MTU calculation for WireGuard](#mtu-calculation-for-wireguard) * [Version compatibility](#version-compatibility) @@ -82,7 +87,9 @@ Phantun creates TUN interface for both the Client and Server. For **Client**, Ph `192.168.200.2` and `fcc8::2` by default. For **Server**, it assigns `192.168.201.2` and `fcc9::2` by default. Therefore, your Kernel must have IPv4/IPv6 forwarding enabled and setup appropriate iptables/nftables rules for NAT between your physical -NIC address and Phantun's Tun interface address. +NIC address and Phantun's Tun interface address. On macOS the same applies, with `sysctl(8)` +and `pf(4)` in place of `sysctl.conf` and `iptables`: see +[step 1](#1-enable-kernel-ip-forwarding) and [step 2](#2-add-required-firewall-rules). You may customize the name of Tun interface created by Phantun and the assigned addresses. Please run the executable with `-h` options to see how to change them. @@ -109,6 +116,8 @@ with `-h` to see detailed options on how to control the IPv6 behavior. ## 1. Enable Kernel IP forwarding +### On Linux + Edit `/etc/sysctl.conf`, add `net.ipv4.ip_forward=1` and run `sudo sysctl -p /etc/sysctl.conf`.
@@ -119,6 +128,22 @@ Edit `/etc/sysctl.conf`, add `net.ipv4.ip_forward=1` and run `sudo sysctl -p /et [Back to TOC](#table-of-contents) +### On macOS + +`sysctl(8)` applies the setting immediately, `/etc/sysctl.conf` at the next boot: + +``` +sudo sysctl -w net.inet.ip.forwarding=1 +``` + +
+ IPv6 specific config + + `sudo sysctl -w net.inet6.ip6.forwarding=1` will need to be set as well. +
+ +[Back to TOC](#table-of-contents) + ## 2. Add required firewall rules @@ -156,6 +181,22 @@ ip6tables -t nat -A POSTROUTING -o eth0 -j MASQUERADE [Back to TOC](#table-of-contents) +#### Using pf (macOS) + +With `pf(4)`, SNAT is written as `nat`. Add the rules below to `/etc/pf.conf`, changing `en0` +to whatever actual physical interface name is, and load them with `sudo pfctl -f /etc/pf.conf`. + +`pf` requires translation rules to come before filtering ones, so insert them after the +`nat-anchor`/`rdr-anchor` lines instead of appending them at the end of the file, which fails +with `Rules must be in order`. `sudo pfctl -n -f /etc/pf.conf` checks the file without loading it. + +``` +nat on en0 inet from any to any -> (en0) +nat on en0 inet6 from any to any -> (en0) +``` + +[Back to TOC](#table-of-contents) + ### Server Server needs to DNAT the TCP listening port to Phantun's TUN interface address. @@ -188,6 +229,19 @@ ip6tables -t nat -A PREROUTING -p tcp -i eth0 --dport 4567 -j DNAT --to-destinat [Back to TOC](#table-of-contents) +#### Using pf (macOS) + +With `pf(4)`, DNAT is written as `rdr`. Add the rules below to `/etc/pf.conf`, changing `en0` +to whatever actual physical interface name is, and load them with `sudo pfctl -f /etc/pf.conf`. +See the [Client](#using-pf-macos) rules above for where in the file they belong. + +``` +rdr on en0 inet proto tcp to port 4567 -> 192.168.201.2 port 4567 +rdr on en0 inet6 proto tcp to port 4567 -> fcc9::2 port 4567 +``` + +[Back to TOC](#table-of-contents) + ## 3. Run Phantun binaries as non-root (Optional) It is ill-advised to run network facing applications as root user. Phantun can be run fully @@ -255,6 +309,21 @@ RUST_LOG=info /usr/local/bin/phantun_client --local 127.0.0.1:1234 --remote exam [Back to TOC](#table-of-contents) +# macOS + +Phantun supports macOS by using the `utun(4)` driver in place of Linux's TUN device. The usage +steps above apply, with the following differences: + +* Creating a `utun` interface requires root and macOS has no equivalent of `cap_net_admin`, so + run the binaries with `sudo` and skip [step 3](#3-run-phantun-binaries-as-non-root-optional). +* The interface is called `utunN`, use `--tun utun4` to pin a specific unit. If that unit is + already taken, Phantun fails to start instead of picking another one. +* macOS refuses to connect a second `SO_REUSEPORT` socket to a peer that another one is + already connected to (`EADDRINUSE`), so a connection gets a single fastpath worker + instead of one per core. + +[Back to TOC](#table-of-contents) + # MTU overhead Phantun aims to keep tunneling overhead to the minimum. The overhead compared to a plain UDP packet diff --git a/fake-tcp/Cargo.toml b/fake-tcp/Cargo.toml index cf3757ad..54f47065 100644 --- a/fake-tcp/Cargo.toml +++ b/fake-tcp/Cargo.toml @@ -19,7 +19,12 @@ bytes = "1" pnet = "0" rand = { version = "0", features = ["small_rng"] } internet-checksum = "0" -tokio-tun = "0" flume = "0" tokio = { workspace = true } log = { workspace = true } + +[target.'cfg(not(target_os = "macos"))'.dependencies] +tokio-tun = "0" + +[target.'cfg(target_os = "macos")'.dependencies] +libc = "0.2" diff --git a/fake-tcp/src/lib.rs b/fake-tcp/src/lib.rs index 6a81880e..f732910f 100644 --- a/fake-tcp/src/lib.rs +++ b/fake-tcp/src/lib.rs @@ -41,6 +41,7 @@ #![cfg_attr(feature = "benchmark", feature(test))] pub mod packet; +pub mod tun; use bytes::{Bytes, BytesMut}; use log::{error, info, trace, warn}; @@ -57,7 +58,7 @@ use std::sync::{ use tokio::sync::broadcast; use tokio::sync::mpsc; use tokio::time; -use tokio_tun::Tun; +use tun::Tun; const TIMEOUT: time::Duration = time::Duration::from_secs(1); const RETRIES: usize = 6; @@ -357,8 +358,8 @@ impl fmt::Display for Socket { /// A userspace TCP state machine impl Stack { - /// Create a new stack, `tun` is an array of [`Tun`](tokio_tun::Tun). - /// When more than one [`Tun`](tokio_tun::Tun) object is passed in, same amount + /// Create a new stack, `tun` is an array of [`Tun`](crate::tun::Tun). + /// When more than one [`Tun`](crate::tun::Tun) object is passed in, same amount /// of reader will be spawned later. This allows user to utilize the performance /// benefit of Multiqueue Tun support on machines with SMP. pub fn new(tun: Vec, local_ip: Ipv4Addr, local_ip6: Option) -> Stack { diff --git a/fake-tcp/src/tun.rs b/fake-tcp/src/tun.rs new file mode 100644 index 00000000..164a38bf --- /dev/null +++ b/fake-tcp/src/tun.rs @@ -0,0 +1,305 @@ +//! TUN interface abstraction. +//! +//! Everywhere except macOS this is a re-export of [`tokio_tun`]. macOS's `utun(4)` +//! driver is not a character device but a kernel control socket, so it gets its own +//! implementation here. Both expose the same API for the [`Stack`](crate::Stack). + +#[cfg(not(target_os = "macos"))] +mod platform { + pub use tokio_tun::{Tun, TunBuilder}; +} + +#[cfg(target_os = "macos")] +mod platform { + use std::io; + use std::mem; + use std::net::Ipv4Addr; + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; + use std::process::Command; + use tokio::io::unix::AsyncFd; + + /// `_IOWR('N', 3, struct ctl_info)`, from ``. + const CTLIOCGINFO: libc::c_ulong = 0xc064_4e03; + + /// `UTUN_OPT_IFNAME` and the kernel control it belongs to, from ``. + const UTUN_OPT_IFNAME: libc::c_int = 2; + const UTUN_CONTROL_NAME: &[u8] = b"com.apple.net.utun_control"; + + /// A macOS `utun(4)` interface. + /// + /// Unlike Linux's Tun this is a socket, and every packet read from or written to + /// it is prefixed with a 4 byte address family in network byte order. That prefix + /// is handled here so that callers only ever deal in bare IP packets. + pub struct Tun { + name: String, + io: AsyncFd, + } + + impl Tun { + /// Receives a packet from the utun interface. + pub async fn recv(&self, buf: &mut [u8]) -> io::Result { + loop { + let mut guard = self.io.readable().await?; + match guard.try_io(|inner| recv_packet(inner.as_raw_fd(), buf)) { + Ok(res) => return res, + Err(_) => continue, + } + } + } + + /// Sends a packet to the utun interface. + pub async fn send(&self, buf: &[u8]) -> io::Result { + loop { + let mut guard = self.io.writable().await?; + match guard.try_io(|inner| send_packet(inner.as_raw_fd(), buf)) { + Ok(res) => return res, + Err(_) => continue, + } + } + } + + /// Tries to send a packet to the utun interface. + /// + /// When the interface queue is full, `Err(io::ErrorKind::WouldBlock)` is returned. + pub fn try_send(&self, buf: &[u8]) -> io::Result { + send_packet(self.io.as_raw_fd(), buf) + } + + /// Returns the name of the utun interface, for example `utun4`. + pub fn name(&self) -> &str { + &self.name + } + } + + /// Builds a [`Tun`], mirroring the subset of `tokio_tun::TunBuilder` that Phantun uses. + #[derive(Default)] + pub struct TunBuilder { + name: String, + address: Option, + destination: Option, + up: bool, + } + + impl TunBuilder { + pub fn new() -> Self { + Self::default() + } + + /// Sets the name of device, if it is empty, then the device name is set by the kernel. + pub fn name(mut self, name: &str) -> Self { + self.name = name.into(); + self + } + + /// Sets the IPv4 address of the device. + pub fn address(mut self, address: Ipv4Addr) -> Self { + self.address = Some(address); + self + } + + /// Sets the IPv4 destination (peer) address of the device. + pub fn destination(mut self, destination: Ipv4Addr) -> Self { + self.destination = Some(destination); + self + } + + /// Sets up the device. + pub fn up(mut self) -> Self { + self.up = true; + self + } + + /// Accepted for compatibility with `tokio_tun::TunBuilder`, utun is single queue + /// so a single [`Tun`] is always built. + pub fn queues(self, _queues: usize) -> Self { + self + } + + /// Builds a new instance of [`Tun`]. + pub fn build(self) -> io::Result> { + let (fd, name) = open_utun(&self.name)?; + + // Sets the addresses and brings the interface up, the equivalent of what + // tokio-tun does with netlink on Linux. ifconfig(8) is used instead of the + // SIOCSIFADDR family of ioctls to keep the unsafe surface down to the utun + // socket itself. + let mut ifconfig = Command::new("/sbin/ifconfig"); + ifconfig.arg(&name); + match (self.address, self.destination) { + // Netmask has to be explicit, otherwise ifconfig picks one from the + // address class and installs a route for the whole subnet. + (Some(address), Some(destination)) => { + ifconfig + .arg(address.to_string()) + .arg(destination.to_string()) + .args(["netmask", "255.255.255.255"]); + } + (Some(address), None) => { + ifconfig.arg(address.to_string()); + } + (None, _) => {} + } + if self.up { + ifconfig.arg("up"); + } + let output = ifconfig.output()?; + if !output.status.success() { + return Err(io::Error::other(format!( + "unable to configure {}: {}", + name, + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + + Ok(vec![Tun { + name, + io: AsyncFd::new(fd)?, + }]) + } + } + + /// Creates a utun interface and returns its file descriptor and kernel assigned name. + /// + /// Creating a utun interface requires root. + fn open_utun(name: &str) -> io::Result<(OwnedFd, String)> { + // an empty name lets the kernel pick the first free interface. Control units are + // numbered from 1 while interface units are numbered from 0, hence the + 1. + let unit = if name.is_empty() { + 0 + } else { + match name + .strip_prefix("utun") + .and_then(|unit| unit.parse::().ok()) + { + Some(unit) => unit + 1, + None => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} is not a valid utun interface name"), + )); + } + } + }; + + let fd = unsafe { libc::socket(libc::PF_SYSTEM, libc::SOCK_DGRAM, libc::SYSPROTO_CONTROL) }; + if fd < 0 { + return Err(io::Error::last_os_error()); + } + let fd = unsafe { OwnedFd::from_raw_fd(fd) }; + + let mut info: libc::ctl_info = unsafe { mem::zeroed() }; + for (dst, src) in info.ctl_name.iter_mut().zip(UTUN_CONTROL_NAME) { + *dst = *src as libc::c_char; + } + if unsafe { libc::ioctl(fd.as_raw_fd(), CTLIOCGINFO, &mut info) } < 0 { + return Err(io::Error::last_os_error()); + } + + let addr = libc::sockaddr_ctl { + sc_len: mem::size_of::() as libc::c_uchar, + sc_family: libc::AF_SYSTEM as libc::c_uchar, + ss_sysaddr: libc::AF_SYS_CONTROL as u16, + sc_id: info.ctl_id, + sc_unit: unit, + sc_reserved: [0; 5], + }; + let sockaddr = std::ptr::from_ref(&addr).cast::(); + if unsafe { + libc::connect( + fd.as_raw_fd(), + sockaddr, + mem::size_of_val(&addr) as libc::socklen_t, + ) + } < 0 + { + return Err(io::Error::last_os_error()); + } + + let mut name = [0u8; libc::IF_NAMESIZE]; + let mut len = name.len() as libc::socklen_t; + if unsafe { + libc::getsockopt( + fd.as_raw_fd(), + libc::SYSPROTO_CONTROL, + UTUN_OPT_IFNAME, + name.as_mut_ptr().cast(), + &mut len, + ) + } < 0 + { + return Err(io::Error::last_os_error()); + } + // the returned length includes the terminating NUL + let name = String::from_utf8_lossy(&name[..len.saturating_sub(1) as usize]).into_owned(); + + // AsyncFd requires a non-blocking file descriptor + let flags = unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFL) }; + if flags < 0 + || unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_SETFL, flags | libc::O_NONBLOCK) } < 0 + { + return Err(io::Error::last_os_error()); + } + + Ok((fd, name)) + } + + /// Reads a single packet, dropping the utun address family prefix. + fn recv_packet(fd: RawFd, buf: &mut [u8]) -> io::Result { + let mut af = [0u8; 4]; + let mut iov = [ + libc::iovec { + iov_base: af.as_mut_ptr().cast(), + iov_len: af.len(), + }, + libc::iovec { + iov_base: buf.as_mut_ptr().cast(), + iov_len: buf.len(), + }, + ]; + let size = unsafe { libc::readv(fd, iov.as_mut_ptr(), iov.len() as libc::c_int) }; + if size < 0 { + return Err(io::Error::last_os_error()); + } + if (size as usize) < af.len() { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "truncated utun packet", + )); + } + + Ok(size as usize - af.len()) + } + + /// Writes a single packet, prefixing it with the utun address family header. + fn send_packet(fd: RawFd, buf: &[u8]) -> io::Result { + let af = match buf.first().map(|version| version >> 4) { + Some(4) => libc::AF_INET, + Some(6) => libc::AF_INET6, + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "not an IP packet", + )); + } + } + .to_be_bytes(); + let iov = [ + libc::iovec { + iov_base: af.as_ptr() as *mut libc::c_void, + iov_len: af.len(), + }, + libc::iovec { + iov_base: buf.as_ptr() as *mut libc::c_void, + iov_len: buf.len(), + }, + ]; + let size = unsafe { libc::writev(fd, iov.as_ptr(), iov.len() as libc::c_int) }; + if size < 0 { + return Err(io::Error::last_os_error()); + } + + Ok(size as usize - af.len()) + } +} + +pub use platform::{Tun, TunBuilder}; diff --git a/phantun/Cargo.toml b/phantun/Cargo.toml index 8b5f317c..91777113 100644 --- a/phantun/Cargo.toml +++ b/phantun/Cargo.toml @@ -16,9 +16,10 @@ socket2 = { version = "0", features = ["all"] } fake-tcp = { path = "../fake-tcp", version = "0" } tokio-util = "0" pretty_env_logger = "0" -tokio-tun = "0" num_cpus = "1" -neli = "0" nix = { version = "0", features = ["net", "uio", "socket"] } tokio = { workspace = true } log = { workspace = true } + +[target.'cfg(not(target_os = "macos"))'.dependencies] +neli = "0" diff --git a/phantun/src/bin/client.rs b/phantun/src/bin/client.rs index 9d539d87..bdedfe24 100644 --- a/phantun/src/bin/client.rs +++ b/phantun/src/bin/client.rs @@ -1,8 +1,9 @@ use clap::{crate_version, Arg, ArgAction, Command}; use fake_tcp::packet::MAX_PACKET_LEN; +use fake_tcp::tun::TunBuilder; use fake_tcp::{Socket, Stack}; use log::{debug, error, info}; -use phantun::utils::{assign_ipv6_address, new_udp_reuseport, udp_recv_pktinfo}; +use phantun::utils::{assign_ipv6_address, new_udp_reuseport, udp_fastpath_workers, udp_recv_pktinfo}; use std::collections::HashMap; use std::fs; use std::io; @@ -10,7 +11,6 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4, SocketAddrV6}; use std::sync::Arc; use tokio::sync::{Notify, RwLock}; use tokio::time; -use tokio_tun::TunBuilder; use tokio_util::sync::CancellationToken; use phantun::UDP_TTL; @@ -220,7 +220,7 @@ async fn main() -> io::Result<()> { let packet_received = Arc::new(Notify::new()); let quit = CancellationToken::new(); - for i in 0..num_cpus { + for i in 0..udp_fastpath_workers() { let sock = sock.clone(); let quit = quit.clone(); let packet_received = packet_received.clone(); @@ -259,7 +259,14 @@ async fn main() -> io::Result<()> { loop { tokio::select! { - Ok(size) = udp_sock.recv(&mut buf_udp) => { + Ok((size, src_addr)) = udp_sock.recv_from(&mut buf_udp) => { + // macOS delivers datagrams addressed to another client to a + // socket that is bound but not yet connected, see README + if src_addr != udp_remote_addr { + debug!("Dropped datagram from {} addressed to {}", src_addr, udp_remote_addr); + continue; + } + if sock.send(&buf_udp[..size]).await.is_none() { debug!("removed fake TCP socket from connections table"); quit.cancel(); diff --git a/phantun/src/bin/server.rs b/phantun/src/bin/server.rs index 960fb995..40d8d1a0 100644 --- a/phantun/src/bin/server.rs +++ b/phantun/src/bin/server.rs @@ -1,8 +1,9 @@ use clap::{crate_version, Arg, ArgAction, Command}; use fake_tcp::packet::MAX_PACKET_LEN; use fake_tcp::Stack; +use fake_tcp::tun::TunBuilder; use log::{debug, error, info}; -use phantun::utils::{assign_ipv6_address, new_udp_reuseport}; +use phantun::utils::{assign_ipv6_address, new_udp_reuseport, udp_fastpath_workers}; use std::fs; use std::io; use std::net::Ipv4Addr; @@ -10,7 +11,6 @@ use std::sync::Arc; use tokio::net::UdpSocket; use tokio::sync::Notify; use tokio::time; -use tokio_tun::TunBuilder; use tokio_util::sync::CancellationToken; use phantun::UDP_TTL; @@ -197,7 +197,7 @@ async fn main() -> io::Result<()> { let local_addr = udp_sock.local_addr()?; drop(udp_sock); - for i in 0..num_cpus { + for i in 0..udp_fastpath_workers() { let sock = sock.clone(); let quit = quit.clone(); let packet_received = packet_received.clone(); diff --git a/phantun/src/utils.rs b/phantun/src/utils.rs index 1b68c1fe..4f355b1d 100644 --- a/phantun/src/utils.rs +++ b/phantun/src/utils.rs @@ -1,3 +1,4 @@ +#[cfg(not(target_os = "macos"))] use neli::{ consts::{ nl::NlmF, @@ -15,9 +16,23 @@ use nix::sys::socket::{ }; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}; use std::os::unix::io::AsRawFd; +#[cfg(target_os = "macos")] +use std::process::Command; use tokio::io::Interest; use tokio::net::UdpSocket; +/// Number of "fastpath" worker tasks to spawn for a single UDP connection. +/// +/// macOS refuses to connect a second socket bound to the same address to a peer that +/// another one is already connected to, so a connection gets a single worker there. +pub fn udp_fastpath_workers() -> usize { + if cfg!(target_os = "macos") { + 1 + } else { + num_cpus::get() + } +} + pub fn new_udp_reuseport(local_addr: SocketAddr) -> UdpSocket { let udp_sock = socket2::Socket::new( if local_addr.is_ipv4() { @@ -112,6 +127,7 @@ fn dst_addr_from_cmsgs(cmsgs: CmsgIterator) -> Option { None } +#[cfg(not(target_os = "macos"))] pub fn assign_ipv6_address(device_name: &str, local: Ipv6Addr, peer: Ipv6Addr) { let index = nix::net::if_::if_nametoindex(device_name).unwrap(); @@ -150,6 +166,27 @@ pub fn assign_ipv6_address(device_name: &str, local: Ipv6Addr, peer: Ipv6Addr) { rtnl.send(&nl_header).unwrap(); } +#[cfg(target_os = "macos")] +pub fn assign_ipv6_address(device_name: &str, local: Ipv6Addr, peer: Ipv6Addr) { + let output = Command::new("/sbin/ifconfig") + .args([ + device_name, + "inet6", + &local.to_string(), + &peer.to_string(), + "prefixlen", + "128", + ]) + .output() + .expect("unable to run ifconfig"); + assert!( + output.status.success(), + "unable to assign IPv6 address to {}: {}", + device_name, + String::from_utf8_lossy(&output.stderr).trim() + ); +} + const fn max_usize(a: usize, b: usize) -> usize { if a > b { a } else { b } }