diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0188b5..9a87d50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,83 +1,149 @@ name: Continuous Integration on: + merge_group: push: - branches: [master] + branches: [main] pull_request: - branches: [master] + branches: [main] + schedule: + - cron: "17 5 * * 1" env: CARGO_TERM_COLOR: always +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: style: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@stable with: - components: rustfmt, clippy + components: clippy,rustfmt + - uses: Swatinem/rust-cache@v2 + - run: cargo fmt --all -- --check + - run: cargo clippy --workspace --all-targets --all-features -- -D warnings + - run: RUSTDOCFLAGS="-D warnings" cargo doc --workspace --all-features --no-deps - - run: cargo fmt -- --check - - run: cargo check - - run: cargo clippy --tests --examples - - documentation: + semver: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - - - uses: dtolnay/rust-toolchain@stable - - - run: cargo doc + - uses: actions/checkout@v6 + - uses: obi1kenobi/cargo-semver-checks-action@v2 + with: + package: minimq,mqtt-staging - audit: + python: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - - uses: actions/checkout@v4 - - uses: taiki-e/install-action@cargo-audit - - run: cargo audit + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: | + mqtt-rpc/py/pyproject.toml + mqtt-staging/py/pyproject.toml + - run: python -m pip install -e './mqtt-rpc/py[dev,test]' -e './mqtt-staging/py[dev]' + - run: python -m compileall -q mqtt_rpc tests + working-directory: mqtt-rpc/py + - run: python -m ruff format --check mqtt_rpc tests + working-directory: mqtt-rpc/py + - run: python -m ruff check mqtt_rpc tests + working-directory: mqtt-rpc/py + - run: python -m pytest -q -o cache_dir=/tmp/mqtt-rpc-pytest tests + working-directory: mqtt-rpc/py + - run: python -m compileall -q mqtt_staging.py tests + working-directory: mqtt-staging/py + - run: python -m ruff format --check mqtt_staging.py tests + working-directory: mqtt-staging/py + - run: python -m ruff check mqtt_staging.py tests + working-directory: mqtt-staging/py + - run: python -m unittest discover -s tests -p 'test_*.py' + working-directory: mqtt-staging/py - compile: + msrv: runs-on: ubuntu-latest + timeout-minutes: 10 strategy: matrix: - toolchain: - - stable - - beta + include: + - toolchain: "1.88.0" + packages: -p minimq -p mqtt-rpc + - toolchain: "1.93.0" + packages: -p mqtt-staging steps: - - uses: actions/checkout@v4 - - name: Install Rust ${{ matrix.toolchain }} - uses: dtolnay/rust-toolchain@master + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ matrix.toolchain }} - target: thumbv7em-none-eabihf - - run: cargo check --verbose - - run: cargo build - - run: cargo build --release - - run: cargo build --examples + targets: thumbv7em-none-eabihf + - uses: Swatinem/rust-cache@v2 + - run: cargo check ${{ matrix.packages }} --lib + - run: cargo check ${{ matrix.packages }} --lib --target thumbv7em-none-eabihf test: runs-on: ubuntu-latest - strategy: - matrix: - toolchain: - - stable + timeout-minutes: 15 + env: + BROKER: 127.0.0.1:1883 steps: - - uses: actions/checkout@v4 - - - name: Start Mosquitto + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: | + mqtt-rpc/py/pyproject.toml + mqtt-staging/py/pyproject.toml + - run: python -m pip install './mqtt-rpc/py[test]' ./mqtt-staging/py + - name: Start MQTT broker run: | sudo apt-get update sudo apt-get install -y mosquitto mosquitto-clients sudo service mosquitto start + - run: cargo test --workspace --all-targets --all-features + - run: python -m pytest -q -o cache_dir=/tmp/mqtt-rpc-pytest mqtt-rpc/py/tests + - run: cargo test -p mqtt-staging --test end_to_end -- --ignored - - name: Install Rust ${{ matrix.toolchain }} - uses: dtolnay/rust-toolchain@master + package: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + - uses: actions/setup-python@v6 with: - toolchain: ${{ matrix.toolchain }} + python-version: "3.11" + cache: pip + cache-dependency-path: | + mqtt-rpc/py/pyproject.toml + mqtt-staging/py/pyproject.toml + - run: cargo package -p minimq + - run: cargo package -p mqtt-rpc + - run: cargo package -p mqtt-staging + - run: python -m pip install build + - run: python -m build mqtt-rpc/py + - run: python -m build mqtt-staging/py - - run: cargo test + audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: taiki-e/install-action@cargo-audit + - run: cargo audit fuzz: runs-on: ubuntu-latest @@ -88,9 +154,7 @@ jobs: - uses: taiki-e/install-action@v2 with: tool: cargo-fuzz - - name: Install nightly Rust - uses: dtolnay/rust-toolchain@nightly - + - uses: dtolnay/rust-toolchain@nightly - run: cargo +nightly fuzz run fuzz_received_packet --target host-tuple -- -dict=dictionary.txt -max_total_time=3 working-directory: fuzz - run: cargo +nightly fuzz run fuzz_packet_reader --target host-tuple -- -dict=dictionary.txt -max_total_time=3 diff --git a/.gitignore b/.gitignore index 431557f..ba072bd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ -ejabberd/*/ -/target +target/ +build/ +dist/ Cargo.lock diff --git a/Cargo.toml b/Cargo.toml index adc50c0..fbf4d03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,49 +1,12 @@ -[package] -name = "minimq" -version = "0.13.0" -authors = [ - "Ryan Summers ", - "Robert Jördens ", - "Max Rottenkolber ", -] +[workspace] +members = ["minimq", "mqtt-rpc", "mqtt-staging"] +resolver = "3" + +[workspace.package] edition = "2024" -description = "An MQTT5 client" -homepage = "https://github.com/quartiq/minimq" -repository = "https://github.com/quartiq/minimq" -documentation = "https://docs.rs/minimq" -readme = "README.md" -categories = ["embedded", "no-std", "database", "encoding"] -keywords = ["mqtt", "embedded", "client"] license = "MIT" -rust-version = "1.85.0" - -[features] -default = ["defmt"] -defmt = [ - "dep:defmt", - "heapless/defmt", - "embedded-io-async/defmt", - "embassy-time/defmt", -] -fuzzing = [] - -[dependencies] -defmt = { version = "1.0.1", optional = true } -num_enum = { version = "0.7", default-features = false } -heapless = "0.9.1" -embedded-io-async = "0.7" -embassy-time = "0.5.1" -serde = { version = "1", features = ["derive"], default-features = false } -thiserror = { version = "2", default-features = false } - -[dev-dependencies] -defmt2log = "0.1.0" -env_logger = "0.11" -embassy-time = { version = "0.5.1", features = ["std", "generic-queue-8"] } -embedded-tls = "0.18.0" -embedded-io-adapters = { version = "0.7.0", features = ["tokio-1"] } -rand = "0.8" -tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "time"] } +repository = "https://github.com/quartiq/minimq.git" +rust-version = "1.88" -[[example]] -name = "tls_public_broker" +[workspace.lints.rust] +unsafe_code = "forbid" diff --git a/README.md b/README.md index 8110494..6726596 100644 --- a/README.md +++ b/README.md @@ -1,169 +1,20 @@ -[![QUARTIQ Matrix Chat](https://img.shields.io/matrix/quartiq:matrix.org)](https://matrix.to/#/#quartiq:matrix.org) -[![Continuous Integration](https://github.com/quartiq/minimq/actions/workflows/ci.yml/badge.svg)](https://github.com/quartiq/minimq/actions/workflows/ci.yml) +# Minimq workspace -# Minimq +Opinionated, allocation-free MQTT 5 components for embedded systems: -`minimq` is a small `no_std`, no-alloc, `async` MQTT v5 client for embedded systems. +- [`minimq`](minimq/) is the async MQTT 5 client. +- [`mqtt-rpc`](mqtt-rpc/) is a minimal MQTT 5 request/reply protocol with a + matching Python client. +- [`mqtt-staging`](mqtt-staging/) stages bounded objects into + application-owned storage and includes a matching Python sender. -Use it when your application already has async network I/O and needs one long-lived MQTT session -with explicit buffers and reconnect handling. +Minimq owns MQTT session mechanics, not the network stack or application. The +protocol crates likewise leave dispatch, storage, activation, and reboot policy +to their applications. Each package has an independent version and release +boundary. -The main API is [`Session`]. +## Releasing -## What You Use - -- [`Buffers`]: caller-owned RX/TX memory -- [`ConfigBuilder`]: session configuration -- [`Disconnect`]: graceful disconnect options -- [`Io`]: transport boundary for an established byte stream -- [`Session`]: the client you drive -- [`InboundPublish`]: output of [`Connection::recv()`] - -## Example - -```no_run -# use std::io; -# struct MyIo; -# use embedded_io_async::{ErrorType, Read, Write}; -# impl ErrorType for MyIo { -# type Error = io::Error; -# } -# impl Read for MyIo { -# async fn read(&mut self, _buf: &mut [u8]) -> Result { -# todo!() -# } -# } -# impl Write for MyIo { -# async fn write(&mut self, _buf: &[u8]) -> Result { -# todo!() -# } -# async fn flush(&mut self) -> Result<(), Self::Error> { -# todo!() -# } -# } -# async fn open_io(_addr: SocketAddr) -> Result { todo!() } -use core::net::SocketAddr; -use minimq::{Buffers, ConfigBuilder, ConnectEvent, Error, Session, TopicFilter}; - -async fn run() { - let rx = &mut [0u8; 256]; - let tx = &mut [0u8; 768]; - let addr: SocketAddr = "127.0.0.1:1883".parse().unwrap(); - let mut session = Session::new( - ConfigBuilder::new(Buffers::new(rx, tx)) - .client_id("demo") - .unwrap(), - ); - - loop { - let io = open_io(addr).await.unwrap(); - // `connect` returns a connection handle that owns the transport and borrows - // the session. Dropping it at the end of the loop releases both for the next - // reconnect. - let mut conn = session.connect(io).await.unwrap(); - match conn.connect_event() { - ConnectEvent::Connected => { - conn.subscribe(&[TopicFilter::new("demo/in")], &[]) - .await - .unwrap(); - } - ConnectEvent::Reconnected => {} - } - - loop { - match conn.recv().await { - Ok(message) => println!("topic={}", message.topic()), - Err(Error::Disconnected) => break, - Err(err) => panic!("{err}"), - } - } - } -} - -# fn main() {} -``` - -The attached transport must implement [`embedded_io_async::Read`] and -[`embedded_io_async::Write`]. -Ordinary lack of inbound data must keep the read future pending; if the transport returns -`TimedOut` or `Interrupted`, [`Connection::poll()`] treats that as transport failure and -disconnects the connection. - -For a TLS MQTT v5 request/reply example that preserves a subscription across reconnects and reuses -the TLS record buffers, see `examples/tls_public_broker.rs`. - -## Errors - -`ConfigBuilder` reports setup-time validation failures through [`ConfigError`]. -Connected session operations report [`Error`]: -- broker rejections and invalid inbound MQTT data surface as [`PeerError`] -- local buffer and capacity limits surface as [`ResourceError`] -- transport failures surface as [`Error::Transport`] - -## Session Model - -You provide packet buffers plus an already-established transport, and a loop that explicitly -passes that transport into [`Session::connect()`] to establish or resume the broker session. - -[`Session::connect()`] takes ownership of the provided transport and performs the unbounded MQTT -`CONNECT` / `CONNACK` handshake. It returns a [`Connection`] that borrows the session. Once -connected: -- [`Connection::recv()`] blocks until the next inbound publish arrives or the connection is lost. -- [`Connection::poll()`] blocks until any session progress happens and returns `Ok(None)` for - internal-only progress such as ACK handling, replay, or keepalive traffic. - -Dropping the connection releases the session for a later reconnect. Call -[`Connection::disconnect()`] first for a graceful MQTT close. - -- [`ConnectEvent::Connected`] means the broker created a fresh session. Re-establish subscriptions - here. -- [`ConnectEvent::Reconnected`] means the broker resumed the existing MQTT session. Existing - subscriptions and in-flight QoS state were kept. -- [`Connection::recv()`] yields one inbound publish. - -If [`Connection::recv()`] or [`Connection::poll()`] returns [`Error::Disconnected`], the caller -discards the handle and decides -when to call -[`Session::connect()`] with a fresh transport again. -Other transport/protocol errors mark the handle dead; callers should handle the error and reconnect -rather than retrying network operations on that handle. - -For cooperative driving: -- use [`Connection::drive()`] for immediate local progress without waiting for future inbound reads - or future session deadlines -- wrap cancel-safe blocking [`Connection::poll()`] or [`Connection::recv()`] in an external timeout - such as [`embassy_time::with_timeout()`] or [`embassy_time::with_deadline()`] -- if you need real wall-clock limits, enforce them in the transport's `read`, `write`, and - `flush` futures; using the same budget as minimq's internal MQTT round-trip timeout keeps - keepalive and transport liveness aligned - -## Buffers - -You supply two buffers. - -- `rx` stores one inbound MQTT packet at a time. Size it for the largest inbound publish, - including topic, properties, and payload. -- `tx` stores outbound encodes and retained in-flight state. Size it for the largest outbound - packet plus the QoS/session state you want to keep active. - -If `tx` is exhausted, `publish()` and other outbound operations can return [`Error::NotReady`]. -Malformed broker varints and undersized local encode buffers are rejected with errors rather than -causing panics. - -Use [`Buffers::split()`] if you prefer one contiguous slab. - -## Request / Reply - -[`InboundPublish`] exposes MQTT v5 request/reply properties directly. - -- [`InboundPublish::response_topic()`] -- [`InboundPublish::correlation_data()`] -- [`InboundPublish::reply()`] -- [`InboundPublish::reply_owned()`] - -## Transport And Time - -`minimq` uses: - -- [`embedded_io_async`] for byte I/O -- [`embassy_time`] for timing +Packages are released independently with `cargo release -p `; +inspect the default dry run before adding `--execute`. Protocol-crate releases +also update the matching Python package version, which is published separately. diff --git a/fuzz/.gitignore b/fuzz/.gitignore index 5208f22..89259bf 100644 --- a/fuzz/.gitignore +++ b/fuzz/.gitignore @@ -1,3 +1,2 @@ artifacts/ coverage/ -target/ diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 42e68b9..007b6f9 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true libfuzzer-sys = "0.4" [dependencies.minimq] -path = ".." +path = "../minimq" features = ["fuzzing"] [[bin]] diff --git a/fuzz/README.md b/fuzz/README.md index 1235e05..85e307f 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -14,5 +14,8 @@ Run the focused packet-boundary fuzzers with nightly Rust: - `cargo +nightly fuzz cmin fuzz_packet_reader --target host-tuple` - `cargo +nightly fuzz cmin fuzz_serializer --target host-tuple` +`host-tuple` makes Cargo use the machine's host target even when `cargo-fuzz` +itself came from a prebuilt binary for a different target. + The fuzz crate enables the main crate's `fuzzing` feature to expose a minimal, fuzz-only API surface for serializer, parser, and `PacketReader` entry points. diff --git a/CHANGELOG.md b/minimq/CHANGELOG.md similarity index 98% rename from CHANGELOG.md rename to minimq/CHANGELOG.md index 4b2164f..25de3b4 100644 --- a/CHANGELOG.md +++ b/minimq/CHANGELOG.md @@ -8,6 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED](https://github.com/quartiq/minimq/compare/v0.13.0...HEAD) - DATE +## Fixed + +* Match acknowledgements to the operation kind and avoid reusing live packet identifiers. +* Reject publishes above the broker's Maximum QoS unless automatic downgrade is enabled. + ## [0.13.0](https://github.com/quartiq/minimq/compare/v0.12.1...v0.13.0) - 2026-07-14 ## Changed diff --git a/minimq/Cargo.toml b/minimq/Cargo.toml new file mode 100644 index 0000000..6fa6348 --- /dev/null +++ b/minimq/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "minimq" +version = "0.13.0" +authors = [ + "Ryan Summers ", + "Robert Jördens ", + "Max Rottenkolber ", +] +edition.workspace = true +description = "An MQTT 5 client" +repository.workspace = true +categories = ["embedded", "no-std", "database", "encoding"] +keywords = ["mqtt", "embedded", "client"] +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[features] +default = ["defmt"] +defmt = [ + "dep:defmt", + "heapless/defmt", + "embedded-io-async/defmt", + "embassy-time/defmt", +] +fuzzing = [] + +[dependencies] +defmt = { version = "1.0.1", optional = true } +num_enum = { version = "0.7", default-features = false } +heapless = "0.9.1" +embedded-io-async = "0.7" +embassy-time = "0.5.1" +serde = { version = "1", features = ["derive"], default-features = false } +thiserror = { version = "2", default-features = false } + +[dev-dependencies] +defmt2log = "0.1.0" +env_logger = "0.11" +embassy-time = { version = "0.5.1", features = ["std", "generic-queue-8"] } +embedded-tls = "0.18.0" +embedded-io-adapters = { version = "0.7.0", features = ["tokio-1"] } +rand = "0.8" +tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "time"] } + +[[example]] +name = "tls_public_broker" diff --git a/minimq/LICENSE-MIT b/minimq/LICENSE-MIT new file mode 100644 index 0000000..ca21c6c --- /dev/null +++ b/minimq/LICENSE-MIT @@ -0,0 +1,27 @@ +Copyright (c) 2020 Quartiq GmbH +Copyright (c) 2020 Ryan Summers +Copyright (c) 2020 Max Rottenkolber + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/minimq/README.md b/minimq/README.md new file mode 100644 index 0000000..0beb471 --- /dev/null +++ b/minimq/README.md @@ -0,0 +1,185 @@ +[![QUARTIQ Matrix Chat](https://img.shields.io/matrix/quartiq:matrix.org)](https://matrix.to/#/#quartiq:matrix.org) +[![Continuous Integration](https://github.com/quartiq/minimq/actions/workflows/ci.yml/badge.svg)](https://github.com/quartiq/minimq/actions/workflows/ci.yml) + +# Minimq + +`minimq` is an opinionated `no_std`, no-alloc, `async` MQTT 5 client for embedded systems. + +It manages one broker session over caller-provided packet buffers and an +[`embedded_io_async`] byte stream. Keepalive, QoS 0/1/2 handshakes, in-flight replay, and resumed +sessions stay inside the client; transport establishment, reconnect timing, and application +dispatch stay outside it. + +The main API is [`Session`]. + +## Scope + +Choose Minimq when fixed memory use and a small managed client matter more than protocol +extensibility. It deliberately provides one direct session model rather than a general MQTT packet +toolkit: no allocator, executor, network stack, runtime-sized queues, or application framework. + +Downstream users include [`miniconf_mqtt`](https://docs.rs/miniconf_mqtt), which exposes typed +configuration trees over MQTT. + +## What You Use + +- [`Buffers`]: caller-owned RX/TX memory +- [`ConfigBuilder`]: session configuration +- [`Connection`]: one live MQTT connection +- [`Disconnect`]: graceful disconnect options +- [`Io`]: transport boundary for an established byte stream +- [`Session`]: the client you drive +- [`InboundPublish`]: output of [`Connection::recv()`] + +## Example + +```no_run +# use std::io; +# struct MyIo; +# use embedded_io_async::{ErrorType, Read, Write}; +# impl ErrorType for MyIo { +# type Error = io::Error; +# } +# impl Read for MyIo { +# async fn read(&mut self, _buf: &mut [u8]) -> Result { +# todo!() +# } +# } +# impl Write for MyIo { +# async fn write(&mut self, _buf: &[u8]) -> Result { +# todo!() +# } +# async fn flush(&mut self) -> Result<(), Self::Error> { +# todo!() +# } +# } +# async fn open_io(_addr: SocketAddr) -> Result { todo!() } +use core::net::SocketAddr; +use minimq::{Buffers, ConfigBuilder, ConnectEvent, Error, Session, TopicFilter}; + +async fn run() { + let rx = &mut [0u8; 256]; + let tx = &mut [0u8; 768]; + let addr: SocketAddr = "127.0.0.1:1883".parse().unwrap(); + let mut session = Session::new( + ConfigBuilder::new(Buffers::new(rx, tx)) + .client_id("demo") + .unwrap(), + ); + + loop { + let io = open_io(addr).await.unwrap(); + // `connect` returns a connection handle that owns the transport and borrows + // the session. Dropping it at the end of the loop releases both for the next + // reconnect. + let mut conn = session.connect(io).await.unwrap(); + match conn.connect_event() { + ConnectEvent::Connected => { + conn.subscribe(&[TopicFilter::new("demo/in")], &[]) + .await + .unwrap(); + } + ConnectEvent::Reconnected => {} + } + + loop { + match conn.recv().await { + Ok(message) => println!("topic={}", message.topic()), + Err(Error::Disconnected) => break, + Err(err) => panic!("{err}"), + } + } + } +} + +# fn main() {} +``` + +The attached transport must implement [`embedded_io_async::Read`] and +[`embedded_io_async::Write`]. +Ordinary lack of inbound data must keep the read future pending; if the transport returns +`TimedOut` or `Interrupted`, [`Connection::poll()`] treats that as transport failure and +disconnects the connection. + +For a TLS MQTT v5 request/reply example that preserves a subscription across reconnects and reuses +the TLS record buffers, see `examples/tls_public_broker.rs`. + +## Errors + +`ConfigBuilder` reports setup-time validation failures through [`ConfigError`]. +Connected session operations report [`Error`]: +- broker rejections and invalid inbound MQTT data surface as [`PeerError`] +- local buffer and capacity limits surface as [`ResourceError`] +- transport failures surface as [`Error::Transport`] + +## Session Model + +You provide packet buffers plus an already-established transport, and a loop that explicitly +passes that transport into [`Session::connect()`] to establish or resume the broker session. + +[`Session::connect()`] takes ownership of the provided transport and performs the unbounded MQTT +`CONNECT` / `CONNACK` handshake. It returns a [`Connection`] that borrows the session. Once +connected: +- [`Connection::recv()`] blocks until the next inbound publish arrives or the connection is lost. +- [`Connection::poll()`] blocks until any session progress happens and returns `Ok(None)` for + internal-only progress such as ACK handling, replay, or keepalive traffic. + +Dropping the connection releases the session for a later reconnect. Call +[`Connection::disconnect()`] first for a graceful MQTT close. + +- [`ConnectEvent::Connected`] means the broker created a fresh session. Re-establish subscriptions + here. +- [`ConnectEvent::Reconnected`] means the broker resumed the existing MQTT session. Existing + subscriptions and in-flight QoS state were kept. +- [`Connection::recv()`] yields one inbound publish. + +If [`Connection::recv()`] or [`Connection::poll()`] returns [`Error::Disconnected`], the caller +discards the handle and decides +when to call +[`Session::connect()`] with a fresh transport again. +Other transport/protocol errors mark the handle dead; callers should handle the error and reconnect +rather than retrying network operations on that handle. + +A transport write returning `Ok(0)` for a non-empty buffer produces [`Error::WriteZero`] and also +marks the connection dead. This is intentionally conservative: an earlier write may already have +sent a packet prefix, so restarting that packet on the same byte stream is not safe. + +For cooperative driving: +- use [`Connection::drive()`] for immediate local progress without waiting for future inbound reads + or future session deadlines +- wrap cancel-safe blocking [`Connection::poll()`] or [`Connection::recv()`] in an external timeout + such as [`embassy_time::with_timeout()`] or [`embassy_time::with_deadline()`] +- if you need real wall-clock limits, enforce them in the transport's `read`, `write`, and + `flush` futures; using the same budget as minimq's internal MQTT round-trip timeout keeps + keepalive and transport liveness aligned + +## Buffers + +You supply two buffers. + +- `rx` stores one inbound MQTT packet at a time. Size it for the largest inbound publish, + including topic, properties, and payload. +- `tx` stores outbound encodes and retained in-flight state. Size it for the largest outbound + packet plus the QoS/session state you want to keep active. + +If `tx` is exhausted, `publish()` and other outbound operations can return [`Error::NotReady`]. +Malformed broker varints and undersized local encode buffers are rejected with errors rather than +causing panics. + +Use [`Buffers::split()`] if you prefer one contiguous slab. + +## Request / Reply + +[`InboundPublish`] exposes MQTT v5 request/reply properties directly. + +- [`InboundPublish::response_topic()`] +- [`InboundPublish::correlation_data()`] +- [`InboundPublish::reply()`] +- [`InboundPublish::reply_owned()`] + +## Transport And Time + +`minimq` uses: + +- [`embedded_io_async`] for byte I/O +- [`embassy_time`] for timing diff --git a/examples/tls_public_broker.rs b/minimq/examples/tls_public_broker.rs similarity index 100% rename from examples/tls_public_broker.rs rename to minimq/examples/tls_public_broker.rs diff --git a/src/config.rs b/minimq/src/config.rs similarity index 100% rename from src/config.rs rename to minimq/src/config.rs diff --git a/src/de/deserializer.rs b/minimq/src/de/deserializer.rs similarity index 100% rename from src/de/deserializer.rs rename to minimq/src/de/deserializer.rs diff --git a/src/de/mod.rs b/minimq/src/de/mod.rs similarity index 100% rename from src/de/mod.rs rename to minimq/src/de/mod.rs diff --git a/src/de/packet_reader.rs b/minimq/src/de/packet_reader.rs similarity index 100% rename from src/de/packet_reader.rs rename to minimq/src/de/packet_reader.rs diff --git a/src/de/received_packet.rs b/minimq/src/de/received_packet.rs similarity index 100% rename from src/de/received_packet.rs rename to minimq/src/de/received_packet.rs diff --git a/src/fuzzing.rs b/minimq/src/fuzzing.rs similarity index 100% rename from src/fuzzing.rs rename to minimq/src/fuzzing.rs diff --git a/src/lib.rs b/minimq/src/lib.rs similarity index 99% rename from src/lib.rs rename to minimq/src/lib.rs index f044872..e6864f1 100644 --- a/src/lib.rs +++ b/minimq/src/lib.rs @@ -1,5 +1,6 @@ #![cfg_attr(not(test), no_std)] #![doc = include_str!("../README.md")] +#![warn(missing_docs)] mod config; mod de; diff --git a/src/mqtt_client/mod.rs b/minimq/src/mqtt_client/mod.rs similarity index 100% rename from src/mqtt_client/mod.rs rename to minimq/src/mqtt_client/mod.rs diff --git a/src/mqtt_client/outbound.rs b/minimq/src/mqtt_client/outbound.rs similarity index 95% rename from src/mqtt_client/outbound.rs rename to minimq/src/mqtt_client/outbound.rs index 8740f4f..9a9e0de 100644 --- a/src/mqtt_client/outbound.rs +++ b/minimq/src/mqtt_client/outbound.rs @@ -5,7 +5,7 @@ use crate::wire::ControlPacket; use crate::{Error, ProtocolError, PubError, ReasonCode, ResourceError, error, trace}; use heapless::Vec; -use super::Io; +use super::{Io, OpKind}; pub(super) const CONTROL_PACKET_LEN: usize = 9; pub(super) const MAX_RETAINED: usize = 8; @@ -36,6 +36,7 @@ pub(super) struct PendingRelease { #[derive(Debug, Copy, Clone, PartialEq, Eq)] struct RetainedPacket { + kind: OpKind, packet_id: u16, offset: usize, len: usize, @@ -201,11 +202,11 @@ impl<'a> Outbound<'a> { }) } - pub(super) fn ack_packet(&mut self, packet_id: u16) -> bool { + pub(super) fn ack_packet(&mut self, kind: OpKind, packet_id: u16) -> bool { let Some(position) = self .retained .iter() - .position(|entry| entry.packet_id == packet_id) + .position(|entry| entry.kind == kind && entry.packet_id == packet_id) else { return false; }; @@ -286,12 +287,14 @@ impl<'a> Outbound<'a> { pub(super) fn retain_packet( &mut self, + kind: OpKind, packet_id: u16, offset: usize, len: usize, ) -> Result<(), ProtocolError> { self.retained .push(RetainedPacket { + kind, packet_id, offset, len, @@ -608,6 +611,7 @@ pub(super) async fn write_all( #[cfg(test)] mod tests { use super::{ControlAction, MAX_FIXED_HEADER_SIZE, Outbound, OutboundStep, SendState}; + use crate::mqtt_client::OpKind; use crate::{ Error, Properties, PubError, ReasonCode, ResourceError, packets::{PublishHeader, Subscribe}, @@ -621,7 +625,7 @@ mod tests { let mut storage = [0u8; 64]; let mut outbound = Outbound::new(&mut storage); - outbound.retain_packet(7, 0, 10).unwrap(); + outbound.retain_packet(OpKind::Subscribe, 7, 0, 10).unwrap(); let (offset, len) = outbound .encode_packet(&Subscribe { @@ -640,7 +644,7 @@ mod tests { let mut storage = [0u8; MAX_FIXED_HEADER_SIZE + 4]; let mut outbound = Outbound::new(&mut storage); - outbound.retain_packet(7, 0, 5).unwrap(); + outbound.retain_packet(OpKind::Subscribe, 7, 0, 5).unwrap(); assert!(!outbound.can_retain()); } @@ -650,7 +654,7 @@ mod tests { let mut storage = [0u8; MAX_FIXED_HEADER_SIZE + 4]; let mut outbound = Outbound::new(&mut storage); - outbound.retain_packet(7, 0, 5).unwrap(); + outbound.retain_packet(OpKind::Subscribe, 7, 0, 5).unwrap(); let publication = Publication::bytes("a", b"x"); let header = PublishHeader { @@ -690,4 +694,16 @@ mod tests { if step.action == action && step.state == SendState::Write { written: 0 } )); } + + #[test] + fn acknowledgement_must_match_retained_operation() { + let mut storage = [0u8; 16]; + let mut outbound = Outbound::new(&mut storage); + outbound.retain_packet(OpKind::Subscribe, 7, 0, 5).unwrap(); + + assert!(!outbound.ack_packet(OpKind::PublishAtLeastOnce, 7)); + assert!(outbound.has_retained(7)); + assert!(outbound.ack_packet(OpKind::Subscribe, 7)); + assert!(!outbound.has_retained(7)); + } } diff --git a/src/mqtt_client/session/drive.rs b/minimq/src/mqtt_client/session/drive.rs similarity index 99% rename from src/mqtt_client/session/drive.rs rename to minimq/src/mqtt_client/session/drive.rs index cfb456f..466c386 100644 --- a/src/mqtt_client/session/drive.rs +++ b/minimq/src/mqtt_client/session/drive.rs @@ -395,6 +395,10 @@ impl<'buf, IO: Io> Connection<'_, 'buf, IO> { self.handle_disconnect(); return Err(Error::Transport(err)); } + Err(Error::WriteZero) => { + self.handle_disconnect(); + return Err(Error::WriteZero); + } Err(err) => return Err(err), }; let written = written + count; diff --git a/src/mqtt_client/session/handshake.rs b/minimq/src/mqtt_client/session/handshake.rs similarity index 100% rename from src/mqtt_client/session/handshake.rs rename to minimq/src/mqtt_client/session/handshake.rs diff --git a/src/mqtt_client/session/inbound.rs b/minimq/src/mqtt_client/session/inbound.rs similarity index 95% rename from src/mqtt_client/session/inbound.rs rename to minimq/src/mqtt_client/session/inbound.rs index 6618c26..2f4b38b 100644 --- a/src/mqtt_client/session/inbound.rs +++ b/minimq/src/mqtt_client/session/inbound.rs @@ -1,6 +1,7 @@ use core::convert::Infallible; use crate::de::ReceivedPacket; +use crate::mqtt_client::OpKind; use crate::mqtt_client::outbound::{ControlAction, check_control_packet_size, check_pubrel_size}; use crate::{ Connection, Error, InboundPublish, Io, PeerError, ProtocolError, QoS, ReasonCode, @@ -18,7 +19,7 @@ impl<'a> SessionData<'a> { match packet { ReceivedPacket::ConnAck(_) => return Err(ProtocolError::UnexpectedPacket.into()), ReceivedPacket::SubAck(ack) => { - if !self.outbound.ack_packet(ack.packet_id) { + if !self.outbound.ack_packet(OpKind::Subscribe, ack.packet_id) { debug!("Ignoring stale SUBACK for packet id {=u16}", ack.packet_id); return Ok(false); } @@ -28,7 +29,7 @@ impl<'a> SessionData<'a> { } } ReceivedPacket::UnsubAck(ack) => { - if !self.outbound.ack_packet(ack.packet_id) { + if !self.outbound.ack_packet(OpKind::Unsubscribe, ack.packet_id) { debug!( "Ignoring stale UNSUBACK for packet id {=u16}", ack.packet_id @@ -45,7 +46,10 @@ impl<'a> SessionData<'a> { runtime.ping_timeout = None; } ReceivedPacket::PubAck(ack) => { - if !self.outbound.ack_packet(ack.packet_id) { + if !self + .outbound + .ack_packet(OpKind::PublishAtLeastOnce, ack.packet_id) + { debug!("Ignoring stale PUBACK for packet id {=u16}", ack.packet_id); return Ok(false); } @@ -60,7 +64,10 @@ impl<'a> SessionData<'a> { ack.reason.code().as_result()?; } ReceivedPacket::PubRec(rec) => { - let queue_release = match self.outbound.ack_packet(rec.packet_id) { + let queue_release = match self + .outbound + .ack_packet(OpKind::PublishExactlyOnce, rec.packet_id) + { true => { runtime.send_quota = runtime .send_quota diff --git a/src/mqtt_client/session/mod.rs b/minimq/src/mqtt_client/session/mod.rs similarity index 97% rename from src/mqtt_client/session/mod.rs rename to minimq/src/mqtt_client/session/mod.rs index 884eeb7..85ed5f2 100644 --- a/src/mqtt_client/session/mod.rs +++ b/minimq/src/mqtt_client/session/mod.rs @@ -196,6 +196,9 @@ impl<'buf, IO: Io> Connection<'_, 'buf, IO> { self.io } + /// Mark this connection dead without sending MQTT `DISCONNECT`. + /// + /// Use this when the application knows the transport can no longer be used. pub fn handle_disconnect(&mut self) { self.live = false; self.session.handle_disconnect(); diff --git a/src/mqtt_client/session/operations.rs b/minimq/src/mqtt_client/session/operations.rs similarity index 95% rename from src/mqtt_client/session/operations.rs rename to minimq/src/mqtt_client/session/operations.rs index 6bdff6d..d27042b 100644 --- a/src/mqtt_client/session/operations.rs +++ b/minimq/src/mqtt_client/session/operations.rs @@ -40,7 +40,7 @@ impl<'buf, IO: Io> Connection<'_, 'buf, IO> { result } - /// Gracefully close the transport with `DISCONNECT`. + /// Gracefully end the MQTT connection with `DISCONNECT`. /// /// This is the graceful counterpart to simply dropping the handle: it sends the MQTT /// `DISCONNECT` so the broker closes cleanly and suppresses the Will. Just dropping @@ -83,7 +83,7 @@ impl<'buf, IO: Io> Connection<'_, 'buf, IO> { self.session .data .outbound - .retain_packet(packet_id, offset, len)?; + .retain_packet(OpKind::Subscribe, packet_id, offset, len)?; debug!( "Enqueued SUBSCRIBE packet_id={=u16} len={=usize} tx_used={=usize}", packet_id, @@ -133,7 +133,7 @@ impl<'buf, IO: Io> Connection<'_, 'buf, IO> { self.session .data .outbound - .retain_packet(packet_id, offset, len)?; + .retain_packet(OpKind::Unsubscribe, packet_id, offset, len)?; debug!( "Enqueued UNSUBSCRIBE packet_id={=u16} len={=usize} tx_used={=usize}", packet_id, @@ -184,7 +184,8 @@ impl<'buf, IO: Io> Connection<'_, 'buf, IO> { return Err(Error::InvalidRequest.into()); } let qos = match self.session.runtime.max_qos { - Some(max_qos) if self.session.downgrade_qos && qos > max_qos => max_qos, + Some(max_qos) if qos > max_qos && self.session.downgrade_qos => max_qos, + Some(max_qos) if qos > max_qos => return Err(Error::InvalidRequest.into()), _ => qos, }; let packet_id = (qos > QoS::AtMostOnce).then(|| self.session.data.next_packet_id()); @@ -205,6 +206,11 @@ impl<'buf, IO: Io> Connection<'_, 'buf, IO> { } if let Some(packet_id) = packet_id { + let kind = if qos == QoS::ExactlyOnce { + OpKind::PublishExactlyOnce + } else { + OpKind::PublishAtLeastOnce + }; let (offset, len) = self .session .data @@ -214,7 +220,7 @@ impl<'buf, IO: Io> Connection<'_, 'buf, IO> { self.session .data .outbound - .retain_packet(packet_id, offset, len)?; + .retain_packet(kind, packet_id, offset, len)?; self.session.runtime.send_quota = self.session.runtime.send_quota.saturating_sub(1); debug!( "Enqueued PUBLISH packet_id={=u16} qos={} len={=usize} send_quota={=u16}/{=u16} tx_used={=usize}", @@ -226,11 +232,6 @@ impl<'buf, IO: Io> Connection<'_, 'buf, IO> { self.session.data.outbound.used() ); self.flush_outbound().await?; - let kind = if qos == QoS::ExactlyOnce { - OpKind::PublishExactlyOnce - } else { - OpKind::PublishAtLeastOnce - }; return Ok(Some(Op::new( kind, packet_id, @@ -249,9 +250,6 @@ impl<'buf, IO: Io> Connection<'_, 'buf, IO> { return Err(Error::Disconnected.into()); } if let Err(err) = write_all(&mut self.io, packet).await { - if matches!(err, Error::WriteZero) { - return Err(err.into()); - } warn!("QoS0 PUBLISH write failed"); self.handle_disconnect(); return Err(err.into()); diff --git a/src/mqtt_client/session/state.rs b/minimq/src/mqtt_client/session/state.rs similarity index 79% rename from src/mqtt_client/session/state.rs rename to minimq/src/mqtt_client/session/state.rs index ef6b033..a0d4024 100644 --- a/src/mqtt_client/session/state.rs +++ b/minimq/src/mqtt_client/session/state.rs @@ -119,9 +119,34 @@ impl<'a> SessionData<'a> { } pub(super) fn next_packet_id(&mut self) -> u16 { - let packet_id = self.packet_id.get(); - self.packet_id = - NonZeroU16::new(packet_id.wrapping_add(1)).unwrap_or(NonZeroU16::new(1).unwrap()); - packet_id + loop { + let packet_id = self.packet_id.get(); + self.packet_id = NonZeroU16::new(packet_id.wrapping_add(1)).unwrap_or(NonZeroU16::MIN); + if !self.outbound.has_retained(packet_id) + && !self.outbound.has_pending_release(packet_id) + { + return packet_id; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::{NonZeroU16, SessionData}; + use crate::ReasonCode; + use crate::mqtt_client::OpKind; + + #[test] + fn packet_identifier_wrap_skips_live_operations() { + let mut storage = [0; 16]; + let mut data = SessionData::new(&mut storage); + data.packet_id = NonZeroU16::new(u16::MAX).unwrap(); + data.outbound + .retain_packet(OpKind::Subscribe, u16::MAX, 0, 5) + .unwrap(); + data.outbound.queue_release(1, ReasonCode::Success).unwrap(); + + assert_eq!(data.next_packet_id(), 2); } } diff --git a/src/mqtt_client/session/tests.rs b/minimq/src/mqtt_client/session/tests.rs similarity index 98% rename from src/mqtt_client/session/tests.rs rename to minimq/src/mqtt_client/session/tests.rs index fb1ac4d..b5e68e6 100644 --- a/src/mqtt_client/session/tests.rs +++ b/minimq/src/mqtt_client/session/tests.rs @@ -340,7 +340,12 @@ fn can_publish_qos1_false_when_retained_slots_full() { conn.session .data .outbound - .retain_packet(packet_id, offset as usize, 4) + .retain_packet( + crate::mqtt_client::OpKind::PublishAtLeastOnce, + packet_id, + offset as usize, + 4, + ) .unwrap(); } diff --git a/src/packets.rs b/minimq/src/packets.rs similarity index 100% rename from src/packets.rs rename to minimq/src/packets.rs diff --git a/src/properties.rs b/minimq/src/properties.rs similarity index 100% rename from src/properties.rs rename to minimq/src/properties.rs diff --git a/src/publication.rs b/minimq/src/publication.rs similarity index 100% rename from src/publication.rs rename to minimq/src/publication.rs diff --git a/src/reason_codes.rs b/minimq/src/reason_codes.rs similarity index 100% rename from src/reason_codes.rs rename to minimq/src/reason_codes.rs diff --git a/src/ser/mod.rs b/minimq/src/ser/mod.rs similarity index 100% rename from src/ser/mod.rs rename to minimq/src/ser/mod.rs diff --git a/src/types.rs b/minimq/src/types.rs similarity index 100% rename from src/types.rs rename to minimq/src/types.rs diff --git a/src/varint.rs b/minimq/src/varint.rs similarity index 100% rename from src/varint.rs rename to minimq/src/varint.rs diff --git a/src/will.rs b/minimq/src/will.rs similarity index 100% rename from src/will.rs rename to minimq/src/will.rs diff --git a/src/wire.rs b/minimq/src/wire.rs similarity index 100% rename from src/wire.rs rename to minimq/src/wire.rs diff --git a/tests/async_client.rs b/minimq/tests/async_client.rs similarity index 93% rename from tests/async_client.rs rename to minimq/tests/async_client.rs index 481cf64..95a2064 100644 --- a/tests/async_client.rs +++ b/minimq/tests/async_client.rs @@ -21,6 +21,7 @@ struct MockIo { tx: Vec>, read_error: Option, write_error: Option<(usize, ErrorKind)>, + write_zero: Option, pending_reads: usize, pending_writes: usize, pending_flushes: usize, @@ -46,6 +47,10 @@ impl MockConnection { self.inner.borrow_mut().write_error = Some((successful_writes, err)); } + fn zero_write_after(&mut self, successful_writes: usize) { + self.inner.borrow_mut().write_zero = Some(successful_writes); + } + fn fail_read(&mut self, err: ErrorKind) { self.inner.borrow_mut().read_error = Some(err); } @@ -118,6 +123,13 @@ impl Write for MockConnection { } *remaining -= 1; } + if let Some(remaining) = &mut inner.write_zero { + if *remaining == 0 { + inner.write_zero = None; + return Poll::Ready(Ok(0)); + } + *remaining -= 1; + } inner.tx.push(buf.to_vec()); Poll::Ready(Ok(buf.len())) }) @@ -206,6 +218,10 @@ fn connack_max_packet_size(max: u32) -> [u8; 10] { ] } +fn connack_max_qos(max: QoS) -> [u8; 7] { + [0x20, 0x05, 0x00, 0x00, 0x02, 0x24, max as u8] +} + fn puback(id: u16) -> [u8; 4] { [0x40, 0x02, (id >> 8) as u8, id as u8] } @@ -861,6 +877,26 @@ fn outbound_qos_acks_can_arrive_out_of_order() { assert!(poll_now(&mut conn).unwrap().is_none()); } +#[test] +fn acknowledgement_kind_must_match_operation() { + let mut connection = MockConnection::default(); + connection.push_rx(&connack()); + connection.push_rx(&puback(1)); + connection.push_rx(&suback(1, 0x00)); + let connector = MockConnector::new(connection); + let mut session = session(); + let mut conn = expect_connected(&mut session, &connector); + + let op = block_on(conn.subscribe(&[TopicFilter::new("data")], &[])).unwrap(); + assert!(conn.is_pending(&op)); + + assert!(poll_now(&mut conn).unwrap().is_none()); + assert!(conn.is_pending(&op)); + + assert!(poll_now(&mut conn).unwrap().is_none()); + assert!(conn.is_complete(&op)); +} + #[test] fn subscribe_is_replayed_after_disconnect_until_suback() { let mut first = MockConnection::default(); @@ -1100,6 +1136,35 @@ fn connack_receive_maximum_clamps_local_quota() { assert!(matches!(result, Err(PubError::Session(Error::NotReady)))); } +#[test] +fn publish_above_broker_maximum_qos_fails_before_send() { + let mut connection = MockConnection::default(); + let inspect = connection.clone(); + connection.push_rx(&connack_max_qos(QoS::AtMostOnce)); + let connector = MockConnector::new(connection); + let mut session = session(); + let mut conn = expect_connected(&mut session, &connector); + + assert!(matches!( + publish_qos1(&mut conn, "data", b"x"), + Err(PubError::Session(Error::InvalidRequest)) + )); + assert_eq!(inspect.tx().len(), 1); +} + +#[test] +fn publish_can_downgrade_to_broker_maximum_qos() { + let mut connection = MockConnection::default(); + let inspect = connection.clone(); + connection.push_rx(&connack_max_qos(QoS::AtMostOnce)); + let connector = MockConnector::new(connection); + let mut session = Session::new(config().autodowngrade_qos()); + let mut conn = expect_connected(&mut session, &connector); + + assert!(publish_qos1(&mut conn, "data", b"x").unwrap().is_none()); + assert_eq!(inspect.tx().last().unwrap()[0], 0x30); +} + #[test] fn session_allows_publish_after_message_borrow_is_dropped() { let mut connection = MockConnection::default(); @@ -1143,6 +1208,58 @@ fn session_reconnects_after_write_error() { publish_qos1_ok(&mut conn, "reply", b"ok"); } +#[test] +fn qos0_write_zero_kills_connection() { + let mut first = MockConnection::default(); + first.push_rx(&connack()); + first.zero_write_after(1); + + let mut second = MockConnection::default(); + second.push_rx(&connack()); + + let connector = MockConnector::with_connections([first, second]); + let mut session = session(); + let mut conn = expect_connected(&mut session, &connector); + + assert!(matches!( + block_on(conn.publish(Publication::bytes("data", b"x"))), + Err(PubError::Session(Error::WriteZero)) + )); + assert!(matches!( + block_on(conn.publish(Publication::bytes("data", b"x"))), + Err(PubError::Session(Error::Disconnected)) + )); + + drop(conn); + expect_connected(&mut session, &connector); +} + +#[test] +fn retained_write_zero_kills_connection() { + let mut first = MockConnection::default(); + first.push_rx(&connack()); + first.zero_write_after(1); + + let mut second = MockConnection::default(); + second.push_rx(&connack()); + + let connector = MockConnector::with_connections([first, second]); + let mut session = session(); + let mut conn = expect_connected(&mut session, &connector); + + assert!(matches!( + publish_qos1(&mut conn, "data", b"x"), + Err(PubError::Session(Error::WriteZero)) + )); + assert!(matches!( + publish_qos1(&mut conn, "data", b"x"), + Err(PubError::Session(Error::Disconnected)) + )); + + drop(conn); + expect_connected(&mut session, &connector); +} + #[test] fn puback_failure_is_reported_and_clears_inflight() { let mut connection = MockConnection::default(); @@ -1259,7 +1376,7 @@ fn connect_retries_cleanly_after_cancellation_during_pending_connack() { } #[test] -fn disconnect_sends_disconnect_packet_and_drops_connection() { +fn disconnect_sends_disconnect_packet_and_marks_handle_dead() { let mut connection = MockConnection::default(); let inspect = connection.clone(); connection.push_rx(&connack()); @@ -1267,15 +1384,14 @@ fn disconnect_sends_disconnect_packet_and_drops_connection() { let mut session = session(); let mut conn = expect_connected(&mut session, &connector); - // `disconnect` consumes the handle, so after this the session is no - // longer connected (no live Conn borrow). + // `disconnect` marks the handle dead; dropping it releases the session borrow. block_on(conn.disconnect()).unwrap(); assert_eq!(inspect.tx().last().unwrap(), &disconnect_req()); } #[test] -fn disconnect_with_sends_reason_and_drops_connection() { +fn disconnect_with_sends_reason_and_marks_handle_dead() { let mut connection = MockConnection::default(); let inspect = connection.clone(); connection.push_rx(&connack()); @@ -1283,7 +1399,7 @@ fn disconnect_with_sends_reason_and_drops_connection() { let mut session = session(); let mut conn = expect_connected(&mut session, &connector); - // `disconnect_with` consumes the handle. + // `disconnect_with` marks the handle dead. block_on(conn.disconnect_with(Disconnect::with_will())).unwrap(); assert_eq!(inspect.tx().last().unwrap(), &disconnect_with_will()); @@ -1307,7 +1423,7 @@ fn disconnect_uses_dedicated_control_storage_when_tx_arena_is_full() { let payload = [0u8; 80]; block_on(conn.publish(Publication::bytes("data", &payload).qos(QoS::AtLeastOnce))).unwrap(); - // `disconnect` consumes the handle. + // `disconnect` marks the handle dead. block_on(conn.disconnect()).unwrap(); assert_eq!(inspect.tx().last().unwrap(), &disconnect_req()); diff --git a/tests/real_broker.rs b/minimq/tests/real_broker.rs similarity index 100% rename from tests/real_broker.rs rename to minimq/tests/real_broker.rs diff --git a/tests/support/mod.rs b/minimq/tests/support/mod.rs similarity index 78% rename from tests/support/mod.rs rename to minimq/tests/support/mod.rs index d6ed65e..38a1cf3 100644 --- a/tests/support/mod.rs +++ b/minimq/tests/support/mod.rs @@ -1,17 +1,10 @@ use std::{ future::Future, pin::Pin, - sync::Arc, sync::OnceLock, - task::{Context, Poll, Wake, Waker}, + task::{Context, Poll, Waker}, }; -struct NoopWaker; - -impl Wake for NoopWaker { - fn wake(self: Arc) {} -} - pub fn init_host_logging() { static HOST_LOGGING: OnceLock<()> = OnceLock::new(); @@ -40,6 +33,5 @@ pub fn poll_once(future: Pin<&mut F>) -> Poll { } fn noop_context() -> Context<'static> { - let waker = Waker::from(Arc::new(NoopWaker)); - Context::from_waker(Box::leak(Box::new(waker))) + Context::from_waker(Waker::noop()) } diff --git a/mqtt-rpc/CHANGELOG.md b/mqtt-rpc/CHANGELOG.md new file mode 100644 index 0000000..d7eace1 --- /dev/null +++ b/mqtt-rpc/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [UNRELEASED](https://github.com/quartiq/minimq/compare/v0.13.0...HEAD) - DATE + +- Initial MQTT 5 request/reply service and Python client. diff --git a/mqtt-rpc/Cargo.toml b/mqtt-rpc/Cargo.toml new file mode 100644 index 0000000..16f77f4 --- /dev/null +++ b/mqtt-rpc/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "mqtt-rpc" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +authors = ["Ryan Summers "] +license.workspace = true +description = "MQTT 5 request/response transport for embedded services" +keywords = ["request", "response", "embedded", "no-std", "mqtt"] +categories = ["no-std", "embedded", "network-programming"] +repository.workspace = true + +[lints] +workspace = true + +[features] +default = [] +defmt = ["minimq/defmt"] + +[dependencies] +heapless = "0.9.1" +minimq = { path = "../minimq", version = "0.13", default-features = false } + +[dev-dependencies] +defmt2log = "0.2.0" +embedded-io-adapters = { version = "0.7", features = ["tokio-1"] } +embassy-time = { version = "0.5.1", features = ["std", "generic-queue-8"] } +env_logger = "0.11" +tokio = { version = "1", features = ["macros", "net", "rt-multi-thread", "time"] } diff --git a/mqtt-rpc/LICENSE-MIT b/mqtt-rpc/LICENSE-MIT new file mode 100644 index 0000000..aaac2af --- /dev/null +++ b/mqtt-rpc/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 QUARTIQ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/mqtt-rpc/README.md b/mqtt-rpc/README.md new file mode 100644 index 0000000..2c3455a --- /dev/null +++ b/mqtt-rpc/README.md @@ -0,0 +1,56 @@ +# mqtt-rpc + +Minimal `no_std` MQTT 5 request/response transport for embedded services. + +`mqtt-rpc` works with a caller-owned Minimq 0.13 session. It subscribes +`/rpc/#`, routes borrowed inbound requests, retains MQTT response-topic and +correlation data for deferred work, and publishes transient QoS-1 responses. + +It deliberately does not own the MQTT transport, application method registry, payload schema, +executor, persistent storage, or retained device status. + +The [`py`](py/) directory contains the corresponding Python requester and command-line client. + +## Wire contract + +- Requests are published below `/rpc/` and must carry an MQTT 5 Response Topic. +- Correlation Data is optional and copied unchanged to the response. +- Retained requests are rejected and never dispatched. +- Responses are QoS 1, non-retained, and carry one `code` User Property. `Ok` means success; + applications define other codes. +- Request and response payloads are opaque to MQTT RPC. +- QoS 1 provides at-least-once delivery. Application methods with side effects must be idempotent; + Correlation Data routes replies but is not a durable deduplication record. + +The requester chooses the MQTT response topic. Before responding, the application +can inspect `ResponseTarget::topic()` and must rely on an appropriate broker ACL +or reject targets outside its allowed response-topic tree. + +## Device use + +After every successful `Session::connect`, tell the service whether the broker resumed the MQTT +session and drive its subscription to completion. Then pass each inbound publication to `handle`: + +```rust,no_run +use minimq::{Connection, Io}; +use mqtt_rpc::{Handle, Service, SUCCESS_CODE, respond}; + +async fn run(connection: &mut Connection<'_, '_, IO>) { + let mut rpc = Service::new("dt/device").unwrap(); + rpc.begin_connection(connection.connect_event()); + while !rpc.step(connection).await.unwrap() { + let _ = connection.poll().await.unwrap(); + } + + loop { + let inbound = connection.recv().await.unwrap(); + if let Handle::Request(request) = rpc.handle(&inbound) { + assert_eq!(request.method(), "ping"); + let target = request.into_response_target(); + respond(connection, &target, SUCCESS_CODE, b"pong".as_slice()) + .await + .unwrap(); + } + } +} +``` diff --git a/mqtt-rpc/py/README.md b/mqtt-rpc/py/README.md new file mode 100644 index 0000000..b289640 --- /dev/null +++ b/mqtt-rpc/py/README.md @@ -0,0 +1,28 @@ +# `mqtt-rpc` Python client + +Python 3.11+ client for MQTT 5 request/response services implemented with the MQTT RPC Rust crate. + +```sh +python -m pip install mqtt-rpc +mqtt-rpc --broker mqtt dt/device settings/store +``` + +The async API carries opaque `str` or `bytes` request payloads and returns response bytes: + +```python +from mqtt_rpc.client import Client + +async with Client("mqtt", "dt/device") as rpc: + response = await rpc.request("settings/store") +``` + +Each client uses a dedicated MQTT connection and response topic. Requests are QoS 1, non-retained, +and carry MQTT Response Topic, Correlation Data, and Message Expiry properties. Responses must copy +the Correlation Data and carry exactly one `code` User Property. `Ok` returns the payload; any other +code raises `RemoteError`. + +QoS 1 provides at-least-once delivery. Methods with side effects must be idempotent; Correlation +Data routes replies but is not a durable deduplication record. + +MQTT RPC does not provide device discovery, schemas, serialization, persistence policy, or retained +status. Applications compose those separately. diff --git a/mqtt-rpc/py/mqtt_rpc/__init__.py b/mqtt-rpc/py/mqtt_rpc/__init__.py new file mode 100644 index 0000000..77ce140 --- /dev/null +++ b/mqtt-rpc/py/mqtt_rpc/__init__.py @@ -0,0 +1 @@ +"""MQTT 5 request/response client.""" diff --git a/mqtt-rpc/py/mqtt_rpc/__main__.py b/mqtt-rpc/py/mqtt_rpc/__main__.py new file mode 100644 index 0000000..f05e8bc --- /dev/null +++ b/mqtt-rpc/py/mqtt_rpc/__main__.py @@ -0,0 +1,38 @@ +"""MQTT RPC command-line client.""" + +import argparse +import asyncio + +from gmqtt.mqtt.handler import MQTTError + +from .client import Client, ProtocolError, RemoteError + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Call an MQTT RPC method") + parser.add_argument("--broker", default="localhost") + parser.add_argument("--timeout", type=float, default=3.0) + parser.add_argument("prefix") + parser.add_argument("method") + parser.add_argument("payload", nargs="?", default="") + return parser + + +async def _run(args: argparse.Namespace) -> bytes: + async with Client(args.broker, args.prefix) as rpc: + return await rpc.request(args.method, args.payload, timeout=args.timeout) + + +def main() -> None: + parser = _parser() + args = parser.parse_args() + try: + response = asyncio.run(_run(args)) + except (MQTTError, ProtocolError, RemoteError, TimeoutError, ValueError) as error: + parser.exit(1, f"mqtt-rpc: {error}\n") + if response: + print(response.decode(errors="replace")) + + +if __name__ == "__main__": + main() diff --git a/mqtt-rpc/py/mqtt_rpc/client.py b/mqtt-rpc/py/mqtt_rpc/client.py new file mode 100644 index 0000000..be756e5 --- /dev/null +++ b/mqtt-rpc/py/mqtt_rpc/client.py @@ -0,0 +1,185 @@ +"""Asynchronous MQTT RPC client.""" + +from __future__ import annotations + +import asyncio +import logging +import math +import uuid +from urllib.parse import urlsplit + +from gmqtt import Client as GmqttClient +from gmqtt import Subscription +from gmqtt.mqtt.constants import MQTTv50 +from gmqtt.mqtt.handler import MQTTError + +LOGGER = logging.getLogger(__name__) +RESPONSE_CODE_PROPERTY = "code" +SUCCESS_CODE = "Ok" + + +class RemoteError(Exception): + """Error response returned by the remote service.""" + + def __init__(self, code: str, payload: bytes): + self.code = code + self.payload = payload + super().__init__(f"{code}: {payload.decode(errors='replace')}") + + +class ProtocolError(Exception): + """Malformed response from the remote service.""" + + +def _first(properties: dict, name: str): + value = properties.get(name) + if isinstance(value, list): + return value[0] if value else None + return value + + +def _broker_address(broker: str) -> tuple[str, int]: + if broker.count(":") > 1 and not broker.startswith("["): + return broker, 1883 + parsed = urlsplit(f"//{broker}") + if parsed.hostname is None: + raise ValueError("broker must be a hostname or IP address") + return parsed.hostname, 1883 if parsed.port is None else parsed.port + + +class Client: + """One connected MQTT RPC requester with an exclusive MQTT session.""" + + def __init__( + self, + broker: str, + prefix: str, + *, + client_id: str | None = None, + keepalive: int = 60, + ): + self.broker = broker + self.prefix = prefix + self._topic_prefix = f"{prefix}/" + self.response_topic = f"{self._topic_prefix}response/{uuid.uuid4().hex}" + self.keepalive = keepalive + self._mqtt = GmqttClient(client_id or f"mqtt-rpc-py-{uuid.uuid4().hex}") + self._mqtt.on_message = self._on_message + self._mqtt.on_subscribe = self._on_subscribe + self._suback: asyncio.Future[tuple[int, ...]] | None = None + self._inflight: dict[bytes, asyncio.Future[bytes]] = {} + self._connected = False + + async def __aenter__(self) -> Client: + host, port = _broker_address(self.broker) + await self._mqtt.connect(host, port, keepalive=self.keepalive, version=MQTTv50) + self._connected = True + try: + await self._subscribe() + except BaseException: + await self.close() + raise + return self + + async def __aexit__(self, *_exc_info) -> None: + await self.close() + + async def close(self) -> None: + """Cancel outstanding requests and disconnect.""" + + if not self._connected: + return + self._connected = False + for future in self._inflight.values(): + future.cancel() + await self._mqtt.disconnect() + + async def request( + self, + method: str, + payload: str | bytes = b"", + *, + timeout: float = 3.0, + ) -> bytes: + """Call one method and return its successful response payload.""" + + if not self._connected: + raise MQTTError("MQTT RPC client is not connected") + if not method: + raise ValueError("method must not be empty") + if timeout <= 0 or not math.isfinite(timeout): + raise ValueError("timeout must be positive and finite") + + correlation = uuid.uuid4().bytes + future = asyncio.get_running_loop().create_future() + self._inflight[correlation] = future + properties = { + "response_topic": self.response_topic, + "correlation_data": correlation, + "message_expiry_interval": max(1, math.ceil(timeout)), + } + topic = f"{self._topic_prefix}rpc/{method}" + LOGGER.debug("Publishing request to %s", topic) + self._mqtt.publish(topic, payload, qos=1, retain=False, **properties) + try: + return await asyncio.wait_for(future, timeout) + finally: + self._inflight.pop(correlation, None) + + async def _subscribe(self) -> None: + subscription = Subscription( + self.response_topic, + qos=1, + no_local=True, + retain_as_published=False, + retain_handling_options=2, + ) + future = asyncio.get_running_loop().create_future() + self._suback = future + try: + self._mqtt.subscribe(subscription) + reasons = await asyncio.wait_for(future, 3.0) + finally: + self._suback = None + if not reasons or reasons[0] >= 128: + raise MQTTError(f"SUBACK failed for {self.response_topic}: {reasons}") + + def _on_subscribe( + self, + _client: GmqttClient, + _mid: int, + reasons: tuple[int, ...], + _properties: dict, + ) -> None: + if (future := self._suback) is not None and not future.done(): + future.set_result(reasons) + + def _on_message( + self, + _client: GmqttClient, + topic: str, + payload: bytes, + _qos: int, + properties: dict, + ) -> None: + if topic != self.response_topic: + return + correlation = _first(properties, "correlation_data") + future = self._inflight.get(correlation) + if future is None or future.done(): + LOGGER.debug("Discarding response with unknown correlation data") + return + try: + codes = [ + value + for name, value in properties.get("user_property", ()) + if name == RESPONSE_CODE_PROPERTY + ] + except (TypeError, ValueError): + codes = [] + if len(codes) != 1: + future.set_exception(ProtocolError("response must carry exactly one code")) + elif codes[0] == SUCCESS_CODE: + future.set_result(payload) + else: + future.set_exception(RemoteError(codes[0], payload)) diff --git a/mqtt-rpc/py/pyproject.toml b/mqtt-rpc/py/pyproject.toml new file mode 100644 index 0000000..184e389 --- /dev/null +++ b/mqtt-rpc/py/pyproject.toml @@ -0,0 +1,37 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "mqtt-rpc" +version = "0.1.0" +description = "Async MQTT 5 request/response client" +readme = "README.md" +requires-python = ">=3.11" +license = "MIT" +authors = [ + { name = "Ryan Summers", email = "ryan.summers@vertigo-designs.com" }, + { name = "Robert Jördens", email = "rj@quartiq.de" }, +] +dependencies = ["gmqtt>=0.7,<0.8"] + +[project.optional-dependencies] +dev = ["ruff>=0.11"] +test = ["pytest>=8"] + +[project.urls] +Homepage = "https://github.com/quartiq/minimq" +Repository = "https://github.com/quartiq/minimq.git" + +[project.scripts] +mqtt-rpc = "mqtt_rpc.__main__:main" + +[tool.setuptools] +packages = ["mqtt_rpc"] + +[tool.ruff] +target-version = "py311" +src = ["mqtt_rpc"] + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "I"] diff --git a/mqtt-rpc/py/tests/test_client.py b/mqtt-rpc/py/tests/test_client.py new file mode 100644 index 0000000..5fa9325 --- /dev/null +++ b/mqtt-rpc/py/tests/test_client.py @@ -0,0 +1,59 @@ +import asyncio +import os +import uuid + +import pytest +from gmqtt import Client as GmqttClient +from gmqtt import Subscription +from gmqtt.mqtt.constants import MQTTv50 +from mqtt_rpc.client import Client, RemoteError + + +def _first(properties, name): + value = properties.get(name) + return value[0] if isinstance(value, list) else value + + +@pytest.mark.parametrize("trailing_slash", [False, True]) +def test_request_response(trailing_slash): + broker = os.getenv("BROKER") + if broker is None: + pytest.skip("set BROKER=host:port to run the broker test") + asyncio.run(_request_response(broker, trailing_slash)) + + +async def _request_response(broker, trailing_slash): + host, port = broker.rsplit(":", 1) + prefix = f"mqtt-rpc-python-test-{uuid.uuid4().hex}" + service_prefix = f"{prefix}/" if trailing_slash else prefix + rpc_topic = f"{service_prefix}/rpc" + subscribed = asyncio.Event() + device = GmqttClient(f"{prefix}-device") + + def on_subscribe(_client, _mid, _reasons, _properties): + subscribed.set() + + def on_message(client, topic, payload, _qos, properties): + assert topic in {f"{rpc_topic}/ping", f"{rpc_topic}//ping"} + code = "Failed" if payload == b"fail" else "Ok" + client.publish( + _first(properties, "response_topic"), + b"failed" if payload == b"fail" else b"pong", + qos=1, + correlation_data=_first(properties, "correlation_data"), + user_property=[("code", code)], + ) + + device.on_subscribe = on_subscribe + device.on_message = on_message + await device.connect(host, int(port), version=MQTTv50) + device.subscribe(Subscription(f"{rpc_topic}/#", qos=1)) + await asyncio.wait_for(subscribed.wait(), 3.0) + try: + async with Client(broker, service_prefix) as rpc: + assert await rpc.request("ping", b"ping") == b"pong" + assert await rpc.request("/ping", b"ping") == b"pong" + with pytest.raises(RemoteError, match="Failed: failed"): + await rpc.request("ping", b"fail") + finally: + await device.disconnect() diff --git a/mqtt-rpc/release.toml b/mqtt-rpc/release.toml new file mode 100644 index 0000000..3d6d266 --- /dev/null +++ b/mqtt-rpc/release.toml @@ -0,0 +1,13 @@ +[[pre-release-replacements]] +file = "CHANGELOG.md" +search = "## \\[UNRELEASED\\]\\(https?://[^)]+?/compare/(?P[^)]+)\\.\\.\\.HEAD\\) - DATE" +replace = """## [UNRELEASED](https://github.com/quartiq/minimq/compare/{{tag_name}}...HEAD) - DATE + +## [{{version}}](https://github.com/quartiq/minimq/compare/${previous}...{{tag_name}}) - {{date}}""" +exactly = 1 + +[[pre-release-replacements]] +file = "py/pyproject.toml" +search = '^version = "[^"]+"$' +replace = 'version = "{{version}}"' +exactly = 1 diff --git a/mqtt-rpc/src/lib.rs b/mqtt-rpc/src/lib.rs new file mode 100644 index 0000000..0f45a0a --- /dev/null +++ b/mqtt-rpc/src/lib.rs @@ -0,0 +1,330 @@ +#![no_std] +#![warn(missing_docs)] + +//! MQTT 5 request/response transport for embedded services. +//! +//! `mqtt-rpc` owns neither the MQTT session nor application dispatch. It subscribes one +//! `/rpc/#` topic filter, extracts MQTT response topics and correlation data from +//! matching requests, and sends transient correlated responses. Applications remain responsible +//! for request payloads, method dispatch, execution, and durable state publication. + +use heapless::String; +use minimq::{ + ConnectEvent, Connection, Error as MqttError, InboundPublish, Io, Op, OwnedResponseTarget, + Property, PubError, QoS, ResourceError, RetainHandling, SubscriptionOptions, ToPayload, + TopicFilter, +}; + +/// Maximum request and response topic length retained by the service. +pub const MAX_TOPIC_LENGTH: usize = 128; + +/// Maximum MQTT correlation-data length retained for a deferred response. +pub const MAX_CORRELATION_LENGTH: usize = 32; + +/// Expiry applied to transient RPC responses. +pub const RESPONSE_EXPIRY_SECS: u32 = 30; + +/// MQTT user-property name carrying an RPC response code. +pub const RESPONSE_CODE_PROPERTY: &str = "code"; + +/// Response code indicating successful request execution. +pub const SUCCESS_CODE: &str = "Ok"; + +const RPC_SUFFIX: &str = "/rpc"; +const RPC_FILTER_SUFFIX: &str = "/#"; + +/// An owned MQTT response destination and optional correlation data. +pub type ResponseTarget = OwnedResponseTarget; + +/// Invalid service configuration. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ConfigError { + /// The resulting RPC subscription topic exceeds [`MAX_TOPIC_LENGTH`]. + TopicTooLong, +} + +/// Why an inbound RPC publication was rejected before application dispatch. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RejectReason { + /// RPC requests must not be retained broker state. + Retained, + /// The request did not provide an MQTT response topic. + MissingResponseTopic, + /// The response topic or correlation data exceeds fixed local storage. + ResponseTargetTooLong, + /// The request addressed `/rpc` without a method path. + EmptyMethod, +} + +impl RejectReason { + /// Return the stable response code for this rejection. + pub const fn code(self) -> &'static str { + match self { + Self::Retained => "Retained", + Self::MissingResponseTopic => "MissingResponseTopic", + Self::ResponseTargetTooLong => "ResponseTargetTooLong", + Self::EmptyMethod => "EmptyMethod", + } + } +} + +/// A valid request borrowing its method and payload from the inbound MQTT packet. +#[derive(Debug)] +pub struct Request<'a> { + method: &'a str, + payload: &'a [u8], + response: ResponseTarget, +} + +impl<'a> Request<'a> { + /// Return the method topic suffix following `/rpc/`. + pub const fn method(&self) -> &'a str { + self.method + } + + /// Return the request payload. + pub const fn payload(&self) -> &'a [u8] { + self.payload + } + + /// Consume the request and retain its response destination for deferred completion. + pub fn into_response_target(self) -> ResponseTarget { + self.response + } +} + +/// A rejected request and any response destination that could safely be retained. +#[derive(Debug)] +pub struct Rejected { + reason: RejectReason, + response: Option, +} + +impl Rejected { + /// Return the rejection reason. + pub const fn reason(&self) -> RejectReason { + self.reason + } + + /// Consume the rejection and return its response destination, when available. + pub fn into_response_target(self) -> Option { + self.response + } +} + +/// Classification of one inbound MQTT publication. +#[derive(Debug)] +pub enum Handle<'a> { + /// The publication is outside this service's RPC topic tree. + Unhandled, + /// The publication belongs to the service but is not a valid request. + Rejected(Rejected), + /// A valid request ready for application dispatch. + Request(Request<'a>), +} + +/// MQTT RPC topic routing and subscription state. +pub struct Service { + rpc_topic: String, + subscribe: Option, + ready: bool, +} + +impl Service { + /// Construct an RPC service below one device prefix. + pub fn new(device_prefix: &str) -> Result { + if device_prefix.len() + RPC_SUFFIX.len() + RPC_FILTER_SUFFIX.len() > MAX_TOPIC_LENGTH { + return Err(ConfigError::TopicTooLong); + } + + let mut rpc_topic = String::new(); + rpc_topic + .push_str(device_prefix) + .map_err(|_| ConfigError::TopicTooLong)?; + rpc_topic + .push_str(RPC_SUFFIX) + .map_err(|_| ConfigError::TopicTooLong)?; + + Ok(Self { + rpc_topic, + subscribe: None, + ready: false, + }) + } + + /// Begin service startup for a newly connected or resumed MQTT session. + pub fn begin_connection(&mut self, event: ConnectEvent) { + self.subscribe = None; + self.ready = matches!(event, ConnectEvent::Reconnected); + } + + /// Return whether the request subscription is active. + pub const fn is_ready(&self) -> bool { + self.ready + } + + /// Advance subscription startup without consuming inbound publications. + /// + /// Returns `Ok(true)` when the service is ready. The caller must continue driving the MQTT + /// connection between calls while this returns `Ok(false)`. + pub async fn step( + &mut self, + connection: &mut Connection<'_, '_, IO>, + ) -> Result> { + if self.ready { + return Ok(true); + } + + if let Some(op) = self.subscribe { + if connection.is_pending(&op) { + return Ok(false); + } + if connection.is_complete(&op) { + self.subscribe = None; + self.ready = true; + return Ok(true); + } + debug_assert!(connection.is_invalidated(&op)); + self.subscribe = None; + return Err(MqttError::Disconnected); + } + + let options = SubscriptionOptions::default() + .maximum_qos(QoS::AtLeastOnce) + .retain_behavior(RetainHandling::Never) + .retain_as_published() + .ignore_local_messages(); + let mut rpc_filter = self.rpc_topic.clone(); + rpc_filter.push_str(RPC_FILTER_SUFFIX).unwrap(); + match connection + .subscribe(&[TopicFilter::new(&rpc_filter).options(options)], &[]) + .await + { + Ok(op) => self.subscribe = Some(op), + Err(MqttError::NotReady | MqttError::Resource(ResourceError::InflightExhausted)) => {} + Err(err) => return Err(err), + } + Ok(false) + } + + /// Classify an inbound publication and retain the response destination for valid requests. + pub fn handle<'a>(&self, inbound: &'a InboundPublish<'a>) -> Handle<'a> { + let Some(method) = self.method(inbound.topic()) else { + return Handle::Unhandled; + }; + + let response = inbound.reply_owned::(); + if inbound.retained() { + return Handle::Rejected(Rejected { + reason: RejectReason::Retained, + response: response.ok().flatten(), + }); + } + let response = match response { + Ok(response) => response, + Err(_) => { + return Handle::Rejected(Rejected { + reason: RejectReason::ResponseTargetTooLong, + response: None, + }); + } + }; + let Some(response) = response else { + return Handle::Rejected(Rejected { + reason: RejectReason::MissingResponseTopic, + response: None, + }); + }; + if method.is_empty() { + return Handle::Rejected(Rejected { + reason: RejectReason::EmptyMethod, + response: Some(response), + }); + } + + Handle::Request(Request { + method, + payload: inbound.payload(), + response, + }) + } + + fn method<'a>(&self, topic: &'a str) -> Option<&'a str> { + let suffix = topic.strip_prefix(self.rpc_topic.as_str())?; + if suffix.is_empty() { + return Some(""); + } + suffix.strip_prefix('/') + } +} + +/// Publish a transient, correlated RPC response. +/// +/// `code` is attached as the [`RESPONSE_CODE_PROPERTY`] MQTT user property. Use +/// [`SUCCESS_CODE`] for success; other values are application-defined failures. Responses use +/// QoS 1, are never retained, and expire after [`RESPONSE_EXPIRY_SECS`]. Payload interpretation +/// and payload-format properties remain application-owned. +pub async fn respond( + connection: &mut Connection<'_, '_, IO>, + target: &ResponseTarget, + code: &str, + payload: P, +) -> Result, PubError> +where + IO: Io, + P: ToPayload, +{ + let properties = [ + Property::MessageExpiryInterval(RESPONSE_EXPIRY_SECS), + Property::UserProperty(RESPONSE_CODE_PROPERTY, code), + ]; + connection + .publish( + target + .publication(payload) + .properties(&properties) + .qos(QoS::AtLeastOnce), + ) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn joins_prefix() { + assert_eq!( + Service::new("device").unwrap().rpc_topic.as_str(), + "device/rpc" + ); + assert_eq!(Service::new("").unwrap().rpc_topic.as_str(), "/rpc"); + assert_eq!( + Service::new("device/").unwrap().rpc_topic.as_str(), + "device//rpc" + ); + } + + #[test] + fn routes_only_the_rpc_tree() { + let service = Service::new("root/device").unwrap(); + assert_eq!(service.method("root/device/rpc"), Some("")); + assert_eq!( + service.method("root/device/rpc/settings/store"), + Some("settings/store") + ); + // Empty topic levels are valid MQTT syntax and remain application-visible. + assert_eq!( + service.method("root/device/rpc//settings/store"), + Some("/settings/store") + ); + assert_eq!(service.method("root/device/rpcx/store"), None); + assert_eq!(service.method("other/device/rpc/store"), None); + } + + #[test] + fn rejects_oversized_topic() { + let prefix = "x".repeat(MAX_TOPIC_LENGTH); + assert_eq!(Service::new(&prefix).err(), Some(ConfigError::TopicTooLong)); + } +} diff --git a/mqtt-rpc/tests/integration.rs b/mqtt-rpc/tests/integration.rs new file mode 100644 index 0000000..4a6a936 --- /dev/null +++ b/mqtt-rpc/tests/integration.rs @@ -0,0 +1,110 @@ +use embedded_io_adapters::tokio_1::FromTokio; +use minimq::{ + ConfigBuilder, Property, Publication, QoS, Session, SubscriptionOptions, TopicFilter, +}; +use mqtt_rpc::{Handle, SUCCESS_CODE, Service, respond}; +use std::{net::SocketAddr, sync::OnceLock}; +use tokio::{ + net::TcpStream, + time::{Duration, timeout}, +}; + +fn init_logging() { + static INIT: OnceLock<()> = OnceLock::new(); + INIT.get_or_init(|| { + env_logger::builder().is_test(true).try_init().unwrap(); + #[cfg(feature = "defmt")] + defmt2log::init_from_current_exe(); + }); +} + +fn config() -> ConfigBuilder<'static> { + ConfigBuilder::from_buffer(Box::leak(Box::new([0; 2048])), 1024).unwrap() +} + +#[tokio::test] +async fn request_response() { + init_logging(); + let Some(addr) = std::env::var("BROKER") + .ok() + .map(|addr| addr.parse::().unwrap()) + else { + eprintln!("skipping broker test; set BROKER=host:port"); + return; + }; + + let io = FromTokio::new(TcpStream::connect(addr).await.unwrap()); + let mut device_session = Session::new(config()); + let mut device = timeout(Duration::from_secs(5), device_session.connect(io)) + .await + .unwrap() + .unwrap(); + let prefix = format!("mqtt-rpc-test-{}", std::process::id()); + let mut rpc = Service::new(&prefix).unwrap(); + rpc.begin_connection(device.connect_event()); + while !rpc.step(&mut device).await.unwrap() { + let _ = device.poll().await.unwrap(); + } + + let io = FromTokio::new(TcpStream::connect(addr).await.unwrap()); + let mut client_session = Session::new(config()); + let mut client = timeout(Duration::from_secs(5), client_session.connect(io)) + .await + .unwrap() + .unwrap(); + let response_topic = format!("{prefix}/response"); + let subscription = client + .subscribe( + &[TopicFilter::new(&response_topic) + .options(SubscriptionOptions::default().maximum_qos(QoS::AtLeastOnce))], + &[], + ) + .await + .unwrap(); + while client.is_pending(&subscription) { + let _ = client.poll().await.unwrap(); + } + + let correlation = b"test"; + let properties = [ + Property::ResponseTopic(&response_topic), + Property::CorrelationData(correlation), + ]; + client + .publish( + Publication::bytes(&format!("{prefix}/rpc/ping"), b"ping") + .properties(&properties) + .qos(QoS::AtLeastOnce), + ) + .await + .unwrap(); + + let inbound = timeout(Duration::from_secs(5), device.recv()) + .await + .unwrap() + .unwrap(); + let Handle::Request(request) = rpc.handle(&inbound) else { + panic!("request was not routed"); + }; + assert_eq!( + (request.method(), request.payload()), + ("ping", b"ping".as_slice()) + ); + let target = request.into_response_target(); + respond(&mut device, &target, SUCCESS_CODE, b"pong".as_slice()) + .await + .unwrap(); + + let response = timeout(Duration::from_secs(5), client.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(response.payload(), b"pong"); + assert_eq!(response.correlation_data(), Some(correlation.as_slice())); + assert!( + response + .properties() + .iter() + .any(|property| matches!(property, Ok(Property::UserProperty("code", SUCCESS_CODE)))) + ); +} diff --git a/mqtt-staging/CHANGELOG.md b/mqtt-staging/CHANGELOG.md new file mode 100644 index 0000000..6e5ef58 --- /dev/null +++ b/mqtt-staging/CHANGELOG.md @@ -0,0 +1,17 @@ + +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [UNRELEASED](https://github.com/quartiq/minimq/compare/0ae37e5...HEAD) - DATE + +## Fixed + +* Refresh the MQTT receive limit for reconstructed services on resumed sessions. + +## [0.1.0](https://github.com/quartiq/mqtt-staging/releases/tag/v0.1.0) - 2026-09-01 + +* Initial release diff --git a/mqtt-staging/Cargo.toml b/mqtt-staging/Cargo.toml new file mode 100644 index 0000000..65b9fba --- /dev/null +++ b/mqtt-staging/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "mqtt-staging" +version = "0.1.0" +authors = ["Robert Jördens "] +edition.workspace = true +description = "Stage bounded objects into embedded storage over MQTT" +repository.workspace = true +categories = ["embedded", "no-std", "network-programming"] +keywords = ["mqtt", "embedded", "no-std", "staging"] +license = "MIT OR Apache-2.0" +rust-version = "1.93" + +[lints] +workspace = true + +[dependencies] +defmt = "1.0.1" +heapless = "0.9.1" +itoa = "1" +minimq = { path = "../minimq", version = "0.13" } +serde = { version = "1.0", features = ["derive"], default-features = false } +serde-json-core = { version = "0.6.0", default-features = false } + +[dev-dependencies] +critical-section = { version = "1", features = ["std"] } +defmt2log = "0.2.0" +embassy-futures = { version = "0.1.2", default-features = false } +embassy-time = { version = "0.5.1", default-features = false, features = ["std", "generic-queue-8"] } +embedded-io-async = { version = "0.7", features = ["std"] } +env_logger = "0.11" diff --git a/mqtt-staging/LICENSE-APACHE b/mqtt-staging/LICENSE-APACHE new file mode 100644 index 0000000..5e0fd33 --- /dev/null +++ b/mqtt-staging/LICENSE-APACHE @@ -0,0 +1,201 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, +and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by +the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all +other entities that control, are controlled by, or are under common +control with that entity. For the purposes of this definition, +"control" means (i) the power, direct or indirect, to cause the +direction or management of such entity, whether by contract or +otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity +exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, +including but not limited to software source code, documentation +source, and configuration files. + +"Object" form shall mean any form resulting from mechanical +transformation or translation of a Source form, including but +not limited to compiled object code, generated documentation, +and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or +Object form, made available under the License, as indicated by a +copyright notice that is included in or attached to the work +(an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object +form, that is based on (or derived from) the Work and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. For the purposes +of this License, Derivative Works shall not include works that remain +separable from, or merely link (or bind by name) to the interfaces of, +the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including +the original version of the Work and any modifications or additions +to that Work or Derivative Works thereof, that is intentionally +submitted to Licensor for inclusion in the Work by the copyright owner +or by an individual or Legal Entity authorized to submit on behalf of +the copyright owner. For the purposes of this definition, "submitted" +means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, +and issue tracking systems that are managed by, or on behalf of, the +Licensor for the purpose of discussing and improving the Work, but +excluding communication that is conspicuously marked or otherwise +designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity +on behalf of whom a Contribution has been received by Licensor and +subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the +Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except as stated in this section) patent license to make, have made, +use, offer to sell, sell, import, and otherwise transfer the Work, +where such license applies only to those patent claims licensable +by such Contributor that are necessarily infringed by their +Contribution(s) alone or by combination of their Contribution(s) +with the Work to which such Contribution(s) was submitted. If You +institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work +or a Contribution incorporated within the Work constitutes direct +or contributory patent infringement, then any patent licenses +granted to You under this License for that Work shall terminate +as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the +Work or Derivative Works thereof in any medium, with or without +modifications, and in Source or Object form, provided that You +meet the following conditions: + +(a) You must give any other recipients of the Work or +Derivative Works a copy of this License; and + +(b) You must cause any modified files to carry prominent notices +stating that You changed the files; and + +(c) You must retain, in the Source form of any Derivative Works +that You distribute, all copyright, patent, trademark, and +attribution notices from the Source form of the Work, +excluding those notices that do not pertain to any part of +the Derivative Works; and + +(d) If the Work includes a "NOTICE" text file as part of its +distribution, then any Derivative Works that You distribute must +include a readable copy of the attribution notices contained +within such NOTICE file, excluding those notices that do not +pertain to any part of the Derivative Works, in at least one +of the following places: within a NOTICE text file distributed +as part of the Derivative Works; within the Source form or +documentation, if provided along with the Derivative Works; or, +within a display generated by the Derivative Works, if and +wherever such third-party notices normally appear. The contents +of the NOTICE file are for informational purposes only and +do not modify the License. You may add Your own attribution +notices within Derivative Works that You distribute, alongside +or as an addendum to the NOTICE text from the Work, provided +that such additional attribution notices cannot be construed +as modifying the License. + +You may add Your own copyright statement to Your modifications and +may provide additional or different license terms and conditions +for use, reproduction, or distribution of Your modifications, or +for any such Derivative Works as a whole, provided Your use, +reproduction, and distribution of the Work otherwise complies with +the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, +any Contribution intentionally submitted for inclusion in the Work +by You to the Licensor shall be under the terms and conditions of +this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify +the terms of any separate license agreement you may have executed +with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade +names, trademarks, service marks, or product names of the Licensor, +except as required for reasonable and customary use in describing the +origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or +agreed to in writing, Licensor provides the Work (and each +Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied, including, without limitation, any warranties or conditions +of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A +PARTICULAR PURPOSE. You are solely responsible for determining the +appropriateness of using or redistributing the Work and assume any +risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, +whether in tort (including negligence), contract, or otherwise, +unless required by applicable law (such as deliberate and grossly +negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, +incidental, or consequential damages of any character arising as a +result of this License or out of the use or inability to use the +Work (including but not limited to damages for loss of goodwill, +work stoppage, computer failure or malfunction, or any and all +other commercial damages or losses), even if such Contributor +has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing +the Work or Derivative Works thereof, You may choose to offer, +and charge a fee for, acceptance of support, warranty, indemnity, +or other liability obligations and/or rights consistent with this +License. However, in accepting such obligations, You may act only +on Your own behalf and on Your sole responsibility, not on behalf +of any other Contributor, and only if You agree to indemnify, +defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason +of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following +boilerplate notice, with the fields enclosed by brackets "{}" +replaced with your own identifying information. (Don't include +the brackets!) The text should be enclosed in the appropriate +comment syntax for the file format. We also recommend that a +file or class name and description of purpose be included on the +same "printed page" as the copyright notice for easier +identification within third-party archives. + +Copyright {yyyy} {name of copyright owner} + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/mqtt-staging/LICENSE-MIT b/mqtt-staging/LICENSE-MIT new file mode 100644 index 0000000..3f55a09 --- /dev/null +++ b/mqtt-staging/LICENSE-MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2019-2021 QUARTIQ GmbH + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/mqtt-staging/README.md b/mqtt-staging/README.md new file mode 100644 index 0000000..d7d0b2d --- /dev/null +++ b/mqtt-staging/README.md @@ -0,0 +1,113 @@ +# mqtt-staging + +`mqtt-staging` moves one bounded byte object into application-owned storage over +MQTT 5. It provides a `no_std` Rust device state machine and one Python host +command. The device requests sequential, write-aligned chunks, so an embedded +application can stage directly into flash without buffering the whole object. + +Firmware is the prime use case (notably Stabilizer with Embassy flash), but OTA +policy is deliberately outside the protocol. The application still owns MQTT +I/O, storage, allocation, timeouts, checksum verification, activation, and +reboot. FNV-1a detects staging errors; it does not authenticate firmware. + +## Host + +```console +pipx install ./py +mqtt-staging --prefix dt/sinara/stabilizer/ota --file firmware.bin +``` + +`--broker` defaults to `$BROKER`, then `localhost:1883`. From a checkout, +`python py/mqtt_staging.py ...` runs the same command. The prefix is the exact root +chosen by the application; the command appends only `/manifest`, `/status`, and +`/chunk`. + +## Device + +```toml +[dependencies] +mqtt-staging = "0.1" +``` + +Create a service with the staging capacity and direct-write limits: + +```rust,ignore +let mut staging = mqtt_staging::Service::new( + "dt/sinara/stabilizer/ota", + mqtt_staging::Config { + capacity: firmware.capacity(), + max_chunk_size: 4096, + write_size: firmware.write_size(), + }, +)?; +``` + +An Embassy flash worker can handle the emitted requests directly: + +```rust,ignore +let ok = match request { + mqtt_staging::StagingRequest::Prepare { size } => + firmware.prepare(size).await.is_ok(), + mqtt_staging::StagingRequest::Write(write) => { + let mut ok = firmware.write(write.offset, write.payload).await.is_ok(); + if ok && let Some(expected) = write.fnv1a64 { + ok = firmware.finish(write.size, expected).await.is_ok(); + } + ok + } +}; +staging.complete_request(ok); +``` + +Here `firmware.write()` can be a thin call to Embassy +`FirmwareUpdater::write_firmware()`. `finish()` verifies the staged bytes; the +application decides whether to call `mark_updated()` and reboot. + +Reconnects preserve transfer state only while the same `Service` remains alive. +A device reset starts from idle unless the application restores that state. + +Call `begin_startup()` after each MQTT connection, `step()` until it is +quiescent, and route inbound publishes through `handle()`. `StagingWrite` borrows +Minimq's current RX packet. A concurrent worker must copy that payload once +before the next connection operation; a synchronous worker may consume it in +place. + +## Protocol + +| Topic | Publisher | Payload | +| --- | --- | --- | +| `/manifest` | host | JSON manifest | +| `/status` | device | JSON state and next-chunk properties | +| `/chunk` | host | object bytes | + +```json +{"id":"85944171f73967e8","size":6,"fnv1a64":9625390261332436968} +``` + +`id` is a 1–48 byte ASCII token using letters, digits, `-`, `_`, or `.`. The +device echoes it in status and as MQTT `CorrelationData`. Status also contains +`state`, `code`, `next_offset`, `size`, `mtu`, and `write_size`, with: + +- `ResponseTopic`: `/chunk` +- `UserProperty("offset", decimal)`: requested byte offset + +The state path is `idle -> preparing -> ready <-> writing -> complete`; an +active transfer may instead end in `error`. QoS 1 manifest and chunk duplicates +are idempotent. Chunks are stop-and-wait and sequential; future offsets fail, +and final alignment padding must be `0xff`. A new transfer requires a new +service instance after `complete` or `error`. + +## Tests + +```console +python -m unittest discover -s py/tests -p 'test_*.py' +cargo test --all-targets +BROKER=localhost:1883 cargo test --test end_to_end -- --ignored +``` + +The ignored test crosses a real broker from the Python command through Minimq +to mock storage. Set `MQTT_STAGING_FEEDER` to override the command. + +## License + +Licensed under either Apache-2.0 or MIT. diff --git a/mqtt-staging/py/README.md b/mqtt-staging/py/README.md new file mode 100644 index 0000000..1d7d6ba --- /dev/null +++ b/mqtt-staging/py/README.md @@ -0,0 +1,12 @@ +# `mqtt-staging` Python sender + +Python 3.11+ sender for the MQTT Staging protocol implemented by the sibling +Rust crate. + +```sh +python -m pip install mqtt-staging +mqtt-staging --prefix dt/device/staging --file object.bin +``` + +The command sends one bounded object. The device owns storage preparation, +validation, activation, and reboot policy. diff --git a/mqtt-staging/py/mqtt_staging.py b/mqtt-staging/py/mqtt_staging.py new file mode 100644 index 0000000..57d4b0a --- /dev/null +++ b/mqtt-staging/py/mqtt_staging.py @@ -0,0 +1,426 @@ +"""Host-side sender for the mqtt-staging protocol.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +from gmqtt import Client +from gmqtt.mqtt.constants import MQTTv50 +from gmqtt.mqtt.handler import MQTTError + +LOGGER = logging.getLogger(__name__) +FNV1A64_OFFSET = 0xCBF29CE484222325 +FNV1A64_PRIME = 0x100000001B3 +INFO_CHUNK_STRIDE = 128 +QOS_AT_LEAST_ONCE = 1 +MqttProperties = dict[str, Any] + + +@dataclass(frozen=True, slots=True) +class Message: + """One MQTT PUBLISH received by the staging command.""" + + topic: str + payload: bytes + properties: MqttProperties + + +@dataclass(frozen=True, slots=True) +class Transfer: + """Identity of the manifest being served.""" + + id: str + size: int + + @classmethod + def from_manifest(cls, manifest: bytes) -> Transfer: + data = json.loads(manifest) + return cls(data["id"], data["size"]) + + def matches(self, status: Status) -> bool: + return ( + status.id == self.id + and status.correlation_data == self.id.encode() + and status.size == self.size + ) + + +class MqttClient: + """Small gmqtt adapter used by the staging command.""" + + def __init__(self, broker: str): + host, port = broker_endpoint(broker) + self.host = host + self.port = port + self.messages: asyncio.Queue[Message] = asyncio.Queue() + self._client = Client(f"mqtt-staging-{uuid.uuid4().hex}", clean_session=True) + self._client.on_message = self._on_message + self._client.on_subscribe = self._on_subscribe + self._subacks: dict[int, asyncio.Future[tuple[int, ...]]] = {} + + async def __aenter__(self) -> MqttClient: + await self._client.connect(self.host, self.port, version=MQTTv50) + return self + + async def __aexit__(self, *_exc_info) -> None: + await self._client.disconnect() + + async def subscribe(self, topic: str, *, qos: int) -> None: + mid = self._client.subscribe(topic, qos=qos) + if mid is None: + return + future = asyncio.get_running_loop().create_future() + self._subacks[mid] = future + reasons = await asyncio.wait_for(future, 3.0) + if not reasons or reasons[0] >= 128: + raise MQTTError(f"SUBACK failed for {topic}: {reasons}") + + async def publish( + self, + topic: str, + payload: bytes, + *, + qos: int, + properties: MqttProperties, + ) -> None: + self._client.publish(topic, payload, qos=qos, **properties) + await asyncio.sleep(0) + + def _on_message( + self, + _client: Client, + topic: str, + payload: bytes, + _qos: int, + properties: MqttProperties, + ) -> None: + self.messages.put_nowait(Message(topic, payload, properties)) + + def _on_subscribe( + self, + _client: Client, + mid: int, + reasons: tuple[int, ...], + _properties: MqttProperties, + ) -> None: + if future := self._subacks.pop(mid, None): + future.set_result(reasons) + + +@dataclass(frozen=True, slots=True) +class Status: + """One device staging status receipt.""" + + state: str + code: str + id: str + response_topic: str + correlation_data: bytes + offset: int + next_offset: int + size: int + mtu: int + write_size: int + + @classmethod + def from_message(cls, message) -> Status: + data = json.loads(message.payload) + properties = message.properties + user_properties = dict(properties.get("user_property", ())) + return cls( + state=data["state"], + code=data["code"], + id=data["id"], + response_topic=_first(properties, "response_topic"), + correlation_data=_first(properties, "correlation_data"), + offset=int(user_properties["offset"]), + next_offset=data["next_offset"], + size=data["size"], + mtu=data["mtu"], + write_size=data["write_size"], + ) + + def raise_for_error(self) -> None: + if self.state == "error" or self.code in { + "error", + "mtu", + "offset", + "oversize", + "storage", + "unaligned", + }: + raise RuntimeError(f"device rejected staging: {self}") + + +def _first(properties: MqttProperties, name: str, default=None): + value = properties.get(name, default) + if isinstance(value, list): + return value[0] if value else default + return value + + +def aligned_chunk_size(requested: int, mtu: int, write_size: int) -> int: + """Choose a chunk size that meets the device write alignment.""" + + size = min(requested, mtu) + return size - size % write_size + + +def chunk_properties(correlation_data: bytes, offset: int) -> MqttProperties: + return { + "payload_format_id": 0, + "correlation_data": correlation_data, + "user_property": [("offset", str(offset))], + } + + +def should_log_chunk_progress( + *, offset: int, payload_len: int, size: int, chunk_size: int +) -> bool: + if offset == 0: + return True + if offset + payload_len >= size: + return True + if chunk_size <= 0: + return False + return (offset // chunk_size) % INFO_CHUNK_STRIDE == 0 + + +def fnv1a64(data: bytes) -> int: + digest = FNV1A64_OFFSET + for byte in data: + digest ^= byte + digest = digest * FNV1A64_PRIME & 0xFFFFFFFFFFFFFFFF + return digest + + +def json_manifest(data: bytes) -> bytes: + checksum = fnv1a64(data) + manifest = { + "id": f"{checksum:016x}", + "size": len(data), + "fnv1a64": checksum, + } + return json.dumps(manifest, separators=(",", ":")).encode() + + +def broker_endpoint(broker: str) -> tuple[str, int]: + """Normalize one MQTT broker address to host and port.""" + + endpoint = urlsplit(broker if "://" in broker else f"mqtt://{broker}") + return endpoint.hostname or broker, endpoint.port or 1883 + + +async def wait_status( + messages: asyncio.Queue[Message], + topic: str, + transfer: Transfer, + *, + timeout: float, +) -> Status: + """Wait for the next status/request receipt.""" + + async with asyncio.timeout(timeout): + while True: + message = await messages.get() + if message.topic != topic: + continue + status = Status.from_message(message) + if not status.id and not status.correlation_data: + status.raise_for_error() + continue + if not transfer.matches(status): + LOGGER.debug("ignoring status for another staging transfer") + continue + status.raise_for_error() + return status + raise TimeoutError(f"timed out waiting for {topic}") + + +async def stage( + *, + broker: str, + prefix: str, + manifest: bytes, + data: bytes, + chunk_size: int, + prepare_timeout: float, + timeout: float, +) -> None: + """Stage one object through a device's MQTT service.""" + + host, port = broker_endpoint(broker) + LOGGER.info( + "staging broker=%s:%s prefix=%s size=%dB chunk=%dB", + host, + port, + prefix, + len(data), + chunk_size, + ) + transfer = Transfer.from_manifest(manifest) + async with MqttClient(broker) as client: + status_topic = f"{prefix}/status" + manifest_topic = f"{prefix}/manifest" + messages = client.messages + LOGGER.info("subscribing status topic %s", status_topic) + await client.subscribe(status_topic, qos=QOS_AT_LEAST_ONCE) + LOGGER.info("publishing manifest topic %s", manifest_topic) + await client.publish( + manifest_topic, + manifest, + qos=QOS_AT_LEAST_ONCE, + properties={"payload_format_id": 1}, + ) + preparing = True + while True: + status = await wait_status( + messages, + status_topic, + transfer, + timeout=prepare_timeout if preparing else timeout, + ) + LOGGER.debug( + ( + "status state=%s code=%s offset=%s next=%s/%s " + "mtu=%s write=%s response=%s correlation=%dB" + ), + status.state, + status.code, + status.offset, + status.next_offset, + status.size, + status.mtu, + status.write_size, + status.response_topic, + len(status.correlation_data or b""), + ) + if status.code == "complete": + LOGGER.info("staging complete size=%s", status.size) + return + if ( + status.response_topic != f"{prefix}/chunk" + or status.offset != status.next_offset + ): + raise RuntimeError(f"invalid staging chunk request: {status}") + if status.state == "preparing": + LOGGER.info( + "status state=%s code=%s next=%s/%s", + status.state, + status.code, + status.next_offset, + status.size, + ) + continue + preparing = False + if should_log_chunk_progress( + offset=status.offset, + payload_len=max(status.next_offset - status.offset, 0), + size=status.size, + chunk_size=max(status.mtu, 1), + ): + LOGGER.info( + "status code=%s offset=%s next=%s/%s mtu=%s write=%s", + status.code, + status.offset, + status.next_offset, + status.size, + status.mtu, + status.write_size, + ) + size = aligned_chunk_size(chunk_size, status.mtu, status.write_size) + if size == 0: + raise RuntimeError( + "device MTU cannot fit one aligned write: " + f"mtu={status.mtu} write_size={status.write_size}" + ) + offset = status.offset + chunk = data[offset : offset + size] + if not chunk: + raise RuntimeError(f"device requested empty chunk at {offset}") + raw_len = len(chunk) + if offset + len(chunk) >= len(data): + chunk += b"\xff" * ((-len(chunk)) % status.write_size) + if should_log_chunk_progress( + offset=offset, + payload_len=raw_len, + size=len(data), + chunk_size=max(size, 1), + ): + LOGGER.info( + "publishing chunk offset=%s raw=%sB payload=%sB", + offset, + raw_len, + len(chunk), + ) + else: + LOGGER.debug( + "publishing chunk offset=%s raw=%sB payload=%sB", + offset, + raw_len, + len(chunk), + ) + await client.publish( + status.response_topic, + chunk, + qos=QOS_AT_LEAST_ONCE, + properties=chunk_properties(status.correlation_data, offset), + ) + + +async def async_main() -> None: + parser = argparse.ArgumentParser( + description="Stage one object over MQTT.", + ) + parser.add_argument("-v", "--verbose", action="count", default=0) + parser.add_argument( + "-b", + "--broker", + default=os.environ.get("BROKER", "localhost:1883"), + ) + parser.add_argument( + "-p", + "--prefix", + required=True, + help="complete protocol topic root, for example devices/example/staging", + ) + parser.add_argument("--file", type=Path, required=True) + parser.add_argument("--chunk-size", type=int, default=1024) + parser.add_argument("--prepare-timeout", type=float, default=30.0) + parser.add_argument("--timeout", type=float, default=10.0) + args = parser.parse_args() + + logging.basicConfig( + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + level=logging.INFO if args.verbose == 0 else logging.DEBUG, + ) + + data = args.file.read_bytes() + manifest = json_manifest(data) + + await stage( + broker=args.broker, + prefix=args.prefix, + manifest=manifest, + data=data, + chunk_size=args.chunk_size, + prepare_timeout=args.prepare_timeout, + timeout=args.timeout, + ) + + +def main() -> None: + asyncio.run(async_main()) + + +if __name__ == "__main__": + main() diff --git a/mqtt-staging/py/pyproject.toml b/mqtt-staging/py/pyproject.toml new file mode 100644 index 0000000..4c4bbb7 --- /dev/null +++ b/mqtt-staging/py/pyproject.toml @@ -0,0 +1,39 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "mqtt-staging" +# Note: keep this in sync with Cargo.toml +version = "0.1.0" +description = "Stage bounded objects into embedded storage over MQTT" +readme = "README.md" +requires-python = ">=3.11" +license = "MIT OR Apache-2.0" +authors = [{ name = "Robert Jördens", email = "rj@quartiq.de" }] +dependencies = ["gmqtt>=0.7,<0.8"] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] + +[project.optional-dependencies] +dev = ["ruff>=0.11"] + +[project.urls] +Repository = "https://github.com/quartiq/minimq.git" + +[project.scripts] +mqtt-staging = "mqtt_staging:main" + +[tool.setuptools] +py-modules = ["mqtt_staging"] + +[tool.ruff] +target-version = "py311" +src = ["."] + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "I"] diff --git a/mqtt-staging/py/tests/test_host.py b/mqtt-staging/py/tests/test_host.py new file mode 100644 index 0000000..f85a177 --- /dev/null +++ b/mqtt-staging/py/tests/test_host.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import asyncio +import unittest + +from mqtt_staging import ( + Status, + Transfer, + aligned_chunk_size, + broker_endpoint, + chunk_properties, + json_manifest, + wait_status, +) + +PREFIX = "devices/example/staging" +TRANSFER_ID = "85944171f73967e8" + + +class Message: + def __init__( + self, + payload: bytes, + properties, + topic: str = f"{PREFIX}/status", + ) -> None: + self.topic = topic + self.payload = payload + self.properties = properties + + +def status_properties(): + return { + "response_topic": f"{PREFIX}/chunk", + "correlation_data": TRANSFER_ID.encode(), + "user_property": [("offset", "1024")], + } + + +class HostToolTests(unittest.TestCase): + def test_aligned_chunk_size_fits_mtu_and_write_size(self) -> None: + self.assertEqual(aligned_chunk_size(1500, 1024, 32), 1024) + self.assertEqual(aligned_chunk_size(1000, 1024, 32), 992) + self.assertEqual(aligned_chunk_size(64, 16, 32), 0) + + def test_broker_endpoint(self) -> None: + self.assertEqual(broker_endpoint("mqtt"), ("mqtt", 1883)) + self.assertEqual(broker_endpoint("mqtt.example:1884"), ("mqtt.example", 1884)) + self.assertEqual(broker_endpoint("[::1]:1884"), ("::1", 1884)) + self.assertEqual( + broker_endpoint("mqtt://mqtt.example:1884"), + ("mqtt.example", 1884), + ) + + def test_status_parses_receipt_payload(self) -> None: + status = Status.from_message( + Message( + b'{"state":"ready","code":"accepted","id":"85944171f73967e8",' + b'"next_offset":1024,"size":2048,"mtu":1024,"write_size":32}', + status_properties(), + ) + ) + + self.assertEqual(status.state, "ready") + self.assertEqual(status.code, "accepted") + self.assertEqual(status.id, TRANSFER_ID) + self.assertEqual(status.response_topic, f"{PREFIX}/chunk") + self.assertEqual(status.correlation_data, TRANSFER_ID.encode()) + self.assertEqual(status.offset, 1024) + self.assertEqual(status.next_offset, 1024) + self.assertEqual(status.size, 2048) + self.assertEqual(status.mtu, 1024) + self.assertEqual(status.write_size, 32) + + def test_status_raises_on_rejection(self) -> None: + status = Status.from_message( + Message( + b'{"state":"error","code":"offset","id":"85944171f73967e8",' + b'"next_offset":0,"size":2048,"mtu":1024,"write_size":32}', + status_properties(), + ) + ) + + with self.assertRaisesRegex(RuntimeError, "rejected"): + status.raise_for_error() + + def test_status_raises_on_mtu_rejection(self) -> None: + status = Status.from_message( + Message( + b'{"state":"idle","code":"mtu","id":"",' + b'"next_offset":0,"size":0,"mtu":1024,"write_size":32}', + status_properties(), + ) + ) + + with self.assertRaisesRegex(RuntimeError, "rejected"): + status.raise_for_error() + + def test_chunk_properties_echo_request_correlation_and_offset(self) -> None: + properties = chunk_properties(b"opaque", 37) + + self.assertEqual(properties["payload_format_id"], 0) + self.assertEqual(properties["correlation_data"], b"opaque") + self.assertEqual(properties["user_property"], [("offset", "37")]) + + def test_json_manifest_is_compact_and_deterministic(self) -> None: + manifest = json_manifest(b"foobar") + + self.assertEqual( + manifest, + b'{"id":"85944171f73967e8","size":6,"fnv1a64":9625390261332436968}', + ) + + +class HostToolAsyncTests(unittest.IsolatedAsyncioTestCase): + async def test_wait_status_ignores_another_transfer(self) -> None: + messages = asyncio.Queue() + properties = status_properties() + await messages.put( + Message( + b'{"state":"ready","code":"accepted","id":"other",' + b'"next_offset":0,"size":2048,"mtu":1024,"write_size":32}', + properties, + ) + ) + await messages.put( + Message( + b'{"state":"ready","code":"accepted","id":"85944171f73967e8",' + b'"next_offset":1024,"size":2048,"mtu":1024,"write_size":32}', + properties, + ) + ) + + status = await wait_status( + messages, + f"{PREFIX}/status", + Transfer(TRANSFER_ID, 2048), + timeout=0.1, + ) + + self.assertEqual(status.id, TRANSFER_ID) + self.assertEqual(status.next_offset, 1024) + + +if __name__ == "__main__": + unittest.main() diff --git a/mqtt-staging/release.toml b/mqtt-staging/release.toml new file mode 100644 index 0000000..3d6d266 --- /dev/null +++ b/mqtt-staging/release.toml @@ -0,0 +1,13 @@ +[[pre-release-replacements]] +file = "CHANGELOG.md" +search = "## \\[UNRELEASED\\]\\(https?://[^)]+?/compare/(?P[^)]+)\\.\\.\\.HEAD\\) - DATE" +replace = """## [UNRELEASED](https://github.com/quartiq/minimq/compare/{{tag_name}}...HEAD) - DATE + +## [{{version}}](https://github.com/quartiq/minimq/compare/${previous}...{{tag_name}}) - {{date}}""" +exactly = 1 + +[[pre-release-replacements]] +file = "py/pyproject.toml" +search = '^version = "[^"]+"$' +replace = 'version = "{{version}}"' +exactly = 1 diff --git a/mqtt-staging/src/lib.rs b/mqtt-staging/src/lib.rs new file mode 100644 index 0000000..7370d68 --- /dev/null +++ b/mqtt-staging/src/lib.rs @@ -0,0 +1,1371 @@ +#![doc = include_str!("../README.md")] +#![cfg_attr(not(test), no_std)] +#![warn(missing_docs)] + +use defmt::{debug, info, trace, warn}; +use heapless::String; +use minimq::{ + ConnectEvent, Connection, Error as MqttError, InboundPublish, Io, Op, Properties, Property, + PubError, Publication, QoS, ResourceError, RetainHandling, SubscriptionOptions, TopicFilter, +}; +use serde::{Deserialize, Serialize}; + +type TopicString = String<128>; + +const MAX_TRANSFER_ID_BYTES: usize = 48; +const MAX_STATUS_BYTES: usize = 256; +const INFO_CHUNK_STRIDE: usize = 128; +const MANIFEST_SUFFIX: &str = "/manifest"; +const STATUS_SUFFIX: &str = "/status"; +const CHUNK_SUFFIX: &str = "/chunk"; + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Transfer { + id: String, + size: u32, + next_offset: u32, + fnv1a64: u64, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Manifest<'a> { + id: &'a str, + size: u32, + fnv1a64: u64, +} + +/// Immediate result of one cooperative `step()`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[must_use = "inspect whether staging work is still pending"] +pub enum Step { + /// No queued staging work remains after this step. + Quiescent, + /// Staging still has queued or in-flight MQTT work. + Pending, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Error returned while constructing a staging service. +pub enum CreateError { + /// A derived protocol topic does not fit the fixed topic buffer. + Topic(ResourceError), + /// The maximum chunk size or storage write size cannot support aligned writes. + InvalidChunkSize, +} + +/// Storage limits for one staging service. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Config { + /// Maximum staged object size in bytes. + pub capacity: u32, + /// Maximum bytes accepted in one MQTT chunk. + pub max_chunk_size: usize, + /// Required storage-write alignment in bytes. + pub write_size: usize, +} + +impl core::fmt::Display for CreateError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Topic(_) => f.write_str("staging topic does not fit buffer"), + Self::InvalidChunkSize => f.write_str("invalid staging chunk or write size"), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, defmt::Format)] +#[serde(rename_all = "kebab-case")] +/// Current staging phase. +pub enum State { + /// No transfer is active. + Idle, + /// The storage backend is preparing the staging area. + Preparing, + /// The device is ready for the next chunk. + Ready, + /// The storage backend is writing one chunk. + Writing, + /// The full object was staged. + Complete, + /// The transfer failed or was aborted. + Error, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, defmt::Format)] +#[serde(rename_all = "kebab-case")] +/// Result conveyed by a staging status publication. +pub enum StatusCode { + /// No transfer is active. + Idle, + /// The manifest or latest chunk was accepted. + Accepted, + /// An already accepted chunk was received again. + Duplicate, + /// The chunk did not match the requested offset. + Offset, + /// The object or chunk exceeds a configured limit. + Oversize, + /// The MQTT RX packet budget cannot carry a configured chunk. + Mtu, + /// The chunk does not meet the storage write alignment. + Unaligned, + /// The storage backend rejected an operation. + Storage, + /// The object was staged. + Complete, + /// The request or service state was invalid. + Error, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +/// Current staging progress and device limits. +pub struct Status<'a> { + /// Current transfer state. + pub state: State, + /// Result associated with this status publication. + pub code: StatusCode, + /// Manifest identifier, or an empty string while idle. + pub id: &'a str, + /// First object byte not yet written. + pub next_offset: u32, + /// Declared object size in bytes. + pub size: u32, + /// Maximum chunk payload accepted by this service. + pub mtu: usize, + /// Required chunk alignment in bytes. + pub write_size: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Action { + Publish(StatusCode), + Subscribe, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ChunkError { + Queue(StatusCode), + Fail(StatusCode), +} + +struct InFlightAction { + action: Action, + op: Op, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum PendingRequest { + Prepare, + Write { next_offset: u32, final_chunk: bool }, +} + +/// Storage operation requested by the staging state machine. +#[derive(Debug)] +pub enum StagingRequest<'a> { + /// Prepare the staging area for one object. + Prepare { + /// Declared object size in bytes. + size: u32, + }, + /// Write one accepted chunk and optionally finalize the object. + Write(StagingWrite<'a>), +} + +/// One accepted chunk ready for storage. +#[derive(Debug)] +pub struct StagingWrite<'a> { + /// Object byte offset. + pub offset: u32, + /// Chunk bytes borrowed from the current Minimq inbound publish. The + /// owner must consume or copy them before polling the connection again. + pub payload: &'a [u8], + /// Manifest size, excluding any final erased-storage padding. + pub size: u32, + /// Expected FNV-1a checksum. Present only on the final chunk. + pub fnv1a64: Option, +} + +/// Result of handling one inbound MQTT publish. +#[must_use = "submit storage work or route unhandled traffic"] +#[derive(Debug)] +pub enum Handle<'a> { + /// The publish is not staging traffic. + Unhandled, + /// Staging traffic was consumed without new storage work. + Consumed, + /// The owner must execute this storage request and report its result with + /// `Service::complete_request`. + Request(StagingRequest<'a>), +} + +/// Cooperative MQTT staging service. +pub struct Service { + prefix: TopicString, + inflight: Option, + pending: Option, + startup_publish: Option, + state: State, + transfer: Option, + request: Option, + capacity: u32, + max_chunk_size: usize, + max_rx_packet_size: usize, + write_size: usize, +} + +impl Service { + /// Create a staging service at the exact protocol topic root supplied by the + /// application. + pub fn new(prefix: &str, config: Config) -> Result { + if config.write_size == 0 + || config.max_chunk_size == 0 + || !config.max_chunk_size.is_multiple_of(config.write_size) + { + return Err(CreateError::InvalidChunkSize); + } + let prefix = TopicString::try_from(prefix) + .map_err(|_| CreateError::Topic(ResourceError::BufferTooSmall))?; + let _ = topic(&prefix, MANIFEST_SUFFIX).map_err(CreateError::Topic)?; + let _ = topic(&prefix, STATUS_SUFFIX).map_err(CreateError::Topic)?; + let _ = topic(&prefix, CHUNK_SUFFIX).map_err(CreateError::Topic)?; + Ok(Self { + prefix, + inflight: None, + pending: None, + startup_publish: None, + state: State::Idle, + transfer: None, + request: None, + capacity: config.capacity, + max_chunk_size: config.max_chunk_size, + max_rx_packet_size: 0, + write_size: config.write_size, + }) + } + + /// Begin staging startup for one MQTT connect event. + /// + /// Pure local state update. Cancel-safe. + pub fn begin_startup(&mut self, event: ConnectEvent) { + let replay = self.replay_status(); + match event { + ConnectEvent::Connected => { + self.inflight = None; + self.startup_publish = replay; + self.queue(Action::Subscribe); + } + ConnectEvent::Reconnected => { + self.startup_publish = None; + if let Some(code) = replay { + self.queue(Action::Publish(code)); + } + } + } + } + + /// Return the current staging status. + /// + /// Pure query. Cancel-safe. + pub fn status(&self) -> Status<'_> { + self.status_with(self.current_code()) + } + + /// Return whether staging has reached a terminal complete state and there is + /// no remaining queued or in-flight MQTT work. + pub fn is_complete(&self) -> bool { + self.state == State::Complete + && self.request.is_none() + && self.pending.is_none() + && self.inflight.is_none() + && self.startup_publish.is_none() + } + + fn current_code(&self) -> StatusCode { + match self.state { + State::Idle => StatusCode::Idle, + State::Preparing | State::Ready | State::Writing => StatusCode::Accepted, + State::Complete => StatusCode::Complete, + State::Error => StatusCode::Error, + } + } + + fn status_with(&self, code: StatusCode) -> Status<'_> { + let transfer = self.transfer.as_ref(); + Status { + state: self.state, + code, + id: transfer.map_or("", |transfer| transfer.id.as_str()), + next_offset: transfer.map_or(0, |transfer| transfer.next_offset), + size: transfer.map_or(0, |transfer| transfer.size), + mtu: self.max_chunk_size, + write_size: self.write_size, + } + } + + /// Abort an active staging transfer. + /// + /// Pure local state update. Cancel-safe. + /// + /// Returns `true` if a staging transfer was active and is now marked as + /// failed. A later `step()` or reconnect startup replay will publish + /// `error` status for the same transfer. + pub fn abort(&mut self) -> bool { + if !matches!(self.state, State::Preparing | State::Ready | State::Writing) { + return false; + } + warn!( + "Aborting staging transfer state={:?} offset={=u32} size={=u32}", + self.state, + self.status().next_offset, + self.status().size + ); + self.state = State::Error; + self.request = None; + if matches!(self.inflight.as_ref(), Some(inflight) if inflight.action == Action::Subscribe) + || self.pending == Some(Action::Subscribe) + { + self.startup_publish = Some(StatusCode::Error); + } else { + self.queue(Action::Publish(StatusCode::Error)); + } + true + } + + /// Handle one inbound publish. + /// + /// Pure local state update returning chunk data borrowed from `inbound`. + /// This is cancel-safe. + pub fn handle<'a>(&mut self, inbound: &InboundPublish<'a>) -> Handle<'a> { + self.handle_publish_with_properties( + inbound.topic(), + inbound.payload(), + inbound.properties(), + ) + } + + /// Complete one previously emitted storage request. + /// + /// Pure local state update. Cancel-safe. + pub fn complete_request(&mut self, success: bool) { + let Some(request) = self.request.take() else { + return; + }; + let Some(transfer) = self.transfer.as_mut() else { + self.state = State::Error; + self.queue(Action::Publish(StatusCode::Error)); + return; + }; + if !success { + warn!( + "Staging storage step failed state={:?} offset={=u32} size={=u32}", + self.state, transfer.next_offset, transfer.size + ); + self.state = State::Error; + self.queue(Action::Publish(StatusCode::Storage)); + return; + } + match request { + PendingRequest::Prepare => { + self.state = State::Ready; + info!("Accepted staging manifest size={=u32}", transfer.size); + self.queue(Action::Publish(StatusCode::Accepted)); + } + PendingRequest::Write { + next_offset, + final_chunk, + .. + } => { + if final_chunk { + transfer.next_offset = transfer.size; + self.state = State::Complete; + info!("Completed staging object size={=u32}", transfer.size); + self.queue(Action::Publish(StatusCode::Complete)); + } else { + debug!( + "staging chunk write completed next_offset={=u32} size={=u32}", + next_offset, transfer.size + ); + transfer.next_offset = next_offset; + self.state = State::Ready; + self.queue(Action::Publish(StatusCode::Accepted)); + } + } + } + } + + fn handle_publish_with_properties<'a>( + &mut self, + topic: &str, + payload: &'a [u8], + properties: &Properties<'_>, + ) -> Handle<'a> { + if topic.strip_prefix(self.prefix.as_str()) == Some(MANIFEST_SUFFIX) { + debug!("staging manifest received payload={=usize}B", payload.len()); + return self.handle_manifest(payload); + } + if topic.strip_prefix(self.prefix.as_str()) != Some(CHUNK_SUFFIX) { + return Handle::Unhandled; + } + let Some(chunk) = chunk_properties(properties) else { + warn!("Rejecting staging chunk without required properties"); + return Handle::Consumed; + }; + self.handle_chunk(chunk.id, chunk.offset, payload) + } + + /// Advance one queued MQTT operation. + /// + /// This is the cooperative queue-drain API. It performs at most one local + /// queued subscribe or status-publish step and does not wait for future + /// inbound reads on its own. + /// + /// Cancel-safe if the underlying transport I/O futures are cancel-safe. + /// The current action stays at the front of the local queue until the MQTT + /// operation is known to have completed or been invalidated. + pub async fn step( + &mut self, + connection: &mut Connection<'_, '_, IO>, + ) -> Result> + where + IO: Io, + { + self.max_rx_packet_size = connection.session().max_rx_packet_size(); + + if let Some(inflight) = self.inflight.take() { + if connection.is_pending(&inflight.op) { + self.inflight = Some(inflight); + return Ok(Step::Pending); + } + if connection.is_invalidated(&inflight.op) { + self.queue(inflight.action); + return Err(MqttError::Disconnected); + } + if inflight.action == Action::Subscribe + && let Some(code) = self.startup_publish.take() + { + self.queue(Action::Publish(code)); + } + } + + let Some(action) = self.pending.take() else { + return Ok(Step::Quiescent); + }; + let op = match self.start_action(connection, action).await { + Ok(op) => op, + Err(error) => { + self.pending = Some(action); + return Err(error); + } + }; + self.inflight = op.map(|op| InFlightAction { action, op }); + Ok(if self.inflight.is_none() && self.pending.is_none() { + Step::Quiescent + } else { + Step::Pending + }) + } + + fn handle_manifest<'a>(&mut self, payload: &'a [u8]) -> Handle<'a> { + let Ok(manifest) = parse_manifest(payload) else { + warn!("Rejecting staging manifest: invalid manifest"); + if self.state == State::Idle { + self.queue(Action::Publish(StatusCode::Error)); + } + return Handle::Consumed; + }; + if self.state != State::Idle { + if self.transfer.as_ref().is_some_and(|transfer| { + transfer.id.as_str() == manifest.id + && transfer.size == manifest.size + && transfer.fnv1a64 == manifest.fnv1a64 + }) { + debug!("Replaying status for duplicate staging manifest"); + self.queue(Action::Publish(self.current_code())); + return Handle::Consumed; + } + warn!( + "Rejecting overlapping staging manifest state={:?} offset={=u32}", + self.state, + self.status().next_offset + ); + return Handle::Consumed; + } + if manifest.size == 0 { + warn!("Rejecting staging manifest: zero-length object"); + self.queue(Action::Publish(StatusCode::Error)); + return Handle::Consumed; + } + if manifest.size > self.capacity { + warn!( + "Rejecting staging manifest: object oversize size={=u32} capacity={=u32}", + manifest.size, self.capacity + ); + self.queue(Action::Publish(StatusCode::Oversize)); + return Handle::Consumed; + } + if required_rx_bytes( + self.chunk_topic().as_str(), + manifest.id.as_bytes(), + self.max_chunk_size, + ) > self.max_rx_packet_size + { + warn!( + "Rejecting staging manifest: chunk exceeds MQTT rx budget chunk={=usize} max_rx={=usize}", + self.max_chunk_size, self.max_rx_packet_size + ); + self.queue(Action::Publish(StatusCode::Mtu)); + return Handle::Consumed; + } + let size = manifest.size; + self.transfer = Some(manifest); + self.state = State::Preparing; + self.queue(Action::Publish(StatusCode::Accepted)); + self.request = Some(PendingRequest::Prepare); + Handle::Request(StagingRequest::Prepare { size }) + } + + fn handle_chunk<'a>(&mut self, id: &[u8], offset: u32, payload: &'a [u8]) -> Handle<'a> { + trace!( + "staging chunk received offset={=u32} len={=usize}", + offset, + payload.len() + ); + let Some(transfer) = self.transfer.as_mut() else { + debug!("Ignoring staging chunk without active transfer"); + self.queue(Action::Publish(StatusCode::Idle)); + return Handle::Consumed; + }; + if id != transfer.id.as_bytes() { + debug!("Ignoring staging chunk for another transfer"); + return Handle::Consumed; + } + if self.state == State::Preparing { + debug!("Ignoring staging chunk while prepare is still pending"); + return Handle::Consumed; + } + if matches!(self.state, State::Complete | State::Error) { + self.queue(Action::Publish(self.current_code())); + return Handle::Consumed; + } + if self.state == State::Writing { + debug!("Ignoring staging chunk while a write is pending"); + return Handle::Consumed; + } + if self.state != State::Ready { + debug!( + "Ignoring staging chunk while state={:?} offset={=u32}", + self.state, transfer.next_offset + ); + self.queue(Action::Publish(StatusCode::Idle)); + return Handle::Consumed; + } + let next_offset = match validate_chunk( + transfer, + offset, + payload, + self.capacity, + self.max_chunk_size, + self.write_size, + ) { + Ok(next_offset) => next_offset, + Err(ChunkError::Queue(code)) => { + debug!( + "Ignoring staging chunk expected={=u32} got={=u32} len={=usize} code={:?}", + transfer.next_offset, + offset, + payload.len(), + code + ); + self.queue(Action::Publish(code)); + return Handle::Consumed; + } + Err(ChunkError::Fail(code)) => { + warn!( + "Rejecting staging chunk offset={=u32} len={=usize} code={:?}", + offset, + payload.len(), + code + ); + self.state = State::Error; + self.queue(Action::Publish(code)); + return Handle::Consumed; + } + }; + let final_chunk = next_offset == transfer.size; + let fnv1a64 = final_chunk.then_some(transfer.fnv1a64); + self.state = State::Writing; + self.request = Some(PendingRequest::Write { + next_offset, + final_chunk, + }); + debug!( + "Queueing staging write offset={=u32} len={=usize} final={=bool}", + offset, + payload.len(), + final_chunk + ); + Handle::Request(StagingRequest::Write(StagingWrite { + offset, + payload, + size: transfer.size, + fnv1a64, + })) + } + + fn replay_status(&self) -> Option { + (self.state != State::Idle).then(|| self.current_code()) + } + + fn queue(&mut self, action: Action) { + if matches!( + (self.inflight.as_ref(), action), + (Some(inflight), Action::Subscribe) + if inflight.action == Action::Subscribe + ) { + return; + } + self.pending = Some(action); + } + + fn manifest_topic(&self) -> TopicString { + topic(&self.prefix, MANIFEST_SUFFIX).expect("validated staging manifest topic") + } + + fn status_topic(&self) -> TopicString { + topic(&self.prefix, STATUS_SUFFIX).expect("validated staging status topic") + } + + fn chunk_topic(&self) -> TopicString { + topic(&self.prefix, CHUNK_SUFFIX).expect("validated staging chunk topic") + } + + async fn start_action( + &mut self, + connection: &mut Connection<'_, '_, IO>, + action: Action, + ) -> Result, MqttError> + where + IO: Io, + { + match action { + Action::Subscribe => { + info!("Subscribing staging MQTT topics"); + let manifest_topic = self.manifest_topic(); + let chunk_topic = self.chunk_topic(); + let filters = [ + TopicFilter::new(manifest_topic.as_str()).options( + SubscriptionOptions::default() + .maximum_qos(QoS::AtLeastOnce) + .retain_behavior(RetainHandling::Never) + .ignore_local_messages(), + ), + TopicFilter::new(chunk_topic.as_str()).options( + SubscriptionOptions::default() + .maximum_qos(QoS::AtLeastOnce) + .retain_behavior(RetainHandling::Never) + .ignore_local_messages(), + ), + ]; + Ok(Some(connection.subscribe(&filters, &[]).await?)) + } + Action::Publish(code) => self.publish_status(connection, code).await, + } + } + + async fn publish_status( + &self, + connection: &mut Connection<'_, '_, IO>, + code: StatusCode, + ) -> Result, MqttError> + where + IO: Io, + { + let status = self.status_with(code); + let mut payload = [0; MAX_STATUS_BYTES]; + let len = serde_json_core::ser::to_slice(&status, &mut payload) + .map_err(|_| MqttError::Resource(ResourceError::BufferTooSmall))?; + let mut offset = itoa::Buffer::new(); + let chunk_topic = self.chunk_topic(); + let id = self + .transfer + .as_ref() + .map_or(&[][..], |transfer| transfer.id.as_bytes()); + let properties = [ + Property::PayloadFormatIndicator(1), + Property::ResponseTopic(chunk_topic.as_str()), + Property::CorrelationData(id), + Property::UserProperty("offset", offset.format(status.next_offset)), + ]; + match connection + .publish( + Publication::new(self.status_topic().as_str(), &payload[..len]) + .properties(&properties) + .qos(QoS::AtLeastOnce), + ) + .await + { + Ok(op) => { + log_status(code, status); + Ok(op) + } + Err(PubError::Payload(_)) => unreachable!(), + Err(PubError::Session(err)) => Err(err), + } + } +} + +fn log_status(code: StatusCode, status: Status<'_>) { + match code { + StatusCode::Accepted => { + if should_log_chunk_progress(status.next_offset, status.size, status.mtu) { + info!( + "staging status code={:?} offset={=u32}", + code, status.next_offset + ); + } else { + debug!( + "staging status code={:?} offset={=u32}", + code, status.next_offset + ); + } + } + StatusCode::Complete => { + info!( + "staging status code={:?} offset={=u32}", + code, status.next_offset + ); + } + StatusCode::Idle | StatusCode::Duplicate => { + debug!( + "staging status code={:?} offset={=u32}", + code, status.next_offset + ); + } + StatusCode::Offset + | StatusCode::Oversize + | StatusCode::Mtu + | StatusCode::Unaligned + | StatusCode::Storage + | StatusCode::Error => { + warn!( + "staging status code={:?} offset={=u32}", + code, status.next_offset + ); + } + } +} + +fn should_log_chunk_progress(next_offset: u32, size: u32, chunk_size: usize) -> bool { + if next_offset == 0 { + return true; + } + if next_offset.saturating_add(chunk_size.try_into().unwrap_or(u32::MAX)) >= size { + return true; + } + (next_offset as usize / chunk_size).is_multiple_of(INFO_CHUNK_STRIDE) +} + +fn validate_chunk( + transfer: &Transfer, + offset: u32, + payload: &[u8], + capacity: u32, + max_chunk_size: usize, + write_size: usize, +) -> Result { + if offset < transfer.next_offset { + return Err(ChunkError::Queue(StatusCode::Duplicate)); + } + if offset != transfer.next_offset { + return Err(ChunkError::Queue(StatusCode::Offset)); + } + + if payload.is_empty() || payload.len() > max_chunk_size { + return Err(ChunkError::Fail(StatusCode::Mtu)); + } + if !payload.len().is_multiple_of(write_size) { + return Err(ChunkError::Fail(StatusCode::Unaligned)); + } + + let Some(end) = offset.checked_add(payload.len() as u32) else { + return Err(ChunkError::Fail(StatusCode::Oversize)); + }; + if end > capacity { + return Err(ChunkError::Fail(StatusCode::Oversize)); + } + if end <= transfer.size { + return Ok(end); + } + + let Some(padding) = payload_padding(payload, transfer.size - offset) else { + return Err(ChunkError::Fail(StatusCode::Oversize)); + }; + if !padding.iter().all(|&byte| byte == 0xff) { + return Err(ChunkError::Fail(StatusCode::Oversize)); + } + Ok(transfer.size) +} + +fn parse_manifest(payload: &[u8]) -> Result { + let (manifest, used) = serde_json_core::from_slice::>(payload).map_err(|_| ())?; + if !valid_id(manifest.id) || !payload[used..].iter().all(u8::is_ascii_whitespace) { + return Err(()); + } + Ok(Transfer { + id: manifest.id.try_into().map_err(|_| ())?, + size: manifest.size, + next_offset: 0, + fnv1a64: manifest.fnv1a64, + }) +} + +fn valid_id(id: &str) -> bool { + !id.is_empty() + && id.len() <= MAX_TRANSFER_ID_BYTES + && id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-._".contains(&byte)) +} + +struct ChunkProperties<'a> { + id: &'a [u8], + offset: u32, +} + +fn chunk_properties<'a>(properties: &'a Properties<'a>) -> Option> { + let id = properties.correlation_data()?; + let mut offset = None; + for property in properties.iter() { + let Ok(Property::UserProperty(key, value)) = property else { + continue; + }; + if key == "offset" { + offset = value.parse().ok(); + } + } + Some(ChunkProperties { + id, + offset: offset?, + }) +} + +fn required_rx_bytes(topic: &str, id: &[u8], payload_size: usize) -> usize { + let properties = 2 // PayloadFormatIndicator property and value + + 1 // CorrelationData property id + + 2 // binary length prefix + + id.len() + + 1 // UserProperty property id + + 2 // key UTF-8 length prefix + + "offset".len() + + 2 // value UTF-8 length prefix + + 10; // max u32 decimal digits + let remaining = 2 // topic UTF-8 length prefix + + topic.len() + + 2 // QoS 1 packet identifier + + mqtt_varint_len(properties) + + properties + + payload_size; + 1 + mqtt_varint_len(remaining) + remaining +} + +fn mqtt_varint_len(mut value: usize) -> usize { + let mut len = 1; + while value >= 128 { + value /= 128; + len += 1; + } + len +} + +fn payload_padding(payload: &[u8], object_bytes: u32) -> Option<&[u8]> { + let object_bytes = usize::try_from(object_bytes).ok()?; + (object_bytes < payload.len()).then(|| &payload[object_bytes..]) +} + +fn topic(prefix: &TopicString, suffix: &str) -> Result { + let mut topic = prefix.clone(); + topic + .push_str(suffix) + .map_err(|_| ResourceError::BufferTooSmall)?; + Ok(topic) +} + +#[cfg(test)] +mod tests { + extern crate std; + + use super::*; + use std::sync::OnceLock; + + const ID: &[u8] = b"7ea690cc8c2cd8ed"; + const ID_6: &[u8] = b"97463f6d0098b84a"; + const SLOT: usize = 32; + + fn init_host_logging() { + static HOST_LOGGING: OnceLock<()> = OnceLock::new(); + + HOST_LOGGING.get_or_init(|| { + let _ = env_logger::builder().is_test(true).try_init(); + defmt2log::init_from_current_exe(); + }); + } + + #[derive(Default)] + struct Backend { + prepared: Option, + writes: std::vec::Vec<(u32, std::vec::Vec)>, + finished: Option, + fail_finish: bool, + } + + impl Backend { + fn apply(&mut self, request: StagingRequest<'_>) -> bool { + match request { + StagingRequest::Prepare { size } => { + self.prepared = Some(size); + true + } + StagingRequest::Write(write) => { + self.writes.push((write.offset, write.payload.to_vec())); + if write.fnv1a64.is_some() { + if self.fail_finish { + return false; + } + self.finished = Some(write.size); + } + true + } + } + } + } + + fn apply_handle(service: &mut Service, backend: &mut Backend, handle: Handle<'_>) { + if let Handle::Request(request) = handle { + service.complete_request(backend.apply(request)); + } + } + + fn manifest<'a>(service: &mut Service, payload: &'a [u8]) -> Handle<'a> { + service.handle_publish_with_properties( + service.manifest_topic().as_str(), + payload, + &Properties::from_slice(&[]), + ) + } + + fn chunk<'a>(service: &mut Service, id: &[u8], offset: &str, payload: &'a [u8]) -> Handle<'a> { + let properties = [ + Property::CorrelationData(id), + Property::UserProperty("offset", offset), + ]; + service.handle_publish_with_properties( + service.chunk_topic().as_str(), + payload, + &Properties::from_slice(&properties), + ) + } + + fn drive_manifest(service: &mut Service, backend: &mut Backend, payload: &[u8]) { + let handle = manifest(service, payload); + apply_handle(service, backend, handle); + } + + fn drive_chunk( + service: &mut Service, + backend: &mut Backend, + id: &[u8], + offset: &str, + payload: &[u8], + ) { + let handle = chunk(service, id, offset, payload); + apply_handle(service, backend, handle); + } + + fn service(rx: usize) -> Service { + init_host_logging(); + let mut service = Service::new( + "devices/example/staging", + Config { + capacity: 128, + max_chunk_size: SLOT, + write_size: 4, + }, + ) + .unwrap(); + service.max_rx_packet_size = rx; + service + } + + fn ready_service() -> Service { + let mut service = service(128); + service.state = State::Ready; + service.transfer = Some(Transfer { + id: "7ea690cc8c2cd8ed".try_into().unwrap(), + size: 64, + next_offset: 32, + fnv1a64: 0, + }); + service + } + + #[test] + fn connected_startup_queues_subscribe_only_for_idle_service() { + let mut service = service(128); + service.begin_startup(ConnectEvent::Connected); + assert_eq!(service.pending, Some(Action::Subscribe)); + assert_eq!(service.startup_publish, None); + } + + #[test] + fn connected_startup_defers_status_replay_until_after_subscribe() { + let mut service = ready_service(); + service.begin_startup(ConnectEvent::Connected); + assert_eq!(service.pending, Some(Action::Subscribe)); + assert_eq!(service.startup_publish, Some(StatusCode::Accepted)); + } + + #[test] + fn reconnected_startup_replays_current_status_without_subscribe() { + let mut service = ready_service(); + service.begin_startup(ConnectEvent::Reconnected); + assert_eq!(service.pending, Some(Action::Publish(StatusCode::Accepted))); + assert_eq!(service.startup_publish, None); + } + + #[test] + fn status_reports_ready_transfer_state() { + let service = ready_service(); + let status = service.status(); + assert_eq!(status.state, State::Ready); + assert_eq!(status.code, StatusCode::Accepted); + assert_eq!(status.id, "7ea690cc8c2cd8ed"); + assert_eq!(status.next_offset, 32); + assert_eq!(status.size, 64); + assert_eq!(status.mtu, SLOT); + assert_eq!(status.write_size, 4); + } + + #[test] + fn largest_status_fits_the_fixed_buffer() { + let status = Status { + state: State::Preparing, + code: StatusCode::Unaligned, + id: "012345678901234567890123456789012345678901234567", + next_offset: u32::MAX, + size: u32::MAX, + mtu: usize::MAX, + write_size: usize::MAX, + }; + let mut payload = [0; MAX_STATUS_BYTES]; + assert!(serde_json_core::ser::to_slice(&status, &mut payload).is_ok()); + } + + #[test] + fn chunk_progress_handles_the_u32_limit() { + assert!(should_log_chunk_progress(u32::MAX - 1, u32::MAX, 4)); + } + + #[test] + fn abort_marks_ready_transfer_failed() { + let mut service = ready_service(); + assert!(service.abort()); + assert_eq!(service.status().state, State::Error); + assert_eq!(service.status().code, StatusCode::Error); + assert_eq!(service.pending, Some(Action::Publish(StatusCode::Error))); + } + + #[test] + fn abort_is_noop_without_active_transfer() { + let mut service = service(128); + assert!(!service.abort()); + assert_eq!(service.status().state, State::Idle); + } + + #[test] + fn abort_is_noop_after_transfer_completes() { + let mut service = service(128); + service.state = State::Complete; + assert!(!service.abort()); + assert_eq!(service.status().state, State::Complete); + } + + #[test] + fn manifest_reports_preparing_before_storage_prepare_completes() { + let mut service = service(128); + let handle = manifest( + &mut service, + br#"{"id":"7ea690cc8c2cd8ed","size":8,"fnv1a64":9126140903112366317}"#, + ); + assert!(matches!( + handle, + Handle::Request(StagingRequest::Prepare { size: 8 }) + )); + assert_eq!(service.status().state, State::Preparing); + assert_eq!(service.pending, Some(Action::Publish(StatusCode::Accepted))); + } + + mod protocol { + use super::*; + + const JSON_MANIFEST_8: &[u8] = + br#"{"id":"7ea690cc8c2cd8ed","size":8,"fnv1a64":9126140903112366317}"#; + const JSON_MANIFEST_6: &[u8] = + br#"{"id":"97463f6d0098b84a","size":6,"fnv1a64":10900469685490858058}"#; + const JSON_MANIFEST_0: &[u8] = + br#"{"id":"cbf29ce484222325","size":0,"fnv1a64":14695981039346656037}"#; + const JSON_MANIFEST_MISSING_DIGEST: &[u8] = br#"{"id":"7ea690cc8c2cd8ed","size":8}"#; + const JSON_MANIFEST_UNKNOWN_FIELD: &[u8] = + br#"{"id":"7ea690cc8c2cd8ed","size":8,"fnv1a64":9126140903112366317,"hash":"sha256"}"#; + + #[test] + fn manifest_rejects_if_one_aligned_chunk_cannot_fit() { + let mut service = service(64); + assert!(matches!( + manifest(&mut service, JSON_MANIFEST_8), + Handle::Consumed + )); + assert_eq!(service.status().state, State::Idle); + assert_eq!(service.status().code, StatusCode::Idle); + assert_eq!(service.pending, Some(Action::Publish(StatusCode::Mtu))); + } + + #[test] + fn manifest_accepts_exact_qos1_chunk_packet_budget() { + let required = required_rx_bytes("devices/example/staging/chunk", ID, SLOT); + assert!(matches!( + manifest(&mut service(required), JSON_MANIFEST_8), + Handle::Request(StagingRequest::Prepare { .. }) + )); + + let mut undersized = service(required - 1); + assert!(matches!( + manifest(&mut undersized, JSON_MANIFEST_8), + Handle::Consumed + )); + assert_eq!(undersized.pending, Some(Action::Publish(StatusCode::Mtu))); + } + + #[test] + fn manifest_rejects_invalid_json_profiles() { + for payload in [ + JSON_MANIFEST_0, + JSON_MANIFEST_MISSING_DIGEST, + JSON_MANIFEST_UNKNOWN_FIELD, + br#"{"id":"","size":8,"fnv1a64":9126140903112366317}"#, + br#"{"id":"bad/id","size":8,"fnv1a64":9126140903112366317}"#, + br#"{"id":"7ea690cc8c2cd8ed","size":8,"fnv1a64":9126140903112366317}x"#, + ] { + let mut service = service(128); + assert!(matches!(manifest(&mut service, payload), Handle::Consumed)); + assert_eq!(service.status().state, State::Idle); + assert_eq!(service.pending, Some(Action::Publish(StatusCode::Error))); + } + } + + #[test] + fn oversized_manifest_is_one_shot_and_next_manifest_can_succeed() { + let mut service = service(128); + let mut backend = Backend::default(); + let oversized = + br#"{"id":"7ea690cc8c2cd8ed","size":129,"fnv1a64":9126140903112366317}"#; + + assert!(matches!( + manifest(&mut service, oversized), + Handle::Consumed + )); + assert_eq!(service.status().state, State::Idle); + assert_eq!(service.pending, Some(Action::Publish(StatusCode::Oversize))); + + drive_manifest(&mut service, &mut backend, JSON_MANIFEST_8); + assert_eq!(service.status().state, State::Ready); + assert_eq!(backend.prepared, Some(8)); + } + + #[test] + fn ordered_chunks_write_immediately_and_finish() { + let mut service = service(128); + let mut backend = Backend::default(); + drive_manifest(&mut service, &mut backend, JSON_MANIFEST_8); + assert_eq!(service.state, State::Ready); + + drive_chunk(&mut service, &mut backend, ID, "0", &[1, 2, 3, 4]); + assert_eq!(service.state, State::Ready); + + drive_chunk(&mut service, &mut backend, ID, "4", &[5, 6, 7, 8]); + assert_eq!(service.state, State::Complete); + + assert_eq!(backend.prepared, Some(8)); + assert_eq!( + backend.writes, + std::vec![(0, std::vec![1, 2, 3, 4]), (4, std::vec![5, 6, 7, 8])] + ); + assert_eq!(backend.finished, Some(8)); + } + + #[test] + fn duplicate_final_chunk_replays_complete_status() { + let mut service = service(128); + let mut backend = Backend::default(); + drive_manifest(&mut service, &mut backend, JSON_MANIFEST_8); + drive_chunk(&mut service, &mut backend, ID, "0", &[1, 2, 3, 4]); + drive_chunk(&mut service, &mut backend, ID, "4", &[5, 6, 7, 8]); + service.pending = None; + + assert!(matches!( + chunk(&mut service, ID, "4", &[5, 6, 7, 8]), + Handle::Consumed + )); + assert_eq!(service.state, State::Complete); + assert_eq!(service.pending, Some(Action::Publish(StatusCode::Complete))); + assert_eq!(backend.writes.len(), 2); + } + + #[test] + fn unrelated_chunks_do_not_disrupt_active_transfer() { + let mut service = service(128); + let mut backend = Backend::default(); + drive_manifest(&mut service, &mut backend, JSON_MANIFEST_8); + service.pending = None; + + assert!(matches!( + service.handle_publish_with_properties( + service.chunk_topic().as_str(), + &[1, 2, 3, 4], + &Properties::from_slice(&[]), + ), + Handle::Consumed + )); + assert_eq!(service.state, State::Ready); + assert_eq!(service.pending, None); + + assert!(matches!( + chunk(&mut service, b"another-transfer", "0", &[1, 2, 3, 4]), + Handle::Consumed + )); + assert_eq!(service.state, State::Ready); + assert_eq!(service.pending, None); + } + + #[test] + fn future_offset_is_rejected_without_write() { + let mut service = service(128); + let mut backend = Backend::default(); + drive_manifest(&mut service, &mut backend, JSON_MANIFEST_8); + assert!(matches!( + chunk(&mut service, ID, "4", &[5, 6, 7, 8]), + Handle::Consumed + )); + + assert!(backend.writes.is_empty()); + assert_eq!(backend.finished, None); + } + + #[test] + fn chunks_wait_for_the_pending_storage_write() { + let mut service = service(128); + let mut backend = Backend::default(); + drive_manifest(&mut service, &mut backend, JSON_MANIFEST_8); + service.pending = None; + + assert!(matches!( + chunk(&mut service, ID, "0", &[1, 2, 3, 4]), + Handle::Request(StagingRequest::Write(_)) + )); + assert!(matches!( + chunk(&mut service, ID, "0", &[1, 2, 3, 4]), + Handle::Consumed + )); + assert!(matches!( + chunk(&mut service, ID, "4", &[5, 6, 7, 8]), + Handle::Consumed + )); + assert_eq!(service.state, State::Writing); + assert_eq!(service.pending, None); + + service.complete_request(true); + assert_eq!(service.state, State::Ready); + assert_eq!(service.status().next_offset, 4); + } + + #[test] + fn overlapping_manifest_does_not_disrupt_active_transfer() { + let mut service = service(128); + let mut backend = Backend::default(); + drive_manifest(&mut service, &mut backend, JSON_MANIFEST_8); + service.pending = None; + assert!(matches!( + manifest(&mut service, JSON_MANIFEST_6), + Handle::Consumed + )); + + assert_eq!(backend.prepared, Some(8)); + assert_eq!(service.state, State::Ready); + assert_eq!(service.pending, None); + let transfer = service.transfer.unwrap(); + assert_eq!(transfer.size, 8); + assert_eq!(transfer.next_offset, 0); + } + + #[test] + fn duplicate_manifest_replays_active_status() { + let mut service = service(128); + let mut backend = Backend::default(); + drive_manifest(&mut service, &mut backend, JSON_MANIFEST_8); + service.pending = None; + + assert!(matches!( + manifest(&mut service, JSON_MANIFEST_8), + Handle::Consumed + )); + assert_eq!(service.state, State::Ready); + assert_eq!(service.pending, Some(Action::Publish(StatusCode::Accepted))); + assert_eq!(backend.prepared, Some(8)); + } + + #[test] + fn final_chunk_may_be_erased_value_padded() { + let mut service = service(128); + let mut backend = Backend::default(); + drive_manifest(&mut service, &mut backend, JSON_MANIFEST_6); + drive_chunk(&mut service, &mut backend, ID_6, "0", &[1, 2, 3, 4]); + drive_chunk(&mut service, &mut backend, ID_6, "4", &[5, 6, 0xff, 0xff]); + assert_eq!(service.state, State::Complete); + + assert_eq!( + backend.writes, + std::vec![(0, std::vec![1, 2, 3, 4]), (4, std::vec![5, 6, 0xff, 0xff])] + ); + assert_eq!(backend.finished, Some(6)); + } + + #[test] + fn final_chunk_rejects_non_erased_padding() { + let mut service = service(128); + let mut backend = Backend::default(); + drive_manifest(&mut service, &mut backend, JSON_MANIFEST_6); + drive_chunk(&mut service, &mut backend, ID_6, "0", &[1, 2, 3, 4]); + assert!(matches!( + chunk(&mut service, ID_6, "4", &[5, 6, 0, 0]), + Handle::Consumed + )); + + assert_eq!(backend.finished, None); + } + + #[test] + fn backend_failure_marks_transfer_error() { + let mut service = service(128); + let mut backend = Backend { + fail_finish: true, + ..Backend::default() + }; + + drive_manifest(&mut service, &mut backend, JSON_MANIFEST_8); + drive_chunk(&mut service, &mut backend, ID, "0", &[1, 2, 3, 4]); + drive_chunk(&mut service, &mut backend, ID, "4", &[5, 6, 7, 8]); + assert_eq!(service.state, State::Error); + assert_eq!(service.pending, Some(Action::Publish(StatusCode::Storage))); + } + } +} diff --git a/mqtt-staging/tests/end_to_end.rs b/mqtt-staging/tests/end_to_end.rs new file mode 100644 index 0000000..6eae630 --- /dev/null +++ b/mqtt-staging/tests/end_to_end.rs @@ -0,0 +1,279 @@ +//! Broker integration from the Python sender through Minimq to mock storage. + +use embassy_futures::block_on; +use embedded_io_async::{ErrorType, Read, Write}; +use minimq::{Buffers, ConfigBuilder, Connection, Session}; +use mqtt_staging::{Config, Handle, Service, StagingRequest, Step}; +use std::fs; +use std::io::{Read as _, Write as _}; +use std::net::TcpStream; +use std::path::PathBuf; +use std::process::{Child, Command, Output, Stdio}; +use std::sync::OnceLock; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::vec::Vec; + +const DATA: &[u8] = &[1, 2, 3, 4, 5, 6, 7, 8]; +const CHUNK_SIZE: usize = 4; + +fn init_host_logging() { + static HOST_LOGGING: OnceLock<()> = OnceLock::new(); + HOST_LOGGING.get_or_init(|| { + let _ = env_logger::builder().is_test(true).try_init(); + defmt2log::init_from_current_exe(); + }); +} + +struct TcpIo(TcpStream); + +impl TcpIo { + fn connect(endpoint: &str) -> std::io::Result { + let stream = TcpStream::connect(endpoint)?; + let timeout = Some(Duration::from_secs(30)); + stream.set_read_timeout(timeout)?; + stream.set_write_timeout(timeout)?; + Ok(Self(stream)) + } +} + +impl ErrorType for TcpIo { + type Error = std::io::Error; +} + +impl Read for TcpIo { + async fn read(&mut self, buffer: &mut [u8]) -> Result { + self.0.read(buffer) + } +} + +impl Write for TcpIo { + async fn write(&mut self, buffer: &[u8]) -> Result { + self.0.write(buffer) + } + + async fn flush(&mut self) -> Result<(), Self::Error> { + self.0.flush() + } +} + +enum OwnedStagingRequest { + Prepare { + size: u32, + }, + Write { + offset: u32, + payload: Vec, + size: u32, + fnv1a64: Option, + }, +} + +#[derive(Default)] +struct MockStorage { + pending: Option, + prepared: Option, + data: Vec, + finished: bool, +} + +impl MockStorage { + fn submit(&mut self, request: StagingRequest<'_>) { + assert!(self.pending.is_none()); + self.pending = Some(match request { + StagingRequest::Prepare { size } => OwnedStagingRequest::Prepare { size }, + StagingRequest::Write(write) => OwnedStagingRequest::Write { + offset: write.offset, + payload: write.payload.to_vec(), + size: write.size, + fnv1a64: write.fnv1a64, + }, + }); + } + + fn complete(&mut self, service: &mut Service) { + let success = match self.pending.take().unwrap() { + OwnedStagingRequest::Prepare { size } => { + self.prepared = Some(size); + self.data = vec![0xff; size as usize]; + true + } + OwnedStagingRequest::Write { + offset, + payload, + size, + fnv1a64, + } => { + let start = offset as usize; + let end = (start + payload.len()).min(size as usize); + self.data[start..end].copy_from_slice(&payload[..end - start]); + self.finished = + fnv1a64.is_some_and(|expected| expected == fnv1a64_hash(&self.data)); + fnv1a64.is_none() || self.finished + } + }; + service.complete_request(success); + } +} + +struct TempFile(PathBuf); + +impl TempFile { + fn new(unique: u128) -> Self { + let path = + std::env::temp_dir().join(format!("mqtt-staging-{unique}-{}.bin", std::process::id())); + fs::write(&path, DATA).unwrap(); + Self(path) + } +} + +impl Drop for TempFile { + fn drop(&mut self) { + let _ = fs::remove_file(&self.0); + } +} + +struct Feeder(Option); + +impl Feeder { + fn assert_running(&mut self) { + let child = self.0.as_mut().unwrap(); + if child.try_wait().unwrap().is_none() { + return; + } + let output = self.0.take().unwrap().wait_with_output().unwrap(); + panic!( + "staging feeder exited before completion\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + + fn wait(mut self) -> Output { + self.0.take().unwrap().wait_with_output().unwrap() + } +} + +impl Drop for Feeder { + fn drop(&mut self) { + if let Some(child) = &mut self.0 { + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +#[test] +#[ignore = "requires BROKER and the mqtt-staging host command"] +fn python_command_through_broker_to_mock_storage() { + init_host_logging(); + let broker = std::env::var("BROKER").expect("set BROKER=host[:port]"); + let endpoint = if broker.contains(':') { + broker.clone() + } else { + format!("{broker}:1883") + }; + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let prefix = format!("mqtt-staging/test/{unique}/service"); + let client_id = format!("service-device-{unique}"); + + let mut rx = [0; 512]; + let mut tx = [0; 1024]; + let mut session = Session::new( + ConfigBuilder::new(Buffers::new(&mut rx, &mut tx)) + .client_id(&client_id) + .unwrap(), + ); + let mut connection = block_on( + session.connect(TcpIo::connect(&endpoint).expect("connect staging device to broker")), + ) + .unwrap(); + let mut service = Service::new( + &prefix, + Config { + capacity: 16, + max_chunk_size: CHUNK_SIZE, + write_size: CHUNK_SIZE, + }, + ) + .unwrap(); + service.begin_startup(connection.connect_event()); + drain_local(&mut service, &mut connection); + + let file = TempFile::new(unique); + let feeder = std::env::var("MQTT_STAGING_FEEDER").unwrap_or_else(|_| "mqtt-staging".to_owned()); + let mut feeder = Feeder(Some( + Command::new(feeder) + .args(["--broker", &broker, "--prefix", &prefix, "--file"]) + .arg(&file.0) + .args(["--chunk-size", "4", "--timeout", "5"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("start Python staging feeder"), + )); + + let mut storage = MockStorage::default(); + loop { + feeder.assert_running(); + match block_on(service.step(&mut connection)).unwrap() { + Step::Pending => handle_inbound( + &mut service, + &mut storage, + block_on(connection.poll()).unwrap(), + ), + Step::Quiescent if storage.pending.is_some() => storage.complete(&mut service), + Step::Quiescent if service.is_complete() => break, + Step::Quiescent => handle_inbound( + &mut service, + &mut storage, + block_on(connection.poll()).unwrap(), + ), + } + } + + let output = feeder.wait(); + assert!( + output.status.success(), + "Python staging feeder failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(storage.prepared, Some(8)); + assert_eq!(storage.data, DATA); + assert!(storage.finished); +} + +fn drain_local(service: &mut Service, connection: &mut Connection<'_, '_, TcpIo>) { + loop { + match block_on(service.step(connection)).unwrap() { + Step::Quiescent => return, + Step::Pending => { + assert!(block_on(connection.poll()).unwrap().is_none()); + } + } + } +} + +fn handle_inbound( + service: &mut Service, + storage: &mut MockStorage, + inbound: Option>, +) { + let Some(inbound) = inbound else { + return; + }; + match service.handle(&inbound) { + Handle::Request(request) => storage.submit(request), + Handle::Consumed => {} + Handle::Unhandled => panic!("unexpected non-staging publish"), + } +} + +fn fnv1a64_hash(data: &[u8]) -> u64 { + data.iter().fold(0xcbf29ce484222325, |digest, byte| { + (digest ^ u64::from(*byte)).wrapping_mul(0x100000001b3) + }) +} diff --git a/release.toml b/release.toml index 9390a15..5d3f903 100644 --- a/release.toml +++ b/release.toml @@ -1,7 +1,9 @@ +tag-name = "{{crate_name}}-v{{version}}" + [[pre-release-replacements]] file = "CHANGELOG.md" search = "## \\[UNRELEASED\\]\\(https?://[^)]+?/compare/(?P[^)]+)\\.\\.\\.HEAD\\) - DATE" -replace = """## [UNRELEASED]({{repository}}/compare/{{tag_name}}...HEAD) - DATE +replace = """## [UNRELEASED](https://github.com/quartiq/minimq/compare/{{tag_name}}...HEAD) - DATE -## [{{version}}]({{repository}}/compare/${previous}...{{tag_name}}) - {{date}}""" +## [{{version}}](https://github.com/quartiq/minimq/compare/${previous}...{{tag_name}}) - {{date}}""" exactly = 1