deuxyeux/RTNode-HeltecV4
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Boundary mode: performance optimizations + boundary filter

Browse Source 
Performance optimizations:
- Move TLSF allocator pool to PSRAM (frees ~170KB internal SRAM)
- Raise TCP_IF_MAX_CLIENTS from 4 to 8 in BOUNDARY_MODE
- Raise path_table_maxsize from 48 to 128, persist from 16 to 32
- Add -DNDEBUG to boundary build: compiles out TRACE/DEBUG macros
- Log level defaults to LOG_VERBOSE when NDEBUG defined
- Serial baud 115200 -> 921600 in BOUNDARY_MODE (reduces CPU blocking)

Previous changes included in this commit:
- Comprehensive boundary filter with transitive whitelisting (7 checks)
- destination_table erase+insert fix (std::map::insert no-overwrite bug)
- Backbone-to-backbone routing guard in next-hop forwarding
- KISS serial output disabled for boundary mode
- flash.py updates for boundary mode support
This commit is contained in:
James L
2026-02-23 22:43:54 -05:00
parent 5ed70dcca9
commit 5077aa3829
9 changed files with 294 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Config.h
View File

@@ -72,7 +72,11 @@
#endif
// MCU independent configuration parameters
#ifdef BOUNDARY_MODE
const long serial_baudrate = 921600;
#else
const long serial_baudrate = 115200;
#endif
// SX1276 RSSI offset to get dBm value from
// packet RSSI register

9
RNode_Firmware.ino
View File

@@ -647,13 +647,12 @@ void setup() {
// ── Boundary Mode: Load config and optionally set up WiFi + TCP ──
HEAD("Boundary Mode: Initializing...", RNS::LOG_TRACE);
// Reduce table sizes to conserve heap on ESP32.
// Default 100 entries for each table fragments heap to critical levels.
// 48 entries gives enough room for local paths plus some backbone paths.
// With TLSF pool moved to PSRAM we have plenty of room.
// 128 path entries supports ~15-20 devices comfortably.
// cull_path_table() is patched to evict backbone paths first, preserving
// local (LoRa / local-TCP) paths needed for inbound message delivery.
RNS::Transport::path_table_maxsize(48);
RNS::Transport::path_table_maxpersist(16);
RNS::Transport::path_table_maxsize(128);
RNS::Transport::path_table_maxpersist(32);
boundary_load_config();
// Start WiFi if enabled

4
TcpInterface.h
View File

@@ -23,7 +23,11 @@
// ─── TCP Interface Configuration ─────────────────────────────────────────────
#define TCP_IF_DEFAULT_PORT 4242
#ifdef BOUNDARY_MODE
#define TCP_IF_MAX_CLIENTS 8
#else
#define TCP_IF_MAX_CLIENTS 4
#endif
#define TCP_IF_HW_MTU 1064
#define TCP_IF_CONNECT_TIMEOUT 6000 // ms
#define TCP_IF_WRITE_TIMEOUT 2000 // ms — short to avoid WDT

4
Utilities.h
View File

@@ -807,6 +807,10 @@ int8_t led_standby_direction = 0;
#endif
void serial_write(uint8_t byte) {
#ifdef BOUNDARY_MODE
// No KISS serial output in boundary mode - serial is used for debug logging only
return;
#endif
#if HAS_BLUETOOTH || HAS_BLE == true
if (bt_state != BT_STATE_CONNECTED) {
#if HAS_WIFI

190
flash.py
View File

@@ -57,6 +57,38 @@ FIRMWARE_BIN = os.path.join(BUILD_DIR, "rnode_firmware_heltec32v4_boundary.bi
# ESP32 partition table magic bytes (first two bytes of a partition table entry)
PARTITION_TABLE_MAGIC = b'\xaa\x50'
def is_merged_binary(firmware_path):
"""Check whether a firmware file is a merged binary (contains bootloader +
partition table) or an app-only binary.
Returns True for merged, False for app-only.
"""
try:
size = os.path.getsize(firmware_path)
if size > 0x8002:
with open(firmware_path, "rb") as f:
f.seek(0x8000)
return f.read(2) == PARTITION_TABLE_MAGIC
except Exception:
pass
return False
def _find_in_platformio_or_release(build_path, release_name):
"""Find a file in the PlatformIO build output or the bundled Release/ dir."""
# 1. PlatformIO build output
if os.path.isfile(build_path):
return build_path
# 2. Bundled in Release/
bundled = os.path.join(os.path.dirname(__file__), "Release", release_name)
if os.path.isfile(bundled):
return bundled
return None
def find_boot_app0():
"""Find boot_app0.bin from PlatformIO framework packages.
@@ -85,6 +117,17 @@ def find_boot_app0():
return None
def find_bootloader():
"""Find bootloader.bin from PlatformIO build output or Release/ bundle."""
return _find_in_platformio_or_release(BOOTLOADER_BIN, "bootloader.bin")
def find_partitions():
"""Find partitions.bin from PlatformIO build output or Release/ bundle."""
return _find_in_platformio_or_release(PARTITIONS_BIN, "partitions.bin")
BOOT_APP0_BIN = find_boot_app0()
# ── Helpers ────────────────────────────────────────────────────────────────────
@@ -231,34 +274,13 @@ def download_firmware(dest_path):
return True
def merge_firmware(output_path, esptool_cmd):
"""Merge bootloader + partitions + boot_app0 + app into a single binary."""
# Check all required files exist
required = {
"bootloader": BOOTLOADER_BIN,
"partitions": PARTITIONS_BIN,
"firmware": FIRMWARE_BIN,
}
# boot_app0 can come from PlatformIO or be bundled
boot_app0 = BOOT_APP0_BIN
if not boot_app0 or not os.path.isfile(boot_app0):
print("Error: boot_app0.bin not found.")
print(" Run 'pio run -e heltec_V4_boundary' first, or install PlatformIO.")
return False
required["boot_app0"] = boot_app0
for name, path in required.items():
if not os.path.isfile(path):
print(f"Error: {name} not found: {path}")
print("Run 'pio run -e heltec_V4_boundary' to build first.")
return False
def _do_merge(output_path, esptool_cmd, bootloader, partitions, boot_app0, firmware):
"""Low-level merge: combine the four components into a single binary."""
print("Merging firmware components...")
print(f" Bootloader: {BOOTLOADER_BIN} @ 0x{BOOTLOADER_ADDR:04x}")
print(f" Partitions: {PARTITIONS_BIN} @ 0x{PARTITIONS_ADDR:04x}")
print(f" boot_app0: {boot_app0} @ 0x{BOOT_APP0_ADDR:04x}")
print(f" Firmware: {FIRMWARE_BIN} @ 0x{APP_ADDR:05x}")
print(f" Bootloader: {bootloader} @ 0x{BOOTLOADER_ADDR:04x}")
print(f" Partitions: {partitions} @ 0x{PARTITIONS_ADDR:04x}")
print(f" boot_app0: {boot_app0} @ 0x{BOOT_APP0_ADDR:04x}")
print(f" Firmware: {firmware} @ 0x{APP_ADDR:05x}")
cmd = esptool_cmd + [
"--chip", CHIP,
@@ -267,10 +289,10 @@ def merge_firmware(output_path, esptool_cmd):
"--flash_freq", FLASH_FREQ,
"--flash_size", FLASH_SIZE,
"-o", output_path,
f"0x{BOOTLOADER_ADDR:x}", BOOTLOADER_BIN,
f"0x{PARTITIONS_ADDR:x}", PARTITIONS_BIN,
f"0x{BOOTLOADER_ADDR:x}", bootloader,
f"0x{PARTITIONS_ADDR:x}", partitions,
f"0x{BOOT_APP0_ADDR:x}", boot_app0,
f"0x{APP_ADDR:x}", FIRMWARE_BIN,
f"0x{APP_ADDR:x}", firmware,
]
result = subprocess.run(cmd, capture_output=True, text=True)
@@ -284,6 +306,67 @@ def merge_firmware(output_path, esptool_cmd):
return True
def merge_firmware(output_path, esptool_cmd):
"""Merge bootloader + partitions + boot_app0 + app into a single binary.
Uses PlatformIO build output, falling back to bundled Release/ copies
for the boot components.
"""
bootloader = find_bootloader()
partitions = find_partitions()
boot_app0 = BOOT_APP0_BIN
firmware = FIRMWARE_BIN
missing = []
if not bootloader: missing.append(("bootloader", BOOTLOADER_BIN))
if not partitions: missing.append(("partitions", PARTITIONS_BIN))
if not boot_app0: missing.append(("boot_app0", "(not found)"))
if not os.path.isfile(firmware):
missing.append(("firmware", firmware))
if missing:
for name, path in missing:
print(f"Error: {name} not found: {path}")
print("Run 'pio run -e heltec_V4_boundary' to build first.")
return False
return _do_merge(output_path, esptool_cmd, bootloader, partitions, boot_app0, firmware)
def auto_merge_app_binary(app_binary_path, esptool_cmd):
"""Auto-merge an app-only binary with boot components for a full flash.
Finds bootloader, partitions, and boot_app0 from PlatformIO build output
or the bundled Release/ directory, then merges them with the supplied
app binary into a temporary merged file.
Returns the path to the merged binary on success, or None on failure.
"""
bootloader = find_bootloader()
partitions = find_partitions()
boot_app0 = BOOT_APP0_BIN
missing = []
if not bootloader: missing.append("bootloader.bin")
if not partitions: missing.append("partitions.bin")
if not boot_app0: missing.append("boot_app0.bin")
if missing:
print(f"Cannot auto-merge: missing {', '.join(missing)}")
print("Place them in the Release/ folder alongside flash.py, or")
print("build with PlatformIO: pio run -e heltec_V4_boundary")
return None
# Create merged binary next to the app binary
base, ext = os.path.splitext(app_binary_path)
merged_path = f"{base}_merged{ext}"
print("Auto-merging app-only binary with boot components...")
if _do_merge(merged_path, esptool_cmd, bootloader, partitions, boot_app0, app_binary_path):
return merged_path
return None
def reset_to_bootloader(port):
"""Open serial port at 1200 baud to trigger ESP32-S3 USB bootloader reset.
@@ -323,22 +406,7 @@ def flash_firmware(firmware_path, port, esptool_cmd, baud=BAUD_RATE):
print(f" Chip: {CHIP} Baud: {baud} Flash: {FLASH_SIZE}\n")
# Determine if this is a merged binary (flash at 0x0) or app-only (flash at 0x10000)
#
# Both merged and app-only binaries start with 0xE9 (ESP32 image magic), so
# that byte alone cannot distinguish them. Instead, check for the partition
# table magic (0xAA 0x50) at offset 0x8000 — only merged binaries contain
# the partition table embedded at that offset.
size = os.path.getsize(firmware_path)
is_merged = False
try:
with open(firmware_path, "rb") as f:
if size > 0x8002: # Must be large enough to contain partition table area
f.seek(0x8000)
pt_magic = f.read(2)
if pt_magic == PARTITION_TABLE_MAGIC:
is_merged = True
except Exception:
pass
is_merged = is_merged_binary(firmware_path)
if is_merged:
flash_addr = f"0x{BOOTLOADER_ADDR:x}"
@@ -500,6 +568,36 @@ Examples:
if erase_choice == "y":
args.erase = True
# ── Safety check: erase + app-only → auto-merge ────────────────────────
if args.erase and not is_merged_binary(firmware_path):
print()
print("╔══════════════════════════════════════════════════════════════╗")
print("║ Erase selected with app-only binary — auto-merging boot ║")
print("║ components (bootloader + partition table + boot_app0) so ║")
print("║ the device remains bootable after erase. ║")
print("╚══════════════════════════════════════════════════════════════╝")
print()
merged = auto_merge_app_binary(firmware_path, esptool_cmd)
if merged:
firmware_path = merged
print(f"\nUsing auto-merged binary: {firmware_path}")
print(f" Size: {os.path.getsize(firmware_path):,} bytes")
print()
else:
print()
print("Auto-merge failed. Options:")
print(" 1) Skip erase and flash app-only (preserves existing NVS/bootloader)")
print(" 2) Abort")
try:
fallback = input("\nSkip erase and continue with app-only flash? [Y/n] ").strip().lower()
except EOFError:
fallback = ""
if fallback == "n":
print("Aborted.")
sys.exit(1)
args.erase = False
print("Erase skipped. Continuing with app-only flash...\n")
confirm = input("\nFlash firmware? [Y/n] ").strip().lower()
if confirm and confirm != "y":
print("Aborted.")

5
lib/microReticulum/src/Log.cpp
View File

@@ -8,8 +8,11 @@
using namespace RNS;
//LogLevel _level = LOG_VERBOSE;
#ifdef NDEBUG
LogLevel _level = LOG_VERBOSE;
#else
LogLevel _level = LOG_TRACE;
#endif
//LogLevel _level = LOG_MEM;
RNS::log_callback _on_log = nullptr;
char _datetime[20];

133
lib/microReticulum/src/Transport.cpp
View File

@@ -1327,13 +1327,26 @@ static bool is_backbone_interface(const Interface& iface) {
if (accept) {
TRACE("Transport::inbound: Packet accepted by filter");
// BOUNDARY MODE: Gate backbone traffic using two whitelists.
// Whitelist 1: local device addresses (LoRa + LocalTCP)
// Whitelist 2: addresses mentioned in packets from local devices
// BOUNDARY MODE: Comprehensive firewall for backbone traffic.
//
// Three rules:
// 1. Addresses that touch local interfaces (RNode/LoRa, LocalTCP)
// get whitelisted on the backbone interface.
// 2. Every packet referencing a whitelisted address — ALL identifiers
// in that packet also get whitelisted (link hashes, announces,
// requests, proofs, truncated hashes, transport IDs, EVERYTHING).
// 3. Everything else gets blocked on the backbone interface.
//
// Note on ratchets: ratchet public keys are embedded in announce
// payloads and flow through unchanged since we forward the entire
// announce verbatim. Ratchet IDs are derived locally and never
// appear as transport-level routing identifiers, so no special
// handling is needed here.
#ifdef BOUNDARY_MODE
{
bool is_backbone = is_backbone_interface(packet.receiving_interface());
if (is_backbone) {
// === BACKBONE PACKET: gate against all whitelists ===
bool allowed = false;
// Whitelist 1: destination is a local device
if (_boundary_local_addresses.find(packet.destination_hash()) != _boundary_local_addresses.end()) {
@@ -1343,33 +1356,60 @@ static bool is_backbone_interface(const Interface& iface) {
else if (_boundary_mentioned_addresses.find(packet.destination_hash()) != _boundary_mentioned_addresses.end()) {
allowed = true;
}
// Allow return traffic: proofs routed via reverse_table
// (destination is the packet hash of a packet we forwarded)
// Return traffic: proofs routed via reverse_table
else if (_reverse_table.find(packet.destination_hash()) != _reverse_table.end()) {
allowed = true;
}
// Allow return traffic: link proofs and link data routed via link_table
// (destination is the link_id of a link we're transporting)
// Return traffic: link proofs and link data via link_table
else if (_link_table.find(packet.destination_hash()) != _link_table.end()) {
allowed = true;
}
// Allow packets addressed to our own control destinations
// (e.g. path request handler) so backbone nodes can discover
// paths to local devices through us
// Our own control destinations (path requests, tunnel synthesize)
else if (_control_hashes.find(packet.destination_hash()) != _control_hashes.end()) {
allowed = true;
}
// Allow packets addressed to our own registered destinations
// Our own registered destinations
else if (_destinations.find(packet.destination_hash()) != _destinations.end()) {
allowed = true;
}
// HEADER_2 packet addressed to us as transport node — the
// sending node routed this to us so we must accept it even
// if we haven't seen this specific destination before
else if (packet.header_type() == Type::Packet::HEADER_2
&& packet.transport_id() == _identity.hash()) {
allowed = true;
}
if (!allowed) {
return;
}
// === TRANSITIVE WHITELIST ===
// Extract ALL identifiers from this allowed backbone packet
// so that future related traffic (proofs, link data, return
// packets) will also pass through the filter.
_boundary_mentioned_addresses.insert(packet.destination_hash());
if (packet.header_type() == Type::Packet::HEADER_2 && packet.transport_id()) {
_boundary_mentioned_addresses.insert(packet.transport_id());
}
if (packet.packet_type() == Type::Packet::LINKREQUEST) {
_boundary_mentioned_addresses.insert(Link::link_id_from_lr_packet(packet));
}
_boundary_mentioned_addresses.insert(packet.getTruncatedHash());
}
else {
// Packet from local interface: add its destination to Whitelist 2
// === LOCAL DEVICE PACKET ===
// Whitelist ALL identifiers from this packet so future
// related backbone traffic will be allowed through.
// Every identifier that touches a local interface gets
// whitelisted on the backbone — link hashes, announces,
// requests, proofs, EVERYTHING.
_boundary_mentioned_addresses.insert(packet.destination_hash());
if (packet.header_type() == Type::Packet::HEADER_2 && packet.transport_id()) {
_boundary_mentioned_addresses.insert(packet.transport_id());
}
if (packet.packet_type() == Type::Packet::LINKREQUEST) {
_boundary_mentioned_addresses.insert(Link::link_id_from_lr_packet(packet));
}
_boundary_mentioned_addresses.insert(packet.getTruncatedHash());
}
}
#endif
@@ -1545,6 +1585,16 @@ static bool is_backbone_interface(const Interface& iface) {
Interface outbound_interface = destination_entry.receiving_interface();
#ifdef BOUNDARY_MODE
// In boundary mode, never route a packet from backbone back to backbone.
// The upstream server sent us this packet because we are the next hop,
// so the destination must be on our local side.
if (is_backbone_interface(packet.receiving_interface()) && is_backbone_interface(outbound_interface)) {
// Path table incorrectly points to backbone. Skip forwarding.
}
else
#endif
{
if (packet.packet_type() == Type::Packet::LINKREQUEST) {
TRACE("Transport::inbound: Packet is next-hop LINKREQUEST");
double now = OS::time();
@@ -1580,6 +1630,7 @@ static bool is_backbone_interface(const Interface& iface) {
transmit(outbound_interface, new_raw);
#endif
destination_entry._timestamp = OS::time();
} // boundary mode else
}
else {
#ifdef BOUNDARY_MODE
@@ -1696,16 +1747,52 @@ static bool is_backbone_interface(const Interface& iface) {
auto destination_iter = _destination_table.find(packet.destination_hash());
if (destination_iter != _destination_table.end()) {
DestinationEntry& dest_entry = (*destination_iter).second;
Bytes next_hop = dest_entry._received_from;
uint8_t remaining_hops = dest_entry._hops;
Interface outbound_interface = dest_entry.receiving_interface();
// Create reverse_table entry so proof can get back
ReverseEntry reverse_entry(
packet.receiving_interface(), outbound_interface, OS::time()
);
_reverse_table.insert({packet.getTruncatedHash(), reverse_entry});
// Build properly routed packet based on remaining hops,
// mirroring the standard transport forwarding logic.
Bytes new_raw(512);
if (remaining_hops > 1) {
// Multi-hop: wrap with HEADER_2/TRANSPORT
uint8_t new_flags = (Type::Packet::HEADER_2) << 6
| (Type::Transport::TRANSPORT) << 4
| (packet.flags() & 0b00001111);
new_raw << new_flags;
new_raw << packet.hops();
new_raw << next_hop; // transport_id
new_raw << packet.raw().mid(2); // destination_hash + payload
}
else {
// Direct or single-hop: send as HEADER_1
new_raw << packet.raw().left(1);
new_raw << packet.hops();
new_raw << packet.raw().mid(2);
}
DEBUG("BOUNDARY: Forwarding backbone packet to local device for " + packet.destination_hash().toHex() + " via " + outbound_interface.toString());
transmit(outbound_interface, packet.raw());
// Create link_table or reverse_table entry for return traffic
if (packet.packet_type() == Type::Packet::LINKREQUEST) {
double now = OS::time();
double proof_timeout = now + Type::Link::ESTABLISHMENT_TIMEOUT_PER_HOP
* std::max((uint8_t)1, remaining_hops);
LinkEntry link_entry(
now, next_hop, outbound_interface, remaining_hops,
packet.receiving_interface(), packet.hops(),
packet.destination_hash(), false, proof_timeout
);
_link_table.insert({Link::link_id_from_lr_packet(packet), link_entry});
DEBUG("BOUNDARY: Created link_table entry for backbone LINKREQUEST, link_id=" + Link::link_id_from_lr_packet(packet).toHex());
}
else {
ReverseEntry reverse_entry(
packet.receiving_interface(), outbound_interface, OS::time()
);
_reverse_table.insert({packet.getTruncatedHash(), reverse_entry});
}
DEBUG("BOUNDARY: Forwarding backbone packet (" + std::to_string(remaining_hops) + " hops) to local device for " + packet.destination_hash().toHex() + " via " + outbound_interface.toString());
transmit(outbound_interface, new_raw);
dest_entry._timestamp = OS::time();
}
}
@@ -2189,9 +2276,13 @@ static bool is_backbone_interface(const Interface& iface) {
packet.get_hash()
);
// CBA ACCUMULATES
// Erase existing entry so insert overwrites (matching Python dict[key]=value)
bool path_existed = (_destination_table.erase(packet.destination_hash()) > 0);
if (_destination_table.insert({packet.destination_hash(), destination_table_entry}).second) {
++_destinations_added;
cull_path_table();
if (!path_existed) {
++_destinations_added;
cull_path_table();
}
}
DEBUG("Destination " + packet.destination_hash().toHex() + " is now " + std::to_string(announce_hops) + " hops away via " + received_from.toHex() + " on " + packet.receiving_interface().toString());

17
lib/microReticulum/src/Utilities/OS.cpp
View File

@@ -3,6 +3,10 @@
#include "../Type.h"
#include "../Log.h"
#if defined(ESP32) && defined(BOARD_HAS_PSRAM)
#include <esp_heap_caps.h>
#endif
using namespace RNS;
using namespace RNS::Utilities;
@@ -47,7 +51,18 @@ void* operator new(size_t size) {
//if (OS::_tlsf == nullptr) {
if (!_tlsf_init) {
_tlsf_init = true;
#if defined(ESP32)
#if defined(ESP32) && defined(BOARD_HAS_PSRAM)
// Use PSRAM for TLSF pool — frees internal SRAM for WiFi/LoRa/stack.
// PSRAM is slower (QSPI) but has 2MB vs ~170KB free internal.
_contiguous_size = ESP.getMaxAllocPsram();
TRACEF("psram contiguous_size: %u", _contiguous_size);
if (_buffer_size == 0) {
_buffer_size = (_contiguous_size * 4) / 5;
}
size_t align = tlsf_align_size();
_buffer_size &= ~(align - 1);
void* raw_buffer = heap_caps_aligned_alloc(align, _buffer_size, MALLOC_CAP_SPIRAM);
#elif defined(ESP32)
// CBA Still unknown why the call to tlsf_create_with_pool() is so flaky on ESP32 with calculated buffer size. Reuires more research and unit tests.
_contiguous_size = ESP.getMaxAllocHeap();
TRACEF("contiguous_size: %u", _contiguous_size);

2
platformio.ini
View File

@@ -335,12 +335,14 @@ board_build.partitions = default_16MB.csv
board_build.flash_mode = qio
board_build.psram_type = qio
board_build.arduino.memory_type = qio_qspi
monitor_speed = 921600
build_flags =
${env.build_flags}
-DBOARD_MODEL=BOARD_HELTEC32_V4
-DARDUINO_USB_CDC_ON_BOOT=1
-DBOARD_HAS_PSRAM=1
-DBOUNDARY_MODE
-DNDEBUG
; --- Boundary mode defaults (override via EEPROM at runtime) ---
; TCP server mode (0=server, 1=client)
-DBOUNDARY_TCP_MODE=0