From 5dc60ad1d613218de8b65ce0ff5ecf66560ffde6 Mon Sep 17 00:00:00 2001 From: ACETyr <35745290+ACETyr@users.noreply.github.com> Date: Sun, 14 Jun 2026 19:00:57 +0200 Subject: [PATCH 1/2] Emit zone-aware UTC timestamp with real sub-second (was naive local + fake .000000) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The packet/raw `timestamp` was built from localtime() (naive, no zone) at publish time with a hardcoded ".000000" sub-second. Aggregators that assume UTC (e.g. CoreScope) clamp a naive value landing >15 min off their clock to ingest time and flag the observer as clock-skewed — so emitting local time actively degrades ingest. Now format from gmtime() with an explicit Z, and use gettimeofday() for a genuine microsecond fraction (SNTP-maintained on ESP32) instead of the literal ".000000". Real sub-second helps consumers correlating off the raw MQTT feed; second-resolution consumers (CoreScope) simply truncate it. Note: this stamps publish time; threading the queued RX capture (QueuedPacket has a millis() timestamp) for true receive time is a worthwhile follow-up. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/helpers/MQTTMessageBuilder.cpp | 71 ++++++++++++++++++------------ 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/src/helpers/MQTTMessageBuilder.cpp b/src/helpers/MQTTMessageBuilder.cpp index aa8057d9b0..d805b0c439 100644 --- a/src/helpers/MQTTMessageBuilder.cpp +++ b/src/helpers/MQTTMessageBuilder.cpp @@ -1,6 +1,7 @@ #include "MQTTMessageBuilder.h" #include #include +#include #include #include "MeshCore.h" @@ -168,18 +169,25 @@ int MQTTMessageBuilder::buildPacketJSON( if (!packet) return 0; // Get current device time (should be UTC since system timezone is set to UTC) - time_t now = time(nullptr); - - // Convert to local time using timezone library (for timestamp field only) - time_t local_time = timezone ? timezone->toLocal(now) : now; - struct tm* local_timeinfo = localtime(&local_time); - - // Format timestamp in ISO 8601 format (LOCAL TIME) + struct timeval now_tv; + gettimeofday(&now_tv, nullptr); + time_t now = now_tv.tv_sec; + + // Packet timestamp is emitted as zone-aware UTC (RFC3339 "Zulu") via gmtime — not naive + // local time. (timezone is intentionally not applied here; see the timestamp comment below.) + struct tm* utc_ts_info = gmtime(&now); + + // Zone-aware UTC (RFC3339 Zulu) with a REAL sub-second from gettimeofday(). The prior ".000000" + // was a hardcoded literal carrying no information; gettimeofday() yields genuine microseconds + // (SNTP-maintained on ESP32), useful for self-hosted logging/correlation. Aggregators that only + // need second resolution (e.g. CoreScope) simply truncate the fraction. Emitting UTC with an + // explicit Z (not naive local time) also stops such aggregators from clamping the value. char timestamp[32]; - if (local_timeinfo) { - strftime(timestamp, sizeof(timestamp), "%Y-%m-%dT%H:%M:%S.000000", local_timeinfo); + if (utc_ts_info) { + size_t ts_len = strftime(timestamp, sizeof(timestamp), "%Y-%m-%dT%H:%M:%S", utc_ts_info); + snprintf(timestamp + ts_len, sizeof(timestamp) - ts_len, ".%06ldZ", (long)now_tv.tv_usec); } else { - strcpy(timestamp, "2024-01-01T12:00:00.000000"); + strcpy(timestamp, "2024-01-01T12:00:00.000000Z"); } // Get UTC time (since system timezone is UTC, time() returns UTC) @@ -250,18 +258,25 @@ int MQTTMessageBuilder::buildPacketJSONFromRaw( if (!packet || !raw_data || raw_len <= 0) return 0; // Get current device time (should be UTC since system timezone is set to UTC) - time_t now = time(nullptr); - - // Convert to local time using timezone library (for timestamp field only) - time_t local_time = timezone ? timezone->toLocal(now) : now; - struct tm* local_timeinfo = localtime(&local_time); - - // Format timestamp in ISO 8601 format (LOCAL TIME) + struct timeval now_tv; + gettimeofday(&now_tv, nullptr); + time_t now = now_tv.tv_sec; + + // Packet timestamp is emitted as zone-aware UTC (RFC3339 "Zulu") via gmtime — not naive + // local time. (timezone is intentionally not applied here; see the timestamp comment below.) + struct tm* utc_ts_info = gmtime(&now); + + // Zone-aware UTC (RFC3339 Zulu) with a REAL sub-second from gettimeofday(). The prior ".000000" + // was a hardcoded literal carrying no information; gettimeofday() yields genuine microseconds + // (SNTP-maintained on ESP32), useful for self-hosted logging/correlation. Aggregators that only + // need second resolution (e.g. CoreScope) simply truncate the fraction. Emitting UTC with an + // explicit Z (not naive local time) also stops such aggregators from clamping the value. char timestamp[32]; - if (local_timeinfo) { - strftime(timestamp, sizeof(timestamp), "%Y-%m-%dT%H:%M:%S.000000", local_timeinfo); + if (utc_ts_info) { + size_t ts_len = strftime(timestamp, sizeof(timestamp), "%Y-%m-%dT%H:%M:%S", utc_ts_info); + snprintf(timestamp + ts_len, sizeof(timestamp) - ts_len, ".%06ldZ", (long)now_tv.tv_usec); } else { - strcpy(timestamp, "2024-01-01T12:00:00.000000"); + strcpy(timestamp, "2024-01-01T12:00:00.000000Z"); } // Get UTC time (since system timezone is UTC, time() returns UTC) @@ -327,18 +342,20 @@ int MQTTMessageBuilder::buildRawJSON( if (!packet) return 0; // Get current device time - time_t now = time(nullptr); + struct timeval now_tv; + gettimeofday(&now_tv, nullptr); + time_t now = now_tv.tv_sec; - // Convert to local time using timezone library - time_t local_time = timezone ? timezone->toLocal(now) : now; - struct tm* timeinfo = localtime(&local_time); + // Emit zone-aware UTC (RFC3339 Zulu) — consistent with the packet builders above. + struct tm* timeinfo = gmtime(&now); - // Format timestamp in ISO 8601 format + // Format timestamp as zone-aware UTC (RFC3339 Zulu) with real sub-second from gettimeofday() char timestamp[32]; if (timeinfo) { - strftime(timestamp, sizeof(timestamp), "%Y-%m-%dT%H:%M:%S.000000", timeinfo); + size_t ts_len = strftime(timestamp, sizeof(timestamp), "%Y-%m-%dT%H:%M:%S", timeinfo); + snprintf(timestamp + ts_len, sizeof(timestamp) - ts_len, ".%06ldZ", (long)now_tv.tv_usec); } else { - strcpy(timestamp, "2024-01-01T12:00:00.000000"); + strcpy(timestamp, "2024-01-01T12:00:00.000000Z"); } // Convert packet to hex From 7cb7807c52189b0da4d3eded893249cee9e14094 Mon Sep 17 00:00:00 2001 From: ACETyr <35745290+ACETyr@users.noreply.github.com> Date: Mon, 15 Jun 2026 00:18:30 +0200 Subject: [PATCH 2/2] Emit zone-aware UTC for the status-message timestamp too PR #14 fixed the packet timestamp, but the two status-message builders in MQTTBridge.cpp still formatted a hardcoded ".000000" via getLocalTime(). Use the same gettimeofday() + gmtime() + ".%06ldZ" pattern so the status timestamp is unambiguous UTC with a real sub-second, consistent with the packet builder (and not naive local, which a CoreScope-style aggregator would clamp/flag). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/helpers/bridges/MQTTBridge.cpp | 32 ++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 287bc5f0b6..7419a2eed7 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1480,12 +1480,18 @@ bool MQTTBridge::publishStatus() { char timestamp[32]; char radio_info[64]; - // Get current timestamp in ISO 8601 format - struct tm timeinfo; - if (getLocalTime(&timeinfo)) { - strftime(timestamp, sizeof(timestamp), "%Y-%m-%dT%H:%M:%S.000000", &timeinfo); + // Zone-aware UTC timestamp (RFC3339 "Zulu") with a real sub-second from gettimeofday(), matching + // the packet builder (MQTTMessageBuilder). The prior ".000000" literal carried no sub-second and + // (with a local TZ set) could read as naive local; gmtime() + an explicit Z keeps it unambiguous UTC. + struct timeval now_tv; + gettimeofday(&now_tv, nullptr); + time_t now_sec = now_tv.tv_sec; + struct tm* utc_info = gmtime(&now_sec); + if (utc_info) { + size_t ts_len = strftime(timestamp, sizeof(timestamp), "%Y-%m-%dT%H:%M:%S", utc_info); + snprintf(timestamp + ts_len, sizeof(timestamp) - ts_len, ".%06ldZ", (long)now_tv.tv_usec); } else { - strcpy(timestamp, "2024-01-01T12:00:00.000000"); + strcpy(timestamp, "2024-01-01T12:00:00.000000Z"); } // Build radio info string (freq,bw,sf,cr) @@ -2497,12 +2503,18 @@ void MQTTBridge::publishStatusToAnalyzerClient(PsychicMqttClient* client, const char timestamp[32]; char radio_info[64]; - // Get current timestamp in ISO 8601 format - struct tm timeinfo; - if (getLocalTime(&timeinfo)) { - strftime(timestamp, sizeof(timestamp), "%Y-%m-%dT%H:%M:%S.000000", &timeinfo); + // Zone-aware UTC timestamp (RFC3339 "Zulu") with a real sub-second from gettimeofday(), matching + // the packet builder (MQTTMessageBuilder). The prior ".000000" literal carried no sub-second and + // (with a local TZ set) could read as naive local; gmtime() + an explicit Z keeps it unambiguous UTC. + struct timeval now_tv; + gettimeofday(&now_tv, nullptr); + time_t now_sec = now_tv.tv_sec; + struct tm* utc_info = gmtime(&now_sec); + if (utc_info) { + size_t ts_len = strftime(timestamp, sizeof(timestamp), "%Y-%m-%dT%H:%M:%S", utc_info); + snprintf(timestamp + ts_len, sizeof(timestamp) - ts_len, ".%06ldZ", (long)now_tv.tv_usec); } else { - strcpy(timestamp, "2024-01-01T12:00:00.000000"); + strcpy(timestamp, "2024-01-01T12:00:00.000000Z"); } // Build radio info string (freq,bw,sf,cr)