Skip to content

Add NMEA2000 talker ID to the WIBL raw data file format, and support for reading the file, and converting to it. #119

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
brian-r-calder wants to merge 2 commits into from
Open

Add NMEA2000 talker ID to the WIBL raw data file format, and support for reading the file, and converting to it. #119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ad4d4f3
fw: add N2K sender ID to output packets for v1.4 of the file format.
brian-r-calder Aug 11, 2026
0246d17
Added build dependency directory to gitignore for LogConvert build.
brian-r-calder Aug 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ xcshareddata

# Build products
build/
build-deps/
*.o
*.LinkFileList
*.hmap
Expand All @@ -56,4 +57,4 @@ AWS/Repo-information.eml
**/local/
**/tmp/
fp-info-cache
WIBL-backups/
WIBL-backups/
30 changes: 20 additions & 10 deletions LogConvert/src/SerialisableFactory.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,10 +106,11 @@ std::shared_ptr<Serialisable> HandleSystemTime(tN2kMsg& msg, bool& no_data_detec
no_data_detected = true;
}

rtn = std::shared_ptr<Serialisable>(new Serialisable(sizeof(uint16_t) + sizeof(double) + sizeof(unsigned long) + 1));
rtn = std::shared_ptr<Serialisable>(new Serialisable(sizeof(uint16_t) + sizeof(double) + sizeof(unsigned long) + 2));
*rtn += date;
*rtn += timestamp;
*rtn += static_cast<uint32_t>(msg.MsgTime);
*rtn += (uint8_t)msg.Source;
*rtn += (uint8_t)source;
}
}
Expand All @@ -136,8 +137,9 @@ std::shared_ptr<Serialisable> HandleAttitude(tN2kMsg& msg, bool& no_data_detecte
no_data_detected = true;
}

rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 3*sizeof(double)));
rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 1 + 3*sizeof(double)));
t.Serialise(rtn);
*rtn += (uint8_t)msg.Source;
*rtn += yaw;
*rtn += pitch;
*rtn += roll;
Expand All @@ -164,8 +166,9 @@ std::shared_ptr<Serialisable> HandleDepth(tN2kMsg& msg, bool& no_data_detected)
no_data_detected = true;
}

rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 3*sizeof(double)));
rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 1 + 3*sizeof(double)));
t.Serialise(rtn);
*rtn += (uint8_t)msg.Source;
*rtn += depth;
*rtn += offset;
*rtn += range;
Expand Down Expand Up @@ -194,8 +197,9 @@ std::shared_ptr<Serialisable> HandleCOG(tN2kMsg& msg, bool& no_data_detected)
no_data_detected = true;
}

rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 2*sizeof(double)));
rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 1 + 2*sizeof(double)));
t.Serialise(rtn);
*rtn += (uint8_t)msg.Source;
*rtn += cog;
*rtn += sog;
}
Expand Down Expand Up @@ -235,8 +239,9 @@ std::shared_ptr<Serialisable> HandleGNSS(tN2kMsg& msg, bool& no_data_detected)
|| N2kIsNA(sep) || N2kIsNA(nRefStations) || N2kIsNA((uint8_t)refStationType) || N2kIsNA(refStationID) || N2kIsNA(correctionAge)) {
no_data_detected = true;
}
rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 2*sizeof(uint16_t) + 8*sizeof(double) + 5));
rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 1 + 2*sizeof(uint16_t) + 8*sizeof(double) + 5));
t.Serialise(rtn); // Put in the standard timestamp, as well as the in-message one.
*rtn += (uint8_t)msg.Source;
*rtn += datestamp; *rtn += timestamp;
*rtn += latitude; *rtn += longitude; *rtn += altitude;
*rtn += (uint8_t)rec_type; *rtn += (uint8_t)rec_method;
Expand Down Expand Up @@ -272,8 +277,9 @@ std::shared_ptr<Serialisable> HandleEnvironment(tN2kMsg& msg, bool& no_data_dete
no_data_detected = true;
}

rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 3*sizeof(double) + 2));
rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 1 + 3*sizeof(double) + 2));
t.Serialise(rtn);
*rtn += (uint8_t)msg.Source;
*rtn += (uint8_t)t_source;
*rtn += temp;
*rtn += (uint8_t)h_source;
Expand Down Expand Up @@ -305,8 +311,9 @@ std::shared_ptr<Serialisable> HandleTemperature(tN2kMsg& msg, bool& no_data_dete
no_data_detected = true;
}

rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 1 + sizeof(double)));
rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 2 + sizeof(double)));
t.Serialise(rtn);
*rtn += (uint8_t)msg.Source;
*rtn += (uint8_t)t_source;
*rtn += temp;
}
Expand Down Expand Up @@ -336,8 +343,9 @@ std::shared_ptr<Serialisable> HandleHumidity(tN2kMsg& msg, bool& no_data_detecte
no_data_detected = true;
}

rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 1 + sizeof(double)));
rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 2 + sizeof(double)));
t.Serialise(rtn);
*rtn += (uint8_t)msg.Source;
*rtn += (uint8_t)h_source;
*rtn += humidity;
}
Expand Down Expand Up @@ -367,8 +375,9 @@ std::shared_ptr<Serialisable> HandlePressure(tN2kMsg& msg, bool& no_data_detecte
no_data_detected = true;
}

rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 1 + sizeof(double)));
rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 2 + sizeof(double)));
t.Serialise(rtn);
*rtn += (uint8_t)msg.Source;
*rtn += (uint8_t)p_source;
*rtn += pressure;
}
Expand Down Expand Up @@ -398,8 +407,9 @@ std::shared_ptr<Serialisable> HandleExtTemperature(tN2kMsg& msg, bool& no_data_d
no_data_detected = true;
}

rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 1 + sizeof(double)));
rtn = std::shared_ptr<Serialisable>(new Serialisable(t.SerialisationSize() + 2 + sizeof(double)));
t.Serialise(rtn);
*rtn += (uint8_t)msg.Source;
*rtn += (uint8_t)t_source;
*rtn += temp;
}
Expand Down
6 changes: 5 additions & 1 deletion LogConvert/src/logconvert.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ namespace po = boost::program_options;
#include "serialisation.h"
#include "SerialisableFactory.h"

const char *version = "1.2.0";

/// Dummy code to allow the system to pretend that there's a millisecond counter
///
/// \return Nominally the millisecond counter since boot; there, uniformly zero
Expand Down Expand Up @@ -157,7 +159,9 @@ std::string NameOutputPacket(uint32_t packet_id)

void Syntax(po::options_description const& cmdopt)
{
std::cout << "logconvert [" << __DATE__ << ", " << __TIME__ << "] - Convert VGI log output to WIBL for upload." << std::endl;
std::cout << "logconvert [" << __DATE__ << ", " << __TIME__ << "] v." << version
<< " - Convert VGI log output to WIBL for upload." << std::endl;
std::cout << " Serialiser Version: " << SerialiserVersionMajor << "." << SerialiserVersionMinor << std::endl;
std::cout << "Syntax: logconvert [opt] <input><output>" << std::endl;
std::cout << cmdopt << std::endl;
}
Expand Down
2 changes: 1 addition & 1 deletion LogConvert/src/serialisation.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
#undef minor

const int SerialiserVersionMajor = 1; ///< Major version number for the serialiser
const int SerialiserVersionMinor = 3; ///< Minor version number for the serialiser
const int SerialiserVersionMinor = 4; ///< Minor version number for the serialiser

/// \class Serialisable
/// \brief Provide encapsulation for data to be written to store
Expand Down
3 changes: 3 additions & 0 deletions LoggerFirmware/ReleaseNotes.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,11 @@
## Firmware 1.7.0

Firmware 1.7.0 addresses multiple issues in the repository, including:

* [Issue 110](https://github.com/CCOMJHC/WIBL/issues/110), which changes the filename structure for raw files to include a fixed extension (e.g., "raw00000.wbl") rather than encoding the filename in the extension. In addition to allowing for more files, this avoids problems in processing later.

* [Issue 81](https://github.com/CCOMJHC/WIBL/issues/81), which adds source (speaker) information to NMEA2000 output packets in the raw file, stepping the serialiser version to 1.4. This also updates the Python code for reading files to be compatible, and updates logconvert to write version 1.4 data.

## Firmware 1.6.2

Firmware 1.6.2 addresses [issue 107](https://github.com/CCOMJHC/WIBL/issues/107) in the repository, where the size of the JSON documents being constructed in the firmware were getting to the size where they took up enough memory that they caused problems when generating SSL connections during upload. This also seemed to have significantly slowed down the webserver, so this version of the firmware should show much snappier connections on the server as a (very welcome) side effect.
Expand Down
2 changes: 1 addition & 1 deletion LoggerFirmware/include/serialisation.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
#include "FS.h"

const int SerialiserVersionMajor = 1; ///< Major version number for the serialiser
const int SerialiserVersionMinor = 3; ///< Minor version number for the serialiser
const int SerialiserVersionMinor = 4; ///< Minor version number for the serialiser

/// \class Serialisable
/// \brief Provide encapsulation for data to be written to store
Expand Down
30 changes: 20 additions & 10 deletions LoggerFirmware/src/N2kLogger.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,10 +277,11 @@ void Logger::HandleSystemTime(Timestamp::TimeDatum const& t, const tN2kMsg& msg)
logger::DataObs obs(t.RawElapsed(), date, timestamp);
logger::Metrics.RegisterObs(obs);

Serialisable s(sizeof(uint16_t) + sizeof(double) + sizeof(unsigned long) + 1);
Serialisable s(sizeof(uint16_t) + sizeof(double) + sizeof(unsigned long) + 2);
s += date;
s += timestamp;
s += t.RawElapsed();
s += (uint8_t)msg.Source;
s += (uint8_t)source;
m_logManager->Record(logger::Manager::PacketIDs::Pkt_SystemTime, s);
m_logManager->Syslog(String("INF: Time update to: ") + m_timeReference.printable());
Expand All @@ -306,8 +307,9 @@ void Logger::HandleAttitude(Timestamp::TimeDatum const& t, tN2kMsg const& msg)
if (IsNA(t) || N2kIsNA(yaw) || N2kIsNA(pitch) || N2kIsNA(roll))
m_logManager->EmitNoDataReject();

Serialisable s(t.SerialisationSize() + 3*sizeof(double));
Serialisable s(t.SerialisationSize() + 1 + 3*sizeof(double));
t.Serialise(s);
s += (uint8_t)msg.Source;
s += yaw;
s += pitch;
s += roll;
Expand Down Expand Up @@ -338,8 +340,9 @@ void Logger::HandleDepth(Timestamp::TimeDatum const& t, tN2kMsg const& msg)
if (IsNA(t) || N2kIsNA(depth) || N2kIsNA(offset) || N2kIsNA(range))
m_logManager->EmitNoDataReject();

Serialisable s(t.SerialisationSize() + 3*sizeof(double));
Serialisable s(t.SerialisationSize() + 1 + 3*sizeof(double));
t.Serialise(s);
s += (uint8_t)msg.Source;
s += depth;
s += offset;
s += range;
Expand Down Expand Up @@ -370,8 +373,9 @@ void Logger::HandleCOG(Timestamp::TimeDatum const& t, tN2kMsg const& msg)
if (IsNA(t) || N2kIsNA(cog) || N2kIsNA(sog))
m_logManager->EmitNoDataReject();

Serialisable s(t.SerialisationSize() + 2*sizeof(double));
Serialisable s(t.SerialisationSize() + 1 + 2*sizeof(double));
t.Serialise(s);
s += (uint8_t)msg.Source;
s += cog;
s += sog;
m_logManager->Record(logger::Manager::PacketIDs::Pkt_COG, s);
Expand Down Expand Up @@ -420,8 +424,9 @@ void Logger::HandleGNSS(Timestamp::TimeDatum const& t, tN2kMsg const& msg)
N2kIsNA(refStationID) || N2kIsNA(correctionAge))
m_logManager->EmitNoDataReject();

Serialisable s(t.SerialisationSize() + 2*sizeof(uint16_t) + 8*sizeof(double) + 5);
Serialisable s(t.SerialisationSize() + 1 + 2*sizeof(uint16_t) + 8*sizeof(double) + 5);
t.Serialise(s); // Put in the standard timestamp, as well as the in-message one.
s += (uint8_t)msg.Source;
s += datestamp; s += timestamp;
s += latitude; s += longitude; s += altitude;
s += (uint8_t)rec_type; s += (uint8_t)rec_method;
Expand Down Expand Up @@ -470,8 +475,9 @@ void Logger::HandleEnvironment(Timestamp::TimeDatum const& t, tN2kMsg const& msg
N2kIsNA((uint8_t)h_source) || N2kIsNA(humidity) || N2kIsNA(pressure))
m_logManager->EmitNoDataReject();

Serialisable s(t.SerialisationSize() + 3*sizeof(double) + 2);
Serialisable s(t.SerialisationSize() + 1 + 3*sizeof(double) + 2);
t.Serialise(s);
s += (uint8_t)msg.Source;
s += (uint8_t)t_source;
s += temp;
s += (uint8_t)h_source;
Expand Down Expand Up @@ -506,8 +512,9 @@ void Logger::HandleTemperature(Timestamp::TimeDatum const& t, tN2kMsg const& msg
if (IsNA(t) || N2kIsNA((uint8_t)t_source) || N2kIsNA(temp))
m_logManager->EmitNoDataReject();

Serialisable s(t.SerialisationSize() + 1 + sizeof(double));
Serialisable s(t.SerialisationSize() + 2 + sizeof(double));
t.Serialise(s);
s += (uint8_t)msg.Source;
s += (uint8_t)t_source;
s += temp;
m_logManager->Record(logger::Manager::PacketIDs::Pkt_Temperature, s);
Expand Down Expand Up @@ -539,8 +546,9 @@ void Logger::HandleHumidity(Timestamp::TimeDatum const& t, tN2kMsg const& msg)
if (IsNA(t) || N2kIsNA((uint8_t)h_source) || N2kIsNA(humidity))
m_logManager->EmitNoDataReject();

Serialisable s(t.SerialisationSize() + 1 + sizeof(double));
Serialisable s(t.SerialisationSize() + 2 + sizeof(double));
t.Serialise(s);
s += (uint8_t)msg.Source;
s += (uint8_t)h_source;
s += humidity;
m_logManager->Record(logger::Manager::PacketIDs::Pkt_Humidity, s);
Expand Down Expand Up @@ -572,8 +580,9 @@ void Logger::HandlePressure(Timestamp::TimeDatum const& t, tN2kMsg const& msg)
if (IsNA(t) || N2kIsNA((uint8_t)p_source) || N2kIsNA(pressure))
m_logManager->EmitNoDataReject();

Serialisable s(t.SerialisationSize() + 1 + sizeof(double));
Serialisable s(t.SerialisationSize() + 2 + sizeof(double));
t.Serialise(s);
s += (uint8_t)msg.Source;
s += (uint8_t)p_source;
s += pressure;
m_logManager->Record(logger::Manager::PacketIDs::Pkt_Pressure, s);
Expand Down Expand Up @@ -605,8 +614,9 @@ void Logger::HandleExtTemperature(Timestamp::TimeDatum const& t, tN2kMsg const&
if (IsNA(t) || N2kIsNA((uint8_t)t_source) || N2kIsNA(temp))
m_logManager->EmitNoDataReject();

Serialisable s(t.SerialisationSize() + 1 + sizeof(double));
Serialisable s(t.SerialisationSize() + 2 + sizeof(double));
t.Serialise(s);
s += (uint8_t)msg.Source;
s += (uint8_t)t_source;
s += temp;
m_logManager->Record(logger::Manager::PacketIDs::Pkt_Temperature, s);
Expand Down
2 changes: 1 addition & 1 deletion wibl-python/src/wibl/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import argparse
import logging

__version__ = '1.2.0'
__version__ = '1.3.0'


LOGGER_NAME: str = 'wibl-python'
Expand Down
Loading