I have been using the iotawatt happily for many years now, but the thing that kept bothering me was that i could not use the binary files. So i’ve spent the last couple of weeks building a Python script to convert the binary log files from the microSD card to a csv or json file. Correct me if im wrong, but you cannot access the data using Graph(+) after you remove the input channel from the webserver. The data however is still in the binary log files. Using IotaLogConvert you can access that data again.
To be completely honest, i didn’t build IotaLogConvert, ChatGPT did. I learned a ton about c++ and Python because of it. It was fun. So here it is.
The script should work out of the box using python3. No additional libraries are required.
IotaLogConvert is as-is. Use it or change it as you see fit. Im not planning on adding any additional functionality. However, if there are any minor bugs, report them and i’ll ask ChatGPT to fix them :). IotaLogConvert does not output fields like Powerfactor or Watt Hours. I have added a csv and json file as an example (im not able to upload the binary file). I do not give any support running IotaLogConvert on Windows and strongly advice you to copy (or download if possible) the files before using IotaLogConvert.
iotalog.json (2.0 MB)
iotalog.csv (408.0 KB)
IotaLogConvert converted my 1.6GB IOTALOG.LOG to a 2.3GB CSV file in a little less than 4 minutes and to a 7.0GB json file in a little more than 7 minutes.
Command structure:
python3 IotaLogConvert.py --input <binary_file_path> --format <csv|json> --interval <seconds> [--timeformat <unix|normal>] --output <output_file>
Arguments:
--input <binary_file_path> or -i <binary_file_path>:
Required. Specifies the path to the binary log file that you want to process.
--format <csv|json> or -f <csv|json>:
Required. Specifies the export format. Choose csv for CSV file output or json for JSON file output.
--interval <seconds> or -I <seconds>:
Required. Specifies the time interval between records in seconds. You must provide a valid interval value. IOTALOG.LOG has an interval of 5 seconds. HISTLOG.LOG has an interval of 60 seconds.
--timeformat <unix|normal> or -t <unix|normal>:
Optional. Specifies the format for time values.
unix: Outputs time as a UNIX timestamp (default).
normal: Outputs time in a human-readable format (DD/MM/YYYY HH:MM:SS).
--output <output_file> or -o <output_file>:
Required. Specifies the name of the output file, including the file extension (.csv or .json).
IotaLogConvert.py:
# IotaLogConvert - Version 1.0
# This script processes binary log files from Iotawatt devices, converting them into CSV or JSON format.
import struct
import sys
import csv
import json
import os
from datetime import datetime, timezone
# Define the structure of the IotaLogRecord
record_format = "I I d 15d 15d"
record_size = struct.calcsize(record_format)
def parse_iota_log_record(binary_data):
unpacked_data = struct.unpack(record_format, binary_data)
return {
"UNIXtime": unpacked_data[0],
"serial": unpacked_data[1],
"logHours": unpacked_data[2],
"accum1": list(unpacked_data[3:18]),
"accum2": list(unpacked_data[18:33])
}
def read_binary_file(file_path):
records = []
with open(file_path, "rb") as f:
while True:
binary_data = f.read(record_size)
if len(binary_data) < record_size:
break
if struct.unpack("I", binary_data[:4])[0] == 0:
continue
records.append(parse_iota_log_record(binary_data))
return records
def calculate_watt_and_va(accum_data1, accum_data2, time_intervals_hours):
delta_accum1 = [j-i for i, j in zip(accum_data1[:-1], accum_data1[1:])]
delta_accum2 = [j-i for i, j in zip(accum_data2[:-1], accum_data2[1:])]
watt = [value / time_intervals_hours for value in delta_accum1]
va = [value / time_intervals_hours for value in delta_accum2]
return [accum_data1[0]] + watt, [accum_data2[0]] + va
def calculate_volts_and_hz(accum_data1, accum_data2, time_intervals_hours):
delta_accum1 = [j-i for i, j in zip(accum_data1[:-1], accum_data1[1:])]
delta_accum2 = [j-i for i, j in zip(accum_data2[:-1], accum_data2[1:])]
volts = [value / time_intervals_hours for value in delta_accum1]
hz = [value / time_intervals_hours for value in delta_accum2]
return [accum_data1[0]] + volts, [accum_data2[0]] + hz
def confirm_overwrite(filepath):
if os.path.exists(filepath):
response = input(f"File {filepath} already exists. Do you want to overwrite it? (y/n): ").strip().lower()
if response != 'y':
print("Operation cancelled.")
sys.exit(1)
def suggest_interval(file_name):
if "IOTALOG.LOG" in file_name.upper():
return 5
elif "HISTLOG.LOG" in file_name.upper():
return 60
return 60 # Default suggestion
def process_data(records, export_format, output_file, time_format='unix', interval=60):
time_intervals_hours = interval / 3600.0 # Convert interval to hours
data = []
header = ["Time", "Serial", "LogHours"]
UNIXtimes = [record['UNIXtime'] for record in records]
times = (
[datetime.fromtimestamp(t, tz=timezone.utc).strftime('%d/%m/%Y %H:%M:%S') for t in UNIXtimes]
if time_format == 'normal'
else UNIXtimes
)
# Processing Input 0 (Voltage Transformer)
accum_data1 = [record['accum1'][0] for record in records]
accum_data2 = [record['accum2'][0] for record in records]
volts, hz = calculate_volts_and_hz(accum_data1, accum_data2, time_intervals_hours)
header.extend(["Volts", "Hz"])
for j, record in enumerate(records):
time = times[j]
serial = record['serial']
log_hours = record['logHours']
data_row = [time, serial, log_hours, volts[j], hz[j]]
data.append(data_row)
# Processing Inputs 1-14 (Current Transformers)
for i in range(1, 15):
accum_data1 = [record['accum1'][i] for record in records]
accum_data2 = [record['accum2'][i] for record in records]
watt, va = calculate_watt_and_va(accum_data1, accum_data2, time_intervals_hours)
header.extend([f"Watt_{i}", f"VA_{i}"])
for j in range(len(records)):
data[j].extend([watt[j], va[j]])
if export_format == 'csv':
confirm_overwrite(output_file)
with open(output_file, 'w', newline='') as file:
writer = csv.writer(file)
writer.writerow(header)
writer.writerows(data)
print(f"Data exported to {output_file}")
elif export_format == 'json':
confirm_overwrite(output_file)
json_data = [{str(times[i]): {header[j]: data[i][j] for j in range(1, len(header))}} for i in range(len(data))]
with open(output_file, 'w') as file:
json.dump(json_data, file, indent=4)
print(f"Data exported to {output_file}")
if __name__ == "__main__":
if len(sys.argv) < 9:
print("Usage: python script.py --input <binary_file_path> --format <csv|json> --interval <seconds> [--timeformat <unix|normal>] --output <output_file>")
sys.exit(1)
file_path = None
export_format = None
time_format = 'unix' # Default value is UNIX format
interval = None
output_file = None
if '--input' in sys.argv or '-i' in sys.argv:
input_arg = '--input' if '--input' in sys.argv else '-i'
file_path = sys.argv[sys.argv.index(input_arg) + 1]
else:
print("Error: --input or -i argument is required.")
sys.exit(1)
if '--format' in sys.argv or '-f' in sys.argv:
format_arg = '--format' if '--format' in sys.argv else '-f'
export_format = sys.argv[sys.argv.index(format_arg) + 1].lower()
else:
print("Error: --format or -f argument is required.")
sys.exit(1)
if '--interval' in sys.argv or '-I' in sys.argv:
interval_arg = '--interval' if '--interval' in sys.argv else '-I'
interval = int(sys.argv[sys.argv.index(interval_arg) + 1])
else:
print("Error: --interval or -I argument is required.")
sys.exit(1)
if '--timeformat' in sys.argv or '-t' in sys.argv:
timeformat_arg = '--timeformat' if '--timeformat' in sys.argv else '-t'
time_format = sys.argv[sys.argv.index(timeformat_arg) + 1].lower()
if time_format not in ['unix', 'normal']:
print("Error: Invalid --timeformat or -t value. Use 'unix' or 'normal'.")
sys.exit(1)
if '--output' in sys.argv or '-o' in sys.argv:
output_file_arg = '--output' if '--output' in sys.argv else '-o'
output_file = sys.argv[sys.argv.index(output_file_arg) + 1]
else:
print("Error: --output or -o argument is required.")
sys.exit(1)
# Step 1: Read the binary file and get the structured records
records = read_binary_file(file_path)
# Step 2: Process the data from the records and export
process_data(records, export_format, output_file, time_format, interval)