nappMATLAB-toolbox/import/importJeti.m

function dataTable = importJeti(filename)
%IMPORTJETI Import JETI SPECBOS irradiance data (CSV or TXT format)
%  DATATABLE = IMPORTJETI(FILENAME) reads data from text file FILENAME.
%
%  Supports:
%  1. CSV: Semicolon-separated with header metadata
%  2. isd: txt-style format with [sections] and tab-separated data
%
%  Example:
%    table = importJeti("20251212_1606_Jeti.csv");
%    table = importJeti("spectrum_#2.isd");

%% Determine file type and route accordingly
[~, ~, ext] = fileparts(filename);

if strcmpi(ext, '.csv')
    dataTable = importCSV(filename);
elseif strcmpi(ext, '.isd')
    dataTable = importTXT(filename);
else
    error('Unsupported file format: %s. Use .csv or .isd', ext);
end

end

%% ========== CSV IMPORT FUNCTION ==========
function dataTable = importCSV(filename)

fid = fopen(filename, 'r');
if fid == -1
    error('Cannot open file: %s', filename);
end

metadata = struct();
metadata.numDatasets    = NaN;
metadata.numPoints      = NaN;
metadata.Date           = {};
metadata.Time           = {};

lineIdx            = 0;
dataHeaderLineIdx  = NaN;

while true
    thisLine = fgetl(fid);
    if ~ischar(thisLine)
        break
    end
    lineIdx = lineIdx + 1;

    % Detect start of data header line
    if contains(thisLine, 'Wavelength [nm]')
        dataHeaderLineIdx = lineIdx;
        break
    end

    parts = strsplit(thisLine, ';');

    if isempty(parts)
        continue
    end

    key = strtrim(parts{1});

    switch key
        case 'Number of datasets'
            if numel(parts) >= 2
                metadata.numDatasets = str2double(strtrim(parts{2}));
            end
        case 'Number of points'
            if numel(parts) >= 2
                metadata.numPoints = str2double(strtrim(parts{2}));
            end
        case 'Date'
            if numel(parts) >= 2
                metadata.Date = parts(2:end);
            end
        case 'Time'
            if numel(parts) >= 2
                metadata.Time = parts(2:end);
            end
    end
end

if isnan(dataHeaderLineIdx)
    fclose(fid);
    error('Could not find data header line in CSV.');
end

firstDataLineInFile = dataHeaderLineIdx + 1;
fclose(fid);

% Derive number of columns:
% If 'Number of datasets' is present: use it.
% Otherwise infer from the header line (new Spec1/Spec2 format).[file:1]
fid = fopen(filename, 'r');
if fid == -1
    error('Cannot reopen file: %s', filename);
end
headerLine = '';
for k = 1:dataHeaderLineIdx
    headerLine = fgetl(fid);
end
fclose(fid);

headerParts = strsplit(headerLine, ';');
nHeaderCols = numel(headerParts);

if isnan(metadata.numDatasets)
    % New format: first column is wavelength, remaining are datasets
    metadata.numDatasets = nHeaderCols - 1;
end

numCols = 1 + metadata.numDatasets;

% Build dataLines
if isnan(metadata.numPoints)
    dataLines = [firstDataLineInFile, Inf];
else
    lastDataLineInFile = firstDataLineInFile + metadata.numPoints - 1;
    dataLines = [firstDataLineInFile, lastDataLineInFile];
end

% Set up Import Options
opts = delimitedTextImportOptions("NumVariables", numCols);
opts.DataLines = dataLines;
opts.Delimiter = ";";

% Build variable names with units
varNames = cell(1, numCols);
varNames{1} = "Wavelength [nm]";

if ~isempty(metadata.Date) && ~isempty(metadata.Time)
    nMeta = min([numel(metadata.Date), numel(metadata.Time), metadata.numDatasets]);
    for k = 1:nMeta
        dstr = strtrim(metadata.Date{k});
        tstr = strtrim(metadata.Time{k});

        try
            % JETI CSV: e.g. 12/18/2025 and 10:40:14 for 12h clock.[file:1]
            dt = datetime([dstr ' ' tstr], ...
                'InputFormat', 'MM/dd/yyyy HH:mm:ss');
            timestr = datestr(dt, 'yyyymmdd_HHMMss');
        catch
            timestr = sprintf('dataset%02d', k);
        end

        varNames{1 + k} = sprintf("SpectralIrradiance_%s [W/(sqm*nm)]", timestr);
    end

    for k = nMeta+1:metadata.numDatasets
        varNames{1 + k} = sprintf("SpectralIrradiance_%02d [W/(sqm*nm)]", k);
    end
else
    for k = 1:metadata.numDatasets
        varNames{1 + k} = sprintf("SpectralIrradiance_%02d [W/(sqm*nm)]", k);
    end
end

opts.VariableNames = string(varNames);
opts.VariableTypes = repmat("double", 1, numCols);
opts.ExtraColumnsRule = "ignore";
opts.EmptyLineRule   = "read";
opts.VariableNamingRule = "preserve";

dataTable = readtable(filename, opts);

end

%% ========== TXT IMPORT FUNCTION ==========
function dataTable = importTXT(filename)

fid = fopen(filename, 'r');
if fid == -1
    error('Cannot open file: %s', filename);
end

metadata = struct();
metadata.Name           = '';
metadata.NumberOfDataX  = NaN;
metadata.NumberOfDataY  = NaN;
metadata.XMin           = NaN;
metadata.XMax           = NaN;
metadata.XUnit          = '';
metadata.YUnit          = '';

dataStartLineIdx = NaN;
lineIdx = 0;

% Parse header sections
while true
    thisLine = fgetl(fid);
    if ~ischar(thisLine)
        break
    end
    lineIdx = lineIdx + 1;
    
    % Check for data section marker
    if strcmp(strtrim(thisLine), 'Data')
        dataStartLineIdx = lineIdx + 1;
        break
    end
    
    % Parse key=value pairs
    if contains(thisLine, '=')
        parts = strsplit(thisLine, '=');
        if numel(parts) == 2
            key   = strtrim(parts{1});
            value = strtrim(parts{2});
            
            switch key
                case 'Name'
                    metadata.Name = value;
                case 'NumberOfDataX'
                    metadata.NumberOfDataX = str2double(value);
                case 'NumberOfDataY'
                    metadata.NumberOfDataY = str2double(value);
                case 'X-MIN'
                    metadata.XMin = str2double(value);
                case 'X-MAX'
                    metadata.XMax = str2double(value);
                case 'X-Unit'
                    metadata.XUnit = value;
                case 'Y-Unit'
                    metadata.YUnit = value;
            end
        end
    end
end

if isnan(dataStartLineIdx)
    fclose(fid);
    error('Could not find data section in TXT file.');
end

fclose(fid);

% Set up import options for tab-separated data
opts = delimitedTextImportOptions("NumVariables", 2);
opts.DataLines = [dataStartLineIdx, Inf];
opts.Delimiter = '\t';
opts.VariableNamingRule = "preserve";

% Build column names as string array
if ~isempty(metadata.XUnit) && ~isempty(metadata.YUnit)
    varNames = string({
        sprintf("Wavelength [%s]", metadata.XUnit)
        sprintf("SpectralIrradiance_%s [%s]", metadata.Name, metadata.YUnit)
    });
else
    varNames = string({
        sprintf("Wavelength_%s", metadata.Name)
        sprintf("SpectralIrradiance_%s", metadata.Name)
    });
end

opts.VariableNames = varNames;
opts.VariableTypes = ["double", "double"];

dataTable = readtable(filename, opts);

end