Load EC-Lab Data File

#pragma TextEncoding = "UTF-8"
#pragma rtGlobals=3     // Use modern global access method and strict wave access.

// About "Load EC-Lab Data File" procedures

// To use this procedure, execute a command such as:
//      LoadECLabDataFile(pathName, filePath)
// where
//      String pathName                 // Name of an Igor symbolic path or ""
//      String filePath                 // Name of file or full path to file

// If you are not familiar with Igor symbolic paths, execute:
//      DisplayHelpTopic "Symbolic Paths"

// LoadECLabDataFile automatically determines the name line, first data line, and delimiter character.
// If it gets those parameters wrong, you can use LoadECLabDataFile2 instead. With LoadECLabDataFile2,
// you specify those parameters in addition to the pathName and filePath.

// These procedures solve issues raised by the column names in EC-Lab data files.

// There are two categories of wave names in Igor: standard and liberal. Standard names start
// with a letter and can include letters, numbers, and the underscore character. Liberal names
// can include any character except for single-quote, double-quote, colon, and semicolon.

// For further discussion of standard and liberal names, execute:
//      DisplayHelpTopic "Object Names"

// Programming with liberal names is tricky and most Igor procedures do not work with
// liberal names. Consequently, LoadWaves, by default, "cleans up" column names to produce
// standard wave names. Using LoadWave/B, you can load waves with liberal names but this
// is usually a bad idea because liberal names cause issues down the road unless you are
// very careful.

// The EC-Lab data file from Bio-Logic uses column names that contain characters that
// are not legal in Igor standard names. Examples are "<Ewe>/V" and "|Ewe|/V". Furthermore,
// when LoadWaves cleans these up, they both evaluate to X_Ewe__V, causing a name conflict.
// And, there are some EC-Lab data files that contain columns with the exact same name
// more than once - see https://www.wavemetrics.com/node/21003.

// The LoadECLabDataFile procedure handles these issues by removing characters that
// are not legal in standard Igor names and then by making sure that each name is unique.
// As a result, you wind up with waves named, for example, Ewe_V and Ewe_V1.
//
// Here is a more extensive, though not complete, list of wave names produced by LoadECLabDataFile:
//      freq_Hz, ReZ_Ohm, ImZ_Ohm, Z_Ohm, PhaseZ_deg, time_s, Ewe_V, I_mA
//      Cs_uF, Cp_uF, cyclenumber, IRange, Ewe_V1, I_A, Q_Qo_mA_h, ReY_Ohm_1
//      ImY_Ohm_1, Y_Ohm_1, PhaseY_deg, I_mA1, dq_mA_h, x1

static Function/S MakeNameUnique(name, nameList)
    String name             // Name that we are about to add to /B flag
    String nameList         // List of names already added to /B flag
   
    String originalName = name
    Variable index = 0
    do
        if (WhichListItem(name, nameList) < 0)
            break               // name is not in the list
        endif
        index += 1
        sprintf name, "%s%d", originalName, index       // Generate new name by appending a number 
    while(1)
   
    return name
End

// RemoveIllegalCharacters(name)
// Removes characters that we know are not legal in Igor standard names.
// Here are typical resulting wave names:
//      freq_Hz, ReZ_Ohm, ImZ_Ohm, Z_Ohm, PhaseZ_deg, time_s, Ewe_V, I_mA
//      Cs_uF, Cp_uF, cyclenumber, IRange, Ewe_V1, I_A, Q_Qo_mA_h, ReY_Ohm_1
//      ImY_Ohm_1, Y_Ohm_1, PhaseY_deg, I_mA1, dq_mA_h, x1
static Function/S RemoveIllegalCharacters(name)
    String name
   
    name = ReplaceString("|", name, "")
    name = ReplaceString("<", name, "")
    name = ReplaceString(">", name, "")
    name = ReplaceString("(", name, "")
    name = ReplaceString(")", name, "")
    name = ReplaceString("/", name, "_")
    name = ReplaceString(" ", name, "")
   
    return name
End

Function/S GenerateColumnInfoStr(pathName, filePath, nameLine, delimiter)
    String pathName     // Name of an Igor symbolic path or ""
    String filePath     // Name of file or full path to file
    Variable nameLine   // 0-based number of line containing column names
    String delimiter    // Typically "," or "\t" (tab)
   
    Variable refNum
    Open/R/P=$pathName refNum as filePath
   
    Variable lineNumber = 0
    do
        String text
        FReadLine refNum, text
        if (lineNumber == nameLine)
            break
        endif
        lineNumber += 1
    while(1)
   
    text = ReplaceString("\r", text, "")        // Remove any CR terminator character
    text = ReplaceString("\n", text, "")        // Remove any LF terminator character
   
    Close refNum
   
    String nameList = ""    // Accumulates names
   
    String columnInfoStr = ""
    Variable numNames = ItemsInList(text, delimiter)
    Variable index
    for(index=0; index<numNames; index+=1)
        String name = StringFromList(index, text, delimiter)
        if (strlen(name) == 0)
            name = "_skip_"    
        endif
       
        // Remove characters that we know are not legal in Igor standard names
        name = RemoveIllegalCharacters(name)
       
        // Make sure this is a legal standard wave name
        name = CleanupName(name, 0)
       
        // Make the name unique if it conflicts with a name already added to nameList
        name = MakeNameUnique(name, nameList)
       
        columnInfoStr += "N='" + name + "';"
        nameList += name + ";"
    endfor
   
    return columnInfoStr
End

// GetFileToLoad(pathName, filePath)
// If necessary, displays an Open File dialog to allow user to choose the file.
// Returns -1 if cancelled, 0 if OK.
// If the function result is 0 then, on return, pathName and filePath are suitable
// for use by LoadWave.
static Function GetFileToLoad(pathName, filePath)
    String& pathName                // Input and output: Name of an Igor symbolic path or ""
    String& filePath                // Input and output: Name of file or full path to file

    if ((strlen(pathName)==0) || (strlen(filePath)==0))
        // Display dialog looking for file.
        String fileFilters = "Text Files (*.txt):.txt;"
        fileFilters += "CSV Files (*.csv):.csv;"
        fileFilters += "DAT Files (*.dat):.dat;"
        fileFilters += "All Files:.*;"
        Variable refNum
        Open/D/R/F=fileFilters/P=$pathName refNum as filePath
        filePath = S_fileName               // S_fileName is full path set by Open/D
        if (strlen(filePath) == 0)      // User cancelled?
            return -1
        endif
    endif
   
    return 0
End

Function LoadECLabDataFile2(pathName, filePath, nameLine, firstDataLine, delimiter)
    String pathName                 // Name of an Igor symbolic path or ""
    String filePath                 // Name of file or full path to file
    Variable nameLine               // Zero-based line number of names line
    Variable firstDataLine      // Zero-based line number of first data line
    String delimiter                // Delimiter between names and numbers, typically "," or "\t" (tab)

    // First get a valid reference to a file
    Variable result = GetFileToLoad(pathName, filePath)
    if (result != 0)
        return result               // User cancelled Open File dialog
    endif

    String columnInfoStr = GenerateColumnInfoStr(pathName, filePath, nameLine, delimiter)
    // Print columnInfoStr
   
    LoadWave/J/Q/L={nameLine,firstDataLine,0,0,0}/A/B=columnInfoStr/E=1/P=$pathName filePath
   
    return 0
End

static Function FindFirstDataLineAndDelimiter(pathName, filePath, firstDataLine, delimiter)
    String pathName                 // Name of an Igor symbolic path or ""
    String filePath                 // Name of file or full path to file
    Variable& firstDataLine     // Output
    String& delimiter               // Output
   
    Variable refNum = 0
    Open/R/P=$pathName refNum as filePath
    if (refNum == 0)
        return -1                       // Unexpected error
    endif
   
    Variable lineNum = 0
    do
        String text
        FReadLine refNum, text
        if (strlen(text) == 0)
            Close refNum
            return -1                   // Did not find first data line - Should not happen if this is a valid EC-Lab file
        endif
       
        Variable num
        sscanf text, "%g", num
        if (V_Flag == 1)
            // The line starts with a number
            firstDataLine = lineNum
            String junk
            sscanf text, "%[^\t ,]", junk       // Deposit all characters other than tab, space, or comma into junk
            int len = strlen(junk)
            delimiter = text[len]
            break
        endif      
       
        lineNum += 1
    while(1)

    Close refNum
    return 0
End

// LoadECLabDataFile(pathName, filePath)
// Automatically determines the name line, first data line, and delimiter,
// and then calls LoadECLabDataFile2.
// It is assumed that the first line that starts with a valid number is the first
// data line and that the line before the first data line is the name line.
Function LoadECLabDataFile(pathName, filePath)
    String pathName                 // Name of an Igor symbolic path or ""
    String filePath                 // Name of file or full path to file

    // First get a valid reference to a file
    Variable result = GetFileToLoad(pathName, filePath)
    if (result != 0)
        return result               // User cancelled Open File dialog
    endif
   
    Variable nameLine = 0           // Zero-based line number of names line
    Variable firstDataLine = 0  // Zero-based line number of first data line
    String delimiter = ""           // Delimiter between names and numbers, typically "," or "\t" (tab)
    result = FindFirstDataLineAndDelimiter(pathName, filePath, firstDataLine, delimiter)
    if (result != 0)
        return result               // Should not happen if this is a valid EC-Lab file
    endif
    nameLine = firstDataLine - 1
   
    // For debugging only
    // Printf "Name line = %d, first data line = %d, delimiter = %d\r", nameLine, firstDataLine, char2num(delimiter)
   
    String columnInfoStr = GenerateColumnInfoStr(pathName, filePath, nameLine, delimiter)
    // Print columnInfoStr
   
    LoadWave/J/Q/L={nameLine,firstDataLine,0,0,0}/A/B=columnInfoStr/E=1/P=$pathName filePath
   
    return 0
End