Remove Extra Line Breaks and Load

#pragma rtGlobals=1     // Use modern global access method.

//  This is an illustration of loading data from a weird file format.
//
//  In some cases, the simplest approach is to create a cleaned-up version of the file
//  as a temporary file and then to load the data from that.
//
//  In this example, the file has line unwanted line breaks represented by consecutive
//  carriage returns, linefeeds or carriage-return/linefeed pairs.
//
//  We create a temporary file with the extra line breaks removed and load the temporary file.
//  You will need to tweak this for your own situation.
//
//  Please read the code and comments below before using or tweaking for your data file.

Menu "Load Waves"
    "Remove Extra Line Breaks and Load...", RemoveExtraLineBreaksAndLoad("", "")
End

//  FixText(textIn)
//  Returns cleaned up version of text without extra line breaks.
//  Handles carriage returns (CR), linefeeds (LF) and carriage-return/linefeed (CRLF) terminators.
//  To keep it simple, this routine assumes that there is at most one extra line break
//  between each valid line.
//  NOTE: You may need to tweak this for your situation.
static Function/S FixText(textIn)
    String textIn
   
    String textOut = textIn
   
    textOut = ReplaceString("\r\r", textOut, "\r")          // Replace double CRs with one CR

    textOut = ReplaceString("\n\n", textOut, "\n")      // Replace double LFs with one LF

    textOut = ReplaceString("\r\n\r\n", textOut, "\r\n")    // Replace double CRLFs with one CRLF
   
    return textOut
End

//  RemoveExtraLineBreaksAndLoad(pathName, fileName)
//  This routine creates a temporary version of the file without the extra line breaks
//  and loads data from the temporary file.
Function RemoveExtraLineBreaksAndLoad(pathName, fileName)
    String pathName     // Name of an Igor symbolic path or "".
    String fileName         // Name of file or full path to file.

    Variable refNum

    // First get a valid reference to a file.
    if ((strlen(pathName)==0) || (strlen(fileName)==0))
        // Display dialog looking for file.
        // Replace /T="????" with, for example, /T=".dat" if your files are .dat files.
        Open /D /R /P=$pathName /T="????" refNum as fileName
        fileName = S_fileName           // S_fileName is set by Open/D
        if (strlen(fileName) == 0)      // User cancelled?
            return -1
        endif
    endif
   
    // Open source file and read the raw text from it into a string variable
    Open/Z=1/R/P=$pathName refNum as fileName
    if (V_flag != 0)
        return -1                       // Error of some kind
    endif
    FStatus refNum                      // Sets V_logEOF
    Variable numBytesInFile = V_logEOF
    String text = PadString("", numBytesInFile, 0x20)
    FBinRead refNum, text               // Read entire file into variable.
    Close refNum
   
    // Fix the text
    text = FixText(text)                    // Remove extra line breaks
   
    // Write the fixed text to a temporary file
    String tempFileName = fileName + ".noindex" // Use of .noindex prevents Spotlight from indexing the file. Otherwise we get an error when we try to delete the file because Spotlight has it open.
    Open refNum as tempFileName
    FBinWrite refNum, text
    Close refNum   
   
    // Load the temporary file
    // NOTE: You will need to tweak this for your situation
    // If you have string or date/time columns you will need to use LoadWave/J instead of /G.
    LoadWave/G/D/E=1/Q/P=$pathName tempFileName
    if (V_flag == 0)
        Printf "An error occurred while loading data from \"%s\"\r", S_fileName
    else
        Printf "Loaded data from \"%s\"\r", S_fileName
    endif

    // Delete the temporary file   
    DeleteFile /P=$pathName tempFileName
   
    return 0
End