Load JPK Data File

#pragma rtGlobals=3     // Use modern global access method and strict wave access.
 
// Load JPK Data File
// This file loader load a JPK Instruments segmented data file like the one posted at http://www.igorexchange.com/node/7321
// These files contain one or more "segments" of data.
// To use the file loader, choose Data->Load JPK Data File.
// The file loader creates a data folder based on the file name.
// It creates a data folder for each segment in the file. These are named Segment0, Segment1, ...
// The data from each segment is loaded into the corresponding data folder
// in the form of waves. The waves are named according to the "# columns" tag in the segment.
 
Menu "Load Waves"
    "Load JPK Data File...", LoadJPKDataFile("", "")
End
 
Structure JPKDataFileHeaderInfo
    Variable numSegments                // Specified by "# force-settings.segments.size:" tag
    Variable startSegmentsFPos          // File position of first segment, specified by "# segmentIndex: 0" tag
    Variable startSegmentsLineNumber    // Zero-based line number of start of first segment
EndStructure
 
static Function TextStartsWith(text, tagText, tagValueText) // Returns the truth that text starts with tagText
    String text
    String tagText
    String& tagValueText                // The text after the tagText and followign space
    
    tagValueText = ""
    
    Variable len = strlen(tagText)
    String subText = text[0,len-1]
    if (CmpStr(subText,tagText) == 0)
        tagValueText = text[len+1,10000]        // +1 to skip space after tag text
        return 1
    endif
    
    return 0
End
 
// GetJPKDataFileHeaderInfo(refNum, fileInfo)
// Returns information about a JPK file from the file header.
static Function GetJPKDataFileHeaderInfo(refNum, fileInfo)
    Variable refNum         // File reference number from Open command
    STRUCT JPKDataFileHeaderInfo& fileInfo
    
    FSetPos refNum, 0
 
    Variable lineNumber = 0
    do
        Variable fPos
        FStatus refNum
        fPos = V_FilePos            // Current file position for the file in bytes from the start
        
        String text
        FReadLine refNum, text
        if (strlen(text) == 0)
            // Reached end-of-file
            Printf "GetJPKDataFileHeaderInfo hit end of file without reading all required fields"
            return -1
        endif
        
        String tagText, tagValueText
        
        tagText = "# force-settings.segments.size:"
        if (TextStartsWith(text,tagText, tagValueText))
            fileInfo.numSegments = str2num(tagValueText)
        endif
        
        tagText = "# segmentIndex: 0"
        if (TextStartsWith(text,tagText, tagValueText))
            // Hit start of segments
            fileInfo.startSegmentsFPos = fPos
            fileInfo.startSegmentsLineNumber = lineNumber
            break
        endif
        
        lineNumber += 1
    while(1)
    
    return 0
End
 
// MakeJPKCatalog(refNum)
// Returns a 2D wave describing the segments of a file.
//  Column 0: File position of start of segment ("# segmentIndex:" tag)
//  Column 1: Line number of start of segment ("# segmentIndex:" tag)
//  Column 2: Line number of column names line ("# columns:" tag)
//  Column 3: Line number of start of segment data
//  Column 4: Line number of last line of segment data
// All line numbers are zero-based.
static Function/WAVE MakeJPKCatalog(refNum)
    Variable refNum         // File reference number from Open command
    
    FSetPos refNum, 0
 
    // The information about the file is returned in this wave
    Make /O /N=(1000,6) JPKCatalog = NaN            // Assume never more than 1000 segments
    SetDimLabel 1, 0, SegmentStartFPos, JPKCatalog  // Column 0 contains the segment file position
    SetDimLabel 1, 1, SegmentStartLine, JPKCatalog      // Column 1 contains the segment start line number
    SetDimLabel 1, 2, ColumnNamesFPos, JPKCatalog   // Column 2 contains the column names file position
    SetDimLabel 1, 3, ColumnNamesLine, JPKCatalog   // Column 3 contains the column names line number
    SetDimLabel 1, 4, DataStartLine, JPKCatalog         // Column 5 contains the data start line number
    SetDimLabel 1, 5, DataEndLine, JPKCatalog           // Column 6 contains the data end line number
 
    Variable numSegments = 0
    Variable segmentNumber = -1 // No segments yet
    Variable inData = 0
 
    Variable lineNumber = 0
    do
        Variable fPos
        FStatus refNum
        fPos = V_FilePos            // Current file position for the file in bytes from the start
        
        String text
        FReadLine refNum, text
        if (strlen(text) == 0)
            // Reached end-of-file
            if (segmentNumber >= 0)                     // Hit start of segments already?
                JPKCatalog[segmentNumber][%DataEndLine] = lineNumber - 1
            endif
            break
        endif
        
        String tagText, tagValueText
        
        tagText = "# segmentIndex:"
        if (TextStartsWith(text,tagText, tagValueText))
            segmentNumber += 1                          // Start of new segment
            numSegments += 1
            JPKCatalog[segmentNumber][] = NaN
            JPKCatalog[segmentNumber][%SegmentStartFPos] = fPos
            JPKCatalog[segmentNumber][%SegmentStartLine] = lineNumber
        endif
        
        tagText = "# columns:"
        if (TextStartsWith(text,tagText, tagValueText))
            JPKCatalog[segmentNumber][%ColumnNamesFPos] = fPos
            JPKCatalog[segmentNumber][%ColumnNamesLine] = lineNumber
        endif
 
        if (segmentNumber >= 0)             // Hit start of segments already?
            if (!inData)
                if (CmpStr(text[0],"#") != 0)           // A line that does not start with # is a data line
                    JPKCatalog[segmentNumber][%DataStartLine] = lineNumber
                    inData = 1
                endif
            endif
        endif
 
        if (segmentNumber >= 0)             // Hit start of segments already?
            if (inData)
                if (CmpStr(text,"\r") == 0)         // Blank line signifies end of segment data
                    JPKCatalog[segmentNumber][%DataEndLine] = lineNumber - 1
                    inData = 0
                endif
            endif
        endif
        
        lineNumber += 1
    while(1)
    
    Redimension /N=(numSegments,6) JPKCatalog
    
    return JPKCatalog   
End
 
static Function SkipSpaces(text, len, pos)
    String text
    Variable len
    Variable pos
    
    Variable i
    for(i=0; i<len; i+=1)
        if (CmpStr(text[pos]," ") != 0)
            break
        else
            pos += 1
        endif
    endfor
    return pos
End
 
// GetJPKSegmentColumnInfoStr(refNum, columnNamesFPos)
// Returns a string suitable for the LoadWave /B flag. The string specifies the column
// name to use for each column in the segment. The name line in the segment looks
// something like this:
// # columns: tipSampleSeparation vDeflection height error smoothedCapacitiveSensorHeight capacitiveSensorHeight hDeflection seriesTime time xTipPosition yTipPosition
// There may be multiple spaces between names.
static Function/S GetJPKSegmentColumnInfoStr(refNum, columnNamesFPos)
    Variable refNum                 // File reference number from Open command
    Variable columnNamesFPos
    
    FSetPos refNum, columnNamesFPos
    
    String text
    FReadLine refNum, text
    Variable len = strlen(text)
    
    String tagText = "# columns: "
    text = ReplaceString(tagText, text, "")         // Remove tag
    
    String str = ""
    
    Variable pos = 0
    Variable numNames = 0
    
    do
        Variable origPos = pos
        pos = SkipSpaces(text, len, pos)
        String name
        sscanf text[pos,len-1], "%s", name
        
        name = CleanupName(name, 0)
        
        // This converts, e.g., Time into Time0. Time is a built-in function and is not allowed for wave names.
        Variable tmp = Exists(name)
        if (tmp!=0 && tmp!=1)
            name = UniqueName(name, 1, 0)
        endif
 
        String temp = "N='" + name + "';"       // e.g., "N=vDeflection;"
        str += temp
        numNames += 1
        
        pos += strlen(name)
    while(pos < len)
 
    return str
End
 
static Function LoadJPKSegment(pathName, filePath, refNum, segmentNumber, segmentStartFPos, segmentStartLine, columnNamesFPos, dataStartLine, dataNumLines) 
    String pathName             // Name of an Igor symbolic path or ""
    String filePath                 // Name of file or partial path relative to symbolic path or full path to file
    Variable refNum                 // File reference number from Open command
    Variable segmentNumber
    Variable segmentStartFPos
    Variable segmentStartLine
    Variable columnNamesFPos
    Variable dataStartLine
    Variable dataNumLines
    
    // Each segment is loaded into a separate data folder
    String dfName = "Segment" + num2istr(segmentNumber)
    NewDataFolder /O /S $dfName
    
    String columnInfoStr = GetJPKSegmentColumnInfoStr(refNum, columnNamesFPos)
    
    LoadWave /G /O /A=Column /L={0, dataStartLine, dataNumLines, 0, 0} /B=columnInfoStr /P=$pathName /Q filePath
    
    SetDataFolder ::                    // Reset current data folder to original
    
    return 0
End
 
static Function/S FileNameToDFName(filePath)
    String filePath
 
    String fileName = ParseFilePath(3, filePath, ":", 0, 0)
    String dfName = CleanupName(fileName, 0)        // Convert to standard Igor name for easier programming
    return dfName   
End
 
// LoadJPKDataFile(pathName, filePath)
// Loads Ωdata and header information from JPK Instruments segmented .txt files.
// If pathName or filePath is "", an Open File dialog is displayed.
// Creates a main data folder based on the file name.
// Each segment of the file is loaded into a separate data folder in the main data folder.
// For example if your file is named "MyFile" and contains 3 segments and you call this while
// the current data folder is root:, you get the following hierarchy:
//  root:
//      MyFile
//          Segment0
//          Segment1
//          Segment2
// The data and header information for a given segment is loaded into the corresponding
// segment data folder.
// If a data folder already exists, conflicting waves and variables are overwritten.
Function LoadJPKDataFile(pathName, filePath)
    String pathName             // Name of an Igor symbolic path or ""
    String filePath                 // Name of file or partial path relative to symbolic path or full path to file
 
    Variable refNum = 0
 
    // First get a valid reference to a file.
    if ((strlen(pathName)==0) || (strlen(filePath)==0))
        // Display dialog looking for file.
        String fileFilters = "JPK Files (*.txt):.txt;"
        fileFilters += "All Files:.*;"
        Open/D/R/F=fileFilters/P=$pathName refNum as filePath
        filePath = S_fileName                   // S_fileName is set by Open/D
        if (strlen(filePath) == 0)                  // User cancelled?
            return -1
        endif
    endif
 
    // Open the file for reading
    Open /R /P=$pathName refNum as filePath
    if (refNum == 0)
        return -1                               // Error opening file
    endif
 
    STRUCT JPKDataFileHeaderInfo fileInfo
    
    Variable result = GetJPKDataFileHeaderInfo(refNum, fileInfo)
    if (result != 0)
        Close refNum
        return result
    endif
    
    // Print fileInfo           // For debugging only
    
    Wave/Z catalog = MakeJPKCatalog(refNum)
    if (!WaveExists(catalog))
        Print "Error in MakeJPKCatalog"
        Close refNum
        return -1
    endif
    
    // Make a data folder for the file
    String dfName = FileNameToDFName(filePath)
    NewDataFolder /O /S $dfName
    
    Variable numSegments = DimSize(catalog,0)
    Variable segmentNumber
    Variable segmentsLoaded = 0
    for(segmentNumber = 0; segmentNumber<numSegments; segmentNumber+=1)
        Variable segmentStartFPos = catalog[segmentNumber][%SegmentStartFPos]
        Variable segmentStartLine = catalog[segmentNumber][%SegmentStartLine]
        Variable columnNamesFPos = catalog[segmentNumber][%ColumnNamesFPos]
        Variable columnNamesLine = catalog[segmentNumber][%ColumnNamesLine]
        Variable dataStartLine = catalog[segmentNumber][%DataStartLine]
        Variable dataEndLine = catalog[segmentNumber][%DataEndLine]
        Variable dataNumLines = dataEndLine - dataStartLine + 1
        
        result = LoadJPKSegment(pathName, filePath, refNum, segmentNumber, segmentStartFPos, segmentStartLine, columnNamesFPos, dataStartLine, dataNumLines) 
        if (result != 0)
            Printf "Error loading segment %d\r", segmentNumber
            break
        endif
        
        segmentsLoaded += 1
    endfor
    
    SetDataFolder ::                // Reset current data folder to original
 
    Close refNum
    
    Printf "Created data folder %s containing %d segments from \"%s\"\r", dfName, segmentsLoaded, filePath
    
    return 0
End