Load and Concatenate All Files in a Folder

#pragma rtGlobals=3     // Use modern global access method and strict wave access.

// This example shows how to load all of the files in a folder and concatenate the data from each file.
// You will need to customize this for your application.

// In this example, the file contains four columns:
//  Date    Time    CH4_dry CO2_dry
// The date and time column are combined into one date/time wave.
// After all files are loaded, the data is sorted by date/time.

// To customize this, you will need to understand Igor's data loading. For background information, execute:
//  DisplayHelpTopic "Loading Waves"
   
// For background information on loading data programmatically, execute:
//  DisplayHelpTopic "Loading Waves Using Igor Procedures"

Menu "Load Waves"
    "Load One File...", LoadOneFile("", "")
    "Load And Concatenate All Files in Folder...", LoadAndConcatenateAllFiles("")
End

static StrConstant kFileNameExtension = ".dat"

// LoadOneFile(pathName, fileName)
// Produces the following waves: DateTimeW, CH4_dry, CO2_dry
Function LoadOneFile(pathName, fileName)
    String pathName     // Name of an Igor symbolic path or "".
    String fileName         // Name of file or full path to file.
 
    // First get a valid reference to a file.
    if ((strlen(pathName)==0) || (strlen(fileName)==0))
        // Display dialog looking for file.
        Variable refNum
        Open/D/R/F=kFileNameExtension/P=$pathName refNum as fileName
        fileName = S_fileName           // S_fileName is set by Open/D
        if (strlen(fileName) == 0)      // User cancelled?
            return -1
        endif
    endif
 
    String columnInfoStr = " "
    columnInfoStr += "N='DateW,F=6;"            // Load DATE column - will become date/time wave
    columnInfoStr += "N='TimeW',F=7;"           // Load TIME column
    columnInfoStr += "N='CH4_dry';"         // Load CH4_dry
    columnInfoStr += "N='CO2_dry';"         // Load CO2_dry
    LoadWave /J /D /K=0 /A /B=columnInfoStr /Q /R={English, 1, 2, 1, 1, "Year-Month-DayofMonth", 40} /P=$pathName fileName
    Variable numWavesLoaded = V_flag            // V_flag is set by LoadWave
    if (numWavesLoaded != 4)
        Print "Error loading file - wrong number of waves loaded"
        return -1
    endif
 
    Wave DateTimeW,TimeW            // Create reference to waves created by LoadWave
    DateTimeW += TimeW              // Add time to date to obtain date/time
    KillWaves/Z TimeW                   // This is no longer needed

    return 0                            // Success
End

// LoadAndConcatenateAllFiles(pathName)
// Loads all files in specified folder with extension specified by kFileNameExtension.
// The output waves are: DateTimeW, CH4_dry, CO2_dry
// All loaded waves are concatenated, creating the output waves in the current data folder.
// If the output waves already exist in the current data folder, this routine appends to them.
Function LoadAndConcatenateAllFiles(pathName)
    String pathName                 // Name of symbolic path or "" to get dialog
    String fileName
    Variable index=0
   
    Wave/D/Z DateTimeW, CH4_dry, CO2_dry
    if (!WaveExists(DateTimeW))                     // Date/time wave does not exist?
        // Create the output waves because the code below concatenates 
        Make/O/N=0/D DateTimeW, CH4_dry, CO2_dry
        SetScale d, 0, 0, "dat", DateTimeW          // Mark as a date/time wave
    endif

    if (strlen(pathName)==0)            // If no path specified, create one
        NewPath/O temporaryPath     // This will put up a dialog
        if (V_flag != 0)
            return -1                   // User cancelled
        endif
        pathName = "temporaryPath"
    endif

    Variable result
    do          // Loop through each file in folder
        fileName = IndexedFile($pathName, index, kFileNameExtension)
        if (strlen(fileName) == 0)          // No more files?
            break                                   // Break out of loop
        endif

        // Load the new data into a temporary data folder
        String dfName = "TempDataForLoadAndConcatenate"
        NewDataFolder/O/S $dfName

        result = LoadOneFile(pathName, fileName)
        if (result != 0)
            String message
            sprintf message, "An error occurred while loading the file \"%s\". Aborting the load.\r", fileName
            Print message
            DoAlert 0, message
            KillDataFolder $dfName
            break      
        endif
       
        // Create wave references for the waves loaded into the temporary data folder
        Wave DateTimeWNew = :DateTimeW
        Wave CH4_dryNew = :CH4_dry
        Wave CO2_dryNew = :CO2_dry
       
        SetDataFolder ::                // Back to parent data folder

        Wave DateTimeW, CH4_dry, CO2_dry
       
        Concatenate /NP {DateTimeWNew}, DateTimeW
        Concatenate /NP {CH4_dryNew}, CH4_dry
        Concatenate /NP {CO2_dryNew}, CO2_dry

        KillDataFolder $dfName
       
        Printf "Loaded file %d: \"%s\"\r", index, fileName

        index += 1
    while (1)

    if (Exists("temporaryPath"))        // Kill temp path if it exists
        KillPath temporaryPath
    endif
   
    Wave DateTimeW, CH4_dry, CO2_dry
    Sort DateTimeW, CO2_dry, CH4_dry,DateTimeW

    return 0                        // Signifies success.
End