/* ----------------------------------------
 * Origin  : GFI INFORMATIQUE 
 * Project : CDPP Propagation Tool 
 * ----------------------------------------
 * History 
 * Version : 01/05/2015 : SCA : V2.1
 * Version : 13/09/2016 : SCA : V2.2 (PSWS-H2020)
 *  Swap STEREO-A / STEREO-B FOV after 14/07/2015
 *  New Radial catalog DAO access
 * Version : 06/11/2017 : SCA : V2.2.2 
 *  Redmine #5403 - DMPT3 
 * Version : 19/07/2022 : LBE : V2.5
 * Redmine 7278: Build gaps in VPlot from sampling
 * Version : 30/11/2022 : LBE : V2.6
 * DM-ID : Redmine #l0481 : 30/11/2022 : HCS added
 * Version : 06/04/2023 : LBE : V2.7
 * DM-ID : redmine#8642 : 06/04/2023 : Added 3D shockwave
 * Version : 12/09/2023 : LBE : V2.8
 * DM-ID : Redmine#11197 : 12/09/2023 : 3DShocks improvments
 * VERSION : 04/06/2024 : V2.10
 * Redmine#11605 : 04/06/2024 : second swap date added for stereo
 * Version : 05/08/2024 : LBE : V2.11
 * DM-ID : Redmine#l0763 : 05/08/2024 : PFSS cube reading improvment (Scanner=>split)
 * ----------------------------------------
 */
package fr.tlse.irap.server.serviceDAO.impl;

import fr.tlse.irap.fieldLine.Cube;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.List;
import java.util.Locale;
import java.util.Scanner;
import java.util.TimeZone;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;

import fr.tlse.irap.fieldLine.Footpoint;
import fr.tlse.irap.fieldLine.Footpoints;
import fr.tlse.irap.fit.Fit;
import fr.tlse.irap.flare.Flare;
import fr.tlse.irap.flare.Flare.FlareType;
import fr.tlse.irap.flare.FlareFlux;
import fr.tlse.irap.map.CarringtonMapType;
import fr.tlse.irap.map.HCSLine;
import fr.tlse.irap.server.ApplicationContextHolder;
import fr.tlse.irap.server.frontServices.FieldLineImpl;
import fr.tlse.irap.server.serviceDAO.IAsciiDAO;
import fr.tlse.irap.server.serviceDAO.constant.CONFIG;
import fr.tlse.irap.server.serviceDAO.utils.FitsHelper;
import fr.tlse.irap.server.serviceDAO.utils.ScanDirectory;
import fr.tlse.irap.shock.Shock;
import fr.tlse.irap.shock.Shockwave3d;
import fr.tlse.irap.shock.Shockwave3dEvent;
import fr.tlse.irap.shock.Shockwave3dEvents;
import fr.tlse.irap.spaceElement.Probe;
import fr.tlse.irap.spectro.Spectro;
import fr.tlse.irap.tool.corotation.CirCorotation;
import fr.tlse.irap.tool.corotation.CirEnd;
import fr.tlse.irap.tool.radial.CMEType;
import fr.tlse.irap.tool.radial.CmeEnd;
import fr.tlse.irap.tool.radial.CmeRadial;
import fr.tlse.irap.tool.sep.Sep.Particle;
import fr.tlse.irap.utils.CoordConverter;
import fr.tlse.irap.utils.ToolHelper;
import fr.tlse.irap.vplot.DateRange;
import fr.tlse.irap.vplot.SolarSpeed;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import javax.vecmath.Matrix3d;
import javax.vecmath.Vector3d;
import javax.vecmath.Vector3f;
import org.apache.commons.io.FileUtils;

public class AsciiDAOImpl implements IAsciiDAO {

    private static final Logger logger = Logger.getLogger(AsciiDAOImpl.class);
    private static CONFIG prop;
    private static final String ISO_TIME_PATTERN = "\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.*";
    private static final String ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss";

    public AsciiDAOImpl() {
        logger.info("ASCII DAO IMPL LOADED");
        // set default timezone UTC
        TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
        prop = (CONFIG) ApplicationContextHolder.getContext().getBean("confData");
    }

    public AsciiDAOImpl(CONFIG lProp) {
        // set default timezone UTC
        TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
        prop = lProp;
    }

    /**
     * Load list of V solar wind from catalog for a S/C or planet between 2 dates
     *
     * @return sampling in seconds. -1 if not found
     */
    @Override
    public int loadSolarSpeedsSampling(String startName) {
        logger.info("SPEED PATH=" + prop.getDATA_PATH() + "/" + prop.getSPEED_DIR());
        logger.debug("loadSolarSpeeds sampling with startName=" + startName);

        File path = new File(prop.getDATA_PATH(), prop.getSPEED_DIR());
        //read sampling
        File lSamplingFile = new File(path, "sampling.txt");
        int lSampling = -1;
        Scanner lScanner = null;
        try {
            lScanner = new Scanner(lSamplingFile);

            while (lScanner.hasNext()) {
                String lCurrLine = lScanner.nextLine();
                logger.debug("Reading line " + lCurrLine);
                String[] lTokens = lCurrLine.split("\\s+");
                int i = 0;
                if (lTokens.length == 2 && lTokens[0].startsWith(startName)) {
                    lSampling = Integer.valueOf(lTokens[1]);
                    logger.info("Found sampling for SpeedInsitu " + lSampling);
                    break;
                }
            }
        } catch (FileNotFoundException lE) {
            lE.printStackTrace(System.err);
            logger.error("Sampling file not found at " + lSamplingFile.getAbsolutePath());
        } finally {
            if (lScanner != null) {
                lScanner.close();
            }
        }
        return lSampling;
    }

    /**
     * Load list of V solar wind from catalog for a S/C or planet between 2 dates
     */
    @Override
    public List<SolarSpeed> loadSolarSpeeds(Date t1, Date t2, String startName) {
        logger.info("SPEED PATH=" + prop.getDATA_PATH() + "/" + prop.getSPEED_DIR());
        logger.debug("loadSolarSpeeds with t1=" + t1 + " t2=" + t2 + " startName=" + startName);

        List<SolarSpeed> readSpeeds = new ArrayList<SolarSpeed>();
        File path = new File(prop.getDATA_PATH(), prop.getSPEED_DIR());
        if (path != null) {
            path = new File(path.getPath(), startName);
        }

        //read sampling
//        long lGapMillisec = 5*60*1000l;
        File[] files = ScanDirectory.fetchFiles(path, t1, t2);
        if (files != null) {
            for (File file : files) {
                try {
                    List<SolarSpeed> currentReadSpeeds = readSolarSpeeds(t1, t2, file);
//                    long lastTime = -1;
                    for (SolarSpeed solarSpeed : currentReadSpeeds) {
//                        if(lastTime!=-1 && (solarSpeed.getEventTime().getTimeInMillis()-lastTime)>lGapMillisec) {
//                            SolarSpeed lNaNSp = new SolarSpeed();
//                            lNaNSp.setSpeed(Float.NaN);
//                            Calendar lNaNCal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
//                            lNaNCal.setTimeInMillis(solarSpeed.getEventTime().getTimeInMillis()-1);
//                            lNaNSp.setEventTime(lNaNCal);
//                            //readSpeeds.add(lNaNSp);
//                            System.out.println("Adding NaN at "+lNaNSp.getEventTime().getTime());
//                        }
//                        System.out.println("Adding "+solarSpeed.getSpeed()+" at "+solarSpeed.getEventTime().getTime());
                        readSpeeds.add(solarSpeed);
//                        lastTime = solarSpeed.getEventTime().getTimeInMillis();
                    }
                } catch (Exception e) {
                    logger.error("Reading failure for file " + file.getName(), e);
                }
            }
        }
        return readSpeeds;
    }

    /**
     * Load list of spectrogram fluxes of a particle from catalog for a S/C between 2 dates
     *
     * @param t1 range min to search
     * @param t2 range max to search
     * @param startName file names start PSP_HET for exemple
     * @return a list with List<Float> and List<Spectro>
     */
    @Override
    public List<Object> loadSEPFlux(Date t1, Date t2, String startName, Particle particle) {
        logger.info("loadSEPFlux SEP spectro PATH=" + prop.getDATA_PATH() + "/" + prop.getSEP_DIR());
        logger.debug("loadSEPFlux load sep fluxes with t1=" + t1 + " t2=" + t2 + " startName=" + startName);

        List<Spectro> readValues = new ArrayList<Spectro>();
        List<Float[]> energyList = new ArrayList<Float[]>();
        List<Object> retValues = new ArrayList<>();
        File path = new File(prop.getDATA_PATH(), prop.getSEP_DIR());
        path = new File(path.getPath(), startName);
        String lParticleStr = "protons";
        switch (particle) {
            case electron:
                lParticleStr = "electrons";
                break;
            case proton:
            default:
                lParticleStr = "protons";
                break;
        }

        //get energies
        File[] lNrjFile = ScanDirectory.fetchMatchingFiles(path, lParticleStr, ".ascii");
        //logger.debug("loadSEPFlux Found "+lNrjFile.length+" energy files");
        if (lNrjFile != null && lNrjFile.length > 0) {
            energyList = readSEPETable(lNrjFile[0]);
        }
        //get spectro time data
        File[] files = ScanDirectory.fetchFiles(path, t1, t2);
        if (files != null) {
            logger.debug("loadSEPFlux Found " + files.length + " data files ");
            for (File file : files) {
                try {
                    if (file.getName().contains(lParticleStr)) {
                        List<Spectro> lCurrFluxes = readSEPFluxes(t1, t2, file);
                        for (Spectro lFlux : lCurrFluxes) {
                            readValues.add(lFlux);
                        }
                    }
                } catch (Exception e) {
                    logger.error("Reading failure for file " + file.getName(), e);
                }
            }
        }
        retValues.add(energyList);
        retValues.add(readValues);
        return retValues;
    }

    /**
     * Load list of Flares from catalog between 2 dates
     */
    @Override
    public List<Flare> loadFlares(Date t1, Date t2) {
        logger.info("FLARE PATH=" + prop.getDATA_PATH() + "/" + prop.getFLARE_DIR());

        List<Flare> readedFlares = new ArrayList<Flare>();
        File path = new File(prop.getDATA_PATH(), prop.getFLARE_DIR());

        File[] files = ScanDirectory.fetchFiles(path, t1, t2);
        if (files != null) {
            for (File file : files) {
                try {
                    List<Flare> currentReadedFlares = readFlares(t1, t2, file);
                    for (Flare flare : currentReadedFlares) {
                        readedFlares.add(flare);
                    }
                } catch (Exception e) {
                    logger.error("Reading failure for file " + file.getName(), e);
                }
            }
        }
        logger.debug("Number of selected flares=" + readedFlares.size());

        return readedFlares;
    }

    /**
     * Load list of Shocks from catalog between 2 dates
     */
    @Override
    public List<Shock> loadShocks(Date t1, Date t2) {
        logger.info("SHOCK PATH=" + prop.getDATA_PATH() + "/" + prop.getSHOCK_DIR());

        List<Shock> readedShocks = new ArrayList<Shock>();
        File path = new File(prop.getDATA_PATH(), prop.getSHOCK_DIR());

        File[] files = ScanDirectory.fetchFiles(path, t1, t2);
        if (files != null) {
            for (File file : files) {
                try {
                    List<Shock> currentReadedShocks = readShocks(t1, t2, file);
                    for (Shock shock : currentReadedShocks) {
                        readedShocks.add(shock);
                    }
                } catch (Exception e) {
                    logger.error("Reading failure for file " + file.getName(), e);
                }
            }
        }
        logger.debug("Number of selected shocks=" + readedShocks.size());

        return readedShocks;
    }

    /**
     * Load list of Shocks from catalog between 2 dates
     */
    @Override
    public Shockwave3dEvent loadShockwave3d(Date t1) {
        
        Shockwave3dEvent readEvent = new Shockwave3dEvent();
        File path = new File(prop.getDATA_PATH(), prop.getSHOCKWAVES3D_DIR());
        logger.info("Fetch shockwave in path " +path.getAbsolutePath()+" at "+t1);
        SimpleDateFormat lSdf = new SimpleDateFormat("yyyyMMdd'T'HHmm");
        lSdf.setTimeZone(TimeZone.getTimeZone("UTC"));
        File lDir = new File(path, lSdf.format(t1));
        try {
            Shockwave3dEvent lEvent = new Shockwave3dEvent();
            List<Shockwave3d> currentReadedShocks = readShockwaves(lDir);
            lEvent.setShockwaves(currentReadedShocks);
            lEvent.setTime(lSdf.parse(lDir.getName()));
            lEvent.computeMinMax();
            readEvent = lEvent;

        } catch (Exception e) {
            logger.error("Reading failure for file " + lDir.getName(), e);
        }
                
        logger.debug("Number of selected shocks = " + readEvent.getShockwaves().size());

        return readEvent;
    }

    /**
     * Load list of Shocks from catalog between 2 dates
     */
    @Override
    public Shockwave3dEvents loadShockwaves3dList() {
        
        Shockwave3dEvents readEvents = new Shockwave3dEvents();
        File path = new File(prop.getDATA_PATH(), prop.getSHOCKWAVES3D_DIR());
        logger.info("Fetch shockwaves in path " +path.getAbsolutePath());
        SimpleDateFormat lSdf = new SimpleDateFormat("yyyyMMdd'T'HHmm");
        lSdf.setTimeZone(TimeZone.getTimeZone("UTC"));
        File[] dirs = path.listFiles();
        if (dirs != null) {
            for (File lDir : dirs) {
                try {
//                    Shockwave3dEvent lEvent = new Shockwave3dEvent();
//                    List<Shockwave3d> currentReadedShocks = readShockwaves(lDir);
//                    lEvent.setShockwaves(currentReadedShocks);
//                    lEvent.setTime(lSdf.parse(lDir.getName()));
//                    lEvent.computeMinMax();
                    
                    readEvents.addEvent(lSdf.parse(lDir.getName()));
                    
                } catch (Exception e) {
                    logger.error("Reading failure for dir " + lDir.getName(), e);
                }
                
            }
        }
        Collections.sort(readEvents.getEvents(),Collections.reverseOrder());
        logger.debug("Number of shocks read = " + readEvents.getEvents().size());

        return readEvents;
    }
    
    
    
    /**
     * Load list of Flares from LINKCAT catalog between 2 dates
     */
    @Override
    public List<Flare> loadLinkcatFlares(Date t1, Date t2, CMEType cmeType, String observatory) {
        logger.info("LINKCAT CME PATH=" + prop.getDATA_PATH() + "/" + prop.getCMEFIT_DIR());
        logger.info("loadLinkcatFlares with t1=" + t1 + " t2=" + t2 + " obs=" + observatory + " cmeType=" + cmeType);

        List<Flare> readedFlares = new ArrayList<Flare>();
        File path = new File(prop.getDATA_PATH(), prop.getCMEFIT_DIR());
        if (path != null) {
            path = new File(path.getPath(), observatory);
            if (path != null) {
                path = new File(path.getPath(), cmeType.name() + "_LINKCAT");
            }
        }

        File[] files = ScanDirectory.fetchFiles(path, t1, t2);
        if (files != null) {
            for (File file : files) {
                try {
                    List<Flare> currentReadedFlares = readLinkcatFlares(t1, t2, file);
                    for (Flare Flare : currentReadedFlares) {
                        readedFlares.add(Flare);
                    }
                } catch (Exception e) {
                    logger.error("Reading failure for file " + file.getName(), e);
                }
            }
        }
        logger.debug("Number of selected flares=" + readedFlares.size());

        return readedFlares;
    }

    /**
     * Load list of CME fits from catalog for a type of CME (FIXED or SSE) and an observatory between 2 dates
     */
    @Override
    public List<Fit> loadFitsCME(Date t1, Date t2, CMEType cmeType, String sourceName) {

        logger.info("CME Fits PATH=" + prop.getDATA_PATH() + "/" + prop.getCMEFIT_DIR());
        logger.debug("loadFits with t1=" + t1 + " t2=" + t2 + " startName=" + sourceName + " cmeType=" + cmeType);

        List<Fit> readedFits = new ArrayList<Fit>();
        File path = new File(prop.getDATA_PATH(), prop.getCMEFIT_DIR());
        if (path != null) {
            path = new File(path.getPath(), sourceName);
            if (path != null) {
                path = new File(path.getPath(), cmeType.name());
            }
        }

        File[] files = ScanDirectory.fetchFiles(path, t1, t2);
        if (files != null) {
            for (File file : files) {
                try {
                    List<Fit> currentFits = readCMEFits(t1, t2, sourceName, file);
                    for (Fit fit : currentFits) {
                        readedFits.add(fit);
                    }
                } catch (Exception e) {
                    logger.error("Reading failure for file " + file.getName(), e);
                }
            }
        }
        logger.debug("Number of selected fits=" + readedFits.size());

        return readedFits;
    }

    /**
     * Load list of CME fits from LINKCAT catalog for a type of CME (FIXED or SSE) and an observatory between 2 dates
     */
    @Override
    public List<Fit> loadLinkcatFitsCME(Date t1, Date t2, CMEType cmeType, String sourceName) {

        logger.info("CME Fits PATH=" + prop.getDATA_PATH() + "/" + prop.getCMEFIT_DIR());
        logger.debug("loadLinkcatFits with t1=" + t1 + " t2=" + t2 + " startName=" + sourceName + " cmeType=" + cmeType);

        List<Fit> readedFits = new ArrayList<Fit>();
        File path = new File(prop.getDATA_PATH(), prop.getCMEFIT_DIR());
        if (path != null) {
            path = new File(path.getPath(), sourceName);
            if (path != null) {
                path = new File(path.getPath(), cmeType.name() + "_LINKCAT");
            }
        }

        File[] files = ScanDirectory.fetchFiles(path, t1, t2);
        if (files != null) {
            for (File file : files) {
                try {
                    List<Fit> currentFits = readLinkcatCMEFits(t1, t2, sourceName, file);
                    for (Fit fit : currentFits) {
                        readedFits.add(fit);
                    }
                } catch (Exception e) {
                    logger.error("Reading failure for file " + file.getName(), e);
                }
            }
        }
        logger.debug("Number of selected fits=" + readedFits.size());

        return readedFits;

    }

    /**
     * Load list of CIR fits from catalog for an observatory between 2 dates
     */
    @Override
    public List<Fit> loadFitsCIR(Date t1, Date t2, String sourceName) {

        logger.info("CIR Fits PATH=" + prop.getDATA_PATH() + "/" + prop.getCIRFIT_DIR());
        logger.debug("loadFits with t1=" + t1 + " t2=" + t2 + " startName=" + sourceName);

        List<Fit> readedFits = new ArrayList<Fit>();
        File path = new File(prop.getDATA_PATH(), prop.getCIRFIT_DIR());
        if (path != null) {
            path = new File(path.getPath(), sourceName);
        }

        File[] files = ScanDirectory.fetchFiles(path, t1, t2);
        if (files != null) {
            for (File file : files) {
                try {
                    List<Fit> currentFits = readCIRFits(t1, t2, file);
                    for (Fit fit : currentFits) {
                        readedFits.add(fit);
                    }
                } catch (Exception e) {
                    logger.error("Reading failure for file " + file.getName(), e);
                }
            }
        }
        logger.debug("Number of selected fits=" + readedFits.size());

        return readedFits;

    }

    /**
     * Load list of date ranges from Insitu catalog for a category and a probe between 2 dates
     */
    @Override
    public List<DateRange> loadDateRanges(Date t1, Date t2, String catalog, String startName, String catType) {
        logger.info("In situ PATH=" + prop.getDATA_PATH() + "/" + prop.getINSITU_DIR());
        logger.debug("loadDateRanges with t1=" + t1 + " t2=" + t2 + "Catalog = " + catalog + " startName=" + startName + " Cat type = " + catType);

        List<DateRange> readedDates = new ArrayList<DateRange>();
        File path = new File(prop.getDATA_PATH(), prop.getINSITU_DIR());
        if (path != null) {
            path = new File(path.getPath(), catalog);
            if (path != null) {
                path = new File(path.getPath(), startName);
                if (path != null) {
                    path = new File(path.getPath(), catType);
                }
            }
        }

        File[] files = ScanDirectory.fetchFiles(path, t1, t2);
        if (files != null) {
            for (File file : files) {
                try {
                    List<DateRange> currentDateRanges = readDateRanges(t1, t2, file);
                    for (DateRange dateRange : currentDateRanges) {
                        readedDates.add(dateRange);
                    }
                } catch (Exception e) {
                    logger.error("Reading failure for file " + file.getName(), e);
                }
            }
        }
        return readedDates;
    }

    /**
     * Load list of Radial CME from catalog for an element ID, a type of CME (FIXED or SSE), an observatory and an
     * extent between 2 dates
     */
    @Override
    public List<CmeRadial> loadRadialCMEs(Date t1, Date t2, int elementId, CMEType cmeType, String obsName, double extent) {
        logger.info("CME Radial PATH=" + prop.getDATA_PATH() + "/" + prop.getRADIAL_DIR());
        logger.debug("loadRadialCMEs with t1=" + t1 + " t2=" + t2 + " element ID =" + elementId + " observer=" + obsName + " extent=" + extent);

        List<CmeRadial> readedCMEs = new ArrayList<CmeRadial>();
        File path = new File(prop.getDATA_PATH(), prop.getRADIAL_DIR());
        if (path != null) {
            path = new File(path.getPath(), obsName);
            if (path != null) {
                path = new File(path.getPath(), cmeType.name());
            }
        }

        File[] files = ScanDirectory.fetchFiles(path, t1, t2);
        if (files != null) {
            Arrays.sort(files);
            for (File file : files) {
                try {
                    List<CmeRadial> currentCMEs = readRadialCMEs(t1, t2, file, elementId, extent);
                    for (CmeRadial cme : currentCMEs) {
                        readedCMEs.add(cme);
                    }
                } catch (Exception e) {
                    logger.error("Reading failure for file " + file.getName(), e);
                }
            }
        }
        logger.debug("Number of selected Radial CMEs=" + readedCMEs.size());

        return readedCMEs;
    }

    /**
     * Load list of Radial CME from LINKCAT catalog for a type of CME (FIXED or SSE) and an observatory between 2 dates
     */
    @Override
    public List<CmeRadial> loadLinkcatCMEs(Date t1, Date t2, CMEType cmeType, String observer, double extent) {
        logger.info("CME LINKCAT PATH=" + prop.getDATA_PATH() + "/" + prop.getCMEFIT_DIR());
        logger.debug("loadLinkcatCMEs with t1=" + t1 + " t2=" + t2 + " observer=" + observer + " extent=" + extent);

        List<CmeRadial> readedCMEs = new ArrayList<CmeRadial>();
        File path = new File(prop.getDATA_PATH(), prop.getCMEFIT_DIR());
        if (path != null) {
            path = new File(path.getPath(), observer);
            if (path != null) {
                path = new File(path.getPath(), cmeType.name() + "_LINKCAT");
            }
        }

        logger.info("Fetch data in file=" + path.getName());
        File[] files = ScanDirectory.fetchFiles(path, t1, t2);
        if (files != null) {
            Arrays.sort(files);
            for (File file : files) {
                try {
                    List<CmeRadial> currentCMEs = readLinkcatCMEs(t1, t2, file, extent);
                    for (CmeRadial cme : currentCMEs) {
                        readedCMEs.add(cme);
                    }
                } catch (Exception e) {
                    logger.error("Reading failure for file " + file.getName(), e);
                }
            }
        }
        logger.debug("Number of selected Linkcat CMEs=" + readedCMEs.size());

        return readedCMEs;
    }

    /**
     * Load list of Corotation CIRs from catalog for a element ID, an observatory and a spread between 2 dates
     */
    @Override
    public List<CirCorotation> loadCorotationCIRs(Date t1, Date t2, int elementId, String obsName, double spread) {
        logger.info("CIR Corot PATH=" + prop.getDATA_PATH() + "/" + prop.getCOROTATION_DIR());
        logger.debug("loadCorotationCIRs with t1=" + t1 + " t2=" + t2 + " element ID =" + elementId + " observer=" + obsName + " spread=" + spread);

        List<CirCorotation> readedCIRs = new ArrayList<CirCorotation>();
        File path = new File(prop.getDATA_PATH(), prop.getCOROTATION_DIR());
        if (path != null) {
            path = new File(path.getPath(), obsName);
        }

        File[] files = ScanDirectory.fetchFiles(path, t1, t2);
        if (files != null) {
            Arrays.sort(files);
            for (File file : files) {
                try {
                    List<CirCorotation> currentCIRs = readCorotationCIRs(t1, t2, file, elementId, spread);
                    for (CirCorotation cir : currentCIRs) {
                        readedCIRs.add(cir);
                    }
                } catch (Exception e) {
                    logger.error("Reading failure for file " + file.getName(), e);
                }
            }
        }
        logger.debug("Number of selected Radial CMEs=" + readedCIRs.size());

        return readedCIRs;
    }

    /**
     * Load list of Carrington map types from CRMaps.conf config file
     */
    @Override
    public List<CarringtonMapType> loadCarringtonMapTypes() {
        logger.info("CRMap conf file PATH=" + prop.getCONF_CRMAP_FILE());

        List<CarringtonMapType> readedCRTypes = new ArrayList<CarringtonMapType>();
        File file;
        Scanner scanner = null;
        try {
            file = ApplicationContextHolder.getContext().getResource(prop.getCONF_CRMAP_FILE()).getFile();
            logger.info("File found " + file.getAbsolutePath());
            scanner = new Scanner(new FileReader(file));

        } catch (IOException e) {
            e.printStackTrace();
            logger.error("File not found " + prop.getCONF_CRMAP_FILE(), e);
            return null;
        }
        scanner.useLocale(Locale.US); // for reading floats

        while (scanner.hasNext()) {
            if (scanner.findInLine("#") != null) {
                scanner.nextLine();
            } else {
                try {
                    String line = scanner.nextLine();
                    logger.info("Reading line " + line);
                    List<String> fields = Arrays.asList(line.split(";"));
                    if (fields.size() >= 3) {
                        CarringtonMapType cRType = new CarringtonMapType();
                        cRType.setProbe(fields.get(0));
                        cRType.setInstrument(fields.get(1));
                        cRType.setMeasurement(fields.get(2));
                        if (fields.size() >= 4) {
                            List<String> radialDists = Arrays.asList(fields.get(3).split(" "));
                            for (String rd : radialDists) {
                                cRType.getRadialDistances().add(rd);
                            }
                        }
                        readedCRTypes.add(cRType);
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    scanner.close();
                    return null;
                }
            }
        }
        logger.info("CRMap list size : " + readedCRTypes.size());
        scanner.close();

        return readedCRTypes;
    }

    @Override
    public List<FlareFlux> loadFlareFlux(Date t1, Date t2, String fluxType, String fMeasure) {
        logger.info("FLAREFLUX PATH=" + prop.getDATA_PATH() + "/" + prop.getFLAREFLUX_DIR() + "/" + fluxType + "/" + fMeasure);
        logger.debug("loadFlareFlux with t1=" + t1 + " , t2=" + t2 + ", FluxType=" + fluxType + ", FluxMeasure=" + fMeasure);

        List<FlareFlux> flares = new ArrayList<FlareFlux>();
        File path = new File(prop.getDATA_PATH(), prop.getFLAREFLUX_DIR() + "/" + fluxType + "/" + fMeasure);

        File[] files = ScanDirectory.fetchFiles(path, t1, t2);
        if (files != null) {
            for (File file : files) {
                try {
                    List<FlareFlux> readedFlares = readFlareFlux(t1, t2, file);
                    for (FlareFlux flareFlux : readedFlares) {
                        flares.add(flareFlux);
                    }
                } catch (Exception e) {
                    logger.error("ERROR : reading failure for file " + file.getName(), e);
                }
            }
        }

        return flares;
    }

    /**
     * Load End footpoint of B-Field line from start footpoint, a date, a source file and a model
     */
    @Override
    public List<Footpoint> loadEndFieldLineFP(Footpoints startFieldLineFP, Date time, String model, String source) {

        logger.info("FIELDLINE PATH=" + prop.getDATA_PATH() + "/" + prop.getFIELDLINE_DIR());

        File path = new File(prop.getDATA_PATH(), prop.getFIELDLINE_DIR());
        List<Footpoint> endFieldLineFP = null;

        if (path != null) {
            path = new File(path.getPath(), model);
            if (path != null) {
                path = new File(path.getPath(), source);
            }
        }

        logger.info("FIELDLINE inputs=" + path.getAbsolutePath() + " " + Arrays.toString(startFieldLineFP.getFootpoints().toArray()));
        if ("WSO".equals(source)) {
            Integer CRN = (int) FitsHelper.getCarringtonRotationNumber(time);
            if (CRN > 0) {
                String crn = CRN.toString();
                File[] files = ScanDirectory.fetchMatchingFiles(path, "_" + crn + ".", ".dat");
                if (files != null && files.length > 0) {
                    try {
                        endFieldLineFP = readEndFieldLineFP(files[0], startFieldLineFP.getFootpoints());

                    } catch (Exception e) {
                        logger.error("Reading failure for file " + files[0], e);
                    }
                }
            }
        }

        return endFieldLineFP;
    }

    /**
     * Load End footpoint of B-Field line from start footpoint, a date, a source file and a model
     */
    /**
     * Load pfss cube on file system. Name is kind of bestcube_YYYYMMDDTHHMISS_mission.dat
     *
     * @param pTime time to find cube
     * @param model should be PFSS. Used for first directory
     * @param source sould be BEST_OF. used for second directory.
     * @return
     */
    @Override
    public Cube loadPFSSCube(Date pTime, String model, String source) {

        double RSun = 696340;
        logger.info("FIELDLINE PATH=" + prop.getDATA_PATH() + "/" + prop.getFIELDLINE_DIR());

        
        
        File path = new File(prop.getDATA_PATH(), prop.getFIELDLINE_DIR());
        Cube lCube = new Cube();

        if (path != null) {
            path = new File(path.getPath(), model);
            if (path != null) {
                path = new File(path.getPath(), source);
            }
        }
        File lFile = null;
        //build file name to find
        SimpleDateFormat lSdf = new SimpleDateFormat("yyyyMMdd'T120000'");
        lSdf.setTimeZone(TimeZone.getTimeZone("UTC"));
//        Calendar lCal = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
//        lCal.setTime(pTime);
//        if (lCal.get(Calendar.AM_PM) == Calendar.PM) {
//            lSdf.applyPattern("yyyyMMdd'T120000'");
//        }
        logger.info("FIELDLINE finding=" + path+"/"+lSdf.format(pTime) + "_cube" + ".txt");
        File[] files = ScanDirectory.fetchMatchingFiles(path, lSdf.format(pTime) + "_cube", ".txt");
        if (files != null && files.length > 0) {
            lFile = files[0];
        } else {
            logger.error("No file found for " + lSdf.format(pTime));
            return null;
        }

        List<Float> rValues = new ArrayList<Float>(); // r grid values (planet radius)
        /**
         * theta grid values colatitude (radians 0 => PI)
         */
        List<Float> tValues = new ArrayList<Float>();
        /**
         * phi grid values (radians 0 => 2*PI)
         */
        List<Float> pValues = new ArrayList<Float>();
        int nbValues = -1;
        float[][][] bXValues = null;
        float[][][] bYValues = null;
        float[][][] bZValues = null;
        float[][][] bRValues = null;
        float[][][] bTValues = null;
        float[][][] bPValues = null;

        //Scanner lScan = null;

        try {
            
            long lStart = System.currentTimeMillis();
            
            Path lFastFilePath = lFile.toPath();
            logger.info("***********Try to read "+lFastFilePath.toString());
            String lContents = new String(Files.readAllBytes(lFastFilePath));
            logger.info("***********ReadAllBytes lasted "+(System.currentTimeMillis()-lStart)/1000d);
            //lStart = System.currentTimeMillis();
            

            /*Read file*/
//            lScan = new Scanner(lContents);
//            String lLine = null;
//            // skip header
//            while (lScan.hasNextLine()) {
//                lLine = lScan.nextLine();
//                if (lLine != null && lLine.startsWith("--------------------")) {
//                    break;
//                }
//            }
            
            /**First, split lines */
            String[] lLines = lContents.split("\\n");
            int lLineIdx = 0;
            String lLine = null;
            while(lLines[lLineIdx]==null || !lLines[lLineIdx].startsWith("--------------------")) {
                lLineIdx++;
            }
            lLineIdx++;//skip "--------------"
            
            //start reading data
            int lIdxPos = 0;
//            while (lScan.hasNextLine()) {
//                lLine = lScan.nextLine();
            while (lLineIdx<lLines.length) {
                lLine = lLines[lLineIdx++];
                if (lLine != null) {
                    String[] lValues = lLine.trim().split("\\s+");
                    if (lValues.length == 1) {
                        if (lValues[0].isEmpty()) {
                            lIdxPos++;
                            System.out.println("New position: " + lIdxPos);
                            if (lIdxPos == 3) {//end reading R/T/P positions
                                break;
                            }
                        } else {
                            switch (lIdxPos) {
                                case 0:
                                    rValues.add(Float.parseFloat(lValues[0]));
                                    break;
                                case 1:

                                    tValues.add((float) Math.toRadians(Float.parseFloat(lValues[0])));
                                    //tValues.add((float)Float.parseFloat(lValues[0]));
                                    break;
                                case 2:
                                    pValues.add((float) Math.toRadians(Float.parseFloat(lValues[0])));
                                    //pValues.add((float)Float.parseFloat(lValues[0]));
                                    break;
                            }
                        }
                    }
                }
            }
            //lCubeValues = new Vector3d[lRValues.size()][lTValues.size()][lPValues.size()];
            nbValues = rValues.size() * tValues.size() * pValues.size();
            bXValues = new float[rValues.size()][tValues.size()][pValues.size()]; //of dimension RxTxP
            bYValues = new float[rValues.size()][tValues.size()][pValues.size()]; //of dimension RxTxP
            bZValues = new float[rValues.size()][tValues.size()][pValues.size()]; //of dimension RxTxP
            bRValues = new float[rValues.size()][tValues.size()][pValues.size()]; //of dimension RxTxP
            bTValues = new float[rValues.size()][tValues.size()][pValues.size()]; //of dimension RxTxP
            bPValues = new float[rValues.size()][tValues.size()][pValues.size()]; //of dimension RxTxP
            Vector3d lBValues[] = new Vector3d[nbValues]; // flat array for arrow cloud display
            //int lValIdx = 2; //index of value to fetch
            int lValNo = 0;
            int lLineNo = 0;

            System.out.println("Read RTP positions: " + nbValues);

            lIdxPos = 0;
//            while (lScan.hasNextLine()) {
//                lLine = lScan.nextLine();
            while (lLineIdx<=lLines.length) {

                if(lLineIdx<lLines.length) {
                    lLine = lLines[lLineIdx++];
                } else {
                    lLine = "\n";
                }
                lLineNo++;
                //System.out.println("* "+lLine);
                if (lLine != null) {
                    String[] lStrValues = lLine.trim().split("\\s+");
                    if (lStrValues[0].isEmpty()) {
                        lIdxPos++;
                        System.out.println("New value: " + lIdxPos + " lValNo: " + lValNo + " lLineNo: " + lLineNo);
                        lValNo = 0;

                        if (lIdxPos == 3) {//end reading R/T/P positions
                            break;
                        }
                    } else {
                        for (String lStrVal : lStrValues) {
                            switch (lIdxPos) {
                                case 0: //R=Z
                                    lBValues[lValNo] = new Vector3d(0d, 0d, Double.parseDouble(lStrVal));
                                    break;
                                case 1: //theta=y
                                    lBValues[lValNo].setY(-Double.parseDouble(lStrVal));
                                    //System.out.println("Read Y "+lStrVal+" => "+lBValues[lValNo].getY());
                                    break;
                                case 2: //phi=X
                                    lBValues[lValNo].setX(Double.parseDouble(lStrVal));
                                    //System.out.println("Read X "+lStrVal+" => "+lBValues[lValNo].getX());
                                    break;
                            }
                            lValNo++;
                        }

                    }
                } else {
                    logger.warn("Read null line");
                }
            }

            int lRPos = 0;
            int lTPos = 0;
            int lPPos = 0;
            int lNbVals = 0;
            for (float lRVal : rValues) {
                lTPos = 0;
                for (float lTVal : tValues) {
                    lPPos = 0;
                    for (float lPVal : pValues) {
                        Vector3d lPos = new Vector3d(
                                FieldLineImpl.sphericToCart(lPVal, lTVal, lRVal * RSun));
                        Vector3f lVz = new Vector3f(FieldLineImpl.sphericToCart(lPVal, lTVal, 1));
                        Vector3f lVy = new Vector3f(FieldLineImpl.sphericToCart(lPVal, lTVal + Math.PI / 2, 1));
                        Vector3f lVx = new Vector3f();
                        lVx.cross(lVy, lVz);
                        lVx.normalize();
                        lVy.normalize();
                        lVz.normalize();
                        Matrix3d lMat = new Matrix3d(
                                lVx.x, lVy.x, lVz.x,
                                lVx.y, lVy.y, lVz.y,
                                lVx.z, lVy.z, lVz.z);

                        Vector3d lBVec = new Vector3d(lBValues[lNbVals]);
                        lMat.transform(lBVec);
                        //System.out.println("B read/transformed "+lBVec+" / "+lBValues[lNbVals]);

                        //lVectors.add(new Vector3d(lVec.z, lVec.y, lVec.x * j3d.getCenterBody().getSize()/2));
                        bXValues[lRPos][lTPos][lPPos] = (float) lBVec.x;
                        bYValues[lRPos][lTPos][lPPos] = (float) lBVec.y;
                        bZValues[lRPos][lTPos][lPPos] = (float) lBVec.z;
                        bRValues[lRPos][lTPos][lPPos] = (float) lBValues[lNbVals].z;
                        bTValues[lRPos][lTPos][lPPos] = (float) lBValues[lNbVals].y;
                        bPValues[lRPos][lTPos][lPPos] = (float) lBValues[lNbVals].x;
                        lNbVals++;
                        lPPos++;
                    }
                    lTPos++;
                }
                lRPos++;
            }

        } catch (IOException lIoe) {
            logger.error("extract - trying to open or read file: " + lIoe.getMessage());
//        } finally {
//            if (null != lScan) {
//                lScan.close();
//            }
        }

        lCube.setrValues(lstFloatToArr(rValues));
        lCube.settValues(lstFloatToArr(tValues));
        lCube.setpValues(lstFloatToArr(pValues));
        lCube.setbXValues(bXValues);
        lCube.setbYValues(bYValues);
        lCube.setbZValues(bZValues);
        return lCube;
    }

    private float[] lstFloatToArr(List<Float> pLst) {
        float[] lArr = new float[pLst.size()];
        for (int i = 0; i < lArr.length; i++) {
            lArr[i] = pLst.get(i);
        }
        return lArr;
    }

    /**
     * Build file Name and get file from Connect tool. Then read file and send data.
     *
     * @param time
     * @return
     */
    public HCSLine loadHCSLine(Date pTime) throws IOException, MalformedURLException {
        HCSLine lHcsLine = new HCSLine();
        //build file name to get
        SimpleDateFormat lSdf = new SimpleDateFormat("yyyyMMdd'T000000'");
        lSdf.setTimeZone(TimeZone.getTimeZone("UTC"));
        Calendar lCal = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
        lCal.setTime(pTime);
        if (lCal.get(Calendar.AM_PM) == Calendar.PM) {
            lSdf.applyPattern("yyyyMMdd'T120000'");
        }

        logger.debug("Getting HCS line for " + lSdf.format(pTime));

        File lTmpHcsFile;
        try {
            lTmpHcsFile = File.createTempFile("hcs", ".ptl");
        } catch (IOException ex) {
            throw new IOException("Cannot create tmp file to get HCS line", ex);
        }
        Scanner lScan = null;
        try {
            String lHcsUrl = "http://connect-tool.irap.omp.eu/static/connect_img/PARKER/PFSS/SCTIME/SCIENCE/EARTH_ADAPT_"
                    + lSdf.format(pTime)
                    + "/EARTH_PARKER_PFSS_SCTIME_ADAPT_SCIENCE_"
                    + lSdf.format(pTime)
                    + "_filehcs.ascii";

            logger.debug("Get distant HCS line at " + lHcsUrl);

            FileUtils.copyURLToFile(new URL(lHcsUrl), lTmpHcsFile);

            /*Read file*/
            lScan = new Scanner(lTmpHcsFile);
            String lLine = null;
            // skip header
            while (lScan.hasNextLine()) {
                lLine = lScan.nextLine();
                logger.debug("HCSRead skipping line: " + lLine);
                if (lLine != null && lLine.startsWith("npoint")) {
                    break;
                }
            }

            //start reading data
            int lIdxPos = 0;
            List<Footpoint> lFpList = new ArrayList<Footpoint>();
            while (lScan.hasNextLine()) {
                lLine = lScan.nextLine();
                if (lLine != null) {
                    String[] lValues = lLine.trim().split("\\s+");
                    if (lValues.length > 4) {
                        //logger.debug("HCSRead add footpoint: "+Float.parseFloat(lValues[3])+" / "+Float.parseFloat(lValues[2]));
                        lFpList.add(new Footpoint(Float.parseFloat(lValues[3]), Float.parseFloat(lValues[2])));
                    }
                }
            }
            lHcsLine.setPoints(lFpList);
            if (lFpList.isEmpty()) {
                //throw new IOException("No HCS points found in file");
            }
        } catch (MalformedURLException ex) {
            throw new IOException("Bad url on HCS file retrieval", ex);
        }

        lTmpHcsFile.delete();
        return lHcsLine;
    }

    /**
     * Read measurements belonging to [t1,t2] from file containing flares
     */
    private static List<Flare> readFlares(Date t1, Date t2, File file) {
        List<Flare> flaresList = new ArrayList<Flare>();

        // Scanner for reading file
        logger.debug("file=" + file.getName() + " t1=" + t1 + " t2=" + t2);
        Scanner scanner = null;
        int lineCount = 0;
        try {
            scanner = new Scanner(new FileReader(file));
        } catch (FileNotFoundException e) {
            logger.error("File not found " + file.getName(), e);
            return null;
        }
        scanner.useLocale(Locale.US); // for reading floats

        // Skip header lines beginning with #
        while (scanner.findInLine("#") != null) {
            scanner.nextLine();
            lineCount++;
        }

        SimpleDateFormat sdf = new SimpleDateFormat(ISO_TIME_FORMAT);
        Pattern pattern = Pattern.compile(ISO_TIME_PATTERN);
        boolean isAfterT2 = false;
        boolean isValid = true;
        Date dayDate = null;

        while (scanner.hasNext() && (scanner.findInLine("#") == null) && !isAfterT2) {
            lineCount++;
            String currentField = scanner.next();
            int iField = 1;

            // Check that the first field of current line is a date as yyyy-MM-ddThh:mm:ssZ*
            if (pattern.matcher(currentField).matches()) {
                Flare currentFlare = new Flare();
                Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
                try {
                    calendar.setTime(sdf.parse(currentField));
                    currentFlare.setEventTime(calendar.getTime());

                    // if first read date, extract the day date at 0h
                    if (dayDate == null) {
                        SimpleDateFormat sdfDay = new SimpleDateFormat("yyyy-MM-dd");
                        dayDate = sdfDay.parse(currentField);
                    }
                    currentFlare.setDayDate(dayDate);
                    iField++;

                    // Read Carrington latitude of flare
                    int oneInt = scanner.nextInt();
                    currentFlare.setLatitude((double) oneInt);
                    iField++;

                    // Read HEEQ longitude of flare
                    oneInt = scanner.nextInt();
                    currentFlare.setRelativeLongitude((double) oneInt);
                    iField++;

                    // flare_class means (1='A' 2='B' 3='C' 4='M' 5='X')
                    oneInt = scanner.nextInt();
                    boolean isFound = false;
                    for (FlareType flareClass : FlareType.values()) {
                        if (oneInt == flareClass.ordinal() + 1) {
                            currentFlare.setCategory(flareClass);
                            isFound = true;
                            break;
                        }
                    }
                    if (!isFound) {
                        logger.error("Line " + lineCount + ", invalid value flare class in column " + iField + " : " + oneInt);
                        isValid = false;
                    }
                    iField++;

                    // Read flare intensity
                    float oneFloat = scanner.nextFloat();
                    currentFlare.setIntensity(oneFloat);
                    iField++;

                    // Read NOAA: add "1" ahead to have standard number
                    oneInt = scanner.nextInt();
                    String text = '1' + String.valueOf(oneInt);
                    oneInt = Integer.decode(text);
                    currentFlare.setNumberNOAA(oneInt);

                } catch (Exception e) {
                    logger.error("Line " + lineCount + ", invalid Field " + iField, e);
                    isValid = false;
                }

                // Store current measurement if valid and inside [t1,t2]
                if (isValid && t1.compareTo(calendar.getTime()) <= 0 && t2.compareTo(calendar.getTime()) >= 0) {
                    flaresList.add(currentFlare);
                }

                // Stop reading if current measurement is after t2
                isAfterT2 = !t2.after(calendar.getTime());
            } else {
                scanner.nextLine();
            }
        }
        scanner.close();
        logger.debug("Number of selected flares=" + flaresList.size());
        return flaresList;
    }

    /**
     * Read measurements belonging to [t1,t2] from LINKCAT file containing flares
     */
    private static List<Flare> readLinkcatFlares(Date t1, Date t2, File file) {
        List<Flare> flaresList = new ArrayList<Flare>();

        // Scanner for reading file
        logger.info("file=" + file.getName() + " t1=" + t1 + " t2=" + t2);
        Scanner scanner = null;
        int lineCount = 0;
        try {
            scanner = new Scanner(new FileReader(file));
        } catch (FileNotFoundException e) {
            logger.error("File not found " + file.getName(), e);
            return null;
        }
        scanner.useLocale(Locale.US); // for reading floats

        while (scanner.findInLine("#") != null) {
            scanner.nextLine();
            lineCount++;
        }

        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm");
        Pattern pattern = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}.*");
        boolean isAfterT2 = false;
        Date dayDate = null;

        while (scanner.hasNext() && (scanner.findInLine("#") == null) && !isAfterT2) {
            // skip first fields
            scanner.next();
            scanner.next();
            scanner.next();
            scanner.next();
            if (scanner.next().equals("F")) {
                // This is a line of type "Flare"
                String currentField = scanner.next();
                // Check that the field 6 of current line is a date
                if (pattern.matcher(currentField).matches()) {
                    Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
                    try {
                        calendar.setTime(sdf.parse(currentField));

                        // Store current measurement if valid and inside [t1,t2] 
                        if (calendar.getTime().after(t1) && calendar.getTime().before(t2)) {
                            Flare currentFlare = new Flare();
                            currentFlare.setEventTime(calendar.getTime());
                            // if first read date, extract the day date at 0h
                            if (dayDate == null) {
                                SimpleDateFormat sdfDay = new SimpleDateFormat("yyyy-MM-dd");
                                dayDate = sdfDay.parse(currentField);
                            }
                            currentFlare.setDayDate(dayDate);
                            currentFlare.setRelativeLongitude(scanner.nextDouble());
                            currentFlare.setLatitude(scanner.nextDouble());
                            String flareClass = scanner.next();
                            currentFlare.setCategory(FlareType.valueOf(flareClass.substring(0, 1)));
                            currentFlare.setIntensity(Float.parseFloat(flareClass.substring(1)));
                            // all fields are OK => add flare
                            flaresList.add(currentFlare);
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                        logger.error("invalid fields for line " + lineCount, e);
                        scanner.close();
                        return null;
                    }
                    // Stop reading if current measurement is after t2
                    isAfterT2 = calendar.getTime().after(t2);
                    lineCount++;

                } else {
                    scanner.nextLine();
                }
            } else {
                scanner.nextLine();
            }
        }
        scanner.close();
        logger.debug("Number of selected flares=" + flaresList.size());
        return flaresList;
    }

    /**
     *
     * @param t1
     * @param t2
     * @param file
     * @return
     */
    private List<Shock> readShocks(Date t1, Date t2, File file) {
        List<Shock> shocks = new ArrayList<Shock>();

        // Scanner for reading file
        logger.debug("file=" + file.getName());
//		logger.debug("read shocks between " + t1 + " and " + t2);
        Scanner scanner = null;
        int lineCount = 0;
        try {
            scanner = new Scanner(new FileReader(file));
        } catch (FileNotFoundException e) {
            logger.error("File not found " + file.getName(), e);
            return null;
        }
        scanner.useLocale(Locale.US); // for reading floats

        while (scanner.findInLine("#") != null) {
            scanner.nextLine();
            lineCount++;
        }

        SimpleDateFormat sdf = new SimpleDateFormat(ISO_TIME_FORMAT);
        Pattern pattern = Pattern.compile(ISO_TIME_PATTERN);
        boolean isAfterT2 = false;
        Date startDate = new Date(t1.getTime());
        long deltaTime = 3 * 60 * 1000 - 1000; // 3 minutes 

        while (scanner.hasNext() && (scanner.findInLine("#") == null) && !isAfterT2) {
            String currentField = scanner.next();
            // Check that the first field of current line is a date
            if (pattern.matcher(currentField).matches()) {
                Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
                try {
                    calendar.setTime(sdf.parse(currentField));

                    if (calendar.getTime().after(startDate) && calendar.getTime().before(t2)) {
                        // Store current measurement if valid and inside [t1,t2] 
                        Shock shock = new Shock();
                        shock.setShockTime(calendar.getTime());
                        shock.setTheta(CoordConverter.normalizeLongitude(scanner.nextDouble()));
                        shock.setPhi(CoordConverter.normalizeLatitude(scanner.nextDouble()));
                        shock.setRdSun(CoordConverter.rsunToKm(scanner.nextDouble()));
                        shock.setRadiusX(CoordConverter.rsunToKm(scanner.nextDouble()));
                        shock.setRadiusY(CoordConverter.rsunToKm(scanner.nextDouble()));
                        shock.setRadiusZ(CoordConverter.rsunToKm(scanner.nextDouble()));

                        //logger.debug("Add shock for " + shock.getShockTime());
                        shocks.add(shock);

                        startDate.setTime(shock.getShockTime().getTime() + deltaTime);
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    logger.error("invalid fields for line " + lineCount, e);
                    scanner.close();
                    return null;
                }

                // Stop reading if current measurement is after t2
                isAfterT2 = calendar.getTime().after(t2);
                lineCount++;

            } else {
                scanner.nextLine();
            }
        }
        logger.debug("Number of selected shocks=" + shocks.size());
        scanner.close();
        return shocks;
    }

    /**
     * Get all files data in a shockwave 3d folder event
     *
     * @param t1
     * @param t2
     * @param pEventDir
     * @return
     */
    private List<Shockwave3d> readShockwaves(File pEventDir) {
        List<Shockwave3d> shocks = new ArrayList<Shockwave3d>();

        // Scanner for reading file
        logger.debug("Reading shockwaves in " + pEventDir.getName());
//		logger.debug("read shocks between " + t1 + " and " + t2);
        long lStartTime = System.currentTimeMillis();
                
        for (File lFile : pEventDir.listFiles()) {
            logger.debug("Reading file " + lFile.getName());
            Scanner scanner = null;
            int lineCount = 0;
            try {
                scanner = new Scanner(lFile);
            } catch (FileNotFoundException e) {
                logger.error("File not found " + pEventDir.getName(), e);
                return null;
            }
            scanner.useLocale(Locale.US); // for reading floats

//            while (scanner.findInLine("#") != null) {
//                scanner.nextLine();
//                lineCount++;
//            }
            try {
                //first line is # Fit time:2013/10/11 07:09:59
                SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
                sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
                String lLine = scanner.nextLine();
                Date lDate = sdf.parse(lLine.substring(lLine.indexOf(':')+1));
                Shockwave3d lSw3d = new Shockwave3d();
                lSw3d.setTime(lDate);

                //Second line is # 1: X (km) 2: Y(km) 3: Z(km) 4: Shock speed (mk/s)  5: Mach_Alfven 6 :Mach_fm 7: Comp_Ratio 8: Theta_BN
                scanner.nextLine();//skip 2nd line

                while (scanner.hasNextLine() && (scanner.findInLine("#") == null)) {
                    String lStrLine = scanner.nextLine().trim();
                    //logger.debug("Reading line "+lStrLine);
                    String[] lStrValues = lStrLine.split("\\s+");
                    if (lStrValues.length >= 8) {
                        //logger.debug("pos = "+Double.parseDouble(lStrValues[0])+" "+Double.parseDouble(lStrValues[1])+" "+Double.parseDouble(lStrValues[2]));
                        double[] lPos = new double[]{Double.parseDouble(lStrValues[0]), Double.parseDouble(lStrValues[1]), Double.parseDouble(lStrValues[2])};
                        lSw3d.getPositions().add(lPos);
                        double[] lParameters = new double[]{
                            Double.parseDouble(lStrValues[3]),
                            Double.parseDouble(lStrValues[4]),
                            Double.parseDouble(lStrValues[5]),
                            Double.parseDouble(lStrValues[6]),
                            Double.parseDouble(lStrValues[7])
                        };
                        lSw3d.getParameters().add(lParameters);
                    }
                }
                logger.debug("Found " + lSw3d.getParameters().size()+ " values.");
                shocks.add(lSw3d);
            } catch (Exception e) {
                e.printStackTrace();
                //logger.error("invalid fields for line " + lineCount, e);
                scanner.close();
                return null;
            }
            scanner.close();
        }
        logger.debug("Number of step in event: " + shocks.size() + ". Read in "+(System.currentTimeMillis()-lStartTime)/1000d+"s");
        lStartTime = System.currentTimeMillis();
        Collections.sort(shocks);//sort by time
        logger.debug("Sorted shoks in "+(System.currentTimeMillis()-lStartTime)/1000d+"s");
        return shocks;
    }

    /**
     * Read measurements belonging to [t1,t2] from file containing solar speeds
     */
    private static List<SolarSpeed> readSolarSpeeds(Date t1, Date t2, final File file) {
        List<SolarSpeed> speedsList = new ArrayList<SolarSpeed>();

        // Scanner for reading file
        logger.debug("file=" + file.getName());
        Scanner scanner = null;
        int lineCount = 0;
        try {
            scanner = new Scanner(new FileReader(file));
        } catch (FileNotFoundException e) {
            logger.error("File not found " + file.getName(), e);
            return null;
        }
        scanner.useLocale(Locale.US); // for reading floats

        while (scanner.findInLine("#") != null) {
            scanner.nextLine();
            lineCount++;
        }

//		SimpleDateFormat simpleDateFormat = new SimpleDateFormat("dd-MM-yyyy HH:mm:ss");
//		Pattern pattern = Pattern.compile("\\d{2}-\\d{2}-\\d{4}");
        SimpleDateFormat sdf = new SimpleDateFormat(ISO_TIME_FORMAT);
        Pattern pattern = Pattern.compile(ISO_TIME_PATTERN);
        boolean isAfterT2 = false;

        while (scanner.hasNext() && (scanner.findInLine("#") == null) && !isAfterT2) {
            String currentField = scanner.next();
            // Check that the first field of current line is a date
            if (pattern.matcher(currentField).matches()) {
                SolarSpeed currentSpeed = new SolarSpeed();
                Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));

                // Read fields about time
                try {
//					String strDate = currentField + " " + scanner.next();
//					calendar.setTime(simpleDateFormat.parse(strDate));
                    String strDate = currentField;
                    calendar.setTime(sdf.parse(strDate));
                    currentSpeed.setEventTime(calendar);

                } catch (Exception e) {
                    e.printStackTrace();
                    logger.error("invalid fields about time for line " + lineCount, e);
                    scanner.close();
                    return null;
                }
                // Read speed field
                try {
                    Float speed = scanner.nextFloat();
                    currentSpeed.setSpeed(speed);
                    // Store current measurement if valid and inside [t1,t2]
                    //if (!speed.isNaN() && speed > 0 && speed < 4000.0) {
                    if (speed > 0 && speed < 4000.0) {
                        if (calendar.getTime().after(t1) && calendar.getTime().before(t2)) {
                            speedsList.add(currentSpeed);
                            // logger.info("speed added in list : " + currentSpeed.getEventTime().getTime() + ", " + currentSpeed.getSpeed());
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    logger.error("invalid field about speed for line " + lineCount, e);
                    scanner.close();
                    return null;
                }

                // Stop reading if current measurement is after t2
                isAfterT2 = !t2.after(calendar.getTime());
            }
            lineCount++;
        }
        logger.info("speed list size : " + speedsList.size());
        scanner.close();
        return speedsList;
    }

    /**
     * Read measurements belonging to [t1,t2] from file containing SEP fluxes
     */
    private static List<Spectro> readSEPFluxes(Date t1, Date t2, final File file) {
        List<Spectro> lValuesList = new ArrayList<Spectro>();

        // Scanner for reading file
        logger.debug("readSEPFluxes file=" + file.getName());
        Scanner lScanner = null;
        int lineCount = 0;
        try {
            lScanner = new Scanner(new FileReader(file));
        } catch (FileNotFoundException e) {
            logger.error("File not found " + file.getName(), e);
            return null;
        }
        lScanner.useLocale(Locale.US); // for reading floats

        while (lScanner.findInLine("#") != null) {
            lScanner.nextLine();
            lineCount++;
        }

//		SimpleDateFormat simpleDateFormat = new SimpleDateFormat("dd-MM-yyyy HH:mm:ss");
//		Pattern pattern = Pattern.compile("\\d{2}-\\d{2}-\\d{4}");
        SimpleDateFormat sdf = new SimpleDateFormat(ISO_TIME_FORMAT);
        Pattern pattern = Pattern.compile(ISO_TIME_PATTERN);
        boolean isAfterT2 = false;

        while (lScanner.hasNext() && (lScanner.findInLine("#") == null) && !isAfterT2) {
            //String currentField = lScanner.next();
            String lCurrLine[] = lScanner.nextLine().split("\\s+");
            // Check that the first field of current line is a date
            if (lCurrLine.length > 0 && pattern.matcher(lCurrLine[0]).matches()) {
                Spectro lCurrSpectro = new Spectro();
                Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));

                // Read fields about time
                try {
//					String strDate = currentField + " " + scanner.next();
//					calendar.setTime(simpleDateFormat.parse(strDate));
                    String strDate = lCurrLine[0];
                    calendar.setTime(sdf.parse(strDate));
                    lCurrSpectro.setEventTime(calendar);

                } catch (Exception e) {
                    e.printStackTrace();
                    logger.error("Invalid fields about time for line " + lineCount, e);
                    lScanner.close();
                    return null;
                }
                if (calendar.getTime().after(t1) && calendar.getTime().before(t2)) {
                    // Read fluxes values
                    try {
                        List<Float> lFlux = new ArrayList<Float>();
                        for (int i = 1; i < lCurrLine.length; i++) {
                            lFlux.add(Float.valueOf(lCurrLine[i]));
                        }
                        if (!lFlux.isEmpty()) {
                            lCurrSpectro.setFlux(lFlux);
                            // Store current measurement if valid and inside [t1,t2]
                            lValuesList.add(lCurrSpectro);
                            //logger.info("spectro time in list : " + lCurrSpectro.getEventTime().getTime() + ", " + lCurrSpectro.getFlux().size() + ", lineLength: " + lCurrLine.length);
                        } else {
                            logger.info("Empty spectro values at time : " + lCurrSpectro.getEventTime().getTime());
                        }
                        //}
                    } catch (Exception e) {
                        e.printStackTrace();
                        logger.error("Invalid field about sep fluxes for line " + lineCount, e);
                        lScanner.close();
                        return null;
                    }
                }

                // Stop reading if current measurement is after t2
                isAfterT2 = !t2.after(calendar.getTime());
            }
            lineCount++;
        }
        logger.info("readSEPFluxes Spectro flux size : " + lValuesList.size());
        lScanner.close();
        return lValuesList;
    }

    /**
     * Read energy table from spectro meta data file
     */
    private static List<Float[]> readSEPETable(final File file) {
        List<Float[]> lNrjList = new ArrayList<Float[]>();

        // Scanner for reading file
        logger.debug("file=" + file.getName());
        Scanner lScanner = null;
        int lineCount = 0;
        try {
            lScanner = new Scanner(new FileReader(file));
        } catch (FileNotFoundException e) {
            logger.error("File not found " + file.getName(), e);
            return null;
        }
        lScanner.useLocale(Locale.US); // for reading floats

        while (lScanner.findInLine("#") != null) {
            lScanner.nextLine();
            lineCount++;
        }
        while (lScanner.hasNext()) {
            String lCurrLine = lScanner.nextLine();
            logger.info("Reading line " + lCurrLine);
            if (lCurrLine.startsWith("#")) {
                continue; // skip comments
            }
            List<String> lTokens = Arrays.asList(lCurrLine.split("\\s+"));
            lNrjList.add(new Float[]{Float.valueOf(lTokens.get(1)), Float.valueOf(lTokens.get(2))});
        }
        logger.info("energy table size : " + lNrjList.size());
        lScanner.close();
        return lNrjList;
    }

    /**
     * Read measurements belonging to [t1,t2] from file containing flare flux
     */
    private List<FlareFlux> readFlareFlux(Date t1, Date t2, final File file) {
        List<FlareFlux> flares = new ArrayList<FlareFlux>();

        // Scanner for reading file
        logger.debug("file=" + file.getName());
        Scanner scanner = null;
        int lineCount = 0;
        try {
            scanner = new Scanner(new FileReader(file));
        } catch (FileNotFoundException e) {
            logger.error("File not found " + file.getName(), e);
            return null;
        }
        scanner.useLocale(Locale.US); // for reading floats

        while (scanner.findInLine("#") != null) {
            scanner.nextLine();
            lineCount++;
        }

        // Check that the first field of current line is a date
        SimpleDateFormat sdf = new SimpleDateFormat(ISO_TIME_FORMAT);
        Pattern pattern = Pattern.compile(ISO_TIME_PATTERN);
        boolean isAfterT2 = false;

        while (scanner.hasNext() && (scanner.findInLine("#") == null) && !isAfterT2) {
            String currentField = scanner.next();
            // Check that the first field of current line is a date
            if (pattern.matcher(currentField).matches()) {
                FlareFlux currentFlux = new FlareFlux();
                Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));

                // Read fields about time
                try {
                    String strDate = currentField;
                    calendar.setTime(sdf.parse(strDate));
                    currentFlux.setEventTime(calendar);

                } catch (Exception e) {
                    e.printStackTrace();
                    logger.error("invalid fields about time for line " + lineCount, e);
                    scanner.close();
                    return null;
                }

                // Read flux field
                try {
                    Float flux = scanner.nextFloat();
                    currentFlux.setFlux(flux);
                    // Store current measurement if valid and inside [t1,t2]
                    if (flux >= 0 && flux < 4000.0) {
                        if (calendar.getTime().after(t1) && calendar.getTime().before(t2)) {
                            flares.add(currentFlux);
                            // logger.info("speed added in list : " + currentFlux.getEventTime().getTime() + ", " + currentFlux.getFlux());
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    logger.error("invalid field about flux for line " + lineCount, e);
                    scanner.close();
                    return null;
                }

                // Stop reading if current measurement is after t2
                isAfterT2 = !t2.after(calendar.getTime());
            }
            lineCount++;
        }
        logger.info("flux list size : " + flares.size());
        scanner.close();
        return flares;
    }

    /**
     * Read measurements belonging to [t1,t2] from file containing date ranges
     */
    private static List<DateRange> readDateRanges(Date t1, Date t2, final File file) {
        List<DateRange> dateRanges = new ArrayList<DateRange>();

        // Scanner for reading file
        logger.debug("file=" + file.getName());
        Scanner scanner = null;
        int lineCount = 0;
        try {
            scanner = new Scanner(new FileReader(file));
        } catch (FileNotFoundException e) {
            logger.error("File not found " + file.getName(), e);
            return null;
        }
        scanner.useLocale(Locale.US); // for reading floats

        while (scanner.findInLine("#") != null) {
            scanner.nextLine();
            lineCount++;
        }

        SimpleDateFormat sdf = new SimpleDateFormat(ISO_TIME_FORMAT);
        Pattern pattern = Pattern.compile(ISO_TIME_PATTERN);
        Date startTime, endTime;
        boolean isAfterT2 = false;

        while (scanner.hasNext() && (scanner.findInLine("#") == null) && !isAfterT2) {
            String tStartField = scanner.next();
            // Check that the first field of current line is a date
            if (pattern.matcher(tStartField).matches()) {
                GregorianCalendar gc = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
                try {
                    gc.setTime(sdf.parse(tStartField));
                    startTime = gc.getTime();

                } catch (Exception e) {
                    logger.error("Line " + lineCount + ", invalid Field 1 : Date ", e);
                    scanner.close();
                    return null;
                }
                String tEndField = scanner.next();
                gc = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
                try {
                    gc.setTime(sdf.parse(tEndField));
                    endTime = gc.getTime();

                } catch (Exception e) {
                    logger.error("Line " + lineCount + ", invalid Field 2 : Date ", e);
                    scanner.close();
                    return null;
                }

                if (startTime.after(t1) && endTime.before(t2)) {
                    DateRange dr = new DateRange();
                    dr.setStartTime(startTime);
                    dr.setEndTime(endTime);
                    dateRanges.add(dr);
                }

                // Stop reading if current measurement is after t2
                isAfterT2 = endTime.after(t2);
                lineCount++;

            } else {
                scanner.nextLine();
            }
            lineCount++;
        }
        logger.info("date range list size : " + dateRanges.size());
        scanner.close();
        return dateRanges;
    }

    /**
     * Read measurements belonging to [t1,t2] from CIR fits catalog Columns : Sun time / Speed / Speed error / Beta
     * (delta long between start and spiral at start time) / Beta error
     */
    private static List<Fit> readCIRFits(Date t1, Date t2, final File file) {
        List<Fit> fits = new ArrayList<Fit>();

        // Scanner for reading file
        logger.debug("file=" + file.getName());
        Scanner scanner = null;
        int lineCount = 0;
        try {
            scanner = new Scanner(new FileReader(file));
        } catch (FileNotFoundException e) {
            logger.error("File not found " + file.getName(), e);
            return null;
        }
        scanner.useLocale(Locale.US); // for reading floats

        while (scanner.findInLine("#") != null) {
            scanner.nextLine();
            lineCount++;
        }

        SimpleDateFormat sdf = new SimpleDateFormat(ISO_TIME_FORMAT);
        Pattern pattern = Pattern.compile(ISO_TIME_PATTERN);
        boolean isAfterT2 = false;

        while (scanner.hasNext() && (scanner.findInLine("#") == null) && !isAfterT2) {
            String currentField = scanner.next();
            // Check that the first field of current line is a date
            if (pattern.matcher(currentField).matches()) {
                Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
                try {
                    calendar.setTime(sdf.parse(currentField));

                    if (calendar.getTime().after(t1) && calendar.getTime().before(t2)) {
                        // Store current measurement if valid and inside [t1,t2] 
                        Fit fit = new Fit();
                        fit.setSunTime(calendar);
                        fit.setSpeed(scanner.nextDouble());
                        fit.setSpeedError(scanner.nextDouble());
                        fit.setBeta(scanner.nextDouble());
                        fit.setBetaError(scanner.nextDouble());

                        fits.add(fit);
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    logger.error("invalid fields for line " + lineCount, e);
                    scanner.close();
                    return null;
                }

                // Stop reading if current measurement is after t2
                isAfterT2 = calendar.getTime().after(t2);
                lineCount++;

            } else {
                scanner.nextLine();
            }
        }
        logger.debug("Number of selected fits=" + fits.size());
        scanner.close();
        return fits;
    }

    /**
     * Read measurements belonging to [t1,t2] from CME fits catalog Columns : Sun time / Speed / HEEQ Longitude / HEEQ
     * latitude / Central PA / Northern PA / Southern PA
     */
    private static List<Fit> readCMEFits(Date t1, Date t2, String sourceName, final File file) {
        List<Fit> fits = new ArrayList<Fit>();

        // Scanner for reading file
        logger.debug("file=" + file.getName());
        Scanner scanner = null;
        int lineCount = 0;
        try {
            scanner = new Scanner(new FileReader(file));
        } catch (FileNotFoundException e) {
            logger.error("File not found " + file.getName(), e);
            return null;
        }
        scanner.useLocale(Locale.US); // for reading floats

        while (scanner.findInLine("#") != null) {
            scanner.nextLine();
            lineCount++;
        }

        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm");
        Pattern pattern = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}.*");
        boolean isAfterT2 = false;

        while (scanner.hasNext() && (scanner.findInLine("#") == null) && !isAfterT2) {
            String currentField = scanner.next();
            // Check that the first field of current line is a date
            if (pattern.matcher(currentField).matches()) {
                Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
                try {
                    calendar.setTime(sdf.parse(currentField));

                    // Store current measurement if valid and inside [t1,t2] 
                    if (calendar.getTime().after(t1) && calendar.getTime().before(t2)) {
                        Fit fit = new Fit();
                        fit.setSunTime(calendar);
                        fit.setSpeed(scanner.nextDouble());
                        fit.setLongitude(scanner.nextDouble());
                        fit.setLatitude(scanner.nextDouble());
                        fit.setCentralPa(scanner.nextDouble());
                        fit.setNorthPa(scanner.nextDouble());
                        fit.setSouthPa(scanner.nextDouble());

                        switch (sourceName) {
                            case "STEREO-A":
                            case "STEREO-B":
                                if (Probe.isBackward(sourceName,calendar.getTime())) {
                                    if (fit.getSouthPa() < 270 && fit.getNorthPa() > 270) {
                                        fits.add(fit);
                                    }
                                } else {
                                    if (fit.getSouthPa() > 90 && fit.getNorthPa() < 90) {
                                        fits.add(fit);
                                    }
                                }
                                break;
                            default:
                                fits.add(fit);
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    logger.error("invalid fields for line " + lineCount, e);
                    scanner.close();
                    return null;
                }
                // Stop reading if current measurement is after t2
                isAfterT2 = calendar.getTime().after(t2);
                lineCount++;

            } else {
                scanner.nextLine();
            }
        }
        logger.debug("Number of selected fits=" + fits.size());
        scanner.close();
        return fits;
    }

    /**
     * Read measurements belonging to [t1,t2] from LINKCAT catalog Columns : Sun time / Speed / CME HEEQ Longitude / CME
     * HEEQ latitude
     */
    private static List<Fit> readLinkcatCMEFits(Date t1, Date t2, String sourceName, final File file) {
        List<Fit> fits = new ArrayList<Fit>();

        // Scanner for reading file
        logger.debug("file=" + file.getName());
        Scanner scanner = null;
        int lineCount = 0;
        try {
            scanner = new Scanner(new FileReader(file));
        } catch (FileNotFoundException e) {
            logger.error("File not found " + file.getName(), e);
            return null;
        }
        scanner.useLocale(Locale.US); // for reading floats

        while (scanner.findInLine("#") != null) {
            scanner.nextLine();
            lineCount++;
        }

        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm");
        Pattern pattern = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}.*");
        boolean isAfterT2 = false;

        while (scanner.hasNext() && (scanner.findInLine("#") == null) && !isAfterT2) {
            String currentField = scanner.next();
            // Check that the first field of current line is a date
            if (pattern.matcher(currentField).matches()) {
                Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
                try {
                    calendar.setTime(sdf.parse(currentField));

                    // Store current measurement if valid and inside [t1,t2] 
                    if (calendar.getTime().after(t1) && calendar.getTime().before(t2)) {
                        Fit fit = new Fit();
                        fit.setSunTime(calendar);
                        fit.setSpeed(scanner.nextDouble());
                        fit.setLongitude(scanner.nextDouble());
                        fit.setLatitude(scanner.nextDouble());
                        // default values
                        switch (sourceName) {
                            case "STEREO-A":
                                fit.setCentralPa(90d);
                                fit.setNorthPa(60d);
                                fit.setSouthPa(120d);
                                break;
                            case "STEREO-B":
                                fit.setCentralPa(270d);
                                fit.setNorthPa(300d);
                                fit.setSouthPa(240d);
                                break;
                        }
                        fits.add(fit);
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    logger.error("invalid fields for line " + lineCount, e);
                    scanner.close();
                    return null;
                }
                // Stop reading if current measurement is after t2
                isAfterT2 = calendar.getTime().after(t2);
                lineCount++;

            } else {
                scanner.nextLine();
            }
        }
        logger.debug("Number of selected fits=" + fits.size());
        scanner.close();
        return fits;
    }

    /**
     * Read measurements belonging to [t1,t2] from Radial CMEs catalog Defined CME columns: Start time / Solar wind
     * speed / HAE Longitude / Element Naif ID / End time / φEnd(t')-φStart(t) / rSun
     *
     * @param extent
     */
    private static List<CmeRadial> readRadialCMEs(Date t1, Date t2, final File file, int elementId, double extent) {
        List<CmeRadial> radialCMEs = new ArrayList<CmeRadial>();

        // Scanner for reading file
        logger.debug("file=" + file.getName());
        Scanner scanner = null;
        int lineCount = 0;
        try {
            scanner = new Scanner(new FileReader(file));
        } catch (FileNotFoundException e) {
            logger.error("File not found " + file.getName(), e);
            return null;
        }
        scanner.useLocale(Locale.US); // for reading floats

        while (scanner.findInLine("#") != null) {
            scanner.nextLine();
            lineCount++;
        }

        SimpleDateFormat sdf = new SimpleDateFormat(ISO_TIME_FORMAT);
        Pattern pattern = Pattern.compile(ISO_TIME_PATTERN);
        Calendar endTime = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
        String currentDate = null;
        CmeRadial cmeRadial = null;
        boolean isAfterT2 = false;

        while (scanner.hasNext() && (scanner.findInLine("#") == null) && !isAfterT2) {
            String dateField = scanner.next();
            if (!dateField.equals(currentDate)) {
                // Check that the first field of current line is a date
                if (pattern.matcher(dateField).matches()) {
                    Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
                    try {
                        calendar.setTime(sdf.parse(dateField));
                        // Store current measurement if valid and inside [t1,t2] 
                        if (calendar.getTime().after(t1) && calendar.getTime().before(t2)) {
                            cmeRadial = new CmeRadial();
                            cmeRadial.setStartTime(calendar);
                            cmeRadial.setSpeed((Double) scanner.nextDouble());
                            cmeRadial.setLongitude((Double) scanner.nextDouble());
                            cmeRadial.setExtent(extent);
                            currentDate = dateField;
                            // add in catalog
                            radialCMEs.add(cmeRadial);
                        }
                    } catch (Exception e) {
                        logger.error("invalid fields for line " + lineCount, e);
                        scanner.close();
                        return null;
                    }
                    // Stop reading if current measurement is after t2
                    isAfterT2 = calendar.getTime().after(t2);
                    lineCount++;

                } else {
                    scanner.nextLine();
                    //logger.debug("next line");
                }
            } else {
                // skip the CME start fields 
                scanner.next();
                scanner.next();
            }
            if (cmeRadial != null) {
                int naifId = scanner.nextInt();
                if (elementId == 0 || elementId == naifId) {
                    String target = scanner.next();
                    dateField = scanner.next();
                    if (pattern.matcher(dateField).matches()) {
                        try {
                            endTime.setTime(sdf.parse(dateField));

                        } catch (Exception e) {
                            logger.error("invalid date field for line " + lineCount, e);
                            scanner.close();
                            return null;
                        }
                        CmeEnd cmeEnd = new CmeEnd();
                        cmeEnd.setId(naifId);
                        cmeEnd.setName(target);
                        cmeEnd.setEndTime(endTime.getTimeInMillis());
                        cmeEnd.setLongSep((Double) scanner.nextDouble());
                        cmeEnd.setDistEnd((Double) scanner.nextDouble());
                        cmeEnd.setImpacted(ToolHelper.isInsideCME(cmeEnd.getLongSep(), extent / 2.0));
                        // add in CME list
                        cmeRadial.getCmeEnds().add(cmeEnd);
                    }
                } else {
                    // skip the line
                    scanner.nextLine();
                }
            }
        }
        logger.debug("Number of selected Radial CMEs=" + radialCMEs.size());
        scanner.close();
        return radialCMEs;
    }

    /**
     * Read measurements belonging to [t1,t2] from Linkcat CMEs catalog Defined CME columns: Start time / Solar wind
     * speed / HEEQ Longitude / Element Naif ID / End time / φEnd(t')-φStart(t) / rSun
     */
    private static List<CmeRadial> readLinkcatCMEs(Date t1, Date t2, final File file, double extent) {
        List<CmeRadial> radialCMEs = new ArrayList<CmeRadial>();

        // Scanner for reading file
        logger.debug("file=" + file.getName());
        Scanner scanner = null;
        int lineCount = 0;
        try {
            scanner = new Scanner(new FileReader(file));
        } catch (FileNotFoundException e) {
            logger.error("File not found " + file.getName(), e);
            return null;
        }
        scanner.useLocale(Locale.US); // for reading floats

        while (scanner.findInLine("#") != null) {
            scanner.nextLine();
            lineCount++;
        }

        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm");
        Pattern pattern = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}.*");
        Calendar endTime = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
        String currentDate = null;
        CmeRadial cmeRadial = null;
        boolean isAfterT2 = false;

        while (scanner.hasNext() && (scanner.findInLine("#") == null) && !isAfterT2) {
            String dateField = scanner.next();
            if (!dateField.equals(currentDate)) {
                // Check that the first field of current line is a date
                if (pattern.matcher(dateField).matches()) {
                    Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
                    try {
                        calendar.setTime(sdf.parse(dateField));
                        // Store current measurement if valid and inside [t1,t2] 
                        if (calendar.getTime().after(t1) && calendar.getTime().before(t2)) {
                            cmeRadial = new CmeRadial();
                            cmeRadial.setStartTime(calendar);
                            cmeRadial.setSpeed((Double) scanner.nextDouble());
                            cmeRadial.setLongitude((Double) scanner.nextDouble());
                            cmeRadial.setLatitude((Double) scanner.nextDouble());
                            currentDate = dateField;
                            // add in catalog
                            radialCMEs.add(cmeRadial);
                        }

                    } catch (Exception e) {
                        logger.error("invalid fields for line " + lineCount, e);
                        scanner.close();
                        return null;
                    }
                    // Stop reading if current measurement is after t2
                    isAfterT2 = calendar.getTime().after(t2);
                    lineCount++;

                } else {
                    scanner.nextLine();
                    //logger.debug("next line");
                }
            } else {
                // skip the CME start fields 
                scanner.next();
                scanner.next();
                scanner.next();
            }

            if (cmeRadial != null) {
                // skip the flare fields 
                scanner.next();
                scanner.next();
                scanner.next();
                scanner.next();
                scanner.next();
                //int naifId = scanner.nextInt();
                //if (elementId == 0 || elementId == naifId) {
                //String target = scanner.next();
                dateField = scanner.next();
                if (pattern.matcher(dateField).matches()) {
                    try {
                        endTime.setTime(sdf.parse(dateField));

                    } catch (Exception e) {
                        logger.error("invalid date field for line " + lineCount, e);
                        scanner.close();
                        return null;
                    }
                    CmeEnd cmeEnd = new CmeEnd();
                    //cmeEnd.setId(naifId);
                    cmeEnd.setName(scanner.next());
                    cmeEnd.setEndTime(endTime.getTimeInMillis());
                    cmeEnd.setLongSep((Double) scanner.nextDouble() - cmeRadial.getLongitude());
                    scanner.next();
                    cmeEnd.setDistEnd((Double) scanner.nextDouble());
                    cmeEnd.setImpacted(ToolHelper.isInsideCME(cmeEnd.getLongSep(), extent / 2.0));
                    // add in CME list
                    cmeRadial.getCmeEnds().add(cmeEnd);
                }
            } else {
                // skip the line
                scanner.nextLine();
            }
        }
        logger.debug("Number of selected Radial CMEs=" + radialCMEs.size());
        scanner.close();
        return radialCMEs;
    }

    /**
     * Read measurements belonging to [t1,t2] from Corotation CIRs catalog Defined CIR columns: Start time / Solar wind
     * speed / HAE Longitude / Element Naif ID / End time CIR / φEnd(t')-φStart(t) / rSun
     *
     * @param t1
     * @param t2
     * @param file
     * @param elementId
     * @param spread
     * @return
     */
    private static List<CirCorotation> readCorotationCIRs(Date t1, Date t2, final File file, int elementId, double spread) {
        List<CirCorotation> corotCIRs = new ArrayList<CirCorotation>();

        // Scanner for reading file
        logger.debug("file=" + file.getName());
        Scanner scanner = null;
        int lineCount = 0;
        try {
            scanner = new Scanner(new FileReader(file));
        } catch (FileNotFoundException e) {
            logger.error("File not found " + file.getName(), e);
            return null;
        }
        scanner.useLocale(Locale.US); // for reading floats

        while (scanner.findInLine("#") != null) {
            scanner.nextLine();
            lineCount++;
        }

        SimpleDateFormat sdf = new SimpleDateFormat(ISO_TIME_FORMAT);
        Pattern pattern = Pattern.compile(ISO_TIME_PATTERN);
        Calendar endTime = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
        String currentDate = null;
        CirCorotation cirCorot = null;
        boolean isAfterT2 = false;

        while (scanner.hasNext() && (scanner.findInLine("#") == null) && !isAfterT2) {
            String dateField = scanner.next();
            if (!dateField.equals(currentDate)) {
                // Check that the first field of current line is a date
                if (pattern.matcher(dateField).matches()) {
                    Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
                    try {
                        calendar.setTime(sdf.parse(dateField));
                        // Store current measurement if valid and inside [t1,t2] 
                        if (calendar.getTime().after(t1) && calendar.getTime().before(t2)) {
                            cirCorot = new CirCorotation();
                            cirCorot.setStartTime(calendar);
                            cirCorot.setSpeed((Double) scanner.nextDouble());
                            cirCorot.setLongitude((Double) scanner.nextDouble());
                            cirCorot.setSpread(spread);
                            currentDate = dateField;
                            // add in catalog
                            corotCIRs.add(cirCorot);
                        }
                    } catch (Exception e) {
                        logger.error("invalid fields for line " + lineCount, e);
                        scanner.close();
                        return null;
                    }
                    // Stop reading if current measurement is after t2
                    isAfterT2 = calendar.getTime().after(t2);
                    lineCount++;

                } else {
                    scanner.nextLine();
                    //logger.debug("next line");
                }
            } else {
                // skip the CIR start fields 
                scanner.next();
                scanner.next();
            }
            if (cirCorot != null) {
                int naifId = scanner.nextInt();
                if (elementId == 0 || elementId == naifId) {
                    String target = scanner.next();
                    dateField = scanner.next();
                    if (pattern.matcher(dateField).matches()) {
                        try {
                            endTime.setTime(sdf.parse(dateField));

                        } catch (Exception e) {
                            logger.error("invalid date field for line " + lineCount, e);
                            scanner.close();
                            return null;
                        }
                        CirEnd cirEnd = new CirEnd();
                        cirEnd.setId(naifId);
                        cirEnd.setName(target);
                        cirEnd.setEndTime(endTime.getTimeInMillis());
                        cirEnd.setLongSep((Double) scanner.nextDouble());
                        cirEnd.setDistEnd((Double) scanner.nextDouble());
                        cirEnd.setImpacted(true);
                        // add in CIR list
                        cirCorot.getCirEnds().add(cirEnd);
                    }
                } else {
                    // skip the line
                    scanner.nextLine();
                }
            }
        }
        logger.debug("Number of selected Corotation CIRs=" + corotCIRs.size());
        scanner.close();
        return corotCIRs;
    }

    /**
     * Read end footpoints (1 RS) of field line from start footpoints (2.5 RS)
     *
     * @param file
     * @param startFieldLineFP
     * @return
     */
    private List<Footpoint> readEndFieldLineFP(File file, List<Footpoint> startFieldLineFP) {
        List<Footpoint> fpts = new ArrayList<Footpoint>();

        // Scanner for reading file
        logger.debug("file=" + file.getName());
        Scanner scanner = null;
        try {
            scanner = new Scanner(new FileReader(file));
        } catch (FileNotFoundException e) {
            logger.error("File not found " + file.getName(), e);
            return null;
        }
        scanner.useLocale(Locale.US); // for reading floats

        while (scanner.findInLine("#") != null) {
            scanner.nextLine();
        }

        // define dimension and step
        int nbLon = 72;
        int nbLat = 36;
        float step = 2.6f;

        try {
            // read longitude and latitude map grid (in degrees)
            float[] lon = new float[nbLon];
            for (int i = 0; i < nbLon; i++) {
                lon[i] = scanner.nextFloat();
                //logger.info("lon=" + lon[i]);
            }
            float[] lat = new float[nbLat];
            for (int i = 0; i < nbLat; i++) {
                lat[i] = scanner.nextFloat();
                //logger.info("lat=" + lat[i]);
            }

            List<Integer> iLon = new ArrayList<Integer>();
            List<Integer> jLat = new ArrayList<Integer>();
            int iLonPrev = -1;
            int jLatPrev = -1;
            for (Footpoint fpStart : startFieldLineFP) {
//				logger.info("fpStart lon=" + fpStart.getLon());
//				logger.info("fpStart lat=" + fpStart.getLat());
                // get indices of field line start footpoint
                int iLongitude = -1;
                for (int i = 0; i < nbLon; i++) {
                    if (fpStart.getLon() < lon[i] + step && fpStart.getLon() >= lon[i] - step) {
                        iLongitude = i;
                        break;
                    }
                }
                int jLatitude = -1;
                for (int i = 0; i < nbLat; i++) {
                    if (fpStart.getLat() < lat[i] + step && fpStart.getLat() >= lat[i] - step) {
                        jLatitude = i;
                        break;
                    }
                }

                if (iLongitude > -1 && jLatitude > -1
                        && (iLongitude != iLonPrev || jLatitude != jLatPrev)) {
                    // add indices without doublons
                    //logger.info("iLon=" + iLongitude);
                    iLon.add(iLongitude);
                    //logger.info("jLat=" + jLatitude);
                    jLat.add(jLatitude);
                    // add footpoint to the result list
                    fpts.add(new Footpoint());
                }
                iLonPrev = iLongitude;
                jLatPrev = jLatitude;
            }

            // retrieve end footpoints longitudes from start footpoints indices
            for (int j = 0; j < nbLat; j++) {
                for (int i = 0; i < nbLon; i++) {
                    float fValue = (float) CoordConverter.normalizeLongitude(scanner.nextFloat());
                    for (int iFp = 0; iFp < fpts.size(); iFp++) {
                        if (i == iLon.get(iFp) && j == jLat.get(iFp)) {
                            fpts.get(iFp).setLon(fValue);
                            //logger.info("lon=" + fValue);
                            break;
                        }
                    }
                }
            }
            // retrieve end footpoints latitudes from start footpoints indices
            for (int j = 0; j < nbLat; j++) {
                for (int i = 0; i < nbLon; i++) {
                    float fValue = (float) CoordConverter.normalizeLatitude(scanner.nextFloat());
                    for (int iFp = 0; iFp < fpts.size(); iFp++) {
                        if (i == iLon.get(iFp) && j == jLat.get(iFp)) {
                            fpts.get(iFp).setLat(fValue);
                            //logger.info("lat=" + fValue);
                            break;
                        }
                    }
                }
            }

        } catch (Exception e) {
            e.printStackTrace();
            scanner.close();
            return null;
        }
        logger.info("footpoint list size : " + fpts.size());
        scanner.close();
        return fpts;
    }

}
