/* ----------------------------------------
 * Origin  : Akkodis
 * Project : CDPP Propagation Tool 
 * ----------------------------------------
 * History 
 * Version : V2.12
 * DM-ID : redmine#11106 : 15/03/2025 : LBE : creation
 * Version : V2.13
 * DM-ID : redmine#12153 : 09/09/2025 : LBE : added WIND radio spectro
 * ----------------------------------------
 */
package fr.tlse.irap.server.serviceDAO.impl;

import java.io.File;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.TimeZone;

import org.apache.log4j.Logger;

import fr.tlse.irap.server.ApplicationContextHolder;
import fr.tlse.irap.server.serviceDAO.ICdfDAO;
import fr.tlse.irap.server.serviceDAO.constant.CONFIG;
import fr.tlse.irap.spectro.Spectro;
import fr.tlse.irap.vplot.SolarSpeed;
import gsfc.nssdc.cdf.CDF;
import gsfc.nssdc.cdf.CDFConstants;
import gsfc.nssdc.cdf.CDFException;
import gsfc.nssdc.cdf.Variable;
import gsfc.nssdc.cdf.util.CDFTT2000;
import gsfc.nssdc.cdf.util.CDFUtils;
import gsfc.nssdc.cdf.util.Epoch;
import gsfc.nssdc.cdf.util.Epoch16;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.DecimalFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Calendar;
import java.util.GregorianCalendar;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.io.FileUtils;
import org.w3c.dom.Document;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

public class CdfDAOImpl implements ICdfDAO {

    private static final Logger logger = Logger.getLogger(CdfDAOImpl.class);
    private static CONFIG prop;
    private static final String ISO_TIME_PATTERN = "\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.*";
    private static final String ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss";
    private static final String CDAWEB_TIME_FORMAT = "yyyyMMdd'T'HHmmss'Z'";
    private static final String SPDF_TIME_FORMAT = "yyyyMMdd";
    private static final String SOARREQ_TIME_FORMAT = "yyyyMMdd";
    private static final String CDF_TIME_FORMAT = "yyyyMMddHHmmss";

    public CdfDAOImpl() {
        logger.info("CDF DAO IMPL LOADED");
        // set default timezone UTC
        TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
        prop = (CONFIG) ApplicationContextHolder.getContext().getBean("confData");
    }

    public CdfDAOImpl(CONFIG lProp) {
        // set default timezone UTC
        TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
        prop = lProp;
    }

    /**
     * Load list of V solar wind from catalog for a S/C or planet between 2
     * dates
     */
    @Override
    public List<SolarSpeed> loadSolarSpeeds(Date pT1, Date pT2, String startName) {
        //logger.info("SPEED PATH=" + prop.getDATA_PATH() + "/" + prop.getSPEED_DIR());
        logger.debug("loadSolarSpeeds CDF with t1=" + pT1 + " t2=" + pT2 + " startName=" + startName);

        List<SolarSpeed> readSpeeds = null;
        try {
            if (startName.startsWith("SOLO")) {
                readSpeeds = readSWSSpeedFromSOAR(pT1, pT2);
            }
        } catch (IOException lIoe) {
            lIoe.printStackTrace(System.err);
            logger.debug("Error reading speed from CDF for " + startName, lIoe);
        } catch (ParseException lEx) {
            lEx.printStackTrace(System.err);
            logger.debug("Error reading speed from CDF for " + startName, lEx);
        }

        return readSpeeds;
    }

    public List<SolarSpeed> readSWSSpeedFromSOAR(Date pT1, Date pT2) throws IOException, ParseException {
        //logger.info("SPEED PATH=" + prop.getDATA_PATH() + "/" + prop.getSPEED_DIR());
        logger.debug("loadSolarSpeeds SOAR CDF with t1=" + pT1 + " t2=" + pT2);

        SimpleDateFormat lSdf = new SimpleDateFormat(SOARREQ_TIME_FORMAT);
        lSdf.setTimeZone(TimeZone.getTimeZone("UTC"));

        List<SolarSpeed> readSpeeds = new ArrayList<SolarSpeed>();

        //get file(s) from SOAR
        //Tap request https://www.cosmos.esa.int/web/soar/guide-to-using-tap
        //https://soar.esac.esa.int/soar-sl-tap/data?retrieval_type=LATEST_PRODUCT&QUERY=SELECT+filepath,filename+FROM+soar.v_sc_repository_file+WHERE+data_item_id+LIKE+'solo_L2_swa-pas-grnd-mom_20220415'
        //If multiple files (more than 1 day request) files in tar in science\swa\L2\2022 use a flat untar ?
        //loop through all cdf to download
        Calendar lCal = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
        lCal.setTime(pT1);
        lCal.set(Calendar.HOUR, 0);
        lCal.set(Calendar.MINUTE, 0);
        lCal.set(Calendar.SECOND, 0);
        long lTimeInMilli = lCal.getTimeInMillis();
        for (long lTime = lCal.getTimeInMillis(); lTime <= pT2.getTime(); lTime += 86400 * 1000) {
            Date lDate = new Date(lTime);
            //get cdf
            String lReq = "https://soar.esac.esa.int/soar-sl-tap/data?retrieval_type=LATEST_PRODUCT"
                    + "&QUERY=SELECT+filepath,filename+FROM+soar.v_sc_repository_file+"
                    + "WHERE+data_item_id+LIKE+'solo_L2_swa-pas-grnd-mom_" + lSdf.format(lDate) + "'";
            System.out.println("REq solo cdf from SOAR: " + lReq);

            File lCdfFile = File.createTempFile("soar", ".cdf");
            FileUtils.copyURLToFile(new URL(lReq), lCdfFile);
            try {
                //open CDF
                CDF lCdf = CDF.open(lCdfFile.getAbsolutePath());
                //read CDF
                Variable lVarTime = null;
                Variable lVarFreq = null;
                Variable lVarData = null;
//[2025-03-26 13:28:30] [info] ** Found variable: 0 Epoch
//[2025-03-26 13:28:30] [info] ** Found variable: 1 Half_interval
//[2025-03-26 13:28:30] [info] ** Found variable: 2 SCET
//[2025-03-26 13:28:30] [info] ** Found variable: 3 Info
//[2025-03-26 13:28:30] [info] ** Found variable: 4 unrecovered_count
//[2025-03-26 13:28:30] [info] ** Found variable: 5 total_count
//[2025-03-26 13:28:30] [info] ** Found variable: 6 quality_factor
//[2025-03-26 13:28:30] [info] ** Found variable: 7 N
//[2025-03-26 13:28:30] [info] ** Found variable: 8 V_SRF
//[2025-03-26 13:28:30] [info] ** Found variable: 9 V_RTN
//[2025-03-26 13:28:30] [info] ** Found variable: 10 P_SRF
//[2025-03-26 13:28:30] [info] ** Found variable: 11 P_RTN
//[2025-03-26 13:28:30] [info] ** Found variable: 12 TxTyTz_SRF
//[2025-03-26 13:28:30] [info] ** Found variable: 13 TxTyTz_RTN
//[2025-03-26 13:28:30] [info] ** Found variable: 14 T
//[2025-03-26 13:28:30] [info] ** Found variable: 15 V_SOLO_RTN

                /* Find first time column and get min and max time */
                int i = 0;
                for (Object lObj : lCdf.getVariables()) {
                    if (lObj instanceof gsfc.nssdc.cdf.Variable) {
                        gsfc.nssdc.cdf.Variable lVar = (gsfc.nssdc.cdf.Variable) lObj;
                        System.out.println("** Found variable: " + i + " " + lVar.toString());
                        if (CDFUtils.isEpochDataType(lVar.getDataType())) {
                            lVarTime = lVar;
////                    Object lStartObj = lVar.getScalarData(0);
////                    Object lEndObj = lVar.getScalarData(lVar.getNumWrittenRecords() - 1);
////                    lStartTime = getDateFromTimeObject(lStartObj, lVar.getDataType());
////                    lStopTime = getDateFromTimeObject(lEndObj, lVar.getDataType());
                        } else if (lVar.toString().startsWith("V_RTN")) {
                            lVarData = lVar;
                        }
                    }
                    i++;
                }
                //nbVariables = cdfFile.getVariables().size();
                //LOGGER.info("Opened CDF file "+file.getName()+": "+startTime+" to "+stopTime);

                //check all var were found
                if (lVarTime == null || lVarData == null) {
                    return readSpeeds;
                }

                logger.debug("Found all variables in cdf nbRecs: " + lVarTime.getNumWrittenRecords() + " type=" + lVarData.getRecord(0));

                //getFreqs from first record
                //List<Float[]> lNrjRanges = new ArrayList<>();
//                float[] lNrjs = (float[]) lVarFreq.getRecord(0);
//                for (i = 0; i < lNrjs.length - 1; i++) {
//                    pNrjRanges.add(new Float[]{lNrjs[i], lNrjs[i + 1]});
//                }
                //fill readSpeeds
                /*Read Variable from CdfFile and record Time and Param values*/
                for (i = 0; i < lVarTime.getNumWrittenRecords(); i += 15) {//downsample data
                    SolarSpeed lSp = new SolarSpeed();
                    //set record time
                    Date lRecDate = getDateFromTimeObject(lVarTime.getRecord(i), lVarTime.getDataType());
                    Calendar lRecCal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
                    lRecCal.setTime(lRecDate);
                    lSp.setEventTime(lRecCal);
                    //set value
                    float[] lValues = (float[]) lVarData.getRecord(i);
                    Float lSpeed = (float) Math.sqrt(lValues[0] * lValues[0] + lValues[1] * lValues[1] + lValues[2] * lValues[2]);
                    lSp.setSpeed(lSpeed);
                    //add record to result array
                    readSpeeds.add(lSp);
                }
                if (lCdf != null) {
                    lCdf.close();
                }
            } catch (CDFException ex) {
                throw new IOException(ex.fillInStackTrace());
            }
        }

        return readSpeeds;
    }

    /**
     * Load radio spectro from catalog for a S/C or planet between 2 dates
     *
     * @return list with at 0: energies, 1: spectro values
     */
    @Override
    public List<Object> loadSpectro(Date pT1, Date pT2, String startName) {
        //logger.info("SPEED PATH=" + prop.getDATA_PATH() + "/" + prop.getSPEED_DIR());
        logger.debug("loadSpectro CDF with t1=" + pT1 + " t2=" + pT2 + " startName=" + startName);

        //init with an empty wrapper to avoid returning null
        List<Object> retValues = new ArrayList<>();
        List<Spectro> lSpectroValues = new ArrayList<>();
        List<Float[]> lNrjRanges = new ArrayList<>();

        try {
            String lHighDataset = "STA_L3_WAV_HFR";
            String lLowDataset = "STA_L3_WAV_LFR";
            String lHighVar = "PSD_SFU";
            String lLowVar = "PSD_SFU";
            if (startName.startsWith("PSP")) {
                lHighDataset = "PSP_FLD_L3_RFS_HFR";
                lLowDataset = "PSP_FLD_L3_RFS_LFR";
                lHighVar = "psp_fld_l3_rfs_hfr_PSD_SFU";
                lLowVar = "psp_fld_l3_rfs_lfr_PSD_SFU";
            }
            if (startName.startsWith("STEREO-A")) {
                lHighDataset = "STA_L3_WAV_HFR";
                lLowDataset = "STA_L3_WAV_LFR";
                lHighVar = "PSD_SFU";
                lLowVar = "PSD_SFU";
            }
            if (startName.startsWith("WIND")) {
                lHighDataset = "wind/waves";
                lLowDataset = "wind/waves";
                lHighVar = "rad2";
                lLowVar = "rad1";
            }

            //temp date to get small data 
            //Date lDateEnd = new Date(pT1.getTime()+60*60*1000); // 1h
            boolean lRet = readMergeSpectro(pT1, pT2,
                    lHighDataset, lLowDataset,
                    lHighVar, lLowVar,
                    lSpectroValues, lNrjRanges);

        } catch (IOException lIoe) {
            lIoe.printStackTrace(System.err);
            //throw new SOAPException(lIoe.fillInStackTrace());
        } catch (ParserConfigurationException ex) {
            ex.printStackTrace(System.err);
        } catch (SAXException ex) {
            ex.printStackTrace(System.err);
        } catch (ParseException ex) {
            ex.printStackTrace(System.err);
        } catch (CDFException ex) {
            ex.printStackTrace(System.err);
        }

        logger.debug("Found " + lNrjRanges.size() + " nrjs and " + lSpectroValues.size() + " values");

        retValues.add(lNrjRanges);
        retValues.add(lSpectroValues);
        return retValues;
    }

    //
    /**
     * return true if data found;
     */
    public boolean readMergeSpectro(Date pT1, Date pT2,
            String pHFRDataset, String pLFRDataset,
            String pHFRVariable, String pLFRVariable,
            List<Spectro> pReadSpectro, List<Float[]> pNrjRanges)
            throws IOException, ParserConfigurationException, SAXException, ParseException, CDFException {
        //logger.info("SPEED PATH=" + prop.getDATA_PATH() + "/" + prop.getSPEED_DIR());
        logger.debug("Load spectra CDF with t1=" + pT1 + " t2=" + pT2);

        //read High frequency values
        List<Spectro> lHFRSpectro = new ArrayList<>();
        List<Float[]> lHFRNrjRanges = new ArrayList<>();
        if (pHFRDataset.startsWith("wind")) {
            //TODO: unit V²/Hz ???
            readSpectroFromSpdf(pT1, pT2, pHFRDataset, pHFRVariable, "PSD_V2_Z", lHFRSpectro, lHFRNrjRanges);
        } else {
            readSpectroFromCDAWeb(pT1, pT2, pHFRDataset, pHFRVariable, lHFRSpectro, lHFRNrjRanges);
        }
        //read low frequency values
        List<Spectro> lLFRSpectro = new ArrayList<>();
        List<Float[]> lLFRNrjRanges = new ArrayList<>();
        if (pHFRDataset.startsWith("wind")) {
            readSpectroFromSpdf(pT1, pT2, pLFRDataset, pLFRVariable, "PSD_V2_Z", lLFRSpectro, lLFRNrjRanges);
        } else {
            readSpectroFromCDAWeb(pT1, pT2, pLFRDataset, pLFRVariable, lLFRSpectro, lLFRNrjRanges);
        }

        if (!lLFRNrjRanges.isEmpty()) {
            pNrjRanges.add(new Float[]{lLFRNrjRanges.get(0)[0], lLFRNrjRanges.get(0)[0]});
            pNrjRanges.addAll(lLFRNrjRanges);
            pNrjRanges.add(new Float[]{lLFRNrjRanges.get(lLFRNrjRanges.size() - 1)[1], lHFRNrjRanges.get(0)[0]});
            pNrjRanges.addAll(lHFRNrjRanges);
        }
        logger.debug("Got Nrjs L/H: " + lLFRNrjRanges.size() + " / " + lHFRNrjRanges.size() + " = " + pNrjRanges.size());

        if (pHFRDataset.startsWith("wind")) {//only for wind
            lHFRSpectro = interpolatespectra(lHFRSpectro, lLFRSpectro);
        }

//        for (Float[] lNrj : pNrjRanges) {
//            System.out.println("[" + lNrj[0] + "/" + lNrj[1] + "]");
//        }

        //For each record in low freq values, add high freq values
        logger.debug("Got spectro records L/H: " + lLFRSpectro.size() + "/" + lHFRSpectro.size());
        for (int i = 0; i < lLFRSpectro.size(); i++) {
            if (lHFRSpectro.size() > i) {
                for (Float lVal : lHFRSpectro.get(i).getFlux()) {
                    lLFRSpectro.get(i).getFlux().add(lVal);
                }
            } else {
                //not enough data in High freq, fill in with NaN
                for (Float lVal : lHFRSpectro.get(0).getFlux()) {
                    lLFRSpectro.get(i).getFlux().add(Float.NaN);
                }
            }
            //if(i<10)logger.debug("Copy record "+i+" of "+lLFRSpectro.get(i).getFlux().size()+" values. "+lLFRSpectro.get(i).getFlux());
        }
        pReadSpectro.addAll(lLFRSpectro);

        return true;
    }

    /**
     * Interpolate en spectro on times of another
     *
     * @param pSpectro1 Spectro to interpolate
     * @param pSpectro2 Spectro for time reference
     * @return lSpectro1 on times of Spectro2
     */
    public List<Spectro> interpolatespectra(List<Spectro> pSpectro1, List<Spectro> pSpectro2) {
        //show first times to check
//        for (int i = 0; i < 10; i++) {
//            System.out.println("times (" + i + ") = " + pSpectro1.get(i).getEventTime().getTime() + " - " + pSpectro2.get(i).getEventTime().getTime());
//        }
//        System.out.println("Low Nrj Data:");
//        DecimalFormat lDf = new DecimalFormat("0.#E0");
//        for (int i = 0; i < 5; i++) {
//            String lStr = "val (" + i + ") = ";
//            for (int j = 0; j < pSpectro1.get(i).getFlux().size(); j++) {
//                lStr += " " + lDf.format(pSpectro1.get(i).getFlux().get(j));
//            }
//            System.out.println(lStr);
//        }

        //generate low nrj values at same rate as high nrj
        int lNbFloatValues = pSpectro1.get(0).getFlux().size();
        //prepare spectro grid in time for interpolation optimisation
        double[] lSp1Times = new double[pSpectro1.size()];
        List<double[]> lSp1Values = new ArrayList<>();
        for (int i = 0; i < lNbFloatValues; i++) {
            double[] lValues = new double[pSpectro1.size()];
            for (int j = 0; j < pSpectro1.size(); j++) {
                if (i == 0) {
                    lSp1Times[j] = pSpectro1.get(j).getEventTime().getTimeInMillis();
                }
                lValues[j] = pSpectro1.get(j).getFlux().get(i);
            }
            lSp1Values.add(lValues);
        }

        List<Spectro> lNewSpectro = new ArrayList<>();
        for (Spectro lSp : pSpectro2) {
            Spectro lNewSp = new Spectro();
            List<Float> lNewFloats = new ArrayList<>();
            for (int i = 0; i < lNbFloatValues; i++) {
                //get interpolated value
                double lNewValue = interpolate(lSp1Times, lSp1Values.get(i), lSp.getEventTime().getTimeInMillis());
                lNewFloats.add((float) lNewValue);
            }
            lNewSp.setEventTime(lSp.getEventTime());
            lNewSp.setFlux(lNewFloats);
            lNewSpectro.add(lNewSp);
        }

//        for (int i = 0; i < 10; i++) {
//            System.out.println("New times (" + i + ") = " + lNewSpectro.get(i).getEventTime().getTime() + " - " + pSpectro2.get(i).getEventTime().getTime());
//        }

        return lNewSpectro;

    }

    /**
     * return true if data found;
     */
    public boolean readSpectroFromCDAWeb(Date pT1, Date pT2,
            String pDataset, String pVariable,
            List<Spectro> pReadSpectro, List<Float[]> pNrjRanges)
            throws IOException, ParserConfigurationException, SAXException, ParseException, CDFException {
        //logger.info("SPEED PATH=" + prop.getDATA_PATH() + "/" + prop.getSPEED_DIR());
        logger.debug("load spectra CDF with t1=" + pT1 + " t2=" + pT2);

        SimpleDateFormat lSdf = new SimpleDateFormat(CDAWEB_TIME_FORMAT);
        lSdf.setTimeZone(TimeZone.getTimeZone("UTC"));

        //get file(s) from cdaweb
        //https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/datasets/PSP_FLD_L3_RFS_HFR/data/20190601T000000Z,20190603T000000Z/psp_fld_l3_rfs_hfr_PSD_SFU?format=cdf
        //get cdf
        String lHReq = "https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/datasets/"
                + pDataset + "/data/" + lSdf.format(pT1) + "," + lSdf.format(pT2)
                + "/" + pVariable + "?format=cdf";

        boolean lIsData = readSpectroInCDF(Arrays.asList(lHReq), pT1, pT2, pVariable, pReadSpectro, pNrjRanges);

        return lIsData;
    }

    /**
     * Reads CDF spectro files at
     * https://spdf.gsfc.nasa.gov/pub/data/wind/waves/rad1_l2/ or
     * https://spdf.gsfc.nasa.gov/pub/data/wind/waves/rad2_l2/ return true if
     * data found;
     */
    public boolean readSpectroFromSpdf(Date pT1, Date pT2,
            String pUrlDir, String pRadStr, String pVariable,
            List<Spectro> pReadSpectro, List<Float[]> pNrjRanges)
            throws IOException, ParserConfigurationException, SAXException, ParseException, CDFException {
        //logger.info("SPEED PATH=" + prop.getDATA_PATH() + "/" + prop.getSPEED_DIR());
        logger.debug("load spectra CDF with t1=" + pT1 + " t2=" + pT2);

        SimpleDateFormat lSdf = new SimpleDateFormat(SPDF_TIME_FORMAT);
        lSdf.setTimeZone(TimeZone.getTimeZone("UTC"));

        //get file(s) from spdf
        //https://spdf.gsfc.nasa.gov/pub/data/wind/waves/rad1_l2/2025/wi_l2_wav_rad1_20250101_v01.cdf
        //https://spdf.gsfc.nasa.gov/pub/data/wind/waves/rad2_l2/2025/wi_l2_wav_rad2_20250101_v01.cdf
        //generate one url for each day
        List<String> lCdfStrUrls = new ArrayList<>();
        for (long lTime = pT1.getTime(); lTime <= pT2.getTime(); lTime += 86400000l) {
            String lStrDate = lSdf.format(new Date(lTime));
            String lYear = lStrDate.substring(0, 4);
            String lFile = "wi_l2_wav_" + pRadStr + "_" + lStrDate + "_v01.cdf";
            String lUrl = "https://spdf.gsfc.nasa.gov/pub/data/" + pUrlDir + "/" + pRadStr + "_l2/" + lYear + "/" + lFile;
            System.out.println("Built url: " + lUrl);
            lCdfStrUrls.add(lUrl);
        }

        boolean lIsData = readSpectroInCDF(lCdfStrUrls, pT1, pT2, pVariable, pReadSpectro, pNrjRanges);

        return lIsData;
    }

    /**
     * Read Spectro data in a list of CDF between 2 dates
     *
     * @param pStrUrls List or cdf urls
     * @param pT1 Begin time
     * @param pT2 End time
     * @param pVariable Spectro variable to fetch in file
     * @param pReadSpectro Array where to put data
     * @param pNrjRanges Array where to put energies list
     * @return false if no data found
     * @throws CDFException
     * @throws MalformedURLException
     * @throws IOException
     * @throws ParserConfigurationException
     * @throws SAXException
     * @throws ParseException
     */
    private boolean readSpectroInCDF(List<String> pStrUrls, Date pT1, Date pT2, String pVariable,
            List<Spectro> pReadSpectro, List<Float[]> pNrjRanges)
            throws CDFException, MalformedURLException, IOException, ParserConfigurationException, SAXException, ParseException {
        for (String lHReq : pStrUrls) {
            File lCdfFile = File.createTempFile("tmpnasa", ".cdf");
            CDF lCdf = null;

            String lFileUrl = lHReq;
            //if(url is not cdf, get file in returned XML file
            if (lHReq != null && !lHReq.endsWith(".cdf")) {
                //System.out.println("***** request radio spectro url " + lHReq + " to " + lCdfFile.getAbsolutePath());
                FileUtils.copyURLToFile(new URL(lHReq), lCdfFile);

                DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
                DocumentBuilder db = dbf.newDocumentBuilder();
                Document doc = db.parse(lHReq);
                doc.getDocumentElement().normalize();
                NodeList lNdl = doc.getElementsByTagName("Name");
                if (lNdl.getLength() == 0) {
                    logger.debug("No data found for cdaweb request " + lHReq);
                    return false;
                }
                lFileUrl = lNdl.item(0).getTextContent();
            }
            logger.debug("***** getting cdf file " + lFileUrl + " to " + lCdfFile.getAbsolutePath());
            FileUtils.copyURLToFile(new URL(lFileUrl), lCdfFile);

            //open CDF
            lCdf = CDF.open(lCdfFile.getAbsolutePath());
            //read CDF

            Variable lVarTime = null;
            Variable lVarFreq = null;
            Variable lVarData = null;

            /* Find first time column and get min and max time */
            int i = 0;
            for (Object lObj : lCdf.getVariables()) {
                if (lObj instanceof gsfc.nssdc.cdf.Variable) {
                    gsfc.nssdc.cdf.Variable lVar = (gsfc.nssdc.cdf.Variable) lObj;
                    //System.out.println("** Found variable: " + i + " " + lVar.toString());
                    if (CDFUtils.isEpochDataType(lVar.getDataType())) {
                        lVarTime = lVar;
//                    Object lStartObj = lVar.getScalarData(0);
//                    Object lEndObj = lVar.getScalarData(lVar.getNumWrittenRecords() - 1);
//                    lStartTime = getDateFromTimeObject(lStartObj, lVar.getDataType());
//                    lStopTime = getDateFromTimeObject(lEndObj, lVar.getDataType());
                    } else if (lVar.toString().toLowerCase().contains("freq")) {
                        lVarFreq = lVar;
                    } else if (lVar.toString().startsWith(pVariable)) {
                        lVarData = lVar;
                    }
                }
                i++;
            }
            //nbVariables = cdfFile.getVariables().size();
            //LOGGER.info("Opened CDF file "+file.getName()+": "+startTime+" to "+stopTime);

            //check all var were found
            if (lVarTime == null || lVarFreq == null || lVarData == null) {
                logger.error("Found variables time/freq/var: " + lVarTime + " " + lVarFreq + " " + lVarData);
                return false;
            }

            logger.debug("Found all variables in cdf nbRecs: " + lVarTime.getNumWrittenRecords());

            //getFreqs from first record
            //List<Float[]> lNrjRanges = new ArrayList<>();
            if (pNrjRanges.isEmpty()) {//read only once energies
                float[] lNrjs = (float[]) lVarFreq.getRecord(0);
//                float lPadValue = ((Float) lVarFreq.get);
                for (i = 0; i < lNrjs.length - 1; i++) {
//                    if(lNrjs[i + 1]<0) {
//                        System.out.println("Got negative nrj: "+lNrjs[i + 1]+" "+lPadValue+" "+(lNrjs[i + 1] == lPadValue));
//                    }
//                    if (lNrjs[i + 1] == lPadValue ) break;//for wind, part of nrj values are unused and negative.
                    if(lNrjs[i + 1]<0) break;
                    pNrjRanges.add(new Float[]{lNrjs[i], lNrjs[i + 1]});
                }
            }

            /*Read Variable from CdfFile and record Time and Param values*/
            for (i = 0; i < lVarTime.getNumWrittenRecords(); i++) {
                Spectro lSp = new Spectro();
                //set record time
                Date lTime = getDateFromTimeObject(lVarTime.getRecord(i), lVarTime.getDataType());

                if (lTime != null && (lTime.before(pT1) || lTime.after(pT2))) {
                    continue;//record is out of time range, skip to next one
                }

                Calendar lCal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
                lCal.setTime(lTime);
                lSp.setEventTime(lCal);
                //set values for each nrj
                float[] lValues = (float[]) lVarData.getRecord(i);
                List<Float> lObjValues = new ArrayList<>();
                for (int j = 0; j < lValues.length && j < pNrjRanges.size(); j++) {
                    float lVal = lValues[j];
//                    //for (float lVal : lValues) {
//                    //TODO: TEMP REMOVE when wind/waves will be calibrated
//                    if (lVal < 1) {
//                        lVal = (float) (lVal * 1e19);
//                    }
                    lObjValues.add(lVal);
                }
                lSp.setFlux(lObjValues);
                //add record to result array
                pReadSpectro.add(lSp);
            }
            if (lCdf != null) {
                lCdf.close();
            }
        }

        return true;
    }

    private Date getDateFromTimeObject(Object pObj, long pDataType) throws ParseException {
        Date lDate = null;
        SimpleDateFormat lSdf = new SimpleDateFormat(CDF_TIME_FORMAT);
        //logger.debug("Test date " + pDataType);
        if (pDataType == CDFConstants.CDF_EPOCH) {
            lDate = lSdf.parse(Epoch.encode2((Double) pObj));
            //logger.debug("Found epoch date " + lDate);
        }
        if (pDataType == CDFConstants.CDF_EPOCH16) {
            lDate = lSdf.parse(Epoch16.encode2(pObj));
            //logger.debug("Found epoch16 date " + lDate);
        }
        if (pDataType == CDFConstants.CDF_TIME_TT2000) {
            lDate = lSdf.parse(CDFTT2000.encode((Long) pObj, 2));
            //logger.debug("Found tt2000 date " + lDate);
        }
        return lDate;
    }

    public static double interpolate(double x[], double y[], double pNewX) {
        double xi = pNewX;
        double newY;

        //System.out.println("interp: Compare value "+pNewX+" to range ["+x[0]+"-"+x[x.length - 1]+"]");
        // If xi is less than the smallest value in x, return the first y value
        if (xi <= x[0]) {
            newY = y[0];
        } // If xi is greater than the largest value in x, return the last y value
        else if (xi >= x[x.length - 1]) {
            newY = y[y.length - 1];
        } // Otherwise, perform linear interpolation
        else {
            int index = 0;
            while (index < x.length - 1 && x[index + 1] < xi) {
                index++;
            }
            double x1 = x[index], x2 = x[index + 1];
            double y1 = y[index], y2 = y[index + 1];

            // Perform linear interpolation
            newY = y1 + (y2 - y1) * (xi - x1) / (x2 - x1);
        }
        return newY;
    }

}
