schain-jc Commit - r1208:05917071ce48

Fix Madrigal HDF5 reader (v2 & v3)

Juan C. Espinoza -

r1208:05917071ce48

parent child

Context file:

r1208:05917071ce48

Collapse all files

schainpy/model/io/jroIO_base.py +3 -3

             '''
             Created on Jul 2, 2014
             @author: roj-idl71
             '''
             import os
             import sys
             import glob
             import time
             import numpy
             import fnmatch
             import inspect
             import time
             import datetime
             import traceback
             import zmq
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
             from schainpy.utils import log
             import schainpy.admin
             LOCALTIME = True
             def isNumber(cad):
                 """
                 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                 Excepciones:
                     Si un determinado string no puede ser convertido a numero
                 Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                 Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                 """
                 try:
                     float(cad)
                     return True
                 except:
                     return False
             def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
                 """
                 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startUTSeconds      :    fecha inicial del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                     endUTSeconds        :    fecha final del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 try:
                     fp = open(filename, 'rb')
                 except IOError:
                     print("The file %s can't be opened" % (filename))
                     return 0
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 if not(sts):
                     print("Skipping the file %s because it has not a valid header" % (filename))
                     return 0
                 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
                     return 0
                 return 1
             def isTimeInRange(thisTime, startTime, endTime):
                 if endTime >= startTime:
                     if (thisTime < startTime) or (thisTime > endTime):
                         return 0
                     return 1
                 else:
                     if (thisTime < startTime) and (thisTime > endTime):
                         return 0
                     return 1
             def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                     startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                     endTime            :    tiempo final del rango seleccionado en formato datetime.time
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 try:
                     fp = open(filename, 'rb')
                 except IOError:
                     print("The file %s can't be opened" % (filename))
                     return None
                 firstBasicHeaderObj = BasicHeader(LOCALTIME)
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
                 processingHeaderObj = ProcessingHeader()
                 lastBasicHeaderObj = BasicHeader(LOCALTIME)
                 sts = firstBasicHeaderObj.read(fp)
                 if not(sts):
                     print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
                     return None
                 if not systemHeaderObj.read(fp):
                     return None
                 if not radarControllerHeaderObj.read(fp):
                     return None
                 if not processingHeaderObj.read(fp):
                     return None
                 filesize = os.path.getsize(filename)
                 offset = processingHeaderObj.blockSize + 24  # header size
                 if filesize <= offset:
                     print("[Reading] %s: This file has not enough data" % filename)
                     return None
                 fp.seek(-offset, 2)
                 sts = lastBasicHeaderObj.read(fp)
                 fp.close()
                 thisDatetime = lastBasicHeaderObj.datatime
                 thisTime_last_block = thisDatetime.time()
                 thisDatetime = firstBasicHeaderObj.datatime
                 thisDate = thisDatetime.date()
                 thisTime_first_block = thisDatetime.time()
                 # General case
                 #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
                 #-----------o----------------------------o-----------
                 #       startTime                     endTime
                 if endTime >= startTime:
                     if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
                         return None
                     return thisDatetime
                 # If endTime < startTime then endTime belongs to the next day
                 #<<<<<<<<<<<o                            o>>>>>>>>>>>
                 #-----------o----------------------------o-----------
                 #        endTime                    startTime
                 if (thisDate == startDate) and (thisTime_last_block < startTime):
                     return None
                 if (thisDate == endDate) and (thisTime_first_block > endTime):
                     return None
                 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
                     return None
                 return thisDatetime
             def isFolderInDateRange(folder, startDate=None, endDate=None):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     folder            :    nombre completo del directorio.
                                            Su formato deberia ser "/path_root/?YYYYDDD"
                                             siendo:
                                                 YYYY    :    Anio         (ejemplo 2015)
                                                 DDD     :    Dia del anio (ejemplo 305)
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el directorio no tiene el formato adecuado
                 """
                 basename = os.path.basename(folder)
                 if not isRadarFolder(basename):
                     print("The folder %s has not the rigth format" % folder)
                     return 0
                 if startDate and endDate:
                     thisDate = getDateFromRadarFolder(basename)
                     if thisDate < startDate:
                         return 0
                     if thisDate > endDate:
                         return 0
                 return 1
             def isFileInDateRange(filename, startDate=None, endDate=None):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                                            Su formato deberia ser "?YYYYDDDsss"
                                             siendo:
                                                 YYYY    :    Anio         (ejemplo 2015)
                                                 DDD     :    Dia del anio (ejemplo 305)
                                                 sss     :    set
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no tiene el formato adecuado
                 """
                 basename = os.path.basename(filename)
                 if not isRadarFile(basename):
                     print("The filename %s has not the rigth format" % filename)
                     return 0
                 if startDate and endDate:
                     thisDate = getDateFromRadarFile(basename)
                     if thisDate < startDate:
                         return 0
                     if thisDate > endDate:
                         return 0
                 return 1
             def getFileFromSet(path, ext, set):
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     try:
                         year = int(thisFile[1:5])
                         doy = int(thisFile[5:8])
                     except:
                         continue
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 myfile = fnmatch.filter(
                     validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
                 if len(myfile) != 0:
                     return myfile[0]
                 else:
                     filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
                     print('the filename %s does not exist' % filename)
                     print('...going to the last file: ')
                 if validFilelist:
                     validFilelist = sorted(validFilelist, key=str.lower)
                     return validFilelist[-1]
                 return None
             def getlastFileFromPath(path, ext):
                 """
                 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
                 al final de la depuracion devuelve el ultimo file de la lista que quedo.
                 Input:
                     fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                     ext         :    extension de los files contenidos en una carpeta
                 Return:
                     El ultimo file de una determinada carpeta, no se considera el path.
                 """
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     year = thisFile[1:5]
                     if not isNumber(year):
                         continue
                     doy = thisFile[5:8]
                     if not isNumber(doy):
                         continue
                     year = int(year)
                     doy = int(doy)
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 if validFilelist:
                     validFilelist = sorted(validFilelist, key=str.lower)
                     return validFilelist[-1]
                 return None
             def checkForRealPath(path, foldercounter, year, doy, set, ext):
                 """
                 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
                 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
                 el path exacto de un determinado file.
                 Example    :
                     nombre correcto del file es  .../.../D2009307/P2009307367.ext
                     Entonces la funcion prueba con las siguientes combinaciones
                         .../.../y2009307367.ext
                         .../.../Y2009307367.ext
                         .../.../x2009307/y2009307367.ext
                         .../.../x2009307/Y2009307367.ext
                         .../.../X2009307/y2009307367.ext
                         .../.../X2009307/Y2009307367.ext
                     siendo para este caso, la ultima combinacion de letras, identica al file buscado
                 Return:
                     Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
                     caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
                     para el filename
                 """
                 fullfilename = None
                 find_flag = False
                 filename = None
                 prefixDirList = [None, 'd', 'D']
                 if ext.lower() == ".r":  # voltage
                     prefixFileList = ['d', 'D']
                 elif ext.lower() == ".pdata":  # spectra
                     prefixFileList = ['p', 'P']
                 else:
                     return None, filename
                 # barrido por las combinaciones posibles
                 for prefixDir in prefixDirList:
                     thispath = path
                     if prefixDir != None:
                         # formo el nombre del directorio xYYYYDDD (x=d o x=D)
                         if foldercounter == 0:
                             thispath = os.path.join(path, "%s%04d%03d" %
                                                     (prefixDir, year, doy))
                         else:
                             thispath = os.path.join(path, "%s%04d%03d_%02d" % (
                                 prefixDir, year, doy, foldercounter))
                     for prefixFile in prefixFileList:  # barrido por las dos combinaciones posibles de "D"
                         # formo el nombre del file xYYYYDDDSSS.ext
                         filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
                         fullfilename = os.path.join(
                             thispath, filename)  # formo el path completo
                         if os.path.exists(fullfilename):  # verifico que exista
                             find_flag = True
                             break
                     if find_flag:
                         break
                 if not(find_flag):
                     return None, filename
                 return fullfilename, filename
             def isRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
                     doy = int(folder[5:8])
                 except:
                     return 0
                 return 1
             def isRadarFile(file):
                 try:
                     year = int(file[1:5])
                     doy = int(file[5:8])
                     set = int(file[8:11])
                 except:
                     return 0
                 return 1
             def getDateFromRadarFile(file):
                 try:
                     year = int(file[1:5])
                     doy = int(file[5:8])
                     set = int(file[8:11])
                 except:
                     return None
                 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
                 return thisDate
             def getDateFromRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
                     doy = int(folder[5:8])
                 except:
                     return None
                 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
                 return thisDate
             class JRODataIO:
                 c = 3E8
                 isConfig = False
                 basicHeaderObj = None
                 systemHeaderObj = None
                 radarControllerHeaderObj = None
                 processingHeaderObj = None
                 dtype = None
                 pathList = []
                 filenameList = []
                 filename = None
                 ext = None
                 flagIsNewFile = 1
                 flagDiscontinuousBlock = 0
                 flagIsNewBlock = 0
                 fp = None
                 firstHeaderSize = 0
                 basicHeaderSize = 24
                 versionFile = 1103
                 fileSize = None
             #     ippSeconds = None
                 fileSizeByHeader = None
                 fileIndex = None
                 profileIndex = None
                 blockIndex = None
                 nTotalBlocks = None
                 maxTimeStep = 30
                 lastUTTime = None
                 datablock = None
                 dataOut = None
                 blocksize = None
                 getByBlock = False
                 def __init__(self):
                     raise NotImplementedError
                 def run(self):
                     raise NotImplementedError
                 def getDtypeWidth(self):
                     dtype_index = get_dtype_index(self.dtype)
                     dtype_width = get_dtype_width(dtype_index)
                     return dtype_width
                 def getAllowedArgs(self):
                     if hasattr(self, '__attrs__'):
                         return self.__attrs__
                     else:
                         return inspect.getargspec(self.run).args
             class JRODataReader(JRODataIO):
                 online = 0
                 realtime = 0
                 nReadBlocks = 0
                 delay = 10  # number of seconds waiting a new file
                 nTries = 3  # quantity tries
                 nFiles = 3  # number of files for searching
                 path = None
                 foldercounter = 0
                 flagNoMoreFiles = 0
                 datetimeList = []
                 __isFirstTimeOnline = 1
                 __printInfo = True
                 profileIndex = None
                 nTxs = 1
                 txIndex = None
                 # Added--------------------
                 selBlocksize = None
                 selBlocktime = None
                 def __init__(self):
                     """
                     This class is used to find data files
                     Example:
                         reader = JRODataReader()
                         fileList = reader.findDataFiles()
                     """
                     pass
                 def createObjByDefault(self):
                     """
                     """
                     raise NotImplementedError
                 def getBlockDimension(self):
                     raise NotImplementedError
                 def searchFilesOffLine(self,
                                        path,
                                        startDate=None,
                                        endDate=None,
                                        startTime=datetime.time(0, 0, 0),
                                        endTime=datetime.time(23, 59, 59),
                                        set=None,
                                        expLabel='',
                                        ext='.r',
                                        cursor=None,
                                        skip=None,
                                        walk=True):
                     self.filenameList = []
                     self.datetimeList = []
                     pathList = []
                     dateList, pathList = self.findDatafiles(
                         path, startDate, endDate, expLabel, ext, walk, include_path=True)
                     if dateList == []:
                         return [], []
                     if len(dateList) > 1:
                         print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)))
                     else:
                         print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]))
                     filenameList = []
                     datetimeList = []
                     for thisPath in pathList:
                         fileList = glob.glob1(thisPath, "*%s" % ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath, file)
                             if not isFileInDateRange(filename, startDate, endDate):
                                 continue
                             thisDatetime = isFileInTimeRange(
                                 filename, startDate, endDate, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if cursor is not None and skip is not None:
                         filenameList = filenameList[cursor * skip:cursor * skip + skip]
                         datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
                     if not(filenameList):
                         print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path))
                         return [], []
                     print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime))
                     # for i in range(len(filenameList)):
                     #     print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
                     """
                     Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
                     devuelve el archivo encontrado ademas de otros datos.
                     Input:
                         path           :    carpeta donde estan contenidos los files que contiene data
                         expLabel        :     Nombre del subexperimento (subfolder)
                         ext              :    extension de los files
                         walk        :    Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
                     Return:
                         directory   :    eL directorio donde esta el file encontrado
                         filename    :    el ultimo file de una determinada carpeta
                         year        :    el anho
                         doy         :    el numero de dia del anho
                         set         :    el set del archivo
                     """
                     if not os.path.isdir(path):
                         return None, None, None, None, None, None
                     dirList = []
                     if not walk:
                         fullpath = path
                         foldercounter = 0
                     else:
                         # Filtra solo los directorios
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path, thisPath)):
                                 continue
                             if not isRadarFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None, None, None, None, None
                         dirList = sorted(dirList, key=str.lower)
                         doypath = dirList[-1]
                         foldercounter = int(doypath.split('_')[1]) if len(
                             doypath.split('_')) > 1 else 0
                         fullpath = os.path.join(path, doypath, expLabel)
                     print("[Reading] %s folder was found: " % (fullpath))
                     if set == None:
                         filename = getlastFileFromPath(fullpath, ext)
                     else:
                         filename = getFileFromSet(fullpath, ext, set)
                     if not(filename):
                         return None, None, None, None, None, None
                     print("[Reading] %s file was found" % (filename))
                     if not(self.__verifyFile(os.path.join(fullpath, filename))):
                         return None, None, None, None, None, None
                     year = int(filename[1:5])
                     doy = int(filename[5:8])
                     set = int(filename[8:11])
                     return fullpath, foldercounter, filename, year, doy, set
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
             #                 print "[Reading] No more Files"
                             return 0
                         filename = self.filenameList[idFile]
                         if not(self.__verifyFile(filename)):
                             continue
                         fileSize = os.path.getsize(filename)
                         fp = open(filename, 'rb')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fp = fp
             #         print "[Reading] Setting the file: %s"%self.filename
                     return 1
                 def __setNextFileOnline(self):
                     """
                     Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
                     no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
                     siguientes.
                     Affected:
                         self.flagIsNewFile
                         self.filename
                         self.fileSize
                         self.fp
                         self.set
                         self.flagNoMoreFiles
                     Return:
 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
 : si el file fue abierto con exito y esta listo a ser leido
                     Excepciones:
                         Si un determinado file no puede ser abierto
                     """
                     nFiles = 0
                     fileOk_flag = False
                     firstTime_flag = True
                     self.set += 1
                     if self.set > 999:
                         self.set = 0
                         self.foldercounter += 1
                     # busca el 1er file disponible
                     fullfilename, filename = checkForRealPath(
                         self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
                     if fullfilename:
                         if self.__verifyFile(fullfilename, False):
                             fileOk_flag = True
                     # si no encuentra un file entonces espera y vuelve a buscar
                     if not(fileOk_flag):
                         # busco en los siguientes self.nFiles+1 files posibles
                         for nFiles in range(self.nFiles + 1):
                             if firstTime_flag:  # si es la 1era vez entonces hace el for self.nTries veces
                                 tries = self.nTries
                             else:
                                 tries = 1  # si no es la 1era vez entonces solo lo hace una vez
                             for nTries in range(tries):
                                 if firstTime_flag:
                                     print("\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1))
                                     sleep(self.delay)
                                 else:
                                     print("\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext))
                                 fullfilename, filename = checkForRealPath(
                                     self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
                                 if fullfilename:
                                     if self.__verifyFile(fullfilename):
                                         fileOk_flag = True
                                         break
                             if fileOk_flag:
                                 break
                             firstTime_flag = False
                             log.warning('Skipping the file {} due to this file doesn\'t exist'.format(filename))
                             self.set += 1
                             # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
                             if nFiles == (self.nFiles - 1):
                                 self.set = 0
                                 self.doy += 1
                                 self.foldercounter = 0
                     if fileOk_flag:
                         self.fileSize = os.path.getsize(fullfilename)
                         self.filename = fullfilename
                         self.flagIsNewFile = 1
                         if self.fp != None:
                             self.fp.close()
                         self.fp = open(fullfilename, 'rb')
                         self.flagNoMoreFiles = 0
             #             print '[Reading] Setting the file: %s' % fullfilename
                     else:
                         self.fileSize = 0
                         self.filename = None
                         self.flagIsNewFile = 0
                         self.fp = None
                         self.flagNoMoreFiles = 1
             #             print '[Reading] No more files to read'
                     return fileOk_flag
                 def setNextFile(self):
                     if self.fp != None:
                         self.fp.close()
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         self.dataOut.error = 'No more files to read'
                         return 0
                     if self.verbose:
                         print('[Reading] Setting the file: %s' % self.filename)
                     self.__readFirstHeader()
                     self.nReadBlocks = 0
                     return 1
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range(self.nTries):
                         self.fp.close()
                         self.fp = open(self.filename, 'rb')
                         self.fp.seek(currentPointer)
                         self.fileSize = os.path.getsize(self.filename)
                         currentSize = self.fileSize - currentPointer
                         if (currentSize >= neededSize):
                             self.basicHeaderObj.read(self.fp)
                             return 1
                         if self.fileSize == self.fileSizeByHeader:
                             #                self.flagEoF = True
                             return 0
                         print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
                         sleep(self.delay)
                     return 0
                 def waitDataBlock(self, pointer_location):
                     currentPointer = pointer_location
                     neededSize = self.processingHeaderObj.blockSize  # + self.basicHeaderSize
                     for nTries in range(self.nTries):
                         self.fp.close()
                         self.fp = open(self.filename, 'rb')
                         self.fp.seek(currentPointer)
                         self.fileSize = os.path.getsize(self.filename)
                         currentSize = self.fileSize - currentPointer
                         if (currentSize >= neededSize):
                             return 1
                         print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
                         sleep(self.delay)
                     return 0
                 def __jumpToLastBlock(self):
                     if not(self.__isFirstTimeOnline):
                         return
                     csize = self.fileSize - self.fp.tell()
                     blocksize = self.processingHeaderObj.blockSize
                     # salta el primer bloque de datos
                     if csize > self.processingHeaderObj.blockSize:
                         self.fp.seek(self.fp.tell() + blocksize)
                     else:
                         return
                     csize = self.fileSize - self.fp.tell()
                     neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     while True:
                         if self.fp.tell() < self.fileSize:
                             self.fp.seek(self.fp.tell() + neededsize)
                         else:
                             self.fp.seek(self.fp.tell() - neededsize)
                             break
             #        csize = self.fileSize - self.fp.tell()
             #        neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
             #        factor = int(csize/neededsize)
             #        if factor > 0:
             #            self.fp.seek(self.fp.tell() + factor*neededsize)
                     self.flagIsNewFile = 0
                     self.__isFirstTimeOnline = 0
                 def __setNewBlock(self):
                     # if self.server is None:
                     if self.fp == None:
                         return 0
             #         if self.online:
             #             self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if self.realtime:
                         self.flagDiscontinuousBlock = 1
                         if not(self.setNextFile()):
                             return 0
                         else:
                             return 1
                     # if self.server is None:
                     currentSize = self.fileSize - self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     if (currentSize >= neededSize):
                         self.basicHeaderObj.read(self.fp)
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     # else:
                     #     self.basicHeaderObj.read(self.zHeader)
                     #     self.lastUTTime = self.basicHeaderObj.utc
                     #     return 1
                     if self.__waitNewBlock():
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     # if self.server is None:
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.basicHeaderObj.utc - self.lastUTTime
                     self.lastUTTime = self.basicHeaderObj.utc
                     self.flagDiscontinuousBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagDiscontinuousBlock = 1
                     return 1
                 def readNextBlock(self):
                     # Skip block out of startTime and endTime
                     while True:
                         if not(self.__setNewBlock()):
                             self.dataOut.error = 'No more files to read'
                             return 0
                         if not(self.readBlock()):
                             return 0
                         self.getBasicHeader()
                         if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
                             print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
                                                                                   self.processingHeaderObj.dataBlocksPerFile,
                                                                                   self.dataOut.datatime.ctime()))
                             continue
                         break
                     if self.verbose:
                         print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
                                                                    self.processingHeaderObj.dataBlocksPerFile,
                                                                    self.dataOut.datatime.ctime()))
                     return 1
                 def __readFirstHeader(self):
                     self.basicHeaderObj.read(self.fp)
                     self.systemHeaderObj.read(self.fp)
                     self.radarControllerHeaderObj.read(self.fp)
                     self.processingHeaderObj.read(self.fp)
                     self.firstHeaderSize = self.basicHeaderObj.size
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
                                                PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
                     else:
                         raise ValueError('Data type was not defined')
                     self.dtype = datatype_str
                     #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
                         self.firstHeaderSize + self.basicHeaderSize * \
                         (self.processingHeaderObj.dataBlocksPerFile - 1)
                     #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
                     #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                     self.getBlockDimension()
                 def __verifyFile(self, filename, msgFlag=True):
                     msg = None
                     try:
                         fp = open(filename, 'rb')
                     except IOError:
                         if msgFlag:
                             print("[Reading] File %s can't be opened" % (filename))
                         return False
                     currentPosition = fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
                     if neededSize == 0:
                         basicHeaderObj = BasicHeader(LOCALTIME)
                         systemHeaderObj = SystemHeader()
                         radarControllerHeaderObj = RadarControllerHeader()
                         processingHeaderObj = ProcessingHeader()
                         if not(basicHeaderObj.read(fp)):
                             fp.close()
                             return False
                         if not(systemHeaderObj.read(fp)):
                             fp.close()
                             return False
                         if not(radarControllerHeaderObj.read(fp)):
                             fp.close()
                             return False
                         if not(processingHeaderObj.read(fp)):
                             fp.close()
                             return False
                         neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
                     else:
                         msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
                     fp.close()
                     fileSize = os.path.getsize(filename)
                     currentSize = fileSize - currentPosition
                     if currentSize < neededSize:
                         if msgFlag and (msg != None):
                             print(msg)
                         return False
                     return True
                 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
                     path_empty = True
                     dateList = []
                     pathList = []
                     multi_path = path.split(',')
                     if not walk:
                         for single_path in multi_path:
                             if not os.path.isdir(single_path):
                                 continue
                             fileList = glob.glob1(single_path, "*" + ext)
                             if not fileList:
                                 continue
                             path_empty = False
                             fileList.sort()
                             for thisFile in fileList:
                                 if not os.path.isfile(os.path.join(single_path, thisFile)):
                                     continue
                                 if not isRadarFile(thisFile):
                                     continue
                                 if not isFileInDateRange(thisFile, startDate, endDate):
                                     continue
                                 thisDate = getDateFromRadarFile(thisFile)
                                 if thisDate in dateList:
                                     continue
                                 dateList.append(thisDate)
                                 pathList.append(single_path)
                     else:
                         for single_path in multi_path:
                             if not os.path.isdir(single_path):
                                 continue
                             dirList = []
                             for thisPath in os.listdir(single_path):
                                 if not os.path.isdir(os.path.join(single_path, thisPath)):
                                     continue
                                 if not isRadarFolder(thisPath):
                                     continue
                                 if not isFolderInDateRange(thisPath, startDate, endDate):
                                     continue
                                 dirList.append(thisPath)
                             if not dirList:
                                 continue
                             dirList.sort()
                             for thisDir in dirList:
                                 datapath = os.path.join(single_path, thisDir, expLabel)
                                 fileList = glob.glob1(datapath, "*" + ext)
                                 if not fileList:
                                     continue
                                 path_empty = False
                                 thisDate = getDateFromRadarFolder(thisDir)
                                 pathList.append(datapath)
                                 dateList.append(thisDate)
                     dateList.sort()
                     if walk:
                         pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
                     else:
                         pattern_path = multi_path[0]
                     if path_empty:
                         print("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
                     else:
                         if not dateList:
                             print("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
                     if include_path:
                         return dateList, pathList
                     return dateList
                 def setup(self,
                           path=None,
                           startDate=None,
                           endDate=None,
                           startTime=datetime.time(0, 0, 0),
                           endTime=datetime.time(23, 59, 59),
                           set=None,
                           expLabel="",
                           ext=None,
                           online=False,
                           delay=60,
                           walk=True,
                           getblock=False,
                           nTxs=1,
                           realtime=False,
                           blocksize=None,
                           blocktime=None,
                           skip=None,
                           cursor=None,
                           warnings=True,
                           verbose=True,
                           server=None,
                           format=None,
                           oneDDict=None,
                           twoDDict=None,
-                          ind2DList=None):
+                          independentParam=None):
                     if server is not None:
                         if 'tcp://' in server:
                             address = server
                         else:
                             address = 'ipc:///tmp/%s' % server
                         self.server = address
                         self.context = zmq.Context()
                         self.receiver = self.context.socket(zmq.PULL)
                         self.receiver.connect(self.server)
                         time.sleep(0.5)
                         print('[Starting] ReceiverData from {}'.format(self.server))
                     else:
                         self.server = None
                         if path == None:
                             raise ValueError("[Reading] The path is not valid")
                         if ext == None:
                             ext = self.ext
                         if online:
                             print("[Reading] Searching files in online mode...")
                             for nTries in range(self.nTries):
                                 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
                                     path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
                                 if fullpath:
                                     break
                                 print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1))
                                 sleep(self.delay)
                             if not(fullpath):
                                 self.dataOut.error = 'There isn\'t any valid file in {}'.format(path)
                                 return
                             self.year = year
                             self.doy = doy
                             self.set = set - 1
                             self.path = path
                             self.foldercounter = foldercounter
                             last_set = None
                         else:
                             print("[Reading] Searching files in offline mode ...")
                             pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                              startTime=startTime, endTime=endTime,
                                                                              set=set, expLabel=expLabel, ext=ext,
                                                                              walk=walk, cursor=cursor,
                                                                              skip=skip)
                             if not(pathList):
                                 self.fileIndex = -1
                                 self.pathList = []
                                 self.filenameList = []
                                 return
                             self.fileIndex = -1
                             self.pathList = pathList
                             self.filenameList = filenameList
                             file_name = os.path.basename(filenameList[-1])
                             basename, ext = os.path.splitext(file_name)
                             last_set = int(basename[-3:])
                         self.online = online
                         self.realtime = realtime
                         self.delay = delay
                         ext = ext.lower()
                         self.ext = ext
                         self.getByBlock = getblock
                         self.nTxs = nTxs
                         self.startTime = startTime
                         self.endTime = endTime
                         self.endDate = endDate
                         self.startDate = startDate
                         # Added-----------------
                         self.selBlocksize = blocksize
                         self.selBlocktime = blocktime
                         # Verbose-----------
                         self.verbose = verbose
                         self.warnings = warnings
                         if not(self.setNextFile()):
                             if (startDate != None) and (endDate != None):
                                 print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
                             elif startDate != None:
                                 print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
                             else:
                                 print("[Reading] No files")
                                 self.fileIndex = -1
                                 self.pathList = []
                                 self.filenameList = []
                                 return
                 #         self.getBasicHeader()
                         if last_set != None:
                             self.dataOut.last_block = last_set * \
                                 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
                     return
                 def getBasicHeader(self):
                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
 . + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                     self.dataOut.timeZone = self.basicHeaderObj.timeZone
                     self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
                     self.dataOut.errorCount = self.basicHeaderObj.errorCount
                     self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
                     self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
             #         self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
                 def getFirstHeader(self):
                     raise NotImplementedError
                 def getData(self):
                     raise NotImplementedError
                 def hasNotDataInBuffer(self):
                     raise NotImplementedError
                 def readBlock(self):
                     raise NotImplementedError
                 def isEndProcess(self):
                     return self.flagNoMoreFiles
                 def printReadBlocks(self):
                     print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
                 def printTotalBlocks(self):
                     print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
                 def printNumberOfBlock(self):
                     'SPAM!'
             #         if self.flagIsNewBlock:
             #             print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
             #                                                       self.processingHeaderObj.dataBlocksPerFile,
             #                                                       self.dataOut.datatime.ctime())
                 def printInfo(self):
                     if self.__printInfo == False:
                         return
                     self.basicHeaderObj.printInfo()
                     self.systemHeaderObj.printInfo()
                     self.radarControllerHeaderObj.printInfo()
                     self.processingHeaderObj.printInfo()
                     self.__printInfo = False
                 def run(self,
                         path=None,
                         startDate=None,
                         endDate=None,
                         startTime=datetime.time(0, 0, 0),
                         endTime=datetime.time(23, 59, 59),
                         set=None,
                         expLabel="",
                         ext=None,
                         online=False,
                         delay=60,
                         walk=True,
                         getblock=False,
                         nTxs=1,
                         realtime=False,
                         blocksize=None,
                         blocktime=None,
                         skip=None,
                         cursor=None,
                         warnings=True,
                         server=None,
                         verbose=True,
                         format=None,
                         oneDDict=None,
                         twoDDict=None,
-                        ind2DList=None, **kwargs):
+                        independentParam=None, **kwargs):
                     if not(self.isConfig):
                         self.setup(path=path,
                                    startDate=startDate,
                                    endDate=endDate,
                                    startTime=startTime,
                                    endTime=endTime,
                                    set=set,
                                    expLabel=expLabel,
                                    ext=ext,
                                    online=online,
                                    delay=delay,
                                    walk=walk,
                                    getblock=getblock,
                                    nTxs=nTxs,
                                    realtime=realtime,
                                    blocksize=blocksize,
                                    blocktime=blocktime,
                                    skip=skip,
                                    cursor=cursor,
                                    warnings=warnings,
                                    server=server,
                                    verbose=verbose,
                                    format=format,
                                    oneDDict=oneDDict,
                                    twoDDict=twoDDict,
-                                   ind2DList=ind2DList)
+                                   independentParam=independentParam)
                         self.isConfig = True
                     if server is None:
                         self.getData()
                     else:
                         self.getFromServer()
             class JRODataWriter(JRODataIO):
                 """
                 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 blockIndex = 0
                 path = None
                 setFile = None
                 profilesPerBlock = None
                 blocksPerFile = None
                 nWriteBlocks = 0
                 fileDate = None
                 def __init__(self, dataOut=None):
                     raise NotImplementedError
                 def hasAllDataInBuffer(self):
                     raise NotImplementedError
                 def setBlockDimension(self):
                     raise NotImplementedError
                 def writeBlock(self):
                     raise NotImplementedError
                 def putData(self):
                     raise NotImplementedError
                 def getProcessFlags(self):
                     processFlags = 0
                     dtype_index = get_dtype_index(self.dtype)
                     procflag_dtype = get_procflag_dtype(dtype_index)
                     processFlags += procflag_dtype
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code is not None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nCohInt > 1:
                         processFlags += PROCFLAG.COHERENT_INTEGRATION
                     if self.dataOut.type == "Spectra":
                         if self.dataOut.nIncohInt > 1:
                             processFlags += PROCFLAG.INCOHERENT_INTEGRATION
                         if self.dataOut.data_dc is not None:
                             processFlags += PROCFLAG.SAVE_CHANNELS_DC
                         if self.dataOut.flagShiftFFT:
                             processFlags += PROCFLAG.SHIFT_FFT_DATA
                     return processFlags
                 def setBasicHeader(self):
                     self.basicHeaderObj.size = self.basicHeaderSize  # bytes
                     self.basicHeaderObj.version = self.versionFile
                     self.basicHeaderObj.dataBlock = self.nTotalBlocks
                     utc = numpy.floor(self.dataOut.utctime)
                     milisecond = (self.dataOut.utctime - utc) * 1000.0
                     self.basicHeaderObj.utc = utc
                     self.basicHeaderObj.miliSecond = milisecond
                     self.basicHeaderObj.timeZone = self.dataOut.timeZone
                     self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
                     self.basicHeaderObj.errorCount = self.dataOut.errorCount
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj self.
                     Return:
                         None
                     """
                     raise NotImplementedError
                 def __writeFirstHeader(self):
                     """
                     Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
                     Affected:
                         __dataType
                     Return:
                         None
                     """
             #        CALCULAR PARAMETROS
                     sizeLongHeader = self.systemHeaderObj.size + \
                         self.radarControllerHeaderObj.size + self.processingHeaderObj.size
                     self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
                     self.basicHeaderObj.write(self.fp)
                     self.systemHeaderObj.write(self.fp)
                     self.radarControllerHeaderObj.write(self.fp)
                     self.processingHeaderObj.write(self.fp)
                 def __setNewBlock(self):
                     """
                     Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
                     Return:
 :    si no pudo escribir nada
 :    Si escribio el Basic el First Header
                     """
                     if self.fp == None:
                         self.setNextFile()
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
                         self.basicHeaderObj.write(self.fp)
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     """
                     Selecciona el bloque siguiente de datos y los escribe en un file
                     Return:
 :    Si no hizo pudo escribir el bloque de datos
 :    Si no pudo escribir el bloque de datos
                     """
                     if not(self.__setNewBlock()):
                         return 0
                     self.writeBlock()
                     print("[Writing] Block No. %d/%d" % (self.blockIndex,
                                                          self.processingHeaderObj.dataBlocksPerFile))
                     return 1
                 def setNextFile(self):
                     """
                     Determina el siguiente file que sera escrito
                     Affected:
                         self.filename
                         self.subfolder
                         self.fp
                         self.setFile
                         self.flagIsNewFile
                     Return:
 :    Si el archivo no puede ser escrito
 :    Si el archivo esta listo para ser escrito
                     """
                     ext = self.ext
                     path = self.path
                     if self.fp != None:
                         self.fp.close()
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
                     fullpath = os.path.join(path, subfolder)
                     setFile = self.setFile
                     if not(os.path.exists(fullpath)):
                         os.mkdir(fullpath)
                         setFile = -1  # inicializo mi contador de seteo
                     else:
                         filesList = os.listdir(fullpath)
                         if len(filesList) > 0:
                             filesList = sorted(filesList, key=str.lower)
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber(filen[8:11]):
                                 # inicializo mi contador de seteo al seteo del ultimo file
                                 setFile = int(filen[8:11])
                             else:
                                 setFile = -1
                         else:
                             setFile = -1  # inicializo mi contador de seteo
                     setFile += 1
                     # If this is a new day it resets some values
                     if self.dataOut.datatime.date() > self.fileDate:
                         setFile = 0
                         self.nTotalBlocks = 0
                     filen = '{}{:04d}{:03d}{:03d}{}'.format(
                             self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
                     filename = os.path.join(path, subfolder, filen)
                     fp = open(filename, 'wb')
                     self.blockIndex = 0
                     # guardando atributos
                     self.filename = filename
                     self.subfolder = subfolder
                     self.fp = fp
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     self.fileDate = self.dataOut.datatime.date()
                     self.setFirstHeader()
                     print('[Writing] Opening file: %s' % self.filename)
                     self.__writeFirstHeader()
                     return 1
                 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
                     """
                     Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
                     Inputs:
                         path                :    directory where data will be saved
                         profilesPerBlock    :    number of profiles per block
                         set                 :    initial file set
                         datatype            :    An integer number that defines data type:
 : int8  (1 byte)
 : int16 (2 bytes)
 : int32 (4 bytes)
 : int64 (8 bytes)
 : float32 (4 bytes)
 : double64 (8 bytes)
                     Return:
 :    Si no realizo un buen seteo
 :    Si realizo un buen seteo
                     """
                     if ext == None:
                         ext = self.ext
                     self.ext = ext.lower()
                     self.path = path
                     if set is None:
                         self.setFile = -1
                     else:
                         self.setFile = set - 1
                     self.blocksPerFile = blocksPerFile
                     self.profilesPerBlock = profilesPerBlock
                     self.dataOut = dataOut
                     self.fileDate = self.dataOut.datatime.date()
                     # By default
                     self.dtype = self.dataOut.dtype
                     if datatype is not None:
                         self.dtype = get_numpy_dtype(datatype)
                     if not(self.setNextFile()):
                         print("[Writing] There isn't a next file")
                         return 0
                     self.setBlockDimension()
                     return 1
                 def run(self, dataOut, path, blocksPerFile=100, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
                                    set=set, ext=ext, datatype=datatype, **kwargs)
                         self.isConfig = True
                     self.dataOut = dataOut
                     self.putData()
                     return self.dataOut

schainpy/model/io/jroIO_madrigal.py +43 -51

             '''
             Created on Aug 1, 2017
             @author: Juan C. Espinoza
             '''
             import os
             import sys
             import time
             import json
             import glob
             import datetime
             import numpy
             import h5py
             from schainpy.model.io.jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jrodata import Parameters
             from schainpy.utils import log
             try:
                 import madrigal.cedar
             except:
                 log.warning(
                     'You should install "madrigal library" module if you want to read/write Madrigal data'
                     )
+            try:
+                basestring
+            except:
+                basestring = str
             DEF_CATALOG = {
                 'principleInvestigator': 'Marco Milla',
                 'expPurpose': '',
                 'cycleTime': '',
                 'correlativeExp': '',
                 'sciRemarks': '',
                 'instRemarks': ''
                 }
             DEF_HEADER = {
                 'kindatDesc': '',
                 'analyst': 'Jicamarca User',
                 'comments': '',
                 'history': ''
                 }
             MNEMONICS = {
 : 'jro',
 : 'jbr',
 : 'jul',
 : 'jas',
 : 'pbr',
 : 'hbr',
 : 'obr',
 : 'clr'
             }
             UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
             def load_json(obj):
                 '''
                 Parse json as string instead of unicode
                 '''
                 if isinstance(obj, str):
                     iterable = json.loads(obj)
                 else:
                     iterable = obj
                 if isinstance(iterable, dict):
-                    return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, (str,unicode)) else v
+                    return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, basestring) else v
                         for k, v in list(iterable.items())}
                 elif isinstance(iterable, (list, tuple)):
-                    return [str(v) if isinstance(v, str) else v for v in iterable]
+                    return [str(v) if isinstance(v, basestring) else v for v in iterable]
                 return iterable
             @MPDecorator
             class MADReader(JRODataReader, ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut = Parameters()
                     self.counter_records = 0
                     self.nrecords = None
                     self.flagNoMoreFiles = 0
                     self.isConfig = False
                     self.filename = None
                     self.intervals = set()
                 def setup(self,
                           path=None,
                           startDate=None,
                           endDate=None,
                           format=None,
                           startTime=datetime.time(0, 0, 0),
                           endTime=datetime.time(23, 59, 59),
                           **kwargs):
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.datatime = datetime.datetime(1900,1,1)
                     self.oneDDict = load_json(kwargs.get('oneDDict',
                                                          "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
                     self.twoDDict = load_json(kwargs.get('twoDDict',
                                                          "{\"GDALT\": \"heightList\"}"))
-                    self.ind2DList = load_json(kwargs.get('ind2DList',
+                    self.independentParam = 'GDALT'
-                                                          "[\"GDALT\"]"))
                     if self.path is None:
                         raise ValueError('The path is not valid')
                     if format is None:
                         raise ValueError('The format is not valid choose simple or hdf5')
                     elif format.lower() in ('simple', 'txt'):
                         self.ext = '.txt'
                     elif format.lower() in ('cedar',):
                         self.ext = '.001'
                     else:
                         self.ext = '.hdf5'
                     self.search_files(self.path)
                     self.fileId = 0
                     if not self.fileList:
                         raise  Warning('There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path))
                     self.setNextFile()
                 def search_files(self, path):
                     '''
                      Searching for madrigal files in path
                      Creating a list of files to procces included in [startDate,endDate]
                      Input:
                          path - Path to find files
                     '''
                     log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
-                    foldercounter = 0
                     fileList0 = glob.glob1(path, '*{}'.format(self.ext))
                     fileList0.sort()
                     self.fileList = []
                     self.dateFileList = []
                     startDate = self.startDate - datetime.timedelta(1)
                     endDate = self.endDate + datetime.timedelta(1)
                     for thisFile in fileList0:
                         year = thisFile[3:7]
                         if not year.isdigit():
                             continue
                         month = thisFile[7:9]
                         if not month.isdigit():
                             continue
                         day = thisFile[9:11]
                         if not day.isdigit():
                             continue
                         year, month, day = int(year), int(month), int(day)
                         dateFile = datetime.date(year, month, day)
                         if (startDate > dateFile) or (endDate < dateFile):
                             continue
                         self.fileList.append(thisFile)
                         self.dateFileList.append(dateFile)
                     return
                 def parseHeader(self):
                     '''
                     '''
                     self.output = {}
                     self.version = '2'
                     s_parameters = None
                     if self.ext == '.txt':
-                        self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s]
+                        self.parameters = [s.strip().lower() for s in self.fp.readline().decode().strip().split(' ') if s]
                     elif self.ext == '.hdf5':
-                        metadata = self.fp['Metadata']
+                        self.metadata = self.fp['Metadata']
-                        data = self.fp['Data']['Array Layout']
+                        if '_record_layout' in self.metadata:
-                        if 'Independent Spatial Parameters' in metadata:
+                            s_parameters = [s[0].lower().decode() for s in self.metadata['Independent Spatial Parameters']]
-                            s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']]
                             self.version = '3'
-                        one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']]
+                        self.parameters = [s[0].lower().decode() for s in self.metadata['Data Parameters']]
-                        one_d = [1 for s in one]
-                        two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']]
-                        two_d = [2 for s in two]
-                        self.parameters = one + two
-                        self.parameters_d = one_d + two_d
                     log.success('Parameters found: {}'.format(self.parameters),
                                 'MADReader')
                     if s_parameters:
-                        log.success('Spatial parameters: {}'.format(','.join(str(s_parameters))),
+                        log.success('Spatial parameters found: {}'.format(s_parameters),
                                     'MADReader')
                     for param in list(self.oneDDict.keys()):
                         if param.lower() not in self.parameters:
                             log.warning(
                                 'Parameter {} not found will be ignored'.format(
                                     param),
                                 'MADReader')
                             self.oneDDict.pop(param, None)
                     for param, value in list(self.twoDDict.items()):
                         if param.lower() not in self.parameters:
                             log.warning(
                                 'Parameter {} not found, it will be ignored'.format(
                                     param),
                                 'MADReader')
                             self.twoDDict.pop(param, None)
                             continue
                         if isinstance(value, list):
                             if value[0] not in self.output:
                                 self.output[value[0]] = []
-                            self.output[value[0]].append(None)
+                            self.output[value[0]].append([])
                 def parseData(self):
                     '''
                     '''
                     if self.ext == '.txt':
                         self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
                         self.nrecords = self.data.shape[0]
-                        self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())])
+                        self.ranges = numpy.unique(self.data[:,self.parameters.index(self.independentParam.lower())])
+                        self.counter_records = 0
                     elif self.ext == '.hdf5':
-                        self.data = self.fp['Data']['Array Layout']
+                        self.data = self.fp['Data']
-                        self.nrecords = len(self.data['timestamps'].value)
+                        self.ranges = numpy.unique(self.data['Table Layout'][self.independentParam.lower()])
-                        self.ranges = self.data['range'].value
+                        self.times = numpy.unique(self.data['Table Layout']['ut1_unix'])
+                        self.counter_records = int(self.data['Table Layout']['recno'][0])
+                        self.nrecords = int(self.data['Table Layout']['recno'][-1])
                 def setNextFile(self):
                     '''
                     '''
                     file_id = self.fileId
                     if file_id == len(self.fileList):
                         log.success('No more files', 'MADReader')
                         self.flagNoMoreFiles = 1
                         return 0
                     log.success(
                         'Opening: {}'.format(self.fileList[file_id]),
                         'MADReader'
                         )
                     filename = os.path.join(self.path, self.fileList[file_id])
                     if self.filename is not None:
                         self.fp.close()
                     self.filename = filename
                     self.filedate = self.dateFileList[file_id]
                     if self.ext=='.hdf5':
                         self.fp = h5py.File(self.filename, 'r')
                     else:
                         self.fp = open(self.filename, 'rb')
                     self.parseHeader()
                     self.parseData()
                     self.sizeOfFile = os.path.getsize(self.filename)
-                    self.counter_records = 0
                     self.flagIsNewFile = 0
                     self.fileId += 1
                     return 1
                 def readNextBlock(self):
                     while True:
                         self.flagDiscontinuousBlock = 0
                         if self.flagIsNewFile:
                             if not self.setNextFile():
                                 return 0
                         self.readBlock()
                         if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
                            (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
                             log.warning(
                                 'Reading Record No. {}/{} -> {} [Skipping]'.format(
                                     self.counter_records,
                                     self.nrecords,
                                     self.datatime.ctime()),
                                 'MADReader')
                             continue
                         break
                     log.log(
                         'Reading Record No. {}/{} -> {}'.format(
                             self.counter_records,
                             self.nrecords,
                             self.datatime.ctime()),
                         'MADReader')
                     return 1
                 def readBlock(self):
                     '''
                     '''
                     dum = []
                     if self.ext == '.txt':
                         dt = self.data[self.counter_records][:6].astype(int)
                         if datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]).date() > self.datatime.date():
                             self.flagDiscontinuousBlock = 1
                         self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
                         while True:
                             dt = self.data[self.counter_records][:6].astype(int)
                             datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
                             if datatime == self.datatime:
                                 dum.append(self.data[self.counter_records])
                                 self.counter_records += 1
                                 if self.counter_records == self.nrecords:
                                     self.flagIsNewFile = True
                                     break
                                 continue
                             self.intervals.add((datatime-self.datatime).seconds)
                             break
                     elif self.ext == '.hdf5':
                         datatime = datetime.datetime.utcfromtimestamp(
-                            self.data['timestamps'][self.counter_records])
+                            self.times[self.counter_records])
-                        nHeights = len(self.ranges)
+                        dum = self.data['Table Layout'][self.data['Table Layout']['recno']==self.counter_records]
-                        for n, param in enumerate(self.parameters):
-                            if self.parameters_d[n] == 1:
-                                dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records])
-                            else:
-                                if self.version == '2':
-                                    dum.append(self.data['2D Parameters'][param][self.counter_records])
-                                else:
-                                    tmp = self.data['2D Parameters'][param].value.T
-                                    dum.append(tmp[self.counter_records])
                         self.intervals.add((datatime-self.datatime).seconds)
                         if datatime.date()>self.datatime.date():
                             self.flagDiscontinuousBlock = 1
                         self.datatime = datatime
                         self.counter_records += 1
                         if self.counter_records == self.nrecords:
                             self.flagIsNewFile = True
                     self.buffer = numpy.array(dum)
                     return
                 def set_output(self):
                     '''
                     Storing data from buffer to dataOut object
                     '''
                     parameters = [None for __ in self.parameters]
                     for param, attr in list(self.oneDDict.items()):
                         x = self.parameters.index(param.lower())
                         setattr(self.dataOut, attr, self.buffer[0][x])
                     for param, value in list(self.twoDDict.items()):
-                        x = self.parameters.index(param.lower())
+                        dummy = numpy.zeros(self.ranges.shape) + numpy.nan
                         if self.ext == '.txt':
-                            y = self.parameters.index(self.ind2DList[0].lower())
+                            x = self.parameters.index(param.lower())
+                            y = self.parameters.index(self.independentParam.lower())
                             ranges = self.buffer[:,y]
                             #if self.ranges.size == ranges.size:
                             #    continue
                             index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
-                            dummy = numpy.zeros(self.ranges.shape) + numpy.nan
                             dummy[index] = self.buffer[:,x]
                         else:
-                            dummy = self.buffer[x]
+                            ranges = self.buffer[self.independentParam.lower()]
+                            index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
+                            dummy[index] = self.buffer[param.lower()]
                         if isinstance(value, str):
-                            if value not in self.ind2DList:
+                            if value not in self.independentParam:
                                 setattr(self.dataOut, value, dummy.reshape(1,-1))
                         elif isinstance(value, list):
                             self.output[value[0]][value[1]] = dummy
                             parameters[value[1]] = param
                     for key, value in list(self.output.items()):
                         setattr(self.dataOut, key, numpy.array(value))
                     self.dataOut.parameters = [s for s in parameters if s]
                     self.dataOut.heightList = self.ranges
                     self.dataOut.utctime = (self.datatime - datetime.datetime(1970, 1, 1)).total_seconds()
                     self.dataOut.utctimeInit = self.dataOut.utctime
                     self.dataOut.paramInterval = min(self.intervals)
                     self.dataOut.useLocalTime = False
                     self.dataOut.flagNoData = False
                     self.dataOut.nrecords = self.nrecords
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                 def getData(self):
                     '''
                     Storing data from databuffer to dataOut object
                     '''
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         self.dataOut.error = 'No file left to process'
                         return 0
                     if not  self.readNextBlock():
                         self.dataOut.flagNoData = True
                         return 0
                     self.set_output()
                     return 1
             @MPDecorator
             class MADWriter(Operation):
                 missing = -32767
                 def __init__(self):
                     Operation.__init__(self)
                     self.dataOut = Parameters()
                     self.counter = 0
                     self.path = None
                     self.fp = None
-                def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}',
+                def run(self, dataOut, path, oneDDict, independentParam='[]', twoDDict='{}',
                         metadata='{}', format='cedar', **kwargs):
                     '''
                     Inputs:
                         path - path where files will be created
                         oneDDict - json of one-dimensional parameters in record where keys
                         are Madrigal codes (integers or mnemonics) and values the corresponding
                         dataOut attribute e.g: {
                             'gdlatr': 'lat',
                             'gdlonr': 'lon',
                             'gdlat2':'lat',
                             'glon2':'lon'}
-                        ind2DList - list of independent spatial two-dimensional parameters e.g:
+                        independentParam - list of independent spatial two-dimensional parameters e.g:
-                            ['heighList']
+                            ['heigthList']
                         twoDDict - json of two-dimensional parameters in record where keys
                         are Madrigal codes (integers or mnemonics) and values the corresponding
                         dataOut attribute if multidimensional array specify as tupple
                         ('attr', pos) e.g: {
                             'gdalt': 'heightList',
                             'vn1p2': ('data_output', 0),
                             'vn2p2': ('data_output', 1),
                             'vn3': ('data_output', 2),
                             'snl': ('data_SNR', 'db')
                             }
                         metadata - json of madrigal metadata (kinst, kindat, catalog and header)
                     '''
                     if not self.isConfig:
-                        self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs)
+                        self.setup(path, oneDDict, independentParam, twoDDict, metadata, format, **kwargs)
                         self.isConfig = True
                     self.dataOut = dataOut
                     self.putData()
                     return 1
-                def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
+                def setup(self, path, oneDDict, independentParam, twoDDict, metadata, format, **kwargs):
                     '''
                     Configure Operation
                     '''
                     self.path = path
                     self.blocks = kwargs.get('blocks', None)
                     self.counter = 0
                     self.oneDDict = load_json(oneDDict)
                     self.twoDDict = load_json(twoDDict)
-                    self.ind2DList = load_json(ind2DList)
+                    self.independentParam = load_json(independentParam)
                     meta = load_json(metadata)
                     self.kinst = meta.get('kinst')
                     self.kindat = meta.get('kindat')
                     self.catalog = meta.get('catalog', DEF_CATALOG)
                     self.header = meta.get('header', DEF_HEADER)
                     if format == 'cedar':
                         self.ext = '.dat'
                         self.extra_args = {}
                     elif format == 'hdf5':
                         self.ext = '.hdf5'
-                        self.extra_args = {'ind2DList': self.ind2DList}
+                        self.extra_args = {'independentParam': self.independentParam}
                     self.keys = [k.lower() for k in self.twoDDict]
                     if 'range' in self.keys:
                         self.keys.remove('range')
                     if 'gdalt' in self.keys:
                         self.keys.remove('gdalt')
                 def setFile(self):
                     '''
                     Create new cedar file object
                     '''
                     self.mnemonic = MNEMONICS[self.kinst]   #TODO get mnemonic from madrigal
                     date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
                     filename = '{}{}{}'.format(self.mnemonic,
                                                date.strftime('%Y%m%d_%H%M%S'),
                                                self.ext)
                     self.fullname = os.path.join(self.path, filename)
                     if os.path.isfile(self.fullname) :
                         log.warning(
                             'Destination file {} already exists, previous file deleted.'.format(
                                 self.fullname),
                             'MADWriter')
                         os.remove(self.fullname)
                     try:
                         log.success(
                             'Creating file: {}'.format(self.fullname),
                             'MADWriter')
                         self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
                     except ValueError as e:
                         log.error(
                             'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
                             'MADWriter')
                         return
                     return 1
                 def writeBlock(self):
                     '''
                     Add data records to cedar file taking data from oneDDict and twoDDict
                     attributes.
                     Allowed parameters in: parcodes.tab
                     '''
                     startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
                     endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
                     heights = self.dataOut.heightList
                     if self.ext == '.dat':
                         for key, value in list(self.twoDDict.items()):
                             if isinstance(value, str):
                                 data = getattr(self.dataOut, value)
                                 invalid = numpy.isnan(data)
                                 data[invalid] = self.missing
                             elif isinstance(value, (tuple, list)):
                                 attr, key = value
                                 data = getattr(self.dataOut, attr)
                                 invalid = numpy.isnan(data)
                                 data[invalid] = self.missing
                     out = {}
                     for key, value in list(self.twoDDict.items()):
                         key = key.lower()
                         if isinstance(value, str):
                             if 'db' in value.lower():
                                 tmp = getattr(self.dataOut, value.replace('_db', ''))
                                 SNRavg = numpy.average(tmp, axis=0)
                                 tmp = 10*numpy.log10(SNRavg)
                             else:
                                 tmp = getattr(self.dataOut, value)
                             out[key] = tmp.flatten()
                         elif isinstance(value, (tuple, list)):
                             attr, x = value
                             data = getattr(self.dataOut, attr)
                             out[key] = data[int(x)]
                     a = numpy.array([out[k] for k in self.keys])
                     nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])
                     index = numpy.where(nrows == False)[0]
                     rec = madrigal.cedar.MadrigalDataRecord(
                         self.kinst,
                         self.kindat,
                         startTime.year,
                         startTime.month,
                         startTime.day,
                         startTime.hour,
                         startTime.minute,
                         startTime.second,
                         startTime.microsecond/10000,
                         endTime.year,
                         endTime.month,
                         endTime.day,
                         endTime.hour,
                         endTime.minute,
                         endTime.second,
                         endTime.microsecond/10000,
                         list(self.oneDDict.keys()),
                         list(self.twoDDict.keys()),
                         len(index),
                         **self.extra_args
                     )
                     # Setting 1d values
                     for key in self.oneDDict:
                         rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
                     # Setting 2d values
                     nrec = 0
                     for n in index:
                         for key in out:
                             rec.set2D(key, nrec, out[key][n])
                         nrec += 1
                     self.fp.append(rec)
                     if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
                         self.fp.dump()
                     if self.counter % 20 == 0 and self.counter > 0:
                         log.log(
                             'Writing {} records'.format(
                                 self.counter),
                             'MADWriter')
                 def setHeader(self):
                     '''
                     Create an add catalog and header to cedar file
                     '''
                     log.success('Closing file {}'.format(self.fullname), 'MADWriter')
                     if self.ext == '.dat':
                         self.fp.write()
                     else:
                         self.fp.dump()
                         self.fp.close()
                     header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
                     header.createCatalog(**self.catalog)
                     header.createHeader(**self.header)
                     header.write()
                 def putData(self):
                     if self.dataOut.flagNoData:
                         return 0
                     if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:
                         if self.counter > 0:
                             self.setHeader()
                         self.counter = 0
                     if self.counter == 0:
                         self.setFile()
                     self.writeBlock()
                     self.counter += 1
                 def close(self):
                     if self.counter > 0:
                         self.setHeader()

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages